17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5ae115bc7Smrj * Common Development and Distribution License (the "License"). 6ae115bc7Smrj * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22ae115bc7Smrj * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 24*74ecdb51SJohn Levon * Copyright 2018 Joyent, Inc. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #ifndef _VM_HAT_PTE_H 287c478bd9Sstevel@tonic-gate #define _VM_HAT_PTE_H 297c478bd9Sstevel@tonic-gate 307c478bd9Sstevel@tonic-gate #ifdef __cplusplus 317c478bd9Sstevel@tonic-gate extern "C" { 327c478bd9Sstevel@tonic-gate #endif 337c478bd9Sstevel@tonic-gate 347c478bd9Sstevel@tonic-gate #include <sys/types.h> 35ae115bc7Smrj #include <sys/mach_mmu.h> 367c478bd9Sstevel@tonic-gate 377c478bd9Sstevel@tonic-gate /* 387c478bd9Sstevel@tonic-gate * macros to get/set/clear the PTE fields 397c478bd9Sstevel@tonic-gate */ 407c478bd9Sstevel@tonic-gate #define PTE_SET(p, f) ((p) |= (f)) 417c478bd9Sstevel@tonic-gate #define PTE_CLR(p, f) ((p) &= ~(x86pte_t)(f)) 427c478bd9Sstevel@tonic-gate #define PTE_GET(p, f) ((p) & (f)) 437c478bd9Sstevel@tonic-gate 447c478bd9Sstevel@tonic-gate /* 457c478bd9Sstevel@tonic-gate * Handy macro to check if a pagetable entry or pointer is valid 467c478bd9Sstevel@tonic-gate */ 477c478bd9Sstevel@tonic-gate #define PTE_ISVALID(p) PTE_GET(p, PT_VALID) 487c478bd9Sstevel@tonic-gate 497c478bd9Sstevel@tonic-gate /* 507c478bd9Sstevel@tonic-gate * Does a PTE map a large page. 517c478bd9Sstevel@tonic-gate */ 527c478bd9Sstevel@tonic-gate #define PTE_IS_LGPG(p, l) ((l) > 0 && PTE_GET((p), PT_PAGESIZE)) 537c478bd9Sstevel@tonic-gate 547c478bd9Sstevel@tonic-gate /* 557c478bd9Sstevel@tonic-gate * does this PTE represent a page (not a pointer to another page table)? 567c478bd9Sstevel@tonic-gate */ 577c478bd9Sstevel@tonic-gate #define PTE_ISPAGE(p, l) \ 587c478bd9Sstevel@tonic-gate (PTE_ISVALID(p) && ((l) == 0 || PTE_GET(p, PT_PAGESIZE))) 597c478bd9Sstevel@tonic-gate 607c478bd9Sstevel@tonic-gate /* 61843e1988Sjohnlev * Handy macro to check if 2 PTE's are the same - ignores REF/MOD bits. 62ab4a9bebSjohnlev * On the 64 bit hypervisor we also have to ignore the high order 63ab4a9bebSjohnlev * software bits and the global/user bit which are set/cleared 64ab4a9bebSjohnlev * capriciously (by the hypervisor!) 657c478bd9Sstevel@tonic-gate */ 66843e1988Sjohnlev #if defined(__amd64) && defined(__xpv) 67ab4a9bebSjohnlev #define PT_IGNORE ((0x7fful << 52) | PT_GLOBAL | PT_USER) 68843e1988Sjohnlev #else 69843e1988Sjohnlev #define PT_IGNORE (0) 70843e1988Sjohnlev #endif 71843e1988Sjohnlev #define PTE_EQUIV(a, b) (((a) | (PT_IGNORE | PT_REF | PT_MOD)) == \ 72843e1988Sjohnlev ((b) | (PT_IGNORE | PT_REF | PT_MOD))) 737c478bd9Sstevel@tonic-gate 747c478bd9Sstevel@tonic-gate /* 757c478bd9Sstevel@tonic-gate * Shorthand for converting a PTE to it's pfn. 767c478bd9Sstevel@tonic-gate */ 77ae115bc7Smrj #define PTE2MFN(p, l) \ 787c478bd9Sstevel@tonic-gate mmu_btop(PTE_GET((p), PTE_IS_LGPG((p), (l)) ? PT_PADDR_LGPG : PT_PADDR)) 79843e1988Sjohnlev #ifdef __xpv 80843e1988Sjohnlev #define PTE2PFN(p, l) pte2pfn(p, l) 81843e1988Sjohnlev #else 82ae115bc7Smrj #define PTE2PFN(p, l) PTE2MFN(p, l) 83843e1988Sjohnlev #endif 847c478bd9Sstevel@tonic-gate 857c478bd9Sstevel@tonic-gate #define PT_NX (0x8000000000000000ull) 86ae115bc7Smrj #define PT_PADDR (0x000ffffffffff000ull) 87ae115bc7Smrj #define PT_PADDR_LGPG (0x000fffffffffe000ull) /* phys addr for large pages */ 887c478bd9Sstevel@tonic-gate 897c478bd9Sstevel@tonic-gate /* 907c478bd9Sstevel@tonic-gate * Macros to create a PTP or PTE from the pfn and level 917c478bd9Sstevel@tonic-gate */ 92843e1988Sjohnlev #ifdef __xpv 93843e1988Sjohnlev 94843e1988Sjohnlev /* 95843e1988Sjohnlev * we use the highest order bit in physical address pfns to mark foreign mfns 96843e1988Sjohnlev */ 97843e1988Sjohnlev #ifdef _LP64 98843e1988Sjohnlev #define PFN_IS_FOREIGN_MFN (1ul << 51) 99843e1988Sjohnlev #else 100843e1988Sjohnlev #define PFN_IS_FOREIGN_MFN (1ul << 31) 101843e1988Sjohnlev #endif 102843e1988Sjohnlev 103843e1988Sjohnlev #define MAKEPTP(pfn, l) \ 104843e1988Sjohnlev (pa_to_ma(pfn_to_pa(pfn)) | mmu.ptp_bits[(l) + 1]) 105843e1988Sjohnlev #define MAKEPTE(pfn, l) \ 106843e1988Sjohnlev ((pfn & PFN_IS_FOREIGN_MFN) ? \ 107843e1988Sjohnlev ((pfn_to_pa(pfn & ~PFN_IS_FOREIGN_MFN) | mmu.pte_bits[l]) | \ 108843e1988Sjohnlev PT_FOREIGN | PT_REF | PT_MOD) : \ 109843e1988Sjohnlev (pa_to_ma(pfn_to_pa(pfn)) | mmu.pte_bits[l])) 110843e1988Sjohnlev #else 1117c478bd9Sstevel@tonic-gate #define MAKEPTP(pfn, l) \ 112ae115bc7Smrj (pfn_to_pa(pfn) | mmu.ptp_bits[(l) + 1]) 1137c478bd9Sstevel@tonic-gate #define MAKEPTE(pfn, l) \ 114ae115bc7Smrj (pfn_to_pa(pfn) | mmu.pte_bits[l]) 115843e1988Sjohnlev #endif 1167c478bd9Sstevel@tonic-gate 1177c478bd9Sstevel@tonic-gate /* 1187c478bd9Sstevel@tonic-gate * The idea of "level" refers to the level where the page table is used in the 1197c478bd9Sstevel@tonic-gate * the hardware address translation steps. The level values correspond to the 1207c478bd9Sstevel@tonic-gate * following names of tables used in AMD/Intel architecture documents: 1217c478bd9Sstevel@tonic-gate * 1227c478bd9Sstevel@tonic-gate * AMD/INTEL name Level # 1237c478bd9Sstevel@tonic-gate * ---------------------- ------- 1247c478bd9Sstevel@tonic-gate * Page Map Level 4 3 1257c478bd9Sstevel@tonic-gate * Page Directory Pointer 2 1267c478bd9Sstevel@tonic-gate * Page Directory 1 1277c478bd9Sstevel@tonic-gate * Page Table 0 1287c478bd9Sstevel@tonic-gate * 1297c478bd9Sstevel@tonic-gate * The numbering scheme is such that the values of 0 and 1 can correspond to 1307c478bd9Sstevel@tonic-gate * the pagesize codes used for MPSS support. For now the Maximum level at 1317c478bd9Sstevel@tonic-gate * which you can have a large page is a constant, that may change in 1327c478bd9Sstevel@tonic-gate * future processors. 1337c478bd9Sstevel@tonic-gate * 1347c478bd9Sstevel@tonic-gate * The type of "level_t" is signed so that it can be used like: 1357c478bd9Sstevel@tonic-gate * level_t l; 1367c478bd9Sstevel@tonic-gate * ... 1377c478bd9Sstevel@tonic-gate * while (--l >= 0) 1387c478bd9Sstevel@tonic-gate * ... 1397c478bd9Sstevel@tonic-gate */ 1407c478bd9Sstevel@tonic-gate #define MAX_NUM_LEVEL 4 14102bc52beSkchow #define MAX_PAGE_LEVEL 2 142c2e5ad71SSam Gwydir #define MIN_PAGE_LEVEL 0 143ae115bc7Smrj typedef int8_t level_t; 1447c478bd9Sstevel@tonic-gate #define LEVEL_SHIFT(l) (mmu.level_shift[l]) 1457c478bd9Sstevel@tonic-gate #define LEVEL_SIZE(l) (mmu.level_size[l]) 1467c478bd9Sstevel@tonic-gate #define LEVEL_OFFSET(l) (mmu.level_offset[l]) 1477c478bd9Sstevel@tonic-gate #define LEVEL_MASK(l) (mmu.level_mask[l]) 1487c478bd9Sstevel@tonic-gate 1497c478bd9Sstevel@tonic-gate /* 1507c478bd9Sstevel@tonic-gate * Macros to: 1517c478bd9Sstevel@tonic-gate * Check for a PFN above 4Gig and 64Gig for 32 bit PAE support 1527c478bd9Sstevel@tonic-gate */ 1537c478bd9Sstevel@tonic-gate #define PFN_4G (4ull * (1024 * 1024 * 1024 / MMU_PAGESIZE)) 1547c478bd9Sstevel@tonic-gate #define PFN_64G (64ull * (1024 * 1024 * 1024 / MMU_PAGESIZE)) 1557c478bd9Sstevel@tonic-gate #define PFN_ABOVE4G(pfn) ((pfn) >= PFN_4G) 1567c478bd9Sstevel@tonic-gate #define PFN_ABOVE64G(pfn) ((pfn) >= PFN_64G) 1577c478bd9Sstevel@tonic-gate 1587c478bd9Sstevel@tonic-gate /* 159*74ecdb51SJohn Levon * The CR3 register holds the physical address of the top level page table, 160*74ecdb51SJohn Levon * along with the current PCID if any. 1617c478bd9Sstevel@tonic-gate */ 162*74ecdb51SJohn Levon #define MAKECR3(pfn, pcid) (mmu_ptob(pfn) | pcid) 1637c478bd9Sstevel@tonic-gate 1647c478bd9Sstevel@tonic-gate /* 1657c478bd9Sstevel@tonic-gate * HAT/MMU parameters that depend on kernel mode and/or processor type 1667c478bd9Sstevel@tonic-gate */ 1677c478bd9Sstevel@tonic-gate struct htable; 1687c478bd9Sstevel@tonic-gate struct hat_mmu_info { 1697c478bd9Sstevel@tonic-gate x86pte_t pt_nx; /* either 0 or PT_NX */ 1707c478bd9Sstevel@tonic-gate x86pte_t pt_global; /* either 0 or PT_GLOBAL */ 1717c478bd9Sstevel@tonic-gate 1727c478bd9Sstevel@tonic-gate pfn_t highest_pfn; 1737c478bd9Sstevel@tonic-gate 1747c478bd9Sstevel@tonic-gate uint_t num_level; /* number of page table levels in use */ 1757c478bd9Sstevel@tonic-gate uint_t max_level; /* just num_level - 1 */ 1767c478bd9Sstevel@tonic-gate uint_t max_page_level; /* maximum level at which we can map a page */ 17702bc52beSkchow uint_t umax_page_level; /* max user page map level */ 1787c478bd9Sstevel@tonic-gate uint_t ptes_per_table; /* # of entries in lower level page tables */ 179*74ecdb51SJohn Levon uint_t top_level_count; /* # of entries in top-level page table */ 180*74ecdb51SJohn Levon uint_t top_level_uslots; /* # of user slots in top-level page table */ 181*74ecdb51SJohn Levon uint_t num_copied_ents; /* # of PCP-copied PTEs to create */ 182*74ecdb51SJohn Levon /* 32-bit versions of values */ 183*74ecdb51SJohn Levon uint_t top_level_uslots32; 184*74ecdb51SJohn Levon uint_t max_level32; 185*74ecdb51SJohn Levon uint_t num_copied_ents32; 186*74ecdb51SJohn Levon 187*74ecdb51SJohn Levon uint_t hash_cnt; /* cnt of entries in htable_hash_cache */ 188*74ecdb51SJohn Levon uint_t hat32_hash_cnt; /* cnt of entries in 32-bit htable_hash_cache */ 1897c478bd9Sstevel@tonic-gate 1907c478bd9Sstevel@tonic-gate uint_t pae_hat; /* either 0 or 1 */ 1917c478bd9Sstevel@tonic-gate 1927c478bd9Sstevel@tonic-gate uintptr_t hole_start; /* start of VA hole (or -1 if none) */ 1937c478bd9Sstevel@tonic-gate uintptr_t hole_end; /* end of VA hole (or 0 if none) */ 1947c478bd9Sstevel@tonic-gate 1957c478bd9Sstevel@tonic-gate struct htable **kmap_htables; /* htables for segmap + 32 bit heap */ 1967c478bd9Sstevel@tonic-gate x86pte_t *kmap_ptes; /* mapping of pagetables that map kmap */ 1977c478bd9Sstevel@tonic-gate uintptr_t kmap_addr; /* start addr of kmap */ 1987c478bd9Sstevel@tonic-gate uintptr_t kmap_eaddr; /* end addr of kmap */ 1997c478bd9Sstevel@tonic-gate 2007c478bd9Sstevel@tonic-gate uint_t pte_size; /* either 4 or 8 */ 2017c478bd9Sstevel@tonic-gate uint_t pte_size_shift; /* either 2 or 3 */ 2027c478bd9Sstevel@tonic-gate x86pte_t ptp_bits[MAX_NUM_LEVEL]; /* bits set for interior PTP */ 2037c478bd9Sstevel@tonic-gate x86pte_t pte_bits[MAX_NUM_LEVEL]; /* bits set for leaf PTE */ 2047c478bd9Sstevel@tonic-gate 205ae115bc7Smrj /* 206ae115bc7Smrj * A range of VA used to window pages in the i86pc/vm code. 207ae115bc7Smrj * See PWIN_XXX macros. 208ae115bc7Smrj */ 209ae115bc7Smrj caddr_t pwin_base; 210ae115bc7Smrj caddr_t pwin_pte_va; 211ae115bc7Smrj paddr_t pwin_pte_pa; 212ae115bc7Smrj 2137c478bd9Sstevel@tonic-gate /* 2147c478bd9Sstevel@tonic-gate * The following tables are equivalent to PAGEXXXXX at different levels 2157c478bd9Sstevel@tonic-gate * in the page table hierarchy. 2167c478bd9Sstevel@tonic-gate */ 2177c478bd9Sstevel@tonic-gate uint_t level_shift[MAX_NUM_LEVEL]; /* PAGESHIFT for given level */ 2187c478bd9Sstevel@tonic-gate uintptr_t level_size[MAX_NUM_LEVEL]; /* PAGESIZE for given level */ 2197c478bd9Sstevel@tonic-gate uintptr_t level_offset[MAX_NUM_LEVEL]; /* PAGEOFFSET for given level */ 2207c478bd9Sstevel@tonic-gate uintptr_t level_mask[MAX_NUM_LEVEL]; /* PAGEMASK for given level */ 2217c478bd9Sstevel@tonic-gate }; 2227c478bd9Sstevel@tonic-gate 2237c478bd9Sstevel@tonic-gate 2247c478bd9Sstevel@tonic-gate #if defined(_KERNEL) 225ae115bc7Smrj 226ae115bc7Smrj /* 227ae115bc7Smrj * Macros to access the HAT's private page windows. They're used for 228ae115bc7Smrj * accessing pagetables, ppcopy() and page_zero(). 229ae115bc7Smrj * The 1st two macros are used to get an index for the particular use. 230ae115bc7Smrj * The next three give you: 231ae115bc7Smrj * - the virtual address of the window 232ae115bc7Smrj * - the virtual address of the pte that maps the window 233ae115bc7Smrj * - the physical address of the pte that map the window 234ae115bc7Smrj */ 235ae115bc7Smrj #define PWIN_TABLE(cpuid) ((cpuid) * 2) 236ae115bc7Smrj #define PWIN_SRC(cpuid) ((cpuid) * 2 + 1) /* for x86pte_copy() */ 237ae115bc7Smrj #define PWIN_VA(x) (mmu.pwin_base + ((x) << MMU_PAGESHIFT)) 238ae115bc7Smrj #define PWIN_PTE_VA(x) (mmu.pwin_pte_va + ((x) << mmu.pte_size_shift)) 239ae115bc7Smrj #define PWIN_PTE_PA(x) (mmu.pwin_pte_pa + ((x) << mmu.pte_size_shift)) 240ae115bc7Smrj 2417c478bd9Sstevel@tonic-gate /* 2427c478bd9Sstevel@tonic-gate * The concept of a VA hole exists in AMD64. This might need to be made 2437c478bd9Sstevel@tonic-gate * model specific eventually. 244aa2ed9e5Sjosephb * 24575d94465SJosef 'Jeff' Sipek * In the 64 bit kernel PTE loads are atomic, but need atomic_cas_64 on 32 24675d94465SJosef 'Jeff' Sipek * bit kernel. 2477c478bd9Sstevel@tonic-gate */ 2487c478bd9Sstevel@tonic-gate #if defined(__amd64) 2497c478bd9Sstevel@tonic-gate 2507c478bd9Sstevel@tonic-gate #ifdef lint 2517c478bd9Sstevel@tonic-gate #define IN_VA_HOLE(va) (__lintzero) 2527c478bd9Sstevel@tonic-gate #else 2537c478bd9Sstevel@tonic-gate #define IN_VA_HOLE(va) (mmu.hole_start <= (va) && (va) < mmu.hole_end) 2547c478bd9Sstevel@tonic-gate #endif 2557c478bd9Sstevel@tonic-gate 256ae115bc7Smrj #define FMT_PTE "0x%lx" 257ae115bc7Smrj #define GET_PTE(ptr) (*(x86pte_t *)(ptr)) 258ae115bc7Smrj #define SET_PTE(ptr, pte) (*(x86pte_t *)(ptr) = pte) 25975d94465SJosef 'Jeff' Sipek #define CAS_PTE(ptr, x, y) atomic_cas_64(ptr, x, y) 2607c478bd9Sstevel@tonic-gate 2617c478bd9Sstevel@tonic-gate #elif defined(__i386) 2627c478bd9Sstevel@tonic-gate 2637c478bd9Sstevel@tonic-gate #define IN_VA_HOLE(va) (__lintzero) 2647c478bd9Sstevel@tonic-gate 265ae115bc7Smrj #define FMT_PTE "0x%llx" 266ae115bc7Smrj 267ae115bc7Smrj /* on 32 bit kernels, 64 bit loads aren't atomic, use get_pte64() */ 268ae115bc7Smrj extern x86pte_t get_pte64(x86pte_t *ptr); 269ae115bc7Smrj #define GET_PTE(ptr) (mmu.pae_hat ? get_pte64(ptr) : *(x86pte32_t *)(ptr)) 270ae115bc7Smrj #define SET_PTE(ptr, pte) \ 271ae115bc7Smrj ((mmu.pae_hat ? ((x86pte32_t *)(ptr))[1] = (pte >> 32) : 0), \ 272ae115bc7Smrj *(x86pte32_t *)(ptr) = pte) 273ae115bc7Smrj #define CAS_PTE(ptr, x, y) \ 27475d94465SJosef 'Jeff' Sipek (mmu.pae_hat ? atomic_cas_64(ptr, x, y) : \ 27575d94465SJosef 'Jeff' Sipek atomic_cas_32((uint32_t *)(ptr), (uint32_t)(x), (uint32_t)(y))) 2767c478bd9Sstevel@tonic-gate 277aa2ed9e5Sjosephb #endif /* __i386 */ 2787c478bd9Sstevel@tonic-gate 279ae115bc7Smrj /* 280ae115bc7Smrj * Return a pointer to the pte entry at the given index within a page table. 281ae115bc7Smrj */ 282ae115bc7Smrj #define PT_INDEX_PTR(p, x) \ 283ae115bc7Smrj ((x86pte_t *)((uintptr_t)(p) + ((x) << mmu.pte_size_shift))) 284ae115bc7Smrj 285ae115bc7Smrj /* 286ae115bc7Smrj * Return the physical address of the pte entry at the given index within a 287ae115bc7Smrj * page table. 288ae115bc7Smrj */ 289ae115bc7Smrj #define PT_INDEX_PHYSADDR(p, x) \ 290ae115bc7Smrj ((paddr_t)(p) + ((x) << mmu.pte_size_shift)) 291ae115bc7Smrj 292ae115bc7Smrj /* 293ae115bc7Smrj * From pfn to bytes, careful not to lose bits on PAE. 294ae115bc7Smrj */ 295ae115bc7Smrj #define pfn_to_pa(pfn) (mmu_ptob((paddr_t)(pfn))) 2967c478bd9Sstevel@tonic-gate 297843e1988Sjohnlev #ifdef __xpv 298843e1988Sjohnlev extern pfn_t pte2pfn(x86pte_t, level_t); 299843e1988Sjohnlev #endif 300843e1988Sjohnlev 3017c478bd9Sstevel@tonic-gate extern struct hat_mmu_info mmu; 3027c478bd9Sstevel@tonic-gate 3037c478bd9Sstevel@tonic-gate #endif /* _KERNEL */ 3047c478bd9Sstevel@tonic-gate 3057c478bd9Sstevel@tonic-gate 3067c478bd9Sstevel@tonic-gate #ifdef __cplusplus 3077c478bd9Sstevel@tonic-gate } 3087c478bd9Sstevel@tonic-gate #endif 3097c478bd9Sstevel@tonic-gate 3107c478bd9Sstevel@tonic-gate #endif /* _VM_HAT_PTE_H */ 311