14c87aefeSPatrick Mooney /*
24c87aefeSPatrick Mooney * This file and its contents are supplied under the terms of the
34c87aefeSPatrick Mooney * Common Development and Distribution License ("CDDL"), version 1.0.
44c87aefeSPatrick Mooney * You may only use this file in accordance with the terms of version
54c87aefeSPatrick Mooney * 1.0 of the CDDL.
64c87aefeSPatrick Mooney *
74c87aefeSPatrick Mooney * A full copy of the text of the CDDL should have accompanied this
84c87aefeSPatrick Mooney * source. A copy of the CDDL is also available via the Internet at
94c87aefeSPatrick Mooney * http://www.illumos.org/license/CDDL.
104c87aefeSPatrick Mooney */
111fa07ac7SMike Zeller /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */
124c87aefeSPatrick Mooney
134c87aefeSPatrick Mooney /*
144c87aefeSPatrick Mooney * Copyright 2019 Joyent, Inc.
15*b9b43e84SPatrick Mooney * Copyright 2023 Oxide Computer Company
164c87aefeSPatrick Mooney */
174c87aefeSPatrick Mooney
184c87aefeSPatrick Mooney #include <sys/types.h>
194c87aefeSPatrick Mooney #include <sys/param.h>
20b4ceea05SDan Cross #include <sys/atomic.h>
214c87aefeSPatrick Mooney #include <sys/kmem.h>
224c87aefeSPatrick Mooney #include <sys/machsystm.h>
23cf409e3fSDan Cross #include <sys/mman.h>
240153d828SPatrick Mooney #include <sys/x86_archext.h>
250153d828SPatrick Mooney #include <vm/hat_pte.h>
264c87aefeSPatrick Mooney
27b4ceea05SDan Cross #include <sys/vmm_gpt.h>
28cf409e3fSDan Cross #include <sys/vmm_vm.h>
294c87aefeSPatrick Mooney
30b4ceea05SDan Cross #define EPT_R (1 << 0)
31b4ceea05SDan Cross #define EPT_W (1 << 1)
32b4ceea05SDan Cross #define EPT_X (1 << 2)
334c87aefeSPatrick Mooney #define EPT_RWX (EPT_R | EPT_W | EPT_X)
34b4ceea05SDan Cross #define EPT_LGPG (1 << 7)
35b4ceea05SDan Cross #define EPT_ACCESSED (1 << 8)
36b4ceea05SDan Cross #define EPT_DIRTY (1 << 9)
374c87aefeSPatrick Mooney
384c87aefeSPatrick Mooney #define EPT_PA_MASK (0x000ffffffffff000ull)
394c87aefeSPatrick Mooney
400153d828SPatrick Mooney #define EPT_MAX_LEVELS 4
410153d828SPatrick Mooney CTASSERT(EPT_MAX_LEVELS <= MAX_GPT_LEVEL);
420153d828SPatrick Mooney
434ac713daSLuqman Aden #define EPTP_FLAG_ACCESSED_DIRTY (1 << 6)
444ac713daSLuqman Aden
454c87aefeSPatrick Mooney CTASSERT(EPT_R == PROT_READ);
464c87aefeSPatrick Mooney CTASSERT(EPT_W == PROT_WRITE);
474c87aefeSPatrick Mooney CTASSERT(EPT_X == PROT_EXEC);
484c87aefeSPatrick Mooney
49b4ceea05SDan Cross static uint_t
ept_pte_prot(uint64_t pte)50b4ceea05SDan Cross ept_pte_prot(uint64_t pte)
51b4ceea05SDan Cross {
52b4ceea05SDan Cross return (pte & EPT_RWX);
53b4ceea05SDan Cross }
544c87aefeSPatrick Mooney
55b4ceea05SDan Cross static inline uint64_t
ept_attr_to_pat(uint8_t attr)56b4ceea05SDan Cross ept_attr_to_pat(uint8_t attr)
57b4ceea05SDan Cross {
58b4ceea05SDan Cross uint64_t bits = attr & 0x7;
59b4ceea05SDan Cross return (bits << 3);
60b4ceea05SDan Cross }
614c87aefeSPatrick Mooney
62b4ceea05SDan Cross static uint64_t
ept_map_table(uint64_t pfn)63b4ceea05SDan Cross ept_map_table(uint64_t pfn)
64b4ceea05SDan Cross {
65b4ceea05SDan Cross const uint64_t paddr = pfn_to_pa(pfn) & EPT_PA_MASK;
66b4ceea05SDan Cross return (paddr | EPT_RWX);
67b4ceea05SDan Cross }
684c87aefeSPatrick Mooney
69b4ceea05SDan Cross static uint64_t
ept_map_page(uint64_t pfn,uint_t prot,uint8_t attr)70b4ceea05SDan Cross ept_map_page(uint64_t pfn, uint_t prot, uint8_t attr)
71b4ceea05SDan Cross {
72b4ceea05SDan Cross const uint64_t paddr = pfn_to_pa(pfn) & EPT_PA_MASK;
73b4ceea05SDan Cross const uint64_t pat = ept_attr_to_pat(attr);
74b4ceea05SDan Cross const uint64_t rprot = prot & EPT_RWX;
75b4ceea05SDan Cross return (paddr | pat | rprot);
76b4ceea05SDan Cross }
774c87aefeSPatrick Mooney
78b4ceea05SDan Cross static uint64_t
ept_pte_pfn(uint64_t pte)79b4ceea05SDan Cross ept_pte_pfn(uint64_t pte)
80b4ceea05SDan Cross {
81b4ceea05SDan Cross return (mmu_btop(pte & PT_PADDR));
82b4ceea05SDan Cross }
834c87aefeSPatrick Mooney
84b4ceea05SDan Cross static bool
ept_pte_is_present(uint64_t pte)85b4ceea05SDan Cross ept_pte_is_present(uint64_t pte)
864c87aefeSPatrick Mooney {
87b4ceea05SDan Cross return ((pte & EPT_RWX) != 0);
884c87aefeSPatrick Mooney }
894c87aefeSPatrick Mooney
90b4ceea05SDan Cross static uint_t
ept_reset_bits(volatile uint64_t * entry,uint64_t mask,uint64_t bits)91b4ceea05SDan Cross ept_reset_bits(volatile uint64_t *entry, uint64_t mask, uint64_t bits)
924c87aefeSPatrick Mooney {
93b4ceea05SDan Cross uint64_t pte, newpte, oldpte = 0;
94b4ceea05SDan Cross
95b4ceea05SDan Cross /*
96b4ceea05SDan Cross * We use volatile and atomic ops here because we may be
97b4ceea05SDan Cross * racing against hardware modifying these bits.
98b4ceea05SDan Cross */
99b4ceea05SDan Cross VERIFY3P(entry, !=, NULL);
100b4ceea05SDan Cross oldpte = *entry;
101b4ceea05SDan Cross do {
102b4ceea05SDan Cross pte = oldpte;
103b4ceea05SDan Cross newpte = (pte & ~mask) | bits;
104b4ceea05SDan Cross oldpte = atomic_cas_64(entry, pte, newpte);
105b4ceea05SDan Cross } while (oldpte != pte);
106b4ceea05SDan Cross
107b4ceea05SDan Cross return (oldpte & mask);
108b4ceea05SDan Cross }
109b4ceea05SDan Cross
110b4ceea05SDan Cross static uint_t
ept_reset_dirty(uint64_t * entry,bool on)111b4ceea05SDan Cross ept_reset_dirty(uint64_t *entry, bool on)
112b4ceea05SDan Cross {
113b4ceea05SDan Cross return (ept_reset_bits(entry, EPT_DIRTY,
114b4ceea05SDan Cross on ? (EPT_DIRTY | EPT_ACCESSED) : 0));
115b4ceea05SDan Cross }
116b4ceea05SDan Cross
117b4ceea05SDan Cross static uint_t
ept_reset_accessed(uint64_t * entry,bool on)118b4ceea05SDan Cross ept_reset_accessed(uint64_t *entry, bool on)
119b4ceea05SDan Cross {
120b4ceea05SDan Cross return (ept_reset_bits(entry, EPT_DIRTY | EPT_ACCESSED,
121b4ceea05SDan Cross on ? EPT_ACCESSED : 0));
122b4ceea05SDan Cross }
123b4ceea05SDan Cross
124*b9b43e84SPatrick Mooney static bool
ept_query(uint64_t * entry,vmm_gpt_query_t query)125*b9b43e84SPatrick Mooney ept_query(uint64_t *entry, vmm_gpt_query_t query)
126*b9b43e84SPatrick Mooney {
127*b9b43e84SPatrick Mooney ASSERT(entry != NULL);
128*b9b43e84SPatrick Mooney
129*b9b43e84SPatrick Mooney const uint64_t pte = *entry;
130*b9b43e84SPatrick Mooney switch (query) {
131*b9b43e84SPatrick Mooney case VGQ_ACCESSED:
132*b9b43e84SPatrick Mooney return ((pte & EPT_ACCESSED) != 0);
133*b9b43e84SPatrick Mooney case VGQ_DIRTY:
134*b9b43e84SPatrick Mooney return ((pte & EPT_DIRTY) != 0);
135*b9b43e84SPatrick Mooney default:
136*b9b43e84SPatrick Mooney panic("unrecognized query: %d", query);
137*b9b43e84SPatrick Mooney }
138*b9b43e84SPatrick Mooney }
139*b9b43e84SPatrick Mooney
1400153d828SPatrick Mooney static uint64_t
ept_get_pmtp(pfn_t root_pfn,bool track_dirty)1414ac713daSLuqman Aden ept_get_pmtp(pfn_t root_pfn, bool track_dirty)
1420153d828SPatrick Mooney {
1434ac713daSLuqman Aden const uint64_t ad_flag = track_dirty ? EPTP_FLAG_ACCESSED_DIRTY : 0;
1444ac713daSLuqman Aden return ((root_pfn << PAGESHIFT | ad_flag |
1450153d828SPatrick Mooney (EPT_MAX_LEVELS - 1) << 3 | MTRR_TYPE_WB));
1460153d828SPatrick Mooney }
1470153d828SPatrick Mooney
1484ac713daSLuqman Aden static bool
ept_hw_ad_supported(void)1494ac713daSLuqman Aden ept_hw_ad_supported(void)
1504ac713daSLuqman Aden {
1514ac713daSLuqman Aden uint64_t ept_caps = rdmsr(MSR_IA32_VMX_EPT_VPID_CAP);
1524ac713daSLuqman Aden return ((ept_caps & IA32_VMX_EPT_VPID_HW_AD) != 0);
1534ac713daSLuqman Aden }
1544ac713daSLuqman Aden
1550153d828SPatrick Mooney vmm_pte_ops_t ept_pte_ops = {
156b4ceea05SDan Cross .vpeo_map_table = ept_map_table,
157b4ceea05SDan Cross .vpeo_map_page = ept_map_page,
158b4ceea05SDan Cross .vpeo_pte_pfn = ept_pte_pfn,
159b4ceea05SDan Cross .vpeo_pte_is_present = ept_pte_is_present,
160b4ceea05SDan Cross .vpeo_pte_prot = ept_pte_prot,
161b4ceea05SDan Cross .vpeo_reset_dirty = ept_reset_dirty,
162b4ceea05SDan Cross .vpeo_reset_accessed = ept_reset_accessed,
163*b9b43e84SPatrick Mooney .vpeo_query = ept_query,
1640153d828SPatrick Mooney .vpeo_get_pmtp = ept_get_pmtp,
1654ac713daSLuqman Aden .vpeo_hw_ad_supported = ept_hw_ad_supported,
1664c87aefeSPatrick Mooney };
167