14c87aefeSPatrick Mooney /*
24c87aefeSPatrick Mooney * This file and its contents are supplied under the terms of the
34c87aefeSPatrick Mooney * Common Development and Distribution License ("CDDL"), version 1.0.
44c87aefeSPatrick Mooney * You may only use this file in accordance with the terms of version
54c87aefeSPatrick Mooney * 1.0 of the CDDL.
64c87aefeSPatrick Mooney *
74c87aefeSPatrick Mooney * A full copy of the text of the CDDL should have accompanied this
84c87aefeSPatrick Mooney * source. A copy of the CDDL is also available via the Internet at
94c87aefeSPatrick Mooney * http://www.illumos.org/license/CDDL.
104c87aefeSPatrick Mooney */
111fa07ac7SMike Zeller /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */
124c87aefeSPatrick Mooney
134c87aefeSPatrick Mooney /*
144c87aefeSPatrick Mooney * Copyright 2019 Joyent, Inc.
15*b9b43e84SPatrick Mooney * Copyright 2023 Oxide Computer Company
164c87aefeSPatrick Mooney */
174c87aefeSPatrick Mooney
184c87aefeSPatrick Mooney #include <sys/types.h>
194c87aefeSPatrick Mooney #include <sys/param.h>
20b4ceea05SDan Cross #include <sys/atomic.h>
214c87aefeSPatrick Mooney #include <sys/kmem.h>
224c87aefeSPatrick Mooney #include <sys/machsystm.h>
23cf409e3fSDan Cross #include <sys/mach_mmu.h>
24cf409e3fSDan Cross #include <sys/mman.h>
254c87aefeSPatrick Mooney #include <sys/x86_archext.h>
26b4ceea05SDan Cross #include <vm/hat_pte.h>
274c87aefeSPatrick Mooney
28b4ceea05SDan Cross #include <sys/vmm_gpt.h>
29cf409e3fSDan Cross #include <sys/vmm_vm.h>
304c87aefeSPatrick Mooney
31b4ceea05SDan Cross static inline uint64_t
rvi_prot(uint_t prot)32b4ceea05SDan Cross rvi_prot(uint_t prot)
33b4ceea05SDan Cross {
34b4ceea05SDan Cross uint64_t bits;
35b4ceea05SDan Cross
36b4ceea05SDan Cross bits = 0;
37b4ceea05SDan Cross if ((prot & PROT_WRITE) != 0)
38b4ceea05SDan Cross bits |= PT_WRITABLE;
39b4ceea05SDan Cross if ((prot & PROT_EXEC) == 0)
40b4ceea05SDan Cross bits |= PT_NX;
41b4ceea05SDan Cross
42b4ceea05SDan Cross return (bits);
43b4ceea05SDan Cross }
444c87aefeSPatrick Mooney
45b4ceea05SDan Cross static uint_t
rvi_pte_prot(uint64_t pte)46b4ceea05SDan Cross rvi_pte_prot(uint64_t pte)
47b4ceea05SDan Cross {
48b4ceea05SDan Cross uint_t prot;
49b4ceea05SDan Cross
50b4ceea05SDan Cross if ((pte & PT_VALID) == 0)
51b4ceea05SDan Cross return (0);
52b4ceea05SDan Cross
53b4ceea05SDan Cross prot = PROT_READ;
54b4ceea05SDan Cross if ((pte & PT_NX) == 0)
55b4ceea05SDan Cross prot |= PROT_EXEC;
56b4ceea05SDan Cross if ((pte & PT_WRITABLE) != 0)
57b4ceea05SDan Cross prot |= PROT_WRITE;
58b4ceea05SDan Cross
59b4ceea05SDan Cross return (prot);
60b4ceea05SDan Cross }
614c87aefeSPatrick Mooney
624c87aefeSPatrick Mooney /* Make sure that PAT indexes line up as expected */
634c87aefeSPatrick Mooney CTASSERT((PAT_DEFAULT_ATTRIBUTE & 0xf) == MTRR_TYPE_WB);
644c87aefeSPatrick Mooney CTASSERT(((PAT_DEFAULT_ATTRIBUTE >> 24) & 0xf) == MTRR_TYPE_UC);
654c87aefeSPatrick Mooney
664c87aefeSPatrick Mooney static inline uint64_t
rvi_attr_to_pat(uint8_t attr)67b4ceea05SDan Cross rvi_attr_to_pat(uint8_t attr)
684c87aefeSPatrick Mooney {
69b4ceea05SDan Cross
70b4ceea05SDan Cross if (attr == MTRR_TYPE_UC)
71b4ceea05SDan Cross return (PT_NOCACHE | PT_WRITETHRU);
72b4ceea05SDan Cross if (attr == MTRR_TYPE_WB)
734c87aefeSPatrick Mooney return (0);
744c87aefeSPatrick Mooney
754c87aefeSPatrick Mooney panic("unexpected memattr %x", attr);
764c87aefeSPatrick Mooney }
774c87aefeSPatrick Mooney
78b4ceea05SDan Cross static uint64_t
rvi_map_table(uint64_t pfn)79b4ceea05SDan Cross rvi_map_table(uint64_t pfn)
804c87aefeSPatrick Mooney {
81b4ceea05SDan Cross const uint64_t paddr = pfn_to_pa(pfn);
82b4ceea05SDan Cross const uint64_t flags = PT_USER | PT_REF | PT_VALID;
83b4ceea05SDan Cross const uint64_t pat = rvi_attr_to_pat(MTRR_TYPE_WB);
84b4ceea05SDan Cross const uint64_t rprot = PT_WRITABLE;
85b4ceea05SDan Cross return (paddr | flags | pat | rprot);
864c87aefeSPatrick Mooney }
874c87aefeSPatrick Mooney
884c87aefeSPatrick Mooney static uint64_t
rvi_map_page(uint64_t pfn,uint_t prot,uint8_t attr)89b4ceea05SDan Cross rvi_map_page(uint64_t pfn, uint_t prot, uint8_t attr)
90b4ceea05SDan Cross {
91b4ceea05SDan Cross const uint64_t paddr = pfn_to_pa(pfn);
92b4ceea05SDan Cross const uint64_t flags = PT_USER | PT_REF | PT_VALID;
93b4ceea05SDan Cross const uint64_t pat = rvi_attr_to_pat(attr);
94b4ceea05SDan Cross const uint64_t rprot = rvi_prot(prot);
95b4ceea05SDan Cross return (paddr | flags | pat | rprot);
96b4ceea05SDan Cross }
97b4ceea05SDan Cross
98b4ceea05SDan Cross static pfn_t
rvi_pte_pfn(uint64_t pte)99b4ceea05SDan Cross rvi_pte_pfn(uint64_t pte)
100b4ceea05SDan Cross {
101b4ceea05SDan Cross return (mmu_btop(pte & PT_PADDR));
102b4ceea05SDan Cross }
103b4ceea05SDan Cross
104b4ceea05SDan Cross static bool
rvi_pte_is_present(uint64_t pte)105b4ceea05SDan Cross rvi_pte_is_present(uint64_t pte)
106b4ceea05SDan Cross {
107b4ceea05SDan Cross return ((pte & PT_VALID) == PT_VALID);
108b4ceea05SDan Cross }
109b4ceea05SDan Cross
110b4ceea05SDan Cross static uint_t
rvi_reset_bits(volatile uint64_t * entry,uint64_t mask,uint64_t bits)111b4ceea05SDan Cross rvi_reset_bits(volatile uint64_t *entry, uint64_t mask, uint64_t bits)
112b4ceea05SDan Cross {
113b4ceea05SDan Cross uint64_t pte, newpte, oldpte = 0;
114b4ceea05SDan Cross
115b4ceea05SDan Cross /*
116b4ceea05SDan Cross * We use volatile and atomic ops here because we may be
117b4ceea05SDan Cross * racing against hardware modifying these bits.
118b4ceea05SDan Cross */
119b4ceea05SDan Cross VERIFY3P(entry, !=, NULL);
120b4ceea05SDan Cross oldpte = *entry;
121b4ceea05SDan Cross do {
122b4ceea05SDan Cross pte = oldpte;
123b4ceea05SDan Cross newpte = (pte & ~mask) | bits;
124b4ceea05SDan Cross oldpte = atomic_cas_64(entry, pte, newpte);
125b4ceea05SDan Cross } while (oldpte != pte);
126b4ceea05SDan Cross
127b4ceea05SDan Cross return (oldpte & mask);
128b4ceea05SDan Cross }
129b4ceea05SDan Cross
130b4ceea05SDan Cross static uint_t
rvi_reset_dirty(uint64_t * entry,bool on)131b4ceea05SDan Cross rvi_reset_dirty(uint64_t *entry, bool on)
132b4ceea05SDan Cross {
133b4ceea05SDan Cross return (rvi_reset_bits(entry, PT_MOD, on ? (PT_MOD | PT_REF) : 0));
134b4ceea05SDan Cross }
135b4ceea05SDan Cross
136b4ceea05SDan Cross static uint_t
rvi_reset_accessed(uint64_t * entry,bool on)137b4ceea05SDan Cross rvi_reset_accessed(uint64_t *entry, bool on)
1384c87aefeSPatrick Mooney {
139b4ceea05SDan Cross return (rvi_reset_bits(entry, (PT_MOD | PT_REF), on ? PT_REF : 0));
140b4ceea05SDan Cross }
141b4ceea05SDan Cross
142*b9b43e84SPatrick Mooney static bool
rvi_query(uint64_t * entry,vmm_gpt_query_t query)143*b9b43e84SPatrick Mooney rvi_query(uint64_t *entry, vmm_gpt_query_t query)
144*b9b43e84SPatrick Mooney {
145*b9b43e84SPatrick Mooney ASSERT(entry != NULL);
146*b9b43e84SPatrick Mooney
147*b9b43e84SPatrick Mooney const uint64_t pte = *entry;
148*b9b43e84SPatrick Mooney switch (query) {
149*b9b43e84SPatrick Mooney case VGQ_ACCESSED:
150*b9b43e84SPatrick Mooney return ((pte & PT_REF) != 0);
151*b9b43e84SPatrick Mooney case VGQ_DIRTY:
152*b9b43e84SPatrick Mooney return ((pte & PT_MOD) != 0);
153*b9b43e84SPatrick Mooney default:
154*b9b43e84SPatrick Mooney panic("unrecognized query: %d", query);
155*b9b43e84SPatrick Mooney }
156*b9b43e84SPatrick Mooney }
157*b9b43e84SPatrick Mooney
1580153d828SPatrick Mooney static uint64_t
rvi_get_pmtp(pfn_t root_pfn,bool track_dirty)1594ac713daSLuqman Aden rvi_get_pmtp(pfn_t root_pfn, bool track_dirty)
1600153d828SPatrick Mooney {
1610153d828SPatrick Mooney return (root_pfn << PAGESHIFT);
1620153d828SPatrick Mooney }
1630153d828SPatrick Mooney
1644ac713daSLuqman Aden static bool
rvi_hw_ad_supported(void)1654ac713daSLuqman Aden rvi_hw_ad_supported(void)
1664ac713daSLuqman Aden {
1674ac713daSLuqman Aden return (true);
1684ac713daSLuqman Aden }
1694ac713daSLuqman Aden
1704ac713daSLuqman Aden
1710153d828SPatrick Mooney vmm_pte_ops_t rvi_pte_ops = {
172b4ceea05SDan Cross .vpeo_map_table = rvi_map_table,
173b4ceea05SDan Cross .vpeo_map_page = rvi_map_page,
174b4ceea05SDan Cross .vpeo_pte_pfn = rvi_pte_pfn,
175b4ceea05SDan Cross .vpeo_pte_is_present = rvi_pte_is_present,
176b4ceea05SDan Cross .vpeo_pte_prot = rvi_pte_prot,
177b4ceea05SDan Cross .vpeo_reset_dirty = rvi_reset_dirty,
178b4ceea05SDan Cross .vpeo_reset_accessed = rvi_reset_accessed,
179*b9b43e84SPatrick Mooney .vpeo_query = rvi_query,
1800153d828SPatrick Mooney .vpeo_get_pmtp = rvi_get_pmtp,
1814ac713daSLuqman Aden .vpeo_hw_ad_supported = rvi_hw_ad_supported,
1824c87aefeSPatrick Mooney };
183