1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27/*
28 * Kernel Physical Mapping (kpm) segment driver (segkpm).
29 *
30 * This driver delivers along with the hat_kpm* interfaces an alternative
31 * mechanism for kernel mappings within the 64-bit Solaris operating system,
32 * which allows the mapping of all physical memory into the kernel address
33 * space at once. This is feasible in 64 bit kernels, e.g. for Ultrasparc II
34 * and beyond processors, since the available VA range is much larger than
35 * possible physical memory. Momentarily all physical memory is supported,
36 * that is represented by the list of memory segments (memsegs).
37 *
38 * Segkpm mappings have also very low overhead and large pages are used
39 * (when possible) to minimize the TLB and TSB footprint. It is also
40 * extentable for other than Sparc architectures (e.g. AMD64). Main
41 * advantage is the avoidance of the TLB-shootdown X-calls, which are
42 * normally needed when a kernel (global) mapping has to be removed.
43 *
44 * First example of a kernel facility that uses the segkpm mapping scheme
45 * is seg_map, where it is used as an alternative to hat_memload().
46 * See also hat layer for more information about the hat_kpm* routines.
47 * The kpm facilty can be turned off at boot time (e.g. /etc/system).
48 */
49
50#include <sys/types.h>
51#include <sys/param.h>
52#include <sys/sysmacros.h>
53#include <sys/systm.h>
54#include <sys/vnode.h>
55#include <sys/cmn_err.h>
56#include <sys/debug.h>
57#include <sys/thread.h>
58#include <sys/cpuvar.h>
59#include <sys/bitmap.h>
60#include <sys/atomic.h>
61#include <sys/lgrp.h>
62
63#include <vm/seg_kmem.h>
64#include <vm/seg_kpm.h>
65#include <vm/hat.h>
66#include <vm/as.h>
67#include <vm/seg.h>
68#include <vm/page.h>
69
70/*
71 * Global kpm controls.
72 * See also platform and mmu specific controls.
73 *
74 * kpm_enable -- global on/off switch for segkpm.
75 * . Set by default on 64bit platforms that have kpm support.
76 * . Will be disabled from platform layer if not supported.
77 * . Can be disabled via /etc/system.
78 *
79 * kpm_smallpages -- use only regular/system pagesize for kpm mappings.
80 * . Can be useful for critical debugging of kpm clients.
81 * . Set to zero by default for platforms that support kpm large pages.
82 *   The use of kpm large pages reduces the footprint of kpm meta data
83 *   and has all the other advantages of using large pages (e.g TLB
84 *   miss reduction).
85 * . Set by default for platforms that don't support kpm large pages or
86 *   where large pages cannot be used for other reasons (e.g. there are
87 *   only few full associative TLB entries available for large pages).
88 *
89 * segmap_kpm -- separate on/off switch for segmap using segkpm:
90 * . Set by default.
91 * . Will be disabled when kpm_enable is zero.
92 * . Will be disabled when MAXBSIZE != PAGESIZE.
93 * . Can be disabled via /etc/system.
94 *
95 */
96int kpm_enable = 1;
97int kpm_smallpages = 0;
98int segmap_kpm = 1;
99
100/*
101 * Private seg op routines.
102 */
103faultcode_t segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr,
104			size_t len, enum fault_type type, enum seg_rw rw);
105static void	segkpm_dump(struct seg *);
106static void	segkpm_badop(void);
107static int	segkpm_notsup(void);
108static int	segkpm_capable(struct seg *, segcapability_t);
109
110#define	SEGKPM_BADOP(t)	(t(*)())segkpm_badop
111#define	SEGKPM_NOTSUP	(int(*)())segkpm_notsup
112
113static struct seg_ops segkpm_ops = {
114	SEGKPM_BADOP(int),	/* dup */
115	SEGKPM_BADOP(int),	/* unmap */
116	SEGKPM_BADOP(void),	/* free */
117	segkpm_fault,
118	SEGKPM_BADOP(int),	/* faulta */
119	SEGKPM_BADOP(int),	/* setprot */
120	SEGKPM_BADOP(int),	/* checkprot */
121	SEGKPM_BADOP(int),	/* kluster */
122	SEGKPM_BADOP(size_t),	/* swapout */
123	SEGKPM_BADOP(int),	/* sync */
124	SEGKPM_BADOP(size_t),	/* incore */
125	SEGKPM_BADOP(int),	/* lockop */
126	SEGKPM_BADOP(int),	/* getprot */
127	SEGKPM_BADOP(u_offset_t), /* getoffset */
128	SEGKPM_BADOP(int),	/* gettype */
129	SEGKPM_BADOP(int),	/* getvp */
130	SEGKPM_BADOP(int),	/* advise */
131	segkpm_dump,		/* dump */
132	SEGKPM_NOTSUP,		/* pagelock */
133	SEGKPM_BADOP(int),	/* setpgsz */
134	SEGKPM_BADOP(int),	/* getmemid */
135	SEGKPM_BADOP(lgrp_mem_policy_info_t *),	/* getpolicy */
136	segkpm_capable,		/* capable */
137	seg_inherit_notsup	/* inherit */
138};
139
140/*
141 * kpm_pgsz and kpm_pgshft are set by platform layer.
142 */
143size_t		kpm_pgsz;	/* kpm page size */
144uint_t		kpm_pgshft;	/* kpm page shift */
145u_offset_t	kpm_pgoff;	/* kpm page offset mask */
146uint_t		kpmp2pshft;	/* kpm page to page shift */
147pgcnt_t		kpmpnpgs;	/* how many pages per kpm page */
148
149
150#ifdef	SEGKPM_SUPPORT
151
152int
153segkpm_create(struct seg *seg, void *argsp)
154{
155	struct segkpm_data *skd;
156	struct segkpm_crargs *b = (struct segkpm_crargs *)argsp;
157	ushort_t *p;
158	int i, j;
159
160	ASSERT(seg->s_as && RW_WRITE_HELD(&seg->s_as->a_lock));
161	ASSERT(btokpmp(seg->s_size) >= 1 &&
162	    kpmpageoff((uintptr_t)seg->s_base) == 0 &&
163	    kpmpageoff((uintptr_t)seg->s_base + seg->s_size) == 0);
164
165	skd = kmem_zalloc(sizeof (struct segkpm_data), KM_SLEEP);
166
167	seg->s_data = (void *)skd;
168	seg->s_ops = &segkpm_ops;
169	skd->skd_prot = b->prot;
170
171	/*
172	 * (1) Segkpm virtual addresses are based on physical adresses.
173	 * From this and in opposite to other segment drivers it is
174	 * often required to allocate a page first to be able to
175	 * calculate the final segkpm virtual address.
176	 * (2) Page  allocation is done by calling page_create_va(),
177	 * one important input argument is a virtual address (also
178	 * expressed by the "va" in the function name). This function
179	 * is highly optimized to select the right page for an optimal
180	 * processor and platform support (e.g. virtual addressed
181	 * caches (VAC), physical addressed caches, NUMA).
182	 *
183	 * Because of (1) the approach is to generate a faked virtual
184	 * address for calling page_create_va(). In order to exploit
185	 * the abilities of (2), especially to utilize the cache
186	 * hierarchy (3) and to avoid VAC alias conflicts (4) the
187	 * selection has to be done carefully. For each virtual color
188	 * a separate counter is provided (4). The count values are
189	 * used for the utilization of all cache lines (3) and are
190	 * corresponding to the cache bins.
191	 */
192	skd->skd_nvcolors = b->nvcolors;
193
194	p = skd->skd_va_select =
195	    kmem_zalloc(NCPU * b->nvcolors * sizeof (ushort_t), KM_SLEEP);
196
197	for (i = 0; i < NCPU; i++)
198		for (j = 0; j < b->nvcolors; j++, p++)
199			*p = j;
200
201	return (0);
202}
203
204/*
205 * This routine is called via a machine specific fault handling
206 * routine.
207 */
208/* ARGSUSED */
209faultcode_t
210segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len,
211	enum fault_type type, enum seg_rw rw)
212{
213	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
214
215	switch (type) {
216	case F_INVAL:
217		return (hat_kpm_fault(hat, addr));
218	case F_SOFTLOCK:
219	case F_SOFTUNLOCK:
220		return (0);
221	default:
222		return (FC_NOSUPPORT);
223	}
224	/*NOTREACHED*/
225}
226
227#define	addr_to_vcolor(addr, vcolors) \
228	((int)(((uintptr_t)(addr) & ((vcolors << PAGESHIFT) - 1)) >> PAGESHIFT))
229
230/*
231 * Create a virtual address that can be used for invocations of
232 * page_create_va. Goal is to utilize the cache hierarchy (round
233 * robin bins) and to select the right color for virtual indexed
234 * caches. It isn't exact since we also increment the bin counter
235 * when the caller uses VOP_GETPAGE and gets a hit in the page
236 * cache, but we keep the bins turning for cache distribution
237 * (see also segkpm_create block comment).
238 */
239caddr_t
240segkpm_create_va(u_offset_t off)
241{
242	int vcolor;
243	ushort_t *p;
244	struct segkpm_data *skd = (struct segkpm_data *)segkpm->s_data;
245	int nvcolors = skd->skd_nvcolors;
246	caddr_t	va;
247
248	vcolor = (nvcolors > 1) ? addr_to_vcolor(off, nvcolors) : 0;
249	p = &skd->skd_va_select[(CPU->cpu_id * nvcolors) + vcolor];
250	va = (caddr_t)ptob(*p);
251
252	atomic_add_16(p, nvcolors);
253
254	return (va);
255}
256
257/*
258 * Unload mapping if the instance has an active kpm mapping.
259 */
260void
261segkpm_mapout_validkpme(struct kpme *kpme)
262{
263	caddr_t vaddr;
264	page_t *pp;
265
266retry:
267	if ((pp = kpme->kpe_page) == NULL) {
268		return;
269	}
270
271	if (page_lock(pp, SE_SHARED, (kmutex_t *)NULL, P_RECLAIM) == 0)
272		goto retry;
273
274	/*
275	 * Check if segkpm mapping is not unloaded in the meantime
276	 */
277	if (kpme->kpe_page == NULL) {
278		page_unlock(pp);
279		return;
280	}
281
282	vaddr = hat_kpm_page2va(pp, 1);
283	hat_kpm_mapout(pp, kpme, vaddr);
284	page_unlock(pp);
285}
286
287static void
288segkpm_badop()
289{
290	panic("segkpm_badop");
291}
292
293#else	/* SEGKPM_SUPPORT */
294
295/* segkpm stubs */
296
297/*ARGSUSED*/
298int segkpm_create(struct seg *seg, void *argsp) { return (0); }
299
300/* ARGSUSED */
301faultcode_t
302segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len,
303	enum fault_type type, enum seg_rw rw)
304{
305	return ((faultcode_t)0);
306}
307
308/* ARGSUSED */
309caddr_t segkpm_create_va(u_offset_t off) { return (NULL); }
310
311/* ARGSUSED */
312void segkpm_mapout_validkpme(struct kpme *kpme) {}
313
314static void
315segkpm_badop() {}
316
317#endif	/* SEGKPM_SUPPORT */
318
319static int
320segkpm_notsup()
321{
322	return (ENOTSUP);
323}
324
325/*
326 * segkpm pages are not dumped, so we just return
327 */
328/*ARGSUSED*/
329static void
330segkpm_dump(struct seg *seg)
331{}
332
333/*
334 * We claim to have no special capabilities.
335 */
336/*ARGSUSED*/
337static int
338segkpm_capable(struct seg *seg, segcapability_t capability)
339{
340	return (0);
341}
342