17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
5c97ad5cdSakolb * Common Development and Distribution License (the "License").
6c97ad5cdSakolb * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
219ac8606fSraf
227c478bd9Sstevel@tonic-gate /*
23d5493db7SPramod Batni * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
24*6a0b1217SPatrick Mooney * Copyright 2019 Joyent, Inc.
257c478bd9Sstevel@tonic-gate */
267c478bd9Sstevel@tonic-gate
277c478bd9Sstevel@tonic-gate #include <sys/types.h>
287c478bd9Sstevel@tonic-gate #include <sys/param.h>
297c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
307c478bd9Sstevel@tonic-gate #include <sys/cred.h>
317c478bd9Sstevel@tonic-gate #include <sys/proc.h>
327c478bd9Sstevel@tonic-gate #include <sys/strsubr.h>
337c478bd9Sstevel@tonic-gate #include <sys/priocntl.h>
347c478bd9Sstevel@tonic-gate #include <sys/class.h>
357c478bd9Sstevel@tonic-gate #include <sys/disp.h>
367c478bd9Sstevel@tonic-gate #include <sys/procset.h>
377c478bd9Sstevel@tonic-gate #include <sys/debug.h>
387c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
397c478bd9Sstevel@tonic-gate #include <sys/errno.h>
407c478bd9Sstevel@tonic-gate #include <sys/systm.h>
417c478bd9Sstevel@tonic-gate #include <sys/schedctl.h>
427c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h>
437c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
447c478bd9Sstevel@tonic-gate #include <sys/project.h>
457c478bd9Sstevel@tonic-gate #include <sys/modctl.h>
467c478bd9Sstevel@tonic-gate #include <sys/fss.h>
477c478bd9Sstevel@tonic-gate #include <sys/fsspriocntl.h>
487c478bd9Sstevel@tonic-gate #include <sys/cpupart.h>
497c478bd9Sstevel@tonic-gate #include <sys/zone.h>
507c478bd9Sstevel@tonic-gate #include <vm/rm.h>
517c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
527c478bd9Sstevel@tonic-gate #include <sys/policy.h>
537c478bd9Sstevel@tonic-gate #include <sys/sdt.h>
54c97ad5cdSakolb #include <sys/cpucaps.h>
557c478bd9Sstevel@tonic-gate
567c478bd9Sstevel@tonic-gate /*
57482a7749SJerry Jelinek * The fair share scheduling class ensures that collections of processes
58482a7749SJerry Jelinek * (zones and projects) each get their configured share of CPU. This is in
59482a7749SJerry Jelinek * contrast to the TS class which considers individual processes.
60482a7749SJerry Jelinek *
61482a7749SJerry Jelinek * The FSS cpu-share is set on zones using the zone.cpu-shares rctl and on
62482a7749SJerry Jelinek * projects using the project.cpu-shares rctl. By default the value is 1
63482a7749SJerry Jelinek * and it can range from 0 - 64k. A value of 0 means that processes in the
64482a7749SJerry Jelinek * collection will only get CPU resources when there are no other processes
65482a7749SJerry Jelinek * that need CPU. The cpu-share is used as one of the inputs to calculate a
66482a7749SJerry Jelinek * thread's "user-mode" priority (umdpri) for the scheduler. The umdpri falls
67482a7749SJerry Jelinek * in the range 0-59. FSS calculates other, internal, priorities which are not
68482a7749SJerry Jelinek * visible outside of the FSS class.
69482a7749SJerry Jelinek *
70482a7749SJerry Jelinek * The FSS class should approximate TS behavior when there are excess CPU
71482a7749SJerry Jelinek * resources. When there is a backlog of runnable processes, then the share
72482a7749SJerry Jelinek * is used as input into the runnable process's priority calculation, where
73482a7749SJerry Jelinek * the final umdpri is used by the scheduler to determine when the process runs.
74482a7749SJerry Jelinek *
75482a7749SJerry Jelinek * Projects in a zone compete with each other for CPU time, receiving CPU
76482a7749SJerry Jelinek * allocation within a zone proportional to the project's share; at a higher
77482a7749SJerry Jelinek * level zones compete with each other, receiving allocation in a pset
78482a7749SJerry Jelinek * proportional to the zone's share.
79482a7749SJerry Jelinek *
80482a7749SJerry Jelinek * The FSS priority calculation consists of several parts.
81482a7749SJerry Jelinek *
82482a7749SJerry Jelinek * 1) Once per second the fss_update function runs. The first thing it does is
83482a7749SJerry Jelinek * call fss_decay_usage. This function does three things.
84482a7749SJerry Jelinek *
85482a7749SJerry Jelinek * a) fss_decay_usage first decays the maxfsspri value for the pset. This
86482a7749SJerry Jelinek * value is used in the per-process priority calculation described in step
87482a7749SJerry Jelinek * (2b). The maxfsspri is decayed using the following formula:
88482a7749SJerry Jelinek *
89482a7749SJerry Jelinek * maxfsspri * fss_nice_decay[NZERO])
90482a7749SJerry Jelinek * maxfsspri = ------------------------------------
91482a7749SJerry Jelinek * FSS_DECAY_BASE
92482a7749SJerry Jelinek *
93482a7749SJerry Jelinek *
94482a7749SJerry Jelinek * - NZERO is the default process priority (i.e. 20)
95482a7749SJerry Jelinek *
96482a7749SJerry Jelinek * The fss_nice_decay array is a fixed set of values used to adjust the
97482a7749SJerry Jelinek * decay rate of processes based on their nice value. Entries in this
98482a7749SJerry Jelinek * array are initialized in fss_init using the following formula:
99482a7749SJerry Jelinek *
100482a7749SJerry Jelinek * (FSS_DECAY_MAX - FSS_DECAY_MIN) * i
101482a7749SJerry Jelinek * FSS_DECAY_MIN + -------------------------------------
102482a7749SJerry Jelinek * FSS_NICE_RANGE - 1
103482a7749SJerry Jelinek *
104482a7749SJerry Jelinek * - FSS_DECAY_MIN is 82 = approximates 65% (82/128)
105482a7749SJerry Jelinek * - FSS_DECAY_MAX is 108 = approximates 85% (108/128)
106482a7749SJerry Jelinek * - FSS_NICE_RANGE is 40 (range is 0 - 39)
107482a7749SJerry Jelinek *
108482a7749SJerry Jelinek * b) The second thing fss_decay_usage does is update each project's "usage"
109482a7749SJerry Jelinek * for the last second and then recalculates the project's "share usage".
110482a7749SJerry Jelinek *
111482a7749SJerry Jelinek * The usage value is the recent CPU usage for all of the threads in the
112482a7749SJerry Jelinek * project. It is decayed and updated this way:
113482a7749SJerry Jelinek *
114482a7749SJerry Jelinek * (usage * FSS_DECAY_USG)
115482a7749SJerry Jelinek * usage = ------------------------- + ticks;
116482a7749SJerry Jelinek * FSS_DECAY_BASE
117482a7749SJerry Jelinek *
118482a7749SJerry Jelinek * - FSS_DECAY_BASE is 128 - used instead of 100 so we can shift vs divide
119482a7749SJerry Jelinek * - FSS_DECAY_USG is 96 - approximates 75% (96/128)
120482a7749SJerry Jelinek * - ticks is updated whenever a process in this project is running
121482a7749SJerry Jelinek * when the scheduler's tick processing fires. This is not a simple
122482a7749SJerry Jelinek * counter, the values are based on the entries in the fss_nice_tick
123482a7749SJerry Jelinek * array (see section 3 below). ticks is then reset to 0 so it can track
124482a7749SJerry Jelinek * the next seconds worth of nice-adjusted time for the project.
125482a7749SJerry Jelinek *
126482a7749SJerry Jelinek * c) The third thing fss_decay_usage does is update each project's "share
127482a7749SJerry Jelinek * usage" (shusage). This is the normalized usage value for the project and
128482a7749SJerry Jelinek * is calculated this way:
129482a7749SJerry Jelinek *
130482a7749SJerry Jelinek * pset_shares^2 zone_int_shares^2
131482a7749SJerry Jelinek * usage * ------------- * ------------------
132482a7749SJerry Jelinek * kpj_shares^2 zone_ext_shares^2
133482a7749SJerry Jelinek *
134482a7749SJerry Jelinek * - usage - see (1b) for more details
135482a7749SJerry Jelinek * - pset_shares is the total of all *active* zone shares in the pset (by
136482a7749SJerry Jelinek * default there is only one pset)
137482a7749SJerry Jelinek * - kpj_shares is the individual project's share (project.cpu-shares rctl)
138482a7749SJerry Jelinek * - zone_int_shares is the sum of shares of all active projects within the
139482a7749SJerry Jelinek * zone (the zone-internal total)
140482a7749SJerry Jelinek * - zone_ext_shares is the share value for the zone (zone.cpu-shares rctl)
141482a7749SJerry Jelinek *
142482a7749SJerry Jelinek * The shusage is used in step (2b) to calculate the thread's new internal
143482a7749SJerry Jelinek * priority. A larger shusage value leads to a lower priority.
144482a7749SJerry Jelinek *
145482a7749SJerry Jelinek * 2) The fss_update function then calls fss_update_list to update the priority
146482a7749SJerry Jelinek * of all threads. This does two things.
147482a7749SJerry Jelinek *
148482a7749SJerry Jelinek * a) First the thread's internal priority is decayed using the following
149482a7749SJerry Jelinek * formula:
150482a7749SJerry Jelinek *
151482a7749SJerry Jelinek * fsspri * fss_nice_decay[nice_value])
152482a7749SJerry Jelinek * fsspri = ------------------------------------
153482a7749SJerry Jelinek * FSS_DECAY_BASE
154482a7749SJerry Jelinek *
155482a7749SJerry Jelinek * - FSS_DECAY_BASE is 128 as described above
156482a7749SJerry Jelinek *
157482a7749SJerry Jelinek * b) Second, if the thread is runnable (TS_RUN or TS_WAIT) calls fss_newpri
158482a7749SJerry Jelinek * to update the user-mode priority (umdpri) of the runnable thread.
159482a7749SJerry Jelinek * Threads that are running (TS_ONPROC) or waiting for an event (TS_SLEEP)
160482a7749SJerry Jelinek * are not updated at this time. The updated user-mode priority can cause
161482a7749SJerry Jelinek * threads to change their position in the run queue.
162482a7749SJerry Jelinek *
163482a7749SJerry Jelinek * The process's new internal fsspri is calculated using the following
164482a7749SJerry Jelinek * formula. All runnable threads in the project will use the same shusage
165482a7749SJerry Jelinek * and nrunnable values in their calculation.
166482a7749SJerry Jelinek *
167482a7749SJerry Jelinek * fsspri += shusage * nrunnable * ticks
168482a7749SJerry Jelinek *
169482a7749SJerry Jelinek * - shusage is the project's share usage, calculated in (1c)
170482a7749SJerry Jelinek * - nrunnable is the number of runnable threads in the project
171482a7749SJerry Jelinek * - ticks is the number of ticks this thread ran since the last fss_newpri
172482a7749SJerry Jelinek * invocation.
173482a7749SJerry Jelinek *
174482a7749SJerry Jelinek * Finally the process's new user-mode priority is calculated using the
175482a7749SJerry Jelinek * following formula:
176482a7749SJerry Jelinek *
177482a7749SJerry Jelinek * (fsspri * umdprirange)
178482a7749SJerry Jelinek * umdpri = maxumdpri - ------------------------
179482a7749SJerry Jelinek * maxfsspri
180482a7749SJerry Jelinek *
181482a7749SJerry Jelinek * - maxumdpri is MINCLSYSPRI - 1 (i.e. 59)
182482a7749SJerry Jelinek * - umdprirange is maxumdpri - 1 (i.e. 58)
183482a7749SJerry Jelinek * - maxfsspri is the largest fsspri seen so far, as we're iterating all
184482a7749SJerry Jelinek * runnable processes
185482a7749SJerry Jelinek *
186482a7749SJerry Jelinek * Thus, a higher internal priority (fsspri) leads to a lower user-mode
187482a7749SJerry Jelinek * priority which means the thread runs less. The fsspri is higher when
188482a7749SJerry Jelinek * the project's normalized share usage is higher, when the project has
189482a7749SJerry Jelinek * more runnable threads, or when the thread has accumulated more run-time.
190482a7749SJerry Jelinek *
191482a7749SJerry Jelinek * This code has various checks to ensure the resulting umdpri is in the
192482a7749SJerry Jelinek * range 1-59. See fss_newpri for more details.
193482a7749SJerry Jelinek *
194482a7749SJerry Jelinek * To reiterate, the above processing is performed once per second to recompute
195482a7749SJerry Jelinek * the runnable thread user-mode priorities.
196482a7749SJerry Jelinek *
197482a7749SJerry Jelinek * 3) The final major component in the priority calculation is the tick
198482a7749SJerry Jelinek * processing which occurs on a thread that is running when the clock
199482a7749SJerry Jelinek * calls fss_tick.
200482a7749SJerry Jelinek *
201482a7749SJerry Jelinek * A thread can run continuously in user-land (compute-bound) for the
202482a7749SJerry Jelinek * fss_quantum (see "dispadmin -c FSS -g" for the configurable properties).
203482a7749SJerry Jelinek * The fss_quantum defaults to 11 (i.e. 11 ticks).
204482a7749SJerry Jelinek *
205482a7749SJerry Jelinek * Once the quantum has been consumed, the thread will call fss_newpri to
206482a7749SJerry Jelinek * recompute its umdpri priority, as described above in (2b). Threads that
207482a7749SJerry Jelinek * were T_ONPROC at the one second interval when runnable thread priorities
208482a7749SJerry Jelinek * were recalculated will have their umdpri priority recalculated when their
209482a7749SJerry Jelinek * quanta expires.
210482a7749SJerry Jelinek *
211482a7749SJerry Jelinek * To ensure that runnable threads within a project see the expected
212482a7749SJerry Jelinek * round-robin behavior, there is a special case in fss_newpri for a thread
213482a7749SJerry Jelinek * that has run for its quanta within the one second update interval. See
214482a7749SJerry Jelinek * the handling for the quanta_up parameter within fss_newpri.
215482a7749SJerry Jelinek *
216482a7749SJerry Jelinek * Also of interest, the fss_tick code increments the project's tick value
217482a7749SJerry Jelinek * using the fss_nice_tick array entry for the thread's nice value. The idea
218482a7749SJerry Jelinek * behind the fss_nice_tick array is that the cost of a tick is lower at
219482a7749SJerry Jelinek * positive nice values (so that it doesn't increase the project's usage
220482a7749SJerry Jelinek * as much as normal) with a 50% drop at the maximum level and a 50%
221482a7749SJerry Jelinek * increase at the minimum level. See (1b). The fss_nice_tick array is
222482a7749SJerry Jelinek * initialized in fss_init using the following formula:
223482a7749SJerry Jelinek *
224482a7749SJerry Jelinek * FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2) - i)
225482a7749SJerry Jelinek * --------------------------------------------------
226482a7749SJerry Jelinek * FSS_NICE_RANGE
227482a7749SJerry Jelinek *
228482a7749SJerry Jelinek * - FSS_TICK_COST is 1000, the tick cost for threads with nice level 0
229482a7749SJerry Jelinek *
2307c478bd9Sstevel@tonic-gate * FSS Data Structures:
2317c478bd9Sstevel@tonic-gate *
2327c478bd9Sstevel@tonic-gate * fsszone
2337c478bd9Sstevel@tonic-gate * ----- -----
2347c478bd9Sstevel@tonic-gate * ----- | | | |
2357c478bd9Sstevel@tonic-gate * | |-------->| |<------->| |<---->...
2367c478bd9Sstevel@tonic-gate * | | ----- -----
2377c478bd9Sstevel@tonic-gate * | | ^ ^ ^
2387c478bd9Sstevel@tonic-gate * | |--- | \ \
2397c478bd9Sstevel@tonic-gate * ----- | | \ \
2407c478bd9Sstevel@tonic-gate * fsspset | | \ \
2417c478bd9Sstevel@tonic-gate * | | \ \
2427c478bd9Sstevel@tonic-gate * | ----- ----- -----
2437c478bd9Sstevel@tonic-gate * -->| |<--->| |<--->| |
2447c478bd9Sstevel@tonic-gate * | | | | | |
2457c478bd9Sstevel@tonic-gate * ----- ----- -----
2467c478bd9Sstevel@tonic-gate * fssproj
2477c478bd9Sstevel@tonic-gate *
2487c478bd9Sstevel@tonic-gate * That is, fsspsets contain a list of fsszone's that are currently active in
2497c478bd9Sstevel@tonic-gate * the pset, and a list of fssproj's, corresponding to projects with runnable
2507c478bd9Sstevel@tonic-gate * threads on the pset. fssproj's in turn point to the fsszone which they
2517c478bd9Sstevel@tonic-gate * are a member of.
2527c478bd9Sstevel@tonic-gate *
2537c478bd9Sstevel@tonic-gate * An fssproj_t is removed when there are no threads in it.
2547c478bd9Sstevel@tonic-gate *
2557c478bd9Sstevel@tonic-gate * An fsszone_t is removed when there are no projects with threads in it.
2567c478bd9Sstevel@tonic-gate */
2577c478bd9Sstevel@tonic-gate
2587c478bd9Sstevel@tonic-gate static pri_t fss_init(id_t, int, classfuncs_t **);
2597c478bd9Sstevel@tonic-gate
2607c478bd9Sstevel@tonic-gate static struct sclass fss = {
2617c478bd9Sstevel@tonic-gate "FSS",
2627c478bd9Sstevel@tonic-gate fss_init,
2637c478bd9Sstevel@tonic-gate 0
2647c478bd9Sstevel@tonic-gate };
2657c478bd9Sstevel@tonic-gate
2667c478bd9Sstevel@tonic-gate extern struct mod_ops mod_schedops;
2677c478bd9Sstevel@tonic-gate
2687c478bd9Sstevel@tonic-gate /*
2697c478bd9Sstevel@tonic-gate * Module linkage information for the kernel.
2707c478bd9Sstevel@tonic-gate */
2717c478bd9Sstevel@tonic-gate static struct modlsched modlsched = {
2727c478bd9Sstevel@tonic-gate &mod_schedops, "fair share scheduling class", &fss
2737c478bd9Sstevel@tonic-gate };
2747c478bd9Sstevel@tonic-gate
2757c478bd9Sstevel@tonic-gate static struct modlinkage modlinkage = {
2767c478bd9Sstevel@tonic-gate MODREV_1, (void *)&modlsched, NULL
2777c478bd9Sstevel@tonic-gate };
2787c478bd9Sstevel@tonic-gate
2797c478bd9Sstevel@tonic-gate #define FSS_MAXUPRI 60
2807c478bd9Sstevel@tonic-gate
2817c478bd9Sstevel@tonic-gate /*
2827c478bd9Sstevel@tonic-gate * The fssproc_t structures are kept in an array of circular doubly linked
2837c478bd9Sstevel@tonic-gate * lists. A hash on the thread pointer is used to determine which list each
2847c478bd9Sstevel@tonic-gate * thread should be placed in. Each list has a dummy "head" which is never
2857c478bd9Sstevel@tonic-gate * removed, so the list is never empty. fss_update traverses these lists to
2867c478bd9Sstevel@tonic-gate * update the priorities of threads that have been waiting on the run queue.
2877c478bd9Sstevel@tonic-gate */
2887c478bd9Sstevel@tonic-gate #define FSS_LISTS 16 /* number of lists, must be power of 2 */
2897c478bd9Sstevel@tonic-gate #define FSS_LIST_HASH(t) (((uintptr_t)(t) >> 9) & (FSS_LISTS - 1))
2907c478bd9Sstevel@tonic-gate #define FSS_LIST_NEXT(i) (((i) + 1) & (FSS_LISTS - 1))
2917c478bd9Sstevel@tonic-gate
2927c478bd9Sstevel@tonic-gate #define FSS_LIST_INSERT(fssproc) \
2937c478bd9Sstevel@tonic-gate { \
2947c478bd9Sstevel@tonic-gate int index = FSS_LIST_HASH(fssproc->fss_tp); \
2957c478bd9Sstevel@tonic-gate kmutex_t *lockp = &fss_listlock[index]; \
2967c478bd9Sstevel@tonic-gate fssproc_t *headp = &fss_listhead[index]; \
2977c478bd9Sstevel@tonic-gate mutex_enter(lockp); \
2987c478bd9Sstevel@tonic-gate fssproc->fss_next = headp->fss_next; \
2997c478bd9Sstevel@tonic-gate fssproc->fss_prev = headp; \
3007c478bd9Sstevel@tonic-gate headp->fss_next->fss_prev = fssproc; \
3017c478bd9Sstevel@tonic-gate headp->fss_next = fssproc; \
3027c478bd9Sstevel@tonic-gate mutex_exit(lockp); \
3037c478bd9Sstevel@tonic-gate }
3047c478bd9Sstevel@tonic-gate
3057c478bd9Sstevel@tonic-gate #define FSS_LIST_DELETE(fssproc) \
3067c478bd9Sstevel@tonic-gate { \
3077c478bd9Sstevel@tonic-gate int index = FSS_LIST_HASH(fssproc->fss_tp); \
3087c478bd9Sstevel@tonic-gate kmutex_t *lockp = &fss_listlock[index]; \
3097c478bd9Sstevel@tonic-gate mutex_enter(lockp); \
3107c478bd9Sstevel@tonic-gate fssproc->fss_prev->fss_next = fssproc->fss_next; \
3117c478bd9Sstevel@tonic-gate fssproc->fss_next->fss_prev = fssproc->fss_prev; \
3127c478bd9Sstevel@tonic-gate mutex_exit(lockp); \
3137c478bd9Sstevel@tonic-gate }
3147c478bd9Sstevel@tonic-gate
3157c478bd9Sstevel@tonic-gate #define FSS_TICK_COST 1000 /* tick cost for threads with nice level = 0 */
3167c478bd9Sstevel@tonic-gate
3177c478bd9Sstevel@tonic-gate /*
3187c478bd9Sstevel@tonic-gate * Decay rate percentages are based on n/128 rather than n/100 so that
3197c478bd9Sstevel@tonic-gate * calculations can avoid having to do an integer divide by 100 (divide
3207c478bd9Sstevel@tonic-gate * by FSS_DECAY_BASE == 128 optimizes to an arithmetic shift).
3217c478bd9Sstevel@tonic-gate *
3227c478bd9Sstevel@tonic-gate * FSS_DECAY_MIN = 83/128 ~= 65%
3237c478bd9Sstevel@tonic-gate * FSS_DECAY_MAX = 108/128 ~= 85%
3247c478bd9Sstevel@tonic-gate * FSS_DECAY_USG = 96/128 ~= 75%
3257c478bd9Sstevel@tonic-gate */
3267c478bd9Sstevel@tonic-gate #define FSS_DECAY_MIN 83 /* fsspri decay pct for threads w/ nice -20 */
3277c478bd9Sstevel@tonic-gate #define FSS_DECAY_MAX 108 /* fsspri decay pct for threads w/ nice +19 */
3287c478bd9Sstevel@tonic-gate #define FSS_DECAY_USG 96 /* fssusage decay pct for projects */
3297c478bd9Sstevel@tonic-gate #define FSS_DECAY_BASE 128 /* base for decay percentages above */
3307c478bd9Sstevel@tonic-gate
3317c478bd9Sstevel@tonic-gate #define FSS_NICE_MIN 0
3327c478bd9Sstevel@tonic-gate #define FSS_NICE_MAX (2 * NZERO - 1)
3337c478bd9Sstevel@tonic-gate #define FSS_NICE_RANGE (FSS_NICE_MAX - FSS_NICE_MIN + 1)
3347c478bd9Sstevel@tonic-gate
3357c478bd9Sstevel@tonic-gate static int fss_nice_tick[FSS_NICE_RANGE];
3367c478bd9Sstevel@tonic-gate static int fss_nice_decay[FSS_NICE_RANGE];
3377c478bd9Sstevel@tonic-gate
3387c478bd9Sstevel@tonic-gate static pri_t fss_maxupri = FSS_MAXUPRI; /* maximum FSS user priority */
3397c478bd9Sstevel@tonic-gate static pri_t fss_maxumdpri; /* maximum user mode fss priority */
3407c478bd9Sstevel@tonic-gate static pri_t fss_maxglobpri; /* maximum global priority used by fss class */
3417c478bd9Sstevel@tonic-gate static pri_t fss_minglobpri; /* minimum global priority */
3427c478bd9Sstevel@tonic-gate
3437c478bd9Sstevel@tonic-gate static fssproc_t fss_listhead[FSS_LISTS];
3447c478bd9Sstevel@tonic-gate static kmutex_t fss_listlock[FSS_LISTS];
3457c478bd9Sstevel@tonic-gate
3467c478bd9Sstevel@tonic-gate static fsspset_t *fsspsets;
3477c478bd9Sstevel@tonic-gate static kmutex_t fsspsets_lock; /* protects fsspsets */
3487c478bd9Sstevel@tonic-gate
3497c478bd9Sstevel@tonic-gate static id_t fss_cid;
3507c478bd9Sstevel@tonic-gate
3517c478bd9Sstevel@tonic-gate static time_t fss_minrun = 2; /* t_pri becomes 59 within 2 secs */
3527c478bd9Sstevel@tonic-gate static time_t fss_minslp = 2; /* min time on sleep queue for hardswap */
3537c478bd9Sstevel@tonic-gate static int fss_quantum = 11;
3547c478bd9Sstevel@tonic-gate
355482a7749SJerry Jelinek static void fss_newpri(fssproc_t *, boolean_t);
3567c478bd9Sstevel@tonic-gate static void fss_update(void *);
3577c478bd9Sstevel@tonic-gate static int fss_update_list(int);
3587c478bd9Sstevel@tonic-gate static void fss_change_priority(kthread_t *, fssproc_t *);
3597c478bd9Sstevel@tonic-gate
3607c478bd9Sstevel@tonic-gate static int fss_admin(caddr_t, cred_t *);
3617c478bd9Sstevel@tonic-gate static int fss_getclinfo(void *);
3627c478bd9Sstevel@tonic-gate static int fss_parmsin(void *);
3637c478bd9Sstevel@tonic-gate static int fss_parmsout(void *, pc_vaparms_t *);
3647c478bd9Sstevel@tonic-gate static int fss_vaparmsin(void *, pc_vaparms_t *);
3657c478bd9Sstevel@tonic-gate static int fss_vaparmsout(void *, pc_vaparms_t *);
3667c478bd9Sstevel@tonic-gate static int fss_getclpri(pcpri_t *);
3677c478bd9Sstevel@tonic-gate static int fss_alloc(void **, int);
3687c478bd9Sstevel@tonic-gate static void fss_free(void *);
3697c478bd9Sstevel@tonic-gate
3707c478bd9Sstevel@tonic-gate static int fss_enterclass(kthread_t *, id_t, void *, cred_t *, void *);
3717c478bd9Sstevel@tonic-gate static void fss_exitclass(void *);
3727c478bd9Sstevel@tonic-gate static int fss_canexit(kthread_t *, cred_t *);
3737c478bd9Sstevel@tonic-gate static int fss_fork(kthread_t *, kthread_t *, void *);
3747c478bd9Sstevel@tonic-gate static void fss_forkret(kthread_t *, kthread_t *);
3757c478bd9Sstevel@tonic-gate static void fss_parmsget(kthread_t *, void *);
3767c478bd9Sstevel@tonic-gate static int fss_parmsset(kthread_t *, void *, id_t, cred_t *);
3777c478bd9Sstevel@tonic-gate static void fss_stop(kthread_t *, int, int);
3787c478bd9Sstevel@tonic-gate static void fss_exit(kthread_t *);
3797c478bd9Sstevel@tonic-gate static void fss_active(kthread_t *);
3807c478bd9Sstevel@tonic-gate static void fss_inactive(kthread_t *);
3817c478bd9Sstevel@tonic-gate static pri_t fss_swapin(kthread_t *, int);
3827c478bd9Sstevel@tonic-gate static pri_t fss_swapout(kthread_t *, int);
3837c478bd9Sstevel@tonic-gate static void fss_trapret(kthread_t *);
3847c478bd9Sstevel@tonic-gate static void fss_preempt(kthread_t *);
3857c478bd9Sstevel@tonic-gate static void fss_setrun(kthread_t *);
3867c478bd9Sstevel@tonic-gate static void fss_sleep(kthread_t *);
3877c478bd9Sstevel@tonic-gate static void fss_tick(kthread_t *);
3887c478bd9Sstevel@tonic-gate static void fss_wakeup(kthread_t *);
3897c478bd9Sstevel@tonic-gate static int fss_donice(kthread_t *, cred_t *, int, int *);
390d4204c85Sraf static int fss_doprio(kthread_t *, cred_t *, int, int *);
3917c478bd9Sstevel@tonic-gate static pri_t fss_globpri(kthread_t *);
3927c478bd9Sstevel@tonic-gate static void fss_yield(kthread_t *);
3937c478bd9Sstevel@tonic-gate static void fss_nullsys();
3947c478bd9Sstevel@tonic-gate
3957c478bd9Sstevel@tonic-gate static struct classfuncs fss_classfuncs = {
3967c478bd9Sstevel@tonic-gate /* class functions */
3977c478bd9Sstevel@tonic-gate fss_admin,
3987c478bd9Sstevel@tonic-gate fss_getclinfo,
3997c478bd9Sstevel@tonic-gate fss_parmsin,
4007c478bd9Sstevel@tonic-gate fss_parmsout,
4017c478bd9Sstevel@tonic-gate fss_vaparmsin,
4027c478bd9Sstevel@tonic-gate fss_vaparmsout,
4037c478bd9Sstevel@tonic-gate fss_getclpri,
4047c478bd9Sstevel@tonic-gate fss_alloc,
4057c478bd9Sstevel@tonic-gate fss_free,
4067c478bd9Sstevel@tonic-gate
4077c478bd9Sstevel@tonic-gate /* thread functions */
4087c478bd9Sstevel@tonic-gate fss_enterclass,
4097c478bd9Sstevel@tonic-gate fss_exitclass,
4107c478bd9Sstevel@tonic-gate fss_canexit,
4117c478bd9Sstevel@tonic-gate fss_fork,
4127c478bd9Sstevel@tonic-gate fss_forkret,
4137c478bd9Sstevel@tonic-gate fss_parmsget,
4147c478bd9Sstevel@tonic-gate fss_parmsset,
4157c478bd9Sstevel@tonic-gate fss_stop,
4167c478bd9Sstevel@tonic-gate fss_exit,
4177c478bd9Sstevel@tonic-gate fss_active,
4187c478bd9Sstevel@tonic-gate fss_inactive,
4197c478bd9Sstevel@tonic-gate fss_swapin,
4207c478bd9Sstevel@tonic-gate fss_swapout,
4217c478bd9Sstevel@tonic-gate fss_trapret,
4227c478bd9Sstevel@tonic-gate fss_preempt,
4237c478bd9Sstevel@tonic-gate fss_setrun,
4247c478bd9Sstevel@tonic-gate fss_sleep,
4257c478bd9Sstevel@tonic-gate fss_tick,
4267c478bd9Sstevel@tonic-gate fss_wakeup,
4277c478bd9Sstevel@tonic-gate fss_donice,
4287c478bd9Sstevel@tonic-gate fss_globpri,
4297c478bd9Sstevel@tonic-gate fss_nullsys, /* set_process_group */
430d4204c85Sraf fss_yield,
431d4204c85Sraf fss_doprio,
4327c478bd9Sstevel@tonic-gate };
4337c478bd9Sstevel@tonic-gate
4347c478bd9Sstevel@tonic-gate int
_init()4357c478bd9Sstevel@tonic-gate _init()
4367c478bd9Sstevel@tonic-gate {
4377c478bd9Sstevel@tonic-gate return (mod_install(&modlinkage));
4387c478bd9Sstevel@tonic-gate }
4397c478bd9Sstevel@tonic-gate
4407c478bd9Sstevel@tonic-gate int
_fini()4417c478bd9Sstevel@tonic-gate _fini()
4427c478bd9Sstevel@tonic-gate {
4437c478bd9Sstevel@tonic-gate return (EBUSY);
4447c478bd9Sstevel@tonic-gate }
4457c478bd9Sstevel@tonic-gate
4467c478bd9Sstevel@tonic-gate int
_info(struct modinfo * modinfop)4477c478bd9Sstevel@tonic-gate _info(struct modinfo *modinfop)
4487c478bd9Sstevel@tonic-gate {
4497c478bd9Sstevel@tonic-gate return (mod_info(&modlinkage, modinfop));
4507c478bd9Sstevel@tonic-gate }
4517c478bd9Sstevel@tonic-gate
4527c478bd9Sstevel@tonic-gate /*ARGSUSED*/
4537c478bd9Sstevel@tonic-gate static int
fss_project_walker(kproject_t * kpj,void * buf)4547c478bd9Sstevel@tonic-gate fss_project_walker(kproject_t *kpj, void *buf)
4557c478bd9Sstevel@tonic-gate {
4567c478bd9Sstevel@tonic-gate return (0);
4577c478bd9Sstevel@tonic-gate }
4587c478bd9Sstevel@tonic-gate
4597c478bd9Sstevel@tonic-gate void *
fss_allocbuf(int op,int type)4607c478bd9Sstevel@tonic-gate fss_allocbuf(int op, int type)
4617c478bd9Sstevel@tonic-gate {
4627c478bd9Sstevel@tonic-gate fssbuf_t *fssbuf;
4637c478bd9Sstevel@tonic-gate void **fsslist;
4647c478bd9Sstevel@tonic-gate int cnt;
4657c478bd9Sstevel@tonic-gate int i;
4667c478bd9Sstevel@tonic-gate size_t size;
4677c478bd9Sstevel@tonic-gate
4687c478bd9Sstevel@tonic-gate ASSERT(op == FSS_NPSET_BUF || op == FSS_NPROJ_BUF || op == FSS_ONE_BUF);
4697c478bd9Sstevel@tonic-gate ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
4707c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock));
4717c478bd9Sstevel@tonic-gate
4727c478bd9Sstevel@tonic-gate fssbuf = kmem_zalloc(sizeof (fssbuf_t), KM_SLEEP);
4737c478bd9Sstevel@tonic-gate switch (op) {
4747c478bd9Sstevel@tonic-gate case FSS_NPSET_BUF:
4757c478bd9Sstevel@tonic-gate cnt = cpupart_list(NULL, 0, CP_NONEMPTY);
4767c478bd9Sstevel@tonic-gate break;
4777c478bd9Sstevel@tonic-gate case FSS_NPROJ_BUF:
4787c478bd9Sstevel@tonic-gate cnt = project_walk_all(ALL_ZONES, fss_project_walker, NULL);
4797c478bd9Sstevel@tonic-gate break;
4807c478bd9Sstevel@tonic-gate case FSS_ONE_BUF:
4817c478bd9Sstevel@tonic-gate cnt = 1;
4827c478bd9Sstevel@tonic-gate break;
4837c478bd9Sstevel@tonic-gate }
4847c478bd9Sstevel@tonic-gate
4857c478bd9Sstevel@tonic-gate switch (type) {
4867c478bd9Sstevel@tonic-gate case FSS_ALLOC_PROJ:
4877c478bd9Sstevel@tonic-gate size = sizeof (fssproj_t);
4887c478bd9Sstevel@tonic-gate break;
4897c478bd9Sstevel@tonic-gate case FSS_ALLOC_ZONE:
4907c478bd9Sstevel@tonic-gate size = sizeof (fsszone_t);
4917c478bd9Sstevel@tonic-gate break;
4927c478bd9Sstevel@tonic-gate }
4937c478bd9Sstevel@tonic-gate fsslist = kmem_zalloc(cnt * sizeof (void *), KM_SLEEP);
4947c478bd9Sstevel@tonic-gate fssbuf->fssb_size = cnt;
4957c478bd9Sstevel@tonic-gate fssbuf->fssb_list = fsslist;
4967c478bd9Sstevel@tonic-gate for (i = 0; i < cnt; i++)
4977c478bd9Sstevel@tonic-gate fsslist[i] = kmem_zalloc(size, KM_SLEEP);
4987c478bd9Sstevel@tonic-gate return (fssbuf);
4997c478bd9Sstevel@tonic-gate }
5007c478bd9Sstevel@tonic-gate
5017c478bd9Sstevel@tonic-gate void
fss_freebuf(fssbuf_t * fssbuf,int type)5027c478bd9Sstevel@tonic-gate fss_freebuf(fssbuf_t *fssbuf, int type)
5037c478bd9Sstevel@tonic-gate {
5047c478bd9Sstevel@tonic-gate void **fsslist;
5057c478bd9Sstevel@tonic-gate int i;
5067c478bd9Sstevel@tonic-gate size_t size;
5077c478bd9Sstevel@tonic-gate
5087c478bd9Sstevel@tonic-gate ASSERT(fssbuf != NULL);
5097c478bd9Sstevel@tonic-gate ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
5107c478bd9Sstevel@tonic-gate fsslist = fssbuf->fssb_list;
5117c478bd9Sstevel@tonic-gate
5127c478bd9Sstevel@tonic-gate switch (type) {
5137c478bd9Sstevel@tonic-gate case FSS_ALLOC_PROJ:
5147c478bd9Sstevel@tonic-gate size = sizeof (fssproj_t);
5157c478bd9Sstevel@tonic-gate break;
5167c478bd9Sstevel@tonic-gate case FSS_ALLOC_ZONE:
5177c478bd9Sstevel@tonic-gate size = sizeof (fsszone_t);
5187c478bd9Sstevel@tonic-gate break;
5197c478bd9Sstevel@tonic-gate }
5207c478bd9Sstevel@tonic-gate
5217c478bd9Sstevel@tonic-gate for (i = 0; i < fssbuf->fssb_size; i++) {
5227c478bd9Sstevel@tonic-gate if (fsslist[i] != NULL)
5237c478bd9Sstevel@tonic-gate kmem_free(fsslist[i], size);
5247c478bd9Sstevel@tonic-gate }
5257c478bd9Sstevel@tonic-gate kmem_free(fsslist, sizeof (void *) * fssbuf->fssb_size);
5267c478bd9Sstevel@tonic-gate kmem_free(fssbuf, sizeof (fssbuf_t));
5277c478bd9Sstevel@tonic-gate }
5287c478bd9Sstevel@tonic-gate
5297c478bd9Sstevel@tonic-gate static fsspset_t *
fss_find_fsspset(cpupart_t * cpupart)5307c478bd9Sstevel@tonic-gate fss_find_fsspset(cpupart_t *cpupart)
5317c478bd9Sstevel@tonic-gate {
5327c478bd9Sstevel@tonic-gate int i;
5337c478bd9Sstevel@tonic-gate fsspset_t *fsspset = NULL;
5347c478bd9Sstevel@tonic-gate int found = 0;
5357c478bd9Sstevel@tonic-gate
5367c478bd9Sstevel@tonic-gate ASSERT(cpupart != NULL);
5377c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&fsspsets_lock));
5387c478bd9Sstevel@tonic-gate
5397c478bd9Sstevel@tonic-gate /*
5407c478bd9Sstevel@tonic-gate * Search for the cpupart pointer in the array of fsspsets.
5417c478bd9Sstevel@tonic-gate */
5427c478bd9Sstevel@tonic-gate for (i = 0; i < max_ncpus; i++) {
5437c478bd9Sstevel@tonic-gate fsspset = &fsspsets[i];
5447c478bd9Sstevel@tonic-gate if (fsspset->fssps_cpupart == cpupart) {
5457c478bd9Sstevel@tonic-gate ASSERT(fsspset->fssps_nproj > 0);
5467c478bd9Sstevel@tonic-gate found = 1;
5477c478bd9Sstevel@tonic-gate break;
5487c478bd9Sstevel@tonic-gate }
5497c478bd9Sstevel@tonic-gate }
5507c478bd9Sstevel@tonic-gate if (found == 0) {
5517c478bd9Sstevel@tonic-gate /*
5527c478bd9Sstevel@tonic-gate * If we didn't find anything, then use the first
5537c478bd9Sstevel@tonic-gate * available slot in the fsspsets array.
5547c478bd9Sstevel@tonic-gate */
5557c478bd9Sstevel@tonic-gate for (i = 0; i < max_ncpus; i++) {
5567c478bd9Sstevel@tonic-gate fsspset = &fsspsets[i];
5577c478bd9Sstevel@tonic-gate if (fsspset->fssps_cpupart == NULL) {
5587c478bd9Sstevel@tonic-gate ASSERT(fsspset->fssps_nproj == 0);
5597c478bd9Sstevel@tonic-gate found = 1;
5607c478bd9Sstevel@tonic-gate break;
5617c478bd9Sstevel@tonic-gate }
5627c478bd9Sstevel@tonic-gate }
5637c478bd9Sstevel@tonic-gate fsspset->fssps_cpupart = cpupart;
5647c478bd9Sstevel@tonic-gate }
5657c478bd9Sstevel@tonic-gate ASSERT(found == 1);
5667c478bd9Sstevel@tonic-gate return (fsspset);
5677c478bd9Sstevel@tonic-gate }
5687c478bd9Sstevel@tonic-gate
5697c478bd9Sstevel@tonic-gate static void
fss_del_fsspset(fsspset_t * fsspset)5707c478bd9Sstevel@tonic-gate fss_del_fsspset(fsspset_t *fsspset)
5717c478bd9Sstevel@tonic-gate {
5727c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&fsspsets_lock));
5737c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
5747c478bd9Sstevel@tonic-gate ASSERT(fsspset->fssps_nproj == 0);
5757c478bd9Sstevel@tonic-gate ASSERT(fsspset->fssps_list == NULL);
5767c478bd9Sstevel@tonic-gate ASSERT(fsspset->fssps_zones == NULL);
5777c478bd9Sstevel@tonic-gate fsspset->fssps_cpupart = NULL;
5787c478bd9Sstevel@tonic-gate fsspset->fssps_maxfsspri = 0;
5797c478bd9Sstevel@tonic-gate fsspset->fssps_shares = 0;
5807c478bd9Sstevel@tonic-gate }
5817c478bd9Sstevel@tonic-gate
5827c478bd9Sstevel@tonic-gate /*
5837c478bd9Sstevel@tonic-gate * The following routine returns a pointer to the fsszone structure which
5847c478bd9Sstevel@tonic-gate * belongs to zone "zone" and cpu partition fsspset, if such structure exists.
5857c478bd9Sstevel@tonic-gate */
5867c478bd9Sstevel@tonic-gate static fsszone_t *
fss_find_fsszone(fsspset_t * fsspset,zone_t * zone)5877c478bd9Sstevel@tonic-gate fss_find_fsszone(fsspset_t *fsspset, zone_t *zone)
5887c478bd9Sstevel@tonic-gate {
5897c478bd9Sstevel@tonic-gate fsszone_t *fsszone;
5907c478bd9Sstevel@tonic-gate
5917c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
5927c478bd9Sstevel@tonic-gate
5937c478bd9Sstevel@tonic-gate if (fsspset->fssps_list != NULL) {
5947c478bd9Sstevel@tonic-gate /*
5957c478bd9Sstevel@tonic-gate * There are projects/zones active on this cpu partition
5967c478bd9Sstevel@tonic-gate * already. Try to find our zone among them.
5977c478bd9Sstevel@tonic-gate */
5987c478bd9Sstevel@tonic-gate fsszone = fsspset->fssps_zones;
5997c478bd9Sstevel@tonic-gate do {
6007c478bd9Sstevel@tonic-gate if (fsszone->fssz_zone == zone) {
6017c478bd9Sstevel@tonic-gate return (fsszone);
6027c478bd9Sstevel@tonic-gate }
6037c478bd9Sstevel@tonic-gate fsszone = fsszone->fssz_next;
6047c478bd9Sstevel@tonic-gate } while (fsszone != fsspset->fssps_zones);
6057c478bd9Sstevel@tonic-gate }
6067c478bd9Sstevel@tonic-gate return (NULL);
6077c478bd9Sstevel@tonic-gate }
6087c478bd9Sstevel@tonic-gate
6097c478bd9Sstevel@tonic-gate /*
6107c478bd9Sstevel@tonic-gate * The following routine links new fsszone structure into doubly linked list of
6117c478bd9Sstevel@tonic-gate * zones active on the specified cpu partition.
6127c478bd9Sstevel@tonic-gate */
6137c478bd9Sstevel@tonic-gate static void
fss_insert_fsszone(fsspset_t * fsspset,zone_t * zone,fsszone_t * fsszone)6147c478bd9Sstevel@tonic-gate fss_insert_fsszone(fsspset_t *fsspset, zone_t *zone, fsszone_t *fsszone)
6157c478bd9Sstevel@tonic-gate {
6167c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
6177c478bd9Sstevel@tonic-gate
6187c478bd9Sstevel@tonic-gate fsszone->fssz_zone = zone;
6197c478bd9Sstevel@tonic-gate fsszone->fssz_rshares = zone->zone_shares;
6207c478bd9Sstevel@tonic-gate
6217c478bd9Sstevel@tonic-gate if (fsspset->fssps_zones == NULL) {
6227c478bd9Sstevel@tonic-gate /*
6237c478bd9Sstevel@tonic-gate * This will be the first fsszone for this fsspset
6247c478bd9Sstevel@tonic-gate */
6257c478bd9Sstevel@tonic-gate fsszone->fssz_next = fsszone->fssz_prev = fsszone;
6267c478bd9Sstevel@tonic-gate fsspset->fssps_zones = fsszone;
6277c478bd9Sstevel@tonic-gate } else {
6287c478bd9Sstevel@tonic-gate /*
6297c478bd9Sstevel@tonic-gate * Insert this fsszone to the doubly linked list.
6307c478bd9Sstevel@tonic-gate */
6317c478bd9Sstevel@tonic-gate fsszone_t *fssz_head = fsspset->fssps_zones;
6327c478bd9Sstevel@tonic-gate
6337c478bd9Sstevel@tonic-gate fsszone->fssz_next = fssz_head;
6347c478bd9Sstevel@tonic-gate fsszone->fssz_prev = fssz_head->fssz_prev;
6357c478bd9Sstevel@tonic-gate fssz_head->fssz_prev->fssz_next = fsszone;
6367c478bd9Sstevel@tonic-gate fssz_head->fssz_prev = fsszone;
6377c478bd9Sstevel@tonic-gate fsspset->fssps_zones = fsszone;
6387c478bd9Sstevel@tonic-gate }
6397c478bd9Sstevel@tonic-gate }
6407c478bd9Sstevel@tonic-gate
6417c478bd9Sstevel@tonic-gate /*
6427c478bd9Sstevel@tonic-gate * The following routine removes a single fsszone structure from the doubly
6437c478bd9Sstevel@tonic-gate * linked list of zones active on the specified cpu partition. Note that
6447c478bd9Sstevel@tonic-gate * global fsspsets_lock must be held in case this fsszone structure is the last
6457c478bd9Sstevel@tonic-gate * on the above mentioned list. Also note that the fsszone structure is not
6467c478bd9Sstevel@tonic-gate * freed here, it is the responsibility of the caller to call kmem_free for it.
6477c478bd9Sstevel@tonic-gate */
6487c478bd9Sstevel@tonic-gate static void
fss_remove_fsszone(fsspset_t * fsspset,fsszone_t * fsszone)6497c478bd9Sstevel@tonic-gate fss_remove_fsszone(fsspset_t *fsspset, fsszone_t *fsszone)
6507c478bd9Sstevel@tonic-gate {
6517c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
6527c478bd9Sstevel@tonic-gate ASSERT(fsszone->fssz_nproj == 0);
6537c478bd9Sstevel@tonic-gate ASSERT(fsszone->fssz_shares == 0);
6547c478bd9Sstevel@tonic-gate ASSERT(fsszone->fssz_runnable == 0);
6557c478bd9Sstevel@tonic-gate
6567c478bd9Sstevel@tonic-gate if (fsszone->fssz_next != fsszone) {
6577c478bd9Sstevel@tonic-gate /*
6587c478bd9Sstevel@tonic-gate * This is not the last zone in the list.
6597c478bd9Sstevel@tonic-gate */
6607c478bd9Sstevel@tonic-gate fsszone->fssz_prev->fssz_next = fsszone->fssz_next;
6617c478bd9Sstevel@tonic-gate fsszone->fssz_next->fssz_prev = fsszone->fssz_prev;
6627c478bd9Sstevel@tonic-gate if (fsspset->fssps_zones == fsszone)
6637c478bd9Sstevel@tonic-gate fsspset->fssps_zones = fsszone->fssz_next;
6647c478bd9Sstevel@tonic-gate } else {
6657c478bd9Sstevel@tonic-gate /*
6667c478bd9Sstevel@tonic-gate * This was the last zone active in this cpu partition.
6677c478bd9Sstevel@tonic-gate */
6687c478bd9Sstevel@tonic-gate fsspset->fssps_zones = NULL;
6697c478bd9Sstevel@tonic-gate }
6707c478bd9Sstevel@tonic-gate }
6717c478bd9Sstevel@tonic-gate
6727c478bd9Sstevel@tonic-gate /*
6737c478bd9Sstevel@tonic-gate * The following routine returns a pointer to the fssproj structure
6747c478bd9Sstevel@tonic-gate * which belongs to project kpj and cpu partition fsspset, if such structure
6757c478bd9Sstevel@tonic-gate * exists.
6767c478bd9Sstevel@tonic-gate */
6777c478bd9Sstevel@tonic-gate static fssproj_t *
fss_find_fssproj(fsspset_t * fsspset,kproject_t * kpj)6787c478bd9Sstevel@tonic-gate fss_find_fssproj(fsspset_t *fsspset, kproject_t *kpj)
6797c478bd9Sstevel@tonic-gate {
6807c478bd9Sstevel@tonic-gate fssproj_t *fssproj;
6817c478bd9Sstevel@tonic-gate
6827c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
6837c478bd9Sstevel@tonic-gate
6847c478bd9Sstevel@tonic-gate if (fsspset->fssps_list != NULL) {
6857c478bd9Sstevel@tonic-gate /*
6867c478bd9Sstevel@tonic-gate * There are projects running on this cpu partition already.
6877c478bd9Sstevel@tonic-gate * Try to find our project among them.
6887c478bd9Sstevel@tonic-gate */
6897c478bd9Sstevel@tonic-gate fssproj = fsspset->fssps_list;
6907c478bd9Sstevel@tonic-gate do {
6917c478bd9Sstevel@tonic-gate if (fssproj->fssp_proj == kpj) {
6927c478bd9Sstevel@tonic-gate ASSERT(fssproj->fssp_pset == fsspset);
6937c478bd9Sstevel@tonic-gate return (fssproj);
6947c478bd9Sstevel@tonic-gate }
6957c478bd9Sstevel@tonic-gate fssproj = fssproj->fssp_next;
6967c478bd9Sstevel@tonic-gate } while (fssproj != fsspset->fssps_list);
6977c478bd9Sstevel@tonic-gate }
6987c478bd9Sstevel@tonic-gate return (NULL);
6997c478bd9Sstevel@tonic-gate }
7007c478bd9Sstevel@tonic-gate
7017c478bd9Sstevel@tonic-gate /*
7027c478bd9Sstevel@tonic-gate * The following routine links new fssproj structure into doubly linked list
7037c478bd9Sstevel@tonic-gate * of projects running on the specified cpu partition.
7047c478bd9Sstevel@tonic-gate */
7057c478bd9Sstevel@tonic-gate static void
fss_insert_fssproj(fsspset_t * fsspset,kproject_t * kpj,fsszone_t * fsszone,fssproj_t * fssproj)7067c478bd9Sstevel@tonic-gate fss_insert_fssproj(fsspset_t *fsspset, kproject_t *kpj, fsszone_t *fsszone,
7077c478bd9Sstevel@tonic-gate fssproj_t *fssproj)
7087c478bd9Sstevel@tonic-gate {
7097c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
7107c478bd9Sstevel@tonic-gate
7117c478bd9Sstevel@tonic-gate fssproj->fssp_pset = fsspset;
7127c478bd9Sstevel@tonic-gate fssproj->fssp_proj = kpj;
7137c478bd9Sstevel@tonic-gate fssproj->fssp_shares = kpj->kpj_shares;
7147c478bd9Sstevel@tonic-gate
7157c478bd9Sstevel@tonic-gate fsspset->fssps_nproj++;
7167c478bd9Sstevel@tonic-gate
7177c478bd9Sstevel@tonic-gate if (fsspset->fssps_list == NULL) {
7187c478bd9Sstevel@tonic-gate /*
7197c478bd9Sstevel@tonic-gate * This will be the first fssproj for this fsspset
7207c478bd9Sstevel@tonic-gate */
7217c478bd9Sstevel@tonic-gate fssproj->fssp_next = fssproj->fssp_prev = fssproj;
7227c478bd9Sstevel@tonic-gate fsspset->fssps_list = fssproj;
7237c478bd9Sstevel@tonic-gate } else {
7247c478bd9Sstevel@tonic-gate /*
7257c478bd9Sstevel@tonic-gate * Insert this fssproj to the doubly linked list.
7267c478bd9Sstevel@tonic-gate */
7277c478bd9Sstevel@tonic-gate fssproj_t *fssp_head = fsspset->fssps_list;
7287c478bd9Sstevel@tonic-gate
7297c478bd9Sstevel@tonic-gate fssproj->fssp_next = fssp_head;
7307c478bd9Sstevel@tonic-gate fssproj->fssp_prev = fssp_head->fssp_prev;
7317c478bd9Sstevel@tonic-gate fssp_head->fssp_prev->fssp_next = fssproj;
7327c478bd9Sstevel@tonic-gate fssp_head->fssp_prev = fssproj;
7337c478bd9Sstevel@tonic-gate fsspset->fssps_list = fssproj;
7347c478bd9Sstevel@tonic-gate }
7357c478bd9Sstevel@tonic-gate fssproj->fssp_fsszone = fsszone;
7367c478bd9Sstevel@tonic-gate fsszone->fssz_nproj++;
7377c478bd9Sstevel@tonic-gate ASSERT(fsszone->fssz_nproj != 0);
7387c478bd9Sstevel@tonic-gate }
7397c478bd9Sstevel@tonic-gate
7407c478bd9Sstevel@tonic-gate /*
7417c478bd9Sstevel@tonic-gate * The following routine removes a single fssproj structure from the doubly
7427c478bd9Sstevel@tonic-gate * linked list of projects running on the specified cpu partition. Note that
7437c478bd9Sstevel@tonic-gate * global fsspsets_lock must be held in case if this fssproj structure is the
7447c478bd9Sstevel@tonic-gate * last on the above mentioned list. Also note that the fssproj structure is
7457c478bd9Sstevel@tonic-gate * not freed here, it is the responsibility of the caller to call kmem_free
7467c478bd9Sstevel@tonic-gate * for it.
7477c478bd9Sstevel@tonic-gate */
7487c478bd9Sstevel@tonic-gate static void
fss_remove_fssproj(fsspset_t * fsspset,fssproj_t * fssproj)7497c478bd9Sstevel@tonic-gate fss_remove_fssproj(fsspset_t *fsspset, fssproj_t *fssproj)
7507c478bd9Sstevel@tonic-gate {
7517c478bd9Sstevel@tonic-gate fsszone_t *fsszone;
7527c478bd9Sstevel@tonic-gate
7537c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&fsspsets_lock));
7547c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
7557c478bd9Sstevel@tonic-gate ASSERT(fssproj->fssp_runnable == 0);
7567c478bd9Sstevel@tonic-gate
7577c478bd9Sstevel@tonic-gate fsspset->fssps_nproj--;
7587c478bd9Sstevel@tonic-gate
7597c478bd9Sstevel@tonic-gate fsszone = fssproj->fssp_fsszone;
7607c478bd9Sstevel@tonic-gate fsszone->fssz_nproj--;
7617c478bd9Sstevel@tonic-gate
7627c478bd9Sstevel@tonic-gate if (fssproj->fssp_next != fssproj) {
7637c478bd9Sstevel@tonic-gate /*
7647c478bd9Sstevel@tonic-gate * This is not the last part in the list.
7657c478bd9Sstevel@tonic-gate */
7667c478bd9Sstevel@tonic-gate fssproj->fssp_prev->fssp_next = fssproj->fssp_next;
7677c478bd9Sstevel@tonic-gate fssproj->fssp_next->fssp_prev = fssproj->fssp_prev;
7687c478bd9Sstevel@tonic-gate if (fsspset->fssps_list == fssproj)
7697c478bd9Sstevel@tonic-gate fsspset->fssps_list = fssproj->fssp_next;
7707c478bd9Sstevel@tonic-gate if (fsszone->fssz_nproj == 0)
7717c478bd9Sstevel@tonic-gate fss_remove_fsszone(fsspset, fsszone);
7727c478bd9Sstevel@tonic-gate } else {
7737c478bd9Sstevel@tonic-gate /*
7747c478bd9Sstevel@tonic-gate * This was the last project part running
7757c478bd9Sstevel@tonic-gate * at this cpu partition.
7767c478bd9Sstevel@tonic-gate */
7777c478bd9Sstevel@tonic-gate fsspset->fssps_list = NULL;
7787c478bd9Sstevel@tonic-gate ASSERT(fsspset->fssps_nproj == 0);
7797c478bd9Sstevel@tonic-gate ASSERT(fsszone->fssz_nproj == 0);
7807c478bd9Sstevel@tonic-gate fss_remove_fsszone(fsspset, fsszone);
7817c478bd9Sstevel@tonic-gate fss_del_fsspset(fsspset);
7827c478bd9Sstevel@tonic-gate }
7837c478bd9Sstevel@tonic-gate }
7847c478bd9Sstevel@tonic-gate
7857c478bd9Sstevel@tonic-gate static void
fss_inactive(kthread_t * t)7867c478bd9Sstevel@tonic-gate fss_inactive(kthread_t *t)
7877c478bd9Sstevel@tonic-gate {
7887c478bd9Sstevel@tonic-gate fssproc_t *fssproc;
7897c478bd9Sstevel@tonic-gate fssproj_t *fssproj;
7907c478bd9Sstevel@tonic-gate fsspset_t *fsspset;
7917c478bd9Sstevel@tonic-gate fsszone_t *fsszone;
7927c478bd9Sstevel@tonic-gate
7937c478bd9Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(t));
7947c478bd9Sstevel@tonic-gate fssproc = FSSPROC(t);
7957c478bd9Sstevel@tonic-gate fssproj = FSSPROC2FSSPROJ(fssproc);
7967c478bd9Sstevel@tonic-gate if (fssproj == NULL) /* if this thread already exited */
7977c478bd9Sstevel@tonic-gate return;
7987c478bd9Sstevel@tonic-gate fsspset = FSSPROJ2FSSPSET(fssproj);
7997c478bd9Sstevel@tonic-gate fsszone = fssproj->fssp_fsszone;
8007c478bd9Sstevel@tonic-gate disp_lock_enter_high(&fsspset->fssps_displock);
8017c478bd9Sstevel@tonic-gate ASSERT(fssproj->fssp_runnable > 0);
8027c478bd9Sstevel@tonic-gate if (--fssproj->fssp_runnable == 0) {
8037c478bd9Sstevel@tonic-gate fsszone->fssz_shares -= fssproj->fssp_shares;
8047c478bd9Sstevel@tonic-gate if (--fsszone->fssz_runnable == 0)
8057c478bd9Sstevel@tonic-gate fsspset->fssps_shares -= fsszone->fssz_rshares;
8067c478bd9Sstevel@tonic-gate }
8077c478bd9Sstevel@tonic-gate ASSERT(fssproc->fss_runnable == 1);
8087c478bd9Sstevel@tonic-gate fssproc->fss_runnable = 0;
8097c478bd9Sstevel@tonic-gate disp_lock_exit_high(&fsspset->fssps_displock);
8107c478bd9Sstevel@tonic-gate }
8117c478bd9Sstevel@tonic-gate
8127c478bd9Sstevel@tonic-gate static void
fss_active(kthread_t * t)8137c478bd9Sstevel@tonic-gate fss_active(kthread_t *t)
8147c478bd9Sstevel@tonic-gate {
8157c478bd9Sstevel@tonic-gate fssproc_t *fssproc;
8167c478bd9Sstevel@tonic-gate fssproj_t *fssproj;
8177c478bd9Sstevel@tonic-gate fsspset_t *fsspset;
8187c478bd9Sstevel@tonic-gate fsszone_t *fsszone;
8197c478bd9Sstevel@tonic-gate
8207c478bd9Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(t));
8217c478bd9Sstevel@tonic-gate fssproc = FSSPROC(t);
8227c478bd9Sstevel@tonic-gate fssproj = FSSPROC2FSSPROJ(fssproc);
8237c478bd9Sstevel@tonic-gate if (fssproj == NULL) /* if this thread already exited */
8247c478bd9Sstevel@tonic-gate return;
8257c478bd9Sstevel@tonic-gate fsspset = FSSPROJ2FSSPSET(fssproj);
8267c478bd9Sstevel@tonic-gate fsszone = fssproj->fssp_fsszone;
8277c478bd9Sstevel@tonic-gate disp_lock_enter_high(&fsspset->fssps_displock);
8287c478bd9Sstevel@tonic-gate if (++fssproj->fssp_runnable == 1) {
8297c478bd9Sstevel@tonic-gate fsszone->fssz_shares += fssproj->fssp_shares;
8307c478bd9Sstevel@tonic-gate if (++fsszone->fssz_runnable == 1)
8317c478bd9Sstevel@tonic-gate fsspset->fssps_shares += fsszone->fssz_rshares;
8327c478bd9Sstevel@tonic-gate }
8337c478bd9Sstevel@tonic-gate ASSERT(fssproc->fss_runnable == 0);
8347c478bd9Sstevel@tonic-gate fssproc->fss_runnable = 1;
8357c478bd9Sstevel@tonic-gate disp_lock_exit_high(&fsspset->fssps_displock);
8367c478bd9Sstevel@tonic-gate }
8377c478bd9Sstevel@tonic-gate
8387c478bd9Sstevel@tonic-gate /*
8397c478bd9Sstevel@tonic-gate * Fair share scheduler initialization. Called by dispinit() at boot time.
8407c478bd9Sstevel@tonic-gate * We can ignore clparmsz argument since we know that the smallest possible
8417c478bd9Sstevel@tonic-gate * parameter buffer is big enough for us.
8427c478bd9Sstevel@tonic-gate */
8437c478bd9Sstevel@tonic-gate /*ARGSUSED*/
8447c478bd9Sstevel@tonic-gate static pri_t
fss_init(id_t cid,int clparmsz,classfuncs_t ** clfuncspp)8457c478bd9Sstevel@tonic-gate fss_init(id_t cid, int clparmsz, classfuncs_t **clfuncspp)
8467c478bd9Sstevel@tonic-gate {
8477c478bd9Sstevel@tonic-gate int i;
8487c478bd9Sstevel@tonic-gate
8497c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock));
8507c478bd9Sstevel@tonic-gate
8517c478bd9Sstevel@tonic-gate fss_cid = cid;
8527c478bd9Sstevel@tonic-gate fss_maxumdpri = minclsyspri - 1;
8537c478bd9Sstevel@tonic-gate fss_maxglobpri = minclsyspri;
8547c478bd9Sstevel@tonic-gate fss_minglobpri = 0;
8557c478bd9Sstevel@tonic-gate fsspsets = kmem_zalloc(sizeof (fsspset_t) * max_ncpus, KM_SLEEP);
8567c478bd9Sstevel@tonic-gate
8577c478bd9Sstevel@tonic-gate /*
8587c478bd9Sstevel@tonic-gate * Initialize the fssproc hash table.
8597c478bd9Sstevel@tonic-gate */
8607c478bd9Sstevel@tonic-gate for (i = 0; i < FSS_LISTS; i++)
8617c478bd9Sstevel@tonic-gate fss_listhead[i].fss_next = fss_listhead[i].fss_prev =
8627c478bd9Sstevel@tonic-gate &fss_listhead[i];
8637c478bd9Sstevel@tonic-gate
8647c478bd9Sstevel@tonic-gate *clfuncspp = &fss_classfuncs;
8657c478bd9Sstevel@tonic-gate
8667c478bd9Sstevel@tonic-gate /*
8677c478bd9Sstevel@tonic-gate * Fill in fss_nice_tick and fss_nice_decay arrays:
8687c478bd9Sstevel@tonic-gate * The cost of a tick is lower at positive nice values (so that it
8697c478bd9Sstevel@tonic-gate * will not increase its project's usage as much as normal) with 50%
8707c478bd9Sstevel@tonic-gate * drop at the maximum level and 50% increase at the minimum level.
8717c478bd9Sstevel@tonic-gate * The fsspri decay is slower at positive nice values. fsspri values
8727c478bd9Sstevel@tonic-gate * of processes with negative nice levels must decay faster to receive
8737c478bd9Sstevel@tonic-gate * time slices more frequently than normal.
8747c478bd9Sstevel@tonic-gate */
8757c478bd9Sstevel@tonic-gate for (i = 0; i < FSS_NICE_RANGE; i++) {
8767c478bd9Sstevel@tonic-gate fss_nice_tick[i] = (FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2)
8777c478bd9Sstevel@tonic-gate - i)) / FSS_NICE_RANGE;
8787c478bd9Sstevel@tonic-gate fss_nice_decay[i] = FSS_DECAY_MIN +
8797c478bd9Sstevel@tonic-gate ((FSS_DECAY_MAX - FSS_DECAY_MIN) * i) /
8807c478bd9Sstevel@tonic-gate (FSS_NICE_RANGE - 1);
8817c478bd9Sstevel@tonic-gate }
8827c478bd9Sstevel@tonic-gate
8837c478bd9Sstevel@tonic-gate return (fss_maxglobpri);
8847c478bd9Sstevel@tonic-gate }
8857c478bd9Sstevel@tonic-gate
8867c478bd9Sstevel@tonic-gate /*
887482a7749SJerry Jelinek * Calculate the new fss_umdpri based on the usage, the normalized share usage
888482a7749SJerry Jelinek * and the number of active threads. Reset the tick counter for this thread.
889482a7749SJerry Jelinek *
890482a7749SJerry Jelinek * When calculating the new priority using the standard formula we can hit
891482a7749SJerry Jelinek * a scenario where we don't have good round-robin behavior. This would be
892482a7749SJerry Jelinek * most commonly seen when there is a zone with lots of runnable threads.
893482a7749SJerry Jelinek * In the bad scenario we will see the following behavior when using the
894482a7749SJerry Jelinek * standard formula and these conditions:
895482a7749SJerry Jelinek *
896482a7749SJerry Jelinek * - there are multiple runnable threads in the zone (project)
897482a7749SJerry Jelinek * - the fssps_maxfsspri is a very large value
898482a7749SJerry Jelinek * - (we also know all of these threads will use the project's
899482a7749SJerry Jelinek * fssp_shusage)
900482a7749SJerry Jelinek *
901482a7749SJerry Jelinek * Under these conditions, a thread with a low fss_fsspri value is chosen
902482a7749SJerry Jelinek * to run and the thread gets a high fss_umdpri. This thread can run for
903482a7749SJerry Jelinek * its full quanta (fss_timeleft) at which time fss_newpri is called to
904482a7749SJerry Jelinek * calculate the thread's new priority.
905482a7749SJerry Jelinek *
906482a7749SJerry Jelinek * In this case, because the newly calculated fsspri value is much smaller
907482a7749SJerry Jelinek * (orders of magnitude) than the fssps_maxfsspri value, if we used the
908482a7749SJerry Jelinek * standard formula the thread will still get a high fss_umdpri value and
909482a7749SJerry Jelinek * will run again for another quanta, even though there are other runnable
910482a7749SJerry Jelinek * threads in the project.
911482a7749SJerry Jelinek *
912482a7749SJerry Jelinek * For a thread that is runnable for a long time, the thread can continue
913482a7749SJerry Jelinek * to run for many quanta (totaling many seconds) before the thread's fsspri
914482a7749SJerry Jelinek * exceeds the fssps_maxfsspri and the thread's fss_umdpri is reset back
915482a7749SJerry Jelinek * down to 1. This behavior also keeps the fssps_maxfsspr at a high value,
916482a7749SJerry Jelinek * so that the next runnable thread might repeat this cycle.
917482a7749SJerry Jelinek *
918482a7749SJerry Jelinek * This leads to the case where we don't have round-robin behavior at quanta
919482a7749SJerry Jelinek * granularity, but instead, runnable threads within the project only run
920482a7749SJerry Jelinek * at several second intervals.
921482a7749SJerry Jelinek *
922482a7749SJerry Jelinek * To prevent this scenario from occuring, when a thread has consumed its
923482a7749SJerry Jelinek * quanta and there are multiple runnable threads in the project, we
924482a7749SJerry Jelinek * immediately cause the thread to hit fssps_maxfsspri so that it gets
925482a7749SJerry Jelinek * reset back to 1 and another runnable thread in the project can run.
9267c478bd9Sstevel@tonic-gate */
9277c478bd9Sstevel@tonic-gate static void
fss_newpri(fssproc_t * fssproc,boolean_t quanta_up)928482a7749SJerry Jelinek fss_newpri(fssproc_t *fssproc, boolean_t quanta_up)
9297c478bd9Sstevel@tonic-gate {
9307c478bd9Sstevel@tonic-gate kthread_t *tp;
9317c478bd9Sstevel@tonic-gate fssproj_t *fssproj;
9327c478bd9Sstevel@tonic-gate fsspset_t *fsspset;
9337c478bd9Sstevel@tonic-gate fsszone_t *fsszone;
9347c478bd9Sstevel@tonic-gate fsspri_t fsspri, maxfsspri;
935482a7749SJerry Jelinek uint32_t n_runnable;
9367c478bd9Sstevel@tonic-gate pri_t invpri;
9377c478bd9Sstevel@tonic-gate uint32_t ticks;
9387c478bd9Sstevel@tonic-gate
9397c478bd9Sstevel@tonic-gate tp = fssproc->fss_tp;
9407c478bd9Sstevel@tonic-gate ASSERT(tp != NULL);
9417c478bd9Sstevel@tonic-gate
9427c478bd9Sstevel@tonic-gate if (tp->t_cid != fss_cid)
9437c478bd9Sstevel@tonic-gate return;
9447c478bd9Sstevel@tonic-gate
9457c478bd9Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(tp));
9467c478bd9Sstevel@tonic-gate
9477c478bd9Sstevel@tonic-gate fssproj = FSSPROC2FSSPROJ(fssproc);
9487c478bd9Sstevel@tonic-gate fsszone = FSSPROJ2FSSZONE(fssproj);
9497c478bd9Sstevel@tonic-gate if (fssproj == NULL)
9507c478bd9Sstevel@tonic-gate /*
9517c478bd9Sstevel@tonic-gate * No need to change priority of exited threads.
9527c478bd9Sstevel@tonic-gate */
9537c478bd9Sstevel@tonic-gate return;
9547c478bd9Sstevel@tonic-gate
9557c478bd9Sstevel@tonic-gate fsspset = FSSPROJ2FSSPSET(fssproj);
9567c478bd9Sstevel@tonic-gate disp_lock_enter_high(&fsspset->fssps_displock);
9577c478bd9Sstevel@tonic-gate
958482a7749SJerry Jelinek ticks = fssproc->fss_ticks;
959482a7749SJerry Jelinek fssproc->fss_ticks = 0;
960482a7749SJerry Jelinek
9617c478bd9Sstevel@tonic-gate if (fssproj->fssp_shares == 0 || fsszone->fssz_rshares == 0) {
9627c478bd9Sstevel@tonic-gate /*
9637c478bd9Sstevel@tonic-gate * Special case: threads with no shares.
9647c478bd9Sstevel@tonic-gate */
9657c478bd9Sstevel@tonic-gate fssproc->fss_umdpri = fss_minglobpri;
9667c478bd9Sstevel@tonic-gate disp_lock_exit_high(&fsspset->fssps_displock);
9677c478bd9Sstevel@tonic-gate return;
9687c478bd9Sstevel@tonic-gate }
9697c478bd9Sstevel@tonic-gate
970482a7749SJerry Jelinek maxfsspri = fsspset->fssps_maxfsspri;
971482a7749SJerry Jelinek n_runnable = fssproj->fssp_runnable;
972482a7749SJerry Jelinek
973482a7749SJerry Jelinek if (quanta_up && n_runnable > 1) {
974482a7749SJerry Jelinek fsspri = maxfsspri;
975482a7749SJerry Jelinek } else {
976482a7749SJerry Jelinek /*
977482a7749SJerry Jelinek * fsspri += fssp_shusage * nrunnable * ticks
978482a7749SJerry Jelinek * If all three values are non-0, this typically calculates to
979482a7749SJerry Jelinek * a large number (sometimes > 1M, sometimes > 100B) due to
980482a7749SJerry Jelinek * fssp_shusage which can be > 1T.
981482a7749SJerry Jelinek */
982482a7749SJerry Jelinek fsspri = fssproc->fss_fsspri;
983482a7749SJerry Jelinek fsspri += fssproj->fssp_shusage * n_runnable * ticks;
984482a7749SJerry Jelinek }
985482a7749SJerry Jelinek
9867c478bd9Sstevel@tonic-gate fssproc->fss_fsspri = fsspri;
9877c478bd9Sstevel@tonic-gate
988482a7749SJerry Jelinek /*
989482a7749SJerry Jelinek * fss_maxumdpri is normally 59, since FSS priorities are 0-59.
990482a7749SJerry Jelinek * If the previous calculation resulted in 0 (e.g. was 0 and added 0
991482a7749SJerry Jelinek * because ticks == 0), then instead of 0, we use the largest priority,
992482a7749SJerry Jelinek * which is still small in comparison to the large numbers we typically
993482a7749SJerry Jelinek * see.
994482a7749SJerry Jelinek */
9957c478bd9Sstevel@tonic-gate if (fsspri < fss_maxumdpri)
9967c478bd9Sstevel@tonic-gate fsspri = fss_maxumdpri; /* so that maxfsspri is != 0 */
9977c478bd9Sstevel@tonic-gate
9987c478bd9Sstevel@tonic-gate /*
9997c478bd9Sstevel@tonic-gate * The general priority formula:
10007c478bd9Sstevel@tonic-gate *
10017c478bd9Sstevel@tonic-gate * (fsspri * umdprirange)
10027c478bd9Sstevel@tonic-gate * pri = maxumdpri - ------------------------
10037c478bd9Sstevel@tonic-gate * maxfsspri
10047c478bd9Sstevel@tonic-gate *
10057c478bd9Sstevel@tonic-gate * If this thread's fsspri is greater than the previous largest
10067c478bd9Sstevel@tonic-gate * fsspri, then record it as the new high and priority for this
10077c478bd9Sstevel@tonic-gate * thread will be one (the lowest priority assigned to a thread
1008482a7749SJerry Jelinek * that has non-zero shares). Because of this check, maxfsspri can
1009482a7749SJerry Jelinek * change as this function is called via the
1010482a7749SJerry Jelinek * fss_update -> fss_update_list -> fss_newpri code path to update
1011482a7749SJerry Jelinek * all runnable threads. See the code in fss_update for how we
1012482a7749SJerry Jelinek * mitigate this issue.
1013482a7749SJerry Jelinek *
10147c478bd9Sstevel@tonic-gate * Note that this formula cannot produce out of bounds priority
1015482a7749SJerry Jelinek * values (0-59); if it is changed, additional checks may need to be
10167c478bd9Sstevel@tonic-gate * added.
10177c478bd9Sstevel@tonic-gate */
10187c478bd9Sstevel@tonic-gate if (fsspri >= maxfsspri) {
10197c478bd9Sstevel@tonic-gate fsspset->fssps_maxfsspri = fsspri;
10207c478bd9Sstevel@tonic-gate disp_lock_exit_high(&fsspset->fssps_displock);
10217c478bd9Sstevel@tonic-gate fssproc->fss_umdpri = 1;
10227c478bd9Sstevel@tonic-gate } else {
10237c478bd9Sstevel@tonic-gate disp_lock_exit_high(&fsspset->fssps_displock);
10247c478bd9Sstevel@tonic-gate invpri = (fsspri * (fss_maxumdpri - 1)) / maxfsspri;
10257c478bd9Sstevel@tonic-gate fssproc->fss_umdpri = fss_maxumdpri - invpri;
10267c478bd9Sstevel@tonic-gate }
10277c478bd9Sstevel@tonic-gate }
10287c478bd9Sstevel@tonic-gate
10297c478bd9Sstevel@tonic-gate /*
1030482a7749SJerry Jelinek * Decays usages of all running projects, resets their tick counters and
1031482a7749SJerry Jelinek * calcluates the projects normalized share usage. Called once per second from
1032482a7749SJerry Jelinek * fss_update().
10337c478bd9Sstevel@tonic-gate */
10347c478bd9Sstevel@tonic-gate static void
fss_decay_usage()10357c478bd9Sstevel@tonic-gate fss_decay_usage()
10367c478bd9Sstevel@tonic-gate {
10377c478bd9Sstevel@tonic-gate uint32_t zone_ext_shares, zone_int_shares;
10387c478bd9Sstevel@tonic-gate uint32_t kpj_shares, pset_shares;
10397c478bd9Sstevel@tonic-gate fsspset_t *fsspset;
10407c478bd9Sstevel@tonic-gate fssproj_t *fssproj;
10417c478bd9Sstevel@tonic-gate fsszone_t *fsszone;
10427c478bd9Sstevel@tonic-gate fsspri_t maxfsspri;
10437c478bd9Sstevel@tonic-gate int psetid;
1044482a7749SJerry Jelinek struct zone *zp;
10457c478bd9Sstevel@tonic-gate
10467c478bd9Sstevel@tonic-gate mutex_enter(&fsspsets_lock);
10477c478bd9Sstevel@tonic-gate /*
10487c478bd9Sstevel@tonic-gate * Go through all active processor sets and decay usages of projects
10497c478bd9Sstevel@tonic-gate * running on them.
10507c478bd9Sstevel@tonic-gate */
10517c478bd9Sstevel@tonic-gate for (psetid = 0; psetid < max_ncpus; psetid++) {
10527c478bd9Sstevel@tonic-gate fsspset = &fsspsets[psetid];
10537c478bd9Sstevel@tonic-gate mutex_enter(&fsspset->fssps_lock);
10547c478bd9Sstevel@tonic-gate
1055482a7749SJerry Jelinek fsspset->fssps_gen++;
1056482a7749SJerry Jelinek
10577c478bd9Sstevel@tonic-gate if (fsspset->fssps_cpupart == NULL ||
10587c478bd9Sstevel@tonic-gate (fssproj = fsspset->fssps_list) == NULL) {
10597c478bd9Sstevel@tonic-gate mutex_exit(&fsspset->fssps_lock);
10607c478bd9Sstevel@tonic-gate continue;
10617c478bd9Sstevel@tonic-gate }
10627c478bd9Sstevel@tonic-gate
10637c478bd9Sstevel@tonic-gate /*
10647c478bd9Sstevel@tonic-gate * Decay maxfsspri for this cpu partition with the
10657c478bd9Sstevel@tonic-gate * fastest possible decay rate.
10667c478bd9Sstevel@tonic-gate */
10677c478bd9Sstevel@tonic-gate disp_lock_enter(&fsspset->fssps_displock);
10687c478bd9Sstevel@tonic-gate
1069482a7749SJerry Jelinek pset_shares = fsspset->fssps_shares;
1070482a7749SJerry Jelinek
10717c478bd9Sstevel@tonic-gate maxfsspri = (fsspset->fssps_maxfsspri *
10727c478bd9Sstevel@tonic-gate fss_nice_decay[NZERO]) / FSS_DECAY_BASE;
10737c478bd9Sstevel@tonic-gate if (maxfsspri < fss_maxumdpri)
10747c478bd9Sstevel@tonic-gate maxfsspri = fss_maxumdpri;
10757c478bd9Sstevel@tonic-gate fsspset->fssps_maxfsspri = maxfsspri;
10767c478bd9Sstevel@tonic-gate
10777c478bd9Sstevel@tonic-gate do {
1078482a7749SJerry Jelinek fsszone = fssproj->fssp_fsszone;
1079482a7749SJerry Jelinek zp = fsszone->fssz_zone;
1080482a7749SJerry Jelinek
10817c478bd9Sstevel@tonic-gate /*
1082482a7749SJerry Jelinek * Reset zone's FSS stats if they are from a
1083482a7749SJerry Jelinek * previous cycle.
1084482a7749SJerry Jelinek */
1085482a7749SJerry Jelinek if (fsspset->fssps_gen != zp->zone_fss_gen) {
1086482a7749SJerry Jelinek zp->zone_fss_gen = fsspset->fssps_gen;
1087482a7749SJerry Jelinek zp->zone_run_ticks = 0;
1088482a7749SJerry Jelinek }
1089482a7749SJerry Jelinek
1090482a7749SJerry Jelinek /*
1091482a7749SJerry Jelinek * Decay project usage, then add in this cycle's
1092482a7749SJerry Jelinek * nice tick value.
10937c478bd9Sstevel@tonic-gate */
10947c478bd9Sstevel@tonic-gate fssproj->fssp_usage =
10957c478bd9Sstevel@tonic-gate (fssproj->fssp_usage * FSS_DECAY_USG) /
1096482a7749SJerry Jelinek FSS_DECAY_BASE +
1097482a7749SJerry Jelinek fssproj->fssp_ticks;
1098482a7749SJerry Jelinek
10997c478bd9Sstevel@tonic-gate fssproj->fssp_ticks = 0;
1100482a7749SJerry Jelinek zp->zone_run_ticks += fssproj->fssp_tick_cnt;
1101482a7749SJerry Jelinek fssproj->fssp_tick_cnt = 0;
11027c478bd9Sstevel@tonic-gate
11037c478bd9Sstevel@tonic-gate /*
11047c478bd9Sstevel@tonic-gate * Readjust the project's number of shares if it has
11057c478bd9Sstevel@tonic-gate * changed since we checked it last time.
11067c478bd9Sstevel@tonic-gate */
11077c478bd9Sstevel@tonic-gate kpj_shares = fssproj->fssp_proj->kpj_shares;
11087c478bd9Sstevel@tonic-gate if (fssproj->fssp_shares != kpj_shares) {
11097c478bd9Sstevel@tonic-gate if (fssproj->fssp_runnable != 0) {
11107c478bd9Sstevel@tonic-gate fsszone->fssz_shares -=
11117c478bd9Sstevel@tonic-gate fssproj->fssp_shares;
11127c478bd9Sstevel@tonic-gate fsszone->fssz_shares += kpj_shares;
11137c478bd9Sstevel@tonic-gate }
11147c478bd9Sstevel@tonic-gate fssproj->fssp_shares = kpj_shares;
11157c478bd9Sstevel@tonic-gate }
11167c478bd9Sstevel@tonic-gate
11177c478bd9Sstevel@tonic-gate /*
11187c478bd9Sstevel@tonic-gate * Readjust the zone's number of shares if it
11197c478bd9Sstevel@tonic-gate * has changed since we checked it last time.
11207c478bd9Sstevel@tonic-gate */
1121482a7749SJerry Jelinek zone_ext_shares = zp->zone_shares;
11227c478bd9Sstevel@tonic-gate if (fsszone->fssz_rshares != zone_ext_shares) {
11237c478bd9Sstevel@tonic-gate if (fsszone->fssz_runnable != 0) {
11247c478bd9Sstevel@tonic-gate fsspset->fssps_shares -=
11257c478bd9Sstevel@tonic-gate fsszone->fssz_rshares;
11267c478bd9Sstevel@tonic-gate fsspset->fssps_shares +=
11277c478bd9Sstevel@tonic-gate zone_ext_shares;
1128482a7749SJerry Jelinek pset_shares = fsspset->fssps_shares;
11297c478bd9Sstevel@tonic-gate }
11307c478bd9Sstevel@tonic-gate fsszone->fssz_rshares = zone_ext_shares;
11317c478bd9Sstevel@tonic-gate }
11327c478bd9Sstevel@tonic-gate zone_int_shares = fsszone->fssz_shares;
1133482a7749SJerry Jelinek
1134482a7749SJerry Jelinek /*
1135482a7749SJerry Jelinek * If anything is runnable in the project, track the
1136482a7749SJerry Jelinek * overall project share percent for monitoring useage.
1137482a7749SJerry Jelinek */
1138482a7749SJerry Jelinek if (fssproj->fssp_runnable > 0) {
1139482a7749SJerry Jelinek uint32_t zone_shr_pct;
1140482a7749SJerry Jelinek uint32_t int_shr_pct;
1141482a7749SJerry Jelinek
1142482a7749SJerry Jelinek /*
1143482a7749SJerry Jelinek * Times 1000 to get tenths of a percent
1144482a7749SJerry Jelinek *
1145482a7749SJerry Jelinek * zone_ext_shares
1146482a7749SJerry Jelinek * zone_shr_pct = ---------------
1147482a7749SJerry Jelinek * pset_shares
1148482a7749SJerry Jelinek *
1149482a7749SJerry Jelinek * kpj_shares
1150482a7749SJerry Jelinek * int_shr_pct = ---------------
1151482a7749SJerry Jelinek * zone_int_shares
1152482a7749SJerry Jelinek */
1153482a7749SJerry Jelinek if (pset_shares == 0 || zone_int_shares == 0) {
1154482a7749SJerry Jelinek fssproj->fssp_shr_pct = 0;
1155482a7749SJerry Jelinek } else {
1156482a7749SJerry Jelinek zone_shr_pct =
1157482a7749SJerry Jelinek (zone_ext_shares * 1000) /
1158482a7749SJerry Jelinek pset_shares;
1159482a7749SJerry Jelinek int_shr_pct = (kpj_shares * 1000) /
1160482a7749SJerry Jelinek zone_int_shares;
1161482a7749SJerry Jelinek fssproj->fssp_shr_pct =
1162482a7749SJerry Jelinek (zone_shr_pct * int_shr_pct) /
1163482a7749SJerry Jelinek 1000;
1164482a7749SJerry Jelinek }
1165482a7749SJerry Jelinek } else {
1166482a7749SJerry Jelinek DTRACE_PROBE1(fss__prj__norun, fssproj_t *,
1167482a7749SJerry Jelinek fssproj);
1168482a7749SJerry Jelinek }
1169482a7749SJerry Jelinek
11707c478bd9Sstevel@tonic-gate /*
11717c478bd9Sstevel@tonic-gate * Calculate fssp_shusage value to be used
11727c478bd9Sstevel@tonic-gate * for fsspri increments for the next second.
11737c478bd9Sstevel@tonic-gate */
1174a14d39b7Sandrei if (kpj_shares == 0 || zone_ext_shares == 0) {
1175a14d39b7Sandrei fssproj->fssp_shusage = 0;
1176a14d39b7Sandrei } else if (FSSPROJ2KPROJ(fssproj) == proj0p) {
1177482a7749SJerry Jelinek uint32_t zone_shr_pct;
1178482a7749SJerry Jelinek
11797c478bd9Sstevel@tonic-gate /*
11807c478bd9Sstevel@tonic-gate * Project 0 in the global zone has 50%
1181482a7749SJerry Jelinek * of its zone. See calculation above for
1182482a7749SJerry Jelinek * the zone's share percent.
11837c478bd9Sstevel@tonic-gate */
1184482a7749SJerry Jelinek if (pset_shares == 0)
1185482a7749SJerry Jelinek zone_shr_pct = 1000;
1186482a7749SJerry Jelinek else
1187482a7749SJerry Jelinek zone_shr_pct =
1188482a7749SJerry Jelinek (zone_ext_shares * 1000) /
1189482a7749SJerry Jelinek pset_shares;
1190482a7749SJerry Jelinek
1191482a7749SJerry Jelinek fssproj->fssp_shr_pct = zone_shr_pct / 2;
1192482a7749SJerry Jelinek
11937c478bd9Sstevel@tonic-gate fssproj->fssp_shusage = (fssproj->fssp_usage *
11947c478bd9Sstevel@tonic-gate zone_int_shares * zone_int_shares) /
11957c478bd9Sstevel@tonic-gate (zone_ext_shares * zone_ext_shares);
11967c478bd9Sstevel@tonic-gate } else {
11977c478bd9Sstevel@tonic-gate /*
11987c478bd9Sstevel@tonic-gate * Thread's priority is based on its project's
11997c478bd9Sstevel@tonic-gate * normalized usage (shusage) value which gets
12007c478bd9Sstevel@tonic-gate * calculated this way:
12017c478bd9Sstevel@tonic-gate *
12027c478bd9Sstevel@tonic-gate * pset_shares^2 zone_int_shares^2
12037c478bd9Sstevel@tonic-gate * usage * ------------- * ------------------
12047c478bd9Sstevel@tonic-gate * kpj_shares^2 zone_ext_shares^2
12057c478bd9Sstevel@tonic-gate *
12067c478bd9Sstevel@tonic-gate * Where zone_int_shares is the sum of shares
12077c478bd9Sstevel@tonic-gate * of all active projects within the zone (and
12087c478bd9Sstevel@tonic-gate * the pset), and zone_ext_shares is the number
12097c478bd9Sstevel@tonic-gate * of zone shares (ie, zone.cpu-shares).
12107c478bd9Sstevel@tonic-gate *
12117c478bd9Sstevel@tonic-gate * If there is only one zone active on the pset
12127c478bd9Sstevel@tonic-gate * the above reduces to:
12137c478bd9Sstevel@tonic-gate *
1214*6a0b1217SPatrick Mooney * zone_int_shares^2
12157c478bd9Sstevel@tonic-gate * shusage = usage * ---------------------
1216*6a0b1217SPatrick Mooney * kpj_shares^2
12177c478bd9Sstevel@tonic-gate *
12187c478bd9Sstevel@tonic-gate * If there's only one project active in the
12197c478bd9Sstevel@tonic-gate * zone this formula reduces to:
12207c478bd9Sstevel@tonic-gate *
12217c478bd9Sstevel@tonic-gate * pset_shares^2
12227c478bd9Sstevel@tonic-gate * shusage = usage * ----------------------
12237c478bd9Sstevel@tonic-gate * zone_ext_shares^2
1224482a7749SJerry Jelinek *
1225482a7749SJerry Jelinek * shusage is one input to calculating fss_pri
1226482a7749SJerry Jelinek * in fss_newpri(). Larger values tend toward
1227482a7749SJerry Jelinek * lower priorities for processes in the proj.
12287c478bd9Sstevel@tonic-gate */
12297c478bd9Sstevel@tonic-gate fssproj->fssp_shusage = fssproj->fssp_usage *
12307c478bd9Sstevel@tonic-gate pset_shares * zone_int_shares;
12317c478bd9Sstevel@tonic-gate fssproj->fssp_shusage /=
12327c478bd9Sstevel@tonic-gate kpj_shares * zone_ext_shares;
12337c478bd9Sstevel@tonic-gate fssproj->fssp_shusage *=
12347c478bd9Sstevel@tonic-gate pset_shares * zone_int_shares;
12357c478bd9Sstevel@tonic-gate fssproj->fssp_shusage /=
12367c478bd9Sstevel@tonic-gate kpj_shares * zone_ext_shares;
12377c478bd9Sstevel@tonic-gate }
12387c478bd9Sstevel@tonic-gate fssproj = fssproj->fssp_next;
12397c478bd9Sstevel@tonic-gate } while (fssproj != fsspset->fssps_list);
12407c478bd9Sstevel@tonic-gate
12417c478bd9Sstevel@tonic-gate disp_lock_exit(&fsspset->fssps_displock);
12427c478bd9Sstevel@tonic-gate mutex_exit(&fsspset->fssps_lock);
12437c478bd9Sstevel@tonic-gate }
12447c478bd9Sstevel@tonic-gate mutex_exit(&fsspsets_lock);
12457c478bd9Sstevel@tonic-gate }
12467c478bd9Sstevel@tonic-gate
12477c478bd9Sstevel@tonic-gate static void
fss_change_priority(kthread_t * t,fssproc_t * fssproc)12487c478bd9Sstevel@tonic-gate fss_change_priority(kthread_t *t, fssproc_t *fssproc)
12497c478bd9Sstevel@tonic-gate {
12507c478bd9Sstevel@tonic-gate pri_t new_pri;
12517c478bd9Sstevel@tonic-gate
12527c478bd9Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(t));
12537c478bd9Sstevel@tonic-gate new_pri = fssproc->fss_umdpri;
12547c478bd9Sstevel@tonic-gate ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
12557c478bd9Sstevel@tonic-gate
1256d4204c85Sraf t->t_cpri = fssproc->fss_upri;
12579ac8606fSraf fssproc->fss_flags &= ~FSSRESTORE;
12587c478bd9Sstevel@tonic-gate if (t == curthread || t->t_state == TS_ONPROC) {
12597c478bd9Sstevel@tonic-gate /*
12607c478bd9Sstevel@tonic-gate * curthread is always onproc
12617c478bd9Sstevel@tonic-gate */
12627c478bd9Sstevel@tonic-gate cpu_t *cp = t->t_disp_queue->disp_cpu;
12637c478bd9Sstevel@tonic-gate THREAD_CHANGE_PRI(t, new_pri);
12647c478bd9Sstevel@tonic-gate if (t == cp->cpu_dispthread)
12657c478bd9Sstevel@tonic-gate cp->cpu_dispatch_pri = DISP_PRIO(t);
12667c478bd9Sstevel@tonic-gate if (DISP_MUST_SURRENDER(t)) {
12677c478bd9Sstevel@tonic-gate fssproc->fss_flags |= FSSBACKQ;
12687c478bd9Sstevel@tonic-gate cpu_surrender(t);
12697c478bd9Sstevel@tonic-gate } else {
12707c478bd9Sstevel@tonic-gate fssproc->fss_timeleft = fss_quantum;
12717c478bd9Sstevel@tonic-gate }
12727c478bd9Sstevel@tonic-gate } else {
12737c478bd9Sstevel@tonic-gate /*
12747c478bd9Sstevel@tonic-gate * When the priority of a thread is changed, it may be
12757c478bd9Sstevel@tonic-gate * necessary to adjust its position on a sleep queue or
12767c478bd9Sstevel@tonic-gate * dispatch queue. The function thread_change_pri accomplishes
12777c478bd9Sstevel@tonic-gate * this.
12787c478bd9Sstevel@tonic-gate */
12797c478bd9Sstevel@tonic-gate if (thread_change_pri(t, new_pri, 0)) {
12807c478bd9Sstevel@tonic-gate /*
12817c478bd9Sstevel@tonic-gate * The thread was on a run queue.
12827c478bd9Sstevel@tonic-gate */
12837c478bd9Sstevel@tonic-gate fssproc->fss_timeleft = fss_quantum;
12847c478bd9Sstevel@tonic-gate } else {
12857c478bd9Sstevel@tonic-gate fssproc->fss_flags |= FSSBACKQ;
12867c478bd9Sstevel@tonic-gate }
12877c478bd9Sstevel@tonic-gate }
12887c478bd9Sstevel@tonic-gate }
12897c478bd9Sstevel@tonic-gate
12907c478bd9Sstevel@tonic-gate /*
12917c478bd9Sstevel@tonic-gate * Update priorities of all fair-sharing threads that are currently runnable
12927c478bd9Sstevel@tonic-gate * at a user mode priority based on the number of shares and current usage.
12937c478bd9Sstevel@tonic-gate * Called once per second via timeout which we reset here.
12947c478bd9Sstevel@tonic-gate *
12957c478bd9Sstevel@tonic-gate * There are several lists of fair-sharing threads broken up by a hash on the
12967c478bd9Sstevel@tonic-gate * thread pointer. Each list has its own lock. This avoids blocking all
12977c478bd9Sstevel@tonic-gate * fss_enterclass, fss_fork, and fss_exitclass operations while fss_update runs.
12987c478bd9Sstevel@tonic-gate * fss_update traverses each list in turn.
1299482a7749SJerry Jelinek *
1300482a7749SJerry Jelinek * Each time we're run (once/second) we may start at the next list and iterate
1301482a7749SJerry Jelinek * through all of the lists. By starting with a different list, we mitigate any
1302482a7749SJerry Jelinek * effects we would see updating the fssps_maxfsspri value in fss_newpri.
13037c478bd9Sstevel@tonic-gate */
13047c478bd9Sstevel@tonic-gate static void
fss_update(void * arg)13057c478bd9Sstevel@tonic-gate fss_update(void *arg)
13067c478bd9Sstevel@tonic-gate {
13077c478bd9Sstevel@tonic-gate int i;
13087c478bd9Sstevel@tonic-gate int new_marker = -1;
13097c478bd9Sstevel@tonic-gate static int fss_update_marker;
13107c478bd9Sstevel@tonic-gate
13117c478bd9Sstevel@tonic-gate /*
13127c478bd9Sstevel@tonic-gate * Decay and update usages for all projects.
13137c478bd9Sstevel@tonic-gate */
13147c478bd9Sstevel@tonic-gate fss_decay_usage();
13157c478bd9Sstevel@tonic-gate
13167c478bd9Sstevel@tonic-gate /*
13177c478bd9Sstevel@tonic-gate * Start with the fss_update_marker list, then do the rest.
13187c478bd9Sstevel@tonic-gate */
13197c478bd9Sstevel@tonic-gate i = fss_update_marker;
13207c478bd9Sstevel@tonic-gate
13217c478bd9Sstevel@tonic-gate /*
13227c478bd9Sstevel@tonic-gate * Go around all threads, set new priorities and decay
13237c478bd9Sstevel@tonic-gate * per-thread CPU usages.
13247c478bd9Sstevel@tonic-gate */
13257c478bd9Sstevel@tonic-gate do {
13267c478bd9Sstevel@tonic-gate /*
13277c478bd9Sstevel@tonic-gate * If this is the first list after the current marker to have
1328482a7749SJerry Jelinek * threads with priority updates, advance the marker to this
13297c478bd9Sstevel@tonic-gate * list for the next time fss_update runs.
13307c478bd9Sstevel@tonic-gate */
13317c478bd9Sstevel@tonic-gate if (fss_update_list(i) &&
13327c478bd9Sstevel@tonic-gate new_marker == -1 && i != fss_update_marker)
13337c478bd9Sstevel@tonic-gate new_marker = i;
13347c478bd9Sstevel@tonic-gate } while ((i = FSS_LIST_NEXT(i)) != fss_update_marker);
13357c478bd9Sstevel@tonic-gate
13367c478bd9Sstevel@tonic-gate /*
13377c478bd9Sstevel@tonic-gate * Advance marker for the next fss_update call
13387c478bd9Sstevel@tonic-gate */
13397c478bd9Sstevel@tonic-gate if (new_marker != -1)
13407c478bd9Sstevel@tonic-gate fss_update_marker = new_marker;
13417c478bd9Sstevel@tonic-gate
13427c478bd9Sstevel@tonic-gate (void) timeout(fss_update, arg, hz);
13437c478bd9Sstevel@tonic-gate }
13447c478bd9Sstevel@tonic-gate
13457c478bd9Sstevel@tonic-gate /*
13467c478bd9Sstevel@tonic-gate * Updates priority for a list of threads. Returns 1 if the priority of one
13477c478bd9Sstevel@tonic-gate * of the threads was actually updated, 0 if none were for various reasons
13487c478bd9Sstevel@tonic-gate * (thread is no longer in the FSS class, is not runnable, has the preemption
13497c478bd9Sstevel@tonic-gate * control no-preempt bit set, etc.)
13507c478bd9Sstevel@tonic-gate */
13517c478bd9Sstevel@tonic-gate static int
fss_update_list(int i)13527c478bd9Sstevel@tonic-gate fss_update_list(int i)
13537c478bd9Sstevel@tonic-gate {
13547c478bd9Sstevel@tonic-gate fssproc_t *fssproc;
13557c478bd9Sstevel@tonic-gate fssproj_t *fssproj;
13567c478bd9Sstevel@tonic-gate fsspri_t fsspri;
1357482a7749SJerry Jelinek pri_t fss_umdpri;
13587c478bd9Sstevel@tonic-gate kthread_t *t;
13597c478bd9Sstevel@tonic-gate int updated = 0;
13607c478bd9Sstevel@tonic-gate
13617c478bd9Sstevel@tonic-gate mutex_enter(&fss_listlock[i]);
13627c478bd9Sstevel@tonic-gate for (fssproc = fss_listhead[i].fss_next; fssproc != &fss_listhead[i];
13637c478bd9Sstevel@tonic-gate fssproc = fssproc->fss_next) {
13647c478bd9Sstevel@tonic-gate t = fssproc->fss_tp;
13657c478bd9Sstevel@tonic-gate /*
13667c478bd9Sstevel@tonic-gate * Lock the thread and verify the state.
13677c478bd9Sstevel@tonic-gate */
13687c478bd9Sstevel@tonic-gate thread_lock(t);
13697c478bd9Sstevel@tonic-gate /*
13707c478bd9Sstevel@tonic-gate * Skip the thread if it is no longer in the FSS class or
13717c478bd9Sstevel@tonic-gate * is running with kernel mode priority.
13727c478bd9Sstevel@tonic-gate */
13737c478bd9Sstevel@tonic-gate if (t->t_cid != fss_cid)
13747c478bd9Sstevel@tonic-gate goto next;
1375c97ad5cdSakolb
13767c478bd9Sstevel@tonic-gate fssproj = FSSPROC2FSSPROJ(fssproc);
13777c478bd9Sstevel@tonic-gate if (fssproj == NULL)
13787c478bd9Sstevel@tonic-gate goto next;
1379482a7749SJerry Jelinek
13807c478bd9Sstevel@tonic-gate if (fssproj->fssp_shares != 0) {
13817c478bd9Sstevel@tonic-gate /*
13827c478bd9Sstevel@tonic-gate * Decay fsspri value.
13837c478bd9Sstevel@tonic-gate */
13847c478bd9Sstevel@tonic-gate fsspri = fssproc->fss_fsspri;
13857c478bd9Sstevel@tonic-gate fsspri = (fsspri * fss_nice_decay[fssproc->fss_nice]) /
13867c478bd9Sstevel@tonic-gate FSS_DECAY_BASE;
13877c478bd9Sstevel@tonic-gate fssproc->fss_fsspri = fsspri;
13887c478bd9Sstevel@tonic-gate }
13897c478bd9Sstevel@tonic-gate
13907c478bd9Sstevel@tonic-gate if (t->t_schedctl && schedctl_get_nopreempt(t))
13917c478bd9Sstevel@tonic-gate goto next;
1392c97ad5cdSakolb if (t->t_state != TS_RUN && t->t_state != TS_WAIT) {
13937c478bd9Sstevel@tonic-gate /*
13947c478bd9Sstevel@tonic-gate * Make next syscall/trap call fss_trapret
13957c478bd9Sstevel@tonic-gate */
13967c478bd9Sstevel@tonic-gate t->t_trapret = 1;
13977c478bd9Sstevel@tonic-gate aston(t);
1398482a7749SJerry Jelinek if (t->t_state == TS_ONPROC)
1399482a7749SJerry Jelinek DTRACE_PROBE1(fss__onproc, fssproc_t *,
1400482a7749SJerry Jelinek fssproc);
14017c478bd9Sstevel@tonic-gate goto next;
14027c478bd9Sstevel@tonic-gate }
1403482a7749SJerry Jelinek fss_newpri(fssproc, B_FALSE);
14047c478bd9Sstevel@tonic-gate updated = 1;
14057c478bd9Sstevel@tonic-gate
1406482a7749SJerry Jelinek fss_umdpri = fssproc->fss_umdpri;
1407482a7749SJerry Jelinek
14087c478bd9Sstevel@tonic-gate /*
14097c478bd9Sstevel@tonic-gate * Only dequeue the thread if it needs to be moved; otherwise
14107c478bd9Sstevel@tonic-gate * it should just round-robin here.
14117c478bd9Sstevel@tonic-gate */
1412482a7749SJerry Jelinek if (t->t_pri != fss_umdpri)
14137c478bd9Sstevel@tonic-gate fss_change_priority(t, fssproc);
14147c478bd9Sstevel@tonic-gate next:
14157c478bd9Sstevel@tonic-gate thread_unlock(t);
14167c478bd9Sstevel@tonic-gate }
14177c478bd9Sstevel@tonic-gate mutex_exit(&fss_listlock[i]);
14187c478bd9Sstevel@tonic-gate return (updated);
14197c478bd9Sstevel@tonic-gate }
14207c478bd9Sstevel@tonic-gate
14217c478bd9Sstevel@tonic-gate /*ARGSUSED*/
14227c478bd9Sstevel@tonic-gate static int
fss_admin(caddr_t uaddr,cred_t * reqpcredp)14237c478bd9Sstevel@tonic-gate fss_admin(caddr_t uaddr, cred_t *reqpcredp)
14247c478bd9Sstevel@tonic-gate {
14257c478bd9Sstevel@tonic-gate fssadmin_t fssadmin;
14267c478bd9Sstevel@tonic-gate
14277c478bd9Sstevel@tonic-gate if (copyin(uaddr, &fssadmin, sizeof (fssadmin_t)))
14287c478bd9Sstevel@tonic-gate return (EFAULT);
14297c478bd9Sstevel@tonic-gate
14307c478bd9Sstevel@tonic-gate switch (fssadmin.fss_cmd) {
14317c478bd9Sstevel@tonic-gate case FSS_SETADMIN:
14327c478bd9Sstevel@tonic-gate if (secpolicy_dispadm(reqpcredp) != 0)
14337c478bd9Sstevel@tonic-gate return (EPERM);
14347c478bd9Sstevel@tonic-gate if (fssadmin.fss_quantum <= 0 || fssadmin.fss_quantum >= hz)
14357c478bd9Sstevel@tonic-gate return (EINVAL);
14367c478bd9Sstevel@tonic-gate fss_quantum = fssadmin.fss_quantum;
14377c478bd9Sstevel@tonic-gate break;
14387c478bd9Sstevel@tonic-gate case FSS_GETADMIN:
14397c478bd9Sstevel@tonic-gate fssadmin.fss_quantum = fss_quantum;
14407c478bd9Sstevel@tonic-gate if (copyout(&fssadmin, uaddr, sizeof (fssadmin_t)))
14417c478bd9Sstevel@tonic-gate return (EFAULT);
14427c478bd9Sstevel@tonic-gate break;
14437c478bd9Sstevel@tonic-gate default:
14447c478bd9Sstevel@tonic-gate return (EINVAL);
14457c478bd9Sstevel@tonic-gate }
14467c478bd9Sstevel@tonic-gate return (0);
14477c478bd9Sstevel@tonic-gate }
14487c478bd9Sstevel@tonic-gate
14497c478bd9Sstevel@tonic-gate static int
fss_getclinfo(void * infop)14507c478bd9Sstevel@tonic-gate fss_getclinfo(void *infop)
14517c478bd9Sstevel@tonic-gate {
14527c478bd9Sstevel@tonic-gate fssinfo_t *fssinfo = (fssinfo_t *)infop;
14537c478bd9Sstevel@tonic-gate fssinfo->fss_maxupri = fss_maxupri;
14547c478bd9Sstevel@tonic-gate return (0);
14557c478bd9Sstevel@tonic-gate }
14567c478bd9Sstevel@tonic-gate
14577c478bd9Sstevel@tonic-gate static int
fss_parmsin(void * parmsp)14587c478bd9Sstevel@tonic-gate fss_parmsin(void *parmsp)
14597c478bd9Sstevel@tonic-gate {
14607c478bd9Sstevel@tonic-gate fssparms_t *fssparmsp = (fssparms_t *)parmsp;
14617c478bd9Sstevel@tonic-gate
14627c478bd9Sstevel@tonic-gate /*
14637c478bd9Sstevel@tonic-gate * Check validity of parameters.
14647c478bd9Sstevel@tonic-gate */
14657c478bd9Sstevel@tonic-gate if ((fssparmsp->fss_uprilim > fss_maxupri ||
14667c478bd9Sstevel@tonic-gate fssparmsp->fss_uprilim < -fss_maxupri) &&
14677c478bd9Sstevel@tonic-gate fssparmsp->fss_uprilim != FSS_NOCHANGE)
14687c478bd9Sstevel@tonic-gate return (EINVAL);
14697c478bd9Sstevel@tonic-gate
14707c478bd9Sstevel@tonic-gate if ((fssparmsp->fss_upri > fss_maxupri ||
14717c478bd9Sstevel@tonic-gate fssparmsp->fss_upri < -fss_maxupri) &&
14727c478bd9Sstevel@tonic-gate fssparmsp->fss_upri != FSS_NOCHANGE)
14737c478bd9Sstevel@tonic-gate return (EINVAL);
14747c478bd9Sstevel@tonic-gate
14757c478bd9Sstevel@tonic-gate return (0);
14767c478bd9Sstevel@tonic-gate }
14777c478bd9Sstevel@tonic-gate
14787c478bd9Sstevel@tonic-gate /*ARGSUSED*/
14797c478bd9Sstevel@tonic-gate static int
fss_parmsout(void * parmsp,pc_vaparms_t * vaparmsp)14807c478bd9Sstevel@tonic-gate fss_parmsout(void *parmsp, pc_vaparms_t *vaparmsp)
14817c478bd9Sstevel@tonic-gate {
14827c478bd9Sstevel@tonic-gate return (0);
14837c478bd9Sstevel@tonic-gate }
14847c478bd9Sstevel@tonic-gate
14857c478bd9Sstevel@tonic-gate static int
fss_vaparmsin(void * parmsp,pc_vaparms_t * vaparmsp)14867c478bd9Sstevel@tonic-gate fss_vaparmsin(void *parmsp, pc_vaparms_t *vaparmsp)
14877c478bd9Sstevel@tonic-gate {
14887c478bd9Sstevel@tonic-gate fssparms_t *fssparmsp = (fssparms_t *)parmsp;
14897c478bd9Sstevel@tonic-gate int priflag = 0;
14907c478bd9Sstevel@tonic-gate int limflag = 0;
14917c478bd9Sstevel@tonic-gate uint_t cnt;
14927c478bd9Sstevel@tonic-gate pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
14937c478bd9Sstevel@tonic-gate
14947c478bd9Sstevel@tonic-gate /*
14957c478bd9Sstevel@tonic-gate * FSS_NOCHANGE (-32768) is outside of the range of values for
14967c478bd9Sstevel@tonic-gate * fss_uprilim and fss_upri. If the structure fssparms_t is changed,
14977c478bd9Sstevel@tonic-gate * FSS_NOCHANGE should be replaced by a flag word.
14987c478bd9Sstevel@tonic-gate */
14997c478bd9Sstevel@tonic-gate fssparmsp->fss_uprilim = FSS_NOCHANGE;
15007c478bd9Sstevel@tonic-gate fssparmsp->fss_upri = FSS_NOCHANGE;
15017c478bd9Sstevel@tonic-gate
15027c478bd9Sstevel@tonic-gate /*
15037c478bd9Sstevel@tonic-gate * Get the varargs parameter and check validity of parameters.
15047c478bd9Sstevel@tonic-gate */
15057c478bd9Sstevel@tonic-gate if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
15067c478bd9Sstevel@tonic-gate return (EINVAL);
15077c478bd9Sstevel@tonic-gate
15087c478bd9Sstevel@tonic-gate for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
15097c478bd9Sstevel@tonic-gate switch (vpp->pc_key) {
15107c478bd9Sstevel@tonic-gate case FSS_KY_UPRILIM:
15117c478bd9Sstevel@tonic-gate if (limflag++)
15127c478bd9Sstevel@tonic-gate return (EINVAL);
15137c478bd9Sstevel@tonic-gate fssparmsp->fss_uprilim = (pri_t)vpp->pc_parm;
15147c478bd9Sstevel@tonic-gate if (fssparmsp->fss_uprilim > fss_maxupri ||
15157c478bd9Sstevel@tonic-gate fssparmsp->fss_uprilim < -fss_maxupri)
15167c478bd9Sstevel@tonic-gate return (EINVAL);
15177c478bd9Sstevel@tonic-gate break;
15187c478bd9Sstevel@tonic-gate case FSS_KY_UPRI:
15197c478bd9Sstevel@tonic-gate if (priflag++)
15207c478bd9Sstevel@tonic-gate return (EINVAL);
15217c478bd9Sstevel@tonic-gate fssparmsp->fss_upri = (pri_t)vpp->pc_parm;
15227c478bd9Sstevel@tonic-gate if (fssparmsp->fss_upri > fss_maxupri ||
15237c478bd9Sstevel@tonic-gate fssparmsp->fss_upri < -fss_maxupri)
15247c478bd9Sstevel@tonic-gate return (EINVAL);
15257c478bd9Sstevel@tonic-gate break;
15267c478bd9Sstevel@tonic-gate default:
15277c478bd9Sstevel@tonic-gate return (EINVAL);
15287c478bd9Sstevel@tonic-gate }
15297c478bd9Sstevel@tonic-gate }
15307c478bd9Sstevel@tonic-gate
15317c478bd9Sstevel@tonic-gate if (vaparmsp->pc_vaparmscnt == 0) {
15327c478bd9Sstevel@tonic-gate /*
15337c478bd9Sstevel@tonic-gate * Use default parameters.
15347c478bd9Sstevel@tonic-gate */
15357c478bd9Sstevel@tonic-gate fssparmsp->fss_upri = fssparmsp->fss_uprilim = 0;
15367c478bd9Sstevel@tonic-gate }
15377c478bd9Sstevel@tonic-gate
15387c478bd9Sstevel@tonic-gate return (0);
15397c478bd9Sstevel@tonic-gate }
15407c478bd9Sstevel@tonic-gate
15417c478bd9Sstevel@tonic-gate /*
15427c478bd9Sstevel@tonic-gate * Copy all selected fair-sharing class parameters to the user. The parameters
15437c478bd9Sstevel@tonic-gate * are specified by a key.
15447c478bd9Sstevel@tonic-gate */
15457c478bd9Sstevel@tonic-gate static int
fss_vaparmsout(void * parmsp,pc_vaparms_t * vaparmsp)15467c478bd9Sstevel@tonic-gate fss_vaparmsout(void *parmsp, pc_vaparms_t *vaparmsp)
15477c478bd9Sstevel@tonic-gate {
15487c478bd9Sstevel@tonic-gate fssparms_t *fssparmsp = (fssparms_t *)parmsp;
15497c478bd9Sstevel@tonic-gate int priflag = 0;
15507c478bd9Sstevel@tonic-gate int limflag = 0;
15517c478bd9Sstevel@tonic-gate uint_t cnt;
15527c478bd9Sstevel@tonic-gate pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
15537c478bd9Sstevel@tonic-gate
15547c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(&curproc->p_lock));
15557c478bd9Sstevel@tonic-gate
15567c478bd9Sstevel@tonic-gate if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
15577c478bd9Sstevel@tonic-gate return (EINVAL);
15587c478bd9Sstevel@tonic-gate
15597c478bd9Sstevel@tonic-gate for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
15607c478bd9Sstevel@tonic-gate switch (vpp->pc_key) {
15617c478bd9Sstevel@tonic-gate case FSS_KY_UPRILIM:
15627c478bd9Sstevel@tonic-gate if (limflag++)
15637c478bd9Sstevel@tonic-gate return (EINVAL);
15647c478bd9Sstevel@tonic-gate if (copyout(&fssparmsp->fss_uprilim,
15657c478bd9Sstevel@tonic-gate (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
15667c478bd9Sstevel@tonic-gate return (EFAULT);
15677c478bd9Sstevel@tonic-gate break;
15687c478bd9Sstevel@tonic-gate case FSS_KY_UPRI:
15697c478bd9Sstevel@tonic-gate if (priflag++)
15707c478bd9Sstevel@tonic-gate return (EINVAL);
15717c478bd9Sstevel@tonic-gate if (copyout(&fssparmsp->fss_upri,
15727c478bd9Sstevel@tonic-gate (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
15737c478bd9Sstevel@tonic-gate return (EFAULT);
15747c478bd9Sstevel@tonic-gate break;
15757c478bd9Sstevel@tonic-gate default:
15767c478bd9Sstevel@tonic-gate return (EINVAL);
15777c478bd9Sstevel@tonic-gate }
15787c478bd9Sstevel@tonic-gate }
15797c478bd9Sstevel@tonic-gate
15807c478bd9Sstevel@tonic-gate return (0);
15817c478bd9Sstevel@tonic-gate }
15827c478bd9Sstevel@tonic-gate
1583d4204c85Sraf /*
1584d4204c85Sraf * Return the user mode scheduling priority range.
1585d4204c85Sraf */
15867c478bd9Sstevel@tonic-gate static int
fss_getclpri(pcpri_t * pcprip)15877c478bd9Sstevel@tonic-gate fss_getclpri(pcpri_t *pcprip)
15887c478bd9Sstevel@tonic-gate {
1589d4204c85Sraf pcprip->pc_clpmax = fss_maxupri;
1590d4204c85Sraf pcprip->pc_clpmin = -fss_maxupri;
15917c478bd9Sstevel@tonic-gate return (0);
15927c478bd9Sstevel@tonic-gate }
15937c478bd9Sstevel@tonic-gate
15947c478bd9Sstevel@tonic-gate static int
fss_alloc(void ** p,int flag)15957c478bd9Sstevel@tonic-gate fss_alloc(void **p, int flag)
15967c478bd9Sstevel@tonic-gate {
15977c478bd9Sstevel@tonic-gate void *bufp;
15987c478bd9Sstevel@tonic-gate
15997c478bd9Sstevel@tonic-gate if ((bufp = kmem_zalloc(sizeof (fssproc_t), flag)) == NULL) {
16007c478bd9Sstevel@tonic-gate return (ENOMEM);
16017c478bd9Sstevel@tonic-gate } else {
16027c478bd9Sstevel@tonic-gate *p = bufp;
16037c478bd9Sstevel@tonic-gate return (0);
16047c478bd9Sstevel@tonic-gate }
16057c478bd9Sstevel@tonic-gate }
16067c478bd9Sstevel@tonic-gate
16077c478bd9Sstevel@tonic-gate static void
fss_free(void * bufp)16087c478bd9Sstevel@tonic-gate fss_free(void *bufp)
16097c478bd9Sstevel@tonic-gate {
16107c478bd9Sstevel@tonic-gate if (bufp)
16117c478bd9Sstevel@tonic-gate kmem_free(bufp, sizeof (fssproc_t));
16127c478bd9Sstevel@tonic-gate }
16137c478bd9Sstevel@tonic-gate
16147c478bd9Sstevel@tonic-gate /*
16157c478bd9Sstevel@tonic-gate * Thread functions
16167c478bd9Sstevel@tonic-gate */
16177c478bd9Sstevel@tonic-gate static int
fss_enterclass(kthread_t * t,id_t cid,void * parmsp,cred_t * reqpcredp,void * bufp)16187c478bd9Sstevel@tonic-gate fss_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp,
16197c478bd9Sstevel@tonic-gate void *bufp)
16207c478bd9Sstevel@tonic-gate {
16217c478bd9Sstevel@tonic-gate fssparms_t *fssparmsp = (fssparms_t *)parmsp;
16227c478bd9Sstevel@tonic-gate fssproc_t *fssproc;
16237c478bd9Sstevel@tonic-gate pri_t reqfssuprilim;
16247c478bd9Sstevel@tonic-gate pri_t reqfssupri;
16257c478bd9Sstevel@tonic-gate static uint32_t fssexists = 0;
16267c478bd9Sstevel@tonic-gate fsspset_t *fsspset;
16277c478bd9Sstevel@tonic-gate fssproj_t *fssproj;
16287c478bd9Sstevel@tonic-gate fsszone_t *fsszone;
16297c478bd9Sstevel@tonic-gate kproject_t *kpj;
16307c478bd9Sstevel@tonic-gate zone_t *zone;
16317c478bd9Sstevel@tonic-gate int fsszone_allocated = 0;
16327c478bd9Sstevel@tonic-gate
16337c478bd9Sstevel@tonic-gate fssproc = (fssproc_t *)bufp;
16347c478bd9Sstevel@tonic-gate ASSERT(fssproc != NULL);
16357c478bd9Sstevel@tonic-gate
16367c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
16377c478bd9Sstevel@tonic-gate
16387c478bd9Sstevel@tonic-gate /*
16397c478bd9Sstevel@tonic-gate * Only root can move threads to FSS class.
16407c478bd9Sstevel@tonic-gate */
16417c478bd9Sstevel@tonic-gate if (reqpcredp != NULL && secpolicy_setpriority(reqpcredp) != 0)
16427c478bd9Sstevel@tonic-gate return (EPERM);
16437c478bd9Sstevel@tonic-gate /*
16447c478bd9Sstevel@tonic-gate * Initialize the fssproc structure.
16457c478bd9Sstevel@tonic-gate */
16467c478bd9Sstevel@tonic-gate fssproc->fss_umdpri = fss_maxumdpri / 2;
16477c478bd9Sstevel@tonic-gate
16487c478bd9Sstevel@tonic-gate if (fssparmsp == NULL) {
16497c478bd9Sstevel@tonic-gate /*
16507c478bd9Sstevel@tonic-gate * Use default values.
16517c478bd9Sstevel@tonic-gate */
16527c478bd9Sstevel@tonic-gate fssproc->fss_nice = NZERO;
16537c478bd9Sstevel@tonic-gate fssproc->fss_uprilim = fssproc->fss_upri = 0;
16547c478bd9Sstevel@tonic-gate } else {
16557c478bd9Sstevel@tonic-gate /*
16567c478bd9Sstevel@tonic-gate * Use supplied values.
16577c478bd9Sstevel@tonic-gate */
16587c478bd9Sstevel@tonic-gate if (fssparmsp->fss_uprilim == FSS_NOCHANGE) {
16597c478bd9Sstevel@tonic-gate reqfssuprilim = 0;
16607c478bd9Sstevel@tonic-gate } else {
16617c478bd9Sstevel@tonic-gate if (fssparmsp->fss_uprilim > 0 &&
16627c478bd9Sstevel@tonic-gate secpolicy_setpriority(reqpcredp) != 0)
16637c478bd9Sstevel@tonic-gate return (EPERM);
16647c478bd9Sstevel@tonic-gate reqfssuprilim = fssparmsp->fss_uprilim;
16657c478bd9Sstevel@tonic-gate }
16667c478bd9Sstevel@tonic-gate if (fssparmsp->fss_upri == FSS_NOCHANGE) {
16677c478bd9Sstevel@tonic-gate reqfssupri = reqfssuprilim;
16687c478bd9Sstevel@tonic-gate } else {
16697c478bd9Sstevel@tonic-gate if (fssparmsp->fss_upri > 0 &&
16707c478bd9Sstevel@tonic-gate secpolicy_setpriority(reqpcredp) != 0)
16717c478bd9Sstevel@tonic-gate return (EPERM);
16727c478bd9Sstevel@tonic-gate /*
16737c478bd9Sstevel@tonic-gate * Set the user priority to the requested value or
16747c478bd9Sstevel@tonic-gate * the upri limit, whichever is lower.
16757c478bd9Sstevel@tonic-gate */
16767c478bd9Sstevel@tonic-gate reqfssupri = fssparmsp->fss_upri;
16777c478bd9Sstevel@tonic-gate if (reqfssupri > reqfssuprilim)
16787c478bd9Sstevel@tonic-gate reqfssupri = reqfssuprilim;
16797c478bd9Sstevel@tonic-gate }
16807c478bd9Sstevel@tonic-gate fssproc->fss_uprilim = reqfssuprilim;
16817c478bd9Sstevel@tonic-gate fssproc->fss_upri = reqfssupri;
16827c478bd9Sstevel@tonic-gate fssproc->fss_nice = NZERO - (NZERO * reqfssupri) / fss_maxupri;
16837c478bd9Sstevel@tonic-gate if (fssproc->fss_nice > FSS_NICE_MAX)
16847c478bd9Sstevel@tonic-gate fssproc->fss_nice = FSS_NICE_MAX;
16857c478bd9Sstevel@tonic-gate }
16867c478bd9Sstevel@tonic-gate
16877c478bd9Sstevel@tonic-gate fssproc->fss_timeleft = fss_quantum;
16887c478bd9Sstevel@tonic-gate fssproc->fss_tp = t;
1689c97ad5cdSakolb cpucaps_sc_init(&fssproc->fss_caps);
16907c478bd9Sstevel@tonic-gate
16917c478bd9Sstevel@tonic-gate /*
16927c478bd9Sstevel@tonic-gate * Put a lock on our fsspset structure.
16937c478bd9Sstevel@tonic-gate */
16947c478bd9Sstevel@tonic-gate mutex_enter(&fsspsets_lock);
16957c478bd9Sstevel@tonic-gate fsspset = fss_find_fsspset(t->t_cpupart);
16967c478bd9Sstevel@tonic-gate mutex_enter(&fsspset->fssps_lock);
16977c478bd9Sstevel@tonic-gate mutex_exit(&fsspsets_lock);
16987c478bd9Sstevel@tonic-gate
16997c478bd9Sstevel@tonic-gate zone = ttoproc(t)->p_zone;
17007c478bd9Sstevel@tonic-gate if ((fsszone = fss_find_fsszone(fsspset, zone)) == NULL) {
17017c478bd9Sstevel@tonic-gate if ((fsszone = kmem_zalloc(sizeof (fsszone_t), KM_NOSLEEP))
17027c478bd9Sstevel@tonic-gate == NULL) {
17037c478bd9Sstevel@tonic-gate mutex_exit(&fsspset->fssps_lock);
17047c478bd9Sstevel@tonic-gate return (ENOMEM);
17057c478bd9Sstevel@tonic-gate } else {
17067c478bd9Sstevel@tonic-gate fsszone_allocated = 1;
17077c478bd9Sstevel@tonic-gate fss_insert_fsszone(fsspset, zone, fsszone);
17087c478bd9Sstevel@tonic-gate }
17097c478bd9Sstevel@tonic-gate }
17107c478bd9Sstevel@tonic-gate kpj = ttoproj(t);
17117c478bd9Sstevel@tonic-gate if ((fssproj = fss_find_fssproj(fsspset, kpj)) == NULL) {
17127c478bd9Sstevel@tonic-gate if ((fssproj = kmem_zalloc(sizeof (fssproj_t), KM_NOSLEEP))
17137c478bd9Sstevel@tonic-gate == NULL) {
17147c478bd9Sstevel@tonic-gate if (fsszone_allocated) {
17157c478bd9Sstevel@tonic-gate fss_remove_fsszone(fsspset, fsszone);
17167c478bd9Sstevel@tonic-gate kmem_free(fsszone, sizeof (fsszone_t));
17177c478bd9Sstevel@tonic-gate }
17187c478bd9Sstevel@tonic-gate mutex_exit(&fsspset->fssps_lock);
17197c478bd9Sstevel@tonic-gate return (ENOMEM);
17207c478bd9Sstevel@tonic-gate } else {
17217c478bd9Sstevel@tonic-gate fss_insert_fssproj(fsspset, kpj, fsszone, fssproj);
17227c478bd9Sstevel@tonic-gate }
17237c478bd9Sstevel@tonic-gate }
17247c478bd9Sstevel@tonic-gate fssproj->fssp_threads++;
17257c478bd9Sstevel@tonic-gate fssproc->fss_proj = fssproj;
17267c478bd9Sstevel@tonic-gate
17277c478bd9Sstevel@tonic-gate /*
17287c478bd9Sstevel@tonic-gate * Reset priority. Process goes to a "user mode" priority here
17297c478bd9Sstevel@tonic-gate * regardless of whether or not it has slept since entering the kernel.
17307c478bd9Sstevel@tonic-gate */
17317c478bd9Sstevel@tonic-gate thread_lock(t);
17327c478bd9Sstevel@tonic-gate t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
17337c478bd9Sstevel@tonic-gate t->t_cid = cid;
17347c478bd9Sstevel@tonic-gate t->t_cldata = (void *)fssproc;
17357c478bd9Sstevel@tonic-gate t->t_schedflag |= TS_RUNQMATCH;
17367c478bd9Sstevel@tonic-gate fss_change_priority(t, fssproc);
1737c97ad5cdSakolb if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
1738c97ad5cdSakolb t->t_state == TS_WAIT)
17397c478bd9Sstevel@tonic-gate fss_active(t);
17407c478bd9Sstevel@tonic-gate thread_unlock(t);
17417c478bd9Sstevel@tonic-gate
17427c478bd9Sstevel@tonic-gate mutex_exit(&fsspset->fssps_lock);
17437c478bd9Sstevel@tonic-gate
17447c478bd9Sstevel@tonic-gate /*
17457c478bd9Sstevel@tonic-gate * Link new structure into fssproc list.
17467c478bd9Sstevel@tonic-gate */
17477c478bd9Sstevel@tonic-gate FSS_LIST_INSERT(fssproc);
17487c478bd9Sstevel@tonic-gate
17497c478bd9Sstevel@tonic-gate /*
17507c478bd9Sstevel@tonic-gate * If this is the first fair-sharing thread to occur since boot,
17517c478bd9Sstevel@tonic-gate * we set up the initial call to fss_update() here. Use an atomic
17527c478bd9Sstevel@tonic-gate * compare-and-swap since that's easier and faster than a mutex
17537c478bd9Sstevel@tonic-gate * (but check with an ordinary load first since most of the time
17547c478bd9Sstevel@tonic-gate * this will already be done).
17557c478bd9Sstevel@tonic-gate */
175675d94465SJosef 'Jeff' Sipek if (fssexists == 0 && atomic_cas_32(&fssexists, 0, 1) == 0)
17577c478bd9Sstevel@tonic-gate (void) timeout(fss_update, NULL, hz);
17587c478bd9Sstevel@tonic-gate
17597c478bd9Sstevel@tonic-gate return (0);
17607c478bd9Sstevel@tonic-gate }
17617c478bd9Sstevel@tonic-gate
17627c478bd9Sstevel@tonic-gate /*
17637c478bd9Sstevel@tonic-gate * Remove fssproc_t from the list.
17647c478bd9Sstevel@tonic-gate */
17657c478bd9Sstevel@tonic-gate static void
fss_exitclass(void * procp)17667c478bd9Sstevel@tonic-gate fss_exitclass(void *procp)
17677c478bd9Sstevel@tonic-gate {
17687c478bd9Sstevel@tonic-gate fssproc_t *fssproc = (fssproc_t *)procp;
17697c478bd9Sstevel@tonic-gate fssproj_t *fssproj;
17707c478bd9Sstevel@tonic-gate fsspset_t *fsspset;
17717c478bd9Sstevel@tonic-gate fsszone_t *fsszone;
17727c478bd9Sstevel@tonic-gate kthread_t *t = fssproc->fss_tp;
17737c478bd9Sstevel@tonic-gate
17747c478bd9Sstevel@tonic-gate /*
17757c478bd9Sstevel@tonic-gate * We should be either getting this thread off the deathrow or
17767c478bd9Sstevel@tonic-gate * this thread has already moved to another scheduling class and
17777c478bd9Sstevel@tonic-gate * we're being called with its old cldata buffer pointer. In both
17787c478bd9Sstevel@tonic-gate * cases, the content of this buffer can not be changed while we're
17797c478bd9Sstevel@tonic-gate * here.
17807c478bd9Sstevel@tonic-gate */
17817c478bd9Sstevel@tonic-gate mutex_enter(&fsspsets_lock);
17827c478bd9Sstevel@tonic-gate thread_lock(t);
17837c478bd9Sstevel@tonic-gate if (t->t_cid != fss_cid) {
17847c478bd9Sstevel@tonic-gate /*
17857c478bd9Sstevel@tonic-gate * We're being called as a result of the priocntl() system
17867c478bd9Sstevel@tonic-gate * call -- someone is trying to move our thread to another
17877c478bd9Sstevel@tonic-gate * scheduling class. We can't call fss_inactive() here
17887c478bd9Sstevel@tonic-gate * because our thread's t_cldata pointer already points
17897c478bd9Sstevel@tonic-gate * to another scheduling class specific data.
17907c478bd9Sstevel@tonic-gate */
17917c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
17927c478bd9Sstevel@tonic-gate
17937c478bd9Sstevel@tonic-gate fssproj = FSSPROC2FSSPROJ(fssproc);
17947c478bd9Sstevel@tonic-gate fsspset = FSSPROJ2FSSPSET(fssproj);
17957c478bd9Sstevel@tonic-gate fsszone = fssproj->fssp_fsszone;
17967c478bd9Sstevel@tonic-gate
17977c478bd9Sstevel@tonic-gate if (fssproc->fss_runnable) {
17987c478bd9Sstevel@tonic-gate disp_lock_enter_high(&fsspset->fssps_displock);
17997c478bd9Sstevel@tonic-gate if (--fssproj->fssp_runnable == 0) {
18007c478bd9Sstevel@tonic-gate fsszone->fssz_shares -= fssproj->fssp_shares;
18017c478bd9Sstevel@tonic-gate if (--fsszone->fssz_runnable == 0)
18027c478bd9Sstevel@tonic-gate fsspset->fssps_shares -=
18037c478bd9Sstevel@tonic-gate fsszone->fssz_rshares;
18047c478bd9Sstevel@tonic-gate }
18057c478bd9Sstevel@tonic-gate disp_lock_exit_high(&fsspset->fssps_displock);
18067c478bd9Sstevel@tonic-gate }
18077c478bd9Sstevel@tonic-gate thread_unlock(t);
18087c478bd9Sstevel@tonic-gate
18097c478bd9Sstevel@tonic-gate mutex_enter(&fsspset->fssps_lock);
18107c478bd9Sstevel@tonic-gate if (--fssproj->fssp_threads == 0) {
18117c478bd9Sstevel@tonic-gate fss_remove_fssproj(fsspset, fssproj);
18127c478bd9Sstevel@tonic-gate if (fsszone->fssz_nproj == 0)
18137c478bd9Sstevel@tonic-gate kmem_free(fsszone, sizeof (fsszone_t));
18147c478bd9Sstevel@tonic-gate kmem_free(fssproj, sizeof (fssproj_t));
18157c478bd9Sstevel@tonic-gate }
18167c478bd9Sstevel@tonic-gate mutex_exit(&fsspset->fssps_lock);
18177c478bd9Sstevel@tonic-gate
18187c478bd9Sstevel@tonic-gate } else {
18197c478bd9Sstevel@tonic-gate ASSERT(t->t_state == TS_FREE);
18207c478bd9Sstevel@tonic-gate /*
18217c478bd9Sstevel@tonic-gate * We're being called from thread_free() when our thread
18227c478bd9Sstevel@tonic-gate * is removed from the deathrow. There is nothing we need
18237c478bd9Sstevel@tonic-gate * do here since everything should've been done earlier
18247c478bd9Sstevel@tonic-gate * in fss_exit().
18257c478bd9Sstevel@tonic-gate */
18267c478bd9Sstevel@tonic-gate thread_unlock(t);
18277c478bd9Sstevel@tonic-gate }
18287c478bd9Sstevel@tonic-gate mutex_exit(&fsspsets_lock);
18297c478bd9Sstevel@tonic-gate
18307c478bd9Sstevel@tonic-gate FSS_LIST_DELETE(fssproc);
18317c478bd9Sstevel@tonic-gate fss_free(fssproc);
18327c478bd9Sstevel@tonic-gate }
18337c478bd9Sstevel@tonic-gate
18347c478bd9Sstevel@tonic-gate /*ARGSUSED*/
18357c478bd9Sstevel@tonic-gate static int
fss_canexit(kthread_t * t,cred_t * credp)18367c478bd9Sstevel@tonic-gate fss_canexit(kthread_t *t, cred_t *credp)
18377c478bd9Sstevel@tonic-gate {
18387c478bd9Sstevel@tonic-gate /*
18397c478bd9Sstevel@tonic-gate * A thread is allowed to exit FSS only if we have sufficient
18407c478bd9Sstevel@tonic-gate * privileges.
18417c478bd9Sstevel@tonic-gate */
18427c478bd9Sstevel@tonic-gate if (credp != NULL && secpolicy_setpriority(credp) != 0)
18437c478bd9Sstevel@tonic-gate return (EPERM);
18447c478bd9Sstevel@tonic-gate else
18457c478bd9Sstevel@tonic-gate return (0);
18467c478bd9Sstevel@tonic-gate }
18477c478bd9Sstevel@tonic-gate
18487c478bd9Sstevel@tonic-gate /*
18497c478bd9Sstevel@tonic-gate * Initialize fair-share class specific proc structure for a child.
18507c478bd9Sstevel@tonic-gate */
18517c478bd9Sstevel@tonic-gate static int
fss_fork(kthread_t * pt,kthread_t * ct,void * bufp)18527c478bd9Sstevel@tonic-gate fss_fork(kthread_t *pt, kthread_t *ct, void *bufp)
18537c478bd9Sstevel@tonic-gate {
18547c478bd9Sstevel@tonic-gate fssproc_t *pfssproc; /* ptr to parent's fssproc structure */
18557c478bd9Sstevel@tonic-gate fssproc_t *cfssproc; /* ptr to child's fssproc structure */
18567c478bd9Sstevel@tonic-gate fssproj_t *fssproj;
18577c478bd9Sstevel@tonic-gate fsspset_t *fsspset;
18587c478bd9Sstevel@tonic-gate
18597c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ttoproc(pt)->p_lock));
18607c478bd9Sstevel@tonic-gate ASSERT(ct->t_state == TS_STOPPED);
18617c478bd9Sstevel@tonic-gate
18627c478bd9Sstevel@tonic-gate cfssproc = (fssproc_t *)bufp;
18637c478bd9Sstevel@tonic-gate ASSERT(cfssproc != NULL);
18647c478bd9Sstevel@tonic-gate bzero(cfssproc, sizeof (fssproc_t));
18657c478bd9Sstevel@tonic-gate
18667c478bd9Sstevel@tonic-gate thread_lock(pt);
18677c478bd9Sstevel@tonic-gate pfssproc = FSSPROC(pt);
18687c478bd9Sstevel@tonic-gate fssproj = FSSPROC2FSSPROJ(pfssproc);
18697c478bd9Sstevel@tonic-gate fsspset = FSSPROJ2FSSPSET(fssproj);
18707c478bd9Sstevel@tonic-gate thread_unlock(pt);
18717c478bd9Sstevel@tonic-gate
18727c478bd9Sstevel@tonic-gate mutex_enter(&fsspset->fssps_lock);
18737c478bd9Sstevel@tonic-gate /*
18747c478bd9Sstevel@tonic-gate * Initialize child's fssproc structure.
18757c478bd9Sstevel@tonic-gate */
18767c478bd9Sstevel@tonic-gate thread_lock(pt);
18777c478bd9Sstevel@tonic-gate ASSERT(FSSPROJ(pt) == fssproj);
18787c478bd9Sstevel@tonic-gate cfssproc->fss_proj = fssproj;
18797c478bd9Sstevel@tonic-gate cfssproc->fss_timeleft = fss_quantum;
18807c478bd9Sstevel@tonic-gate cfssproc->fss_umdpri = pfssproc->fss_umdpri;
18817c478bd9Sstevel@tonic-gate cfssproc->fss_fsspri = 0;
18827c478bd9Sstevel@tonic-gate cfssproc->fss_uprilim = pfssproc->fss_uprilim;
18837c478bd9Sstevel@tonic-gate cfssproc->fss_upri = pfssproc->fss_upri;
18847c478bd9Sstevel@tonic-gate cfssproc->fss_tp = ct;
18857c478bd9Sstevel@tonic-gate cfssproc->fss_nice = pfssproc->fss_nice;
1886c97ad5cdSakolb cpucaps_sc_init(&cfssproc->fss_caps);
1887c97ad5cdSakolb
18889ac8606fSraf cfssproc->fss_flags =
1889*6a0b1217SPatrick Mooney pfssproc->fss_flags & ~(FSSBACKQ | FSSRESTORE);
18907c478bd9Sstevel@tonic-gate ct->t_cldata = (void *)cfssproc;
18917c478bd9Sstevel@tonic-gate ct->t_schedflag |= TS_RUNQMATCH;
18927c478bd9Sstevel@tonic-gate thread_unlock(pt);
18937c478bd9Sstevel@tonic-gate
18947c478bd9Sstevel@tonic-gate fssproj->fssp_threads++;
18957c478bd9Sstevel@tonic-gate mutex_exit(&fsspset->fssps_lock);
18967c478bd9Sstevel@tonic-gate
18977c478bd9Sstevel@tonic-gate /*
18987c478bd9Sstevel@tonic-gate * Link new structure into fssproc hash table.
18997c478bd9Sstevel@tonic-gate */
19007c478bd9Sstevel@tonic-gate FSS_LIST_INSERT(cfssproc);
19017c478bd9Sstevel@tonic-gate return (0);
19027c478bd9Sstevel@tonic-gate }
19037c478bd9Sstevel@tonic-gate
19047c478bd9Sstevel@tonic-gate /*
19057c478bd9Sstevel@tonic-gate * Child is placed at back of dispatcher queue and parent gives up processor
19067c478bd9Sstevel@tonic-gate * so that the child runs first after the fork. This allows the child
19077c478bd9Sstevel@tonic-gate * immediately execing to break the multiple use of copy on write pages with no
19087c478bd9Sstevel@tonic-gate * disk home. The parent will get to steal them back rather than uselessly
19097c478bd9Sstevel@tonic-gate * copying them.
19107c478bd9Sstevel@tonic-gate */
19117c478bd9Sstevel@tonic-gate static void
fss_forkret(kthread_t * t,kthread_t * ct)19127c478bd9Sstevel@tonic-gate fss_forkret(kthread_t *t, kthread_t *ct)
19137c478bd9Sstevel@tonic-gate {
19147c478bd9Sstevel@tonic-gate proc_t *pp = ttoproc(t);
19157c478bd9Sstevel@tonic-gate proc_t *cp = ttoproc(ct);
19167c478bd9Sstevel@tonic-gate fssproc_t *fssproc;
19177c478bd9Sstevel@tonic-gate
19187c478bd9Sstevel@tonic-gate ASSERT(t == curthread);
19197c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&pidlock));
19207c478bd9Sstevel@tonic-gate
19217c478bd9Sstevel@tonic-gate /*
19227c478bd9Sstevel@tonic-gate * Grab the child's p_lock before dropping pidlock to ensure the
19237c478bd9Sstevel@tonic-gate * process does not disappear before we set it running.
19247c478bd9Sstevel@tonic-gate */
19257c478bd9Sstevel@tonic-gate mutex_enter(&cp->p_lock);
19267c478bd9Sstevel@tonic-gate continuelwps(cp);
19277c478bd9Sstevel@tonic-gate mutex_exit(&cp->p_lock);
19287c478bd9Sstevel@tonic-gate
19297c478bd9Sstevel@tonic-gate mutex_enter(&pp->p_lock);
1930d5493db7SPramod Batni mutex_exit(&pidlock);
19317c478bd9Sstevel@tonic-gate continuelwps(pp);
19327c478bd9Sstevel@tonic-gate
19337c478bd9Sstevel@tonic-gate thread_lock(t);
19347c478bd9Sstevel@tonic-gate
19357c478bd9Sstevel@tonic-gate fssproc = FSSPROC(t);
1936482a7749SJerry Jelinek fss_newpri(fssproc, B_FALSE);
19377c478bd9Sstevel@tonic-gate fssproc->fss_timeleft = fss_quantum;
19387c478bd9Sstevel@tonic-gate t->t_pri = fssproc->fss_umdpri;
19397c478bd9Sstevel@tonic-gate ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
19407c478bd9Sstevel@tonic-gate THREAD_TRANSITION(t);
19417c478bd9Sstevel@tonic-gate
19427c478bd9Sstevel@tonic-gate /*
19437c478bd9Sstevel@tonic-gate * We don't want to call fss_setrun(t) here because it may call
19447c478bd9Sstevel@tonic-gate * fss_active, which we don't need.
19457c478bd9Sstevel@tonic-gate */
19467c478bd9Sstevel@tonic-gate fssproc->fss_flags &= ~FSSBACKQ;
19477c478bd9Sstevel@tonic-gate
1948d3d50737SRafael Vanoni if (t->t_disp_time != ddi_get_lbolt())
19497c478bd9Sstevel@tonic-gate setbackdq(t);
19507c478bd9Sstevel@tonic-gate else
19517c478bd9Sstevel@tonic-gate setfrontdq(t);
19527c478bd9Sstevel@tonic-gate
19537c478bd9Sstevel@tonic-gate thread_unlock(t);
1954d5493db7SPramod Batni /*
1955d5493db7SPramod Batni * Safe to drop p_lock now since it is safe to change
1956d5493db7SPramod Batni * the scheduling class after this point.
1957d5493db7SPramod Batni */
1958d5493db7SPramod Batni mutex_exit(&pp->p_lock);
19597c478bd9Sstevel@tonic-gate
19607c478bd9Sstevel@tonic-gate swtch();
19617c478bd9Sstevel@tonic-gate }
19627c478bd9Sstevel@tonic-gate
19637c478bd9Sstevel@tonic-gate /*
19647c478bd9Sstevel@tonic-gate * Get the fair-sharing parameters of the thread pointed to by fssprocp into
19657c478bd9Sstevel@tonic-gate * the buffer pointed by fssparmsp.
19667c478bd9Sstevel@tonic-gate */
19677c478bd9Sstevel@tonic-gate static void
fss_parmsget(kthread_t * t,void * parmsp)19687c478bd9Sstevel@tonic-gate fss_parmsget(kthread_t *t, void *parmsp)
19697c478bd9Sstevel@tonic-gate {
19707c478bd9Sstevel@tonic-gate fssproc_t *fssproc = FSSPROC(t);
19717c478bd9Sstevel@tonic-gate fssparms_t *fssparmsp = (fssparms_t *)parmsp;
19727c478bd9Sstevel@tonic-gate
19737c478bd9Sstevel@tonic-gate fssparmsp->fss_uprilim = fssproc->fss_uprilim;
19747c478bd9Sstevel@tonic-gate fssparmsp->fss_upri = fssproc->fss_upri;
19757c478bd9Sstevel@tonic-gate }
19767c478bd9Sstevel@tonic-gate
19777c478bd9Sstevel@tonic-gate /*ARGSUSED*/
19787c478bd9Sstevel@tonic-gate static int
fss_parmsset(kthread_t * t,void * parmsp,id_t reqpcid,cred_t * reqpcredp)19797c478bd9Sstevel@tonic-gate fss_parmsset(kthread_t *t, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
19807c478bd9Sstevel@tonic-gate {
19817c478bd9Sstevel@tonic-gate char nice;
19827c478bd9Sstevel@tonic-gate pri_t reqfssuprilim;
19837c478bd9Sstevel@tonic-gate pri_t reqfssupri;
19847c478bd9Sstevel@tonic-gate fssproc_t *fssproc = FSSPROC(t);
19857c478bd9Sstevel@tonic-gate fssparms_t *fssparmsp = (fssparms_t *)parmsp;
19867c478bd9Sstevel@tonic-gate
19877c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
19887c478bd9Sstevel@tonic-gate
19897c478bd9Sstevel@tonic-gate if (fssparmsp->fss_uprilim == FSS_NOCHANGE)
19907c478bd9Sstevel@tonic-gate reqfssuprilim = fssproc->fss_uprilim;
19917c478bd9Sstevel@tonic-gate else
19927c478bd9Sstevel@tonic-gate reqfssuprilim = fssparmsp->fss_uprilim;
19937c478bd9Sstevel@tonic-gate
19947c478bd9Sstevel@tonic-gate if (fssparmsp->fss_upri == FSS_NOCHANGE)
19957c478bd9Sstevel@tonic-gate reqfssupri = fssproc->fss_upri;
19967c478bd9Sstevel@tonic-gate else
19977c478bd9Sstevel@tonic-gate reqfssupri = fssparmsp->fss_upri;
19987c478bd9Sstevel@tonic-gate
19997c478bd9Sstevel@tonic-gate /*
20007c478bd9Sstevel@tonic-gate * Make sure the user priority doesn't exceed the upri limit.
20017c478bd9Sstevel@tonic-gate */
20027c478bd9Sstevel@tonic-gate if (reqfssupri > reqfssuprilim)
20037c478bd9Sstevel@tonic-gate reqfssupri = reqfssuprilim;
20047c478bd9Sstevel@tonic-gate
20057c478bd9Sstevel@tonic-gate /*
20067c478bd9Sstevel@tonic-gate * Basic permissions enforced by generic kernel code for all classes
20077c478bd9Sstevel@tonic-gate * require that a thread attempting to change the scheduling parameters
20087c478bd9Sstevel@tonic-gate * of a target thread be privileged or have a real or effective UID
20097c478bd9Sstevel@tonic-gate * matching that of the target thread. We are not called unless these
20107c478bd9Sstevel@tonic-gate * basic permission checks have already passed. The fair-sharing class
20117c478bd9Sstevel@tonic-gate * requires in addition that the calling thread be privileged if it
20127c478bd9Sstevel@tonic-gate * is attempting to raise the upri limit above its current value.
20137c478bd9Sstevel@tonic-gate * This may have been checked previously but if our caller passed us
20147c478bd9Sstevel@tonic-gate * a non-NULL credential pointer we assume it hasn't and we check it
20157c478bd9Sstevel@tonic-gate * here.
20167c478bd9Sstevel@tonic-gate */
20177c478bd9Sstevel@tonic-gate if ((reqpcredp != NULL) &&
20187c478bd9Sstevel@tonic-gate (reqfssuprilim > fssproc->fss_uprilim) &&
201924d819e6SJerry Jelinek secpolicy_raisepriority(reqpcredp) != 0)
20207c478bd9Sstevel@tonic-gate return (EPERM);
20217c478bd9Sstevel@tonic-gate
20227c478bd9Sstevel@tonic-gate /*
20237c478bd9Sstevel@tonic-gate * Set fss_nice to the nice value corresponding to the user priority we
20247c478bd9Sstevel@tonic-gate * are setting. Note that setting the nice field of the parameter
20257c478bd9Sstevel@tonic-gate * struct won't affect upri or nice.
20267c478bd9Sstevel@tonic-gate */
20277c478bd9Sstevel@tonic-gate nice = NZERO - (reqfssupri * NZERO) / fss_maxupri;
20287c478bd9Sstevel@tonic-gate if (nice > FSS_NICE_MAX)
20297c478bd9Sstevel@tonic-gate nice = FSS_NICE_MAX;
20307c478bd9Sstevel@tonic-gate
20317c478bd9Sstevel@tonic-gate thread_lock(t);
20327c478bd9Sstevel@tonic-gate
20337c478bd9Sstevel@tonic-gate fssproc->fss_uprilim = reqfssuprilim;
20347c478bd9Sstevel@tonic-gate fssproc->fss_upri = reqfssupri;
20357c478bd9Sstevel@tonic-gate fssproc->fss_nice = nice;
2036482a7749SJerry Jelinek fss_newpri(fssproc, B_FALSE);
20377c478bd9Sstevel@tonic-gate
20387c478bd9Sstevel@tonic-gate fss_change_priority(t, fssproc);
20397c478bd9Sstevel@tonic-gate thread_unlock(t);
20407c478bd9Sstevel@tonic-gate return (0);
20417c478bd9Sstevel@tonic-gate
20427c478bd9Sstevel@tonic-gate }
20437c478bd9Sstevel@tonic-gate
20447c478bd9Sstevel@tonic-gate /*
20457c478bd9Sstevel@tonic-gate * The thread is being stopped.
20467c478bd9Sstevel@tonic-gate */
20477c478bd9Sstevel@tonic-gate /*ARGSUSED*/
20487c478bd9Sstevel@tonic-gate static void
fss_stop(kthread_t * t,int why,int what)20497c478bd9Sstevel@tonic-gate fss_stop(kthread_t *t, int why, int what)
20507c478bd9Sstevel@tonic-gate {
20517c478bd9Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(t));
20527c478bd9Sstevel@tonic-gate ASSERT(t == curthread);
20537c478bd9Sstevel@tonic-gate
20547c478bd9Sstevel@tonic-gate fss_inactive(t);
20557c478bd9Sstevel@tonic-gate }
20567c478bd9Sstevel@tonic-gate
20577c478bd9Sstevel@tonic-gate /*
20587c478bd9Sstevel@tonic-gate * The current thread is exiting, do necessary adjustments to its project
20597c478bd9Sstevel@tonic-gate */
20607c478bd9Sstevel@tonic-gate static void
fss_exit(kthread_t * t)20617c478bd9Sstevel@tonic-gate fss_exit(kthread_t *t)
20627c478bd9Sstevel@tonic-gate {
20637c478bd9Sstevel@tonic-gate fsspset_t *fsspset;
20647c478bd9Sstevel@tonic-gate fssproj_t *fssproj;
20657c478bd9Sstevel@tonic-gate fssproc_t *fssproc;
20667c478bd9Sstevel@tonic-gate fsszone_t *fsszone;
20677c478bd9Sstevel@tonic-gate int free = 0;
20687c478bd9Sstevel@tonic-gate
20697c478bd9Sstevel@tonic-gate /*
20707c478bd9Sstevel@tonic-gate * Thread t here is either a current thread (in which case we hold
20717c478bd9Sstevel@tonic-gate * its process' p_lock), or a thread being destroyed by forklwp_fail(),
20727c478bd9Sstevel@tonic-gate * in which case we hold pidlock and thread is no longer on the
20737c478bd9Sstevel@tonic-gate * thread list.
20747c478bd9Sstevel@tonic-gate */
20757c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock) || MUTEX_HELD(&pidlock));
20767c478bd9Sstevel@tonic-gate
20777c478bd9Sstevel@tonic-gate fssproc = FSSPROC(t);
20787c478bd9Sstevel@tonic-gate fssproj = FSSPROC2FSSPROJ(fssproc);
20797c478bd9Sstevel@tonic-gate fsspset = FSSPROJ2FSSPSET(fssproj);
20807c478bd9Sstevel@tonic-gate fsszone = fssproj->fssp_fsszone;
20817c478bd9Sstevel@tonic-gate
20827c478bd9Sstevel@tonic-gate mutex_enter(&fsspsets_lock);
20837c478bd9Sstevel@tonic-gate mutex_enter(&fsspset->fssps_lock);
20847c478bd9Sstevel@tonic-gate
20857c478bd9Sstevel@tonic-gate thread_lock(t);
20867c478bd9Sstevel@tonic-gate disp_lock_enter_high(&fsspset->fssps_displock);
20877c478bd9Sstevel@tonic-gate if (t->t_state == TS_ONPROC || t->t_state == TS_RUN) {
20887c478bd9Sstevel@tonic-gate if (--fssproj->fssp_runnable == 0) {
20897c478bd9Sstevel@tonic-gate fsszone->fssz_shares -= fssproj->fssp_shares;
20907c478bd9Sstevel@tonic-gate if (--fsszone->fssz_runnable == 0)
20917c478bd9Sstevel@tonic-gate fsspset->fssps_shares -= fsszone->fssz_rshares;
20927c478bd9Sstevel@tonic-gate }
20937c478bd9Sstevel@tonic-gate ASSERT(fssproc->fss_runnable == 1);
20947c478bd9Sstevel@tonic-gate fssproc->fss_runnable = 0;
20957c478bd9Sstevel@tonic-gate }
20967c478bd9Sstevel@tonic-gate if (--fssproj->fssp_threads == 0) {
20977c478bd9Sstevel@tonic-gate fss_remove_fssproj(fsspset, fssproj);
20987c478bd9Sstevel@tonic-gate free = 1;
20997c478bd9Sstevel@tonic-gate }
21007c478bd9Sstevel@tonic-gate disp_lock_exit_high(&fsspset->fssps_displock);
21017c478bd9Sstevel@tonic-gate fssproc->fss_proj = NULL; /* mark this thread as already exited */
21027c478bd9Sstevel@tonic-gate thread_unlock(t);
21037c478bd9Sstevel@tonic-gate
21047c478bd9Sstevel@tonic-gate if (free) {
21057c478bd9Sstevel@tonic-gate if (fsszone->fssz_nproj == 0)
21067c478bd9Sstevel@tonic-gate kmem_free(fsszone, sizeof (fsszone_t));
21077c478bd9Sstevel@tonic-gate kmem_free(fssproj, sizeof (fssproj_t));
21087c478bd9Sstevel@tonic-gate }
21097c478bd9Sstevel@tonic-gate mutex_exit(&fsspset->fssps_lock);
21107c478bd9Sstevel@tonic-gate mutex_exit(&fsspsets_lock);
2111c97ad5cdSakolb
21128c34bbb7Sakolb /*
21138c34bbb7Sakolb * A thread could be exiting in between clock ticks, so we need to
21148c34bbb7Sakolb * calculate how much CPU time it used since it was charged last time.
21158c34bbb7Sakolb *
21168c34bbb7Sakolb * CPU caps are not enforced on exiting processes - it is usually
21178c34bbb7Sakolb * desirable to exit as soon as possible to free resources.
21188c34bbb7Sakolb */
2119c97ad5cdSakolb if (CPUCAPS_ON()) {
2120c97ad5cdSakolb thread_lock(t);
2121c97ad5cdSakolb fssproc = FSSPROC(t);
2122c97ad5cdSakolb (void) cpucaps_charge(t, &fssproc->fss_caps,
2123c97ad5cdSakolb CPUCAPS_CHARGE_ONLY);
2124c97ad5cdSakolb thread_unlock(t);
2125c97ad5cdSakolb }
21267c478bd9Sstevel@tonic-gate }
21277c478bd9Sstevel@tonic-gate
21287c478bd9Sstevel@tonic-gate static void
fss_nullsys()21297c478bd9Sstevel@tonic-gate fss_nullsys()
21307c478bd9Sstevel@tonic-gate {
21317c478bd9Sstevel@tonic-gate }
21327c478bd9Sstevel@tonic-gate
21337c478bd9Sstevel@tonic-gate /*
21347c478bd9Sstevel@tonic-gate * fss_swapin() returns -1 if the thread is loaded or is not eligible to be
21357c478bd9Sstevel@tonic-gate * swapped in. Otherwise, it returns the thread's effective priority based
21367c478bd9Sstevel@tonic-gate * on swapout time and size of process (0 <= epri <= 0 SHRT_MAX).
21377c478bd9Sstevel@tonic-gate */
21387c478bd9Sstevel@tonic-gate /*ARGSUSED*/
21397c478bd9Sstevel@tonic-gate static pri_t
fss_swapin(kthread_t * t,int flags)21407c478bd9Sstevel@tonic-gate fss_swapin(kthread_t *t, int flags)
21417c478bd9Sstevel@tonic-gate {
21427c478bd9Sstevel@tonic-gate fssproc_t *fssproc = FSSPROC(t);
21437c478bd9Sstevel@tonic-gate long epri = -1;
21447c478bd9Sstevel@tonic-gate proc_t *pp = ttoproc(t);
21457c478bd9Sstevel@tonic-gate
21467c478bd9Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(t));
21477c478bd9Sstevel@tonic-gate
21487c478bd9Sstevel@tonic-gate if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) {
21497c478bd9Sstevel@tonic-gate time_t swapout_time;
21507c478bd9Sstevel@tonic-gate
2151d3d50737SRafael Vanoni swapout_time = (ddi_get_lbolt() - t->t_stime) / hz;
2152*6a0b1217SPatrick Mooney if (INHERITED(t)) {
21537c478bd9Sstevel@tonic-gate epri = (long)DISP_PRIO(t) + swapout_time;
21547c478bd9Sstevel@tonic-gate } else {
21557c478bd9Sstevel@tonic-gate /*
21567c478bd9Sstevel@tonic-gate * Threads which have been out for a long time,
21577c478bd9Sstevel@tonic-gate * have high user mode priority and are associated
21587c478bd9Sstevel@tonic-gate * with a small address space are more deserving.
21597c478bd9Sstevel@tonic-gate */
21607c478bd9Sstevel@tonic-gate epri = fssproc->fss_umdpri;
21617c478bd9Sstevel@tonic-gate ASSERT(epri >= 0 && epri <= fss_maxumdpri);
21627c478bd9Sstevel@tonic-gate epri += swapout_time - pp->p_swrss / nz(maxpgio)/2;
21637c478bd9Sstevel@tonic-gate }
21647c478bd9Sstevel@tonic-gate /*
21657c478bd9Sstevel@tonic-gate * Scale epri so that SHRT_MAX / 2 represents zero priority.
21667c478bd9Sstevel@tonic-gate */
21677c478bd9Sstevel@tonic-gate epri += SHRT_MAX / 2;
21687c478bd9Sstevel@tonic-gate if (epri < 0)
21697c478bd9Sstevel@tonic-gate epri = 0;
21707c478bd9Sstevel@tonic-gate else if (epri > SHRT_MAX)
21717c478bd9Sstevel@tonic-gate epri = SHRT_MAX;
21727c478bd9Sstevel@tonic-gate }
21737c478bd9Sstevel@tonic-gate return ((pri_t)epri);
21747c478bd9Sstevel@tonic-gate }
21757c478bd9Sstevel@tonic-gate
21767c478bd9Sstevel@tonic-gate /*
21777c478bd9Sstevel@tonic-gate * fss_swapout() returns -1 if the thread isn't loaded or is not eligible to
21787c478bd9Sstevel@tonic-gate * be swapped out. Otherwise, it returns the thread's effective priority
21797c478bd9Sstevel@tonic-gate * based on if the swapper is in softswap or hardswap mode.
21807c478bd9Sstevel@tonic-gate */
21817c478bd9Sstevel@tonic-gate static pri_t
fss_swapout(kthread_t * t,int flags)21827c478bd9Sstevel@tonic-gate fss_swapout(kthread_t *t, int flags)
21837c478bd9Sstevel@tonic-gate {
21847c478bd9Sstevel@tonic-gate long epri = -1;
21857c478bd9Sstevel@tonic-gate proc_t *pp = ttoproc(t);
21867c478bd9Sstevel@tonic-gate time_t swapin_time;
21877c478bd9Sstevel@tonic-gate
21887c478bd9Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(t));
21897c478bd9Sstevel@tonic-gate
21907c478bd9Sstevel@tonic-gate if (INHERITED(t) ||
21917c478bd9Sstevel@tonic-gate (t->t_proc_flag & TP_LWPEXIT) ||
2192d4204c85Sraf (t->t_state & (TS_ZOMB|TS_FREE|TS_STOPPED|TS_ONPROC|TS_WAIT)) ||
21937c478bd9Sstevel@tonic-gate !(t->t_schedflag & TS_LOAD) ||
21947c478bd9Sstevel@tonic-gate !(SWAP_OK(t)))
21957c478bd9Sstevel@tonic-gate return (-1);
21967c478bd9Sstevel@tonic-gate
21977c478bd9Sstevel@tonic-gate ASSERT(t->t_state & (TS_SLEEP | TS_RUN));
21987c478bd9Sstevel@tonic-gate
2199d3d50737SRafael Vanoni swapin_time = (ddi_get_lbolt() - t->t_stime) / hz;
22007c478bd9Sstevel@tonic-gate
22017c478bd9Sstevel@tonic-gate if (flags == SOFTSWAP) {
22027c478bd9Sstevel@tonic-gate if (t->t_state == TS_SLEEP && swapin_time > maxslp) {
22037c478bd9Sstevel@tonic-gate epri = 0;
22047c478bd9Sstevel@tonic-gate } else {
22057c478bd9Sstevel@tonic-gate return ((pri_t)epri);
22067c478bd9Sstevel@tonic-gate }
22077c478bd9Sstevel@tonic-gate } else {
22087c478bd9Sstevel@tonic-gate pri_t pri;
22097c478bd9Sstevel@tonic-gate
22107c478bd9Sstevel@tonic-gate if ((t->t_state == TS_SLEEP && swapin_time > fss_minslp) ||
22117c478bd9Sstevel@tonic-gate (t->t_state == TS_RUN && swapin_time > fss_minrun)) {
22127c478bd9Sstevel@tonic-gate pri = fss_maxumdpri;
22137c478bd9Sstevel@tonic-gate epri = swapin_time -
22147c478bd9Sstevel@tonic-gate (rm_asrss(pp->p_as) / nz(maxpgio)/2) - (long)pri;
22157c478bd9Sstevel@tonic-gate } else {
22167c478bd9Sstevel@tonic-gate return ((pri_t)epri);
22177c478bd9Sstevel@tonic-gate }
22187c478bd9Sstevel@tonic-gate }
22197c478bd9Sstevel@tonic-gate
22207c478bd9Sstevel@tonic-gate /*
22217c478bd9Sstevel@tonic-gate * Scale epri so that SHRT_MAX / 2 represents zero priority.
22227c478bd9Sstevel@tonic-gate */
22237c478bd9Sstevel@tonic-gate epri += SHRT_MAX / 2;
22247c478bd9Sstevel@tonic-gate if (epri < 0)
22257c478bd9Sstevel@tonic-gate epri = 0;
22267c478bd9Sstevel@tonic-gate else if (epri > SHRT_MAX)
22277c478bd9Sstevel@tonic-gate epri = SHRT_MAX;
22287c478bd9Sstevel@tonic-gate
22297c478bd9Sstevel@tonic-gate return ((pri_t)epri);
22307c478bd9Sstevel@tonic-gate }
22317c478bd9Sstevel@tonic-gate
22327c478bd9Sstevel@tonic-gate /*
2233*6a0b1217SPatrick Mooney * Run swap-out checks when returning to userspace.
22347c478bd9Sstevel@tonic-gate */
22357c478bd9Sstevel@tonic-gate static void
fss_trapret(kthread_t * t)22367c478bd9Sstevel@tonic-gate fss_trapret(kthread_t *t)
22377c478bd9Sstevel@tonic-gate {
22387c478bd9Sstevel@tonic-gate cpu_t *cp = CPU;
22397c478bd9Sstevel@tonic-gate
22407c478bd9Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(t));
22417c478bd9Sstevel@tonic-gate ASSERT(t == curthread);
22427c478bd9Sstevel@tonic-gate ASSERT(cp->cpu_dispthread == t);
22437c478bd9Sstevel@tonic-gate ASSERT(t->t_state == TS_ONPROC);
22447c478bd9Sstevel@tonic-gate
22457c478bd9Sstevel@tonic-gate /*
22467c478bd9Sstevel@tonic-gate * Swapout lwp if the swapper is waiting for this thread to reach
22477c478bd9Sstevel@tonic-gate * a safe point.
22487c478bd9Sstevel@tonic-gate */
22497c478bd9Sstevel@tonic-gate if (t->t_schedflag & TS_SWAPENQ) {
22507c478bd9Sstevel@tonic-gate thread_unlock(t);
22517c478bd9Sstevel@tonic-gate swapout_lwp(ttolwp(t));
22527c478bd9Sstevel@tonic-gate thread_lock(t);
22537c478bd9Sstevel@tonic-gate }
22547c478bd9Sstevel@tonic-gate }
22557c478bd9Sstevel@tonic-gate
22567c478bd9Sstevel@tonic-gate /*
22577c478bd9Sstevel@tonic-gate * Arrange for thread to be placed in appropriate location on dispatcher queue.
22587c478bd9Sstevel@tonic-gate * This is called with the current thread in TS_ONPROC and locked.
22597c478bd9Sstevel@tonic-gate */
22607c478bd9Sstevel@tonic-gate static void
fss_preempt(kthread_t * t)22617c478bd9Sstevel@tonic-gate fss_preempt(kthread_t *t)
22627c478bd9Sstevel@tonic-gate {
22637c478bd9Sstevel@tonic-gate fssproc_t *fssproc = FSSPROC(t);
22647c478bd9Sstevel@tonic-gate klwp_t *lwp;
22657c478bd9Sstevel@tonic-gate uint_t flags;
22667c478bd9Sstevel@tonic-gate
22677c478bd9Sstevel@tonic-gate ASSERT(t == curthread);
22687c478bd9Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(curthread));
22697c478bd9Sstevel@tonic-gate ASSERT(t->t_state == TS_ONPROC);
22707c478bd9Sstevel@tonic-gate
2271c97ad5cdSakolb /*
2272c97ad5cdSakolb * This thread may be placed on wait queue by CPU Caps. In this case we
2273c97ad5cdSakolb * do not need to do anything until it is removed from the wait queue.
2274c97ad5cdSakolb * Do not enforce CPU caps on threads running at a kernel priority
2275c97ad5cdSakolb */
2276c97ad5cdSakolb if (CPUCAPS_ON()) {
2277c97ad5cdSakolb (void) cpucaps_charge(t, &fssproc->fss_caps,
22788c34bbb7Sakolb CPUCAPS_CHARGE_ENFORCE);
2279c97ad5cdSakolb
2280*6a0b1217SPatrick Mooney if (CPUCAPS_ENFORCE(t))
2281c97ad5cdSakolb return;
2282c97ad5cdSakolb }
2283c97ad5cdSakolb
22847c478bd9Sstevel@tonic-gate /*
22857c478bd9Sstevel@tonic-gate * If preempted in user-land mark the thread as swappable because it
22867c478bd9Sstevel@tonic-gate * cannot be holding any kernel locks.
22877c478bd9Sstevel@tonic-gate */
22887c478bd9Sstevel@tonic-gate ASSERT(t->t_schedflag & TS_DONT_SWAP);
2289*6a0b1217SPatrick Mooney lwp = ttolwp(t);
22907c478bd9Sstevel@tonic-gate if (lwp != NULL && lwp->lwp_state == LWP_USER)
22917c478bd9Sstevel@tonic-gate t->t_schedflag &= ~TS_DONT_SWAP;
22927c478bd9Sstevel@tonic-gate
22937c478bd9Sstevel@tonic-gate /*
22947c478bd9Sstevel@tonic-gate * Check to see if we're doing "preemption control" here. If
22957c478bd9Sstevel@tonic-gate * we are, and if the user has requested that this thread not
22967c478bd9Sstevel@tonic-gate * be preempted, and if preemptions haven't been put off for
22977c478bd9Sstevel@tonic-gate * too long, let the preemption happen here but try to make
22989ac8606fSraf * sure the thread is rescheduled as soon as possible. We do
22997c478bd9Sstevel@tonic-gate * this by putting it on the front of the highest priority run
23007c478bd9Sstevel@tonic-gate * queue in the FSS class. If the preemption has been put off
23017c478bd9Sstevel@tonic-gate * for too long, clear the "nopreempt" bit and let the thread
23027c478bd9Sstevel@tonic-gate * be preempted.
23037c478bd9Sstevel@tonic-gate */
23047c478bd9Sstevel@tonic-gate if (t->t_schedctl && schedctl_get_nopreempt(t)) {
23057c478bd9Sstevel@tonic-gate if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
23067c478bd9Sstevel@tonic-gate DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
2307*6a0b1217SPatrick Mooney /*
2308*6a0b1217SPatrick Mooney * If not already remembered, remember current
2309*6a0b1217SPatrick Mooney * priority for restoration in fss_yield().
2310*6a0b1217SPatrick Mooney */
2311*6a0b1217SPatrick Mooney if (!(fssproc->fss_flags & FSSRESTORE)) {
2312*6a0b1217SPatrick Mooney fssproc->fss_scpri = t->t_pri;
2313*6a0b1217SPatrick Mooney fssproc->fss_flags |= FSSRESTORE;
23147c478bd9Sstevel@tonic-gate }
2315*6a0b1217SPatrick Mooney THREAD_CHANGE_PRI(t, fss_maxumdpri);
2316*6a0b1217SPatrick Mooney t->t_schedflag |= TS_DONT_SWAP;
23179ac8606fSraf schedctl_set_yield(t, 1);
23187c478bd9Sstevel@tonic-gate setfrontdq(t);
23197c478bd9Sstevel@tonic-gate return;
23207c478bd9Sstevel@tonic-gate } else {
23219ac8606fSraf if (fssproc->fss_flags & FSSRESTORE) {
23229ac8606fSraf THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
23239ac8606fSraf fssproc->fss_flags &= ~FSSRESTORE;
23249ac8606fSraf }
23257c478bd9Sstevel@tonic-gate schedctl_set_nopreempt(t, 0);
23267c478bd9Sstevel@tonic-gate DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
23277c478bd9Sstevel@tonic-gate /*
23287c478bd9Sstevel@tonic-gate * Fall through and be preempted below.
23297c478bd9Sstevel@tonic-gate */
23307c478bd9Sstevel@tonic-gate }
23317c478bd9Sstevel@tonic-gate }
23327c478bd9Sstevel@tonic-gate
2333*6a0b1217SPatrick Mooney flags = fssproc->fss_flags & FSSBACKQ;
23347c478bd9Sstevel@tonic-gate
23357c478bd9Sstevel@tonic-gate if (flags == FSSBACKQ) {
23367c478bd9Sstevel@tonic-gate fssproc->fss_timeleft = fss_quantum;
23377c478bd9Sstevel@tonic-gate fssproc->fss_flags &= ~FSSBACKQ;
23387c478bd9Sstevel@tonic-gate setbackdq(t);
23397c478bd9Sstevel@tonic-gate } else {
23407c478bd9Sstevel@tonic-gate setfrontdq(t);
23417c478bd9Sstevel@tonic-gate }
23427c478bd9Sstevel@tonic-gate }
23437c478bd9Sstevel@tonic-gate
23447c478bd9Sstevel@tonic-gate /*
23457c478bd9Sstevel@tonic-gate * Called when a thread is waking up and is to be placed on the run queue.
23467c478bd9Sstevel@tonic-gate */
23477c478bd9Sstevel@tonic-gate static void
fss_setrun(kthread_t * t)23487c478bd9Sstevel@tonic-gate fss_setrun(kthread_t *t)
23497c478bd9Sstevel@tonic-gate {
23507c478bd9Sstevel@tonic-gate fssproc_t *fssproc = FSSPROC(t);
23517c478bd9Sstevel@tonic-gate
23527c478bd9Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(t)); /* t should be in transition */
23537c478bd9Sstevel@tonic-gate
23547c478bd9Sstevel@tonic-gate if (t->t_state == TS_SLEEP || t->t_state == TS_STOPPED)
23557c478bd9Sstevel@tonic-gate fss_active(t);
23567c478bd9Sstevel@tonic-gate
23577c478bd9Sstevel@tonic-gate fssproc->fss_timeleft = fss_quantum;
23587c478bd9Sstevel@tonic-gate
23597c478bd9Sstevel@tonic-gate fssproc->fss_flags &= ~FSSBACKQ;
2360*6a0b1217SPatrick Mooney THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
23617c478bd9Sstevel@tonic-gate
2362d3d50737SRafael Vanoni if (t->t_disp_time != ddi_get_lbolt())
23637c478bd9Sstevel@tonic-gate setbackdq(t);
23647c478bd9Sstevel@tonic-gate else
23657c478bd9Sstevel@tonic-gate setfrontdq(t);
23667c478bd9Sstevel@tonic-gate }
23677c478bd9Sstevel@tonic-gate
23687c478bd9Sstevel@tonic-gate /*
2369*6a0b1217SPatrick Mooney * Prepare thread for sleep.
23707c478bd9Sstevel@tonic-gate */
23717c478bd9Sstevel@tonic-gate static void
fss_sleep(kthread_t * t)23727c478bd9Sstevel@tonic-gate fss_sleep(kthread_t *t)
23737c478bd9Sstevel@tonic-gate {
23747c478bd9Sstevel@tonic-gate fssproc_t *fssproc = FSSPROC(t);
23757c478bd9Sstevel@tonic-gate
23767c478bd9Sstevel@tonic-gate ASSERT(t == curthread);
23777c478bd9Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(t));
23787c478bd9Sstevel@tonic-gate
23797c478bd9Sstevel@tonic-gate ASSERT(t->t_state == TS_ONPROC);
2380c97ad5cdSakolb
2381c97ad5cdSakolb /*
2382c97ad5cdSakolb * Account for time spent on CPU before going to sleep.
2383c97ad5cdSakolb */
23848c34bbb7Sakolb (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2385c97ad5cdSakolb
23867c478bd9Sstevel@tonic-gate fss_inactive(t);
2387d3d50737SRafael Vanoni t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
23887c478bd9Sstevel@tonic-gate }
23897c478bd9Sstevel@tonic-gate
23907c478bd9Sstevel@tonic-gate /*
23917c478bd9Sstevel@tonic-gate * A tick interrupt has ocurrend on a running thread. Check to see if our
23927c478bd9Sstevel@tonic-gate * time slice has expired. We must also clear the TS_DONT_SWAP flag in
23937c478bd9Sstevel@tonic-gate * t_schedflag if the thread is eligible to be swapped out.
23947c478bd9Sstevel@tonic-gate */
23957c478bd9Sstevel@tonic-gate static void
fss_tick(kthread_t * t)23967c478bd9Sstevel@tonic-gate fss_tick(kthread_t *t)
23977c478bd9Sstevel@tonic-gate {
23987c478bd9Sstevel@tonic-gate fssproc_t *fssproc;
23997c478bd9Sstevel@tonic-gate fssproj_t *fssproj;
24007c478bd9Sstevel@tonic-gate klwp_t *lwp;
2401c97ad5cdSakolb boolean_t call_cpu_surrender = B_FALSE;
2402c97ad5cdSakolb boolean_t cpucaps_enforce = B_FALSE;
24037c478bd9Sstevel@tonic-gate
24047c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
24057c478bd9Sstevel@tonic-gate
24067c478bd9Sstevel@tonic-gate /*
24077c478bd9Sstevel@tonic-gate * It's safe to access fsspset and fssproj structures because we're
24087c478bd9Sstevel@tonic-gate * holding our p_lock here.
24097c478bd9Sstevel@tonic-gate */
24107c478bd9Sstevel@tonic-gate thread_lock(t);
24117c478bd9Sstevel@tonic-gate fssproc = FSSPROC(t);
24127c478bd9Sstevel@tonic-gate fssproj = FSSPROC2FSSPROJ(fssproc);
24137c478bd9Sstevel@tonic-gate if (fssproj != NULL) {
24147c478bd9Sstevel@tonic-gate fsspset_t *fsspset = FSSPROJ2FSSPSET(fssproj);
24157c478bd9Sstevel@tonic-gate disp_lock_enter_high(&fsspset->fssps_displock);
24167c478bd9Sstevel@tonic-gate fssproj->fssp_ticks += fss_nice_tick[fssproc->fss_nice];
2417482a7749SJerry Jelinek fssproj->fssp_tick_cnt++;
24187c478bd9Sstevel@tonic-gate fssproc->fss_ticks++;
24197c478bd9Sstevel@tonic-gate disp_lock_exit_high(&fsspset->fssps_displock);
24207c478bd9Sstevel@tonic-gate }
24217c478bd9Sstevel@tonic-gate
2422c97ad5cdSakolb /*
2423c97ad5cdSakolb * Keep track of thread's project CPU usage. Note that projects
2424c97ad5cdSakolb * get charged even when threads are running in the kernel.
2425c97ad5cdSakolb * Do not surrender CPU if running in the SYS class.
2426c97ad5cdSakolb */
2427c97ad5cdSakolb if (CPUCAPS_ON()) {
2428*6a0b1217SPatrick Mooney cpucaps_enforce = cpucaps_charge(t, &fssproc->fss_caps,
2429*6a0b1217SPatrick Mooney CPUCAPS_CHARGE_ENFORCE);
2430c97ad5cdSakolb }
2431c97ad5cdSakolb
2432*6a0b1217SPatrick Mooney if (--fssproc->fss_timeleft <= 0) {
2433*6a0b1217SPatrick Mooney pri_t new_pri;
2434*6a0b1217SPatrick Mooney
24357c478bd9Sstevel@tonic-gate /*
2436*6a0b1217SPatrick Mooney * If we're doing preemption control and trying to avoid
2437*6a0b1217SPatrick Mooney * preempting this thread, just note that the thread should
2438*6a0b1217SPatrick Mooney * yield soon and let it keep running (unless it's been a
2439*6a0b1217SPatrick Mooney * while).
24407c478bd9Sstevel@tonic-gate */
2441*6a0b1217SPatrick Mooney if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2442*6a0b1217SPatrick Mooney if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2443*6a0b1217SPatrick Mooney DTRACE_SCHED1(schedctl__nopreempt,
2444*6a0b1217SPatrick Mooney kthread_t *, t);
2445*6a0b1217SPatrick Mooney schedctl_set_yield(t, 1);
2446*6a0b1217SPatrick Mooney thread_unlock_nopreempt(t);
2447*6a0b1217SPatrick Mooney return;
24487c478bd9Sstevel@tonic-gate }
2449*6a0b1217SPatrick Mooney }
2450*6a0b1217SPatrick Mooney fssproc->fss_flags &= ~FSSRESTORE;
24517c478bd9Sstevel@tonic-gate
2452*6a0b1217SPatrick Mooney fss_newpri(fssproc, B_TRUE);
2453*6a0b1217SPatrick Mooney new_pri = fssproc->fss_umdpri;
2454*6a0b1217SPatrick Mooney ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
24557c478bd9Sstevel@tonic-gate
2456*6a0b1217SPatrick Mooney /*
2457*6a0b1217SPatrick Mooney * When the priority of a thread is changed, it may be
2458*6a0b1217SPatrick Mooney * necessary to adjust its position on a sleep queue or
2459*6a0b1217SPatrick Mooney * dispatch queue. The function thread_change_pri accomplishes
2460*6a0b1217SPatrick Mooney * this.
2461*6a0b1217SPatrick Mooney */
2462*6a0b1217SPatrick Mooney if (thread_change_pri(t, new_pri, 0)) {
2463*6a0b1217SPatrick Mooney if ((t->t_schedflag & TS_LOAD) &&
2464*6a0b1217SPatrick Mooney (lwp = t->t_lwp) &&
2465*6a0b1217SPatrick Mooney lwp->lwp_state == LWP_USER)
2466*6a0b1217SPatrick Mooney t->t_schedflag &= ~TS_DONT_SWAP;
2467*6a0b1217SPatrick Mooney fssproc->fss_timeleft = fss_quantum;
2468*6a0b1217SPatrick Mooney } else {
2469c97ad5cdSakolb call_cpu_surrender = B_TRUE;
24707c478bd9Sstevel@tonic-gate }
2471*6a0b1217SPatrick Mooney } else if (t->t_state == TS_ONPROC &&
2472*6a0b1217SPatrick Mooney t->t_pri < t->t_disp_queue->disp_maxrunpri) {
2473*6a0b1217SPatrick Mooney /*
2474*6a0b1217SPatrick Mooney * If there is a higher-priority thread which is waiting for a
2475*6a0b1217SPatrick Mooney * processor, then thread surrenders the processor.
2476*6a0b1217SPatrick Mooney */
2477*6a0b1217SPatrick Mooney call_cpu_surrender = B_TRUE;
24787c478bd9Sstevel@tonic-gate }
2479c97ad5cdSakolb
2480c97ad5cdSakolb if (cpucaps_enforce && 2 * fssproc->fss_timeleft > fss_quantum) {
2481c97ad5cdSakolb /*
2482c97ad5cdSakolb * The thread used more than half of its quantum, so assume that
2483c97ad5cdSakolb * it used the whole quantum.
2484c97ad5cdSakolb *
2485c97ad5cdSakolb * Update thread's priority just before putting it on the wait
2486c97ad5cdSakolb * queue so that it gets charged for the CPU time from its
2487c97ad5cdSakolb * quantum even before that quantum expires.
2488c97ad5cdSakolb */
2489482a7749SJerry Jelinek fss_newpri(fssproc, B_FALSE);
2490c97ad5cdSakolb if (t->t_pri != fssproc->fss_umdpri)
2491c97ad5cdSakolb fss_change_priority(t, fssproc);
2492c97ad5cdSakolb
2493c97ad5cdSakolb /*
2494c97ad5cdSakolb * We need to call cpu_surrender for this thread due to cpucaps
2495c97ad5cdSakolb * enforcement, but fss_change_priority may have already done
2496c97ad5cdSakolb * so. In this case FSSBACKQ is set and there is no need to call
2497c97ad5cdSakolb * cpu-surrender again.
2498c97ad5cdSakolb */
2499c97ad5cdSakolb if (!(fssproc->fss_flags & FSSBACKQ))
2500c97ad5cdSakolb call_cpu_surrender = B_TRUE;
2501c97ad5cdSakolb }
2502c97ad5cdSakolb
2503c97ad5cdSakolb if (call_cpu_surrender) {
2504c97ad5cdSakolb fssproc->fss_flags |= FSSBACKQ;
2505c97ad5cdSakolb cpu_surrender(t);
2506c97ad5cdSakolb }
2507c97ad5cdSakolb
25087c478bd9Sstevel@tonic-gate thread_unlock_nopreempt(t); /* clock thread can't be preempted */
25097c478bd9Sstevel@tonic-gate }
25107c478bd9Sstevel@tonic-gate
25117c478bd9Sstevel@tonic-gate /*
25127c478bd9Sstevel@tonic-gate * Processes waking up go to the back of their queue. We don't need to assign
25137c478bd9Sstevel@tonic-gate * a time quantum here because thread is still at a kernel mode priority and
25147c478bd9Sstevel@tonic-gate * the time slicing is not done for threads running in the kernel after
25157c478bd9Sstevel@tonic-gate * sleeping. The proper time quantum will be assigned by fss_trapret before the
25167c478bd9Sstevel@tonic-gate * thread returns to user mode.
25177c478bd9Sstevel@tonic-gate */
25187c478bd9Sstevel@tonic-gate static void
fss_wakeup(kthread_t * t)25197c478bd9Sstevel@tonic-gate fss_wakeup(kthread_t *t)
25207c478bd9Sstevel@tonic-gate {
25217c478bd9Sstevel@tonic-gate fssproc_t *fssproc;
25227c478bd9Sstevel@tonic-gate
25237c478bd9Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(t));
25247c478bd9Sstevel@tonic-gate ASSERT(t->t_state == TS_SLEEP);
25257c478bd9Sstevel@tonic-gate
25267c478bd9Sstevel@tonic-gate fss_active(t);
25277c478bd9Sstevel@tonic-gate
2528d3d50737SRafael Vanoni t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
25297c478bd9Sstevel@tonic-gate fssproc = FSSPROC(t);
25307c478bd9Sstevel@tonic-gate fssproc->fss_flags &= ~FSSBACKQ;
25317c478bd9Sstevel@tonic-gate
2532*6a0b1217SPatrick Mooney /* Recalculate the priority. */
2533*6a0b1217SPatrick Mooney if (t->t_disp_time == ddi_get_lbolt()) {
2534*6a0b1217SPatrick Mooney setfrontdq(t);
25357c478bd9Sstevel@tonic-gate } else {
2536*6a0b1217SPatrick Mooney fssproc->fss_timeleft = fss_quantum;
2537*6a0b1217SPatrick Mooney THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2538*6a0b1217SPatrick Mooney setbackdq(t);
25397c478bd9Sstevel@tonic-gate }
25407c478bd9Sstevel@tonic-gate }
25417c478bd9Sstevel@tonic-gate
25427c478bd9Sstevel@tonic-gate /*
25437c478bd9Sstevel@tonic-gate * fss_donice() is called when a nice(1) command is issued on the thread to
25447c478bd9Sstevel@tonic-gate * alter the priority. The nice(1) command exists in Solaris for compatibility.
25457c478bd9Sstevel@tonic-gate * Thread priority adjustments should be done via priocntl(1).
25467c478bd9Sstevel@tonic-gate */
25477c478bd9Sstevel@tonic-gate static int
fss_donice(kthread_t * t,cred_t * cr,int incr,int * retvalp)25487c478bd9Sstevel@tonic-gate fss_donice(kthread_t *t, cred_t *cr, int incr, int *retvalp)
25497c478bd9Sstevel@tonic-gate {
25507c478bd9Sstevel@tonic-gate int newnice;
25517c478bd9Sstevel@tonic-gate fssproc_t *fssproc = FSSPROC(t);
25527c478bd9Sstevel@tonic-gate fssparms_t fssparms;
25537c478bd9Sstevel@tonic-gate
25547c478bd9Sstevel@tonic-gate /*
25557c478bd9Sstevel@tonic-gate * If there is no change to priority, just return current setting.
25567c478bd9Sstevel@tonic-gate */
25577c478bd9Sstevel@tonic-gate if (incr == 0) {
25587c478bd9Sstevel@tonic-gate if (retvalp)
25597c478bd9Sstevel@tonic-gate *retvalp = fssproc->fss_nice - NZERO;
25607c478bd9Sstevel@tonic-gate return (0);
25617c478bd9Sstevel@tonic-gate }
25627c478bd9Sstevel@tonic-gate
256324d819e6SJerry Jelinek if ((incr < 0 || incr > 2 * NZERO) && secpolicy_raisepriority(cr) != 0)
25647c478bd9Sstevel@tonic-gate return (EPERM);
25657c478bd9Sstevel@tonic-gate
25667c478bd9Sstevel@tonic-gate /*
25677c478bd9Sstevel@tonic-gate * Specifying a nice increment greater than the upper limit of
25687c478bd9Sstevel@tonic-gate * FSS_NICE_MAX (== 2 * NZERO - 1) will result in the thread's nice
25697c478bd9Sstevel@tonic-gate * value being set to the upper limit. We check for this before
25707c478bd9Sstevel@tonic-gate * computing the new value because otherwise we could get overflow
25717c478bd9Sstevel@tonic-gate * if a privileged user specified some ridiculous increment.
25727c478bd9Sstevel@tonic-gate */
25737c478bd9Sstevel@tonic-gate if (incr > FSS_NICE_MAX)
25747c478bd9Sstevel@tonic-gate incr = FSS_NICE_MAX;
25757c478bd9Sstevel@tonic-gate
25767c478bd9Sstevel@tonic-gate newnice = fssproc->fss_nice + incr;
25777c478bd9Sstevel@tonic-gate if (newnice > FSS_NICE_MAX)
25787c478bd9Sstevel@tonic-gate newnice = FSS_NICE_MAX;
25797c478bd9Sstevel@tonic-gate else if (newnice < FSS_NICE_MIN)
25807c478bd9Sstevel@tonic-gate newnice = FSS_NICE_MIN;
25817c478bd9Sstevel@tonic-gate
25827c478bd9Sstevel@tonic-gate fssparms.fss_uprilim = fssparms.fss_upri =
25837c478bd9Sstevel@tonic-gate -((newnice - NZERO) * fss_maxupri) / NZERO;
25847c478bd9Sstevel@tonic-gate
25857c478bd9Sstevel@tonic-gate /*
25867c478bd9Sstevel@tonic-gate * Reset the uprilim and upri values of the thread.
25877c478bd9Sstevel@tonic-gate */
25887c478bd9Sstevel@tonic-gate (void) fss_parmsset(t, (void *)&fssparms, (id_t)0, (cred_t *)NULL);
25897c478bd9Sstevel@tonic-gate
25907c478bd9Sstevel@tonic-gate /*
25917c478bd9Sstevel@tonic-gate * Although fss_parmsset already reset fss_nice it may not have been
25927c478bd9Sstevel@tonic-gate * set to precisely the value calculated above because fss_parmsset
25937c478bd9Sstevel@tonic-gate * determines the nice value from the user priority and we may have
25947c478bd9Sstevel@tonic-gate * truncated during the integer conversion from nice value to user
25957c478bd9Sstevel@tonic-gate * priority and back. We reset fss_nice to the value we calculated
25967c478bd9Sstevel@tonic-gate * above.
25977c478bd9Sstevel@tonic-gate */
25987c478bd9Sstevel@tonic-gate fssproc->fss_nice = (char)newnice;
25997c478bd9Sstevel@tonic-gate
26007c478bd9Sstevel@tonic-gate if (retvalp)
26017c478bd9Sstevel@tonic-gate *retvalp = newnice - NZERO;
26027c478bd9Sstevel@tonic-gate return (0);
26037c478bd9Sstevel@tonic-gate }
26047c478bd9Sstevel@tonic-gate
2605d4204c85Sraf /*
2606d4204c85Sraf * Increment the priority of the specified thread by incr and
2607d4204c85Sraf * return the new value in *retvalp.
2608d4204c85Sraf */
2609d4204c85Sraf static int
fss_doprio(kthread_t * t,cred_t * cr,int incr,int * retvalp)2610d4204c85Sraf fss_doprio(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2611d4204c85Sraf {
2612d4204c85Sraf int newpri;
2613d4204c85Sraf fssproc_t *fssproc = FSSPROC(t);
2614d4204c85Sraf fssparms_t fssparms;
2615d4204c85Sraf
2616d4204c85Sraf /*
2617d4204c85Sraf * If there is no change to priority, just return current setting.
2618d4204c85Sraf */
2619d4204c85Sraf if (incr == 0) {
2620d4204c85Sraf *retvalp = fssproc->fss_upri;
2621d4204c85Sraf return (0);
2622d4204c85Sraf }
2623d4204c85Sraf
2624d4204c85Sraf newpri = fssproc->fss_upri + incr;
2625d4204c85Sraf if (newpri > fss_maxupri || newpri < -fss_maxupri)
2626d4204c85Sraf return (EINVAL);
2627d4204c85Sraf
2628d4204c85Sraf *retvalp = newpri;
2629d4204c85Sraf fssparms.fss_uprilim = fssparms.fss_upri = newpri;
2630d4204c85Sraf
2631d4204c85Sraf /*
2632d4204c85Sraf * Reset the uprilim and upri values of the thread.
2633d4204c85Sraf */
2634d4204c85Sraf return (fss_parmsset(t, &fssparms, (id_t)0, cr));
2635d4204c85Sraf }
2636d4204c85Sraf
26377c478bd9Sstevel@tonic-gate /*
26387c478bd9Sstevel@tonic-gate * Return the global scheduling priority that would be assigned to a thread
26397c478bd9Sstevel@tonic-gate * entering the fair-sharing class with the fss_upri.
26407c478bd9Sstevel@tonic-gate */
26417c478bd9Sstevel@tonic-gate /*ARGSUSED*/
26427c478bd9Sstevel@tonic-gate static pri_t
fss_globpri(kthread_t * t)26437c478bd9Sstevel@tonic-gate fss_globpri(kthread_t *t)
26447c478bd9Sstevel@tonic-gate {
26457c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
26467c478bd9Sstevel@tonic-gate
26477c478bd9Sstevel@tonic-gate return (fss_maxumdpri / 2);
26487c478bd9Sstevel@tonic-gate }
26497c478bd9Sstevel@tonic-gate
26507c478bd9Sstevel@tonic-gate /*
26517c478bd9Sstevel@tonic-gate * Called from the yield(2) system call when a thread is yielding (surrendering)
26527c478bd9Sstevel@tonic-gate * the processor. The kernel thread is placed at the back of a dispatch queue.
26537c478bd9Sstevel@tonic-gate */
26547c478bd9Sstevel@tonic-gate static void
fss_yield(kthread_t * t)26557c478bd9Sstevel@tonic-gate fss_yield(kthread_t *t)
26567c478bd9Sstevel@tonic-gate {
26577c478bd9Sstevel@tonic-gate fssproc_t *fssproc = FSSPROC(t);
26587c478bd9Sstevel@tonic-gate
26597c478bd9Sstevel@tonic-gate ASSERT(t == curthread);
26607c478bd9Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(t));
26617c478bd9Sstevel@tonic-gate
2662c97ad5cdSakolb /*
2663c97ad5cdSakolb * Collect CPU usage spent before yielding
2664c97ad5cdSakolb */
26658c34bbb7Sakolb (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2666c97ad5cdSakolb
26677c478bd9Sstevel@tonic-gate /*
26687c478bd9Sstevel@tonic-gate * Clear the preemption control "yield" bit since the user is
26697c478bd9Sstevel@tonic-gate * doing a yield.
26707c478bd9Sstevel@tonic-gate */
26717c478bd9Sstevel@tonic-gate if (t->t_schedctl)
26727c478bd9Sstevel@tonic-gate schedctl_set_yield(t, 0);
26739ac8606fSraf /*
26749ac8606fSraf * If fss_preempt() artifically increased the thread's priority
26759ac8606fSraf * to avoid preemption, restore the original priority now.
26769ac8606fSraf */
26779ac8606fSraf if (fssproc->fss_flags & FSSRESTORE) {
26789ac8606fSraf THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
26799ac8606fSraf fssproc->fss_flags &= ~FSSRESTORE;
26809ac8606fSraf }
26817c478bd9Sstevel@tonic-gate if (fssproc->fss_timeleft < 0) {
26827c478bd9Sstevel@tonic-gate /*
26837c478bd9Sstevel@tonic-gate * Time slice was artificially extended to avoid preemption,
26847c478bd9Sstevel@tonic-gate * so pretend we're preempting it now.
26857c478bd9Sstevel@tonic-gate */
26867c478bd9Sstevel@tonic-gate DTRACE_SCHED1(schedctl__yield, int, -fssproc->fss_timeleft);
26877c478bd9Sstevel@tonic-gate fssproc->fss_timeleft = fss_quantum;
26887c478bd9Sstevel@tonic-gate }
26897c478bd9Sstevel@tonic-gate fssproc->fss_flags &= ~FSSBACKQ;
26907c478bd9Sstevel@tonic-gate setbackdq(t);
26917c478bd9Sstevel@tonic-gate }
26927c478bd9Sstevel@tonic-gate
26937c478bd9Sstevel@tonic-gate void
fss_changeproj(kthread_t * t,void * kp,void * zp,fssbuf_t * projbuf,fssbuf_t * zonebuf)26947c478bd9Sstevel@tonic-gate fss_changeproj(kthread_t *t, void *kp, void *zp, fssbuf_t *projbuf,
26957c478bd9Sstevel@tonic-gate fssbuf_t *zonebuf)
26967c478bd9Sstevel@tonic-gate {
26977c478bd9Sstevel@tonic-gate kproject_t *kpj_new = kp;
26987c478bd9Sstevel@tonic-gate zone_t *zone = zp;
26997c478bd9Sstevel@tonic-gate fssproj_t *fssproj_old, *fssproj_new;
27007c478bd9Sstevel@tonic-gate fsspset_t *fsspset;
27017c478bd9Sstevel@tonic-gate kproject_t *kpj_old;
27027c478bd9Sstevel@tonic-gate fssproc_t *fssproc;
27037c478bd9Sstevel@tonic-gate fsszone_t *fsszone_old, *fsszone_new;
27047c478bd9Sstevel@tonic-gate int free = 0;
27057c478bd9Sstevel@tonic-gate int id;
27067c478bd9Sstevel@tonic-gate
27077c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock));
27087c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&pidlock));
27097c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
27107c478bd9Sstevel@tonic-gate
27117c478bd9Sstevel@tonic-gate if (t->t_cid != fss_cid)
27127c478bd9Sstevel@tonic-gate return;
27137c478bd9Sstevel@tonic-gate
27147c478bd9Sstevel@tonic-gate fssproc = FSSPROC(t);
27157c478bd9Sstevel@tonic-gate mutex_enter(&fsspsets_lock);
27167c478bd9Sstevel@tonic-gate fssproj_old = FSSPROC2FSSPROJ(fssproc);
27177c478bd9Sstevel@tonic-gate if (fssproj_old == NULL) {
27187c478bd9Sstevel@tonic-gate mutex_exit(&fsspsets_lock);
27197c478bd9Sstevel@tonic-gate return;
27207c478bd9Sstevel@tonic-gate }
27217c478bd9Sstevel@tonic-gate
27227c478bd9Sstevel@tonic-gate fsspset = FSSPROJ2FSSPSET(fssproj_old);
27237c478bd9Sstevel@tonic-gate mutex_enter(&fsspset->fssps_lock);
27247c478bd9Sstevel@tonic-gate kpj_old = FSSPROJ2KPROJ(fssproj_old);
27257c478bd9Sstevel@tonic-gate fsszone_old = fssproj_old->fssp_fsszone;
27267c478bd9Sstevel@tonic-gate
27277c478bd9Sstevel@tonic-gate ASSERT(t->t_cpupart == fsspset->fssps_cpupart);
27287c478bd9Sstevel@tonic-gate
27297c478bd9Sstevel@tonic-gate if (kpj_old == kpj_new) {
27307c478bd9Sstevel@tonic-gate mutex_exit(&fsspset->fssps_lock);
27317c478bd9Sstevel@tonic-gate mutex_exit(&fsspsets_lock);
27327c478bd9Sstevel@tonic-gate return;
27337c478bd9Sstevel@tonic-gate }
27347c478bd9Sstevel@tonic-gate
27357c478bd9Sstevel@tonic-gate if ((fsszone_new = fss_find_fsszone(fsspset, zone)) == NULL) {
27367c478bd9Sstevel@tonic-gate /*
27377c478bd9Sstevel@tonic-gate * If the zone for the new project is not currently active on
27387c478bd9Sstevel@tonic-gate * the cpu partition we're on, get one of the pre-allocated
27397c478bd9Sstevel@tonic-gate * buffers and link it in our per-pset zone list. Such buffers
27407c478bd9Sstevel@tonic-gate * should already exist.
27417c478bd9Sstevel@tonic-gate */
27427c478bd9Sstevel@tonic-gate for (id = 0; id < zonebuf->fssb_size; id++) {
27437c478bd9Sstevel@tonic-gate if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
27447c478bd9Sstevel@tonic-gate fss_insert_fsszone(fsspset, zone, fsszone_new);
27457c478bd9Sstevel@tonic-gate zonebuf->fssb_list[id] = NULL;
27467c478bd9Sstevel@tonic-gate break;
27477c478bd9Sstevel@tonic-gate }
27487c478bd9Sstevel@tonic-gate }
27497c478bd9Sstevel@tonic-gate }
27507c478bd9Sstevel@tonic-gate ASSERT(fsszone_new != NULL);
27517c478bd9Sstevel@tonic-gate if ((fssproj_new = fss_find_fssproj(fsspset, kpj_new)) == NULL) {
27527c478bd9Sstevel@tonic-gate /*
27537c478bd9Sstevel@tonic-gate * If our new project is not currently running
27547c478bd9Sstevel@tonic-gate * on the cpu partition we're on, get one of the
27557c478bd9Sstevel@tonic-gate * pre-allocated buffers and link it in our new cpu
27567c478bd9Sstevel@tonic-gate * partition doubly linked list. Such buffers should already
27577c478bd9Sstevel@tonic-gate * exist.
27587c478bd9Sstevel@tonic-gate */
27597c478bd9Sstevel@tonic-gate for (id = 0; id < projbuf->fssb_size; id++) {
27607c478bd9Sstevel@tonic-gate if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
27617c478bd9Sstevel@tonic-gate fss_insert_fssproj(fsspset, kpj_new,
27627c478bd9Sstevel@tonic-gate fsszone_new, fssproj_new);
27637c478bd9Sstevel@tonic-gate projbuf->fssb_list[id] = NULL;
27647c478bd9Sstevel@tonic-gate break;
27657c478bd9Sstevel@tonic-gate }
27667c478bd9Sstevel@tonic-gate }
27677c478bd9Sstevel@tonic-gate }
27687c478bd9Sstevel@tonic-gate ASSERT(fssproj_new != NULL);
27697c478bd9Sstevel@tonic-gate
27707c478bd9Sstevel@tonic-gate thread_lock(t);
2771c97ad5cdSakolb if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2772c97ad5cdSakolb t->t_state == TS_WAIT)
27737c478bd9Sstevel@tonic-gate fss_inactive(t);
27747c478bd9Sstevel@tonic-gate ASSERT(fssproj_old->fssp_threads > 0);
27757c478bd9Sstevel@tonic-gate if (--fssproj_old->fssp_threads == 0) {
27767c478bd9Sstevel@tonic-gate fss_remove_fssproj(fsspset, fssproj_old);
27777c478bd9Sstevel@tonic-gate free = 1;
27787c478bd9Sstevel@tonic-gate }
27797c478bd9Sstevel@tonic-gate fssproc->fss_proj = fssproj_new;
27807c478bd9Sstevel@tonic-gate fssproc->fss_fsspri = 0;
27817c478bd9Sstevel@tonic-gate fssproj_new->fssp_threads++;
2782c97ad5cdSakolb if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2783c97ad5cdSakolb t->t_state == TS_WAIT)
27847c478bd9Sstevel@tonic-gate fss_active(t);
27857c478bd9Sstevel@tonic-gate thread_unlock(t);
27867c478bd9Sstevel@tonic-gate if (free) {
27877c478bd9Sstevel@tonic-gate if (fsszone_old->fssz_nproj == 0)
27887c478bd9Sstevel@tonic-gate kmem_free(fsszone_old, sizeof (fsszone_t));
27897c478bd9Sstevel@tonic-gate kmem_free(fssproj_old, sizeof (fssproj_t));
27907c478bd9Sstevel@tonic-gate }
27917c478bd9Sstevel@tonic-gate
27927c478bd9Sstevel@tonic-gate mutex_exit(&fsspset->fssps_lock);
27937c478bd9Sstevel@tonic-gate mutex_exit(&fsspsets_lock);
27947c478bd9Sstevel@tonic-gate }
27957c478bd9Sstevel@tonic-gate
27967c478bd9Sstevel@tonic-gate void
fss_changepset(kthread_t * t,void * newcp,fssbuf_t * projbuf,fssbuf_t * zonebuf)27977c478bd9Sstevel@tonic-gate fss_changepset(kthread_t *t, void *newcp, fssbuf_t *projbuf,
27987c478bd9Sstevel@tonic-gate fssbuf_t *zonebuf)
27997c478bd9Sstevel@tonic-gate {
28007c478bd9Sstevel@tonic-gate fsspset_t *fsspset_old, *fsspset_new;
28017c478bd9Sstevel@tonic-gate fssproj_t *fssproj_old, *fssproj_new;
28027c478bd9Sstevel@tonic-gate fsszone_t *fsszone_old, *fsszone_new;
28037c478bd9Sstevel@tonic-gate fssproc_t *fssproc;
28047c478bd9Sstevel@tonic-gate kproject_t *kpj;
28057c478bd9Sstevel@tonic-gate zone_t *zone;
28067c478bd9Sstevel@tonic-gate int id;
28077c478bd9Sstevel@tonic-gate
28087c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock));
28097c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&pidlock));
28107c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
28117c478bd9Sstevel@tonic-gate
28127c478bd9Sstevel@tonic-gate if (t->t_cid != fss_cid)
28137c478bd9Sstevel@tonic-gate return;
28147c478bd9Sstevel@tonic-gate
28157c478bd9Sstevel@tonic-gate fssproc = FSSPROC(t);
28167c478bd9Sstevel@tonic-gate zone = ttoproc(t)->p_zone;
28177c478bd9Sstevel@tonic-gate mutex_enter(&fsspsets_lock);
28187c478bd9Sstevel@tonic-gate fssproj_old = FSSPROC2FSSPROJ(fssproc);
28197c478bd9Sstevel@tonic-gate if (fssproj_old == NULL) {
28207c478bd9Sstevel@tonic-gate mutex_exit(&fsspsets_lock);
28217c478bd9Sstevel@tonic-gate return;
28227c478bd9Sstevel@tonic-gate }
28237c478bd9Sstevel@tonic-gate fsszone_old = fssproj_old->fssp_fsszone;
28247c478bd9Sstevel@tonic-gate fsspset_old = FSSPROJ2FSSPSET(fssproj_old);
28257c478bd9Sstevel@tonic-gate kpj = FSSPROJ2KPROJ(fssproj_old);
28267c478bd9Sstevel@tonic-gate
28277c478bd9Sstevel@tonic-gate if (fsspset_old->fssps_cpupart == newcp) {
28287c478bd9Sstevel@tonic-gate mutex_exit(&fsspsets_lock);
28297c478bd9Sstevel@tonic-gate return;
28307c478bd9Sstevel@tonic-gate }
28317c478bd9Sstevel@tonic-gate
28327c478bd9Sstevel@tonic-gate ASSERT(ttoproj(t) == kpj);
28337c478bd9Sstevel@tonic-gate
28347c478bd9Sstevel@tonic-gate fsspset_new = fss_find_fsspset(newcp);
28357c478bd9Sstevel@tonic-gate
28367c478bd9Sstevel@tonic-gate mutex_enter(&fsspset_new->fssps_lock);
28377c478bd9Sstevel@tonic-gate if ((fsszone_new = fss_find_fsszone(fsspset_new, zone)) == NULL) {
28387c478bd9Sstevel@tonic-gate for (id = 0; id < zonebuf->fssb_size; id++) {
28397c478bd9Sstevel@tonic-gate if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
28407c478bd9Sstevel@tonic-gate fss_insert_fsszone(fsspset_new, zone,
28417c478bd9Sstevel@tonic-gate fsszone_new);
28427c478bd9Sstevel@tonic-gate zonebuf->fssb_list[id] = NULL;
28437c478bd9Sstevel@tonic-gate break;
28447c478bd9Sstevel@tonic-gate }
28457c478bd9Sstevel@tonic-gate }
28467c478bd9Sstevel@tonic-gate }
28477c478bd9Sstevel@tonic-gate ASSERT(fsszone_new != NULL);
28487c478bd9Sstevel@tonic-gate if ((fssproj_new = fss_find_fssproj(fsspset_new, kpj)) == NULL) {
28497c478bd9Sstevel@tonic-gate for (id = 0; id < projbuf->fssb_size; id++) {
28507c478bd9Sstevel@tonic-gate if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
28517c478bd9Sstevel@tonic-gate fss_insert_fssproj(fsspset_new, kpj,
28527c478bd9Sstevel@tonic-gate fsszone_new, fssproj_new);
28537c478bd9Sstevel@tonic-gate projbuf->fssb_list[id] = NULL;
28547c478bd9Sstevel@tonic-gate break;
28557c478bd9Sstevel@tonic-gate }
28567c478bd9Sstevel@tonic-gate }
28577c478bd9Sstevel@tonic-gate }
28587c478bd9Sstevel@tonic-gate ASSERT(fssproj_new != NULL);
28597c478bd9Sstevel@tonic-gate
28607c478bd9Sstevel@tonic-gate fssproj_new->fssp_threads++;
28617c478bd9Sstevel@tonic-gate thread_lock(t);
2862c97ad5cdSakolb if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2863c97ad5cdSakolb t->t_state == TS_WAIT)
2864d4204c85Sraf fss_inactive(t);
28657c478bd9Sstevel@tonic-gate fssproc->fss_proj = fssproj_new;
28667c478bd9Sstevel@tonic-gate fssproc->fss_fsspri = 0;
2867c97ad5cdSakolb if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2868c97ad5cdSakolb t->t_state == TS_WAIT)
2869d4204c85Sraf fss_active(t);
28707c478bd9Sstevel@tonic-gate thread_unlock(t);
28717c478bd9Sstevel@tonic-gate mutex_exit(&fsspset_new->fssps_lock);
28727c478bd9Sstevel@tonic-gate
28737c478bd9Sstevel@tonic-gate mutex_enter(&fsspset_old->fssps_lock);
28747c478bd9Sstevel@tonic-gate if (--fssproj_old->fssp_threads == 0) {
28757c478bd9Sstevel@tonic-gate fss_remove_fssproj(fsspset_old, fssproj_old);
28767c478bd9Sstevel@tonic-gate if (fsszone_old->fssz_nproj == 0)
28777c478bd9Sstevel@tonic-gate kmem_free(fsszone_old, sizeof (fsszone_t));
28787c478bd9Sstevel@tonic-gate kmem_free(fssproj_old, sizeof (fssproj_t));
28797c478bd9Sstevel@tonic-gate }
28807c478bd9Sstevel@tonic-gate mutex_exit(&fsspset_old->fssps_lock);
28817c478bd9Sstevel@tonic-gate
28827c478bd9Sstevel@tonic-gate mutex_exit(&fsspsets_lock);
28837c478bd9Sstevel@tonic-gate }
2884