Lines Matching refs:tp
105 void disp_swapped_enq(kthread_t *tp);
106 static void disp_swapped_setrun(kthread_t *tp);
118 static void setkpdq(kthread_t *tp, int borf);
160 static kthread_t *disp_ratify(kthread_t *tp, disp_t *kpq);
727 kthread_t *tp; in disp() local
750 (tp = disp_getbest(kpq)) != NULL) { in disp()
751 if (disp_ratify(tp, kpq) != NULL) { in disp()
753 "disp_end:tid %p", tp); in disp()
754 return (tp); in disp()
771 if ((tp = disp_getwork(cpup)) == NULL || in disp()
772 tp == T_DONTSTEAL) { in disp()
773 tp = cpup->cpu_idle_thread; in disp()
775 THREAD_ONPROC(tp, cpup); in disp()
776 cpup->cpu_dispthread = tp; in disp()
783 tp = cpup->cpu_idle_thread; in disp()
784 THREAD_ONPROC(tp, cpup); in disp()
785 cpup->cpu_dispthread = tp; in disp()
791 "disp_end:tid %p", tp); in disp()
792 return (tp); in disp()
796 tp = dq->dq_first; in disp()
798 ASSERT(tp != NULL); in disp()
799 ASSERT(tp->t_schedflag & TS_LOAD); /* thread must be swapped in */ in disp()
801 DTRACE_SCHED2(dequeue, kthread_t *, tp, disp_t *, dp); in disp()
808 if ((dq->dq_first = tp->t_link) == NULL) { in disp()
835 tp->t_link = NULL; in disp()
843 tp->t_schedflag |= TS_DONT_SWAP; in disp()
844 cpup->cpu_dispthread = tp; /* protected by spl only */ in disp()
846 ASSERT(pri == DISP_PRIO(tp)); in disp()
847 thread_onproc(tp, cpup); /* set t_state to TS_ONPROC */ in disp()
850 ASSERT(tp != NULL); in disp()
852 "disp_end:tid %p", tp); in disp()
854 if (disp_ratify(tp, kpq) == NULL) in disp()
857 return (tp); in disp()
1023 search_disp_queues(disp_t *dp, kthread_t *tp) in search_disp_queues() argument
1036 if (tp == rp) { in search_disp_queues()
1052 thread_on_queue(kthread_t *tp) in thread_on_queue() argument
1064 if (search_disp_queues(cp->cpu_disp, tp)) in thread_on_queue()
1073 if (search_disp_queues(&part->cp_kp_queue, tp)) in thread_on_queue()
1082 #define thread_on_queue(tp) 0 /* ASSERT must be !thread_on_queue */ argument
1199 setbackdq(kthread_t *tp) in setbackdq() argument
1208 ASSERT(THREAD_LOCK_HELD(tp)); in setbackdq()
1209 ASSERT((tp->t_schedflag & TS_ALLSTART) == 0); in setbackdq()
1210 ASSERT(!thread_on_queue(tp)); /* make sure tp isn't on a runq */ in setbackdq()
1216 if ((tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)) != TS_LOAD) { in setbackdq()
1217 disp_swapped_setrun(tp); in setbackdq()
1221 self = (tp == curthread); in setbackdq()
1223 if (tp->t_bound_cpu || tp->t_weakbound_cpu) in setbackdq()
1228 tpri = DISP_PRIO(tp); in setbackdq()
1230 cp = tp->t_cpu; in setbackdq()
1233 setkpdq(tp, SETKP_BACK); in setbackdq()
1246 if ((!THREAD_HAS_CACHE_WARMTH(tp)) || in setbackdq()
1247 !smt_should_run(tp, tp->t_cpu) || in setbackdq()
1248 (tp->t_cpu == cpu_inmotion) || in setbackdq()
1249 !LGRP_CONTAINS_CPU(tp->t_lpl->lpl_lgrp, tp->t_cpu)) { in setbackdq()
1250 cp = disp_lowpri_cpu(tp->t_cpu, tp, tpri); in setbackdq()
1252 cp = tp->t_cpu; in setbackdq()
1255 if (tp->t_cpupart == cp->cpu_part) { in setbackdq()
1261 cp = cmt_balance(tp, cp); in setbackdq()
1268 !(tp->t_schedflag & TS_RUNQMATCH)) in setbackdq()
1273 if (tp->t_lpl->lpl_lgrpid == LGRP_ROOTID) { in setbackdq()
1279 if (smt_should_run(tp, newcp) && in setbackdq()
1282 kthread_t *, tp, in setbackdq()
1291 cp = disp_lowpri_cpu(tp->t_cpupart->cp_cpulist, tp, in setbackdq()
1292 tp->t_pri); in setbackdq()
1302 cp = tp->t_weakbound_cpu ? in setbackdq()
1303 tp->t_weakbound_cpu : tp->t_bound_cpu; in setbackdq()
1313 if ((!self) && (tp->t_waitrq == 0)) { in setbackdq()
1317 (void) cpu_update_pct(tp, curtime); in setbackdq()
1318 tp->t_waitrq = curtime; in setbackdq()
1320 (void) cpu_update_pct(tp, gethrtime_unscaled()); in setbackdq()
1326 DTRACE_SCHED3(enqueue, kthread_t *, tp, disp_t *, dp, int, 0); in setbackdq()
1328 tpri, cp, tp); in setbackdq()
1332 THREAD_RUN(tp, &dp->disp_lock); /* set t_state to TS_RUN */ in setbackdq()
1333 tp->t_disp_queue = dp; in setbackdq()
1334 tp->t_link = NULL; in setbackdq()
1344 dq->dq_last->t_link = tp; in setbackdq()
1345 dq->dq_last = tp; in setbackdq()
1349 dq->dq_first = dq->dq_last = tp; in setbackdq()
1384 setfrontdq(kthread_t *tp) in setfrontdq() argument
1392 ASSERT(THREAD_LOCK_HELD(tp)); in setfrontdq()
1393 ASSERT((tp->t_schedflag & TS_ALLSTART) == 0); in setfrontdq()
1394 ASSERT(!thread_on_queue(tp)); /* make sure tp isn't on a runq */ in setfrontdq()
1400 if ((tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)) != TS_LOAD) { in setfrontdq()
1401 disp_swapped_setrun(tp); in setfrontdq()
1405 if (tp->t_bound_cpu || tp->t_weakbound_cpu) in setfrontdq()
1410 tpri = DISP_PRIO(tp); in setfrontdq()
1412 cp = tp->t_cpu; in setfrontdq()
1415 setkpdq(tp, SETKP_FRONT); in setfrontdq()
1418 cp = tp->t_cpu; in setfrontdq()
1419 if (tp->t_cpupart == cp->cpu_part) { in setfrontdq()
1431 if (!LGRP_CONTAINS_CPU(tp->t_lpl->lpl_lgrp, cp) || in setfrontdq()
1434 !THREAD_HAS_CACHE_WARMTH(tp))) { in setfrontdq()
1435 cp = disp_lowpri_cpu(tp->t_cpu, tp, tpri); in setfrontdq()
1441 cp = disp_lowpri_cpu(tp->t_cpupart->cp_cpulist, in setfrontdq()
1442 tp, tp->t_pri); in setfrontdq()
1452 cp = tp->t_weakbound_cpu ? in setfrontdq()
1453 tp->t_weakbound_cpu : tp->t_bound_cpu; in setfrontdq()
1464 if ((tp != curthread) && (tp->t_waitrq == 0)) { in setfrontdq()
1468 (void) cpu_update_pct(tp, curtime); in setfrontdq()
1469 tp->t_waitrq = curtime; in setfrontdq()
1471 (void) cpu_update_pct(tp, gethrtime_unscaled()); in setfrontdq()
1477 TRACE_2(TR_FAC_DISP, TR_FRONTQ, "frontq:pri %d tid %p", tpri, tp); in setfrontdq()
1478 DTRACE_SCHED3(enqueue, kthread_t *, tp, disp_t *, dp, int, 1); in setfrontdq()
1482 THREAD_RUN(tp, &dp->disp_lock); /* set TS_RUN state and lock */ in setfrontdq()
1483 tp->t_disp_queue = dp; in setfrontdq()
1493 tp->t_link = dq->dq_first; in setfrontdq()
1494 dq->dq_first = tp; in setfrontdq()
1498 tp->t_link = NULL; in setfrontdq()
1499 dq->dq_first = dq->dq_last = tp; in setfrontdq()
1509 if (tp == curthread && dp->disp_max_unbound_pri == -1 && in setfrontdq()
1530 setkpdq(kthread_t *tp, int borf) in setkpdq() argument
1537 tpri = DISP_PRIO(tp); in setkpdq()
1539 dp = &tp->t_cpupart->cp_kp_queue; in setkpdq()
1542 TRACE_2(TR_FAC_DISP, TR_FRONTQ, "frontq:pri %d tid %p", tpri, tp); in setkpdq()
1545 DTRACE_SCHED3(enqueue, kthread_t *, tp, disp_t *, dp, int, borf); in setkpdq()
1546 THREAD_RUN(tp, &dp->disp_lock); /* set t_state to TS_RUN */ in setkpdq()
1547 tp->t_disp_queue = dp; in setkpdq()
1554 tp->t_link = NULL; in setkpdq()
1555 dq->dq_last->t_link = tp; in setkpdq()
1556 dq->dq_last = tp; in setkpdq()
1559 tp->t_link = dq->dq_first; in setkpdq()
1560 dq->dq_first = tp; in setkpdq()
1566 dq->dq_first = dq->dq_last = tp; in setkpdq()
1570 tp->t_link = NULL; in setkpdq()
1571 dq->dq_first = dq->dq_last = tp; in setkpdq()
1582 cp = tp->t_cpu; in setkpdq()
1583 if (tp->t_cpupart != cp->cpu_part) { in setkpdq()
1585 cp = tp->t_cpupart->cp_cpulist; in setkpdq()
1587 cp = disp_lowpri_cpu(cp, tp, tp->t_pri); in setkpdq()
1604 dispdeq(kthread_t *tp) in dispdeq() argument
1613 ASSERT(THREAD_LOCK_HELD(tp)); in dispdeq()
1615 if (tp->t_state != TS_RUN) in dispdeq()
1622 if ((tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)) != TS_LOAD) in dispdeq()
1625 tpri = DISP_PRIO(tp); in dispdeq()
1626 dp = tp->t_disp_queue; in dispdeq()
1639 while (rp != tp && rp != NULL) { in dispdeq()
1649 DTRACE_SCHED2(dequeue, kthread_t *, tp, disp_t *, dp); in dispdeq()
1673 tp->t_link = NULL; in dispdeq()
1674 THREAD_TRANSITION(tp); /* put in intermediate state */ in dispdeq()
1725 disp_swapped_enq(kthread_t *tp) in disp_swapped_enq() argument
1727 ASSERT(THREAD_LOCK_HELD(tp)); in disp_swapped_enq()
1728 ASSERT(tp->t_schedflag & TS_LOAD); in disp_swapped_enq()
1730 switch (tp->t_state) { in disp_swapped_enq()
1733 THREAD_SWAP(tp, &swapped_lock); /* set TS_RUN state and lock */ in disp_swapped_enq()
1737 THREAD_TRANSITION(tp); in disp_swapped_enq()
1739 THREAD_SWAP(tp, &swapped_lock); /* set TS_RUN state and lock */ in disp_swapped_enq()
1742 panic("disp_swapped: tp: %p bad t_state", (void *)tp); in disp_swapped_enq()
1757 disp_swapped_setrun(kthread_t *tp) in disp_swapped_setrun() argument
1759 ASSERT(THREAD_LOCK_HELD(tp)); in disp_swapped_setrun()
1760 ASSERT((tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)) != TS_LOAD); in disp_swapped_setrun()
1762 switch (tp->t_state) { in disp_swapped_setrun()
1769 if (DISP_PRIO(tp) > maxclsyspri) in disp_swapped_setrun()
1773 THREAD_RUN(tp, &swapped_lock); /* set TS_RUN state and lock */ in disp_swapped_setrun()
1778 panic("disp_swapped_setrun: tp: %p bad t_state", (void *)tp); in disp_swapped_setrun()
1794 cpu_surrender(kthread_t *tp) in cpu_surrender() argument
1801 ASSERT(THREAD_LOCK_HELD(tp)); in cpu_surrender()
1803 if (tp->t_state != TS_ONPROC) in cpu_surrender()
1805 cpup = tp->t_disp_queue->disp_cpu; /* CPU thread dispatched to */ in cpu_surrender()
1811 if (tp->t_cid == sysdccid) { in cpu_surrender()
1812 uint_t t_pri = DISP_PRIO(tp); in cpu_surrender()
1828 DTRACE_SCHED1(surrender, kthread_t *, tp); in cpu_surrender()
1835 if (tp != curthread || (lwp = tp->t_lwp) == NULL || in cpu_surrender()
1837 aston(tp); in cpu_surrender()
1842 "cpu_surrender:tid %p cpu %p", tp, cpup); in cpu_surrender()
1850 disp_ratify(kthread_t *tp, disp_t *kpq) in disp_ratify() argument
1856 ASSERT(tp != NULL); in disp_ratify()
1868 tpri = DISP_PRIO(tp); in disp_ratify()
1880 thread_lock_high(tp); in disp_ratify()
1881 THREAD_TRANSITION(tp); in disp_ratify()
1882 setfrontdq(tp); in disp_ratify()
1883 thread_unlock_nopreempt(tp); in disp_ratify()
1885 tp = NULL; in disp_ratify()
1887 return (tp); in disp_ratify()
1900 kthread_t *tp; in disp_getwork() local
1917 tp = (disp_getbest(kpq)); in disp_getwork()
1918 if (tp) in disp_getwork()
1919 return (disp_ratify(tp, kpq)); in disp_getwork()
2058 tp = disp_getbest(tcp->cpu_disp); in disp_getwork()
2059 if (tp == NULL || tp == T_DONTSTEAL) in disp_getwork()
2060 return (tp); in disp_getwork()
2061 return (disp_ratify(tp, kpq)); in disp_getwork()
2082 kthread_t *tp; in disp_fix_unbound_pri() local
2122 tp = dq->dq_first; in disp_fix_unbound_pri()
2124 while (tp && (tp->t_bound_cpu || tp->t_weakbound_cpu)) { in disp_fix_unbound_pri()
2125 tp = tp->t_link; in disp_fix_unbound_pri()
2131 } while (tp == NULL); in disp_fix_unbound_pri()
2146 disp_adjust_unbound_pri(kthread_t *tp) in disp_adjust_unbound_pri() argument
2151 ASSERT(THREAD_LOCK_HELD(tp)); in disp_adjust_unbound_pri()
2157 if (tp->t_bound_cpu == NULL || in disp_adjust_unbound_pri()
2158 tp->t_state != TS_RUN || in disp_adjust_unbound_pri()
2159 tp->t_schedflag & TS_ON_SWAPQ) in disp_adjust_unbound_pri()
2162 tpri = DISP_PRIO(tp); in disp_adjust_unbound_pri()
2163 dp = tp->t_bound_cpu->cpu_disp; in disp_adjust_unbound_pri()
2183 kthread_t *tp; in disp_getbest() local
2213 for (tp = dq->dq_first; tp != NULL; tp = tp->t_link) { in disp_getbest()
2220 if (tp->t_bound_cpu || tp->t_weakbound_cpu) in disp_getbest()
2260 tp->t_cpu != tcp) in disp_getbest()
2279 rqtime = now - tp->t_waitrq; in disp_getbest()
2293 DTRACE_PROBE4(nosteal, kthread_t *, tp, in disp_getbest()
2323 if (tp == NULL) { in disp_getbest()
2340 thread_was_on_queue = dispdeq(tp); /* drops disp_lock */ in disp_getbest()
2345 (void) dispdeq(tp); /* drops disp_lock */ in disp_getbest()
2354 tp->t_schedflag |= TS_DONT_SWAP; in disp_getbest()
2359 tp->t_disp_queue = cp->cpu_disp; in disp_getbest()
2361 cp->cpu_dispthread = tp; /* protected by spl only */ in disp_getbest()
2386 ASSERT(pri == DISP_PRIO(tp)); in disp_getbest()
2388 DTRACE_PROBE3(steal, kthread_t *, tp, cpu_t *, tcp, cpu_t *, cp); in disp_getbest()
2390 thread_onproc(tp, cp); /* set t_state to TS_ONPROC */ in disp_getbest()
2397 return (tp); in disp_getbest()
2411 kthread_t *tp; in disp_bound_common() local
2417 tp = curthread; /* faster than allthreads */ in disp_bound_common()
2419 if (tp->t_state != TS_FREE) { in disp_bound_common()
2425 if ((tp->t_flag & T_INTR_THREAD) && in disp_bound_common()
2433 if (tp == cp->cpu_idle_thread) in disp_bound_common()
2440 if (tp == cp->cpu_pause_thread) in disp_bound_common()
2444 (tp->t_bound_cpu == cp || in disp_bound_common()
2445 tp->t_bind_cpu == cp->cpu_id || in disp_bound_common()
2446 tp->t_weakbound_cpu == cp)) { in disp_bound_common()
2452 (tp->t_cpupart == cp->cpu_part)) { in disp_bound_common()
2457 } while ((tp = tp->t_next) != curthread && found == 0); in disp_bound_common()
2503 kthread_t *tp; in disp_cpu_inactive() local
2512 tp = dq->dq_first; in disp_cpu_inactive()
2517 while (tp != NULL && tp->t_bound_cpu != NULL) { in disp_cpu_inactive()
2518 tp = tp->t_link; in disp_cpu_inactive()
2521 if (tp == NULL) { in disp_cpu_inactive()
2527 wasonq = dispdeq(tp); /* drops disp_lock */ in disp_cpu_inactive()
2529 ASSERT(tp->t_weakbound_cpu == NULL); in disp_cpu_inactive()
2531 setbackdq(tp); in disp_cpu_inactive()
2545 ASSERT(tp->t_cpu != cp); in disp_cpu_inactive()
2546 thread_unlock(tp); in disp_cpu_inactive()
2567 cpu_score(cpu_t *cp, kthread_t *tp) in cpu_score() argument
2571 if (tp == curthread && cp == curthread->t_cpu) in cpu_score()
2583 return (smt_adjust_cpu_score(tp, cp, score)); in cpu_score()
2608 disp_lowpri_cpu(cpu_t *hint, kthread_t *tp, pri_t tpri) in disp_lowpri_cpu() argument
2619 ASSERT(tp->t_lpl->lpl_ncpu > 0); in disp_lowpri_cpu()
2624 lpl_iter = tp->t_lpl; in disp_lowpri_cpu()
2645 pri_t score = cpu_score(cp, tp); in disp_lowpri_cpu()