Lines Matching refs:pg
134 #define IS_CMT_PG(pg) (((pg_t *)(pg))->pg_class->pgc_id == pg_cmt_class_id) argument
228 pg_cmt_free(pg_t *pg) in pg_cmt_free() argument
230 ASSERT(pg != NULL); in pg_cmt_free()
231 ASSERT(IS_CMT_PG(pg)); in pg_cmt_free()
233 kmem_free((pg_cmt_t *)pg, sizeof (pg_cmt_t)); in pg_cmt_free()
300 cmt_callback_init(pg_t *pg) in cmt_callback_init() argument
306 if (((pg_cmt_t *)pg)->cmt_policy == CMT_NO_POLICY) in cmt_callback_init()
309 switch (((pghw_t *)pg)->pghw_hw) { in cmt_callback_init()
311 pg->pg_cb.thread_swtch = cmt_ev_thread_swtch_pwr; in cmt_callback_init()
312 pg->pg_cb.thread_remain = cmt_ev_thread_remain_pwr; in cmt_callback_init()
315 pg->pg_cb.thread_swtch = cmt_ev_thread_swtch; in cmt_callback_init()
331 cmt_hier_promote(pg_cmt_t *pg, cpu_pg_t *pgdata) in cmt_hier_promote() argument
344 parent = pg->cmt_parent; in cmt_hier_promote()
352 ASSERT(PG_NUM_CPUS((pg_t *)pg) >= PG_NUM_CPUS((pg_t *)parent)); in cmt_hier_promote()
367 r = group_add(parent->cmt_siblings, pg, GRP_NORESIZE); in cmt_hier_promote()
380 r = group_add(&cmt_root->cl_pgs, pg, GRP_NORESIZE); in cmt_hier_promote()
393 if (group_remove(parent->cmt_children, pg, GRP_NORESIZE) != -1) { in cmt_hier_promote()
398 children = pg->cmt_children; in cmt_hier_promote()
399 pg->cmt_children = parent->cmt_children; in cmt_hier_promote()
402 nchildren = pg->cmt_nchildren; in cmt_hier_promote()
403 pg->cmt_nchildren = parent->cmt_nchildren; in cmt_hier_promote()
409 pg->cmt_siblings = parent->cmt_siblings; in cmt_hier_promote()
410 parent->cmt_siblings = pg->cmt_children; in cmt_hier_promote()
415 PG_CPU_ITR_INIT(pg, cpu_iter); in cmt_hier_promote()
437 if (cpu_pg->cmt_parent == pg) { in cmt_hier_promote()
445 if ((idx = group_find(&pgd->cmt_pgs, (void *)pg)) == -1) { in cmt_hier_promote()
466 err = group_add_at(&pgd->cmt_pgs, pg, idx - 1); in cmt_hier_promote()
481 pg->cmt_parent = parent->cmt_parent; in cmt_hier_promote()
482 parent->cmt_parent = pg; in cmt_hier_promote()
502 pg_cmt_t *pg; in pg_cmt_cpu_init() local
558 pg = (pg_cmt_t *)pghw_place_cpu(cp, hw); in pg_cmt_cpu_init()
559 if (pg == NULL) { in pg_cmt_cpu_init()
564 pg = (pg_cmt_t *)pg_create(pg_cmt_class_id); in pg_cmt_cpu_init()
567 pghw_init((pghw_t *)pg, cp, hw); in pg_cmt_cpu_init()
573 pg->cmt_policy = policy; in pg_cmt_cpu_init()
576 cmt_callback_init((pg_t *)pg); in pg_cmt_cpu_init()
578 bitset_init(&pg->cmt_cpus_actv_set); in pg_cmt_cpu_init()
579 group_create(&pg->cmt_cpus_actv); in pg_cmt_cpu_init()
581 ASSERT(IS_CMT_PG(pg)); in pg_cmt_cpu_init()
584 ((pghw_t *)pg)->pghw_generation++; in pg_cmt_cpu_init()
587 pg_cpu_add((pg_t *)pg, cp, pgdata); in pg_cmt_cpu_init()
592 group_expand(&pg->cmt_cpus_actv, in pg_cmt_cpu_init()
593 GROUP_SIZE(&((pg_t *)pg)->pg_cpus)); in pg_cmt_cpu_init()
596 bitset_capacity(&pg->cmt_cpus_actv_set)) { in pg_cmt_cpu_init()
597 bitset_resize(&pg->cmt_cpus_actv_set, in pg_cmt_cpu_init()
605 cpu_cmt_hier[levels++] = pg; in pg_cmt_cpu_init()
610 pg_cache = (pg_t *)pg; in pg_cmt_cpu_init()
661 pg = cpu_cmt_hier[level]; in pg_cmt_cpu_init()
666 while (pg->cmt_parent && in pg_cmt_cpu_init()
667 pg->cmt_parent != cpu_cmt_hier[level + 1]) { in pg_cmt_cpu_init()
668 cmt_hier_promote(pg, pgdata); in pg_cmt_cpu_init()
688 pg = cpu_cmt_hier[level]; in pg_cmt_cpu_init()
689 err = group_add_at(cmt_pgs, pg, levels - level - 1); in pg_cmt_cpu_init()
693 pgdata->cmt_lineage = (pg_t *)pg; in pg_cmt_cpu_init()
695 if (pg->cmt_siblings != NULL) { in pg_cmt_cpu_init()
697 ASSERT(pg->cmt_parent == NULL || in pg_cmt_cpu_init()
698 pg->cmt_parent == cpu_cmt_hier[level + 1]); in pg_cmt_cpu_init()
699 ASSERT(pg->cmt_siblings == &lgrp->cl_pgs || in pg_cmt_cpu_init()
700 ((pg->cmt_parent != NULL) && in pg_cmt_cpu_init()
701 pg->cmt_siblings == pg->cmt_parent->cmt_children)); in pg_cmt_cpu_init()
706 pg->cmt_parent = NULL; in pg_cmt_cpu_init()
708 pg->cmt_siblings = &lgrp->cl_pgs; in pg_cmt_cpu_init()
713 pg->cmt_parent = cpu_cmt_hier[level + 1]; in pg_cmt_cpu_init()
720 if (pg->cmt_parent->cmt_children == NULL) { in pg_cmt_cpu_init()
721 pg->cmt_parent->cmt_children = in pg_cmt_cpu_init()
723 group_create(pg->cmt_parent->cmt_children); in pg_cmt_cpu_init()
725 pg->cmt_siblings = pg->cmt_parent->cmt_children; in pg_cmt_cpu_init()
726 children = ++pg->cmt_parent->cmt_nchildren; in pg_cmt_cpu_init()
729 group_expand(pg->cmt_siblings, children); in pg_cmt_cpu_init()
772 pg_cmt_t *pg; in pg_cmt_cpu_fini() local
818 pg = (pg_cmt_t *)pgdata->cmt_lineage; in pg_cmt_cpu_fini()
819 while (pg != NULL) { in pg_cmt_cpu_fini()
821 ((pghw_t *)pg)->pghw_generation++; in pg_cmt_cpu_fini()
826 (void) group_remove(cmt_pgs, pg, GRP_RESIZE); in pg_cmt_cpu_fini()
834 if (GROUP_SIZE(&((pg_t *)pg)->pg_cpus) == 1) { in pg_cmt_cpu_fini()
835 if (pg->cmt_children != NULL) in pg_cmt_cpu_fini()
836 group_destroy(pg->cmt_children); in pg_cmt_cpu_fini()
837 if (pg->cmt_siblings != NULL) { in pg_cmt_cpu_fini()
838 if (pg->cmt_siblings == &lgrp->cl_pgs) in pg_cmt_cpu_fini()
841 pg->cmt_parent->cmt_nchildren--; in pg_cmt_cpu_fini()
844 pg = pg->cmt_parent; in pg_cmt_cpu_fini()
854 while ((pg = group_iterate(pgs, &i)) != NULL) { in pg_cmt_cpu_fini()
855 if (IS_CMT_PG(pg) == 0) in pg_cmt_cpu_fini()
858 pg_cpu_delete((pg_t *)pg, cp, pgdata); in pg_cmt_cpu_fini()
866 if (GROUP_SIZE(&((pg_t *)pg)->pg_cpus) == 0) { in pg_cmt_cpu_fini()
871 group_destroy(&pg->cmt_cpus_actv); in pg_cmt_cpu_fini()
872 bitset_fini(&pg->cmt_cpus_actv_set); in pg_cmt_cpu_fini()
873 pghw_fini((pghw_t *)pg); in pg_cmt_cpu_fini()
875 pg_destroy((pg_t *)pg); in pg_cmt_cpu_fini()
887 pg_t *pg; in pg_cmt_cpupart_in() local
903 while ((pg = group_iterate(pgs, &i)) != NULL) { in pg_cmt_cpupart_in()
904 if (IS_CMT_PG(pg) == 0) in pg_cmt_cpupart_in()
907 if (bitset_capacity(&pp->cp_cmt_pgs) <= pg->pg_id) in pg_cmt_cpupart_in()
908 bitset_resize(&pp->cp_cmt_pgs, pg->pg_id + 1); in pg_cmt_cpupart_in()
920 pg_t *pg; in pg_cmt_cpupart_move() local
936 while ((pg = group_iterate(pgs, &pg_iter)) != NULL) { in pg_cmt_cpupart_move()
938 if (IS_CMT_PG(pg) == 0) in pg_cmt_cpupart_move()
944 bitset_add(&newpp->cp_cmt_pgs, pg->pg_id); in pg_cmt_cpupart_move()
951 PG_CPU_ITR_INIT(pg, cpu_iter); in pg_cmt_cpupart_move()
962 bitset_del(&cp->cpu_part->cp_cmt_pgs, pg->pg_id); in pg_cmt_cpupart_move()
976 pg_cmt_t *pg; in pg_cmt_cpu_active() local
990 while ((pg = group_iterate(pgs, &i)) != NULL) { in pg_cmt_cpu_active()
992 if (IS_CMT_PG(pg) == 0) in pg_cmt_cpu_active()
998 ((pghw_t *)pg)->pghw_generation++; in pg_cmt_cpu_active()
1000 err = group_add(&pg->cmt_cpus_actv, cp, GRP_NORESIZE); in pg_cmt_cpu_active()
1009 if (GROUP_SIZE(&pg->cmt_cpus_actv) == 1 && in pg_cmt_cpu_active()
1010 (pg->cmt_policy & (CMT_BALANCE | CMT_COALESCE))) { in pg_cmt_cpu_active()
1011 err = group_add(pg->cmt_siblings, pg, GRP_NORESIZE); in pg_cmt_cpu_active()
1018 if (pg->cmt_parent == NULL && in pg_cmt_cpu_active()
1019 pg->cmt_siblings != &cmt_root->cl_pgs) { in pg_cmt_cpu_active()
1020 err = group_add(&cmt_root->cl_pgs, pg, in pg_cmt_cpu_active()
1031 bitset_add(&pg->cmt_cpus_actv_set, cp->cpu_seqid); in pg_cmt_cpu_active()
1032 bitset_add(&cp->cpu_part->cp_cmt_pgs, ((pg_t *)pg)->pg_id); in pg_cmt_cpu_active()
1046 pg_cmt_t *pg; in pg_cmt_cpu_inactive() local
1060 while ((pg = group_iterate(pgs, &i)) != NULL) { in pg_cmt_cpu_inactive()
1062 if (IS_CMT_PG(pg) == 0) in pg_cmt_cpu_inactive()
1068 ((pghw_t *)pg)->pghw_generation++; in pg_cmt_cpu_inactive()
1074 err = group_remove(&pg->cmt_cpus_actv, cp, GRP_NORESIZE); in pg_cmt_cpu_inactive()
1077 bitset_del(&pg->cmt_cpus_actv_set, cp->cpu_seqid); in pg_cmt_cpu_inactive()
1083 if (GROUP_SIZE(&pg->cmt_cpus_actv) == 0 && in pg_cmt_cpu_inactive()
1084 (pg->cmt_policy & (CMT_BALANCE | CMT_COALESCE))) { in pg_cmt_cpu_inactive()
1085 err = group_remove(pg->cmt_siblings, pg, GRP_NORESIZE); in pg_cmt_cpu_inactive()
1088 if (pg->cmt_parent == NULL && in pg_cmt_cpu_inactive()
1089 pg->cmt_siblings != &cmt_root->cl_pgs) { in pg_cmt_cpu_inactive()
1090 err = group_remove(&cmt_root->cl_pgs, pg, in pg_cmt_cpu_inactive()
1100 ASSERT(GROUP_SIZE(&pg->cmt_cpus_actv) <= in pg_cmt_cpu_inactive()
1101 GROUP_SIZE(&((pg_t *)pg)->pg_cpus)); in pg_cmt_cpu_inactive()
1107 PG_CPU_ITR_INIT(pg, cpu_itr); in pg_cmt_cpu_inactive()
1119 ((pg_t *)pg)->pg_id); in pg_cmt_cpu_inactive()
1128 pg_cmt_cpu_belongs(pg_t *pg, cpu_t *cp) in pg_cmt_cpu_belongs() argument
1132 pg_cpu = GROUP_ACCESS(&pg->pg_cpus, 0); in pg_cmt_cpu_belongs()
1141 if (pg_plat_cpus_share(cp, pg_cpu, ((pghw_t *)pg)->pghw_hw)) in pg_cmt_cpu_belongs()
1261 pg_cmt_t *pg; in cmt_pad_enable() local
1290 while ((pg = group_iterate(hwset, &iter)) != NULL) { in cmt_pad_enable()
1296 pg->cmt_policy = pg_cmt_policy(((pghw_t *)pg)->pghw_hw); in cmt_pad_enable()
1297 while ((pg->cmt_parent != NULL) && in cmt_pad_enable()
1298 (pg->cmt_parent->cmt_policy != pg->cmt_policy) && in cmt_pad_enable()
1299 (PG_NUM_CPUS((pg_t *)pg) == in cmt_pad_enable()
1300 PG_NUM_CPUS((pg_t *)pg->cmt_parent))) { in cmt_pad_enable()
1301 cmt_hier_promote(pg, NULL); in cmt_pad_enable()
1313 pg_cmt_t *pg; in cmt_pad_disable() local
1334 while ((pg = group_iterate(hwset, &iter)) != NULL) { in cmt_pad_disable()
1341 if (pg->cmt_children != NULL && in cmt_pad_disable()
1342 GROUP_SIZE(pg->cmt_children) == 1) { in cmt_pad_disable()
1343 child = GROUP_ACCESS(pg->cmt_children, 0); in cmt_pad_disable()
1348 pg->cmt_policy = CMT_BALANCE; in cmt_pad_disable()
1355 cmt_ev_thread_swtch(pg_t *pg, cpu_t *cp, hrtime_t now, kthread_t *old, in cmt_ev_thread_swtch() argument
1358 pg_cmt_t *cmt_pg = (pg_cmt_t *)pg; in cmt_ev_thread_swtch()
1370 #define THREAD_RUNNABLE_IN_PG(t, pg) \ argument
1373 bitset_in_set(&(pg)->cmt_cpus_actv_set, \
1377 cmt_ev_thread_swtch_pwr(pg_t *pg, cpu_t *cp, hrtime_t now, kthread_t *old, in cmt_ev_thread_swtch_pwr() argument
1380 pg_cmt_t *cmt = (pg_cmt_t *)pg; in cmt_ev_thread_swtch_pwr()
1418 cmt_ev_thread_remain_pwr(pg_t *pg, cpu_t *cp, kthread_t *t) in cmt_ev_thread_remain_pwr() argument
1420 pg_cmt_t *cmt = (pg_cmt_t *)pg; in cmt_ev_thread_remain_pwr()
1432 pg_cmt_policy_name(pg_t *pg) in pg_cmt_policy_name() argument
1436 policy = ((pg_cmt_t *)pg)->cmt_policy; in pg_cmt_policy_name()
1471 pg_cmt_t *pg, *child; in pg_cmt_prune() local
1528 while ((pg = group_iterate(hwset, &hw_iter)) != NULL) { in pg_cmt_prune()
1534 if (pg->cmt_children && in pg_cmt_prune()
1535 GROUP_SIZE(pg->cmt_children) > 1) { in pg_cmt_prune()
1536 cap_needed = GROUP_SIZE(pg->cmt_children) - 1; in pg_cmt_prune()
1538 group_expand(pg->cmt_siblings, in pg_cmt_prune()
1539 GROUP_SIZE(pg->cmt_siblings) + cap_needed); in pg_cmt_prune()
1545 if (pg->cmt_parent == NULL && in pg_cmt_prune()
1546 pg->cmt_siblings != &cmt_root->cl_pgs) { in pg_cmt_prune()
1565 while ((pg = group_iterate(hwset, &hw_iter)) != NULL) { in pg_cmt_prune()
1570 if (pg->cmt_siblings) { in pg_cmt_prune()
1571 (void) group_remove(pg->cmt_siblings, pg, GRP_NORESIZE); in pg_cmt_prune()
1573 if (pg->cmt_parent == NULL && in pg_cmt_prune()
1574 pg->cmt_siblings != &cmt_root->cl_pgs) { in pg_cmt_prune()
1575 (void) group_remove(&cmt_root->cl_pgs, pg, in pg_cmt_prune()
1583 pg->cmt_policy = CMT_NO_POLICY; in pg_cmt_prune()
1595 if (pg->cmt_children != NULL) { in pg_cmt_prune()
1596 children = pg->cmt_children; in pg_cmt_prune()
1601 if (pg->cmt_siblings != NULL) { in pg_cmt_prune()
1602 r = group_add(pg->cmt_siblings, child, in pg_cmt_prune()
1606 if (pg->cmt_parent == NULL && in pg_cmt_prune()
1607 pg->cmt_siblings != in pg_cmt_prune()
1615 group_empty(pg->cmt_children); in pg_cmt_prune()
1621 pg_callback_set_defaults((pg_t *)pg); in pg_cmt_prune()
1626 PG_CPU_ITR_INIT(pg, cpu_iter); in pg_cmt_prune()
1649 if (cpu_pg->cmt_parent == pg) { in pg_cmt_prune()
1650 cpu_pg->cmt_parent = pg->cmt_parent; in pg_cmt_prune()
1651 cpu_pg->cmt_siblings = pg->cmt_siblings; in pg_cmt_prune()
1661 (void) group_remove(&cpd->cmt_pgs, pg, GRP_NORESIZE); in pg_cmt_prune()
1744 pg_cmt_t *pg, *pg_next, *pg_bad, *pg_tmp, *parent; in pg_cmt_lineage_validate() local
1750 pg = NULL; in pg_cmt_lineage_validate()
1759 pg = lineage[i]; in pg_cmt_lineage_validate()
1770 (PG_NUM_CPUS((pg_t *)pg) <= PG_NUM_CPUS((pg_t *)pg_next))); in pg_cmt_lineage_validate()
1794 parent = pg->cmt_parent; in pg_cmt_lineage_validate()
1819 PG_NUM_CPUS((pg_t *)pg)) { in pg_cmt_lineage_validate()
1834 PG_CPU_ITR_INIT((pg_t *)pg, cpu_iter); in pg_cmt_lineage_validate()
1889 if (((pghw_t *)pg)->pghw_hw == PGHW_CHIP) { in pg_cmt_lineage_validate()
1890 if (pg_cmt_prune(pg, lineage, sz, pgdata) == 0) { in pg_cmt_lineage_validate()
1902 if (PG_CMT_HW_SUSPECT(((pghw_t *)pg)->pghw_hw)) { in pg_cmt_lineage_validate()
1903 if (pg_cmt_prune(pg, lineage, sz, pgdata) == 0) { in pg_cmt_lineage_validate()
1934 if ((PG_NUM_CPUS((pg_t *)pg) < PG_NUM_CPUS((pg_t *)pg_next)) && in pg_cmt_lineage_validate()
1935 (PG_CMT_HW_SUSPECT(((pghw_t *)pg)->pghw_hw))) { in pg_cmt_lineage_validate()
1936 pg_bad = pg; in pg_cmt_lineage_validate()
1937 } else if (PG_NUM_CPUS((pg_t *)pg) == in pg_cmt_lineage_validate()
1941 } else if (PG_CMT_HW_SUSPECT(((pghw_t *)pg)->pghw_hw)) { in pg_cmt_lineage_validate()
1942 pg_bad = pg; in pg_cmt_lineage_validate()