1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2008-2009, Intel Corporation.
23 * All Rights Reserved.
24 */
25
26#include <stdlib.h>
27#include <stdio.h>
28#include <memory.h>
29#include <string.h>
30#include <limits.h>
31#include <sys/stat.h>
32
33#include "latencytop.h"
34
35/* Statistics for each process/thread. */
36typedef struct _lt_stat_collection lt_stat_collection_t;
37typedef gboolean (*check_child_func_t) (gpointer key,
38    lt_stat_collection_t *stat, void *user);
39
40typedef struct {
41	lt_stat_entry_t lt_grp_summary;
42	/* cause_id -> stat entry */
43	GHashTable *lt_grp_cidlist;
44} lt_datagroup_t;
45
46#define	NGROUPS			2
47#define	GROUP_CAUSE		0
48#define	GROUP_SOBJ		1
49
50/*
51 * A data collection hierarchy involving three entities - system, process
52 * and thread. The hierarchic relationship is as follows :
53 *
54 *		1 system -> 1 or more processes -> 1 or more threads
55 */
56struct _lt_stat_collection {
57	lt_stat_level_t lt_sc_level;
58	unsigned int lt_sc_id;
59	char *lt_sc_name;
60	lt_datagroup_t lt_sc_groups[NGROUPS];
61	/*
62	 * The following fields: lt_sc_parent, lt_sc_children and
63	 * lt_sc_check_child_func maintain the tree structure.
64	 */
65	lt_stat_collection_t *lt_sc_parent;		/* Parent node */
66	GHashTable *lt_sc_children;	/* pid/tid -> lt_stat_collection_t */
67	check_child_func_t lt_sc_check_child_func; /* Release dead children */
68};
69
70/* Internal data structure to back up a stat_list */
71typedef struct _lt_stat_list lt_stat_list_t;
72typedef void (*free_list_func_t)(lt_stat_list_t *);
73struct _lt_stat_list {
74	int lt_sl_entry_count;
75	lt_stat_entry_t **lt_sl_entries;
76	uint64_t lt_sl_gtotal;
77	free_list_func_t lt_sl_free_func;
78};
79
80/* Root of the collection hierarchy: system level statistics */
81static lt_stat_collection_t *stat_system = NULL;
82
83/*
84 * Data structure to hold synchronization objects.
85 * We don't use normal "cause table" because this needs to be cleared
86 * every time we refresh in order to make sure that stale synchronization
87 * objects don't consume memory.
88 */
89typedef struct {
90	int lt_soi_type;
91	unsigned long long lt_soi_addr;
92} lt_sobj_id_t;
93
94typedef struct {
95	lt_sobj_id_t lt_so_oid;
96	int lt_so_cause_id;
97	char lt_so_string[32];	/* Enough to hold "%s: 0x%llX" */
98} lt_sobj_t;
99
100static GHashTable *sobj_table = NULL;
101static int sobj_table_len = 0;
102
103/*
104 * Lower 32-bit of the address of synchronization objects is used to hash
105 * them.
106 */
107static guint
108sobj_id_hash(lt_sobj_id_t *id)
109{
110	g_assert(id != NULL);
111	return (id->lt_soi_addr & 0xFFFFFFFF);
112}
113
114/*
115 * Test if two synchronization objects are the same.
116 */
117static gboolean
118sobj_id_equal(lt_sobj_id_t *a, lt_sobj_id_t *b)
119{
120	g_assert(a != NULL && b != NULL);
121	return (a->lt_soi_type == b->lt_soi_type &&
122	    a->lt_soi_addr == b->lt_soi_addr);
123}
124
125/*
126 * Look up the cause_id of a synchronization object.
127 * Note that this cause_id is only unique in GROUP_SOBJ, and changes after
128 * a refresh.
129 */
130static lt_sobj_t *
131lookup_sobj(lt_sobj_id_t *id)
132{
133	const char *stype_str[] = {
134		"None",
135		"Mutex",
136		"RWLock",
137		"CV",
138		"Sema",
139		"User",
140		"User_PI",
141		"Shuttle"
142	};
143	const int stype_str_len =
144	    sizeof (stype_str) / sizeof (stype_str[0]);
145	lt_sobj_t *ret = NULL;
146	g_assert(id != NULL);
147
148	if (id->lt_soi_type < 0 || id->lt_soi_type >= stype_str_len) {
149		return (NULL);
150	}
151
152	if (sobj_table != NULL) {
153		ret = (lt_sobj_t *)g_hash_table_lookup(sobj_table, id);
154	} else {
155		sobj_table = g_hash_table_new_full(
156		    (GHashFunc)sobj_id_hash, (GEqualFunc)sobj_id_equal,
157		    NULL, (GDestroyNotify)free);
158		lt_check_null(sobj_table);
159	}
160
161	if (ret == NULL) {
162		ret = (lt_sobj_t *)lt_zalloc(sizeof (lt_sobj_t));
163		ret->lt_so_cause_id = ++sobj_table_len;
164		(void) snprintf(ret->lt_so_string, sizeof (ret->lt_so_string),
165		    "%s: 0x%llX", stype_str[id->lt_soi_type], id->lt_soi_addr);
166		ret->lt_so_oid.lt_soi_type = id->lt_soi_type;
167		ret->lt_so_oid.lt_soi_addr = id->lt_soi_addr;
168
169		g_hash_table_insert(sobj_table, &ret->lt_so_oid, ret);
170	}
171
172	return (ret);
173}
174
175/*
176 * Check if a process exists by using /proc/pid
177 */
178/* ARGSUSED */
179static gboolean
180check_process(gpointer key, lt_stat_collection_t *stat, void *user)
181{
182	char name[PATH_MAX];
183
184	(void) snprintf(name, PATH_MAX, "/proc/%u", stat->lt_sc_id);
185	return (lt_file_exist(name) ? FALSE : TRUE);
186}
187
188/*
189 * Check if a thread exists by using /proc/pid/lwp/tid
190 */
191/* ARGSUSED */
192static gboolean
193check_thread(gpointer key, lt_stat_collection_t *stat, void *user)
194{
195	char name[PATH_MAX];
196
197	g_assert(stat->lt_sc_parent != NULL);
198	g_assert(stat->lt_sc_parent->lt_sc_level == LT_LEVEL_PROCESS);
199
200	(void) snprintf(name, PATH_MAX, "/proc/%u/lwp/%u",
201	    stat->lt_sc_parent->lt_sc_id, stat->lt_sc_id);
202	return (lt_file_exist(name) ? FALSE : TRUE);
203}
204
205/*
206 * Helper function to free a stat node.
207 */
208static void
209free_stat(lt_stat_collection_t *stat)
210{
211	int i;
212
213	if (stat == NULL) {
214		return;
215	}
216
217	for (i = 0; i < NGROUPS; ++i) {
218		if (stat->lt_sc_groups[i].lt_grp_cidlist != NULL) {
219			g_hash_table_destroy(stat->lt_sc_groups[i].
220			    lt_grp_cidlist);
221		}
222	}
223
224	if (stat->lt_sc_children != NULL) {
225		g_hash_table_destroy(stat->lt_sc_children);
226	}
227
228	if (stat->lt_sc_name != NULL) {
229		free(stat->lt_sc_name);
230	}
231
232	free(stat);
233}
234
235/*
236 * Helper function to initialize a stat node.
237 */
238/* ARGSUSED */
239static void
240clear_stat(gpointer key, lt_stat_collection_t *stat, void *user)
241{
242	int i;
243
244	g_assert(stat != NULL);
245
246	for (i = 0; i < NGROUPS; ++i) {
247		if (stat->lt_sc_groups[i].lt_grp_cidlist != NULL) {
248			g_hash_table_destroy(stat->lt_sc_groups[i].
249			    lt_grp_cidlist);
250			stat->lt_sc_groups[i].lt_grp_cidlist = NULL;
251		}
252
253		stat->lt_sc_groups[i].lt_grp_summary.lt_se_data.lt_s_count = 0;
254		stat->lt_sc_groups[i].lt_grp_summary.lt_se_data.lt_s_total = 0;
255		stat->lt_sc_groups[i].lt_grp_summary.lt_se_data.lt_s_max = 0;
256	}
257
258	if (stat->lt_sc_children != NULL) {
259		g_hash_table_foreach_remove(stat->lt_sc_children,
260		    (GHRFunc)stat->lt_sc_check_child_func, NULL);
261		g_hash_table_foreach(stat->lt_sc_children,
262		    (GHFunc)clear_stat, NULL);
263	}
264}
265
266/*
267 * Update a collection with the given value.
268 * Recursively update parents in the hierarchy  until the root is reached.
269 */
270static void
271update_stat_entry(lt_stat_collection_t *stat, int cause_id,
272		lt_stat_type_t type, uint64_t value,
273		const char *string, int group_to_use)
274{
275	lt_stat_entry_t *entry = NULL;
276	lt_datagroup_t *group;
277
278	if (group_to_use < 0 || group_to_use >= NGROUPS) {
279		return;
280	}
281
282	group = &(stat->lt_sc_groups[group_to_use]);
283
284	if (group->lt_grp_cidlist != NULL) {
285		entry = (lt_stat_entry_t *)g_hash_table_lookup(
286		    group->lt_grp_cidlist, LT_INT_TO_POINTER(cause_id));
287	} else   {
288		group->lt_grp_cidlist = g_hash_table_new_full(
289		    g_direct_hash, g_direct_equal,
290		    NULL, (GDestroyNotify)free);
291		lt_check_null(group->lt_grp_cidlist);
292	}
293
294	if (entry == NULL) {
295		entry = (lt_stat_entry_t *)lt_zalloc(sizeof (lt_stat_entry_t));
296		entry->lt_se_string = string;
297
298		switch (group_to_use) {
299		case GROUP_CAUSE:
300			entry->lt_se_type = STAT_CAUSE;
301			entry->lt_se_tsdata.lt_se_t_cause.lt_se_c_id = cause_id;
302			entry->lt_se_tsdata.lt_se_t_cause.lt_se_c_flags =
303			    lt_table_get_cause_flag(cause_id, CAUSE_ALL_FLAGS);
304
305			/* hide the first '#' */
306			if ((entry->lt_se_tsdata.lt_se_t_cause.lt_se_c_flags
307			    & CAUSE_FLAG_HIDE_IN_SUMMARY) != 0) {
308				++entry->lt_se_string;
309			}
310
311			break;
312		case GROUP_SOBJ:
313			entry->lt_se_type = STAT_SOBJ;
314			entry->lt_se_tsdata.lt_se_t_sobj.lt_se_s_id = cause_id;
315			break;
316		}
317
318		g_hash_table_insert(group->lt_grp_cidlist,
319		    LT_INT_TO_POINTER(cause_id), entry);
320	}
321
322	lt_update_stat_value(&entry->lt_se_data, type, value);
323
324	if (group_to_use == GROUP_SOBJ ||
325	    (entry->lt_se_tsdata.lt_se_t_cause.lt_se_c_flags &
326	    CAUSE_FLAG_HIDE_IN_SUMMARY) == 0) {
327		lt_update_stat_value(&group->lt_grp_summary.lt_se_data, type,
328		    value);
329	}
330
331	if (stat->lt_sc_parent != NULL) {
332		update_stat_entry(stat->lt_sc_parent, cause_id, type, value,
333		    string, group_to_use);
334	}
335}
336
337/*
338 * Identify the cause of latency from the given stack trace.
339 * Return cause_id.
340 */
341static void
342find_cause(char *stack, int *cause_id, int *cause_priority)
343{
344	int cause_temp;
345	int prio_temp;
346	int cause = INVALID_CAUSE;
347	int priority = 0;
348	int found = 0;
349
350	g_assert(cause_id != NULL);
351	g_assert(cause_priority != NULL);
352
353	while (stack != NULL) {
354		char *sep;
355		sep = strchr(stack, ' ');
356
357		if (sep != NULL) {
358			*sep = '\0';
359		}
360
361		found = lt_table_cause_from_stack(stack, &cause_temp,
362		    &prio_temp);
363
364		if (found && (cause == INVALID_CAUSE ||
365		    HIGHER_PRIORITY(prio_temp, priority))) {
366			cause = cause_temp;
367			priority = prio_temp;
368		}
369
370		if (sep != NULL) {
371			*sep = ' ';
372			stack = sep + 1;
373		} else   {
374			stack = NULL;
375		}
376	}
377
378	*cause_id = cause;
379	*cause_priority = priority;
380}
381
382/*
383 * Create a new collection and hook it to the parent.
384 */
385static lt_stat_collection_t *
386new_collection(lt_stat_level_t level, unsigned int id, char *name,
387    lt_stat_collection_t *parent, check_child_func_t check_child_func)
388{
389	int i;
390	lt_stat_collection_t *ret;
391
392	ret = (lt_stat_collection_t *)
393	    lt_zalloc(sizeof (lt_stat_collection_t));
394
395	ret->lt_sc_level = level;
396	ret->lt_sc_check_child_func = check_child_func;
397	ret->lt_sc_id = id;
398	ret->lt_sc_name = name;
399
400	for (i = 0; i < NGROUPS; ++i) {
401		ret->lt_sc_groups[i].lt_grp_summary.lt_se_string =
402		    (const char *)name;
403	}
404
405	if (parent != NULL) {
406		ret->lt_sc_parent = parent;
407
408		if (parent->lt_sc_children == NULL) {
409			parent->lt_sc_children = g_hash_table_new_full(
410			    g_direct_hash, g_direct_equal,
411			    NULL, (GDestroyNotify)free_stat);
412			lt_check_null(parent->lt_sc_children);
413		}
414
415		g_hash_table_insert(parent->lt_sc_children,
416		    LT_INT_TO_POINTER((int)id), ret);
417	}
418
419	return (ret);
420}
421
422/*
423 * Find the "leaf" in the collection hierarchy, using the given pid and tid.
424 */
425static lt_stat_collection_t *
426get_stat_c(pid_t pid, id_t tid)
427{
428	lt_stat_collection_t *stat_p = NULL;
429	lt_stat_collection_t *stat_t = NULL;
430
431	if (stat_system == NULL) {
432		stat_system = new_collection(LT_LEVEL_GLOBAL,
433		    PID_SYS_GLOBAL, lt_strdup("SYSTEM"), NULL, check_process);
434	} else if (stat_system->lt_sc_children != NULL) {
435		stat_p = (lt_stat_collection_t *)
436		    g_hash_table_lookup(stat_system->lt_sc_children,
437		    LT_INT_TO_POINTER(pid));
438	}
439
440	if (stat_p == NULL) {
441		char *fname;
442		fname = lt_get_proc_field(pid, LT_FIELD_FNAME);
443
444		if (fname == NULL) {
445			/*
446			 * we could not get the executable name of the
447			 * process; the process is probably already dead.
448			 */
449			return (NULL);
450		}
451
452		stat_p = new_collection(LT_LEVEL_PROCESS,
453		    (unsigned int)pid, fname, stat_system, check_thread);
454	} else if (stat_p->lt_sc_children != NULL) {
455		stat_t = (lt_stat_collection_t *)
456		    g_hash_table_lookup(stat_p->lt_sc_children,
457		    LT_INT_TO_POINTER(tid));
458	}
459
460	if (stat_t == NULL) {
461		const int tname_size = 16; /* Enough for "Thread %d" */
462		char *tname;
463
464		tname = (char *)lt_zalloc(tname_size);
465		(void) snprintf(tname, tname_size, "Thread %d", tid);
466
467		stat_t = new_collection(LT_LEVEL_THREAD,
468		    (unsigned int)tid, tname, stat_p, NULL);
469	}
470
471	return (stat_t);
472}
473
474/*
475 * Update statistics with the given cause_id. Values will be added to
476 * internal statistics.
477 */
478void
479lt_stat_update_cause(pid_t pid, id_t tid, int cause_id, lt_stat_type_t type,
480    uint64_t value)
481{
482	const char *string;
483	lt_stat_collection_t *stat_t = NULL;
484
485	if (cause_id < 0 || value == 0) {
486		return;
487	}
488
489	if (lt_table_get_cause_flag(cause_id, CAUSE_FLAG_DISABLED)) {
490		/* Ignore this cause */
491		return;
492	}
493
494	stat_t = get_stat_c(pid, tid);
495
496	if (stat_t == NULL) {
497		/* Process must be dead. */
498		return;
499	}
500
501	string = lt_table_get_cause_name(cause_id);
502
503	update_stat_entry(stat_t, cause_id, type, value, string, GROUP_CAUSE);
504}
505
506/*
507 * Update statistics with the given stack trace.
508 * The stack trace is mapped to a cause and lt_stat_update_cause() is called
509 * to update statistics.
510 */
511void
512lt_stat_update(pid_t pid, id_t tid, char *stack, char *tag,
513    unsigned int tag_priority, lt_stat_type_t type, uint64_t value)
514{
515	int tag_cause_id = INVALID_CAUSE;
516	int stack_cause_id = INVALID_CAUSE;
517	int cause_id = INVALID_CAUSE;
518	int stack_priority = 0;
519
520	if (value == 0) {
521		return;
522	}
523
524	find_cause(stack, &stack_cause_id, &stack_priority);
525
526	if (tag_priority != 0) {
527		tag_cause_id = lt_table_cause_from_name(tag, 0, 0);
528
529		if (tag_cause_id == INVALID_CAUSE) {
530			/* This must be a syscall tag */
531			char tmp[64];
532			(void) snprintf(tmp, sizeof (tmp), "Syscall: %s", tag);
533			tag_cause_id = lt_table_cause_from_name(tmp, 1, 0);
534		}
535	}
536
537	cause_id = (tag_priority > stack_priority) ? tag_cause_id :
538	    stack_cause_id;
539
540	if (cause_id == INVALID_CAUSE) {
541		/*
542		 * We got an unmapped stack. Set SPECIAL flag to display it
543		 * in pane 2. This makes it easier to find the cause.
544		 */
545		cause_id = lt_table_cause_from_name(stack, 1,
546		    CAUSE_FLAG_SPECIAL);
547		lt_klog_log(LT_KLOG_LEVEL_UNMAPPED, pid, stack, type, value);
548	} else   {
549		lt_klog_log(LT_KLOG_LEVEL_MAPPED, pid, stack, type, value);
550	}
551
552	lt_stat_update_cause(pid, tid, cause_id, type, value);
553}
554
555/*
556 * Zero out all statistics, but keep the data structures in memory
557 * to be used to hold new data immediately following.
558 */
559void
560lt_stat_clear_all(void)
561{
562	if (stat_system != NULL) {
563		clear_stat(NULL, stat_system, NULL);
564	}
565
566	if (sobj_table != NULL) {
567		g_hash_table_destroy(sobj_table);
568		sobj_table = NULL;
569	}
570}
571
572/*
573 * Clean up function that frees all memory used for statistics.
574 */
575void
576lt_stat_free_all(void)
577{
578	if (stat_system != NULL) {
579		free_stat(stat_system);
580		stat_system = NULL;
581	}
582
583	if (sobj_table != NULL) {
584		g_hash_table_destroy(sobj_table);
585		sobj_table = NULL;
586	}
587}
588
589/*
590 * Get top N causes of latency for a process. Return handle to a stat_list.
591 * Use pid = PID_SYS_GLOBAL to get global top list.
592 * Call lt_stat_list_free after use to clean up.
593 */
594void *
595lt_stat_list_create(lt_list_type_t list_type, lt_stat_level_t level,
596    pid_t pid, id_t tid, int count, lt_sort_t sort_by)
597{
598	GCompareFunc func;
599	GList *list, *walk;
600	lt_stat_collection_t *stat_c = NULL;
601	lt_stat_list_t *ret;
602	lt_datagroup_t *group;
603
604	if (level == LT_LEVEL_GLOBAL) {
605		/* Use global entry */
606		stat_c = stat_system;
607	} else if (stat_system != NULL && stat_system->lt_sc_children != NULL) {
608		/* Find process entry first */
609		stat_c = (lt_stat_collection_t *)g_hash_table_lookup(
610		    stat_system->lt_sc_children, LT_INT_TO_POINTER(pid));
611
612		if (level == LT_LEVEL_THREAD) {
613			/*
614			 * If thread entry is requested, find it based on
615			 * process entry.
616			 */
617			if (stat_c != NULL && stat_c->lt_sc_children != NULL) {
618				stat_c = (lt_stat_collection_t *)
619				    g_hash_table_lookup(stat_c->lt_sc_children,
620				    LT_INT_TO_POINTER(tid));
621			} else {
622				/*
623				 * Thread entry was not found; set it to NULL,
624				 * so that we can return empty list later.
625				 */
626				stat_c = NULL;
627			}
628		}
629	}
630
631	ret = (lt_stat_list_t *)lt_zalloc(sizeof (lt_stat_list_t));
632	ret->lt_sl_entries = (lt_stat_entry_t **)
633	    lt_zalloc(count * sizeof (lt_stat_entry_t *));
634
635	if (stat_c == NULL) {
636		/* Empty list */
637		return (ret);
638	}
639
640	if (list_type == LT_LIST_SOBJ) {
641		group = &(stat_c->lt_sc_groups[GROUP_SOBJ]);
642	} else {
643		group = &(stat_c->lt_sc_groups[GROUP_CAUSE]);
644	}
645
646	if (group->lt_grp_cidlist == NULL) {
647		/* Empty list */
648		return (ret);
649	}
650
651	ret->lt_sl_gtotal = group->lt_grp_summary.lt_se_data.lt_s_total;
652
653	list = g_hash_table_get_values(group->lt_grp_cidlist);
654
655	switch (sort_by) {
656	case LT_SORT_TOTAL:
657		func = (GCompareFunc)lt_sort_by_total_desc;
658		break;
659	case LT_SORT_MAX:
660		func = (GCompareFunc)lt_sort_by_max_desc;
661		break;
662	case LT_SORT_AVG:
663		func = (GCompareFunc)lt_sort_by_avg_desc;
664		break;
665	case LT_SORT_COUNT:
666		func = (GCompareFunc)lt_sort_by_count_desc;
667		break;
668	}
669	list = g_list_sort(list, func);
670
671	for (walk = list;
672	    walk != NULL && count > 0;
673	    walk = g_list_next(walk), --count) {
674		lt_stat_entry_t *data = (lt_stat_entry_t *)walk->data;
675
676		if (list_type == LT_LIST_CAUSE &&
677		    data->lt_se_type == STAT_CAUSE &&
678		    (data->lt_se_tsdata.lt_se_t_cause.lt_se_c_flags &
679		    CAUSE_FLAG_HIDE_IN_SUMMARY) != 0) {
680			continue;
681		}
682
683		if (list_type == LT_LIST_SPECIALS &&
684		    data->lt_se_type == STAT_CAUSE &&
685		    (data->lt_se_tsdata.lt_se_t_cause.lt_se_c_flags &
686		    CAUSE_FLAG_SPECIAL) == 0) {
687			continue;
688		}
689
690		if (data->lt_se_data.lt_s_count == 0) {
691			break;
692		}
693
694		ret->lt_sl_entries[ret->lt_sl_entry_count++] = data;
695	}
696
697	g_list_free(list);
698
699	return (ret);
700}
701
702/*
703 * Free memory allocated by lt_stat_list_create().
704 */
705void
706lt_stat_list_free(void *ptr)
707{
708	lt_stat_list_t *list = (lt_stat_list_t *)ptr;
709
710	if (list == NULL) {
711		return;
712	}
713
714	if (list->lt_sl_free_func != NULL) {
715		list->lt_sl_free_func(list);
716	}
717
718	if (list->lt_sl_entries != NULL) {
719		free(list->lt_sl_entries);
720	}
721
722	free(list);
723}
724
725/*
726 * Check if the given list contains the given item.
727 */
728int
729lt_stat_list_has_item(void *ptr, int i)
730{
731	lt_stat_list_t *list = (lt_stat_list_t *)ptr;
732
733	if (list == NULL || i < 0 || i >= list->lt_sl_entry_count ||
734	    list->lt_sl_entries[i] == NULL) {
735		return (0);
736	}
737
738	return (1);
739}
740
741/*
742 * Get display name of the given item i in the given list.
743 */
744const char *
745lt_stat_list_get_reason(void *ptr, int i)
746{
747	lt_stat_list_t *list = (lt_stat_list_t *)ptr;
748
749	if (list == NULL || i < 0 || i >= list->lt_sl_entry_count ||
750	    list->lt_sl_entries[i] == NULL) {
751		return (NULL);
752	}
753
754	g_assert(list->lt_sl_entries[i]->lt_se_string != NULL);
755
756	return (list->lt_sl_entries[i]->lt_se_string);
757}
758
759/*
760 * Get maximum value of the given item i in the given list.
761 */
762uint64_t
763lt_stat_list_get_max(void *ptr, int i)
764{
765	lt_stat_list_t *list = (lt_stat_list_t *)ptr;
766
767	if (list == NULL || i < 0 || i >= list->lt_sl_entry_count ||
768	    list->lt_sl_entries[i] == NULL) {
769		return (0);
770	}
771
772	return (list->lt_sl_entries[i]->lt_se_data.lt_s_max);
773}
774
775/*
776 * Get total value of the given item i in the given list.
777 */
778uint64_t
779lt_stat_list_get_sum(void *ptr, int i)
780{
781	lt_stat_list_t *list = (lt_stat_list_t *)ptr;
782
783	if (list == NULL || i < 0 || i >= list->lt_sl_entry_count ||
784	    list->lt_sl_entries[i] == NULL) {
785		return (0);
786	}
787
788	return (list->lt_sl_entries[i]->lt_se_data.lt_s_total);
789}
790
791/*
792 * Get count value of the given item i in the given list.
793 */
794uint64_t
795lt_stat_list_get_count(void *ptr, int i)
796{
797	lt_stat_list_t *list = (lt_stat_list_t *)ptr;
798
799	if (list == NULL || i < 0 || i >= list->lt_sl_entry_count ||
800	    list->lt_sl_entries[i] == NULL) {
801		return (0);
802	}
803
804	return (list->lt_sl_entries[i]->lt_se_data.lt_s_count);
805}
806
807/*
808 * Get grand total of all latency in the list.
809 */
810uint64_t
811lt_stat_list_get_gtotal(void *ptr)
812{
813	lt_stat_list_t *list = (lt_stat_list_t *)ptr;
814
815	if (list == NULL) {
816		return (0);
817	}
818
819	return (list->lt_sl_gtotal);
820}
821
822/*
823 * ============================================================================
824 * Process and thread list.
825 * They share a lot of the static variables that are used for keeping
826 * statistics, hence they are located in this file.
827 */
828
829/*
830 * Helper function, sort by PID/TID ascend.
831 */
832static int
833sort_id(lt_stat_collection_t *a, lt_stat_collection_t *b)
834{
835	return ((int)(a->lt_sc_id - b->lt_sc_id));
836}
837
838/*
839 * Get the current list of processes. Call lt_stat_proc_list_free after use
840 * to clean up.
841 */
842static int
843plist_create(pid_t ** list)
844{
845	GList *pid_list, *walk;
846	int ret, count;
847
848	ret = g_hash_table_size(stat_system->lt_sc_children);
849	*list = (pid_t *)lt_malloc(sizeof (pid_t) * ret);
850
851	pid_list = g_hash_table_get_values(stat_system->lt_sc_children);
852	pid_list = g_list_sort(pid_list, (GCompareFunc)sort_id);
853
854	for (walk = pid_list, count = 0;
855	    walk != NULL && count < ret;
856	    walk = g_list_next(walk), ++count) {
857		(*list)[count] = (int)
858		    ((lt_stat_collection_t *)(walk->data))->lt_sc_id;
859	}
860
861	g_list_free(pid_list);
862
863	return (ret);
864}
865
866/*
867 * Count the no. of threads currently present in a process.
868 * Only thread that have SSLEEP are counted.
869 */
870/* ARGSUSED */
871static void
872count_threads(gpointer key, lt_stat_collection_t *stat_c, int *ret)
873{
874	g_assert(ret != NULL);
875
876	if (stat_c->lt_sc_children != NULL) {
877		*ret += g_hash_table_size(stat_c->lt_sc_children);
878	}
879}
880
881/*
882 * Get current list of processes and threads.
883 * Call lt_stat_proc_list_free after use to clean up.
884 */
885static int
886tlist_create(pid_t ** plist, id_t ** tlist)
887{
888	GList *pid_list, *walk_p;
889	GList *tid_list, *walk_t;
890	int ret = 0;
891	int count = 0;
892
893	g_hash_table_foreach(stat_system->lt_sc_children,
894	    (GHFunc)count_threads, &ret);
895
896	*plist = (pid_t *)lt_malloc(sizeof (pid_t) * ret);
897	*tlist = (id_t *)lt_malloc(sizeof (id_t) * ret);
898
899	pid_list = g_hash_table_get_values(stat_system->lt_sc_children);
900	pid_list = g_list_sort(pid_list, (GCompareFunc)sort_id);
901
902	for (walk_p = pid_list; walk_p != NULL;
903	    walk_p = g_list_next(walk_p)) {
904		lt_stat_collection_t *stat_p =
905		    (lt_stat_collection_t *)walk_p->data;
906
907		if (stat_p->lt_sc_children == NULL) {
908			continue;
909		}
910
911		tid_list = g_hash_table_get_values(stat_p->lt_sc_children);
912		tid_list = g_list_sort(tid_list, (GCompareFunc)sort_id);
913
914		for (walk_t = tid_list; walk_t != NULL;
915		    walk_t = g_list_next(walk_t)) {
916			lt_stat_collection_t *stat_t =
917			    (lt_stat_collection_t *)walk_t->data;
918
919			(*plist)[count] = (int)stat_p->lt_sc_id;
920			(*tlist)[count] = (int)stat_t->lt_sc_id;
921
922			++count;
923		}
924		g_list_free(tid_list);
925	}
926
927	g_list_free(pid_list);
928	g_assert(count == ret);
929
930	return (ret);
931}
932
933/*
934 * List of processes that are tracked by LatencyTOP.
935 */
936int
937lt_stat_proc_list_create(pid_t ** plist, id_t ** tlist)
938{
939	if (plist == NULL) {
940		return (-1);
941	}
942
943	if (stat_system == NULL || stat_system->lt_sc_children == NULL) {
944		*plist = NULL;
945
946		if (tlist != NULL) {
947			*tlist = NULL;
948		}
949
950		return (0);
951	}
952
953	if (tlist == NULL) {
954		return (plist_create(plist));
955	} else {
956		return (tlist_create(plist, tlist));
957	}
958}
959
960/*
961 * Free memory allocated by lt_stat_proc_list_create().
962 */
963void
964lt_stat_proc_list_free(pid_t *plist, id_t *tlist)
965{
966	if (plist != NULL) {
967		free(plist);
968	}
969
970	if (tlist != NULL) {
971		free(tlist);
972	}
973}
974
975/*
976 * Get executable name of the given process (ID).
977 */
978const char *
979lt_stat_proc_get_name(pid_t pid)
980{
981	lt_stat_collection_t *stat_p = NULL;
982
983	if (stat_system == NULL || stat_system->lt_sc_children == NULL) {
984		return (NULL);
985	}
986
987	stat_p = (lt_stat_collection_t *)g_hash_table_lookup(
988	    stat_system->lt_sc_children, LT_INT_TO_POINTER(pid));
989
990	if (stat_p != NULL) {
991		return (stat_p->lt_sc_name);
992	} else   {
993		return (NULL);
994	}
995}
996
997/*
998 * Get number of threads.
999 */
1000int
1001lt_stat_proc_get_nthreads(pid_t pid)
1002{
1003	lt_stat_collection_t *stat_p = NULL;
1004
1005	if (stat_system == NULL || stat_system->lt_sc_children == NULL) {
1006		return (0);
1007	}
1008
1009	stat_p = (lt_stat_collection_t *)g_hash_table_lookup(
1010	    stat_system->lt_sc_children, LT_INT_TO_POINTER(pid));
1011
1012	if (stat_p != NULL) {
1013		return (g_hash_table_size(stat_p->lt_sc_children));
1014	} else   {
1015		return (0);
1016	}
1017}
1018
1019/*
1020 * Update statistics for synchronization objects.
1021 */
1022void
1023lt_stat_update_sobj(pid_t pid, id_t tid, int stype,
1024    unsigned long long wchan,
1025    lt_stat_type_t type, uint64_t value)
1026{
1027	lt_sobj_id_t id;
1028	lt_sobj_t *sobj;
1029	int cause_id;
1030	lt_stat_collection_t *stat_t = NULL;
1031
1032	stat_t = get_stat_c(pid, tid);
1033
1034	if (stat_t == NULL) {
1035		return;
1036	}
1037
1038	id.lt_soi_type = stype;
1039	id.lt_soi_addr = wchan;
1040	sobj = lookup_sobj(&id);
1041
1042	if (sobj == NULL) {
1043		return;
1044	}
1045
1046	cause_id = sobj->lt_so_cause_id;
1047
1048	update_stat_entry(stat_t, cause_id, type, value,
1049	    sobj->lt_so_string, GROUP_SOBJ);
1050}
1051