1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23/*
24 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
25 * Use is subject to license terms.
26 */
27
28/*
29 * Copyright (c) 2012 by Delphix. All rights reserved.
30 * Copyright 2017 Joyent, Inc.
31 */
32
33#include <sys/types.h>
34#include <sys/sysmacros.h>
35#include <sys/isa_defs.h>
36
37#include <strings.h>
38#include <stdlib.h>
39#include <setjmp.h>
40#include <assert.h>
41#include <errno.h>
42
43#include <dt_impl.h>
44#include <dt_grammar.h>
45#include <dt_parser.h>
46#include <dt_provider.h>
47
48static void dt_cg_node(dt_node_t *, dt_irlist_t *, dt_regset_t *);
49
50static dt_irnode_t *
51dt_cg_node_alloc(uint_t label, dif_instr_t instr)
52{
53	dt_irnode_t *dip = malloc(sizeof (dt_irnode_t));
54
55	if (dip == NULL)
56		longjmp(yypcb->pcb_jmpbuf, EDT_NOMEM);
57
58	dip->di_label = label;
59	dip->di_instr = instr;
60	dip->di_extern = NULL;
61	dip->di_next = NULL;
62
63	return (dip);
64}
65
66/*
67 * Code generator wrapper function for ctf_member_info.  If we are given a
68 * reference to a forward declaration tag, search the entire type space for
69 * the actual definition and then call ctf_member_info on the result.
70 */
71static ctf_file_t *
72dt_cg_membinfo(ctf_file_t *fp, ctf_id_t type, const char *s, ctf_membinfo_t *mp)
73{
74	while (ctf_type_kind(fp, type) == CTF_K_FORWARD) {
75		char n[DT_TYPE_NAMELEN];
76		dtrace_typeinfo_t dtt;
77
78		if (ctf_type_name(fp, type, n, sizeof (n)) == NULL ||
79		    dt_type_lookup(n, &dtt) == -1 || (
80		    dtt.dtt_ctfp == fp && dtt.dtt_type == type))
81			break; /* unable to improve our position */
82
83		fp = dtt.dtt_ctfp;
84		type = ctf_type_resolve(fp, dtt.dtt_type);
85	}
86
87	if (ctf_member_info(fp, type, s, mp) == CTF_ERR)
88		return (NULL); /* ctf_errno is set for us */
89
90	return (fp);
91}
92
93static void
94dt_cg_xsetx(dt_irlist_t *dlp, dt_ident_t *idp, uint_t lbl, int reg, uint64_t x)
95{
96	int flag = idp != NULL ? DT_INT_PRIVATE : DT_INT_SHARED;
97	int intoff = dt_inttab_insert(yypcb->pcb_inttab, x, flag);
98	dif_instr_t instr = DIF_INSTR_SETX((uint_t)intoff, reg);
99
100	if (intoff == -1)
101		longjmp(yypcb->pcb_jmpbuf, EDT_NOMEM);
102
103	if (intoff > DIF_INTOFF_MAX)
104		longjmp(yypcb->pcb_jmpbuf, EDT_INT2BIG);
105
106	dt_irlist_append(dlp, dt_cg_node_alloc(lbl, instr));
107
108	if (idp != NULL)
109		dlp->dl_last->di_extern = idp;
110}
111
112static void
113dt_cg_setx(dt_irlist_t *dlp, int reg, uint64_t x)
114{
115	dt_cg_xsetx(dlp, NULL, DT_LBL_NONE, reg, x);
116}
117
118/*
119 * When loading bit-fields, we want to convert a byte count in the range
120 * 1-8 to the closest power of 2 (e.g. 3->4, 5->8, etc).  The clp2() function
121 * is a clever implementation from "Hacker's Delight" by Henry Warren, Jr.
122 */
123static size_t
124clp2(size_t x)
125{
126	x--;
127
128	x |= (x >> 1);
129	x |= (x >> 2);
130	x |= (x >> 4);
131	x |= (x >> 8);
132	x |= (x >> 16);
133
134	return (x + 1);
135}
136
137/*
138 * Lookup the correct load opcode to use for the specified node and CTF type.
139 * We determine the size and convert it to a 3-bit index.  Our lookup table
140 * is constructed to use a 5-bit index, consisting of the 3-bit size 0-7, a
141 * bit for the sign, and a bit for userland address.  For example, a 4-byte
142 * signed load from userland would be at the following table index:
143 * user=1 sign=1 size=4 => binary index 11011 = decimal index 27
144 */
145static uint_t
146dt_cg_load(dt_node_t *dnp, ctf_file_t *ctfp, ctf_id_t type)
147{
148	static const uint_t ops[] = {
149		DIF_OP_LDUB,	DIF_OP_LDUH,	0,	DIF_OP_LDUW,
150		0,		0,		0,	DIF_OP_LDX,
151		DIF_OP_LDSB,	DIF_OP_LDSH,	0,	DIF_OP_LDSW,
152		0,		0,		0,	DIF_OP_LDX,
153		DIF_OP_ULDUB,	DIF_OP_ULDUH,	0,	DIF_OP_ULDUW,
154		0,		0,		0,	DIF_OP_ULDX,
155		DIF_OP_ULDSB,	DIF_OP_ULDSH,	0,	DIF_OP_ULDSW,
156		0,		0,		0,	DIF_OP_ULDX,
157	};
158
159	ctf_encoding_t e;
160	ssize_t size;
161
162	/*
163	 * If we're loading a bit-field, the size of our load is found by
164	 * rounding cte_bits up to a byte boundary and then finding the
165	 * nearest power of two to this value (see clp2(), above).
166	 */
167	if ((dnp->dn_flags & DT_NF_BITFIELD) &&
168	    ctf_type_encoding(ctfp, type, &e) != CTF_ERR)
169		size = clp2(P2ROUNDUP(e.cte_bits, NBBY) / NBBY);
170	else
171		size = ctf_type_size(ctfp, type);
172
173	if (size < 1 || size > 8 || (size & (size - 1)) != 0) {
174		xyerror(D_UNKNOWN, "internal error -- cg cannot load "
175		    "size %ld when passed by value\n", (long)size);
176	}
177
178	size--; /* convert size to 3-bit index */
179
180	if (dnp->dn_flags & DT_NF_SIGNED)
181		size |= 0x08;
182	if (dnp->dn_flags & DT_NF_USERLAND)
183		size |= 0x10;
184
185	return (ops[size]);
186}
187
188static void
189dt_cg_ptrsize(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp,
190    uint_t op, int dreg)
191{
192	ctf_file_t *ctfp = dnp->dn_ctfp;
193	ctf_arinfo_t r;
194	dif_instr_t instr;
195	ctf_id_t type;
196	uint_t kind;
197	ssize_t size;
198	int sreg;
199
200	type = ctf_type_resolve(ctfp, dnp->dn_type);
201	kind = ctf_type_kind(ctfp, type);
202	assert(kind == CTF_K_POINTER || kind == CTF_K_ARRAY);
203
204	if (kind == CTF_K_ARRAY) {
205		if (ctf_array_info(ctfp, type, &r) != 0) {
206			yypcb->pcb_hdl->dt_ctferr = ctf_errno(ctfp);
207			longjmp(yypcb->pcb_jmpbuf, EDT_CTF);
208		}
209		type = r.ctr_contents;
210	} else
211		type = ctf_type_reference(ctfp, type);
212
213	if ((size = ctf_type_size(ctfp, type)) == 1)
214		return; /* multiply or divide by one can be omitted */
215
216	sreg = dt_regset_alloc(drp);
217	dt_cg_setx(dlp, sreg, size);
218	instr = DIF_INSTR_FMT(op, dreg, sreg, dreg);
219	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
220	dt_regset_free(drp, sreg);
221}
222
223/*
224 * If the result of a "." or "->" operation is a bit-field, we use this routine
225 * to generate an epilogue to the load instruction that extracts the value.  In
226 * the diagrams below the "ld??" is the load instruction that is generated to
227 * load the containing word that is generating prior to calling this function.
228 *
229 * Epilogue for unsigned fields:	Epilogue for signed fields:
230 *
231 * ldu?	[r1], r1			lds? [r1], r1
232 * setx	USHIFT, r2			setx 64 - SSHIFT, r2
233 * srl	r1, r2, r1			sll  r1, r2, r1
234 * setx	(1 << bits) - 1, r2		setx 64 - bits, r2
235 * and	r1, r2, r1			sra  r1, r2, r1
236 *
237 * The *SHIFT constants above changes value depending on the endian-ness of our
238 * target architecture.  Refer to the comments below for more details.
239 */
240static void
241dt_cg_field_get(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp,
242    ctf_file_t *fp, const ctf_membinfo_t *mp)
243{
244	ctf_encoding_t e;
245	dif_instr_t instr;
246	uint64_t shift;
247	int r1, r2;
248
249	if (ctf_type_encoding(fp, mp->ctm_type, &e) != 0 || e.cte_bits > 64) {
250		xyerror(D_UNKNOWN, "cg: bad field: off %lu type <%ld> "
251		    "bits %u\n", mp->ctm_offset, mp->ctm_type, e.cte_bits);
252	}
253
254	assert(dnp->dn_op == DT_TOK_PTR || dnp->dn_op == DT_TOK_DOT);
255	r1 = dnp->dn_left->dn_reg;
256	r2 = dt_regset_alloc(drp);
257
258	/*
259	 * On little-endian architectures, ctm_offset counts from the right so
260	 * ctm_offset % NBBY itself is the amount we want to shift right to
261	 * move the value bits to the little end of the register to mask them.
262	 * On big-endian architectures, ctm_offset counts from the left so we
263	 * must subtract (ctm_offset % NBBY + cte_bits) from the size in bits
264	 * we used for the load.  The size of our load in turn is found by
265	 * rounding cte_bits up to a byte boundary and then finding the
266	 * nearest power of two to this value (see clp2(), above).  These
267	 * properties are used to compute shift as USHIFT or SSHIFT, below.
268	 */
269	if (dnp->dn_flags & DT_NF_SIGNED) {
270#ifdef _BIG_ENDIAN
271		shift = clp2(P2ROUNDUP(e.cte_bits, NBBY) / NBBY) * NBBY -
272		    mp->ctm_offset % NBBY;
273#else
274		shift = mp->ctm_offset % NBBY + e.cte_bits;
275#endif
276		dt_cg_setx(dlp, r2, 64 - shift);
277		instr = DIF_INSTR_FMT(DIF_OP_SLL, r1, r2, r1);
278		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
279
280		dt_cg_setx(dlp, r2, 64 - e.cte_bits);
281		instr = DIF_INSTR_FMT(DIF_OP_SRA, r1, r2, r1);
282		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
283	} else {
284#ifdef _BIG_ENDIAN
285		shift = clp2(P2ROUNDUP(e.cte_bits, NBBY) / NBBY) * NBBY -
286		    (mp->ctm_offset % NBBY + e.cte_bits);
287#else
288		shift = mp->ctm_offset % NBBY;
289#endif
290		dt_cg_setx(dlp, r2, shift);
291		instr = DIF_INSTR_FMT(DIF_OP_SRL, r1, r2, r1);
292		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
293
294		dt_cg_setx(dlp, r2, (1ULL << e.cte_bits) - 1);
295		instr = DIF_INSTR_FMT(DIF_OP_AND, r1, r2, r1);
296		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
297	}
298
299	dt_regset_free(drp, r2);
300}
301
302/*
303 * If the destination of a store operation is a bit-field, we use this routine
304 * to generate a prologue to the store instruction that loads the surrounding
305 * bits, clears the destination field, and ORs in the new value of the field.
306 * In the diagram below the "st?" is the store instruction that is generated to
307 * store the containing word that is generating after calling this function.
308 *
309 * ld	[dst->dn_reg], r1
310 * setx	~(((1 << cte_bits) - 1) << (ctm_offset % NBBY)), r2
311 * and	r1, r2, r1
312 *
313 * setx	(1 << cte_bits) - 1, r2
314 * and	src->dn_reg, r2, r2
315 * setx ctm_offset % NBBY, r3
316 * sll	r2, r3, r2
317 *
318 * or	r1, r2, r1
319 * st?	r1, [dst->dn_reg]
320 *
321 * This routine allocates a new register to hold the value to be stored and
322 * returns it.  The caller is responsible for freeing this register later.
323 */
324static int
325dt_cg_field_set(dt_node_t *src, dt_irlist_t *dlp,
326    dt_regset_t *drp, dt_node_t *dst)
327{
328	uint64_t cmask, fmask, shift;
329	dif_instr_t instr;
330	int r1, r2, r3;
331
332	ctf_membinfo_t m;
333	ctf_encoding_t e;
334	ctf_file_t *fp, *ofp;
335	ctf_id_t type;
336
337	assert(dst->dn_op == DT_TOK_PTR || dst->dn_op == DT_TOK_DOT);
338	assert(dst->dn_right->dn_kind == DT_NODE_IDENT);
339
340	fp = dst->dn_left->dn_ctfp;
341	type = ctf_type_resolve(fp, dst->dn_left->dn_type);
342
343	if (dst->dn_op == DT_TOK_PTR) {
344		type = ctf_type_reference(fp, type);
345		type = ctf_type_resolve(fp, type);
346	}
347
348	if ((fp = dt_cg_membinfo(ofp = fp, type,
349	    dst->dn_right->dn_string, &m)) == NULL) {
350		yypcb->pcb_hdl->dt_ctferr = ctf_errno(ofp);
351		longjmp(yypcb->pcb_jmpbuf, EDT_CTF);
352	}
353
354	if (ctf_type_encoding(fp, m.ctm_type, &e) != 0 || e.cte_bits > 64) {
355		xyerror(D_UNKNOWN, "cg: bad field: off %lu type <%ld> "
356		    "bits %u\n", m.ctm_offset, m.ctm_type, e.cte_bits);
357	}
358
359	r1 = dt_regset_alloc(drp);
360	r2 = dt_regset_alloc(drp);
361	r3 = dt_regset_alloc(drp);
362
363	/*
364	 * Compute shifts and masks.  We need to compute "shift" as the amount
365	 * we need to shift left to position our field in the containing word.
366	 * Refer to the comments in dt_cg_field_get(), above, for more info.
367	 * We then compute fmask as the mask that truncates the value in the
368	 * input register to width cte_bits, and cmask as the mask used to
369	 * pass through the containing bits and zero the field bits.
370	 */
371#ifdef _BIG_ENDIAN
372	shift = clp2(P2ROUNDUP(e.cte_bits, NBBY) / NBBY) * NBBY -
373	    (m.ctm_offset % NBBY + e.cte_bits);
374#else
375	shift = m.ctm_offset % NBBY;
376#endif
377	fmask = (1ULL << e.cte_bits) - 1;
378	cmask = ~(fmask << shift);
379
380	instr = DIF_INSTR_LOAD(
381	    dt_cg_load(dst, fp, m.ctm_type), dst->dn_reg, r1);
382	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
383
384	dt_cg_setx(dlp, r2, cmask);
385	instr = DIF_INSTR_FMT(DIF_OP_AND, r1, r2, r1);
386	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
387
388	dt_cg_setx(dlp, r2, fmask);
389	instr = DIF_INSTR_FMT(DIF_OP_AND, src->dn_reg, r2, r2);
390	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
391
392	dt_cg_setx(dlp, r3, shift);
393	instr = DIF_INSTR_FMT(DIF_OP_SLL, r2, r3, r2);
394	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
395
396	instr = DIF_INSTR_FMT(DIF_OP_OR, r1, r2, r1);
397	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
398
399	dt_regset_free(drp, r3);
400	dt_regset_free(drp, r2);
401
402	return (r1);
403}
404
405static void
406dt_cg_store(dt_node_t *src, dt_irlist_t *dlp, dt_regset_t *drp, dt_node_t *dst)
407{
408	ctf_encoding_t e;
409	dif_instr_t instr;
410	size_t size;
411	int reg;
412
413	/*
414	 * If we're loading a bit-field, the size of our store is found by
415	 * rounding dst's cte_bits up to a byte boundary and then finding the
416	 * nearest power of two to this value (see clp2(), above).
417	 */
418	if ((dst->dn_flags & DT_NF_BITFIELD) &&
419	    ctf_type_encoding(dst->dn_ctfp, dst->dn_type, &e) != CTF_ERR)
420		size = clp2(P2ROUNDUP(e.cte_bits, NBBY) / NBBY);
421	else
422		size = dt_node_type_size(src);
423
424	if (src->dn_flags & DT_NF_REF) {
425		reg = dt_regset_alloc(drp);
426		dt_cg_setx(dlp, reg, size);
427		instr = DIF_INSTR_COPYS(src->dn_reg, reg, dst->dn_reg);
428		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
429		dt_regset_free(drp, reg);
430	} else {
431		if (dst->dn_flags & DT_NF_BITFIELD)
432			reg = dt_cg_field_set(src, dlp, drp, dst);
433		else
434			reg = src->dn_reg;
435
436		switch (size) {
437		case 1:
438			instr = DIF_INSTR_STORE(DIF_OP_STB, reg, dst->dn_reg);
439			break;
440		case 2:
441			instr = DIF_INSTR_STORE(DIF_OP_STH, reg, dst->dn_reg);
442			break;
443		case 4:
444			instr = DIF_INSTR_STORE(DIF_OP_STW, reg, dst->dn_reg);
445			break;
446		case 8:
447			instr = DIF_INSTR_STORE(DIF_OP_STX, reg, dst->dn_reg);
448			break;
449		default:
450			xyerror(D_UNKNOWN, "internal error -- cg cannot store "
451			    "size %lu when passed by value\n", (ulong_t)size);
452		}
453		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
454
455		if (dst->dn_flags & DT_NF_BITFIELD)
456			dt_regset_free(drp, reg);
457	}
458}
459
460/*
461 * Generate code for a typecast or for argument promotion from the type of the
462 * actual to the type of the formal.  We need to generate code for casts when
463 * a scalar type is being narrowed or changing signed-ness.  We first shift the
464 * desired bits high (losing excess bits if narrowing) and then shift them down
465 * using logical shift (unsigned result) or arithmetic shift (signed result).
466 */
467static void
468dt_cg_typecast(const dt_node_t *src, const dt_node_t *dst,
469    dt_irlist_t *dlp, dt_regset_t *drp)
470{
471	size_t srcsize = dt_node_type_size(src);
472	size_t dstsize = dt_node_type_size(dst);
473
474	dif_instr_t instr;
475	int rg;
476
477	if (!dt_node_is_scalar(dst))
478		return; /* not a scalar */
479	if (dstsize == srcsize &&
480	    ((src->dn_flags ^ dst->dn_flags) & DT_NF_SIGNED) == 0)
481		return; /* not narrowing or changing signed-ness */
482	if (dstsize > srcsize && (src->dn_flags & DT_NF_SIGNED) == 0)
483		return; /* nothing to do in this case */
484
485	rg = dt_regset_alloc(drp);
486
487	if (dstsize > srcsize) {
488		int n = sizeof (uint64_t) * NBBY - srcsize * NBBY;
489		int s = (dstsize - srcsize) * NBBY;
490
491		dt_cg_setx(dlp, rg, n);
492
493		instr = DIF_INSTR_FMT(DIF_OP_SLL, src->dn_reg, rg, dst->dn_reg);
494		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
495
496		if ((dst->dn_flags & DT_NF_SIGNED) || n == s) {
497			instr = DIF_INSTR_FMT(DIF_OP_SRA,
498			    dst->dn_reg, rg, dst->dn_reg);
499			dt_irlist_append(dlp,
500			    dt_cg_node_alloc(DT_LBL_NONE, instr));
501		} else {
502			dt_cg_setx(dlp, rg, s);
503			instr = DIF_INSTR_FMT(DIF_OP_SRA,
504			    dst->dn_reg, rg, dst->dn_reg);
505			dt_irlist_append(dlp,
506			    dt_cg_node_alloc(DT_LBL_NONE, instr));
507			dt_cg_setx(dlp, rg, n - s);
508			instr = DIF_INSTR_FMT(DIF_OP_SRL,
509			    dst->dn_reg, rg, dst->dn_reg);
510			dt_irlist_append(dlp,
511			    dt_cg_node_alloc(DT_LBL_NONE, instr));
512		}
513	} else if (dstsize != sizeof (uint64_t)) {
514		int n = sizeof (uint64_t) * NBBY - dstsize * NBBY;
515
516		dt_cg_setx(dlp, rg, n);
517
518		instr = DIF_INSTR_FMT(DIF_OP_SLL, src->dn_reg, rg, dst->dn_reg);
519		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
520
521		instr = DIF_INSTR_FMT((dst->dn_flags & DT_NF_SIGNED) ?
522		    DIF_OP_SRA : DIF_OP_SRL, dst->dn_reg, rg, dst->dn_reg);
523		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
524	}
525
526	dt_regset_free(drp, rg);
527}
528
529/*
530 * Generate code to push the specified argument list on to the tuple stack.
531 * We use this routine for handling subroutine calls and associative arrays.
532 * We must first generate code for all subexpressions before loading the stack
533 * because any subexpression could itself require the use of the tuple stack.
534 * This holds a number of registers equal to the number of arguments, but this
535 * is not a huge problem because the number of arguments can't exceed the
536 * number of tuple register stack elements anyway.  At most one extra register
537 * is required (either by dt_cg_typecast() or for dtdt_size, below).  This
538 * implies that a DIF implementation should offer a number of general purpose
539 * registers at least one greater than the number of tuple registers.
540 */
541static void
542dt_cg_arglist(dt_ident_t *idp, dt_node_t *args,
543    dt_irlist_t *dlp, dt_regset_t *drp)
544{
545	const dt_idsig_t *isp = idp->di_data;
546	dt_node_t *dnp;
547	int i = 0;
548
549	for (dnp = args; dnp != NULL; dnp = dnp->dn_list)
550		dt_cg_node(dnp, dlp, drp);
551
552	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, DIF_INSTR_FLUSHTS));
553
554	for (dnp = args; dnp != NULL; dnp = dnp->dn_list, i++) {
555		dtrace_diftype_t t;
556		dif_instr_t instr;
557		uint_t op;
558		int reg;
559
560		dt_node_diftype(yypcb->pcb_hdl, dnp, &t);
561
562		isp->dis_args[i].dn_reg = dnp->dn_reg; /* re-use register */
563		dt_cg_typecast(dnp, &isp->dis_args[i], dlp, drp);
564		isp->dis_args[i].dn_reg = -1;
565
566		if (t.dtdt_flags & DIF_TF_BYREF) {
567			op = DIF_OP_PUSHTR;
568			if (t.dtdt_size != 0) {
569				reg = dt_regset_alloc(drp);
570				dt_cg_setx(dlp, reg, t.dtdt_size);
571			} else {
572				reg = DIF_REG_R0;
573			}
574		} else {
575			op = DIF_OP_PUSHTV;
576			reg = DIF_REG_R0;
577		}
578
579		instr = DIF_INSTR_PUSHTS(op, t.dtdt_kind, reg, dnp->dn_reg);
580		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
581		dt_regset_free(drp, dnp->dn_reg);
582
583		if (reg != DIF_REG_R0)
584			dt_regset_free(drp, reg);
585	}
586
587	if (i > yypcb->pcb_hdl->dt_conf.dtc_diftupregs)
588		longjmp(yypcb->pcb_jmpbuf, EDT_NOTUPREG);
589}
590
591static void
592dt_cg_arithmetic_op(dt_node_t *dnp, dt_irlist_t *dlp,
593    dt_regset_t *drp, uint_t op)
594{
595	int is_ptr_op = (dnp->dn_op == DT_TOK_ADD || dnp->dn_op == DT_TOK_SUB ||
596	    dnp->dn_op == DT_TOK_ADD_EQ || dnp->dn_op == DT_TOK_SUB_EQ);
597
598	int lp_is_ptr = dt_node_is_pointer(dnp->dn_left);
599	int rp_is_ptr = dt_node_is_pointer(dnp->dn_right);
600
601	dif_instr_t instr;
602
603	if (lp_is_ptr && rp_is_ptr) {
604		assert(dnp->dn_op == DT_TOK_SUB);
605		is_ptr_op = 0;
606	}
607
608	dt_cg_node(dnp->dn_left, dlp, drp);
609	if (is_ptr_op && rp_is_ptr)
610		dt_cg_ptrsize(dnp, dlp, drp, DIF_OP_MUL, dnp->dn_left->dn_reg);
611
612	dt_cg_node(dnp->dn_right, dlp, drp);
613	if (is_ptr_op && lp_is_ptr)
614		dt_cg_ptrsize(dnp, dlp, drp, DIF_OP_MUL, dnp->dn_right->dn_reg);
615
616	instr = DIF_INSTR_FMT(op, dnp->dn_left->dn_reg,
617	    dnp->dn_right->dn_reg, dnp->dn_left->dn_reg);
618
619	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
620	dt_regset_free(drp, dnp->dn_right->dn_reg);
621	dnp->dn_reg = dnp->dn_left->dn_reg;
622
623	if (lp_is_ptr && rp_is_ptr)
624		dt_cg_ptrsize(dnp->dn_right,
625		    dlp, drp, DIF_OP_UDIV, dnp->dn_reg);
626}
627
628static uint_t
629dt_cg_stvar(const dt_ident_t *idp)
630{
631	static const uint_t aops[] = { DIF_OP_STGAA, DIF_OP_STTAA, DIF_OP_NOP };
632	static const uint_t sops[] = { DIF_OP_STGS, DIF_OP_STTS, DIF_OP_STLS };
633
634	uint_t i = (((idp->di_flags & DT_IDFLG_LOCAL) != 0) << 1) |
635	    ((idp->di_flags & DT_IDFLG_TLS) != 0);
636
637	return (idp->di_kind == DT_IDENT_ARRAY ? aops[i] : sops[i]);
638}
639
640static void
641dt_cg_prearith_op(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp, uint_t op)
642{
643	ctf_file_t *ctfp = dnp->dn_ctfp;
644	dif_instr_t instr;
645	ctf_id_t type;
646	ssize_t size = 1;
647	int reg;
648
649	if (dt_node_is_pointer(dnp)) {
650		type = ctf_type_resolve(ctfp, dnp->dn_type);
651		assert(ctf_type_kind(ctfp, type) == CTF_K_POINTER);
652		size = ctf_type_size(ctfp, ctf_type_reference(ctfp, type));
653	}
654
655	dt_cg_node(dnp->dn_child, dlp, drp);
656	dnp->dn_reg = dnp->dn_child->dn_reg;
657
658	reg = dt_regset_alloc(drp);
659	dt_cg_setx(dlp, reg, size);
660
661	instr = DIF_INSTR_FMT(op, dnp->dn_reg, reg, dnp->dn_reg);
662	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
663	dt_regset_free(drp, reg);
664
665	/*
666	 * If we are modifying a variable, generate an stv instruction from
667	 * the variable specified by the identifier.  If we are storing to a
668	 * memory address, generate code again for the left-hand side using
669	 * DT_NF_REF to get the address, and then generate a store to it.
670	 * In both paths, we store the value in dnp->dn_reg (the new value).
671	 */
672	if (dnp->dn_child->dn_kind == DT_NODE_VAR) {
673		dt_ident_t *idp = dt_ident_resolve(dnp->dn_child->dn_ident);
674
675		idp->di_flags |= DT_IDFLG_DIFW;
676		instr = DIF_INSTR_STV(dt_cg_stvar(idp),
677		    idp->di_id, dnp->dn_reg);
678		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
679	} else {
680		uint_t rbit = dnp->dn_child->dn_flags & DT_NF_REF;
681
682		assert(dnp->dn_child->dn_flags & DT_NF_WRITABLE);
683		assert(dnp->dn_child->dn_flags & DT_NF_LVALUE);
684
685		dnp->dn_child->dn_flags |= DT_NF_REF; /* force pass-by-ref */
686		dt_cg_node(dnp->dn_child, dlp, drp);
687
688		dt_cg_store(dnp, dlp, drp, dnp->dn_child);
689		dt_regset_free(drp, dnp->dn_child->dn_reg);
690
691		dnp->dn_left->dn_flags &= ~DT_NF_REF;
692		dnp->dn_left->dn_flags |= rbit;
693	}
694}
695
696static void
697dt_cg_postarith_op(dt_node_t *dnp, dt_irlist_t *dlp,
698    dt_regset_t *drp, uint_t op)
699{
700	ctf_file_t *ctfp = dnp->dn_ctfp;
701	dif_instr_t instr;
702	ctf_id_t type;
703	ssize_t size = 1;
704	int nreg;
705
706	if (dt_node_is_pointer(dnp)) {
707		type = ctf_type_resolve(ctfp, dnp->dn_type);
708		assert(ctf_type_kind(ctfp, type) == CTF_K_POINTER);
709		size = ctf_type_size(ctfp, ctf_type_reference(ctfp, type));
710	}
711
712	dt_cg_node(dnp->dn_child, dlp, drp);
713	dnp->dn_reg = dnp->dn_child->dn_reg;
714
715	nreg = dt_regset_alloc(drp);
716	dt_cg_setx(dlp, nreg, size);
717	instr = DIF_INSTR_FMT(op, dnp->dn_reg, nreg, nreg);
718	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
719
720	/*
721	 * If we are modifying a variable, generate an stv instruction from
722	 * the variable specified by the identifier.  If we are storing to a
723	 * memory address, generate code again for the left-hand side using
724	 * DT_NF_REF to get the address, and then generate a store to it.
725	 * In both paths, we store the value from 'nreg' (the new value).
726	 */
727	if (dnp->dn_child->dn_kind == DT_NODE_VAR) {
728		dt_ident_t *idp = dt_ident_resolve(dnp->dn_child->dn_ident);
729
730		idp->di_flags |= DT_IDFLG_DIFW;
731		instr = DIF_INSTR_STV(dt_cg_stvar(idp), idp->di_id, nreg);
732		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
733	} else {
734		uint_t rbit = dnp->dn_child->dn_flags & DT_NF_REF;
735		int oreg = dnp->dn_reg;
736
737		assert(dnp->dn_child->dn_flags & DT_NF_WRITABLE);
738		assert(dnp->dn_child->dn_flags & DT_NF_LVALUE);
739
740		dnp->dn_child->dn_flags |= DT_NF_REF; /* force pass-by-ref */
741		dt_cg_node(dnp->dn_child, dlp, drp);
742
743		dnp->dn_reg = nreg;
744		dt_cg_store(dnp, dlp, drp, dnp->dn_child);
745		dnp->dn_reg = oreg;
746
747		dt_regset_free(drp, dnp->dn_child->dn_reg);
748		dnp->dn_left->dn_flags &= ~DT_NF_REF;
749		dnp->dn_left->dn_flags |= rbit;
750	}
751
752	dt_regset_free(drp, nreg);
753}
754
755/*
756 * Determine if we should perform signed or unsigned comparison for an OP2.
757 * If both operands are of arithmetic type, perform the usual arithmetic
758 * conversions to determine the common real type for comparison [ISOC 6.5.8.3].
759 */
760static int
761dt_cg_compare_signed(dt_node_t *dnp)
762{
763	dt_node_t dn;
764
765	if (dt_node_is_string(dnp->dn_left) ||
766	    dt_node_is_string(dnp->dn_right))
767		return (1); /* strings always compare signed */
768	else if (!dt_node_is_arith(dnp->dn_left) ||
769	    !dt_node_is_arith(dnp->dn_right))
770		return (0); /* non-arithmetic types always compare unsigned */
771
772	bzero(&dn, sizeof (dn));
773	dt_node_promote(dnp->dn_left, dnp->dn_right, &dn);
774	return (dn.dn_flags & DT_NF_SIGNED);
775}
776
777static void
778dt_cg_compare_op(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp, uint_t op)
779{
780	uint_t lbl_true = dt_irlist_label(dlp);
781	uint_t lbl_post = dt_irlist_label(dlp);
782
783	dif_instr_t instr;
784	uint_t opc;
785
786	dt_cg_node(dnp->dn_left, dlp, drp);
787	dt_cg_node(dnp->dn_right, dlp, drp);
788
789	if (dt_node_is_string(dnp->dn_left) || dt_node_is_string(dnp->dn_right))
790		opc = DIF_OP_SCMP;
791	else
792		opc = DIF_OP_CMP;
793
794	instr = DIF_INSTR_CMP(opc, dnp->dn_left->dn_reg, dnp->dn_right->dn_reg);
795	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
796	dt_regset_free(drp, dnp->dn_right->dn_reg);
797	dnp->dn_reg = dnp->dn_left->dn_reg;
798
799	instr = DIF_INSTR_BRANCH(op, lbl_true);
800	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
801
802	instr = DIF_INSTR_MOV(DIF_REG_R0, dnp->dn_reg);
803	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
804
805	instr = DIF_INSTR_BRANCH(DIF_OP_BA, lbl_post);
806	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
807
808	dt_cg_xsetx(dlp, NULL, lbl_true, dnp->dn_reg, 1);
809	dt_irlist_append(dlp, dt_cg_node_alloc(lbl_post, DIF_INSTR_NOP));
810}
811
812/*
813 * Code generation for the ternary op requires some trickery with the assembler
814 * in order to conserve registers.  We generate code for dn_expr and dn_left
815 * and free their registers so they do not have be consumed across codegen for
816 * dn_right.  We insert a dummy MOV at the end of dn_left into the destination
817 * register, which is not yet known because we haven't done dn_right yet, and
818 * save the pointer to this instruction node.  We then generate code for
819 * dn_right and use its register as our output.  Finally, we reach back and
820 * patch the instruction for dn_left to move its output into this register.
821 */
822static void
823dt_cg_ternary_op(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp)
824{
825	uint_t lbl_false = dt_irlist_label(dlp);
826	uint_t lbl_post = dt_irlist_label(dlp);
827
828	dif_instr_t instr;
829	dt_irnode_t *dip;
830
831	dt_cg_node(dnp->dn_expr, dlp, drp);
832	instr = DIF_INSTR_TST(dnp->dn_expr->dn_reg);
833	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
834	dt_regset_free(drp, dnp->dn_expr->dn_reg);
835
836	instr = DIF_INSTR_BRANCH(DIF_OP_BE, lbl_false);
837	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
838
839	dt_cg_node(dnp->dn_left, dlp, drp);
840	instr = DIF_INSTR_MOV(dnp->dn_left->dn_reg, DIF_REG_R0);
841	dip = dt_cg_node_alloc(DT_LBL_NONE, instr); /* save dip for below */
842	dt_irlist_append(dlp, dip);
843	dt_regset_free(drp, dnp->dn_left->dn_reg);
844
845	instr = DIF_INSTR_BRANCH(DIF_OP_BA, lbl_post);
846	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
847
848	dt_irlist_append(dlp, dt_cg_node_alloc(lbl_false, DIF_INSTR_NOP));
849	dt_cg_node(dnp->dn_right, dlp, drp);
850	dnp->dn_reg = dnp->dn_right->dn_reg;
851
852	/*
853	 * Now that dn_reg is assigned, reach back and patch the correct MOV
854	 * instruction into the tail of dn_left.  We know dn_reg was unused
855	 * at that point because otherwise dn_right couldn't have allocated it.
856	 */
857	dip->di_instr = DIF_INSTR_MOV(dnp->dn_left->dn_reg, dnp->dn_reg);
858	dt_irlist_append(dlp, dt_cg_node_alloc(lbl_post, DIF_INSTR_NOP));
859}
860
861static void
862dt_cg_logical_and(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp)
863{
864	uint_t lbl_false = dt_irlist_label(dlp);
865	uint_t lbl_post = dt_irlist_label(dlp);
866
867	dif_instr_t instr;
868
869	dt_cg_node(dnp->dn_left, dlp, drp);
870	instr = DIF_INSTR_TST(dnp->dn_left->dn_reg);
871	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
872	dt_regset_free(drp, dnp->dn_left->dn_reg);
873
874	instr = DIF_INSTR_BRANCH(DIF_OP_BE, lbl_false);
875	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
876
877	dt_cg_node(dnp->dn_right, dlp, drp);
878	instr = DIF_INSTR_TST(dnp->dn_right->dn_reg);
879	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
880	dnp->dn_reg = dnp->dn_right->dn_reg;
881
882	instr = DIF_INSTR_BRANCH(DIF_OP_BE, lbl_false);
883	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
884
885	dt_cg_setx(dlp, dnp->dn_reg, 1);
886
887	instr = DIF_INSTR_BRANCH(DIF_OP_BA, lbl_post);
888	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
889
890	instr = DIF_INSTR_MOV(DIF_REG_R0, dnp->dn_reg);
891	dt_irlist_append(dlp, dt_cg_node_alloc(lbl_false, instr));
892
893	dt_irlist_append(dlp, dt_cg_node_alloc(lbl_post, DIF_INSTR_NOP));
894}
895
896static void
897dt_cg_logical_xor(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp)
898{
899	uint_t lbl_next = dt_irlist_label(dlp);
900	uint_t lbl_tail = dt_irlist_label(dlp);
901
902	dif_instr_t instr;
903
904	dt_cg_node(dnp->dn_left, dlp, drp);
905	instr = DIF_INSTR_TST(dnp->dn_left->dn_reg);
906	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
907
908	instr = DIF_INSTR_BRANCH(DIF_OP_BE, lbl_next);
909	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
910	dt_cg_setx(dlp, dnp->dn_left->dn_reg, 1);
911
912	dt_irlist_append(dlp, dt_cg_node_alloc(lbl_next, DIF_INSTR_NOP));
913	dt_cg_node(dnp->dn_right, dlp, drp);
914
915	instr = DIF_INSTR_TST(dnp->dn_right->dn_reg);
916	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
917
918	instr = DIF_INSTR_BRANCH(DIF_OP_BE, lbl_tail);
919	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
920	dt_cg_setx(dlp, dnp->dn_right->dn_reg, 1);
921
922	instr = DIF_INSTR_FMT(DIF_OP_XOR, dnp->dn_left->dn_reg,
923	    dnp->dn_right->dn_reg, dnp->dn_left->dn_reg);
924
925	dt_irlist_append(dlp, dt_cg_node_alloc(lbl_tail, instr));
926
927	dt_regset_free(drp, dnp->dn_right->dn_reg);
928	dnp->dn_reg = dnp->dn_left->dn_reg;
929}
930
931static void
932dt_cg_logical_or(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp)
933{
934	uint_t lbl_true = dt_irlist_label(dlp);
935	uint_t lbl_false = dt_irlist_label(dlp);
936	uint_t lbl_post = dt_irlist_label(dlp);
937
938	dif_instr_t instr;
939
940	dt_cg_node(dnp->dn_left, dlp, drp);
941	instr = DIF_INSTR_TST(dnp->dn_left->dn_reg);
942	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
943	dt_regset_free(drp, dnp->dn_left->dn_reg);
944
945	instr = DIF_INSTR_BRANCH(DIF_OP_BNE, lbl_true);
946	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
947
948	dt_cg_node(dnp->dn_right, dlp, drp);
949	instr = DIF_INSTR_TST(dnp->dn_right->dn_reg);
950	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
951	dnp->dn_reg = dnp->dn_right->dn_reg;
952
953	instr = DIF_INSTR_BRANCH(DIF_OP_BE, lbl_false);
954	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
955
956	dt_cg_xsetx(dlp, NULL, lbl_true, dnp->dn_reg, 1);
957
958	instr = DIF_INSTR_BRANCH(DIF_OP_BA, lbl_post);
959	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
960
961	instr = DIF_INSTR_MOV(DIF_REG_R0, dnp->dn_reg);
962	dt_irlist_append(dlp, dt_cg_node_alloc(lbl_false, instr));
963
964	dt_irlist_append(dlp, dt_cg_node_alloc(lbl_post, DIF_INSTR_NOP));
965}
966
967static void
968dt_cg_logical_neg(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp)
969{
970	uint_t lbl_zero = dt_irlist_label(dlp);
971	uint_t lbl_post = dt_irlist_label(dlp);
972
973	dif_instr_t instr;
974
975	dt_cg_node(dnp->dn_child, dlp, drp);
976	dnp->dn_reg = dnp->dn_child->dn_reg;
977
978	instr = DIF_INSTR_TST(dnp->dn_reg);
979	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
980
981	instr = DIF_INSTR_BRANCH(DIF_OP_BE, lbl_zero);
982	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
983
984	instr = DIF_INSTR_MOV(DIF_REG_R0, dnp->dn_reg);
985	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
986
987	instr = DIF_INSTR_BRANCH(DIF_OP_BA, lbl_post);
988	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
989
990	dt_cg_xsetx(dlp, NULL, lbl_zero, dnp->dn_reg, 1);
991	dt_irlist_append(dlp, dt_cg_node_alloc(lbl_post, DIF_INSTR_NOP));
992}
993
994static void
995dt_cg_asgn_op(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp)
996{
997	dif_instr_t instr;
998	dt_ident_t *idp;
999
1000	/*
1001	 * If we are performing a structure assignment of a translated type,
1002	 * we must instantiate all members and create a snapshot of the object
1003	 * in scratch space.  We allocs a chunk of memory, generate code for
1004	 * each member, and then set dnp->dn_reg to the scratch object address.
1005	 */
1006	if ((idp = dt_node_resolve(dnp->dn_right, DT_IDENT_XLSOU)) != NULL) {
1007		ctf_membinfo_t ctm;
1008		dt_xlator_t *dxp = idp->di_data;
1009		dt_node_t *mnp, dn, mn;
1010		int r1, r2;
1011
1012		/*
1013		 * Create two fake dt_node_t's representing operator "." and a
1014		 * right-hand identifier child node.  These will be repeatedly
1015		 * modified according to each instantiated member so that we
1016		 * can pass them to dt_cg_store() and effect a member store.
1017		 */
1018		bzero(&dn, sizeof (dt_node_t));
1019		dn.dn_kind = DT_NODE_OP2;
1020		dn.dn_op = DT_TOK_DOT;
1021		dn.dn_left = dnp;
1022		dn.dn_right = &mn;
1023
1024		bzero(&mn, sizeof (dt_node_t));
1025		mn.dn_kind = DT_NODE_IDENT;
1026		mn.dn_op = DT_TOK_IDENT;
1027
1028		/*
1029		 * Allocate a register for our scratch data pointer.  First we
1030		 * set it to the size of our data structure, and then replace
1031		 * it with the result of an allocs of the specified size.
1032		 */
1033		r1 = dt_regset_alloc(drp);
1034		dt_cg_setx(dlp, r1,
1035		    ctf_type_size(dxp->dx_dst_ctfp, dxp->dx_dst_base));
1036
1037		instr = DIF_INSTR_ALLOCS(r1, r1);
1038		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1039
1040		/*
1041		 * When dt_cg_asgn_op() is called, we have already generated
1042		 * code for dnp->dn_right, which is the translator input.  We
1043		 * now associate this register with the translator's input
1044		 * identifier so it can be referenced during our member loop.
1045		 */
1046		dxp->dx_ident->di_flags |= DT_IDFLG_CGREG;
1047		dxp->dx_ident->di_id = dnp->dn_right->dn_reg;
1048
1049		for (mnp = dxp->dx_members; mnp != NULL; mnp = mnp->dn_list) {
1050			/*
1051			 * Generate code for the translator member expression,
1052			 * and then cast the result to the member type.
1053			 */
1054			dt_cg_node(mnp->dn_membexpr, dlp, drp);
1055			mnp->dn_reg = mnp->dn_membexpr->dn_reg;
1056			dt_cg_typecast(mnp->dn_membexpr, mnp, dlp, drp);
1057
1058			/*
1059			 * Ask CTF for the offset of the member so we can store
1060			 * to the appropriate offset.  This call has already
1061			 * been done once by the parser, so it should succeed.
1062			 */
1063			if (ctf_member_info(dxp->dx_dst_ctfp, dxp->dx_dst_base,
1064			    mnp->dn_membname, &ctm) == CTF_ERR) {
1065				yypcb->pcb_hdl->dt_ctferr =
1066				    ctf_errno(dxp->dx_dst_ctfp);
1067				longjmp(yypcb->pcb_jmpbuf, EDT_CTF);
1068			}
1069
1070			/*
1071			 * If the destination member is at offset 0, store the
1072			 * result directly to r1 (the scratch buffer address).
1073			 * Otherwise allocate another temporary for the offset
1074			 * and add r1 to it before storing the result.
1075			 */
1076			if (ctm.ctm_offset != 0) {
1077				r2 = dt_regset_alloc(drp);
1078
1079				/*
1080				 * Add the member offset rounded down to the
1081				 * nearest byte.  If the offset was not aligned
1082				 * on a byte boundary, this member is a bit-
1083				 * field and dt_cg_store() will handle masking.
1084				 */
1085				dt_cg_setx(dlp, r2, ctm.ctm_offset / NBBY);
1086				instr = DIF_INSTR_FMT(DIF_OP_ADD, r1, r2, r2);
1087				dt_irlist_append(dlp,
1088				    dt_cg_node_alloc(DT_LBL_NONE, instr));
1089
1090				dt_node_type_propagate(mnp, &dn);
1091				dn.dn_right->dn_string = mnp->dn_membname;
1092				dn.dn_reg = r2;
1093
1094				dt_cg_store(mnp, dlp, drp, &dn);
1095				dt_regset_free(drp, r2);
1096
1097			} else {
1098				dt_node_type_propagate(mnp, &dn);
1099				dn.dn_right->dn_string = mnp->dn_membname;
1100				dn.dn_reg = r1;
1101
1102				dt_cg_store(mnp, dlp, drp, &dn);
1103			}
1104
1105			dt_regset_free(drp, mnp->dn_reg);
1106		}
1107
1108		dxp->dx_ident->di_flags &= ~DT_IDFLG_CGREG;
1109		dxp->dx_ident->di_id = 0;
1110
1111		if (dnp->dn_right->dn_reg != -1)
1112			dt_regset_free(drp, dnp->dn_right->dn_reg);
1113
1114		assert(dnp->dn_reg == dnp->dn_right->dn_reg);
1115		dnp->dn_reg = r1;
1116	}
1117
1118	/*
1119	 * If we are storing to a memory address, generate code again for the
1120	 * left-hand side using DT_NF_REF to get the address, and then generate
1121	 * a store to it.
1122	 *
1123	 * Both here and the other variable-store paths, we assume dnp->dn_reg
1124	 * already has the new value.
1125	 */
1126	if (dnp->dn_left->dn_kind != DT_NODE_VAR) {
1127		uint_t rbit = dnp->dn_left->dn_flags & DT_NF_REF;
1128
1129		assert(dnp->dn_left->dn_flags & DT_NF_WRITABLE);
1130		assert(dnp->dn_left->dn_flags & DT_NF_LVALUE);
1131
1132		dnp->dn_left->dn_flags |= DT_NF_REF; /* force pass-by-ref */
1133
1134		dt_cg_node(dnp->dn_left, dlp, drp);
1135		dt_cg_store(dnp, dlp, drp, dnp->dn_left);
1136		dt_regset_free(drp, dnp->dn_left->dn_reg);
1137
1138		dnp->dn_left->dn_flags &= ~DT_NF_REF;
1139		dnp->dn_left->dn_flags |= rbit;
1140		return;
1141	}
1142
1143	idp = dt_ident_resolve(dnp->dn_left->dn_ident);
1144	idp->di_flags |= DT_IDFLG_DIFW;
1145
1146	/*
1147	 * Storing to an array variable is a special case.
1148	 * Only 'uregs[]' supports this for the time being.
1149	 */
1150	if (idp->di_kind == DT_IDENT_ARRAY &&
1151	    idp->di_id <= DIF_VAR_ARRAY_MAX) {
1152		dt_node_t *idx = dnp->dn_left->dn_args;
1153
1154		dt_cg_node(idx, dlp, drp);
1155		instr = DIF_INSTR_FMT(DIF_OP_STGA, idp->di_id, idx->dn_reg,
1156		    dnp->dn_reg);
1157		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1158		dt_regset_free(drp, idx->dn_reg);
1159		return;
1160	}
1161
1162	if (idp->di_kind == DT_IDENT_ARRAY)
1163		dt_cg_arglist(idp, dnp->dn_left->dn_args, dlp, drp);
1164
1165	instr = DIF_INSTR_STV(dt_cg_stvar(idp), idp->di_id, dnp->dn_reg);
1166	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1167}
1168
1169static void
1170dt_cg_assoc_op(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp)
1171{
1172	dif_instr_t instr;
1173	uint_t op;
1174
1175	assert(dnp->dn_kind == DT_NODE_VAR);
1176	assert(!(dnp->dn_ident->di_flags & DT_IDFLG_LOCAL));
1177	assert(dnp->dn_args != NULL);
1178
1179	dt_cg_arglist(dnp->dn_ident, dnp->dn_args, dlp, drp);
1180
1181	dnp->dn_reg = dt_regset_alloc(drp);
1182
1183	if (dnp->dn_ident->di_flags & DT_IDFLG_TLS)
1184		op = DIF_OP_LDTAA;
1185	else
1186		op = DIF_OP_LDGAA;
1187
1188	dnp->dn_ident->di_flags |= DT_IDFLG_DIFR;
1189	instr = DIF_INSTR_LDV(op, dnp->dn_ident->di_id, dnp->dn_reg);
1190	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1191
1192	/*
1193	 * If the associative array is a pass-by-reference type, then we are
1194	 * loading its value as a pointer to either load or store through it.
1195	 * The array element in question may not have been faulted in yet, in
1196	 * which case DIF_OP_LD*AA will return zero.  We append an epilogue
1197	 * of instructions similar to the following:
1198	 *
1199	 *	  ld?aa	 id, %r1	! base ld?aa instruction above
1200	 *	  tst	 %r1		! start of epilogue
1201	 *   +--- bne	 label
1202	 *   |    setx	 size, %r1
1203	 *   |    allocs %r1, %r1
1204	 *   |    st?aa	 id, %r1
1205	 *   |    ld?aa	 id, %r1
1206	 *   v
1207	 * label: < rest of code >
1208	 *
1209	 * The idea is that we allocs a zero-filled chunk of scratch space and
1210	 * do a DIF_OP_ST*AA to fault in and initialize the array element, and
1211	 * then reload it to get the faulted-in address of the new variable
1212	 * storage.  This isn't cheap, but pass-by-ref associative array values
1213	 * are (thus far) uncommon and the allocs cost only occurs once.  If
1214	 * this path becomes important to DTrace users, we can improve things
1215	 * by adding a new DIF opcode to fault in associative array elements.
1216	 */
1217	if (dnp->dn_flags & DT_NF_REF) {
1218		uint_t stvop = op == DIF_OP_LDTAA ? DIF_OP_STTAA : DIF_OP_STGAA;
1219		uint_t label = dt_irlist_label(dlp);
1220
1221		instr = DIF_INSTR_TST(dnp->dn_reg);
1222		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1223
1224		instr = DIF_INSTR_BRANCH(DIF_OP_BNE, label);
1225		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1226
1227		dt_cg_setx(dlp, dnp->dn_reg, dt_node_type_size(dnp));
1228		instr = DIF_INSTR_ALLOCS(dnp->dn_reg, dnp->dn_reg);
1229		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1230
1231		dnp->dn_ident->di_flags |= DT_IDFLG_DIFW;
1232		instr = DIF_INSTR_STV(stvop, dnp->dn_ident->di_id, dnp->dn_reg);
1233		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1234
1235		instr = DIF_INSTR_LDV(op, dnp->dn_ident->di_id, dnp->dn_reg);
1236		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1237
1238		dt_irlist_append(dlp, dt_cg_node_alloc(label, DIF_INSTR_NOP));
1239	}
1240}
1241
1242static void
1243dt_cg_array_op(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp)
1244{
1245	dt_probe_t *prp = yypcb->pcb_probe;
1246	uintmax_t saved = dnp->dn_args->dn_value;
1247	dt_ident_t *idp = dnp->dn_ident;
1248
1249	dif_instr_t instr;
1250	uint_t op;
1251	size_t size;
1252	int reg, n;
1253
1254	assert(dnp->dn_kind == DT_NODE_VAR);
1255	assert(!(idp->di_flags & DT_IDFLG_LOCAL));
1256
1257	assert(dnp->dn_args->dn_kind == DT_NODE_INT);
1258	assert(dnp->dn_args->dn_list == NULL);
1259
1260	/*
1261	 * If this is a reference in the args[] array, temporarily modify the
1262	 * array index according to the static argument mapping (if any),
1263	 * unless the argument reference is provided by a dynamic translator.
1264	 * If we're using a dynamic translator for args[], then just set dn_reg
1265	 * to an invalid reg and return: DIF_OP_XLARG will fetch the arg later.
1266	 */
1267	if (idp->di_id == DIF_VAR_ARGS) {
1268		if ((idp->di_kind == DT_IDENT_XLPTR ||
1269		    idp->di_kind == DT_IDENT_XLSOU) &&
1270		    dt_xlator_dynamic(idp->di_data)) {
1271			dnp->dn_reg = -1;
1272			return;
1273		}
1274		dnp->dn_args->dn_value = prp->pr_mapping[saved];
1275	}
1276
1277	dt_cg_node(dnp->dn_args, dlp, drp);
1278	dnp->dn_args->dn_value = saved;
1279
1280	dnp->dn_reg = dnp->dn_args->dn_reg;
1281
1282	if (idp->di_flags & DT_IDFLG_TLS)
1283		op = DIF_OP_LDTA;
1284	else
1285		op = DIF_OP_LDGA;
1286
1287	idp->di_flags |= DT_IDFLG_DIFR;
1288
1289	instr = DIF_INSTR_LDA(op, idp->di_id,
1290	    dnp->dn_args->dn_reg, dnp->dn_reg);
1291
1292	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1293
1294	/*
1295	 * If this is a reference to the args[] array, we need to take the
1296	 * additional step of explicitly eliminating any bits larger than the
1297	 * type size: the DIF interpreter in the kernel will always give us
1298	 * the raw (64-bit) argument value, and any bits larger than the type
1299	 * size may be junk.  As a practical matter, this arises only on 64-bit
1300	 * architectures and only when the argument index is larger than the
1301	 * number of arguments passed directly to DTrace: if a 8-, 16- or
1302	 * 32-bit argument must be retrieved from the stack, it is possible
1303	 * (and it some cases, likely) that the upper bits will be garbage.
1304	 */
1305	if (idp->di_id != DIF_VAR_ARGS || !dt_node_is_scalar(dnp))
1306		return;
1307
1308	if ((size = dt_node_type_size(dnp)) == sizeof (uint64_t))
1309		return;
1310
1311	reg = dt_regset_alloc(drp);
1312	assert(size < sizeof (uint64_t));
1313	n = sizeof (uint64_t) * NBBY - size * NBBY;
1314
1315	dt_cg_setx(dlp, reg, n);
1316
1317	instr = DIF_INSTR_FMT(DIF_OP_SLL, dnp->dn_reg, reg, dnp->dn_reg);
1318	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1319
1320	instr = DIF_INSTR_FMT((dnp->dn_flags & DT_NF_SIGNED) ?
1321	    DIF_OP_SRA : DIF_OP_SRL, dnp->dn_reg, reg, dnp->dn_reg);
1322
1323	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1324	dt_regset_free(drp, reg);
1325}
1326
1327/*
1328 * Generate code for an inlined variable reference.  Inlines can be used to
1329 * define either scalar or associative array substitutions.  For scalars, we
1330 * simply generate code for the parse tree saved in the identifier's din_root,
1331 * and then cast the resulting expression to the inline's declaration type.
1332 * For arrays, we take the input parameter subtrees from dnp->dn_args and
1333 * temporarily store them in the din_root of each din_argv[i] identifier,
1334 * which are themselves inlines and were set up for us by the parser.  The
1335 * result is that any reference to the inlined parameter inside the top-level
1336 * din_root will turn into a recursive call to dt_cg_inline() for a scalar
1337 * inline whose din_root will refer to the subtree pointed to by the argument.
1338 */
1339static void
1340dt_cg_inline(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp)
1341{
1342	dt_ident_t *idp = dnp->dn_ident;
1343	dt_idnode_t *inp = idp->di_iarg;
1344
1345	dt_idnode_t *pinp;
1346	dt_node_t *pnp;
1347	int i;
1348
1349	assert(idp->di_flags & DT_IDFLG_INLINE);
1350	assert(idp->di_ops == &dt_idops_inline);
1351
1352	if (idp->di_kind == DT_IDENT_ARRAY) {
1353		for (i = 0, pnp = dnp->dn_args;
1354		    pnp != NULL; pnp = pnp->dn_list, i++) {
1355			if (inp->din_argv[i] != NULL) {
1356				pinp = inp->din_argv[i]->di_iarg;
1357				pinp->din_root = pnp;
1358			}
1359		}
1360	}
1361
1362	dt_cg_node(inp->din_root, dlp, drp);
1363	dnp->dn_reg = inp->din_root->dn_reg;
1364	dt_cg_typecast(inp->din_root, dnp, dlp, drp);
1365
1366	if (idp->di_kind == DT_IDENT_ARRAY) {
1367		for (i = 0; i < inp->din_argc; i++) {
1368			pinp = inp->din_argv[i]->di_iarg;
1369			pinp->din_root = NULL;
1370		}
1371	}
1372}
1373
1374typedef struct dt_xlmemb {
1375	dt_ident_t *dtxl_idp;		/* translated ident */
1376	dt_irlist_t *dtxl_dlp;		/* instruction list */
1377	dt_regset_t *dtxl_drp;		/* register set */
1378	int dtxl_sreg;			/* location of the translation input */
1379	int dtxl_dreg;			/* location of our allocated buffer */
1380} dt_xlmemb_t;
1381
1382/*ARGSUSED*/
1383static int
1384dt_cg_xlate_member(const char *name, ctf_id_t type, ulong_t off, void *arg)
1385{
1386	dt_xlmemb_t *dx = arg;
1387	dt_ident_t *idp = dx->dtxl_idp;
1388	dt_irlist_t *dlp = dx->dtxl_dlp;
1389	dt_regset_t *drp = dx->dtxl_drp;
1390
1391	dt_node_t *mnp;
1392	dt_xlator_t *dxp;
1393
1394	int reg, treg;
1395	uint32_t instr;
1396	size_t size;
1397
1398	/* Generate code for the translation. */
1399	dxp = idp->di_data;
1400	mnp = dt_xlator_member(dxp, name);
1401
1402	/* If there's no translator for the given member, skip it. */
1403	if (mnp == NULL)
1404		return (0);
1405
1406	dxp->dx_ident->di_flags |= DT_IDFLG_CGREG;
1407	dxp->dx_ident->di_id = dx->dtxl_sreg;
1408
1409	dt_cg_node(mnp->dn_membexpr, dlp, drp);
1410
1411	dxp->dx_ident->di_flags &= ~DT_IDFLG_CGREG;
1412	dxp->dx_ident->di_id = 0;
1413
1414	treg = mnp->dn_membexpr->dn_reg;
1415
1416	/* Compute the offset into our buffer and store the result there. */
1417	reg = dt_regset_alloc(drp);
1418
1419	dt_cg_setx(dlp, reg, off / NBBY);
1420	instr = DIF_INSTR_FMT(DIF_OP_ADD, dx->dtxl_dreg, reg, reg);
1421	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1422
1423	size = ctf_type_size(mnp->dn_membexpr->dn_ctfp,
1424	    mnp->dn_membexpr->dn_type);
1425	if (dt_node_is_scalar(mnp->dn_membexpr)) {
1426		/*
1427		 * Copying scalars is simple.
1428		 */
1429		switch (size) {
1430		case 1:
1431			instr = DIF_INSTR_STORE(DIF_OP_STB, treg, reg);
1432			break;
1433		case 2:
1434			instr = DIF_INSTR_STORE(DIF_OP_STH, treg, reg);
1435			break;
1436		case 4:
1437			instr = DIF_INSTR_STORE(DIF_OP_STW, treg, reg);
1438			break;
1439		case 8:
1440			instr = DIF_INSTR_STORE(DIF_OP_STX, treg, reg);
1441			break;
1442		default:
1443			xyerror(D_UNKNOWN, "internal error -- unexpected "
1444			    "size: %lu\n", (ulong_t)size);
1445		}
1446
1447		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1448
1449	} else if (dt_node_is_string(mnp->dn_membexpr)) {
1450		int szreg;
1451
1452		/*
1453		 * Use the copys instruction for strings.
1454		 */
1455		szreg = dt_regset_alloc(drp);
1456		dt_cg_setx(dlp, szreg, size);
1457		instr = DIF_INSTR_COPYS(treg, szreg, reg);
1458		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1459		dt_regset_free(drp, szreg);
1460	} else {
1461		int szreg;
1462
1463		/*
1464		 * If it's anything else then we'll just bcopy it.
1465		 */
1466		szreg = dt_regset_alloc(drp);
1467		dt_cg_setx(dlp, szreg, size);
1468		dt_irlist_append(dlp,
1469		    dt_cg_node_alloc(DT_LBL_NONE, DIF_INSTR_FLUSHTS));
1470		instr = DIF_INSTR_PUSHTS(DIF_OP_PUSHTV, DIF_TYPE_CTF,
1471		    DIF_REG_R0, treg);
1472		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1473		instr = DIF_INSTR_PUSHTS(DIF_OP_PUSHTV, DIF_TYPE_CTF,
1474		    DIF_REG_R0, reg);
1475		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1476		instr = DIF_INSTR_PUSHTS(DIF_OP_PUSHTV, DIF_TYPE_CTF,
1477		    DIF_REG_R0, szreg);
1478		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1479		instr = DIF_INSTR_CALL(DIF_SUBR_BCOPY, szreg);
1480		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1481		dt_regset_free(drp, szreg);
1482	}
1483
1484	dt_regset_free(drp, reg);
1485	dt_regset_free(drp, treg);
1486
1487	return (0);
1488}
1489
1490/*
1491 * If we're expanding a translated type, we create an appropriately sized
1492 * buffer with alloca() and then translate each member into it.
1493 */
1494static int
1495dt_cg_xlate_expand(dt_node_t *dnp, dt_ident_t *idp, dt_irlist_t *dlp,
1496    dt_regset_t *drp)
1497{
1498	dt_xlmemb_t dlm;
1499	uint32_t instr;
1500	int dreg;
1501	size_t size;
1502
1503	dreg = dt_regset_alloc(drp);
1504	size = ctf_type_size(dnp->dn_ident->di_ctfp, dnp->dn_ident->di_type);
1505
1506	/* Call alloca() to create the buffer. */
1507	dt_cg_setx(dlp, dreg, size);
1508
1509	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, DIF_INSTR_FLUSHTS));
1510
1511	instr = DIF_INSTR_PUSHTS(DIF_OP_PUSHTV, DIF_TYPE_CTF, DIF_REG_R0, dreg);
1512	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1513
1514	instr = DIF_INSTR_CALL(DIF_SUBR_ALLOCA, dreg);
1515	dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1516
1517	/* Generate the translation for each member. */
1518	dlm.dtxl_idp = idp;
1519	dlm.dtxl_dlp = dlp;
1520	dlm.dtxl_drp = drp;
1521	dlm.dtxl_sreg = dnp->dn_reg;
1522	dlm.dtxl_dreg = dreg;
1523	(void) ctf_member_iter(dnp->dn_ident->di_ctfp,
1524	    dnp->dn_ident->di_type, dt_cg_xlate_member,
1525	    &dlm);
1526
1527	return (dreg);
1528}
1529
1530static void
1531dt_cg_node(dt_node_t *dnp, dt_irlist_t *dlp, dt_regset_t *drp)
1532{
1533	ctf_file_t *ctfp = dnp->dn_ctfp;
1534	ctf_file_t *octfp;
1535	ctf_membinfo_t m;
1536	ctf_id_t type;
1537
1538	dif_instr_t instr;
1539	dt_ident_t *idp;
1540	ssize_t stroff;
1541	uint_t op;
1542
1543	switch (dnp->dn_op) {
1544	case DT_TOK_COMMA:
1545		dt_cg_node(dnp->dn_left, dlp, drp);
1546		dt_regset_free(drp, dnp->dn_left->dn_reg);
1547		dt_cg_node(dnp->dn_right, dlp, drp);
1548		dnp->dn_reg = dnp->dn_right->dn_reg;
1549		break;
1550
1551	case DT_TOK_ASGN:
1552		dt_cg_node(dnp->dn_right, dlp, drp);
1553		dnp->dn_reg = dnp->dn_right->dn_reg;
1554		dt_cg_asgn_op(dnp, dlp, drp);
1555		break;
1556
1557	case DT_TOK_ADD_EQ:
1558		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_ADD);
1559		dt_cg_asgn_op(dnp, dlp, drp);
1560		break;
1561
1562	case DT_TOK_SUB_EQ:
1563		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_SUB);
1564		dt_cg_asgn_op(dnp, dlp, drp);
1565		break;
1566
1567	case DT_TOK_MUL_EQ:
1568		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_MUL);
1569		dt_cg_asgn_op(dnp, dlp, drp);
1570		break;
1571
1572	case DT_TOK_DIV_EQ:
1573		dt_cg_arithmetic_op(dnp, dlp, drp,
1574		    (dnp->dn_flags & DT_NF_SIGNED) ? DIF_OP_SDIV : DIF_OP_UDIV);
1575		dt_cg_asgn_op(dnp, dlp, drp);
1576		break;
1577
1578	case DT_TOK_MOD_EQ:
1579		dt_cg_arithmetic_op(dnp, dlp, drp,
1580		    (dnp->dn_flags & DT_NF_SIGNED) ? DIF_OP_SREM : DIF_OP_UREM);
1581		dt_cg_asgn_op(dnp, dlp, drp);
1582		break;
1583
1584	case DT_TOK_AND_EQ:
1585		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_AND);
1586		dt_cg_asgn_op(dnp, dlp, drp);
1587		break;
1588
1589	case DT_TOK_XOR_EQ:
1590		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_XOR);
1591		dt_cg_asgn_op(dnp, dlp, drp);
1592		break;
1593
1594	case DT_TOK_OR_EQ:
1595		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_OR);
1596		dt_cg_asgn_op(dnp, dlp, drp);
1597		break;
1598
1599	case DT_TOK_LSH_EQ:
1600		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_SLL);
1601		dt_cg_asgn_op(dnp, dlp, drp);
1602		break;
1603
1604	case DT_TOK_RSH_EQ:
1605		dt_cg_arithmetic_op(dnp, dlp, drp,
1606		    (dnp->dn_flags & DT_NF_SIGNED) ? DIF_OP_SRA : DIF_OP_SRL);
1607		dt_cg_asgn_op(dnp, dlp, drp);
1608		break;
1609
1610	case DT_TOK_QUESTION:
1611		dt_cg_ternary_op(dnp, dlp, drp);
1612		break;
1613
1614	case DT_TOK_LOR:
1615		dt_cg_logical_or(dnp, dlp, drp);
1616		break;
1617
1618	case DT_TOK_LXOR:
1619		dt_cg_logical_xor(dnp, dlp, drp);
1620		break;
1621
1622	case DT_TOK_LAND:
1623		dt_cg_logical_and(dnp, dlp, drp);
1624		break;
1625
1626	case DT_TOK_BOR:
1627		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_OR);
1628		break;
1629
1630	case DT_TOK_XOR:
1631		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_XOR);
1632		break;
1633
1634	case DT_TOK_BAND:
1635		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_AND);
1636		break;
1637
1638	case DT_TOK_EQU:
1639		dt_cg_compare_op(dnp, dlp, drp, DIF_OP_BE);
1640		break;
1641
1642	case DT_TOK_NEQ:
1643		dt_cg_compare_op(dnp, dlp, drp, DIF_OP_BNE);
1644		break;
1645
1646	case DT_TOK_LT:
1647		dt_cg_compare_op(dnp, dlp, drp,
1648		    dt_cg_compare_signed(dnp) ? DIF_OP_BL : DIF_OP_BLU);
1649		break;
1650
1651	case DT_TOK_LE:
1652		dt_cg_compare_op(dnp, dlp, drp,
1653		    dt_cg_compare_signed(dnp) ? DIF_OP_BLE : DIF_OP_BLEU);
1654		break;
1655
1656	case DT_TOK_GT:
1657		dt_cg_compare_op(dnp, dlp, drp,
1658		    dt_cg_compare_signed(dnp) ? DIF_OP_BG : DIF_OP_BGU);
1659		break;
1660
1661	case DT_TOK_GE:
1662		dt_cg_compare_op(dnp, dlp, drp,
1663		    dt_cg_compare_signed(dnp) ? DIF_OP_BGE : DIF_OP_BGEU);
1664		break;
1665
1666	case DT_TOK_LSH:
1667		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_SLL);
1668		break;
1669
1670	case DT_TOK_RSH:
1671		dt_cg_arithmetic_op(dnp, dlp, drp,
1672		    (dnp->dn_flags & DT_NF_SIGNED) ? DIF_OP_SRA : DIF_OP_SRL);
1673		break;
1674
1675	case DT_TOK_ADD:
1676		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_ADD);
1677		break;
1678
1679	case DT_TOK_SUB:
1680		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_SUB);
1681		break;
1682
1683	case DT_TOK_MUL:
1684		dt_cg_arithmetic_op(dnp, dlp, drp, DIF_OP_MUL);
1685		break;
1686
1687	case DT_TOK_DIV:
1688		dt_cg_arithmetic_op(dnp, dlp, drp,
1689		    (dnp->dn_flags & DT_NF_SIGNED) ? DIF_OP_SDIV : DIF_OP_UDIV);
1690		break;
1691
1692	case DT_TOK_MOD:
1693		dt_cg_arithmetic_op(dnp, dlp, drp,
1694		    (dnp->dn_flags & DT_NF_SIGNED) ? DIF_OP_SREM : DIF_OP_UREM);
1695		break;
1696
1697	case DT_TOK_LNEG:
1698		dt_cg_logical_neg(dnp, dlp, drp);
1699		break;
1700
1701	case DT_TOK_BNEG:
1702		dt_cg_node(dnp->dn_child, dlp, drp);
1703		dnp->dn_reg = dnp->dn_child->dn_reg;
1704		instr = DIF_INSTR_NOT(dnp->dn_reg, dnp->dn_reg);
1705		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1706		break;
1707
1708	case DT_TOK_PREINC:
1709		dt_cg_prearith_op(dnp, dlp, drp, DIF_OP_ADD);
1710		break;
1711
1712	case DT_TOK_POSTINC:
1713		dt_cg_postarith_op(dnp, dlp, drp, DIF_OP_ADD);
1714		break;
1715
1716	case DT_TOK_PREDEC:
1717		dt_cg_prearith_op(dnp, dlp, drp, DIF_OP_SUB);
1718		break;
1719
1720	case DT_TOK_POSTDEC:
1721		dt_cg_postarith_op(dnp, dlp, drp, DIF_OP_SUB);
1722		break;
1723
1724	case DT_TOK_IPOS:
1725		dt_cg_node(dnp->dn_child, dlp, drp);
1726		dnp->dn_reg = dnp->dn_child->dn_reg;
1727		break;
1728
1729	case DT_TOK_INEG:
1730		dt_cg_node(dnp->dn_child, dlp, drp);
1731		dnp->dn_reg = dnp->dn_child->dn_reg;
1732
1733		instr = DIF_INSTR_FMT(DIF_OP_SUB, DIF_REG_R0,
1734		    dnp->dn_reg, dnp->dn_reg);
1735
1736		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1737		break;
1738
1739	case DT_TOK_DEREF:
1740		dt_cg_node(dnp->dn_child, dlp, drp);
1741		dnp->dn_reg = dnp->dn_child->dn_reg;
1742
1743		if (dt_node_is_dynamic(dnp->dn_child)) {
1744			int reg;
1745			idp = dt_node_resolve(dnp->dn_child, DT_IDENT_XLPTR);
1746			assert(idp != NULL);
1747			reg = dt_cg_xlate_expand(dnp, idp, dlp, drp);
1748
1749			dt_regset_free(drp, dnp->dn_child->dn_reg);
1750			dnp->dn_reg = reg;
1751
1752		} else if (!(dnp->dn_flags & DT_NF_REF)) {
1753			uint_t ubit = dnp->dn_flags & DT_NF_USERLAND;
1754
1755			/*
1756			 * Save and restore DT_NF_USERLAND across dt_cg_load():
1757			 * we need the sign bit from dnp and the user bit from
1758			 * dnp->dn_child in order to get the proper opcode.
1759			 */
1760			dnp->dn_flags |=
1761			    (dnp->dn_child->dn_flags & DT_NF_USERLAND);
1762
1763			instr = DIF_INSTR_LOAD(dt_cg_load(dnp, ctfp,
1764			    dnp->dn_type), dnp->dn_reg, dnp->dn_reg);
1765
1766			dnp->dn_flags &= ~DT_NF_USERLAND;
1767			dnp->dn_flags |= ubit;
1768
1769			dt_irlist_append(dlp,
1770			    dt_cg_node_alloc(DT_LBL_NONE, instr));
1771		}
1772		break;
1773
1774	case DT_TOK_ADDROF: {
1775		uint_t rbit = dnp->dn_child->dn_flags & DT_NF_REF;
1776
1777		dnp->dn_child->dn_flags |= DT_NF_REF; /* force pass-by-ref */
1778		dt_cg_node(dnp->dn_child, dlp, drp);
1779		dnp->dn_reg = dnp->dn_child->dn_reg;
1780
1781		dnp->dn_child->dn_flags &= ~DT_NF_REF;
1782		dnp->dn_child->dn_flags |= rbit;
1783		break;
1784	}
1785
1786	case DT_TOK_SIZEOF: {
1787		size_t size = dt_node_sizeof(dnp->dn_child);
1788		dnp->dn_reg = dt_regset_alloc(drp);
1789		assert(size != 0);
1790		dt_cg_setx(dlp, dnp->dn_reg, size);
1791		break;
1792	}
1793
1794	case DT_TOK_STRINGOF:
1795		dt_cg_node(dnp->dn_child, dlp, drp);
1796		dnp->dn_reg = dnp->dn_child->dn_reg;
1797		break;
1798
1799	case DT_TOK_XLATE:
1800		/*
1801		 * An xlate operator appears in either an XLATOR, indicating a
1802		 * reference to a dynamic translator, or an OP2, indicating
1803		 * use of the xlate operator in the user's program.  For the
1804		 * dynamic case, generate an xlate opcode with a reference to
1805		 * the corresponding member, pre-computed for us in dn_members.
1806		 */
1807		if (dnp->dn_kind == DT_NODE_XLATOR) {
1808			dt_xlator_t *dxp = dnp->dn_xlator;
1809
1810			assert(dxp->dx_ident->di_flags & DT_IDFLG_CGREG);
1811			assert(dxp->dx_ident->di_id != 0);
1812
1813			dnp->dn_reg = dt_regset_alloc(drp);
1814
1815			if (dxp->dx_arg == -1) {
1816				instr = DIF_INSTR_MOV(
1817				    dxp->dx_ident->di_id, dnp->dn_reg);
1818				dt_irlist_append(dlp,
1819				    dt_cg_node_alloc(DT_LBL_NONE, instr));
1820				op = DIF_OP_XLATE;
1821			} else
1822				op = DIF_OP_XLARG;
1823
1824			instr = DIF_INSTR_XLATE(op, 0, dnp->dn_reg);
1825			dt_irlist_append(dlp,
1826			    dt_cg_node_alloc(DT_LBL_NONE, instr));
1827
1828			dlp->dl_last->di_extern = dnp->dn_xmember;
1829			break;
1830		}
1831
1832		assert(dnp->dn_kind == DT_NODE_OP2);
1833		dt_cg_node(dnp->dn_right, dlp, drp);
1834		dnp->dn_reg = dnp->dn_right->dn_reg;
1835		break;
1836
1837	case DT_TOK_LPAR:
1838		dt_cg_node(dnp->dn_right, dlp, drp);
1839		dnp->dn_reg = dnp->dn_right->dn_reg;
1840		dt_cg_typecast(dnp->dn_right, dnp, dlp, drp);
1841		break;
1842
1843	case DT_TOK_PTR:
1844	case DT_TOK_DOT:
1845		assert(dnp->dn_right->dn_kind == DT_NODE_IDENT);
1846		dt_cg_node(dnp->dn_left, dlp, drp);
1847
1848		/*
1849		 * If the left-hand side of PTR or DOT is a dynamic variable,
1850		 * we expect it to be the output of a D translator.   In this
1851		 * case, we look up the parse tree corresponding to the member
1852		 * that is being accessed and run the code generator over it.
1853		 * We then cast the result as if by the assignment operator.
1854		 */
1855		if ((idp = dt_node_resolve(
1856		    dnp->dn_left, DT_IDENT_XLSOU)) != NULL ||
1857		    (idp = dt_node_resolve(
1858		    dnp->dn_left, DT_IDENT_XLPTR)) != NULL) {
1859
1860			dt_xlator_t *dxp;
1861			dt_node_t *mnp;
1862
1863			dxp = idp->di_data;
1864			mnp = dt_xlator_member(dxp, dnp->dn_right->dn_string);
1865			assert(mnp != NULL);
1866
1867			dxp->dx_ident->di_flags |= DT_IDFLG_CGREG;
1868			dxp->dx_ident->di_id = dnp->dn_left->dn_reg;
1869
1870			dt_cg_node(mnp->dn_membexpr, dlp, drp);
1871			dnp->dn_reg = mnp->dn_membexpr->dn_reg;
1872			dt_cg_typecast(mnp->dn_membexpr, dnp, dlp, drp);
1873
1874			dxp->dx_ident->di_flags &= ~DT_IDFLG_CGREG;
1875			dxp->dx_ident->di_id = 0;
1876
1877			if (dnp->dn_left->dn_reg != -1)
1878				dt_regset_free(drp, dnp->dn_left->dn_reg);
1879			break;
1880		}
1881
1882		ctfp = dnp->dn_left->dn_ctfp;
1883		type = ctf_type_resolve(ctfp, dnp->dn_left->dn_type);
1884
1885		if (dnp->dn_op == DT_TOK_PTR) {
1886			type = ctf_type_reference(ctfp, type);
1887			type = ctf_type_resolve(ctfp, type);
1888		}
1889
1890		if ((ctfp = dt_cg_membinfo(octfp = ctfp, type,
1891		    dnp->dn_right->dn_string, &m)) == NULL) {
1892			yypcb->pcb_hdl->dt_ctferr = ctf_errno(octfp);
1893			longjmp(yypcb->pcb_jmpbuf, EDT_CTF);
1894		}
1895
1896		if (m.ctm_offset != 0) {
1897			int reg;
1898
1899			reg = dt_regset_alloc(drp);
1900
1901			/*
1902			 * If the offset is not aligned on a byte boundary, it
1903			 * is a bit-field member and we will extract the value
1904			 * bits below after we generate the appropriate load.
1905			 */
1906			dt_cg_setx(dlp, reg, m.ctm_offset / NBBY);
1907
1908			instr = DIF_INSTR_FMT(DIF_OP_ADD,
1909			    dnp->dn_left->dn_reg, reg, dnp->dn_left->dn_reg);
1910
1911			dt_irlist_append(dlp,
1912			    dt_cg_node_alloc(DT_LBL_NONE, instr));
1913			dt_regset_free(drp, reg);
1914		}
1915
1916		if (!(dnp->dn_flags & DT_NF_REF)) {
1917			uint_t ubit = dnp->dn_flags & DT_NF_USERLAND;
1918
1919			/*
1920			 * Save and restore DT_NF_USERLAND across dt_cg_load():
1921			 * we need the sign bit from dnp and the user bit from
1922			 * dnp->dn_left in order to get the proper opcode.
1923			 */
1924			dnp->dn_flags |=
1925			    (dnp->dn_left->dn_flags & DT_NF_USERLAND);
1926
1927			instr = DIF_INSTR_LOAD(dt_cg_load(dnp,
1928			    ctfp, m.ctm_type), dnp->dn_left->dn_reg,
1929			    dnp->dn_left->dn_reg);
1930
1931			dnp->dn_flags &= ~DT_NF_USERLAND;
1932			dnp->dn_flags |= ubit;
1933
1934			dt_irlist_append(dlp,
1935			    dt_cg_node_alloc(DT_LBL_NONE, instr));
1936
1937			if (dnp->dn_flags & DT_NF_BITFIELD)
1938				dt_cg_field_get(dnp, dlp, drp, ctfp, &m);
1939		}
1940
1941		dnp->dn_reg = dnp->dn_left->dn_reg;
1942		break;
1943
1944	case DT_TOK_STRING:
1945		dnp->dn_reg = dt_regset_alloc(drp);
1946
1947		assert(dnp->dn_kind == DT_NODE_STRING);
1948		stroff = dt_strtab_insert(yypcb->pcb_strtab, dnp->dn_string);
1949
1950		if (stroff == -1L)
1951			longjmp(yypcb->pcb_jmpbuf, EDT_NOMEM);
1952		if (stroff > DIF_STROFF_MAX)
1953			longjmp(yypcb->pcb_jmpbuf, EDT_STR2BIG);
1954
1955		instr = DIF_INSTR_SETS((ulong_t)stroff, dnp->dn_reg);
1956		dt_irlist_append(dlp, dt_cg_node_alloc(DT_LBL_NONE, instr));
1957		break;
1958
1959	case DT_TOK_IDENT:
1960		/*
1961		 * If the specified identifier is a variable on which we have
1962		 * set the code generator register flag, then this variable
1963		 * has already had code generated for it and saved in di_id.
1964		 * Allocate a new register and copy the existing value to it.
1965		 */
1966		if (dnp->dn_kind == DT_NODE_VAR &&
1967		    (dnp->dn_ident->di_flags & DT_IDFLG_CGREG)) {
1968			dnp->dn_reg = dt_regset_alloc(drp);
1969			instr = DIF_INSTR_MOV(dnp->dn_ident->di_id,
1970			    dnp->dn_reg);
1971			dt_irlist_append(dlp,
1972			    dt_cg_node_alloc(DT_LBL_NONE, instr));
1973			break;
1974		}
1975
1976		/*
1977		 * Identifiers can represent function calls, variable refs, or
1978		 * symbols.  First we check for inlined variables, and handle
1979		 * them by generating code for the inline parse tree.
1980		 */
1981		if (dnp->dn_kind == DT_NODE_VAR &&
1982		    (dnp->dn_ident->di_flags & DT_IDFLG_INLINE)) {
1983			dt_cg_inline(dnp, dlp, drp);
1984			break;
1985		}
1986
1987		switch (dnp->dn_kind) {
1988		case DT_NODE_FUNC:
1989			if ((idp = dnp->dn_ident)->di_kind != DT_IDENT_FUNC) {
1990				dnerror(dnp, D_CG_EXPR, "%s %s( ) may not be "
1991				    "called from a D expression (D program "
1992				    "context required)\n",
1993				    dt_idkind_name(idp->di_kind), idp->di_name);
1994			}
1995
1996			dt_cg_arglist(dnp->dn_ident, dnp->dn_args, dlp, drp);
1997
1998			dnp->dn_reg = dt_regset_alloc(drp);
1999			instr = DIF_INSTR_CALL(dnp->dn_ident->di_id,
2000			    dnp->dn_reg);
2001
2002			dt_irlist_append(dlp,
2003			    dt_cg_node_alloc(DT_LBL_NONE, instr));
2004
2005			break;
2006
2007		case DT_NODE_VAR:
2008			if (dnp->dn_ident->di_kind == DT_IDENT_XLSOU ||
2009			    dnp->dn_ident->di_kind == DT_IDENT_XLPTR) {
2010				/*
2011				 * This can only happen if we have translated
2012				 * args[].  See dt_idcook_args() for details.
2013				 */
2014				assert(dnp->dn_ident->di_id == DIF_VAR_ARGS);
2015				dt_cg_array_op(dnp, dlp, drp);
2016				break;
2017			}
2018
2019			if (dnp->dn_ident->di_kind == DT_IDENT_ARRAY) {
2020				if (dnp->dn_ident->di_id > DIF_VAR_ARRAY_MAX)
2021					dt_cg_assoc_op(dnp, dlp, drp);
2022				else
2023					dt_cg_array_op(dnp, dlp, drp);
2024				break;
2025			}
2026
2027			dnp->dn_reg = dt_regset_alloc(drp);
2028
2029			if (dnp->dn_ident->di_flags & DT_IDFLG_LOCAL)
2030				op = DIF_OP_LDLS;
2031			else if (dnp->dn_ident->di_flags & DT_IDFLG_TLS)
2032				op = DIF_OP_LDTS;
2033			else
2034				op = DIF_OP_LDGS;
2035
2036			dnp->dn_ident->di_flags |= DT_IDFLG_DIFR;
2037
2038			instr = DIF_INSTR_LDV(op,
2039			    dnp->dn_ident->di_id, dnp->dn_reg);
2040
2041			dt_irlist_append(dlp,
2042			    dt_cg_node_alloc(DT_LBL_NONE, instr));
2043			break;
2044
2045		case DT_NODE_SYM: {
2046			dtrace_hdl_t *dtp = yypcb->pcb_hdl;
2047			dtrace_syminfo_t *sip = dnp->dn_ident->di_data;
2048			GElf_Sym sym;
2049
2050			if (dtrace_lookup_by_name(dtp,
2051			    sip->dts_object, sip->dts_name, &sym, NULL) == -1) {
2052				xyerror(D_UNKNOWN, "cg failed for symbol %s`%s:"
2053				    " %s\n", sip->dts_object, sip->dts_name,
2054				    dtrace_errmsg(dtp, dtrace_errno(dtp)));
2055			}
2056
2057			dnp->dn_reg = dt_regset_alloc(drp);
2058			dt_cg_xsetx(dlp, dnp->dn_ident,
2059			    DT_LBL_NONE, dnp->dn_reg, sym.st_value);
2060
2061			if (!(dnp->dn_flags & DT_NF_REF)) {
2062				instr = DIF_INSTR_LOAD(dt_cg_load(dnp, ctfp,
2063				    dnp->dn_type), dnp->dn_reg, dnp->dn_reg);
2064				dt_irlist_append(dlp,
2065				    dt_cg_node_alloc(DT_LBL_NONE, instr));
2066			}
2067			break;
2068		}
2069
2070		default:
2071			xyerror(D_UNKNOWN, "internal error -- node type %u is "
2072			    "not valid for an identifier\n", dnp->dn_kind);
2073		}
2074		break;
2075
2076	case DT_TOK_INT:
2077		dnp->dn_reg = dt_regset_alloc(drp);
2078		dt_cg_setx(dlp, dnp->dn_reg, dnp->dn_value);
2079		break;
2080
2081	default:
2082		xyerror(D_UNKNOWN, "internal error -- token type %u is not a "
2083		    "valid D compilation token\n", dnp->dn_op);
2084	}
2085}
2086
2087void
2088dt_cg(dt_pcb_t *pcb, dt_node_t *dnp)
2089{
2090	dif_instr_t instr;
2091	dt_xlator_t *dxp;
2092	dt_ident_t *idp;
2093
2094	if (pcb->pcb_regs == NULL && (pcb->pcb_regs =
2095	    dt_regset_create(pcb->pcb_hdl->dt_conf.dtc_difintregs)) == NULL)
2096		longjmp(pcb->pcb_jmpbuf, EDT_NOMEM);
2097
2098	dt_regset_reset(pcb->pcb_regs);
2099	(void) dt_regset_alloc(pcb->pcb_regs); /* allocate %r0 */
2100
2101	if (pcb->pcb_inttab != NULL)
2102		dt_inttab_destroy(pcb->pcb_inttab);
2103
2104	if ((pcb->pcb_inttab = dt_inttab_create(yypcb->pcb_hdl)) == NULL)
2105		longjmp(pcb->pcb_jmpbuf, EDT_NOMEM);
2106
2107	if (pcb->pcb_strtab != NULL)
2108		dt_strtab_destroy(pcb->pcb_strtab);
2109
2110	if ((pcb->pcb_strtab = dt_strtab_create(BUFSIZ)) == NULL)
2111		longjmp(pcb->pcb_jmpbuf, EDT_NOMEM);
2112
2113	dt_irlist_destroy(&pcb->pcb_ir);
2114	dt_irlist_create(&pcb->pcb_ir);
2115
2116	assert(pcb->pcb_dret == NULL);
2117	pcb->pcb_dret = dnp;
2118
2119	if (dt_node_resolve(dnp, DT_IDENT_XLPTR) != NULL) {
2120		dnerror(dnp, D_CG_DYN, "expression cannot evaluate to result "
2121		    "of a translated pointer\n");
2122	}
2123
2124	/*
2125	 * If we're generating code for a translator body, assign the input
2126	 * parameter to the first available register (i.e. caller passes %r1).
2127	 */
2128	if (dnp->dn_kind == DT_NODE_MEMBER) {
2129		dxp = dnp->dn_membxlator;
2130		dnp = dnp->dn_membexpr;
2131
2132		dxp->dx_ident->di_flags |= DT_IDFLG_CGREG;
2133		dxp->dx_ident->di_id = dt_regset_alloc(pcb->pcb_regs);
2134	}
2135
2136	dt_cg_node(dnp, &pcb->pcb_ir, pcb->pcb_regs);
2137
2138	if ((idp = dt_node_resolve(dnp, DT_IDENT_XLSOU)) != NULL) {
2139		int reg = dt_cg_xlate_expand(dnp, idp,
2140		    &pcb->pcb_ir, pcb->pcb_regs);
2141		dt_regset_free(pcb->pcb_regs, dnp->dn_reg);
2142		dnp->dn_reg = reg;
2143	}
2144
2145	instr = DIF_INSTR_RET(dnp->dn_reg);
2146	dt_regset_free(pcb->pcb_regs, dnp->dn_reg);
2147	dt_irlist_append(&pcb->pcb_ir, dt_cg_node_alloc(DT_LBL_NONE, instr));
2148
2149	if (dnp->dn_kind == DT_NODE_MEMBER) {
2150		dt_regset_free(pcb->pcb_regs, dxp->dx_ident->di_id);
2151		dxp->dx_ident->di_id = 0;
2152		dxp->dx_ident->di_flags &= ~DT_IDFLG_CGREG;
2153	}
2154
2155	dt_regset_free(pcb->pcb_regs, 0);
2156	dt_regset_assert_free(pcb->pcb_regs);
2157}
2158