xref: /illumos-gate/usr/src/uts/common/fs/smbsrv/smb2_dispatch.c (revision 58ccc3dc6cf31bbb97afd9f13137fe67844f1c95)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2016 Nexenta Systems, Inc.  All rights reserved.
14  */
15 
16 
17 #include <smbsrv/smb2_kproto.h>
18 #include <smbsrv/smb_kstat.h>
19 #include <smbsrv/smb2.h>
20 
21 /*
22  * Saved state for a command that "goes async".  When a compound request
23  * contains a command that may block indefinitely, the compound reply is
24  * composed with an "interim response" for that command, and information
25  * needed to actually dispatch that command is saved on a list of "async"
26  * commands for this compound request.  After the compound reply is sent,
27  * the list of async commands is processed, and those may block as long
28  * as they need to without affecting the initial compound request.
29  *
30  * Now interestingly, this "async" mechanism is not used with the full
31  * range of asynchrony that one might imagine.  The design of async
32  * request processing can be drastically simplified if we can assume
33  * that there's no need to run more than one async command at a time.
34  * With that simplifying assumption, we can continue using the current
35  * "one worker thread per request message" model, which has very simple
36  * locking rules etc.  The same worker thread that handles the initial
37  * compound request can handle the list of async requests.
38  *
39  * As it turns out, SMB2 clients do not try to use more than one "async"
40  * command in a compound.  If they were to do so, the [MS-SMB2] spec.
41  * allows us to decline additional async requests with an error.
42  *
43  * smb_async_req_t is the struct used to save an "async" request on
44  * the list of requests that had an interim reply in the initial
45  * compound reply.  This includes everything needed to restart
46  * processing at the async command.
47  */
48 
49 typedef struct smb2_async_req {
50 
51 	smb_sdrc_t		(*ar_func)(smb_request_t *);
52 
53 	int ar_cmd_hdr;		/* smb2_cmd_hdr offset */
54 	int ar_cmd_len;		/* length from hdr */
55 
56 	/*
57 	 * SMB2 header fields.
58 	 */
59 	uint16_t		ar_cmd_code;
60 	uint16_t		ar_uid;
61 	uint16_t		ar_tid;
62 	uint32_t		ar_pid;
63 	uint32_t		ar_hdr_flags;
64 	uint64_t		ar_messageid;
65 } smb2_async_req_t;
66 
67 void smb2sr_do_async(smb_request_t *);
68 smb_sdrc_t smb2_invalid_cmd(smb_request_t *);
69 static void smb2_tq_work(void *);
70 
71 static const smb_disp_entry_t
72 smb2_disp_table[SMB2__NCMDS] = {
73 
74 	/* text-name, pre, func, post, cmd-code, dialect, flags */
75 
76 	{  "smb2_negotiate", NULL,
77 	    smb2_negotiate, NULL, 0, 0,
78 	    SDDF_SUPPRESS_TID | SDDF_SUPPRESS_UID },
79 
80 	{  "smb2_session_setup", NULL,
81 	    smb2_session_setup, NULL, 0, 0,
82 	    SDDF_SUPPRESS_TID | SDDF_SUPPRESS_UID },
83 
84 	{  "smb2_logoff", NULL,
85 	    smb2_logoff, NULL, 0, 0,
86 	    SDDF_SUPPRESS_TID },
87 
88 	{  "smb2_tree_connect", NULL,
89 	    smb2_tree_connect, NULL, 0, 0,
90 	    SDDF_SUPPRESS_TID },
91 
92 	{  "smb2_tree_disconn", NULL,
93 	    smb2_tree_disconn, NULL, 0, 0 },
94 
95 	{  "smb2_create", NULL,
96 	    smb2_create, NULL, 0, 0 },
97 
98 	{  "smb2_close", NULL,
99 	    smb2_close, NULL, 0, 0 },
100 
101 	{  "smb2_flush", NULL,
102 	    smb2_flush, NULL, 0, 0 },
103 
104 	{  "smb2_read", NULL,
105 	    smb2_read, NULL, 0, 0 },
106 
107 	{  "smb2_write", NULL,
108 	    smb2_write, NULL, 0, 0 },
109 
110 	{  "smb2_lock", NULL,
111 	    smb2_lock, NULL, 0, 0 },
112 
113 	{  "smb2_ioctl", NULL,
114 	    smb2_ioctl, NULL, 0, 0 },
115 
116 	/*
117 	 * Note: Cancel gets the "invalid command" handler because
118 	 * that's always handled directly in the reader.  We should
119 	 * never get to the function using this table, but note:
120 	 * We CAN get here if a nasty client adds cancel to some
121 	 * compound message, which is a protocol violation.
122 	 */
123 	{  "smb2_cancel", NULL,
124 	    smb2_invalid_cmd, NULL, 0, 0 },
125 
126 	{  "smb2_echo", NULL,
127 	    smb2_echo, NULL, 0, 0,
128 	    SDDF_SUPPRESS_UID | SDDF_SUPPRESS_TID },
129 
130 	{  "smb2_query_dir", NULL,
131 	    smb2_query_dir, NULL, 0, 0 },
132 
133 	{  "smb2_change_notify", NULL,
134 	    smb2_change_notify, NULL, 0, 0 },
135 
136 	{  "smb2_query_info", NULL,
137 	    smb2_query_info, NULL, 0, 0 },
138 
139 	{  "smb2_set_info", NULL,
140 	    smb2_set_info, NULL, 0, 0 },
141 
142 	{  "smb2_oplock_break_ack", NULL,
143 	    smb2_oplock_break_ack, NULL, 0, 0 },
144 
145 	{  "smb2_invalid_cmd", NULL,
146 	    smb2_invalid_cmd, NULL, 0, 0,
147 	    SDDF_SUPPRESS_UID | SDDF_SUPPRESS_TID },
148 };
149 
150 smb_sdrc_t
151 smb2_invalid_cmd(smb_request_t *sr)
152 {
153 #ifdef	DEBUG
154 	cmn_err(CE_NOTE, "clnt %s bad SMB2 cmd code",
155 	    sr->session->ip_addr_str);
156 #endif
157 	sr->smb2_status = NT_STATUS_INVALID_PARAMETER;
158 	return (SDRC_DROP_VC);
159 }
160 
161 /*
162  * This is the SMB2 handler for new smb requests, called from
163  * smb_session_reader after SMB negotiate is done.  For most SMB2
164  * requests, we just enqueue them for the smb_session_worker to
165  * execute via the task queue, so they can block for resources
166  * without stopping the reader thread.  A few protocol messages
167  * are special cases and are handled directly here in the reader
168  * thread so they don't wait for taskq scheduling.
169  *
170  * This function must either enqueue the new request for
171  * execution via the task queue, or execute it directly
172  * and then free it.  If this returns non-zero, the caller
173  * will drop the session.
174  */
175 int
176 smb2sr_newrq(smb_request_t *sr)
177 {
178 	uint32_t magic;
179 	uint16_t command;
180 	int rc;
181 
182 	magic = LE_IN32(sr->sr_request_buf);
183 	if (magic != SMB2_PROTOCOL_MAGIC) {
184 		smb_request_free(sr);
185 		/* will drop the connection */
186 		return (EPROTO);
187 	}
188 
189 	/*
190 	 * Execute Cancel requests immediately, (here in the
191 	 * reader thread) so they won't wait for any other
192 	 * commands we might already have in the task queue.
193 	 * Cancel also skips signature verification and
194 	 * does not consume a sequence number.
195 	 * [MS-SMB2] 3.2.4.24 Cancellation...
196 	 */
197 	command = LE_IN16((uint8_t *)sr->sr_request_buf + 12);
198 	if (command == SMB2_CANCEL) {
199 		rc = smb2sr_newrq_cancel(sr);
200 		smb_request_free(sr);
201 		return (rc);
202 	}
203 
204 	/*
205 	 * Submit the request to the task queue, which calls
206 	 * smb2_tq_work when the workload permits.
207 	 */
208 	sr->sr_time_submitted = gethrtime();
209 	sr->sr_state = SMB_REQ_STATE_SUBMITTED;
210 	smb_srqueue_waitq_enter(sr->session->s_srqueue);
211 	(void) taskq_dispatch(sr->sr_server->sv_worker_pool,
212 	    smb2_tq_work, sr, TQ_SLEEP);
213 
214 	return (0);
215 }
216 
217 static void
218 smb2_tq_work(void *arg)
219 {
220 	smb_request_t	*sr;
221 	smb_srqueue_t	*srq;
222 
223 	sr = (smb_request_t *)arg;
224 	SMB_REQ_VALID(sr);
225 
226 	srq = sr->session->s_srqueue;
227 	smb_srqueue_waitq_to_runq(srq);
228 	sr->sr_worker = curthread;
229 	sr->sr_time_active = gethrtime();
230 
231 	/*
232 	 * In contrast with SMB1, SMB2 must _always_ dispatch to
233 	 * the handler function, because cancelled requests need
234 	 * an error reply (NT_STATUS_CANCELLED).
235 	 */
236 	smb2sr_work(sr);
237 
238 	smb_srqueue_runq_exit(srq);
239 }
240 
241 /*
242  * smb2sr_work
243  *
244  * This function processes each SMB command in the current request
245  * (which may be a compound request) building a reply containing
246  * SMB reply messages, one-to-one with the SMB commands.  Some SMB
247  * commands (change notify, blocking locks) may require both an
248  * "interim response" and a later "async response" at completion.
249  * In such cases, we'll encode the interim response in the reply
250  * compound we're building, and put the (now async) command on a
251  * list of commands that need further processing.  After we've
252  * finished processing the commands in this compound and building
253  * the compound reply, we'll send the compound reply, and finally
254  * process the list of async commands.
255  *
256  * As we work our way through the compound request and reply,
257  * we need to keep track of the bounds of the current request
258  * and reply.  For the request, this uses an MBC_SHADOW_CHAIN
259  * that begins at smb2_cmd_hdr.  The reply is appended to the
260  * sr->reply chain starting at smb2_reply_hdr.
261  *
262  * This function must always free the smb request.
263  */
264 void
265 smb2sr_work(struct smb_request *sr)
266 {
267 	const smb_disp_entry_t	*sdd;
268 	smb_disp_stats_t	*sds;
269 	smb_session_t		*session;
270 	uint32_t		msg_len;
271 	uint16_t		cmd_idx;
272 	int			rc = 0;
273 	boolean_t		disconnect = B_FALSE;
274 	boolean_t		related;
275 
276 	session = sr->session;
277 
278 	ASSERT(sr->tid_tree == 0);
279 	ASSERT(sr->uid_user == 0);
280 	ASSERT(sr->fid_ofile == 0);
281 	sr->smb_fid = (uint16_t)-1;
282 	sr->smb2_status = 0;
283 
284 	/* temporary until we identify a user */
285 	sr->user_cr = zone_kcred();
286 
287 	mutex_enter(&sr->sr_mutex);
288 	switch (sr->sr_state) {
289 	case SMB_REQ_STATE_SUBMITTED:
290 	case SMB_REQ_STATE_CLEANED_UP:
291 		sr->sr_state = SMB_REQ_STATE_ACTIVE;
292 		break;
293 	default:
294 		ASSERT(0);
295 		/* FALLTHROUGH */
296 	case SMB_REQ_STATE_CANCELED:
297 		sr->smb2_status = NT_STATUS_CANCELLED;
298 		break;
299 	}
300 	mutex_exit(&sr->sr_mutex);
301 
302 cmd_start:
303 	/*
304 	 * Decode the request header
305 	 *
306 	 * Most problems with decoding will result in the error
307 	 * STATUS_INVALID_PARAMETER.  If the decoding problem
308 	 * prevents continuing, we'll close the connection.
309 	 * [MS-SMB2] 3.3.5.2.6 Handling Incorrectly Formatted...
310 	 *
311 	 * We treat some status codes as if "sticky", meaning
312 	 * once they're set after some command handler returns,
313 	 * all remaining commands get this status without even
314 	 * calling the command-specific handler. The cancelled
315 	 * status is used above, and insufficient_resources is
316 	 * used when smb2sr_go_async declines to "go async".
317 	 * Otherwise initialize to zero (success).
318 	 */
319 	if (sr->smb2_status != NT_STATUS_CANCELLED &&
320 	    sr->smb2_status != NT_STATUS_INSUFFICIENT_RESOURCES)
321 		sr->smb2_status = 0;
322 
323 	sr->smb2_cmd_hdr = sr->command.chain_offset;
324 	if ((rc = smb2_decode_header(sr)) != 0) {
325 		cmn_err(CE_WARN, "clnt %s bad SMB2 header",
326 		    session->ip_addr_str);
327 		disconnect = B_TRUE;
328 		goto cleanup;
329 	}
330 
331 	/*
332 	 * The SMB2_FLAGS_SERVER_TO_REDIR should only appear
333 	 * in messages from the server back to the client.
334 	 */
335 	if ((sr->smb2_hdr_flags & SMB2_FLAGS_SERVER_TO_REDIR) != 0) {
336 		cmn_err(CE_WARN, "clnt %s bad SMB2 flags",
337 		    session->ip_addr_str);
338 		disconnect = B_TRUE;
339 		goto cleanup;
340 	}
341 	related = (sr->smb2_hdr_flags & SMB2_FLAGS_RELATED_OPERATIONS);
342 
343 	/*
344 	 * In case we bail out with an error before we get to the
345 	 * section that computes the credit grant, initialize the
346 	 * response header fields so that credits won't change.
347 	 * Note: SMB 2.02 clients may send credit charge zero.
348 	 */
349 	if (sr->smb2_credit_charge == 0)
350 		sr->smb2_credit_charge = 1;
351 	sr->smb2_credit_response = sr->smb2_credit_charge;
352 
353 	/*
354 	 * Reserve space for the reply header, and save the offset.
355 	 * The reply header will be overwritten later.  If we have
356 	 * already exhausted the output space, then this client is
357 	 * trying something funny.  Log it and kill 'em.
358 	 */
359 	sr->smb2_reply_hdr = sr->reply.chain_offset;
360 	if ((rc = smb2_encode_header(sr, B_FALSE)) != 0) {
361 		cmn_err(CE_WARN, "clnt %s excessive reply",
362 		    session->ip_addr_str);
363 		disconnect = B_TRUE;
364 		goto cleanup;
365 	}
366 
367 	/*
368 	 * Figure out the length of data following the SMB2 header.
369 	 * It ends at either the next SMB2 header if there is one
370 	 * (smb2_next_command != 0) or at the end of the message.
371 	 */
372 	if (sr->smb2_next_command != 0) {
373 		/* [MS-SMB2] says this is 8-byte aligned */
374 		msg_len = sr->smb2_next_command;
375 		if ((msg_len & 7) != 0 || (msg_len < SMB2_HDR_SIZE) ||
376 		    ((sr->smb2_cmd_hdr + msg_len) > sr->command.max_bytes)) {
377 			cmn_err(CE_WARN, "clnt %s bad SMB2 next cmd",
378 			    session->ip_addr_str);
379 			disconnect = B_TRUE;
380 			goto cleanup;
381 		}
382 	} else {
383 		msg_len = sr->command.max_bytes - sr->smb2_cmd_hdr;
384 	}
385 
386 	/*
387 	 * Setup a shadow chain for this SMB2 command, starting
388 	 * with the header and ending at either the next command
389 	 * or the end of the message.  The signing check below
390 	 * needs the entire SMB2 command.  After that's done, we
391 	 * advance chain_offset to the end of the header where
392 	 * the command specific handlers continue decoding.
393 	 */
394 	(void) MBC_SHADOW_CHAIN(&sr->smb_data, &sr->command,
395 	    sr->smb2_cmd_hdr, msg_len);
396 
397 	/*
398 	 * Validate the commmand code, get dispatch table entries.
399 	 * [MS-SMB2] 3.3.5.2.6 Handling Incorrectly Formatted...
400 	 *
401 	 * The last slot in the dispatch table is used to handle
402 	 * invalid commands.  Same for statistics.
403 	 */
404 	if (sr->smb2_cmd_code < SMB2_INVALID_CMD)
405 		cmd_idx = sr->smb2_cmd_code;
406 	else
407 		cmd_idx = SMB2_INVALID_CMD;
408 	sdd = &smb2_disp_table[cmd_idx];
409 	sds = &session->s_server->sv_disp_stats2[cmd_idx];
410 
411 	/*
412 	 * If this command is NOT "related" to the previous,
413 	 * clear out the UID, TID, FID state that might be
414 	 * left over from the previous command.
415 	 *
416 	 * If the command IS related, any new IDs are ignored,
417 	 * and we simply continue with the previous user, tree,
418 	 * and open file.
419 	 */
420 	if (!related) {
421 		/*
422 		 * Drop user, tree, file; carefully ordered to
423 		 * avoid dangling references: file, tree, user
424 		 */
425 		if (sr->fid_ofile != NULL) {
426 			smb_ofile_request_complete(sr->fid_ofile);
427 			smb_ofile_release(sr->fid_ofile);
428 			sr->fid_ofile = NULL;
429 		}
430 		if (sr->tid_tree != NULL) {
431 			smb_tree_release(sr->tid_tree);
432 			sr->tid_tree = NULL;
433 		}
434 		if (sr->uid_user != NULL) {
435 			smb_user_release(sr->uid_user);
436 			sr->uid_user = NULL;
437 			sr->user_cr = zone_kcred();
438 		}
439 	}
440 
441 	/*
442 	 * Make sure we have a user and tree as needed
443 	 * according to the flags for the this command.
444 	 * Note that we may have inherited these.
445 	 */
446 	if ((sdd->sdt_flags & SDDF_SUPPRESS_UID) == 0) {
447 		/*
448 		 * This command requires a user session.
449 		 */
450 		if (related) {
451 			/*
452 			 * Previous command should have given us a user.
453 			 * [MS-SMB2] 3.3.5.2 Handling Related Requests
454 			 */
455 			if (sr->uid_user == NULL) {
456 				smb2sr_put_error(sr,
457 				    NT_STATUS_INVALID_PARAMETER);
458 				goto cmd_done;
459 			}
460 			sr->smb_uid = sr->uid_user->u_uid;
461 		} else {
462 			/*
463 			 * Lookup the UID
464 			 * [MS-SMB2] 3.3.5.2 Verifying the Session
465 			 */
466 			ASSERT(sr->uid_user == NULL);
467 			sr->uid_user = smb_session_lookup_uid(session,
468 			    sr->smb_uid);
469 			if (sr->uid_user == NULL) {
470 				smb2sr_put_error(sr,
471 				    NT_STATUS_USER_SESSION_DELETED);
472 				goto cmd_done;
473 			}
474 			sr->user_cr = smb_user_getcred(sr->uid_user);
475 		}
476 		ASSERT(sr->uid_user != NULL);
477 	}
478 
479 	if ((sdd->sdt_flags & SDDF_SUPPRESS_TID) == 0) {
480 		/*
481 		 * This command requires a tree connection.
482 		 */
483 		if (related) {
484 			/*
485 			 * Previous command should have given us a tree.
486 			 * [MS-SMB2] 3.3.5.2 Handling Related Requests
487 			 */
488 			if (sr->tid_tree == NULL) {
489 				smb2sr_put_error(sr,
490 				    NT_STATUS_INVALID_PARAMETER);
491 				goto cmd_done;
492 			}
493 			sr->smb_tid = sr->tid_tree->t_tid;
494 		} else {
495 			/*
496 			 * Lookup the TID
497 			 * [MS-SMB2] 3.3.5.2 Verifying the Tree Connect
498 			 */
499 			ASSERT(sr->tid_tree == NULL);
500 			sr->tid_tree = smb_session_lookup_tree(session,
501 			    sr->smb_tid);
502 			if (sr->tid_tree == NULL) {
503 				smb2sr_put_error(sr,
504 				    NT_STATUS_NETWORK_NAME_DELETED);
505 				goto cmd_done;
506 			}
507 		}
508 		ASSERT(sr->tid_tree != NULL);
509 	}
510 
511 	/*
512 	 * SMB2 signature verification, two parts:
513 	 * (a) Require SMB2_FLAGS_SIGNED (for most request types)
514 	 * (b) If SMB2_FLAGS_SIGNED is set, check the signature.
515 	 * [MS-SMB2] 3.3.5.2.4 Verifying the Signature
516 	 */
517 
518 	/*
519 	 * No user session means no signature check.  That's OK,
520 	 * i.e. for commands marked SDDF_SUPPRESS_UID above.
521 	 * Note, this also means we won't sign the reply.
522 	 */
523 	if (sr->uid_user == NULL)
524 		sr->smb2_hdr_flags &= ~SMB2_FLAGS_SIGNED;
525 
526 	/*
527 	 * The SDDF_SUPPRESS_UID dispatch is set for requests that
528 	 * don't need a UID (user).  These also don't require a
529 	 * signature check here.
530 	 */
531 	if ((sdd->sdt_flags & SDDF_SUPPRESS_UID) == 0 &&
532 	    sr->uid_user != NULL &&
533 	    (sr->uid_user->u_sign_flags & SMB_SIGNING_CHECK) != 0) {
534 		/*
535 		 * This request type should be signed, and
536 		 * we're configured to require signatures.
537 		 */
538 		if ((sr->smb2_hdr_flags & SMB2_FLAGS_SIGNED) == 0) {
539 			smb2sr_put_error(sr, NT_STATUS_ACCESS_DENIED);
540 			goto cmd_done;
541 		}
542 		rc = smb2_sign_check_request(sr);
543 		if (rc != 0) {
544 			DTRACE_PROBE1(smb2__sign__check, smb_request_t, sr);
545 			smb2sr_put_error(sr, NT_STATUS_ACCESS_DENIED);
546 			goto cmd_done;
547 		}
548 	}
549 
550 	/*
551 	 * Now that the signing check is done with smb_data,
552 	 * advance past the SMB2 header we decoded earlier.
553 	 * This leaves sr->smb_data correctly positioned
554 	 * for command-specific decoding in the dispatch
555 	 * function called next.
556 	 */
557 	sr->smb_data.chain_offset = sr->smb2_cmd_hdr + SMB2_HDR_SIZE;
558 
559 	/*
560 	 * SMB2 credits determine how many simultaneous commands the
561 	 * client may issue, and bounds the range of message IDs those
562 	 * commands may use.  With multi-credit support, commands may
563 	 * use ranges of message IDs, where the credits used by each
564 	 * command are proportional to their data transfer size.
565 	 *
566 	 * Every command may request an increase or decrease of
567 	 * the currently granted credits, based on the difference
568 	 * between the credit request and the credit charge.
569 	 * [MS-SMB2] 3.3.1.2 Algorithm for the Granting of Credits
570 	 *
571 	 * Most commands have credit_request=1, credit_charge=1,
572 	 * which keeps the credit grant unchanged.
573 	 *
574 	 * All we're really doing here (for now) is reducing the
575 	 * credit_response if the client requests a credit increase
576 	 * that would take their credit over the maximum, and
577 	 * limiting the decrease so they don't run out of credits.
578 	 *
579 	 * Later, this could do something dynamic based on load.
580 	 *
581 	 * One other non-obvious bit about credits: We keep the
582 	 * session s_max_credits low until the 1st authentication,
583 	 * at which point we'll set the normal maximum_credits.
584 	 * Some clients ask for more credits with session setup,
585 	 * and we need to handle that requested increase _after_
586 	 * the command-specific handler returns so it won't be
587 	 * restricted to the lower (pre-auth) limit.
588 	 */
589 	sr->smb2_credit_response = sr->smb2_credit_request;
590 	if (sr->smb2_credit_request < sr->smb2_credit_charge) {
591 		uint16_t cur, d;
592 
593 		mutex_enter(&session->s_credits_mutex);
594 		cur = session->s_cur_credits;
595 
596 		/* Handle credit decrease. */
597 		d = sr->smb2_credit_charge - sr->smb2_credit_request;
598 		cur -= d;
599 		if (cur & 0x8000) {
600 			/*
601 			 * underflow (bad credit charge or request)
602 			 * leave credits unchanged (response=charge)
603 			 */
604 			cur = session->s_cur_credits;
605 			sr->smb2_credit_response = sr->smb2_credit_charge;
606 			DTRACE_PROBE1(smb2__credit__neg, smb_request_t, sr);
607 		}
608 
609 		/*
610 		 * The server MUST ensure that the number of credits
611 		 * held by the client is never reduced to zero.
612 		 * [MS-SMB2] 3.3.1.2
613 		 */
614 		if (cur == 0) {
615 			cur = 1;
616 			sr->smb2_credit_response += 1;
617 			DTRACE_PROBE1(smb2__credit__min, smb_request_t, sr);
618 		}
619 
620 		DTRACE_PROBE3(smb2__credit__decrease,
621 		    smb_request_t, sr, int, (int)cur,
622 		    int, (int)session->s_cur_credits);
623 
624 		session->s_cur_credits = cur;
625 		mutex_exit(&session->s_credits_mutex);
626 	}
627 
628 	/*
629 	 * The real work: call the SMB2 command handler
630 	 * (except for "sticky" smb2_status - see above)
631 	 */
632 	sr->sr_time_start = gethrtime();
633 	rc = SDRC_SUCCESS;
634 	if (sr->smb2_status == 0) {
635 		/* NB: not using pre_op */
636 		rc = (*sdd->sdt_function)(sr);
637 		/* NB: not using post_op */
638 	}
639 
640 	MBC_FLUSH(&sr->raw_data);
641 
642 	/*
643 	 * Second half of SMB2 credit handling (increases)
644 	 */
645 	if (sr->smb2_credit_request > sr->smb2_credit_charge) {
646 		uint16_t cur, d;
647 
648 		mutex_enter(&session->s_credits_mutex);
649 		cur = session->s_cur_credits;
650 
651 		/* Handle credit increase. */
652 		d = sr->smb2_credit_request - sr->smb2_credit_charge;
653 		cur += d;
654 
655 		/*
656 		 * If new credits would be above max,
657 		 * reduce the credit grant.
658 		 */
659 		if (cur > session->s_max_credits) {
660 			d = cur - session->s_max_credits;
661 			cur = session->s_max_credits;
662 			sr->smb2_credit_response -= d;
663 			DTRACE_PROBE1(smb2__credit__max, smb_request_t, sr);
664 		}
665 
666 		DTRACE_PROBE3(smb2__credit__increase,
667 		    smb_request_t, sr, int, (int)cur,
668 		    int, (int)session->s_cur_credits);
669 
670 		session->s_cur_credits = cur;
671 		mutex_exit(&session->s_credits_mutex);
672 	}
673 
674 cmd_done:
675 	/*
676 	 * Pad the reply to align(8) if necessary.
677 	 */
678 	if (sr->reply.chain_offset & 7) {
679 		int padsz = 8 - (sr->reply.chain_offset & 7);
680 		(void) smb_mbc_encodef(&sr->reply, "#.", padsz);
681 	}
682 	ASSERT((sr->reply.chain_offset & 7) == 0);
683 
684 	/*
685 	 * Record some statistics: latency, rx bytes, tx bytes.
686 	 */
687 	smb_latency_add_sample(&sds->sdt_lat,
688 	    gethrtime() - sr->sr_time_start);
689 	atomic_add_64(&sds->sdt_rxb,
690 	    (int64_t)(sr->command.chain_offset - sr->smb2_cmd_hdr));
691 	atomic_add_64(&sds->sdt_txb,
692 	    (int64_t)(sr->reply.chain_offset - sr->smb2_reply_hdr));
693 
694 	switch (rc) {
695 	case SDRC_SUCCESS:
696 		break;
697 	default:
698 		/*
699 		 * SMB2 does not use the other dispatch return codes.
700 		 * If we see something else, log an event so we'll
701 		 * know something is returning bogus status codes.
702 		 * If you see these in the log, use dtrace to find
703 		 * the code returning something else.
704 		 */
705 #ifdef	DEBUG
706 		cmn_err(CE_NOTE, "handler for %u returned 0x%x",
707 		    sr->smb2_cmd_code, rc);
708 #endif
709 		/* FALLTHROUGH */
710 	case SDRC_ERROR:
711 		if (sr->smb2_status == 0)
712 			sr->smb2_status = NT_STATUS_INTERNAL_ERROR;
713 		break;
714 	case SDRC_DROP_VC:
715 		disconnect = B_TRUE;
716 		goto cleanup;
717 	}
718 
719 	/*
720 	 * If there's a next command, figure out where it starts,
721 	 * and fill in the next command offset for the reply.
722 	 * Note: We sanity checked smb2_next_command above
723 	 * (the offset to the next command).  Similarly set
724 	 * smb2_next_reply as the offset to the next reply.
725 	 */
726 	if (sr->smb2_next_command != 0) {
727 		sr->command.chain_offset =
728 		    sr->smb2_cmd_hdr + sr->smb2_next_command;
729 		sr->smb2_next_reply =
730 		    sr->reply.chain_offset - sr->smb2_reply_hdr;
731 	} else {
732 		sr->smb2_next_reply = 0;
733 	}
734 
735 	/*
736 	 * Overwrite the SMB2 header for the response of
737 	 * this command (possibly part of a compound).
738 	 * encode_header adds: SMB2_FLAGS_SERVER_TO_REDIR
739 	 */
740 	(void) smb2_encode_header(sr, B_TRUE);
741 
742 	if (sr->smb2_hdr_flags & SMB2_FLAGS_SIGNED)
743 		smb2_sign_reply(sr);
744 
745 	if (sr->smb2_next_command != 0)
746 		goto cmd_start;
747 
748 	/*
749 	 * We've done all the commands in this compound.
750 	 * Send it out.
751 	 */
752 	smb2_send_reply(sr);
753 
754 	/*
755 	 * If any of the requests "went async", process those now.
756 	 * The async. function "keeps" this sr, changing its state
757 	 * to completed and calling smb_request_free().
758 	 */
759 	if (sr->sr_async_req != NULL) {
760 		smb2sr_do_async(sr);
761 		return;
762 	}
763 
764 cleanup:
765 	if (disconnect) {
766 		smb_rwx_rwenter(&session->s_lock, RW_WRITER);
767 		switch (session->s_state) {
768 		case SMB_SESSION_STATE_DISCONNECTED:
769 		case SMB_SESSION_STATE_TERMINATED:
770 			break;
771 		default:
772 			smb_soshutdown(session->sock);
773 			session->s_state = SMB_SESSION_STATE_DISCONNECTED;
774 			break;
775 		}
776 		smb_rwx_rwexit(&session->s_lock);
777 	}
778 
779 	mutex_enter(&sr->sr_mutex);
780 	sr->sr_state = SMB_REQ_STATE_COMPLETED;
781 	mutex_exit(&sr->sr_mutex);
782 
783 	smb_request_free(sr);
784 }
785 
786 /*
787  * Dispatch an async request using saved information.
788  * See smb2sr_save_async and [MS-SMB2] 3.3.4.2
789  *
790  * This is sort of a "lite" version of smb2sr_work.  Initialize the
791  * command and reply areas as they were when the command-speicific
792  * handler started (in case it needs to decode anything again).
793  * Call the async function, which builds the command-specific part
794  * of the response.  Finally, send the response and free the sr.
795  */
796 void
797 smb2sr_do_async(smb_request_t *sr)
798 {
799 	const smb_disp_entry_t	*sdd;
800 	smb_disp_stats_t	*sds;
801 	smb2_async_req_t	*ar;
802 	int rc = 0;
803 
804 	/*
805 	 * Restore what smb2_decode_header found.
806 	 * (In lieu of decoding it again.)
807 	 */
808 	ar = sr->sr_async_req;
809 	sr->smb2_cmd_hdr   = ar->ar_cmd_hdr;
810 	sr->smb2_cmd_code  = ar->ar_cmd_code;
811 	sr->smb2_hdr_flags = ar->ar_hdr_flags;
812 	sr->smb2_async_id  = (uintptr_t)ar;
813 	sr->smb2_messageid = ar->ar_messageid;
814 	sr->smb_pid = ar->ar_pid;
815 	sr->smb_tid = ar->ar_tid;
816 	sr->smb_uid = ar->ar_uid;
817 	sr->smb2_status = 0;
818 
819 	/*
820 	 * Async requests don't grant credits, because any credits
821 	 * should have gone out with the interim reply.
822 	 * An async reply goes alone (no next reply).
823 	 */
824 	sr->smb2_credit_response = 0;
825 	sr->smb2_next_reply = 0;
826 
827 	/*
828 	 * Setup input mbuf_chain
829 	 */
830 	ASSERT(ar->ar_cmd_len >= SMB2_HDR_SIZE);
831 	(void) MBC_SHADOW_CHAIN(&sr->smb_data, &sr->command,
832 	    sr->smb2_cmd_hdr + SMB2_HDR_SIZE,
833 	    ar->ar_cmd_len - SMB2_HDR_SIZE);
834 
835 	/*
836 	 * Setup output mbuf_chain
837 	 */
838 	MBC_FLUSH(&sr->reply);
839 	sr->smb2_reply_hdr = sr->reply.chain_offset;
840 	(void) smb2_encode_header(sr, B_FALSE);
841 
842 	VERIFY3U(sr->smb2_cmd_code, <, SMB2_INVALID_CMD);
843 	sdd = &smb2_disp_table[sr->smb2_cmd_code];
844 	sds = sr->session->s_server->sv_disp_stats2;
845 	sds = &sds[sr->smb2_cmd_code];
846 
847 	/*
848 	 * Keep the UID, TID, ofile we have.
849 	 */
850 	if ((sdd->sdt_flags & SDDF_SUPPRESS_UID) == 0 &&
851 	    sr->uid_user == NULL) {
852 		smb2sr_put_error(sr, NT_STATUS_USER_SESSION_DELETED);
853 		goto cmd_done;
854 	}
855 	if ((sdd->sdt_flags & SDDF_SUPPRESS_TID) == 0 &&
856 	    sr->tid_tree == NULL) {
857 		smb2sr_put_error(sr, NT_STATUS_NETWORK_NAME_DELETED);
858 		goto cmd_done;
859 	}
860 
861 	/*
862 	 * Signature already verified
863 	 * Credits handled...
864 	 *
865 	 * Just call the async handler function.
866 	 */
867 	rc = ar->ar_func(sr);
868 	if (rc != 0 && sr->smb2_status == 0)
869 		sr->smb2_status = NT_STATUS_INTERNAL_ERROR;
870 
871 cmd_done:
872 	/*
873 	 * Pad the reply to align(8) if necessary.
874 	 */
875 	if (sr->reply.chain_offset & 7) {
876 		int padsz = 8 - (sr->reply.chain_offset & 7);
877 		(void) smb_mbc_encodef(&sr->reply, "#.", padsz);
878 	}
879 	ASSERT((sr->reply.chain_offset & 7) == 0);
880 
881 	/*
882 	 * Record some statistics: (just tx bytes here)
883 	 */
884 	atomic_add_64(&sds->sdt_txb,
885 	    (int64_t)(sr->reply.chain_offset - sr->smb2_reply_hdr));
886 
887 	/*
888 	 * Overwrite the SMB2 header for the response of
889 	 * this command (possibly part of a compound).
890 	 * The call adds: SMB2_FLAGS_SERVER_TO_REDIR
891 	 */
892 	(void) smb2_encode_header(sr, B_TRUE);
893 
894 	if (sr->smb2_hdr_flags & SMB2_FLAGS_SIGNED)
895 		smb2_sign_reply(sr);
896 
897 	smb2_send_reply(sr);
898 
899 	/*
900 	 * Done.  Unlink and free.
901 	 */
902 	sr->sr_async_req = NULL;
903 	kmem_free(ar, sizeof (*ar));
904 
905 	mutex_enter(&sr->sr_mutex);
906 	sr->sr_state = SMB_REQ_STATE_COMPLETED;
907 	mutex_exit(&sr->sr_mutex);
908 
909 	smb_request_free(sr);
910 }
911 
912 /*
913  * In preparation for sending an "interim response", save
914  * all the state we'll need to run an async command later,
915  * and assign an "async id" for this (now async) command.
916  * See [MS-SMB2] 3.3.4.2
917  *
918  * If more than one request in a compound request tries to
919  * "go async", we can "say no".  See [MS-SMB2] 3.3.4.2
920  *	If an operation would require asynchronous processing
921  *	but resources are constrained, the server MAY choose to
922  *	fail that operation with STATUS_INSUFFICIENT_RESOURCES.
923  *
924  * For simplicity, we further restrict the cases where we're
925  * willing to "go async", and only allow the last command in a
926  * compound to "go async".  It happens that this is the only
927  * case where we're actually asked to go async anyway. This
928  * simplification also means there can be at most one command
929  * in a compound that "goes async" (the last one).
930  *
931  * If we agree to "go async", this should return STATUS_PENDING.
932  * Otherwise return STATUS_INSUFFICIENT_RESOURCES for this and
933  * all requests following this request.  (See the comments re.
934  * "sticky" smb2_status values in smb2sr_work).
935  *
936  * Note: the Async ID we assign here is arbitrary, and need only
937  * be unique among pending async responses on this connection, so
938  * this just uses an object address as the Async ID.
939  *
940  * Also, the assigned worker is the ONLY thread using this
941  * async request object (sr_async_req) so no locking.
942  */
943 uint32_t
944 smb2sr_go_async(smb_request_t *sr,
945     smb_sdrc_t (*async_func)(smb_request_t *))
946 {
947 	smb2_async_req_t *ar;
948 
949 	if (sr->smb2_next_command != 0)
950 		return (NT_STATUS_INSUFFICIENT_RESOURCES);
951 
952 	ASSERT(sr->sr_async_req == NULL);
953 	ar = kmem_zalloc(sizeof (*ar), KM_SLEEP);
954 
955 	/*
956 	 * Place an interim response in the compound reply.
957 	 *
958 	 * Turn on the "async" flag for both the (synchronous)
959 	 * interim response and the (later) async response,
960 	 * by storing that in flags before coping into ar.
961 	 */
962 	sr->smb2_hdr_flags |= SMB2_FLAGS_ASYNC_COMMAND;
963 	sr->smb2_async_id = (uintptr_t)ar;
964 
965 	ar->ar_func = async_func;
966 	ar->ar_cmd_hdr = sr->smb2_cmd_hdr;
967 	ar->ar_cmd_len = sr->smb_data.max_bytes - sr->smb2_cmd_hdr;
968 
969 	ar->ar_cmd_code = sr->smb2_cmd_code;
970 	ar->ar_hdr_flags = sr->smb2_hdr_flags;
971 	ar->ar_messageid = sr->smb2_messageid;
972 	ar->ar_pid = sr->smb_pid;
973 	ar->ar_tid = sr->smb_tid;
974 	ar->ar_uid = sr->smb_uid;
975 
976 	sr->sr_async_req = ar;
977 
978 	/* Interim responses are NOT signed. */
979 	sr->smb2_hdr_flags &= ~SMB2_FLAGS_SIGNED;
980 
981 	return (NT_STATUS_PENDING);
982 }
983 
984 int
985 smb2_decode_header(smb_request_t *sr)
986 {
987 	uint64_t ssnid;
988 	uint32_t pid, tid;
989 	uint16_t hdr_len;
990 	int rc;
991 
992 	rc = smb_mbc_decodef(
993 	    &sr->command, "Nwww..wwllqllq16c",
994 	    &hdr_len,			/* w */
995 	    &sr->smb2_credit_charge,	/* w */
996 	    &sr->smb2_chan_seq,		/* w */
997 	    /* reserved			  .. */
998 	    &sr->smb2_cmd_code,		/* w */
999 	    &sr->smb2_credit_request,	/* w */
1000 	    &sr->smb2_hdr_flags,	/* l */
1001 	    &sr->smb2_next_command,	/* l */
1002 	    &sr->smb2_messageid,	/* q */
1003 	    &pid,			/* l */
1004 	    &tid,			/* l */
1005 	    &ssnid,			/* q */
1006 	    sr->smb2_sig);		/* 16c */
1007 	if (rc)
1008 		return (rc);
1009 
1010 	if (hdr_len != SMB2_HDR_SIZE)
1011 		return (-1);
1012 
1013 	sr->smb_uid = (uint16_t)ssnid;	/* XXX wide UIDs */
1014 
1015 	if (sr->smb2_hdr_flags & SMB2_FLAGS_ASYNC_COMMAND) {
1016 		sr->smb2_async_id = pid |
1017 		    ((uint64_t)tid) << 32;
1018 	} else {
1019 		sr->smb_pid = pid;
1020 		sr->smb_tid = (uint16_t)tid; /* XXX wide TIDs */
1021 	}
1022 
1023 	return (rc);
1024 }
1025 
1026 int
1027 smb2_encode_header(smb_request_t *sr, boolean_t overwrite)
1028 {
1029 	uint64_t ssnid = sr->smb_uid;
1030 	uint64_t pid_tid_aid; /* pid+tid, or async id */
1031 	uint32_t reply_hdr_flags;
1032 	int rc;
1033 
1034 	if (sr->smb2_hdr_flags & SMB2_FLAGS_ASYNC_COMMAND) {
1035 		pid_tid_aid = sr->smb2_async_id;
1036 	} else {
1037 		pid_tid_aid = sr->smb_pid |
1038 		    ((uint64_t)sr->smb_tid) << 32;
1039 	}
1040 	reply_hdr_flags = sr->smb2_hdr_flags | SMB2_FLAGS_SERVER_TO_REDIR;
1041 
1042 	if (overwrite) {
1043 		rc = smb_mbc_poke(&sr->reply,
1044 		    sr->smb2_reply_hdr,
1045 		    "Nwwlwwllqqq16c",
1046 		    SMB2_HDR_SIZE,		/* w */
1047 		    sr->smb2_credit_charge,	/* w */
1048 		    sr->smb2_status,		/* l */
1049 		    sr->smb2_cmd_code,		/* w */
1050 		    sr->smb2_credit_response,	/* w */
1051 		    reply_hdr_flags,		/* l */
1052 		    sr->smb2_next_reply,	/* l */
1053 		    sr->smb2_messageid,		/* q */
1054 		    pid_tid_aid,		/* q */
1055 		    ssnid,			/* q */
1056 		    sr->smb2_sig);		/* 16c */
1057 	} else {
1058 		rc = smb_mbc_encodef(&sr->reply,
1059 		    "Nwwlwwllqqq16c",
1060 		    SMB2_HDR_SIZE,		/* w */
1061 		    sr->smb2_credit_charge,	/* w */
1062 		    sr->smb2_status,		/* l */
1063 		    sr->smb2_cmd_code,		/* w */
1064 		    sr->smb2_credit_response,	/* w */
1065 		    reply_hdr_flags,		/* l */
1066 		    sr->smb2_next_reply,	/* l */
1067 		    sr->smb2_messageid,		/* q */
1068 		    pid_tid_aid,		/* q */
1069 		    ssnid,			/* q */
1070 		    sr->smb2_sig);		/* 16c */
1071 	}
1072 
1073 	return (rc);
1074 }
1075 
1076 void
1077 smb2_send_reply(smb_request_t *sr)
1078 {
1079 
1080 	if (smb_session_send(sr->session, 0, &sr->reply) == 0)
1081 		sr->reply.chain = 0;
1082 }
1083 
1084 /*
1085  * This wrapper function exists to help catch calls to smbsr_status()
1086  * (which is SMB1-specific) in common code.  See smbsr_status().
1087  * If the log message below is seen, put a dtrace probe on this
1088  * function with a stack() action to see who is calling the SMB1
1089  * "put error" from common code, and fix it.
1090  */
1091 void
1092 smbsr_status_smb2(smb_request_t *sr, DWORD status)
1093 {
1094 	const char *name;
1095 
1096 	if (sr->smb2_cmd_code < SMB2__NCMDS)
1097 		name = smb2_disp_table[sr->smb2_cmd_code].sdt_name;
1098 	else
1099 		name = "<unknown>";
1100 #ifdef	DEBUG
1101 	cmn_err(CE_NOTE, "smbsr_status called for %s", name);
1102 #endif
1103 
1104 	smb2sr_put_error_data(sr, status, NULL);
1105 }
1106 
1107 void
1108 smb2sr_put_errno(struct smb_request *sr, int errnum)
1109 {
1110 	uint32_t status = smb_errno2status(errnum);
1111 	smb2sr_put_error_data(sr, status, NULL);
1112 }
1113 
1114 void
1115 smb2sr_put_error(smb_request_t *sr, uint32_t status)
1116 {
1117 	smb2sr_put_error_data(sr, status, NULL);
1118 }
1119 
1120 /*
1121  * Build an SMB2 error response.  [MS-SMB2] 2.2.2
1122  */
1123 void
1124 smb2sr_put_error_data(smb_request_t *sr, uint32_t status, mbuf_chain_t *mbc)
1125 {
1126 	DWORD len;
1127 
1128 	/*
1129 	 * The common dispatch code writes this when it
1130 	 * updates the SMB2 header before sending.
1131 	 */
1132 	sr->smb2_status = status;
1133 
1134 	/* Rewind to the end of the SMB header. */
1135 	sr->reply.chain_offset = sr->smb2_reply_hdr + SMB2_HDR_SIZE;
1136 
1137 	/*
1138 	 * NB: Must provide at least one byte of error data,
1139 	 * per [MS-SMB2] 2.2.2
1140 	 */
1141 	if (mbc != NULL && (len = MBC_LENGTH(mbc)) != 0) {
1142 		(void) smb_mbc_encodef(
1143 		    &sr->reply,
1144 		    "wwlC",
1145 		    9,	/* StructSize */	/* w */
1146 		    0,	/* reserved */		/* w */
1147 		    len,			/* l */
1148 		    mbc);			/* C */
1149 	} else {
1150 		(void) smb_mbc_encodef(
1151 		    &sr->reply,
1152 		    "wwl.",
1153 		    9,	/* StructSize */	/* w */
1154 		    0,	/* reserved */		/* w */
1155 		    0);				/* l. */
1156 	}
1157 }
1158 
1159 /*
1160  * smb2sr_lookup_fid
1161  *
1162  * Setup sr->fid_ofile, either inherited from a related command,
1163  * or obtained via FID lookup.  Similar inheritance logic as in
1164  * smb2sr_work.
1165  */
1166 uint32_t
1167 smb2sr_lookup_fid(smb_request_t *sr, smb2fid_t *fid)
1168 {
1169 	boolean_t related = sr->smb2_hdr_flags &
1170 	    SMB2_FLAGS_RELATED_OPERATIONS;
1171 
1172 	if (related) {
1173 		if (sr->fid_ofile == NULL)
1174 			return (NT_STATUS_INVALID_PARAMETER);
1175 		sr->smb_fid = sr->fid_ofile->f_fid;
1176 		return (0);
1177 	}
1178 
1179 	/*
1180 	 * If we could be sure this is called only once per cmd,
1181 	 * we could simply ASSERT(sr->fid_ofile == NULL) here.
1182 	 * However, there are cases where it can be called again
1183 	 * handling the same command, so let's tolerate that.
1184 	 */
1185 	if (sr->fid_ofile == NULL) {
1186 		sr->smb_fid = (uint16_t)fid->temporal;
1187 		sr->fid_ofile = smb_ofile_lookup_by_fid(sr, sr->smb_fid);
1188 	}
1189 	if (sr->fid_ofile == NULL)
1190 		return (NT_STATUS_FILE_CLOSED);
1191 
1192 	return (0);
1193 }
1194 
1195 /*
1196  * smb2_dispatch_stats_init
1197  *
1198  * Initializes dispatch statistics for SMB2.
1199  * See also smb_dispatch_stats_init(), which fills in
1200  * the lower part of the statistics array, from zero
1201  * through SMB_COM_NUM;
1202  */
1203 void
1204 smb2_dispatch_stats_init(smb_server_t *sv)
1205 {
1206 	smb_disp_stats_t *sds = sv->sv_disp_stats2;
1207 	smb_kstat_req_t *ksr;
1208 	int		i;
1209 
1210 	ksr = ((smbsrv_kstats_t *)sv->sv_ksp->ks_data)->ks_reqs2;
1211 
1212 	for (i = 0; i < SMB2__NCMDS; i++, ksr++) {
1213 		smb_latency_init(&sds[i].sdt_lat);
1214 		(void) strlcpy(ksr->kr_name, smb2_disp_table[i].sdt_name,
1215 		    sizeof (ksr->kr_name));
1216 	}
1217 }
1218 
1219 /*
1220  * smb2_dispatch_stats_fini
1221  *
1222  * Frees and destroyes the resources used for statistics.
1223  */
1224 void
1225 smb2_dispatch_stats_fini(smb_server_t *sv)
1226 {
1227 	smb_disp_stats_t *sds = sv->sv_disp_stats2;
1228 	int	i;
1229 
1230 	for (i = 0; i < SMB2__NCMDS; i++)
1231 		smb_latency_destroy(&sds[i].sdt_lat);
1232 }
1233 
1234 void
1235 smb2_dispatch_stats_update(smb_server_t *sv,
1236     smb_kstat_req_t *ksr, int first, int nreq)
1237 {
1238 	smb_disp_stats_t *sds = sv->sv_disp_stats2;
1239 	int	i;
1240 	int	last;
1241 
1242 	last = first + nreq - 1;
1243 
1244 	if ((first < SMB2__NCMDS) && (last < SMB2__NCMDS))  {
1245 		for (i = first; i <= last; i++, ksr++) {
1246 			ksr->kr_rxb = sds[i].sdt_rxb;
1247 			ksr->kr_txb = sds[i].sdt_txb;
1248 			mutex_enter(&sds[i].sdt_lat.ly_mutex);
1249 			ksr->kr_nreq = sds[i].sdt_lat.ly_a_nreq;
1250 			ksr->kr_sum = sds[i].sdt_lat.ly_a_sum;
1251 			ksr->kr_a_mean = sds[i].sdt_lat.ly_a_mean;
1252 			ksr->kr_a_stddev =
1253 			    sds[i].sdt_lat.ly_a_stddev;
1254 			ksr->kr_d_mean = sds[i].sdt_lat.ly_d_mean;
1255 			ksr->kr_d_stddev =
1256 			    sds[i].sdt_lat.ly_d_stddev;
1257 			sds[i].sdt_lat.ly_d_mean = 0;
1258 			sds[i].sdt_lat.ly_d_nreq = 0;
1259 			sds[i].sdt_lat.ly_d_stddev = 0;
1260 			sds[i].sdt_lat.ly_d_sum = 0;
1261 			mutex_exit(&sds[i].sdt_lat.ly_mutex);
1262 		}
1263 	}
1264 }
1265