xref: /illumos-gate/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_timers.c (revision 6a634c9d)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * This file implements the timer setup and timeout handling functions.
 */

#include <sys/ib/mgt/ibmf/ibmf_impl.h>

extern int ibmf_trace_level;

/*
 * ibmf_i_set_timer():
 *	Set the timer to the response or transaction time interval
 */
void
ibmf_i_set_timer(void (*func)(void *), ibmf_msg_impl_t *msgimplp,
    ibmf_timer_t type)
{
	clock_t		interval;
	ibmf_rmpp_ctx_t	*rmpp_ctx;

	ASSERT(MUTEX_HELD(&msgimplp->im_mutex));

	IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_set_timer_start,
	    IBMF_TNF_TRACE, "", "ibmf_i_set_timer: msgp = %p, "
	    "timer_type = 0x%x, func_cb = 0x%p\n",
	    tnf_opaque, msgimplp, msgimplp, tnf_opaque, timer_type, type,
	    tnf_opaque, func_cb, func);

	if (type == IBMF_RESP_TIMER) {

		/*
		 * The response timer interval is the sum of the IBA
		 * defined RespTimeValue (Vol. 1, Section 13.4.6.2.2),
		 * and the round trip time value. Both values are determined
		 * by the IBMF client and passed in the retrans_rtv and
		 * retrans_rttv fields respectively, when calling
		 * ibmf_msg_transport()
		 */
		ASSERT(msgimplp->im_rp_timeout_id == 0);
		interval = msgimplp->im_retrans.retrans_rtv +
		    msgimplp->im_retrans.retrans_rttv;

		IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_set_timer,
		    IBMF_TNF_TRACE, "", "ibmf_i_set_timer: %s, interval = %ld "
		    "resp_time %x round trip time %x\n",
		    tnf_string, msg, "setting response timer",
		    tnf_long, interval, interval,
		    tnf_uint, resp_time, msgimplp->im_retrans.retrans_rtv,
		    tnf_uint, interval, msgimplp->im_retrans.retrans_rttv);

		msgimplp->im_rp_timeout_id = timeout(func,
		    (void *)msgimplp, drv_usectohz(interval));
	} else if (type == IBMF_TRANS_TIMER) {
		rmpp_ctx = &msgimplp->im_rmpp_ctx;

		ASSERT(msgimplp->im_tr_timeout_id == 0);
		if (rmpp_ctx->rmpp_flags & IBMF_CTX_RMPP_FLAGS_DYN_PYLD) {
			/*
			 * if payload was not specified use IB spec default
			 * of 40 seconds
			 */
			interval = IBMF_RETRANS_DEF_TRANS_TO;

			IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
			    ibmf_i_set_timer, IBMF_TNF_TRACE, "",
			    "ibmf_i_set_timer: %s, interval = %ld\n",
			    tnf_string, msg,
			    "payload size unknown.  Using default trans_to",
			    tnf_long, interval, interval);
		} else {
			/*
			 * if payload was specified, use a variation of IB
			 * spec equation (13.6.3.2) that accounts for average
			 * window size
			 */
			interval = (msgimplp->im_retrans.retrans_rtv +
			    msgimplp->im_retrans.retrans_rttv) /
			    IBMF_RMPP_DEFAULT_WIN_SZ * 4 *
			    msgimplp->im_rmpp_ctx.rmpp_num_pkts;

			IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L3,
			    ibmf_i_set_timer, IBMF_TNF_TRACE, "",
			    "ibmf_i_set_timer: %s, num_pkts = %d, rttv ="
			    " %x, window_size = %d, interval = %ld\n",
			    tnf_string, msg, "setting trans timer",
			    tnf_uint, num_pkts,
			    msgimplp->im_rmpp_ctx.rmpp_num_pkts, tnf_uint, rtv,
			    msgimplp->im_retrans.retrans_rttv,
			    tnf_uint, window_size, IBMF_RMPP_DEFAULT_WIN_SZ,
			    tnf_long, interval, interval);
		}

		/*
		 * Use the client specified transaction timeout value if
		 * smaller than the calculated value
		 */
		if ((msgimplp->im_retrans.retrans_trans_to != 0) &&
		    (msgimplp->im_retrans.retrans_trans_to < interval)) {

			interval = msgimplp->im_retrans.retrans_trans_to;

			IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
			    ibmf_i_set_timer, IBMF_TNF_TRACE, "",
			    "ibmf_i_set_timer: %s, new_interval = %ld\n",
			    tnf_string, msg, "user trans_to is smaller",
			    tnf_long, new_interval, interval);
		}

		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_set_timer,
		    IBMF_TNF_TRACE, "", "ibmf_i_set_timer: %s, interval = %ld"
		    "\n", tnf_string, msg, "setting transaction timer",
		    tnf_long, interval, interval);

		msgimplp->im_tr_timeout_id = timeout(func,
		    (void *)msgimplp, drv_usectohz(interval));
	}

	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_set_timer_end,
	    IBMF_TNF_TRACE, "", "ibmf_i_set_timer() exit\n");
}

/*
 * ibmf_i_unset_timer():
 *	Unset the timer
 */
void
ibmf_i_unset_timer(ibmf_msg_impl_t *msgimplp, ibmf_timer_t type)
{
	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_unset_timer_start,
	    IBMF_TNF_TRACE, "", "ibmf_i_unset_timer(): msgp = %p, \n",
	    tnf_opaque, msgimplp, msgimplp);

	ASSERT(MUTEX_HELD(&msgimplp->im_mutex));

	if (type == IBMF_RESP_TIMER) {
		if (msgimplp->im_rp_timeout_id != 0) {
			msgimplp->im_rp_unset_timeout_id =
			    msgimplp->im_rp_timeout_id;
			msgimplp->im_rp_timeout_id = 0;
		}
	} else if (type == IBMF_TRANS_TIMER) {
		if (msgimplp->im_tr_timeout_id != 0) {
			msgimplp->im_tr_unset_timeout_id =
			    msgimplp->im_tr_timeout_id;
			msgimplp->im_tr_timeout_id = 0;
		}
	}

	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_unset_timer_end,
	    IBMF_TNF_TRACE, "", "ibmf_i_unset_timer() exit\n");
}

/*
 * ibmf_i_recv_timeout:
 *
 *	Perform "receive" timeout processing for the message.
 *	This timeout handler is only used in RMPP processing.
 */
void
ibmf_i_recv_timeout(void *argp)
{
	ibmf_msg_impl_t *msgimplp = (ibmf_msg_impl_t *)argp;
	ibmf_client_t	*clientp = (ibmf_client_t *)msgimplp->im_client;
	ibmf_rmpp_ctx_t	*rmpp_ctx;
	int		msg_flags;
	uint_t		ref_cnt;
	int		status;

	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4,
	    ibmf_i_recv_timeout_start, IBMF_TNF_TRACE, "",
	    "ibmf_i_recv_timeout(): msgp = 0x%p\n", tnf_opaque, msg, msgimplp);

	mutex_enter(&msgimplp->im_mutex);

	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
	    ibmf_i_recv_timeout, IBMF_TNF_TRACE, "",
	    "ibmf_i_recv_timeout(): resetting id time %llx\n",
	    tnf_opaque, time, gethrtime());

	/*
	 * If the message has been marked unitialized or done
	 * release the message mutex and return
	 */
	if ((msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_UNINIT) ||
	    (msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE)) {

		mutex_exit(&msgimplp->im_mutex);

		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
		    ibmf_i_recv_timeout, IBMF_TNF_TRACE, "",
		    "ibmf_i_recv_timeout(): %s, msgp = 0x%p\n", tnf_string, msg,
		    "Message marked for removal, return without processing "
		    "recv timeout",
		    tnf_opaque, msgimplp, msgimplp);

		return;
	}

	/*
	 * Unset the response and trans timers if they haven't fired (unlikely)
	 */
	ibmf_i_unset_timer(msgimplp, IBMF_RESP_TIMER);
	ibmf_i_unset_timer(msgimplp, IBMF_TRANS_TIMER);

	rmpp_ctx = &msgimplp->im_rmpp_ctx;

	/* Perform timeout processing for the RMPP transaction */
	if (rmpp_ctx->rmpp_state == IBMF_RMPP_STATE_RECEVR_ACTIVE) {

		IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
		    ibmf_i_recv_timeout, IBMF_TNF_TRACE, "",
		    "ibmf_i_recv_timeout(): %s\n", tnf_string, msg,
		    "RMPP context is Receiver Active, sending ABORT T2L");

		status = ibmf_i_send_rmpp(msgimplp, IBMF_RMPP_TYPE_ABORT,
		    IBMF_RMPP_STATUS_T2L, 0, 0, IBMF_NO_BLOCK);
		if (status != IBMF_SUCCESS) {
			IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
			    ibmf_i_recv_timeout_err, IBMF_TNF_ERROR, "",
			    "ibmf_i_recv_timeout(): %s\n", tnf_string, msg,
			    "RMPP ABORT send failed");
			msgimplp->im_trans_state_flags |=
			    IBMF_TRANS_STATE_FLAG_SEND_DONE;
		}

		mutex_enter(&clientp->ic_kstat_mutex);
		IBMF_ADD32_KSTATS(clientp, rmpp_errors, 1);
		mutex_exit(&clientp->ic_kstat_mutex);

		rmpp_ctx->rmpp_state = IBMF_RMPP_STATE_ABORT;

		IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
		    ibmf_i_recv_timeout, IBMF_TNF_ERROR, "",
		    "ibmf_i_recv_timeout(): %s\n", tnf_string, msg,
		    "RMPP context is Receiver Active, terminating transaction");

		ibmf_i_terminate_transaction(msgimplp->im_client,
		    msgimplp, IBMF_TRANS_TIMEOUT);

	} else if (rmpp_ctx->rmpp_state == IBMF_RMPP_STATE_RECEVR_TERMINATE) {

		IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
		    ibmf_i_recv_timeout, IBMF_TNF_TRACE, "",
		    "ibmf_i_recv_timeout(): %s\n", tnf_string, msg,
		    "RMPP context is Receiver Terminate, "
		    "terminating transaction");
		rmpp_ctx->rmpp_state = IBMF_RMPP_STATE_DONE;
		ibmf_i_terminate_transaction(msgimplp->im_client, msgimplp,
		    IBMF_SUCCESS);
	}

	/*
	 * Save the transaction state flags and the timeout IDs
	 * before releasing the mutex as they may be changed after that.
	 */
	msg_flags = msgimplp->im_trans_state_flags;

	mutex_exit(&msgimplp->im_mutex);

	IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
	    ibmf_i_recv_timeout, IBMF_TNF_TRACE, "",
	    "ibmf_i_recv_timeout(): %s, msgp = 0x%p, refcnt = %d\n", tnf_string,
	    msg, "recv timeout done.  Dec ref count", tnf_opaque, msgimplp,
	    msgimplp, tnf_uint, flags, msg_flags);

	/*
	 * If the transaction flags indicate a completed transaction,
	 * notify the client
	 */
	if (msg_flags & IBMF_TRANS_STATE_FLAG_DONE) {
		/* Remove the message from the client's message list */
		ibmf_i_client_rem_msg(clientp, msgimplp, &ref_cnt);

		/*
		 * Notify the client if the message reference count is zero.
		 * At this point, we know that the transaction is done and
		 * the message has been removed from the client's message list.
		 * So, we only need to make sure the reference count is zero
		 * before notifying the client.
		 */
		if (ref_cnt == 0) {
			_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msgimplp))
			if (msgimplp->im_flags & IBMF_MSG_FLAGS_TERMINATION) {

				/*
				 * If the message is a termination message,
				 * free it at this time.
				 */
				IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
				    ibmf_i_recv_timeout, IBMF_TNF_TRACE, "",
				    "ibmf_i_recv_timeout(): freeing terminate "
				    "message %p\n", tnf_opaque, msgp, msgimplp);

				/* free up the UD destination resource */
				if (msgimplp->im_ibmf_ud_dest != NULL) {
					ibmf_i_free_ud_dest(clientp, msgimplp);
					ibmf_i_clean_ud_dest_list(
					    clientp->ic_myci, B_FALSE);
				}

				/* Free the receive buffer */
				kmem_free(
				    msgimplp->im_msgbufs_recv.im_bufs_mad_hdr,
				    IBMF_MAD_SIZE);

				/* destroy the message mutex */
				mutex_destroy(&msgimplp->im_mutex);

				/* Free the termination message context */
				kmem_free(msgimplp, sizeof (ibmf_msg_impl_t));

				/*
				 * Decrease the "messages allocated" count
				 * so that an ibmf_unregister() can succeed
				 * for this client.
				 */
				mutex_enter(&clientp->ic_mutex);
				clientp->ic_msgs_alloced--;
				mutex_exit(&clientp->ic_mutex);

			} else {

				IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
				    ibmf_i_recv_timeout, IBMF_TNF_TRACE, "",
				    "ibmf_i_recv_timeout(): calling "
				    "notify %p\n", tnf_opaque, msgp, msgimplp);

				ibmf_i_notify_client(msgimplp);
			}
		}
	}

	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
	    ibmf_i_recv_timeout_end, IBMF_TNF_TRACE, "",
	    "ibmf_i_recv_timeout() exit\n");
}

/*
 * ibmf_i_send_timeout:
 *
 *	Perform "send" timeout processing for the message.
 *	This timeout handler is used in non-RMPP and RMPP processing.
 */
void
ibmf_i_send_timeout(void *argp)
{
	ibmf_msg_impl_t *msgimplp = (ibmf_msg_impl_t *)argp;
	ibmf_client_t	*clientp = (ibmf_client_t *)msgimplp->im_client;
	ibmf_rmpp_ctx_t	*rmpp_ctx;
	int		msg_flags;
	uint_t		ref_cnt;
	int		status;

	IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L4,
	    ibmf_i_send_timeout_start, IBMF_TNF_TRACE, "",
	    "ibmf_i_send_timeout_client(): msgp = 0x%p mgt_class = 0x%x "
	    "local lid 0x%x remote lid 0x%x remote q# 0x%x\n",
	    tnf_opaque, msg, msgimplp,
	    tnf_uint, mgt_class, msgimplp->im_mgt_class,
	    tnf_uint, local_lid, msgimplp->im_local_addr.ia_local_lid,
	    tnf_uint, remote_lid, msgimplp->im_local_addr.ia_remote_lid,
	    tnf_uint, qno, msgimplp->im_local_addr.ia_remote_qno);

	mutex_enter(&msgimplp->im_mutex);

	/*
	 * If the message has been marked uninitialized or done, release the
	 * message mutex and return
	 */
	if ((msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_UNINIT) ||
	    (msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE)) {

		mutex_exit(&msgimplp->im_mutex);

		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
		    ibmf_i_send_timeout, IBMF_TNF_TRACE, "",
		    "ibmf_i_send_timeout(): %s, msgp = 0x%p\n", tnf_string, msg,
		    "Message is done, return without processing send timeout",
		    tnf_opaque, msgimplp, msgimplp);

		return;
	}

	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_send_timeout,
	    IBMF_TNF_TRACE, "", "ibmf_i_send_timeout(): resetting id %d\n",
	    tnf_opaque, timeout_id, msgimplp->im_rp_timeout_id);

	/*
	 * If the timer fired, but the corresponding MAD was received before
	 * we got to this point in the timeout code, then do nothing in the
	 * timeout handler and return
	 */
	if ((msgimplp->im_flags & IBMF_MSG_FLAGS_RECV_RMPP) &&
	    (msgimplp->im_rp_timeout_id == 0)) {

		mutex_exit(&msgimplp->im_mutex);

		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
		    ibmf_i_send_timeout, IBMF_TNF_TRACE, "",
		    "ibmf_i_send_timeout(): %s, msgp = 0x%p\n", tnf_string, msg,
		    "Message not in undefined state, return without processing "
		    "send timeout",
		    tnf_opaque, msgimplp, msgimplp);

		return;
	}

	/* Clear the response timer */
	if (msgimplp->im_rp_timeout_id != 0)
		ibmf_i_unset_timer(msgimplp, IBMF_RESP_TIMER);

	rmpp_ctx = &msgimplp->im_rmpp_ctx;

	/*
	 * Non-RMPP send transaction timeout processing
	 */
	if ((msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP) == 0) {

		/*
		 * We use the RMPP context to store the retry count even if
		 * the response does not use RMPP
		 */
		if (rmpp_ctx->rmpp_retry_cnt <
		    msgimplp->im_retrans.retrans_retries) {

			IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
			    ibmf_i_send_timeout, IBMF_TNF_TRACE, "",
			    "ibmf_i_send_timeout(): %s, msgp = 0x%p, "
			    "retry_cnt = %d, max_retries = %d\n",
			    tnf_string, msg, "Non-RMPP send timed out",
			    tnf_opaque, msgimplp, msgimplp,
			    tnf_uint, retry_cnt, rmpp_ctx->rmpp_retry_cnt,
			    tnf_uint, max_retries,
			    msgimplp->im_retrans.retrans_retries);

			rmpp_ctx->rmpp_retry_cnt++;

			status = ibmf_i_send_single_pkt(msgimplp->im_client,
			    msgimplp->im_qp_hdl, msgimplp, IBMF_NO_BLOCK);
			if (status == IBMF_SUCCESS) {

				mutex_exit(&msgimplp->im_mutex);

				IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
				    ibmf_i_send_timeout, IBMF_TNF_TRACE, "",
				    "ibmf_i_send_timeout(): %s, msgp = 0x%p\n",
				    tnf_string, msg, "Resent send",
				    tnf_opaque, msgimplp, msgimplp);

				return;
			}

			IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1,
			    ibmf_i_send_timeout, IBMF_TNF_ERROR, "",
			    "ibmf_i_send_timeout(): %s, msgp = 0x%p, "
			    "status = %d\n", tnf_string, msg,
			    "Retry send failed; terminating transaction",
			    tnf_opaque, msgimplp, msgimplp,
			    tnf_opaque, status, status);

		} else {

			IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
			    ibmf_i_send_timeout, IBMF_TNF_ERROR, "",
			    "ibmf_i_send_timeout(): %s\n",  tnf_string, msg,
			    "Not RMPP SEND, terminate transaction with "
			    "IBMF_TRANS_TIMEOUT");
		}

		/*
		 * If we are in receive RMPP mode, then an ABORT should
		 * be sent after the required number of retries.
		 */
		if (msgimplp->im_flags & IBMF_MSG_FLAGS_RECV_RMPP) {
			status = ibmf_i_send_rmpp(msgimplp,
			    IBMF_RMPP_TYPE_ABORT, IBMF_RMPP_STATUS_TMR, 0, 0,
			    IBMF_NO_BLOCK);
			if (status != IBMF_SUCCESS) {
				IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
				    ibmf_i_send_timeout_err, IBMF_TNF_ERROR, "",
				    "ibmf_i_send_timeout(): %s\n", tnf_string,
				    msg, "RMPP ABORT send failed");
				msgimplp->im_trans_state_flags |=
				    IBMF_TRANS_STATE_FLAG_SEND_DONE;
			}
			rmpp_ctx->rmpp_state = IBMF_RMPP_STATE_ABORT;
		}

		ibmf_i_terminate_transaction(msgimplp->im_client,
		    msgimplp, IBMF_TRANS_TIMEOUT);

		msg_flags = msgimplp->im_trans_state_flags;

		mutex_exit(&msgimplp->im_mutex);

		/* Notify the client if the transaction is done */
		if (msg_flags & IBMF_TRANS_STATE_FLAG_DONE) {

			IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
			    ibmf_i_send_timeout, IBMF_TNF_TRACE, "",
			    "ibmf_i_send_timeout(): %s, msgp = 0x%p\n",
			    tnf_string, msg, "calling notify",
			    tnf_opaque, msgimplp, msgimplp);
			/* Remove the message from the client's message list */
			ibmf_i_client_rem_msg(clientp, msgimplp, &ref_cnt);
			/*
			 * Notify the client if the message reference count is
			 * zero. At this point, we know that the transaction is
			 * done and the message has been removed from the
			 * client's message list. So, we need to be sure the
			 * reference count is zero before notifying the client.
			 */
			if (ref_cnt == 0) {
				ibmf_i_notify_client(msgimplp);
			}
		}

		IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_timeout,
		    IBMF_TNF_TRACE, "", "ibmf_i_send_timeout() exit\n");

		return;
	}

	IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
	    ibmf_i_send_timeout, IBMF_TNF_TRACE, "",
	    "ibmf_i_send_timeout(): %s, msgp = 0x%p, retry_cnt = %d, "
	    "max_retries = %d\n", tnf_string, msg, "RMPP send timed out",
	    tnf_opaque, msgimplp, msgimplp,
	    tnf_uint, retry_cnt, rmpp_ctx->rmpp_retry_cnt,
	    tnf_uint, max_retries, msgimplp->im_retrans.retrans_retries);

	/* RMPP send transaction timeout processing */
	if (rmpp_ctx->rmpp_retry_cnt == msgimplp->im_retrans.retrans_retries) {

		IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
		    ibmf_i_send_timeout, IBMF_TNF_TRACE, "",
		    "ibmf_i_send_timeout(): %s\n", tnf_string, msg,
		    "Maximum retries done, sending ABORT TMR");

		status = ibmf_i_send_rmpp(msgimplp, IBMF_RMPP_TYPE_ABORT,
		    IBMF_RMPP_STATUS_TMR, 0, 0, IBMF_NO_BLOCK);
		if (status != IBMF_SUCCESS) {
			IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
			    ibmf_i_send_timeout_err, IBMF_TNF_ERROR, "",
			    "ibmf_i_send_timeout(): %s\n", tnf_string, msg,
			    "RMPP ABORT send failed");
			msgimplp->im_trans_state_flags |=
			    IBMF_TRANS_STATE_FLAG_SEND_DONE;
		}

		rmpp_ctx->rmpp_state = IBMF_RMPP_STATE_ABORT;

		mutex_enter(&clientp->ic_kstat_mutex);
		IBMF_ADD32_KSTATS(clientp, rmpp_errors, 1);
		mutex_exit(&clientp->ic_kstat_mutex);

		IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
		    ibmf_i_send_timeout, IBMF_TNF_ERROR, "",
		    "ibmf_i_send_timeout(): %s\n", tnf_string, msg,
		    "Maximum retries done, terminate transaction with "
		    "IBMF_TRANS_TIMEOUT");

		ibmf_i_terminate_transaction(msgimplp->im_client,
		    msgimplp, IBMF_TRANS_TIMEOUT);

	} else {

		if (rmpp_ctx->rmpp_state == IBMF_RMPP_STATE_SENDER_ACTIVE) {

			IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
			    ibmf_i_send_timeout, IBMF_TNF_TRACE, "",
			    "ibmf_i_send_timeout(): %s\n", tnf_string, msg,
			    "RMPP context is Sender Active, Resending window");

			/*
			 * resend the window
			 */
			rmpp_ctx->rmpp_ns = rmpp_ctx->rmpp_wf;

			ibmf_i_send_rmpp_window(msgimplp, IBMF_NO_BLOCK);
		} else if (rmpp_ctx->rmpp_state ==
		    IBMF_RMPP_STATE_SENDER_SWITCH) {

			IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
			    ibmf_i_send_timeout, IBMF_TNF_TRACE, "",
			    "ibmf_i_send_timeout(): %s\n", tnf_string, msg,
			    "RMPP context is Sender Terminate, sending ACK");

			/* send ACK */
			(void) ibmf_i_send_rmpp(msgimplp, IBMF_RMPP_TYPE_ACK,
			    IBMF_RMPP_STATUS_NORMAL, 0, 1, IBMF_NO_BLOCK);

			IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
			    ibmf_i_send_timeout, IBMF_TNF_TRACE, "",
			    "ibmf_i_send_timeout(): setting timer %d %p\n",
			    tnf_opaque, msgp, msgimplp, tnf_opaque,
			    timeout_id, msgimplp->im_rp_timeout_id);

			/* set response timer */
			ibmf_i_set_timer(ibmf_i_send_timeout, msgimplp,
			    IBMF_RESP_TIMER);
		}

		rmpp_ctx->rmpp_retry_cnt++;

	}

	msg_flags = msgimplp->im_trans_state_flags;

	mutex_exit(&msgimplp->im_mutex);

	clientp = (ibmf_client_t *)msgimplp->im_client;

	IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
	    ibmf_i_send_timeout, IBMF_TNF_TRACE, "",
	    "ibmf_i_send_timeout(): %s, msgp = 0x%p\n", tnf_string, msg,
	    "Send timeout done", tnf_opaque, msgimplp, msgimplp);

	if (msg_flags & IBMF_TRANS_STATE_FLAG_DONE) {
		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
		    ibmf_i_send_timeout, IBMF_TNF_TRACE, "",
		    "ibmf_i_send_timeout(): %s, msgp = 0x%p\n", tnf_string, msg,
		    "calling notify", tnf_opaque, msgimplp, msgimplp);
		/* Remove the message from the client's message list */
		ibmf_i_client_rem_msg(clientp, msgimplp, &ref_cnt);
		/*
		 * Notify the client if the message reference count is zero.
		 * At this point, we know that the transaction is done and
		 * the message has been removed from the client's message list.
		 * So, we only need to make sure the reference count is zero
		 * before notifying the client.
		 */
		if (ref_cnt == 0) {
			ibmf_i_notify_client(msgimplp);
		}
	}

	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_timeout_end,
	    IBMF_TNF_TRACE, "", "ibmf_i_send_timeout() exit\n");
}

void
ibmf_i_err_terminate_timeout(void *argp)
{
	ibmf_msg_impl_t *msgimplp = (ibmf_msg_impl_t *)argp;
	ibmf_client_t	*clientp = (ibmf_client_t *)msgimplp->im_client;
	int		msg_flags;
	uint_t		ref_cnt;

	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4,
	    ibmf_i_err_terminate_timeout_start, IBMF_TNF_TRACE, "",
	    "ibmf_i_err_terminate_timeout_client(): msgp = 0x%p\n",
	    tnf_opaque, msg, msgimplp);

	mutex_enter(&msgimplp->im_mutex);

	/*
	 * If the message has been marked uninitialized or done, release the
	 * message mutex and return
	 */
	if ((msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_UNINIT) ||
	    (msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE)) {

		mutex_exit(&msgimplp->im_mutex);

		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
		    ibmf_i_err_terminate_timeout, IBMF_TNF_TRACE, "",
		    "ibmf_i_err_terminate_timeout(): %s, msgp = 0x%p\n",
		    tnf_string, msg, "Message is done, return without "
		    "processing error terminate timeout",
		    tnf_opaque, msgimplp, msgimplp);

		return;
	}

	IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_err_terminate_timeout,
	    IBMF_TNF_TRACE, "", "ibmf_i_err_terminate_timeout(): resetting "
	    "id %d\n", tnf_opaque, timeout_id, msgimplp->im_rp_timeout_id);

	/* Clear the response timer */
	if (msgimplp->im_rp_timeout_id != 0)
		msgimplp->im_rp_timeout_id = 0;

	/* Mark the transaction as terminated */
	ibmf_i_terminate_transaction(msgimplp->im_client, msgimplp,
	    IBMF_TRANS_FAILURE);

	msg_flags = msgimplp->im_trans_state_flags;

	mutex_exit(&msgimplp->im_mutex);

	clientp = (ibmf_client_t *)msgimplp->im_client;

	IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_err_terminate_timeout,
	    IBMF_TNF_TRACE, "", "ibmf_i_err_terminate_timeout(): %s, "
	    "msgp = 0x%p\n", tnf_string, msg,
	    "Error terminate timeout done", tnf_opaque, msgimplp, msgimplp);

	if (msg_flags & IBMF_TRANS_STATE_FLAG_DONE) {
		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
		    ibmf_i_err_terminate_timeout, IBMF_TNF_TRACE, "",
		    "ibmf_i_err_terminate_timeout(): %s, msgp = 0x%p\n",
		    tnf_string, msg,
		    "calling notify", tnf_opaque, msgimplp, msgimplp);
		/* Remove the message from the client's message list */
		ibmf_i_client_rem_msg(clientp, msgimplp, &ref_cnt);
		/*
		 * Notify the client if the message reference count is zero.
		 * At this point, we know that the transaction is done and
		 * the message has been removed from the client's message list.
		 * So, we only need to make sure the reference count is zero
		 * before notifying the client.
		 */
		if (ref_cnt == 0) {
			_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msgimplp))
			if (msgimplp->im_flags & IBMF_MSG_FLAGS_TERMINATION) {

				/*
				 * If the message is a termination message,
				 * free it at this time.
				 */

				IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
				    ibmf_i_err_terminate_timeout,
				    IBMF_TNF_TRACE, "",
				    "ibmf_i_recv_timeout(): freeing terminate "
				    "message %p\n", tnf_opaque, msgp, msgimplp);

				/* free up the UD destination resource */
				if (msgimplp->im_ibmf_ud_dest != NULL) {
					ibmf_i_free_ud_dest(clientp, msgimplp);
					ibmf_i_clean_ud_dest_list(
					    clientp->ic_myci, B_FALSE);
				}

				/* Free the receive buffer */
				kmem_free(
				    msgimplp->im_msgbufs_recv.im_bufs_mad_hdr,
				    IBMF_MAD_SIZE);

				/* destroy the message mutex */
				mutex_destroy(&msgimplp->im_mutex);

				/* Free the termination message context */
				kmem_free(msgimplp, sizeof (ibmf_msg_impl_t));

				/*
				 * Decrease the "messages allocated" count
				 * so that an ibmf_unregister() can succeed
				 * for this client.
				 */
				mutex_enter(&clientp->ic_mutex);
				clientp->ic_msgs_alloced--;
				mutex_exit(&clientp->ic_mutex);

			} else {

				IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
				    ibmf_i_err_terminate_timeout,
				    IBMF_TNF_TRACE, "",
				    "ibmf_i_recv_timeout(): calling "
				    "notify %p\n", tnf_opaque, msgp, msgimplp);

				ibmf_i_notify_client(msgimplp);
			}
		}
	}

	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
	    ibmf_i_err_terminate_timeout_end, IBMF_TNF_TRACE, "",
	    "ibmf_i_err_terminate_timeout() exit\n");
}