17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
5f317a3a3Skrishna * Common Development and Distribution License (the "License").
6f317a3a3Skrishna * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
229b009fc1SValerie Bubb Fenwick * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
237c478bd9Sstevel@tonic-gate */
247c478bd9Sstevel@tonic-gate
256ea3c060SGarrett D'Amore /*
2657a30242SGarrett D'Amore * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
276ea3c060SGarrett D'Amore */
286ea3c060SGarrett D'Amore
297c478bd9Sstevel@tonic-gate /*
307c478bd9Sstevel@tonic-gate * This file contains the core framework routines for the
317c478bd9Sstevel@tonic-gate * kernel cryptographic framework. These routines are at the
327c478bd9Sstevel@tonic-gate * layer, between the kernel API/ioctls and the SPI.
337c478bd9Sstevel@tonic-gate */
347c478bd9Sstevel@tonic-gate
357c478bd9Sstevel@tonic-gate #include <sys/types.h>
367c478bd9Sstevel@tonic-gate #include <sys/errno.h>
377c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
387c478bd9Sstevel@tonic-gate #include <sys/proc.h>
397c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
407c478bd9Sstevel@tonic-gate #include <sys/cpupart.h>
417c478bd9Sstevel@tonic-gate #include <sys/ksynch.h>
427c478bd9Sstevel@tonic-gate #include <sys/callb.h>
437c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
447c478bd9Sstevel@tonic-gate #include <sys/systm.h>
457c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
467c478bd9Sstevel@tonic-gate #include <sys/kstat.h>
477c478bd9Sstevel@tonic-gate #include <sys/crypto/common.h>
487c478bd9Sstevel@tonic-gate #include <sys/crypto/impl.h>
497c478bd9Sstevel@tonic-gate #include <sys/crypto/sched_impl.h>
507c478bd9Sstevel@tonic-gate #include <sys/crypto/api.h>
517c478bd9Sstevel@tonic-gate #include <sys/crypto/spi.h>
527c478bd9Sstevel@tonic-gate #include <sys/taskq_impl.h>
537c478bd9Sstevel@tonic-gate #include <sys/ddi.h>
547c478bd9Sstevel@tonic-gate #include <sys/sunddi.h>
557c478bd9Sstevel@tonic-gate
567c478bd9Sstevel@tonic-gate
577c478bd9Sstevel@tonic-gate kcf_global_swq_t *gswq; /* Global software queue */
587c478bd9Sstevel@tonic-gate
597c478bd9Sstevel@tonic-gate /* Thread pool related variables */
607c478bd9Sstevel@tonic-gate static kcf_pool_t *kcfpool; /* Thread pool of kcfd LWPs */
61455859fbSDarren Moffat int kcf_maxthreads = 2;
62455859fbSDarren Moffat int kcf_minthreads = 1;
637c478bd9Sstevel@tonic-gate int kcf_thr_multiple = 2; /* Boot-time tunable for experimentation */
647c478bd9Sstevel@tonic-gate static ulong_t kcf_idlethr_timeout;
657c478bd9Sstevel@tonic-gate static boolean_t kcf_sched_running = B_FALSE;
667c478bd9Sstevel@tonic-gate #define KCF_DEFAULT_THRTIMEOUT 60000000 /* 60 seconds */
677c478bd9Sstevel@tonic-gate
687c478bd9Sstevel@tonic-gate /* kmem caches used by the scheduler */
697c478bd9Sstevel@tonic-gate static struct kmem_cache *kcf_sreq_cache;
707c478bd9Sstevel@tonic-gate static struct kmem_cache *kcf_areq_cache;
717c478bd9Sstevel@tonic-gate static struct kmem_cache *kcf_context_cache;
727c478bd9Sstevel@tonic-gate
737c478bd9Sstevel@tonic-gate /* Global request ID table */
747c478bd9Sstevel@tonic-gate static kcf_reqid_table_t *kcf_reqid_table[REQID_TABLES];
757c478bd9Sstevel@tonic-gate
767c478bd9Sstevel@tonic-gate /* KCF stats. Not protected. */
777c478bd9Sstevel@tonic-gate static kcf_stats_t kcf_ksdata = {
787c478bd9Sstevel@tonic-gate { "total threads in pool", KSTAT_DATA_UINT32},
797c478bd9Sstevel@tonic-gate { "idle threads in pool", KSTAT_DATA_UINT32},
807c478bd9Sstevel@tonic-gate { "min threads in pool", KSTAT_DATA_UINT32},
817c478bd9Sstevel@tonic-gate { "max threads in pool", KSTAT_DATA_UINT32},
827c478bd9Sstevel@tonic-gate { "requests in gswq", KSTAT_DATA_UINT32},
837c478bd9Sstevel@tonic-gate { "max requests in gswq", KSTAT_DATA_UINT32},
84c41e7ccaSkrishna { "threads for HW taskq", KSTAT_DATA_UINT32},
85c41e7ccaSkrishna { "minalloc for HW taskq", KSTAT_DATA_UINT32},
86c41e7ccaSkrishna { "maxalloc for HW taskq", KSTAT_DATA_UINT32}
877c478bd9Sstevel@tonic-gate };
887c478bd9Sstevel@tonic-gate
897c478bd9Sstevel@tonic-gate static kstat_t *kcf_misc_kstat = NULL;
907c478bd9Sstevel@tonic-gate ulong_t kcf_swprov_hndl = 0;
917c478bd9Sstevel@tonic-gate
927c478bd9Sstevel@tonic-gate static kcf_areq_node_t *kcf_areqnode_alloc(kcf_provider_desc_t *,
937c478bd9Sstevel@tonic-gate kcf_context_t *, crypto_call_req_t *, kcf_req_params_t *, boolean_t);
947c478bd9Sstevel@tonic-gate static int kcf_disp_sw_request(kcf_areq_node_t *);
957c478bd9Sstevel@tonic-gate static void process_req_hwp(void *);
966ea3c060SGarrett D'Amore static kcf_areq_node_t *kcf_dequeue(void);
977c478bd9Sstevel@tonic-gate static int kcf_enqueue(kcf_areq_node_t *);
986ea3c060SGarrett D'Amore static void kcfpool_alloc(void);
997c478bd9Sstevel@tonic-gate static void kcf_reqid_delete(kcf_areq_node_t *areq);
1007c478bd9Sstevel@tonic-gate static crypto_req_id_t kcf_reqid_insert(kcf_areq_node_t *areq);
1017c478bd9Sstevel@tonic-gate static int kcf_misc_kstat_update(kstat_t *ksp, int rw);
1026ea3c060SGarrett D'Amore static void compute_min_max_threads(void);
1036ea3c060SGarrett D'Amore static void kcfpool_svc(void *);
1046ea3c060SGarrett D'Amore static void kcfpoold(void *);
1057c478bd9Sstevel@tonic-gate
1067c478bd9Sstevel@tonic-gate
1077c478bd9Sstevel@tonic-gate /*
1087c478bd9Sstevel@tonic-gate * Create a new context.
1097c478bd9Sstevel@tonic-gate */
1107c478bd9Sstevel@tonic-gate crypto_ctx_t *
kcf_new_ctx(crypto_call_req_t * crq,kcf_provider_desc_t * pd,crypto_session_id_t sid)1117c478bd9Sstevel@tonic-gate kcf_new_ctx(crypto_call_req_t *crq, kcf_provider_desc_t *pd,
1127c478bd9Sstevel@tonic-gate crypto_session_id_t sid)
1137c478bd9Sstevel@tonic-gate {
1147c478bd9Sstevel@tonic-gate crypto_ctx_t *ctx;
1157c478bd9Sstevel@tonic-gate kcf_context_t *kcf_ctx;
1167c478bd9Sstevel@tonic-gate
1177c478bd9Sstevel@tonic-gate kcf_ctx = kmem_cache_alloc(kcf_context_cache,
1187c478bd9Sstevel@tonic-gate (crq == NULL) ? KM_SLEEP : KM_NOSLEEP);
1197c478bd9Sstevel@tonic-gate if (kcf_ctx == NULL)
1207c478bd9Sstevel@tonic-gate return (NULL);
1217c478bd9Sstevel@tonic-gate
1227c478bd9Sstevel@tonic-gate /* initialize the context for the consumer */
1237c478bd9Sstevel@tonic-gate kcf_ctx->kc_refcnt = 1;
1247c478bd9Sstevel@tonic-gate kcf_ctx->kc_req_chain_first = NULL;
1257c478bd9Sstevel@tonic-gate kcf_ctx->kc_req_chain_last = NULL;
1267c478bd9Sstevel@tonic-gate kcf_ctx->kc_secondctx = NULL;
1277c478bd9Sstevel@tonic-gate KCF_PROV_REFHOLD(pd);
1287c478bd9Sstevel@tonic-gate kcf_ctx->kc_prov_desc = pd;
1296a1073f8Skrishna kcf_ctx->kc_sw_prov_desc = NULL;
1306a1073f8Skrishna kcf_ctx->kc_mech = NULL;
1317c478bd9Sstevel@tonic-gate
1327c478bd9Sstevel@tonic-gate ctx = &kcf_ctx->kc_glbl_ctx;
1337c478bd9Sstevel@tonic-gate ctx->cc_provider = pd->pd_prov_handle;
1347c478bd9Sstevel@tonic-gate ctx->cc_session = sid;
1357c478bd9Sstevel@tonic-gate ctx->cc_provider_private = NULL;
1367c478bd9Sstevel@tonic-gate ctx->cc_framework_private = (void *)kcf_ctx;
1376a1073f8Skrishna ctx->cc_flags = 0;
1386a1073f8Skrishna ctx->cc_opstate = NULL;
1397c478bd9Sstevel@tonic-gate
1407c478bd9Sstevel@tonic-gate return (ctx);
1417c478bd9Sstevel@tonic-gate }
1427c478bd9Sstevel@tonic-gate
1437c478bd9Sstevel@tonic-gate /*
1447c478bd9Sstevel@tonic-gate * Allocate a new async request node.
1457c478bd9Sstevel@tonic-gate *
1467c478bd9Sstevel@tonic-gate * ictx - Framework private context pointer
1477c478bd9Sstevel@tonic-gate * crq - Has callback function and argument. Should be non NULL.
1487c478bd9Sstevel@tonic-gate * req - The parameters to pass to the SPI
1497c478bd9Sstevel@tonic-gate */
1507c478bd9Sstevel@tonic-gate static kcf_areq_node_t *
kcf_areqnode_alloc(kcf_provider_desc_t * pd,kcf_context_t * ictx,crypto_call_req_t * crq,kcf_req_params_t * req,boolean_t isdual)1517c478bd9Sstevel@tonic-gate kcf_areqnode_alloc(kcf_provider_desc_t *pd, kcf_context_t *ictx,
1527c478bd9Sstevel@tonic-gate crypto_call_req_t *crq, kcf_req_params_t *req, boolean_t isdual)
1537c478bd9Sstevel@tonic-gate {
1547c478bd9Sstevel@tonic-gate kcf_areq_node_t *arptr, *areq;
1557c478bd9Sstevel@tonic-gate
1567c478bd9Sstevel@tonic-gate ASSERT(crq != NULL);
1577c478bd9Sstevel@tonic-gate arptr = kmem_cache_alloc(kcf_areq_cache, KM_NOSLEEP);
1587c478bd9Sstevel@tonic-gate if (arptr == NULL)
1597c478bd9Sstevel@tonic-gate return (NULL);
1607c478bd9Sstevel@tonic-gate
1617c478bd9Sstevel@tonic-gate arptr->an_state = REQ_ALLOCATED;
1627c478bd9Sstevel@tonic-gate arptr->an_reqarg = *crq;
1637c478bd9Sstevel@tonic-gate arptr->an_params = *req;
1647c478bd9Sstevel@tonic-gate arptr->an_context = ictx;
1657c478bd9Sstevel@tonic-gate arptr->an_isdual = isdual;
1667c478bd9Sstevel@tonic-gate
1677c478bd9Sstevel@tonic-gate arptr->an_next = arptr->an_prev = NULL;
1687c478bd9Sstevel@tonic-gate KCF_PROV_REFHOLD(pd);
1697c478bd9Sstevel@tonic-gate arptr->an_provider = pd;
1707c478bd9Sstevel@tonic-gate arptr->an_tried_plist = NULL;
1717c478bd9Sstevel@tonic-gate arptr->an_refcnt = 1;
1727c478bd9Sstevel@tonic-gate arptr->an_idnext = arptr->an_idprev = NULL;
1737c478bd9Sstevel@tonic-gate
1747c478bd9Sstevel@tonic-gate /*
1757c478bd9Sstevel@tonic-gate * Requests for context-less operations do not use the
1767c478bd9Sstevel@tonic-gate * fields - an_is_my_turn, and an_ctxchain_next.
1777c478bd9Sstevel@tonic-gate */
1787c478bd9Sstevel@tonic-gate if (ictx == NULL)
1797c478bd9Sstevel@tonic-gate return (arptr);
1807c478bd9Sstevel@tonic-gate
1817c478bd9Sstevel@tonic-gate KCF_CONTEXT_REFHOLD(ictx);
1827c478bd9Sstevel@tonic-gate /*
1837c478bd9Sstevel@tonic-gate * Chain this request to the context.
1847c478bd9Sstevel@tonic-gate */
1857c478bd9Sstevel@tonic-gate mutex_enter(&ictx->kc_in_use_lock);
1867c478bd9Sstevel@tonic-gate arptr->an_ctxchain_next = NULL;
1877c478bd9Sstevel@tonic-gate if ((areq = ictx->kc_req_chain_last) == NULL) {
1887c478bd9Sstevel@tonic-gate arptr->an_is_my_turn = B_TRUE;
1897c478bd9Sstevel@tonic-gate ictx->kc_req_chain_last =
1907c478bd9Sstevel@tonic-gate ictx->kc_req_chain_first = arptr;
1917c478bd9Sstevel@tonic-gate } else {
1927c478bd9Sstevel@tonic-gate ASSERT(ictx->kc_req_chain_first != NULL);
1937c478bd9Sstevel@tonic-gate arptr->an_is_my_turn = B_FALSE;
1947c478bd9Sstevel@tonic-gate /* Insert the new request to the end of the chain. */
1957c478bd9Sstevel@tonic-gate areq->an_ctxchain_next = arptr;
1967c478bd9Sstevel@tonic-gate ictx->kc_req_chain_last = arptr;
1977c478bd9Sstevel@tonic-gate }
1987c478bd9Sstevel@tonic-gate mutex_exit(&ictx->kc_in_use_lock);
1997c478bd9Sstevel@tonic-gate
2007c478bd9Sstevel@tonic-gate return (arptr);
2017c478bd9Sstevel@tonic-gate }
2027c478bd9Sstevel@tonic-gate
2037c478bd9Sstevel@tonic-gate /*
2047c478bd9Sstevel@tonic-gate * Queue the request node and do one of the following:
2057c478bd9Sstevel@tonic-gate * - If there is an idle thread signal it to run.
2066ea3c060SGarrett D'Amore * - Else, signal the creator thread to possibly create more threads.
2077c478bd9Sstevel@tonic-gate */
2087c478bd9Sstevel@tonic-gate static int
kcf_disp_sw_request(kcf_areq_node_t * areq)2097c478bd9Sstevel@tonic-gate kcf_disp_sw_request(kcf_areq_node_t *areq)
2107c478bd9Sstevel@tonic-gate {
2117c478bd9Sstevel@tonic-gate int err;
2127c478bd9Sstevel@tonic-gate
2137c478bd9Sstevel@tonic-gate if ((err = kcf_enqueue(areq)) != 0)
2147c478bd9Sstevel@tonic-gate return (err);
2157c478bd9Sstevel@tonic-gate
2167c478bd9Sstevel@tonic-gate if (kcfpool->kp_idlethreads > 0) {
2177c478bd9Sstevel@tonic-gate /* Signal an idle thread to run */
2187c478bd9Sstevel@tonic-gate mutex_enter(&gswq->gs_lock);
2197c478bd9Sstevel@tonic-gate cv_signal(&gswq->gs_cv);
2207c478bd9Sstevel@tonic-gate mutex_exit(&gswq->gs_lock);
2217c478bd9Sstevel@tonic-gate
2227c478bd9Sstevel@tonic-gate return (CRYPTO_QUEUED);
2237c478bd9Sstevel@tonic-gate }
2247c478bd9Sstevel@tonic-gate
2256ea3c060SGarrett D'Amore /* Signal the creator thread for more threads */
2266ea3c060SGarrett D'Amore mutex_enter(&kcfpool->kp_lock);
2276ea3c060SGarrett D'Amore cv_signal(&kcfpool->kp_cv);
2286ea3c060SGarrett D'Amore mutex_exit(&kcfpool->kp_lock);
2297c478bd9Sstevel@tonic-gate
2307c478bd9Sstevel@tonic-gate return (CRYPTO_QUEUED);
2317c478bd9Sstevel@tonic-gate }
2327c478bd9Sstevel@tonic-gate
2337c478bd9Sstevel@tonic-gate /*
2347c478bd9Sstevel@tonic-gate * This routine is called by the taskq associated with
2357c478bd9Sstevel@tonic-gate * each hardware provider. We notify the kernel consumer
2367c478bd9Sstevel@tonic-gate * via the callback routine in case of CRYPTO_SUCCESS or
2377c478bd9Sstevel@tonic-gate * a failure.
2387c478bd9Sstevel@tonic-gate *
2397c478bd9Sstevel@tonic-gate * A request can be of type kcf_areq_node_t or of type
2407c478bd9Sstevel@tonic-gate * kcf_sreq_node_t.
2417c478bd9Sstevel@tonic-gate */
2427c478bd9Sstevel@tonic-gate static void
process_req_hwp(void * ireq)2437c478bd9Sstevel@tonic-gate process_req_hwp(void *ireq)
2447c478bd9Sstevel@tonic-gate {
2457c478bd9Sstevel@tonic-gate int error = 0;
2467c478bd9Sstevel@tonic-gate crypto_ctx_t *ctx;
2477c478bd9Sstevel@tonic-gate kcf_call_type_t ctype;
2487c478bd9Sstevel@tonic-gate kcf_provider_desc_t *pd;
2497c478bd9Sstevel@tonic-gate kcf_areq_node_t *areq = (kcf_areq_node_t *)ireq;
2507c478bd9Sstevel@tonic-gate kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)ireq;
251ef56a3c5SKrishna Yenduri kcf_prov_cpu_t *mp;
2527c478bd9Sstevel@tonic-gate
2537c478bd9Sstevel@tonic-gate pd = ((ctype = GET_REQ_TYPE(ireq)) == CRYPTO_SYNCH) ?
2547c478bd9Sstevel@tonic-gate sreq->sn_provider : areq->an_provider;
2557c478bd9Sstevel@tonic-gate
2567c478bd9Sstevel@tonic-gate /*
2577c478bd9Sstevel@tonic-gate * Wait if flow control is in effect for the provider. A
2587c478bd9Sstevel@tonic-gate * CRYPTO_PROVIDER_READY or CRYPTO_PROVIDER_FAILED
2597c478bd9Sstevel@tonic-gate * notification will signal us. We also get signaled if
2607c478bd9Sstevel@tonic-gate * the provider is unregistering.
2617c478bd9Sstevel@tonic-gate */
2622d794da1Skrishna if (pd->pd_state == KCF_PROV_BUSY) {
2632d794da1Skrishna mutex_enter(&pd->pd_lock);
2642d794da1Skrishna while (pd->pd_state == KCF_PROV_BUSY)
2652d794da1Skrishna cv_wait(&pd->pd_resume_cv, &pd->pd_lock);
2662d794da1Skrishna mutex_exit(&pd->pd_lock);
2672d794da1Skrishna }
2687c478bd9Sstevel@tonic-gate
2697c478bd9Sstevel@tonic-gate /*
2707c478bd9Sstevel@tonic-gate * Bump the internal reference count while the request is being
2717c478bd9Sstevel@tonic-gate * processed. This is how we know when it's safe to unregister
2727c478bd9Sstevel@tonic-gate * a provider. This step must precede the pd_state check below.
2737c478bd9Sstevel@tonic-gate */
274ef56a3c5SKrishna Yenduri mp = &(pd->pd_percpu_bins[CPU_SEQID]);
275ef56a3c5SKrishna Yenduri KCF_PROV_JOB_HOLD(mp);
2767c478bd9Sstevel@tonic-gate
2777c478bd9Sstevel@tonic-gate /*
2787c478bd9Sstevel@tonic-gate * Fail the request if the provider has failed. We return a
2797c478bd9Sstevel@tonic-gate * recoverable error and the notified clients attempt any
2807c478bd9Sstevel@tonic-gate * recovery. For async clients this is done in kcf_aop_done()
2817c478bd9Sstevel@tonic-gate * and for sync clients it is done in the k-api routines.
2827c478bd9Sstevel@tonic-gate */
2837c478bd9Sstevel@tonic-gate if (pd->pd_state >= KCF_PROV_FAILED) {
2847c478bd9Sstevel@tonic-gate error = CRYPTO_DEVICE_ERROR;
2857c478bd9Sstevel@tonic-gate goto bail;
2867c478bd9Sstevel@tonic-gate }
2877c478bd9Sstevel@tonic-gate
2887c478bd9Sstevel@tonic-gate if (ctype == CRYPTO_SYNCH) {
2897c478bd9Sstevel@tonic-gate mutex_enter(&sreq->sn_lock);
2907c478bd9Sstevel@tonic-gate sreq->sn_state = REQ_INPROGRESS;
291ef56a3c5SKrishna Yenduri sreq->sn_mp = mp;
2927c478bd9Sstevel@tonic-gate mutex_exit(&sreq->sn_lock);
2937c478bd9Sstevel@tonic-gate
2947c478bd9Sstevel@tonic-gate ctx = sreq->sn_context ? &sreq->sn_context->kc_glbl_ctx : NULL;
2957c478bd9Sstevel@tonic-gate error = common_submit_request(sreq->sn_provider, ctx,
2967c478bd9Sstevel@tonic-gate sreq->sn_params, sreq);
2977c478bd9Sstevel@tonic-gate } else {
298c41e7ccaSkrishna kcf_context_t *ictx;
2997c478bd9Sstevel@tonic-gate ASSERT(ctype == CRYPTO_ASYNCH);
3007c478bd9Sstevel@tonic-gate
3017c478bd9Sstevel@tonic-gate /*
3027c478bd9Sstevel@tonic-gate * We are in the per-hardware provider thread context and
3037c478bd9Sstevel@tonic-gate * hence can sleep. Note that the caller would have done
3047c478bd9Sstevel@tonic-gate * a taskq_dispatch(..., TQ_NOSLEEP) and would have returned.
3057c478bd9Sstevel@tonic-gate */
306c41e7ccaSkrishna ctx = (ictx = areq->an_context) ? &ictx->kc_glbl_ctx : NULL;
307c41e7ccaSkrishna
308c41e7ccaSkrishna mutex_enter(&areq->an_lock);
309c41e7ccaSkrishna /*
310c41e7ccaSkrishna * We need to maintain ordering for multi-part requests.
311c41e7ccaSkrishna * an_is_my_turn is set to B_TRUE initially for a request
312c41e7ccaSkrishna * when it is enqueued and there are no other requests
313c41e7ccaSkrishna * for that context. It is set later from kcf_aop_done() when
314c41e7ccaSkrishna * the request before us in the chain of requests for the
315c41e7ccaSkrishna * context completes. We get signaled at that point.
316c41e7ccaSkrishna */
317c41e7ccaSkrishna if (ictx != NULL) {
318c41e7ccaSkrishna ASSERT(ictx->kc_prov_desc == areq->an_provider);
319c41e7ccaSkrishna
320c41e7ccaSkrishna while (areq->an_is_my_turn == B_FALSE) {
321c41e7ccaSkrishna cv_wait(&areq->an_turn_cv, &areq->an_lock);
322c41e7ccaSkrishna }
323c41e7ccaSkrishna }
324c41e7ccaSkrishna areq->an_state = REQ_INPROGRESS;
325ef56a3c5SKrishna Yenduri areq->an_mp = mp;
326c41e7ccaSkrishna mutex_exit(&areq->an_lock);
327c41e7ccaSkrishna
3287c478bd9Sstevel@tonic-gate error = common_submit_request(areq->an_provider, ctx,
3297c478bd9Sstevel@tonic-gate &areq->an_params, areq);
3307c478bd9Sstevel@tonic-gate }
3317c478bd9Sstevel@tonic-gate
3327c478bd9Sstevel@tonic-gate bail:
3337c478bd9Sstevel@tonic-gate if (error == CRYPTO_QUEUED) {
3347c478bd9Sstevel@tonic-gate /*
3357c478bd9Sstevel@tonic-gate * The request is queued by the provider and we should
3367c478bd9Sstevel@tonic-gate * get a crypto_op_notification() from the provider later.
3377c478bd9Sstevel@tonic-gate * We notify the consumer at that time.
3387c478bd9Sstevel@tonic-gate */
3397c478bd9Sstevel@tonic-gate return;
3407c478bd9Sstevel@tonic-gate } else { /* CRYPTO_SUCCESS or other failure */
341ef56a3c5SKrishna Yenduri KCF_PROV_JOB_RELE(mp);
3427c478bd9Sstevel@tonic-gate if (ctype == CRYPTO_SYNCH)
3437c478bd9Sstevel@tonic-gate kcf_sop_done(sreq, error);
3447c478bd9Sstevel@tonic-gate else
3457c478bd9Sstevel@tonic-gate kcf_aop_done(areq, error);
3467c478bd9Sstevel@tonic-gate }
3477c478bd9Sstevel@tonic-gate }
3487c478bd9Sstevel@tonic-gate
3497c478bd9Sstevel@tonic-gate /*
3507c478bd9Sstevel@tonic-gate * This routine checks if a request can be retried on another
3517c478bd9Sstevel@tonic-gate * provider. If true, mech1 is initialized to point to the mechanism
3527c478bd9Sstevel@tonic-gate * structure. mech2 is also initialized in case of a dual operation. fg
3537c478bd9Sstevel@tonic-gate * is initialized to the correct crypto_func_group_t bit flag. They are
3547c478bd9Sstevel@tonic-gate * initialized by this routine, so that the caller can pass them to a
3557c478bd9Sstevel@tonic-gate * kcf_get_mech_provider() or kcf_get_dual_provider() with no further change.
3567c478bd9Sstevel@tonic-gate *
3577c478bd9Sstevel@tonic-gate * We check that the request is for a init or atomic routine and that
3587c478bd9Sstevel@tonic-gate * it is for one of the operation groups used from k-api .
3597c478bd9Sstevel@tonic-gate */
3607c478bd9Sstevel@tonic-gate static boolean_t
can_resubmit(kcf_areq_node_t * areq,crypto_mechanism_t ** mech1,crypto_mechanism_t ** mech2,crypto_func_group_t * fg)3617c478bd9Sstevel@tonic-gate can_resubmit(kcf_areq_node_t *areq, crypto_mechanism_t **mech1,
3627c478bd9Sstevel@tonic-gate crypto_mechanism_t **mech2, crypto_func_group_t *fg)
3637c478bd9Sstevel@tonic-gate {
3647c478bd9Sstevel@tonic-gate kcf_req_params_t *params;
3657c478bd9Sstevel@tonic-gate kcf_op_type_t optype;
3667c478bd9Sstevel@tonic-gate
3677c478bd9Sstevel@tonic-gate params = &areq->an_params;
3687c478bd9Sstevel@tonic-gate optype = params->rp_optype;
3697c478bd9Sstevel@tonic-gate
3707c478bd9Sstevel@tonic-gate if (!(IS_INIT_OP(optype) || IS_ATOMIC_OP(optype)))
3717c478bd9Sstevel@tonic-gate return (B_FALSE);
3727c478bd9Sstevel@tonic-gate
3737c478bd9Sstevel@tonic-gate switch (params->rp_opgrp) {
3747c478bd9Sstevel@tonic-gate case KCF_OG_DIGEST: {
3757c478bd9Sstevel@tonic-gate kcf_digest_ops_params_t *dops = ¶ms->rp_u.digest_params;
3767c478bd9Sstevel@tonic-gate
3777c478bd9Sstevel@tonic-gate dops->do_mech.cm_type = dops->do_framework_mechtype;
3787c478bd9Sstevel@tonic-gate *mech1 = &dops->do_mech;
3797c478bd9Sstevel@tonic-gate *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_DIGEST :
3807c478bd9Sstevel@tonic-gate CRYPTO_FG_DIGEST_ATOMIC;
3817c478bd9Sstevel@tonic-gate break;
3827c478bd9Sstevel@tonic-gate }
3837c478bd9Sstevel@tonic-gate
3847c478bd9Sstevel@tonic-gate case KCF_OG_MAC: {
3857c478bd9Sstevel@tonic-gate kcf_mac_ops_params_t *mops = ¶ms->rp_u.mac_params;
3867c478bd9Sstevel@tonic-gate
3877c478bd9Sstevel@tonic-gate mops->mo_mech.cm_type = mops->mo_framework_mechtype;
3887c478bd9Sstevel@tonic-gate *mech1 = &mops->mo_mech;
3897c478bd9Sstevel@tonic-gate *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_MAC :
3907c478bd9Sstevel@tonic-gate CRYPTO_FG_MAC_ATOMIC;
3917c478bd9Sstevel@tonic-gate break;
3927c478bd9Sstevel@tonic-gate }
3937c478bd9Sstevel@tonic-gate
3947c478bd9Sstevel@tonic-gate case KCF_OG_SIGN: {
3957c478bd9Sstevel@tonic-gate kcf_sign_ops_params_t *sops = ¶ms->rp_u.sign_params;
3967c478bd9Sstevel@tonic-gate
3977c478bd9Sstevel@tonic-gate sops->so_mech.cm_type = sops->so_framework_mechtype;
3987c478bd9Sstevel@tonic-gate *mech1 = &sops->so_mech;
3997c478bd9Sstevel@tonic-gate switch (optype) {
4007c478bd9Sstevel@tonic-gate case KCF_OP_INIT:
4017c478bd9Sstevel@tonic-gate *fg = CRYPTO_FG_SIGN;
4027c478bd9Sstevel@tonic-gate break;
4037c478bd9Sstevel@tonic-gate case KCF_OP_ATOMIC:
4047c478bd9Sstevel@tonic-gate *fg = CRYPTO_FG_SIGN_ATOMIC;
4057c478bd9Sstevel@tonic-gate break;
4067c478bd9Sstevel@tonic-gate default:
4077c478bd9Sstevel@tonic-gate ASSERT(optype == KCF_OP_SIGN_RECOVER_ATOMIC);
4087c478bd9Sstevel@tonic-gate *fg = CRYPTO_FG_SIGN_RECOVER_ATOMIC;
4097c478bd9Sstevel@tonic-gate }
4107c478bd9Sstevel@tonic-gate break;
4117c478bd9Sstevel@tonic-gate }
4127c478bd9Sstevel@tonic-gate
4137c478bd9Sstevel@tonic-gate case KCF_OG_VERIFY: {
4147c478bd9Sstevel@tonic-gate kcf_verify_ops_params_t *vops = ¶ms->rp_u.verify_params;
4157c478bd9Sstevel@tonic-gate
4167c478bd9Sstevel@tonic-gate vops->vo_mech.cm_type = vops->vo_framework_mechtype;
4177c478bd9Sstevel@tonic-gate *mech1 = &vops->vo_mech;
4187c478bd9Sstevel@tonic-gate switch (optype) {
4197c478bd9Sstevel@tonic-gate case KCF_OP_INIT:
4207c478bd9Sstevel@tonic-gate *fg = CRYPTO_FG_VERIFY;
4217c478bd9Sstevel@tonic-gate break;
4227c478bd9Sstevel@tonic-gate case KCF_OP_ATOMIC:
4237c478bd9Sstevel@tonic-gate *fg = CRYPTO_FG_VERIFY_ATOMIC;
4247c478bd9Sstevel@tonic-gate break;
4257c478bd9Sstevel@tonic-gate default:
4267c478bd9Sstevel@tonic-gate ASSERT(optype == KCF_OP_VERIFY_RECOVER_ATOMIC);
4277c478bd9Sstevel@tonic-gate *fg = CRYPTO_FG_VERIFY_RECOVER_ATOMIC;
4287c478bd9Sstevel@tonic-gate }
4297c478bd9Sstevel@tonic-gate break;
4307c478bd9Sstevel@tonic-gate }
4317c478bd9Sstevel@tonic-gate
4327c478bd9Sstevel@tonic-gate case KCF_OG_ENCRYPT: {
4337c478bd9Sstevel@tonic-gate kcf_encrypt_ops_params_t *eops = ¶ms->rp_u.encrypt_params;
4347c478bd9Sstevel@tonic-gate
4357c478bd9Sstevel@tonic-gate eops->eo_mech.cm_type = eops->eo_framework_mechtype;
4367c478bd9Sstevel@tonic-gate *mech1 = &eops->eo_mech;
4377c478bd9Sstevel@tonic-gate *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_ENCRYPT :
4387c478bd9Sstevel@tonic-gate CRYPTO_FG_ENCRYPT_ATOMIC;
4397c478bd9Sstevel@tonic-gate break;
4407c478bd9Sstevel@tonic-gate }
4417c478bd9Sstevel@tonic-gate
4427c478bd9Sstevel@tonic-gate case KCF_OG_DECRYPT: {
4437c478bd9Sstevel@tonic-gate kcf_decrypt_ops_params_t *dcrops = ¶ms->rp_u.decrypt_params;
4447c478bd9Sstevel@tonic-gate
4457c478bd9Sstevel@tonic-gate dcrops->dop_mech.cm_type = dcrops->dop_framework_mechtype;
4467c478bd9Sstevel@tonic-gate *mech1 = &dcrops->dop_mech;
4477c478bd9Sstevel@tonic-gate *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_DECRYPT :
4487c478bd9Sstevel@tonic-gate CRYPTO_FG_DECRYPT_ATOMIC;
4497c478bd9Sstevel@tonic-gate break;
4507c478bd9Sstevel@tonic-gate }
4517c478bd9Sstevel@tonic-gate
4527c478bd9Sstevel@tonic-gate case KCF_OG_ENCRYPT_MAC: {
4537c478bd9Sstevel@tonic-gate kcf_encrypt_mac_ops_params_t *eops =
4547c478bd9Sstevel@tonic-gate ¶ms->rp_u.encrypt_mac_params;
4557c478bd9Sstevel@tonic-gate
4567c478bd9Sstevel@tonic-gate eops->em_encr_mech.cm_type = eops->em_framework_encr_mechtype;
4577c478bd9Sstevel@tonic-gate *mech1 = &eops->em_encr_mech;
4587c478bd9Sstevel@tonic-gate eops->em_mac_mech.cm_type = eops->em_framework_mac_mechtype;
4597c478bd9Sstevel@tonic-gate *mech2 = &eops->em_mac_mech;
4607c478bd9Sstevel@tonic-gate *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_ENCRYPT_MAC :
4617c478bd9Sstevel@tonic-gate CRYPTO_FG_ENCRYPT_MAC_ATOMIC;
4627c478bd9Sstevel@tonic-gate break;
4637c478bd9Sstevel@tonic-gate }
4647c478bd9Sstevel@tonic-gate
4657c478bd9Sstevel@tonic-gate case KCF_OG_MAC_DECRYPT: {
4667c478bd9Sstevel@tonic-gate kcf_mac_decrypt_ops_params_t *dops =
4677c478bd9Sstevel@tonic-gate ¶ms->rp_u.mac_decrypt_params;
4687c478bd9Sstevel@tonic-gate
4697c478bd9Sstevel@tonic-gate dops->md_mac_mech.cm_type = dops->md_framework_mac_mechtype;
4707c478bd9Sstevel@tonic-gate *mech1 = &dops->md_mac_mech;
4717c478bd9Sstevel@tonic-gate dops->md_decr_mech.cm_type = dops->md_framework_decr_mechtype;
4727c478bd9Sstevel@tonic-gate *mech2 = &dops->md_decr_mech;
4737c478bd9Sstevel@tonic-gate *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_MAC_DECRYPT :
4747c478bd9Sstevel@tonic-gate CRYPTO_FG_MAC_DECRYPT_ATOMIC;
4757c478bd9Sstevel@tonic-gate break;
4767c478bd9Sstevel@tonic-gate }
4777c478bd9Sstevel@tonic-gate
4787c478bd9Sstevel@tonic-gate default:
4797c478bd9Sstevel@tonic-gate return (B_FALSE);
4807c478bd9Sstevel@tonic-gate }
4817c478bd9Sstevel@tonic-gate
4827c478bd9Sstevel@tonic-gate return (B_TRUE);
4837c478bd9Sstevel@tonic-gate }
4847c478bd9Sstevel@tonic-gate
4857c478bd9Sstevel@tonic-gate /*
4867c478bd9Sstevel@tonic-gate * This routine is called when a request to a provider has failed
4877c478bd9Sstevel@tonic-gate * with a recoverable error. This routine tries to find another provider
4887c478bd9Sstevel@tonic-gate * and dispatches the request to the new provider, if one is available.
4897c478bd9Sstevel@tonic-gate * We reuse the request structure.
4907c478bd9Sstevel@tonic-gate *
4917c478bd9Sstevel@tonic-gate * A return value of NULL from kcf_get_mech_provider() indicates
4927c478bd9Sstevel@tonic-gate * we have tried the last provider.
4937c478bd9Sstevel@tonic-gate */
4947c478bd9Sstevel@tonic-gate static int
kcf_resubmit_request(kcf_areq_node_t * areq)4957c478bd9Sstevel@tonic-gate kcf_resubmit_request(kcf_areq_node_t *areq)
4967c478bd9Sstevel@tonic-gate {
4977c478bd9Sstevel@tonic-gate int error = CRYPTO_FAILED;
4987c478bd9Sstevel@tonic-gate kcf_context_t *ictx;
4997c478bd9Sstevel@tonic-gate kcf_provider_desc_t *old_pd;
5007c478bd9Sstevel@tonic-gate kcf_provider_desc_t *new_pd;
5017c478bd9Sstevel@tonic-gate crypto_mechanism_t *mech1 = NULL, *mech2 = NULL;
5027c478bd9Sstevel@tonic-gate crypto_mech_type_t prov_mt1, prov_mt2;
5037c478bd9Sstevel@tonic-gate crypto_func_group_t fg;
5047c478bd9Sstevel@tonic-gate
5057c478bd9Sstevel@tonic-gate if (!can_resubmit(areq, &mech1, &mech2, &fg))
5067c478bd9Sstevel@tonic-gate return (error);
5077c478bd9Sstevel@tonic-gate
5087c478bd9Sstevel@tonic-gate old_pd = areq->an_provider;
5097c478bd9Sstevel@tonic-gate /*
510ef56a3c5SKrishna Yenduri * Add old_pd to the list of providers already tried.
511ef56a3c5SKrishna Yenduri * We release the new hold on old_pd in kcf_free_triedlist().
5127c478bd9Sstevel@tonic-gate */
5137c478bd9Sstevel@tonic-gate if (kcf_insert_triedlist(&areq->an_tried_plist, old_pd,
514ef56a3c5SKrishna Yenduri KM_NOSLEEP | KCF_HOLD_PROV) == NULL)
5157c478bd9Sstevel@tonic-gate return (error);
5167c478bd9Sstevel@tonic-gate
5177c478bd9Sstevel@tonic-gate if (mech1 && !mech2) {
518436935a1SVladimir Kotal new_pd = kcf_get_mech_provider(mech1->cm_type, NULL, NULL,
5199b009fc1SValerie Bubb Fenwick &error, areq->an_tried_plist, fg, 0);
5207c478bd9Sstevel@tonic-gate } else {
5217c478bd9Sstevel@tonic-gate ASSERT(mech1 != NULL && mech2 != NULL);
5227c478bd9Sstevel@tonic-gate
523436935a1SVladimir Kotal new_pd = kcf_get_dual_provider(mech1, NULL, mech2, NULL,
524436935a1SVladimir Kotal NULL, &prov_mt1,
5259b009fc1SValerie Bubb Fenwick &prov_mt2, &error, areq->an_tried_plist, fg, fg, 0);
5267c478bd9Sstevel@tonic-gate }
5277c478bd9Sstevel@tonic-gate
5287c478bd9Sstevel@tonic-gate if (new_pd == NULL)
5297c478bd9Sstevel@tonic-gate return (error);
5307c478bd9Sstevel@tonic-gate
5317c478bd9Sstevel@tonic-gate /*
5327c478bd9Sstevel@tonic-gate * We reuse the old context by resetting provider specific
5337c478bd9Sstevel@tonic-gate * fields in it.
5347c478bd9Sstevel@tonic-gate */
5357c478bd9Sstevel@tonic-gate if ((ictx = areq->an_context) != NULL) {
5367c478bd9Sstevel@tonic-gate crypto_ctx_t *ctx;
5377c478bd9Sstevel@tonic-gate
5387c478bd9Sstevel@tonic-gate ASSERT(old_pd == ictx->kc_prov_desc);
5397c478bd9Sstevel@tonic-gate KCF_PROV_REFRELE(ictx->kc_prov_desc);
5407c478bd9Sstevel@tonic-gate KCF_PROV_REFHOLD(new_pd);
5417c478bd9Sstevel@tonic-gate ictx->kc_prov_desc = new_pd;
5427c478bd9Sstevel@tonic-gate
5437c478bd9Sstevel@tonic-gate ctx = &ictx->kc_glbl_ctx;
5447c478bd9Sstevel@tonic-gate ctx->cc_provider = new_pd->pd_prov_handle;
5457c478bd9Sstevel@tonic-gate ctx->cc_session = new_pd->pd_sid;
5467c478bd9Sstevel@tonic-gate ctx->cc_provider_private = NULL;
5477c478bd9Sstevel@tonic-gate }
5487c478bd9Sstevel@tonic-gate
5497c478bd9Sstevel@tonic-gate /* We reuse areq. by resetting the provider and context fields. */
5507c478bd9Sstevel@tonic-gate KCF_PROV_REFRELE(old_pd);
5517c478bd9Sstevel@tonic-gate KCF_PROV_REFHOLD(new_pd);
5527c478bd9Sstevel@tonic-gate areq->an_provider = new_pd;
5537c478bd9Sstevel@tonic-gate mutex_enter(&areq->an_lock);
5547c478bd9Sstevel@tonic-gate areq->an_state = REQ_WAITING;
5557c478bd9Sstevel@tonic-gate mutex_exit(&areq->an_lock);
5567c478bd9Sstevel@tonic-gate
5577c478bd9Sstevel@tonic-gate switch (new_pd->pd_prov_type) {
5587c478bd9Sstevel@tonic-gate case CRYPTO_SW_PROVIDER:
5597c478bd9Sstevel@tonic-gate error = kcf_disp_sw_request(areq);
5607c478bd9Sstevel@tonic-gate break;
5617c478bd9Sstevel@tonic-gate
5627c478bd9Sstevel@tonic-gate case CRYPTO_HW_PROVIDER: {
563ef56a3c5SKrishna Yenduri taskq_t *taskq = new_pd->pd_taskq;
5647c478bd9Sstevel@tonic-gate
5657c478bd9Sstevel@tonic-gate if (taskq_dispatch(taskq, process_req_hwp, areq, TQ_NOSLEEP) ==
566*fc8ae2ecSToomas Soome TASKQID_INVALID) {
5677c478bd9Sstevel@tonic-gate error = CRYPTO_HOST_MEMORY;
5687c478bd9Sstevel@tonic-gate } else {
5697c478bd9Sstevel@tonic-gate error = CRYPTO_QUEUED;
5707c478bd9Sstevel@tonic-gate }
5717c478bd9Sstevel@tonic-gate
5727c478bd9Sstevel@tonic-gate break;
5737c478bd9Sstevel@tonic-gate }
5747c478bd9Sstevel@tonic-gate }
5757c478bd9Sstevel@tonic-gate
576ef56a3c5SKrishna Yenduri KCF_PROV_REFRELE(new_pd);
5777c478bd9Sstevel@tonic-gate return (error);
5787c478bd9Sstevel@tonic-gate }
5797c478bd9Sstevel@tonic-gate
580f317a3a3Skrishna #define EMPTY_TASKQ(tq) ((tq)->tq_task.tqent_next == &(tq)->tq_task)
5817c478bd9Sstevel@tonic-gate
5827c478bd9Sstevel@tonic-gate /*
5837c478bd9Sstevel@tonic-gate * Routine called by both ioctl and k-api. The consumer should
5847c478bd9Sstevel@tonic-gate * bundle the parameters into a kcf_req_params_t structure. A bunch
5857c478bd9Sstevel@tonic-gate * of macros are available in ops_impl.h for this bundling. They are:
5867c478bd9Sstevel@tonic-gate *
5877c478bd9Sstevel@tonic-gate * KCF_WRAP_DIGEST_OPS_PARAMS()
5887c478bd9Sstevel@tonic-gate * KCF_WRAP_MAC_OPS_PARAMS()
5897c478bd9Sstevel@tonic-gate * KCF_WRAP_ENCRYPT_OPS_PARAMS()
5907c478bd9Sstevel@tonic-gate * KCF_WRAP_DECRYPT_OPS_PARAMS() ... etc.
5917c478bd9Sstevel@tonic-gate *
5927c478bd9Sstevel@tonic-gate * It is the caller's responsibility to free the ctx argument when
5937c478bd9Sstevel@tonic-gate * appropriate. See the KCF_CONTEXT_COND_RELEASE macro for details.
5947c478bd9Sstevel@tonic-gate */
5957c478bd9Sstevel@tonic-gate int
kcf_submit_request(kcf_provider_desc_t * pd,crypto_ctx_t * ctx,crypto_call_req_t * crq,kcf_req_params_t * params,boolean_t cont)5967c478bd9Sstevel@tonic-gate kcf_submit_request(kcf_provider_desc_t *pd, crypto_ctx_t *ctx,
5977c478bd9Sstevel@tonic-gate crypto_call_req_t *crq, kcf_req_params_t *params, boolean_t cont)
5987c478bd9Sstevel@tonic-gate {
599ef56a3c5SKrishna Yenduri int error;
6007c478bd9Sstevel@tonic-gate kcf_areq_node_t *areq;
6017c478bd9Sstevel@tonic-gate kcf_sreq_node_t *sreq;
6027c478bd9Sstevel@tonic-gate kcf_context_t *kcf_ctx;
603ef56a3c5SKrishna Yenduri taskq_t *taskq;
604ef56a3c5SKrishna Yenduri kcf_prov_cpu_t *mp;
6057c478bd9Sstevel@tonic-gate
6067c478bd9Sstevel@tonic-gate kcf_ctx = ctx ? (kcf_context_t *)ctx->cc_framework_private : NULL;
6077c478bd9Sstevel@tonic-gate
6087c478bd9Sstevel@tonic-gate /* Synchronous cases */
6097c478bd9Sstevel@tonic-gate if (crq == NULL) {
6107c478bd9Sstevel@tonic-gate switch (pd->pd_prov_type) {
6117c478bd9Sstevel@tonic-gate case CRYPTO_SW_PROVIDER:
6127c478bd9Sstevel@tonic-gate error = common_submit_request(pd, ctx, params,
6137c478bd9Sstevel@tonic-gate KCF_RHNDL(KM_SLEEP));
6147c478bd9Sstevel@tonic-gate break;
6157c478bd9Sstevel@tonic-gate
6167c478bd9Sstevel@tonic-gate case CRYPTO_HW_PROVIDER:
617ef56a3c5SKrishna Yenduri taskq = pd->pd_taskq;
618ef56a3c5SKrishna Yenduri
619c1591d22SKrishna Yenduri /*
620c1591d22SKrishna Yenduri * Special case for CRYPTO_SYNCHRONOUS providers that
621c1591d22SKrishna Yenduri * never return a CRYPTO_QUEUED error. We skip any
622c1591d22SKrishna Yenduri * request allocation and call the SPI directly.
623c1591d22SKrishna Yenduri */
624c1591d22SKrishna Yenduri if ((pd->pd_flags & CRYPTO_SYNCHRONOUS) &&
625c1591d22SKrishna Yenduri EMPTY_TASKQ(taskq)) {
626ef56a3c5SKrishna Yenduri mp = &(pd->pd_percpu_bins[CPU_SEQID]);
627ef56a3c5SKrishna Yenduri KCF_PROV_JOB_HOLD(mp);
628ef56a3c5SKrishna Yenduri
629c1591d22SKrishna Yenduri if (pd->pd_state == KCF_PROV_READY) {
630c1591d22SKrishna Yenduri error = common_submit_request(pd, ctx,
631c1591d22SKrishna Yenduri params, KCF_RHNDL(KM_SLEEP));
632ef56a3c5SKrishna Yenduri KCF_PROV_JOB_RELE(mp);
633c1591d22SKrishna Yenduri ASSERT(error != CRYPTO_QUEUED);
634c1591d22SKrishna Yenduri break;
635c1591d22SKrishna Yenduri }
636ef56a3c5SKrishna Yenduri KCF_PROV_JOB_RELE(mp);
637c1591d22SKrishna Yenduri }
638c1591d22SKrishna Yenduri
6397c478bd9Sstevel@tonic-gate sreq = kmem_cache_alloc(kcf_sreq_cache, KM_SLEEP);
6407c478bd9Sstevel@tonic-gate sreq->sn_state = REQ_ALLOCATED;
6417c478bd9Sstevel@tonic-gate sreq->sn_rv = CRYPTO_FAILED;
6427c478bd9Sstevel@tonic-gate sreq->sn_params = params;
6437c478bd9Sstevel@tonic-gate
6447c478bd9Sstevel@tonic-gate /*
6457c478bd9Sstevel@tonic-gate * Note that we do not need to hold the context
6467c478bd9Sstevel@tonic-gate * for synchronous case as the context will never
647ac129f9eSKrishna Yenduri * become invalid underneath us. We do not need to hold
648ac129f9eSKrishna Yenduri * the provider here either as the caller has a hold.
6497c478bd9Sstevel@tonic-gate */
6507c478bd9Sstevel@tonic-gate sreq->sn_context = kcf_ctx;
651ac129f9eSKrishna Yenduri ASSERT(KCF_PROV_REFHELD(pd));
652ac129f9eSKrishna Yenduri sreq->sn_provider = pd;
6537c478bd9Sstevel@tonic-gate
6547c478bd9Sstevel@tonic-gate ASSERT(taskq != NULL);
6557c478bd9Sstevel@tonic-gate /*
656f317a3a3Skrishna * Call the SPI directly if the taskq is empty and the
657f317a3a3Skrishna * provider is not busy, else dispatch to the taskq.
658f317a3a3Skrishna * Calling directly is fine as this is the synchronous
659f317a3a3Skrishna * case. This is unlike the asynchronous case where we
660f317a3a3Skrishna * must always dispatch to the taskq.
6617c478bd9Sstevel@tonic-gate */
662f317a3a3Skrishna if (EMPTY_TASKQ(taskq) &&
663f317a3a3Skrishna pd->pd_state == KCF_PROV_READY) {
664f317a3a3Skrishna process_req_hwp(sreq);
665f317a3a3Skrishna } else {
666f317a3a3Skrishna /*
667f317a3a3Skrishna * We can not tell from taskq_dispatch() return
668f317a3a3Skrishna * value if we exceeded maxalloc. Hence the
669f317a3a3Skrishna * check here. Since we are allowed to wait in
670f317a3a3Skrishna * the synchronous case, we wait for the taskq
671f317a3a3Skrishna * to become empty.
672f317a3a3Skrishna */
673f317a3a3Skrishna if (taskq->tq_nalloc >= crypto_taskq_maxalloc) {
674f317a3a3Skrishna taskq_wait(taskq);
675f317a3a3Skrishna }
676c41e7ccaSkrishna
677c41e7ccaSkrishna (void) taskq_dispatch(taskq, process_req_hwp,
678c41e7ccaSkrishna sreq, TQ_SLEEP);
6797c478bd9Sstevel@tonic-gate }
6807c478bd9Sstevel@tonic-gate
6817c478bd9Sstevel@tonic-gate /*
6827c478bd9Sstevel@tonic-gate * Wait for the notification to arrive,
6837c478bd9Sstevel@tonic-gate * if the operation is not done yet.
6847c478bd9Sstevel@tonic-gate * Bug# 4722589 will make the wait a cv_wait_sig().
6857c478bd9Sstevel@tonic-gate */
6867c478bd9Sstevel@tonic-gate mutex_enter(&sreq->sn_lock);
6877c478bd9Sstevel@tonic-gate while (sreq->sn_state < REQ_DONE)
6887c478bd9Sstevel@tonic-gate cv_wait(&sreq->sn_cv, &sreq->sn_lock);
6897c478bd9Sstevel@tonic-gate mutex_exit(&sreq->sn_lock);
6907c478bd9Sstevel@tonic-gate
6917c478bd9Sstevel@tonic-gate error = sreq->sn_rv;
6927c478bd9Sstevel@tonic-gate kmem_cache_free(kcf_sreq_cache, sreq);
6937c478bd9Sstevel@tonic-gate
6947c478bd9Sstevel@tonic-gate break;
6957c478bd9Sstevel@tonic-gate
6967c478bd9Sstevel@tonic-gate default:
6977c478bd9Sstevel@tonic-gate error = CRYPTO_FAILED;
6987c478bd9Sstevel@tonic-gate break;
6997c478bd9Sstevel@tonic-gate }
7007c478bd9Sstevel@tonic-gate
7017c478bd9Sstevel@tonic-gate } else { /* Asynchronous cases */
7027c478bd9Sstevel@tonic-gate switch (pd->pd_prov_type) {
7037c478bd9Sstevel@tonic-gate case CRYPTO_SW_PROVIDER:
7047c478bd9Sstevel@tonic-gate if (!(crq->cr_flag & CRYPTO_ALWAYS_QUEUE)) {
7057c478bd9Sstevel@tonic-gate /*
7067c478bd9Sstevel@tonic-gate * This case has less overhead since there is
7077c478bd9Sstevel@tonic-gate * no switching of context.
7087c478bd9Sstevel@tonic-gate */
7097c478bd9Sstevel@tonic-gate error = common_submit_request(pd, ctx, params,
7107c478bd9Sstevel@tonic-gate KCF_RHNDL(KM_NOSLEEP));
7117c478bd9Sstevel@tonic-gate } else {
7127c478bd9Sstevel@tonic-gate /*
7137c478bd9Sstevel@tonic-gate * CRYPTO_ALWAYS_QUEUE is set. We need to
7147c478bd9Sstevel@tonic-gate * queue the request and return.
7157c478bd9Sstevel@tonic-gate */
7167c478bd9Sstevel@tonic-gate areq = kcf_areqnode_alloc(pd, kcf_ctx, crq,
7177c478bd9Sstevel@tonic-gate params, cont);
7187c478bd9Sstevel@tonic-gate if (areq == NULL)
7197c478bd9Sstevel@tonic-gate error = CRYPTO_HOST_MEMORY;
7207c478bd9Sstevel@tonic-gate else {
7217c478bd9Sstevel@tonic-gate if (!(crq->cr_flag
7227c478bd9Sstevel@tonic-gate & CRYPTO_SKIP_REQID)) {
7237c478bd9Sstevel@tonic-gate /*
7247c478bd9Sstevel@tonic-gate * Set the request handle. This handle
7257c478bd9Sstevel@tonic-gate * is used for any crypto_cancel_req(9f)
7267c478bd9Sstevel@tonic-gate * calls from the consumer. We have to
7277c478bd9Sstevel@tonic-gate * do this before dispatching the
7287c478bd9Sstevel@tonic-gate * request.
7297c478bd9Sstevel@tonic-gate */
7307c478bd9Sstevel@tonic-gate crq->cr_reqid = kcf_reqid_insert(areq);
7317c478bd9Sstevel@tonic-gate }
7327c478bd9Sstevel@tonic-gate
7337c478bd9Sstevel@tonic-gate error = kcf_disp_sw_request(areq);
7347c478bd9Sstevel@tonic-gate /*
7357c478bd9Sstevel@tonic-gate * There is an error processing this
7367c478bd9Sstevel@tonic-gate * request. Remove the handle and
7377c478bd9Sstevel@tonic-gate * release the request structure.
7387c478bd9Sstevel@tonic-gate */
7397c478bd9Sstevel@tonic-gate if (error != CRYPTO_QUEUED) {
7407c478bd9Sstevel@tonic-gate if (!(crq->cr_flag
7417c478bd9Sstevel@tonic-gate & CRYPTO_SKIP_REQID))
7427c478bd9Sstevel@tonic-gate kcf_reqid_delete(areq);
7437c478bd9Sstevel@tonic-gate KCF_AREQ_REFRELE(areq);
7447c478bd9Sstevel@tonic-gate }
7457c478bd9Sstevel@tonic-gate }
7467c478bd9Sstevel@tonic-gate }
7477c478bd9Sstevel@tonic-gate break;
7487c478bd9Sstevel@tonic-gate
7497c478bd9Sstevel@tonic-gate case CRYPTO_HW_PROVIDER:
7507c478bd9Sstevel@tonic-gate /*
7517c478bd9Sstevel@tonic-gate * We need to queue the request and return.
7527c478bd9Sstevel@tonic-gate */
7537c478bd9Sstevel@tonic-gate areq = kcf_areqnode_alloc(pd, kcf_ctx, crq, params,
7547c478bd9Sstevel@tonic-gate cont);
7557c478bd9Sstevel@tonic-gate if (areq == NULL) {
7567c478bd9Sstevel@tonic-gate error = CRYPTO_HOST_MEMORY;
7577c478bd9Sstevel@tonic-gate goto done;
7587c478bd9Sstevel@tonic-gate }
7597c478bd9Sstevel@tonic-gate
760ef56a3c5SKrishna Yenduri taskq = pd->pd_taskq;
7617c478bd9Sstevel@tonic-gate ASSERT(taskq != NULL);
7627c478bd9Sstevel@tonic-gate /*
7637c478bd9Sstevel@tonic-gate * We can not tell from taskq_dispatch() return
7647c478bd9Sstevel@tonic-gate * value if we exceeded maxalloc. Hence the check
7657c478bd9Sstevel@tonic-gate * here.
7667c478bd9Sstevel@tonic-gate */
7677c478bd9Sstevel@tonic-gate if (taskq->tq_nalloc >= crypto_taskq_maxalloc) {
7687c478bd9Sstevel@tonic-gate error = CRYPTO_BUSY;
7697c478bd9Sstevel@tonic-gate KCF_AREQ_REFRELE(areq);
7707c478bd9Sstevel@tonic-gate goto done;
7717c478bd9Sstevel@tonic-gate }
7727c478bd9Sstevel@tonic-gate
7737c478bd9Sstevel@tonic-gate if (!(crq->cr_flag & CRYPTO_SKIP_REQID)) {
7747c478bd9Sstevel@tonic-gate /*
7757c478bd9Sstevel@tonic-gate * Set the request handle. This handle is used
7767c478bd9Sstevel@tonic-gate * for any crypto_cancel_req(9f) calls from the
7777c478bd9Sstevel@tonic-gate * consumer. We have to do this before dispatching
7787c478bd9Sstevel@tonic-gate * the request.
7797c478bd9Sstevel@tonic-gate */
7807c478bd9Sstevel@tonic-gate crq->cr_reqid = kcf_reqid_insert(areq);
7817c478bd9Sstevel@tonic-gate }
7827c478bd9Sstevel@tonic-gate
7837c478bd9Sstevel@tonic-gate if (taskq_dispatch(taskq,
7847c478bd9Sstevel@tonic-gate process_req_hwp, areq, TQ_NOSLEEP) ==
785*fc8ae2ecSToomas Soome TASKQID_INVALID) {
7867c478bd9Sstevel@tonic-gate error = CRYPTO_HOST_MEMORY;
7877c478bd9Sstevel@tonic-gate if (!(crq->cr_flag & CRYPTO_SKIP_REQID))
7887c478bd9Sstevel@tonic-gate kcf_reqid_delete(areq);
7897c478bd9Sstevel@tonic-gate KCF_AREQ_REFRELE(areq);
7907c478bd9Sstevel@tonic-gate } else {
7917c478bd9Sstevel@tonic-gate error = CRYPTO_QUEUED;
7927c478bd9Sstevel@tonic-gate }
7937c478bd9Sstevel@tonic-gate break;
7947c478bd9Sstevel@tonic-gate
7957c478bd9Sstevel@tonic-gate default:
7967c478bd9Sstevel@tonic-gate error = CRYPTO_FAILED;
7977c478bd9Sstevel@tonic-gate break;
7987c478bd9Sstevel@tonic-gate }
7997c478bd9Sstevel@tonic-gate }
8007c478bd9Sstevel@tonic-gate
8017c478bd9Sstevel@tonic-gate done:
8027c478bd9Sstevel@tonic-gate return (error);
8037c478bd9Sstevel@tonic-gate }
8047c478bd9Sstevel@tonic-gate
8057c478bd9Sstevel@tonic-gate /*
8067c478bd9Sstevel@tonic-gate * We're done with this framework context, so free it. Note that freeing
8077c478bd9Sstevel@tonic-gate * framework context (kcf_context) frees the global context (crypto_ctx).
8087c478bd9Sstevel@tonic-gate *
8097c478bd9Sstevel@tonic-gate * The provider is responsible for freeing provider private context after a
8107c478bd9Sstevel@tonic-gate * final or single operation and resetting the cc_provider_private field
8117c478bd9Sstevel@tonic-gate * to NULL. It should do this before it notifies the framework of the
8127c478bd9Sstevel@tonic-gate * completion. We still need to call KCF_PROV_FREE_CONTEXT to handle cases
8137c478bd9Sstevel@tonic-gate * like crypto_cancel_ctx(9f).
8147c478bd9Sstevel@tonic-gate */
8157c478bd9Sstevel@tonic-gate void
kcf_free_context(kcf_context_t * kcf_ctx)8167c478bd9Sstevel@tonic-gate kcf_free_context(kcf_context_t *kcf_ctx)
8177c478bd9Sstevel@tonic-gate {
8187c478bd9Sstevel@tonic-gate kcf_provider_desc_t *pd = kcf_ctx->kc_prov_desc;
8197c478bd9Sstevel@tonic-gate crypto_ctx_t *gctx = &kcf_ctx->kc_glbl_ctx;
8207c478bd9Sstevel@tonic-gate kcf_context_t *kcf_secondctx = kcf_ctx->kc_secondctx;
821ef56a3c5SKrishna Yenduri kcf_prov_cpu_t *mp;
8227c478bd9Sstevel@tonic-gate
8237c478bd9Sstevel@tonic-gate /* Release the second context, if any */
8247c478bd9Sstevel@tonic-gate
8257c478bd9Sstevel@tonic-gate if (kcf_secondctx != NULL)
8267c478bd9Sstevel@tonic-gate KCF_CONTEXT_REFRELE(kcf_secondctx);
8277c478bd9Sstevel@tonic-gate
8287c478bd9Sstevel@tonic-gate if (gctx->cc_provider_private != NULL) {
8297c478bd9Sstevel@tonic-gate mutex_enter(&pd->pd_lock);
8307c478bd9Sstevel@tonic-gate if (!KCF_IS_PROV_REMOVED(pd)) {
8317c478bd9Sstevel@tonic-gate /*
8327c478bd9Sstevel@tonic-gate * Increment the provider's internal refcnt so it
8337c478bd9Sstevel@tonic-gate * doesn't unregister from the framework while
8347c478bd9Sstevel@tonic-gate * we're calling the entry point.
8357c478bd9Sstevel@tonic-gate */
836ef56a3c5SKrishna Yenduri mp = &(pd->pd_percpu_bins[CPU_SEQID]);
837ef56a3c5SKrishna Yenduri KCF_PROV_JOB_HOLD(mp);
8387c478bd9Sstevel@tonic-gate mutex_exit(&pd->pd_lock);
8397c478bd9Sstevel@tonic-gate (void) KCF_PROV_FREE_CONTEXT(pd, gctx);
840ef56a3c5SKrishna Yenduri KCF_PROV_JOB_RELE(mp);
8417c478bd9Sstevel@tonic-gate } else {
8427c478bd9Sstevel@tonic-gate mutex_exit(&pd->pd_lock);
8437c478bd9Sstevel@tonic-gate }
8447c478bd9Sstevel@tonic-gate }
8457c478bd9Sstevel@tonic-gate
8467c478bd9Sstevel@tonic-gate /* kcf_ctx->kc_prov_desc has a hold on pd */
8477c478bd9Sstevel@tonic-gate KCF_PROV_REFRELE(kcf_ctx->kc_prov_desc);
8487c478bd9Sstevel@tonic-gate
8496a1073f8Skrishna /* check if this context is shared with a software provider */
8506a1073f8Skrishna if ((gctx->cc_flags & CRYPTO_INIT_OPSTATE) &&
8516a1073f8Skrishna kcf_ctx->kc_sw_prov_desc != NULL) {
8526a1073f8Skrishna KCF_PROV_REFRELE(kcf_ctx->kc_sw_prov_desc);
8536a1073f8Skrishna }
8546a1073f8Skrishna
8557c478bd9Sstevel@tonic-gate kmem_cache_free(kcf_context_cache, kcf_ctx);
8567c478bd9Sstevel@tonic-gate }
8577c478bd9Sstevel@tonic-gate
8587c478bd9Sstevel@tonic-gate /*
8597c478bd9Sstevel@tonic-gate * Free the request after releasing all the holds.
8607c478bd9Sstevel@tonic-gate */
8617c478bd9Sstevel@tonic-gate void
kcf_free_req(kcf_areq_node_t * areq)8627c478bd9Sstevel@tonic-gate kcf_free_req(kcf_areq_node_t *areq)
8637c478bd9Sstevel@tonic-gate {
8647c478bd9Sstevel@tonic-gate KCF_PROV_REFRELE(areq->an_provider);
8657c478bd9Sstevel@tonic-gate if (areq->an_context != NULL)
8667c478bd9Sstevel@tonic-gate KCF_CONTEXT_REFRELE(areq->an_context);
8677c478bd9Sstevel@tonic-gate
8687c478bd9Sstevel@tonic-gate if (areq->an_tried_plist != NULL)
8697c478bd9Sstevel@tonic-gate kcf_free_triedlist(areq->an_tried_plist);
8707c478bd9Sstevel@tonic-gate kmem_cache_free(kcf_areq_cache, areq);
8717c478bd9Sstevel@tonic-gate }
8727c478bd9Sstevel@tonic-gate
8737c478bd9Sstevel@tonic-gate /*
8747c478bd9Sstevel@tonic-gate * Utility routine to remove a request from the chain of requests
8757c478bd9Sstevel@tonic-gate * hanging off a context.
8767c478bd9Sstevel@tonic-gate */
8777c478bd9Sstevel@tonic-gate void
kcf_removereq_in_ctxchain(kcf_context_t * ictx,kcf_areq_node_t * areq)8787c478bd9Sstevel@tonic-gate kcf_removereq_in_ctxchain(kcf_context_t *ictx, kcf_areq_node_t *areq)
8797c478bd9Sstevel@tonic-gate {
8807c478bd9Sstevel@tonic-gate kcf_areq_node_t *cur, *prev;
8817c478bd9Sstevel@tonic-gate
8827c478bd9Sstevel@tonic-gate /*
8837c478bd9Sstevel@tonic-gate * Get context lock, search for areq in the chain and remove it.
8847c478bd9Sstevel@tonic-gate */
8857c478bd9Sstevel@tonic-gate ASSERT(ictx != NULL);
8867c478bd9Sstevel@tonic-gate mutex_enter(&ictx->kc_in_use_lock);
8877c478bd9Sstevel@tonic-gate prev = cur = ictx->kc_req_chain_first;
8887c478bd9Sstevel@tonic-gate
8897c478bd9Sstevel@tonic-gate while (cur != NULL) {
8907c478bd9Sstevel@tonic-gate if (cur == areq) {
8917c478bd9Sstevel@tonic-gate if (prev == cur) {
8927c478bd9Sstevel@tonic-gate if ((ictx->kc_req_chain_first =
8937c478bd9Sstevel@tonic-gate cur->an_ctxchain_next) == NULL)
8947c478bd9Sstevel@tonic-gate ictx->kc_req_chain_last = NULL;
8957c478bd9Sstevel@tonic-gate } else {
8967c478bd9Sstevel@tonic-gate if (cur == ictx->kc_req_chain_last)
8977c478bd9Sstevel@tonic-gate ictx->kc_req_chain_last = prev;
8987c478bd9Sstevel@tonic-gate prev->an_ctxchain_next = cur->an_ctxchain_next;
8997c478bd9Sstevel@tonic-gate }
9007c478bd9Sstevel@tonic-gate
9017c478bd9Sstevel@tonic-gate break;
9027c478bd9Sstevel@tonic-gate }
9037c478bd9Sstevel@tonic-gate prev = cur;
9047c478bd9Sstevel@tonic-gate cur = cur->an_ctxchain_next;
9057c478bd9Sstevel@tonic-gate }
9067c478bd9Sstevel@tonic-gate mutex_exit(&ictx->kc_in_use_lock);
9077c478bd9Sstevel@tonic-gate }
9087c478bd9Sstevel@tonic-gate
9097c478bd9Sstevel@tonic-gate /*
9107c478bd9Sstevel@tonic-gate * Remove the specified node from the global software queue.
9117c478bd9Sstevel@tonic-gate *
9127c478bd9Sstevel@tonic-gate * The caller must hold the queue lock and request lock (an_lock).
9137c478bd9Sstevel@tonic-gate */
9147c478bd9Sstevel@tonic-gate void
kcf_remove_node(kcf_areq_node_t * node)9157c478bd9Sstevel@tonic-gate kcf_remove_node(kcf_areq_node_t *node)
9167c478bd9Sstevel@tonic-gate {
9177c478bd9Sstevel@tonic-gate kcf_areq_node_t *nextp = node->an_next;
9187c478bd9Sstevel@tonic-gate kcf_areq_node_t *prevp = node->an_prev;
9197c478bd9Sstevel@tonic-gate
9207c478bd9Sstevel@tonic-gate ASSERT(mutex_owned(&gswq->gs_lock));
9217c478bd9Sstevel@tonic-gate
9227c478bd9Sstevel@tonic-gate if (nextp != NULL)
9237c478bd9Sstevel@tonic-gate nextp->an_prev = prevp;
9247c478bd9Sstevel@tonic-gate else
9257c478bd9Sstevel@tonic-gate gswq->gs_last = prevp;
9267c478bd9Sstevel@tonic-gate
9277c478bd9Sstevel@tonic-gate if (prevp != NULL)
9287c478bd9Sstevel@tonic-gate prevp->an_next = nextp;
9297c478bd9Sstevel@tonic-gate else
9307c478bd9Sstevel@tonic-gate gswq->gs_first = nextp;
9317c478bd9Sstevel@tonic-gate
9327c478bd9Sstevel@tonic-gate ASSERT(mutex_owned(&node->an_lock));
9337c478bd9Sstevel@tonic-gate node->an_state = REQ_CANCELED;
9347c478bd9Sstevel@tonic-gate }
9357c478bd9Sstevel@tonic-gate
9367c478bd9Sstevel@tonic-gate /*
9377c478bd9Sstevel@tonic-gate * Remove and return the first node in the global software queue.
9387c478bd9Sstevel@tonic-gate *
9397c478bd9Sstevel@tonic-gate * The caller must hold the queue lock.
9407c478bd9Sstevel@tonic-gate */
9417c478bd9Sstevel@tonic-gate static kcf_areq_node_t *
kcf_dequeue(void)9426ea3c060SGarrett D'Amore kcf_dequeue(void)
9437c478bd9Sstevel@tonic-gate {
9447c478bd9Sstevel@tonic-gate kcf_areq_node_t *tnode = NULL;
9457c478bd9Sstevel@tonic-gate
9467c478bd9Sstevel@tonic-gate ASSERT(mutex_owned(&gswq->gs_lock));
9477c478bd9Sstevel@tonic-gate if ((tnode = gswq->gs_first) == NULL) {
9487c478bd9Sstevel@tonic-gate return (NULL);
9497c478bd9Sstevel@tonic-gate } else {
9507c478bd9Sstevel@tonic-gate ASSERT(gswq->gs_first->an_prev == NULL);
9517c478bd9Sstevel@tonic-gate gswq->gs_first = tnode->an_next;
9527c478bd9Sstevel@tonic-gate if (tnode->an_next == NULL)
9537c478bd9Sstevel@tonic-gate gswq->gs_last = NULL;
9547c478bd9Sstevel@tonic-gate else
9557c478bd9Sstevel@tonic-gate tnode->an_next->an_prev = NULL;
9567c478bd9Sstevel@tonic-gate }
9577c478bd9Sstevel@tonic-gate
9587c478bd9Sstevel@tonic-gate gswq->gs_njobs--;
9597c478bd9Sstevel@tonic-gate return (tnode);
9607c478bd9Sstevel@tonic-gate }
9617c478bd9Sstevel@tonic-gate
9627c478bd9Sstevel@tonic-gate /*
9637c478bd9Sstevel@tonic-gate * Add the request node to the end of the global software queue.
9647c478bd9Sstevel@tonic-gate *
9657c478bd9Sstevel@tonic-gate * The caller should not hold the queue lock. Returns 0 if the
9667c478bd9Sstevel@tonic-gate * request is successfully queued. Returns CRYPTO_BUSY if the limit
9677c478bd9Sstevel@tonic-gate * on the number of jobs is exceeded.
9687c478bd9Sstevel@tonic-gate */
9697c478bd9Sstevel@tonic-gate static int
kcf_enqueue(kcf_areq_node_t * node)9707c478bd9Sstevel@tonic-gate kcf_enqueue(kcf_areq_node_t *node)
9717c478bd9Sstevel@tonic-gate {
9727c478bd9Sstevel@tonic-gate kcf_areq_node_t *tnode;
9737c478bd9Sstevel@tonic-gate
9747c478bd9Sstevel@tonic-gate mutex_enter(&gswq->gs_lock);
9757c478bd9Sstevel@tonic-gate
9767c478bd9Sstevel@tonic-gate if (gswq->gs_njobs >= gswq->gs_maxjobs) {
9777c478bd9Sstevel@tonic-gate mutex_exit(&gswq->gs_lock);
9787c478bd9Sstevel@tonic-gate return (CRYPTO_BUSY);
9797c478bd9Sstevel@tonic-gate }
9807c478bd9Sstevel@tonic-gate
9817c478bd9Sstevel@tonic-gate if (gswq->gs_last == NULL) {
9827c478bd9Sstevel@tonic-gate gswq->gs_first = gswq->gs_last = node;
9837c478bd9Sstevel@tonic-gate } else {
9847c478bd9Sstevel@tonic-gate ASSERT(gswq->gs_last->an_next == NULL);
9857c478bd9Sstevel@tonic-gate tnode = gswq->gs_last;
9867c478bd9Sstevel@tonic-gate tnode->an_next = node;
9877c478bd9Sstevel@tonic-gate gswq->gs_last = node;
9887c478bd9Sstevel@tonic-gate node->an_prev = tnode;
9897c478bd9Sstevel@tonic-gate }
9907c478bd9Sstevel@tonic-gate
9917c478bd9Sstevel@tonic-gate gswq->gs_njobs++;
9927c478bd9Sstevel@tonic-gate
9937c478bd9Sstevel@tonic-gate /* an_lock not needed here as we hold gs_lock */
9947c478bd9Sstevel@tonic-gate node->an_state = REQ_WAITING;
9957c478bd9Sstevel@tonic-gate
9967c478bd9Sstevel@tonic-gate mutex_exit(&gswq->gs_lock);
9977c478bd9Sstevel@tonic-gate
9987c478bd9Sstevel@tonic-gate return (0);
9997c478bd9Sstevel@tonic-gate }
10007c478bd9Sstevel@tonic-gate
10017c478bd9Sstevel@tonic-gate /*
10027c478bd9Sstevel@tonic-gate * Function run by a thread from kcfpool to work on global software queue.
10037c478bd9Sstevel@tonic-gate */
10046ea3c060SGarrett D'Amore void
kcfpool_svc(void * arg)10056ea3c060SGarrett D'Amore kcfpool_svc(void *arg)
10067c478bd9Sstevel@tonic-gate {
10076ea3c060SGarrett D'Amore _NOTE(ARGUNUSED(arg));
10087c478bd9Sstevel@tonic-gate int error = 0;
10097c478bd9Sstevel@tonic-gate clock_t rv;
1010d3d50737SRafael Vanoni clock_t timeout_val = drv_usectohz(kcf_idlethr_timeout);
10117c478bd9Sstevel@tonic-gate kcf_areq_node_t *req;
10127c478bd9Sstevel@tonic-gate kcf_context_t *ictx;
10137c478bd9Sstevel@tonic-gate kcf_provider_desc_t *pd;
10147c478bd9Sstevel@tonic-gate
10157c478bd9Sstevel@tonic-gate KCF_ATOMIC_INCR(kcfpool->kp_threads);
10167c478bd9Sstevel@tonic-gate
10177c478bd9Sstevel@tonic-gate for (;;) {
10187c478bd9Sstevel@tonic-gate mutex_enter(&gswq->gs_lock);
10197c478bd9Sstevel@tonic-gate
10207c478bd9Sstevel@tonic-gate while ((req = kcf_dequeue()) == NULL) {
10217c478bd9Sstevel@tonic-gate KCF_ATOMIC_INCR(kcfpool->kp_idlethreads);
10226ea3c060SGarrett D'Amore rv = cv_reltimedwait(&gswq->gs_cv,
1023d3d50737SRafael Vanoni &gswq->gs_lock, timeout_val, TR_CLOCK_TICK);
10247c478bd9Sstevel@tonic-gate KCF_ATOMIC_DECR(kcfpool->kp_idlethreads);
10257c478bd9Sstevel@tonic-gate
10267c478bd9Sstevel@tonic-gate switch (rv) {
10277c478bd9Sstevel@tonic-gate case 0:
10287c478bd9Sstevel@tonic-gate case -1:
10297c478bd9Sstevel@tonic-gate /*
10306ea3c060SGarrett D'Amore * Woke up with no work to do. Check
10316ea3c060SGarrett D'Amore * if this thread should exit. We keep
10326ea3c060SGarrett D'Amore * at least kcf_minthreads.
10337c478bd9Sstevel@tonic-gate */
10347c478bd9Sstevel@tonic-gate if (kcfpool->kp_threads > kcf_minthreads) {
10356ea3c060SGarrett D'Amore KCF_ATOMIC_DECR(kcfpool->kp_threads);
10367c478bd9Sstevel@tonic-gate mutex_exit(&gswq->gs_lock);
103757a30242SGarrett D'Amore
103857a30242SGarrett D'Amore /*
103957a30242SGarrett D'Amore * lwp_exit() assumes it is called
104057a30242SGarrett D'Amore * with the proc lock held. But the
104157a30242SGarrett D'Amore * first thing it does is drop it.
104257a30242SGarrett D'Amore * This ensures that lwp does not
104357a30242SGarrett D'Amore * exit before lwp_create is done
104457a30242SGarrett D'Amore * with it.
104557a30242SGarrett D'Amore */
104657a30242SGarrett D'Amore mutex_enter(&curproc->p_lock);
104757a30242SGarrett D'Amore lwp_exit(); /* does not return */
10487c478bd9Sstevel@tonic-gate }
10497c478bd9Sstevel@tonic-gate
10506ea3c060SGarrett D'Amore /* Resume the wait for work. */
10517c478bd9Sstevel@tonic-gate break;
10527c478bd9Sstevel@tonic-gate
10537c478bd9Sstevel@tonic-gate default:
10547c478bd9Sstevel@tonic-gate /*
10557c478bd9Sstevel@tonic-gate * We are signaled to work on the queue.
10567c478bd9Sstevel@tonic-gate */
10577c478bd9Sstevel@tonic-gate break;
10587c478bd9Sstevel@tonic-gate }
10597c478bd9Sstevel@tonic-gate }
10607c478bd9Sstevel@tonic-gate
10617c478bd9Sstevel@tonic-gate mutex_exit(&gswq->gs_lock);
10627c478bd9Sstevel@tonic-gate
10637c478bd9Sstevel@tonic-gate ictx = req->an_context;
10647c478bd9Sstevel@tonic-gate if (ictx == NULL) { /* Context-less operation */
10657c478bd9Sstevel@tonic-gate pd = req->an_provider;
10667c478bd9Sstevel@tonic-gate error = common_submit_request(pd, NULL,
10677c478bd9Sstevel@tonic-gate &req->an_params, req);
10687c478bd9Sstevel@tonic-gate kcf_aop_done(req, error);
10697c478bd9Sstevel@tonic-gate continue;
10707c478bd9Sstevel@tonic-gate }
10717c478bd9Sstevel@tonic-gate
10727c478bd9Sstevel@tonic-gate /*
10737c478bd9Sstevel@tonic-gate * We check if we can work on the request now.
10747c478bd9Sstevel@tonic-gate * Solaris does not guarantee any order on how the threads
10757c478bd9Sstevel@tonic-gate * are scheduled or how the waiters on a mutex are chosen.
10767c478bd9Sstevel@tonic-gate * So, we need to maintain our own order.
10777c478bd9Sstevel@tonic-gate *
10787c478bd9Sstevel@tonic-gate * is_my_turn is set to B_TRUE initially for a request when
10797c478bd9Sstevel@tonic-gate * it is enqueued and there are no other requests
10807c478bd9Sstevel@tonic-gate * for that context. Note that a thread sleeping on
1081c41e7ccaSkrishna * an_turn_cv is not counted as an idle thread. This is
10827c478bd9Sstevel@tonic-gate * because we define an idle thread as one that sleeps on the
10837c478bd9Sstevel@tonic-gate * global queue waiting for new requests.
10847c478bd9Sstevel@tonic-gate */
1085c41e7ccaSkrishna mutex_enter(&req->an_lock);
10867c478bd9Sstevel@tonic-gate while (req->an_is_my_turn == B_FALSE) {
10877c478bd9Sstevel@tonic-gate KCF_ATOMIC_INCR(kcfpool->kp_blockedthreads);
1088c41e7ccaSkrishna cv_wait(&req->an_turn_cv, &req->an_lock);
10897c478bd9Sstevel@tonic-gate KCF_ATOMIC_DECR(kcfpool->kp_blockedthreads);
10907c478bd9Sstevel@tonic-gate }
10917c478bd9Sstevel@tonic-gate
10927c478bd9Sstevel@tonic-gate req->an_state = REQ_INPROGRESS;
10937c478bd9Sstevel@tonic-gate mutex_exit(&req->an_lock);
10947c478bd9Sstevel@tonic-gate
10957c478bd9Sstevel@tonic-gate pd = ictx->kc_prov_desc;
10967c478bd9Sstevel@tonic-gate ASSERT(pd == req->an_provider);
10977c478bd9Sstevel@tonic-gate error = common_submit_request(pd, &ictx->kc_glbl_ctx,
10987c478bd9Sstevel@tonic-gate &req->an_params, req);
10997c478bd9Sstevel@tonic-gate
11007c478bd9Sstevel@tonic-gate kcf_aop_done(req, error);
11017c478bd9Sstevel@tonic-gate }
11027c478bd9Sstevel@tonic-gate }
11037c478bd9Sstevel@tonic-gate
11047c478bd9Sstevel@tonic-gate /*
11057c478bd9Sstevel@tonic-gate * kmem_cache_alloc constructor for sync request structure.
11067c478bd9Sstevel@tonic-gate */
11077c478bd9Sstevel@tonic-gate /* ARGSUSED */
11087c478bd9Sstevel@tonic-gate static int
kcf_sreq_cache_constructor(void * buf,void * cdrarg,int kmflags)11097c478bd9Sstevel@tonic-gate kcf_sreq_cache_constructor(void *buf, void *cdrarg, int kmflags)
11107c478bd9Sstevel@tonic-gate {
11117c478bd9Sstevel@tonic-gate kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)buf;
11127c478bd9Sstevel@tonic-gate
11137c478bd9Sstevel@tonic-gate sreq->sn_type = CRYPTO_SYNCH;
11147c478bd9Sstevel@tonic-gate cv_init(&sreq->sn_cv, NULL, CV_DEFAULT, NULL);
11157c478bd9Sstevel@tonic-gate mutex_init(&sreq->sn_lock, NULL, MUTEX_DEFAULT, NULL);
11167c478bd9Sstevel@tonic-gate
11177c478bd9Sstevel@tonic-gate return (0);
11187c478bd9Sstevel@tonic-gate }
11197c478bd9Sstevel@tonic-gate
11207c478bd9Sstevel@tonic-gate /* ARGSUSED */
11217c478bd9Sstevel@tonic-gate static void
kcf_sreq_cache_destructor(void * buf,void * cdrarg)11227c478bd9Sstevel@tonic-gate kcf_sreq_cache_destructor(void *buf, void *cdrarg)
11237c478bd9Sstevel@tonic-gate {
11247c478bd9Sstevel@tonic-gate kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)buf;
11257c478bd9Sstevel@tonic-gate
11267c478bd9Sstevel@tonic-gate mutex_destroy(&sreq->sn_lock);
11277c478bd9Sstevel@tonic-gate cv_destroy(&sreq->sn_cv);
11287c478bd9Sstevel@tonic-gate }
11297c478bd9Sstevel@tonic-gate
11307c478bd9Sstevel@tonic-gate /*
11317c478bd9Sstevel@tonic-gate * kmem_cache_alloc constructor for async request structure.
11327c478bd9Sstevel@tonic-gate */
11337c478bd9Sstevel@tonic-gate /* ARGSUSED */
11347c478bd9Sstevel@tonic-gate static int
kcf_areq_cache_constructor(void * buf,void * cdrarg,int kmflags)11357c478bd9Sstevel@tonic-gate kcf_areq_cache_constructor(void *buf, void *cdrarg, int kmflags)
11367c478bd9Sstevel@tonic-gate {
11377c478bd9Sstevel@tonic-gate kcf_areq_node_t *areq = (kcf_areq_node_t *)buf;
11387c478bd9Sstevel@tonic-gate
11397c478bd9Sstevel@tonic-gate areq->an_type = CRYPTO_ASYNCH;
1140b5fca8f8Stomee areq->an_refcnt = 0;
11417c478bd9Sstevel@tonic-gate mutex_init(&areq->an_lock, NULL, MUTEX_DEFAULT, NULL);
11427c478bd9Sstevel@tonic-gate cv_init(&areq->an_done, NULL, CV_DEFAULT, NULL);
1143c41e7ccaSkrishna cv_init(&areq->an_turn_cv, NULL, CV_DEFAULT, NULL);
11447c478bd9Sstevel@tonic-gate
11457c478bd9Sstevel@tonic-gate return (0);
11467c478bd9Sstevel@tonic-gate }
11477c478bd9Sstevel@tonic-gate
11487c478bd9Sstevel@tonic-gate /* ARGSUSED */
11497c478bd9Sstevel@tonic-gate static void
kcf_areq_cache_destructor(void * buf,void * cdrarg)11507c478bd9Sstevel@tonic-gate kcf_areq_cache_destructor(void *buf, void *cdrarg)
11517c478bd9Sstevel@tonic-gate {
11527c478bd9Sstevel@tonic-gate kcf_areq_node_t *areq = (kcf_areq_node_t *)buf;
11537c478bd9Sstevel@tonic-gate
11547c478bd9Sstevel@tonic-gate ASSERT(areq->an_refcnt == 0);
11557c478bd9Sstevel@tonic-gate mutex_destroy(&areq->an_lock);
11567c478bd9Sstevel@tonic-gate cv_destroy(&areq->an_done);
1157c41e7ccaSkrishna cv_destroy(&areq->an_turn_cv);
11587c478bd9Sstevel@tonic-gate }
11597c478bd9Sstevel@tonic-gate
11607c478bd9Sstevel@tonic-gate /*
11617c478bd9Sstevel@tonic-gate * kmem_cache_alloc constructor for kcf_context structure.
11627c478bd9Sstevel@tonic-gate */
11637c478bd9Sstevel@tonic-gate /* ARGSUSED */
11647c478bd9Sstevel@tonic-gate static int
kcf_context_cache_constructor(void * buf,void * cdrarg,int kmflags)11657c478bd9Sstevel@tonic-gate kcf_context_cache_constructor(void *buf, void *cdrarg, int kmflags)
11667c478bd9Sstevel@tonic-gate {
11677c478bd9Sstevel@tonic-gate kcf_context_t *kctx = (kcf_context_t *)buf;
11687c478bd9Sstevel@tonic-gate
1169b5fca8f8Stomee kctx->kc_refcnt = 0;
11707c478bd9Sstevel@tonic-gate mutex_init(&kctx->kc_in_use_lock, NULL, MUTEX_DEFAULT, NULL);
11717c478bd9Sstevel@tonic-gate
11727c478bd9Sstevel@tonic-gate return (0);
11737c478bd9Sstevel@tonic-gate }
11747c478bd9Sstevel@tonic-gate
11757c478bd9Sstevel@tonic-gate /* ARGSUSED */
11767c478bd9Sstevel@tonic-gate static void
kcf_context_cache_destructor(void * buf,void * cdrarg)11777c478bd9Sstevel@tonic-gate kcf_context_cache_destructor(void *buf, void *cdrarg)
11787c478bd9Sstevel@tonic-gate {
11797c478bd9Sstevel@tonic-gate kcf_context_t *kctx = (kcf_context_t *)buf;
11807c478bd9Sstevel@tonic-gate
11817c478bd9Sstevel@tonic-gate ASSERT(kctx->kc_refcnt == 0);
11827c478bd9Sstevel@tonic-gate mutex_destroy(&kctx->kc_in_use_lock);
11837c478bd9Sstevel@tonic-gate }
11847c478bd9Sstevel@tonic-gate
11857c478bd9Sstevel@tonic-gate /*
11867c478bd9Sstevel@tonic-gate * Creates and initializes all the structures needed by the framework.
11877c478bd9Sstevel@tonic-gate */
11887c478bd9Sstevel@tonic-gate void
kcf_sched_init(void)11897c478bd9Sstevel@tonic-gate kcf_sched_init(void)
11907c478bd9Sstevel@tonic-gate {
11917c478bd9Sstevel@tonic-gate int i;
11927c478bd9Sstevel@tonic-gate kcf_reqid_table_t *rt;
11937c478bd9Sstevel@tonic-gate
11947c478bd9Sstevel@tonic-gate /*
11957c478bd9Sstevel@tonic-gate * Create all the kmem caches needed by the framework. We set the
11967c478bd9Sstevel@tonic-gate * align argument to 64, to get a slab aligned to 64-byte as well as
11977c478bd9Sstevel@tonic-gate * have the objects (cache_chunksize) to be a 64-byte multiple.
11987c478bd9Sstevel@tonic-gate * This helps to avoid false sharing as this is the size of the
11997c478bd9Sstevel@tonic-gate * CPU cache line.
12007c478bd9Sstevel@tonic-gate */
12017c478bd9Sstevel@tonic-gate kcf_sreq_cache = kmem_cache_create("kcf_sreq_cache",
12027c478bd9Sstevel@tonic-gate sizeof (struct kcf_sreq_node), 64, kcf_sreq_cache_constructor,
12037c478bd9Sstevel@tonic-gate kcf_sreq_cache_destructor, NULL, NULL, NULL, 0);
12047c478bd9Sstevel@tonic-gate
12057c478bd9Sstevel@tonic-gate kcf_areq_cache = kmem_cache_create("kcf_areq_cache",
12067c478bd9Sstevel@tonic-gate sizeof (struct kcf_areq_node), 64, kcf_areq_cache_constructor,
12077c478bd9Sstevel@tonic-gate kcf_areq_cache_destructor, NULL, NULL, NULL, 0);
12087c478bd9Sstevel@tonic-gate
12097c478bd9Sstevel@tonic-gate kcf_context_cache = kmem_cache_create("kcf_context_cache",
12107c478bd9Sstevel@tonic-gate sizeof (struct kcf_context), 64, kcf_context_cache_constructor,
12117c478bd9Sstevel@tonic-gate kcf_context_cache_destructor, NULL, NULL, NULL, 0);
12127c478bd9Sstevel@tonic-gate
12137c478bd9Sstevel@tonic-gate gswq = kmem_alloc(sizeof (kcf_global_swq_t), KM_SLEEP);
12147c478bd9Sstevel@tonic-gate
12157c478bd9Sstevel@tonic-gate mutex_init(&gswq->gs_lock, NULL, MUTEX_DEFAULT, NULL);
12167c478bd9Sstevel@tonic-gate cv_init(&gswq->gs_cv, NULL, CV_DEFAULT, NULL);
12177c478bd9Sstevel@tonic-gate gswq->gs_njobs = 0;
1218455859fbSDarren Moffat gswq->gs_maxjobs = kcf_maxthreads * crypto_taskq_maxalloc;
12197c478bd9Sstevel@tonic-gate gswq->gs_first = gswq->gs_last = NULL;
12207c478bd9Sstevel@tonic-gate
12217c478bd9Sstevel@tonic-gate /* Initialize the global reqid table */
12227c478bd9Sstevel@tonic-gate for (i = 0; i < REQID_TABLES; i++) {
12237c478bd9Sstevel@tonic-gate rt = kmem_zalloc(sizeof (kcf_reqid_table_t), KM_SLEEP);
12247c478bd9Sstevel@tonic-gate kcf_reqid_table[i] = rt;
12257c478bd9Sstevel@tonic-gate mutex_init(&rt->rt_lock, NULL, MUTEX_DEFAULT, NULL);
12267c478bd9Sstevel@tonic-gate rt->rt_curid = i;
12277c478bd9Sstevel@tonic-gate }
12287c478bd9Sstevel@tonic-gate
12297c478bd9Sstevel@tonic-gate /* Allocate and initialize the thread pool */
12307c478bd9Sstevel@tonic-gate kcfpool_alloc();
12317c478bd9Sstevel@tonic-gate
12327c478bd9Sstevel@tonic-gate /* Initialize the event notification list variables */
12337c478bd9Sstevel@tonic-gate mutex_init(&ntfy_list_lock, NULL, MUTEX_DEFAULT, NULL);
12347c478bd9Sstevel@tonic-gate cv_init(&ntfy_list_cv, NULL, CV_DEFAULT, NULL);
12357c478bd9Sstevel@tonic-gate
12367c478bd9Sstevel@tonic-gate /* Initialize the crypto_bufcall list variables */
12377c478bd9Sstevel@tonic-gate mutex_init(&cbuf_list_lock, NULL, MUTEX_DEFAULT, NULL);
12387c478bd9Sstevel@tonic-gate cv_init(&cbuf_list_cv, NULL, CV_DEFAULT, NULL);
12397c478bd9Sstevel@tonic-gate
12407c478bd9Sstevel@tonic-gate /* Create the kcf kstat */
12417c478bd9Sstevel@tonic-gate kcf_misc_kstat = kstat_create("kcf", 0, "framework_stats", "crypto",
12427c478bd9Sstevel@tonic-gate KSTAT_TYPE_NAMED, sizeof (kcf_stats_t) / sizeof (kstat_named_t),
12437c478bd9Sstevel@tonic-gate KSTAT_FLAG_VIRTUAL);
12447c478bd9Sstevel@tonic-gate
12457c478bd9Sstevel@tonic-gate if (kcf_misc_kstat != NULL) {
12467c478bd9Sstevel@tonic-gate kcf_misc_kstat->ks_data = &kcf_ksdata;
12477c478bd9Sstevel@tonic-gate kcf_misc_kstat->ks_update = kcf_misc_kstat_update;
12487c478bd9Sstevel@tonic-gate kstat_install(kcf_misc_kstat);
12497c478bd9Sstevel@tonic-gate }
12507c478bd9Sstevel@tonic-gate }
12517c478bd9Sstevel@tonic-gate
12527c478bd9Sstevel@tonic-gate /*
12537c478bd9Sstevel@tonic-gate * This routine should only be called by drv/cryptoadm.
12547c478bd9Sstevel@tonic-gate *
12557c478bd9Sstevel@tonic-gate * kcf_sched_running flag isn't protected by a lock. But, we are safe because
12567c478bd9Sstevel@tonic-gate * the first thread ("cryptoadm refresh") calling this routine during
12577c478bd9Sstevel@tonic-gate * boot time completes before any other thread that can call this routine.
12587c478bd9Sstevel@tonic-gate */
12597c478bd9Sstevel@tonic-gate void
kcf_sched_start(void)12607c478bd9Sstevel@tonic-gate kcf_sched_start(void)
12617c478bd9Sstevel@tonic-gate {
12627c478bd9Sstevel@tonic-gate if (kcf_sched_running)
12637c478bd9Sstevel@tonic-gate return;
12647c478bd9Sstevel@tonic-gate
12657c478bd9Sstevel@tonic-gate /* Start the background processing thread. */
12667c478bd9Sstevel@tonic-gate (void) thread_create(NULL, 0, &crypto_bufcall_service, 0, 0, &p0,
12677c478bd9Sstevel@tonic-gate TS_RUN, minclsyspri);
12687c478bd9Sstevel@tonic-gate
12697c478bd9Sstevel@tonic-gate kcf_sched_running = B_TRUE;
12707c478bd9Sstevel@tonic-gate }
12717c478bd9Sstevel@tonic-gate
12727c478bd9Sstevel@tonic-gate /*
12737c478bd9Sstevel@tonic-gate * Signal the waiting sync client.
12747c478bd9Sstevel@tonic-gate */
12757c478bd9Sstevel@tonic-gate void
kcf_sop_done(kcf_sreq_node_t * sreq,int error)12767c478bd9Sstevel@tonic-gate kcf_sop_done(kcf_sreq_node_t *sreq, int error)
12777c478bd9Sstevel@tonic-gate {
12787c478bd9Sstevel@tonic-gate mutex_enter(&sreq->sn_lock);
12797c478bd9Sstevel@tonic-gate sreq->sn_state = REQ_DONE;
12807c478bd9Sstevel@tonic-gate sreq->sn_rv = error;
12817c478bd9Sstevel@tonic-gate cv_signal(&sreq->sn_cv);
12827c478bd9Sstevel@tonic-gate mutex_exit(&sreq->sn_lock);
12837c478bd9Sstevel@tonic-gate }
12847c478bd9Sstevel@tonic-gate
12857c478bd9Sstevel@tonic-gate /*
12867c478bd9Sstevel@tonic-gate * Callback the async client with the operation status.
12877c478bd9Sstevel@tonic-gate * We free the async request node and possibly the context.
12887c478bd9Sstevel@tonic-gate * We also handle any chain of requests hanging off of
12897c478bd9Sstevel@tonic-gate * the context.
12907c478bd9Sstevel@tonic-gate */
12917c478bd9Sstevel@tonic-gate void
kcf_aop_done(kcf_areq_node_t * areq,int error)12927c478bd9Sstevel@tonic-gate kcf_aop_done(kcf_areq_node_t *areq, int error)
12937c478bd9Sstevel@tonic-gate {
12947c478bd9Sstevel@tonic-gate kcf_op_type_t optype;
12957c478bd9Sstevel@tonic-gate boolean_t skip_notify = B_FALSE;
12967c478bd9Sstevel@tonic-gate kcf_context_t *ictx;
12977c478bd9Sstevel@tonic-gate kcf_areq_node_t *nextreq;
12987c478bd9Sstevel@tonic-gate
12997c478bd9Sstevel@tonic-gate /*
13007c478bd9Sstevel@tonic-gate * Handle recoverable errors. This has to be done first
13017c478bd9Sstevel@tonic-gate * before doing any thing else in this routine so that
13027c478bd9Sstevel@tonic-gate * we do not change the state of the request.
13037c478bd9Sstevel@tonic-gate */
13047c478bd9Sstevel@tonic-gate if (error != CRYPTO_SUCCESS && IS_RECOVERABLE(error)) {
13057c478bd9Sstevel@tonic-gate /*
13067c478bd9Sstevel@tonic-gate * We try another provider, if one is available. Else
13077c478bd9Sstevel@tonic-gate * we continue with the failure notification to the
13087c478bd9Sstevel@tonic-gate * client.
13097c478bd9Sstevel@tonic-gate */
13107c478bd9Sstevel@tonic-gate if (kcf_resubmit_request(areq) == CRYPTO_QUEUED)
13117c478bd9Sstevel@tonic-gate return;
13127c478bd9Sstevel@tonic-gate }
13137c478bd9Sstevel@tonic-gate
13147c478bd9Sstevel@tonic-gate mutex_enter(&areq->an_lock);
13157c478bd9Sstevel@tonic-gate areq->an_state = REQ_DONE;
13167c478bd9Sstevel@tonic-gate mutex_exit(&areq->an_lock);
13177c478bd9Sstevel@tonic-gate
13187c478bd9Sstevel@tonic-gate optype = (&areq->an_params)->rp_optype;
13197c478bd9Sstevel@tonic-gate if ((ictx = areq->an_context) != NULL) {
13207c478bd9Sstevel@tonic-gate /*
13217c478bd9Sstevel@tonic-gate * A request after it is removed from the request
13227c478bd9Sstevel@tonic-gate * queue, still stays on a chain of requests hanging
13237c478bd9Sstevel@tonic-gate * of its context structure. It needs to be removed
13247c478bd9Sstevel@tonic-gate * from this chain at this point.
13257c478bd9Sstevel@tonic-gate */
13267c478bd9Sstevel@tonic-gate mutex_enter(&ictx->kc_in_use_lock);
13277c478bd9Sstevel@tonic-gate nextreq = areq->an_ctxchain_next;
13287c478bd9Sstevel@tonic-gate if (nextreq != NULL) {
1329c41e7ccaSkrishna mutex_enter(&nextreq->an_lock);
13307c478bd9Sstevel@tonic-gate nextreq->an_is_my_turn = B_TRUE;
1331c41e7ccaSkrishna cv_signal(&nextreq->an_turn_cv);
1332c41e7ccaSkrishna mutex_exit(&nextreq->an_lock);
13337c478bd9Sstevel@tonic-gate }
13347c478bd9Sstevel@tonic-gate
13357c478bd9Sstevel@tonic-gate ictx->kc_req_chain_first = nextreq;
13367c478bd9Sstevel@tonic-gate if (nextreq == NULL)
13377c478bd9Sstevel@tonic-gate ictx->kc_req_chain_last = NULL;
13387c478bd9Sstevel@tonic-gate mutex_exit(&ictx->kc_in_use_lock);
13397c478bd9Sstevel@tonic-gate
13407c478bd9Sstevel@tonic-gate if (IS_SINGLE_OP(optype) || IS_FINAL_OP(optype)) {
13417c478bd9Sstevel@tonic-gate ASSERT(nextreq == NULL);
13427c478bd9Sstevel@tonic-gate KCF_CONTEXT_REFRELE(ictx);
13437c478bd9Sstevel@tonic-gate } else if (error != CRYPTO_SUCCESS && IS_INIT_OP(optype)) {
13447c478bd9Sstevel@tonic-gate /*
13457c478bd9Sstevel@tonic-gate * NOTE - We do not release the context in case of update
13467c478bd9Sstevel@tonic-gate * operations. We require the consumer to free it explicitly,
13477c478bd9Sstevel@tonic-gate * in case it wants to abandon an update operation. This is done
13487c478bd9Sstevel@tonic-gate * as there may be mechanisms in ECB mode that can continue
13497c478bd9Sstevel@tonic-gate * even if an operation on a block fails.
13507c478bd9Sstevel@tonic-gate */
13517c478bd9Sstevel@tonic-gate KCF_CONTEXT_REFRELE(ictx);
13527c478bd9Sstevel@tonic-gate }
13537c478bd9Sstevel@tonic-gate }
13547c478bd9Sstevel@tonic-gate
13557c478bd9Sstevel@tonic-gate /* Deal with the internal continuation to this request first */
13567c478bd9Sstevel@tonic-gate
13577c478bd9Sstevel@tonic-gate if (areq->an_isdual) {
13587c478bd9Sstevel@tonic-gate kcf_dual_req_t *next_arg;
13597c478bd9Sstevel@tonic-gate next_arg = (kcf_dual_req_t *)areq->an_reqarg.cr_callback_arg;
13607c478bd9Sstevel@tonic-gate next_arg->kr_areq = areq;
13617c478bd9Sstevel@tonic-gate KCF_AREQ_REFHOLD(areq);
13627c478bd9Sstevel@tonic-gate areq->an_isdual = B_FALSE;
13637c478bd9Sstevel@tonic-gate
13647c478bd9Sstevel@tonic-gate NOTIFY_CLIENT(areq, error);
13657c478bd9Sstevel@tonic-gate return;
13667c478bd9Sstevel@tonic-gate }
13677c478bd9Sstevel@tonic-gate
13687c478bd9Sstevel@tonic-gate /*
13697c478bd9Sstevel@tonic-gate * If CRYPTO_NOTIFY_OPDONE flag is set, we should notify
13707c478bd9Sstevel@tonic-gate * always. If this flag is clear, we skip the notification
13717c478bd9Sstevel@tonic-gate * provided there are no errors. We check this flag for only
13727c478bd9Sstevel@tonic-gate * init or update operations. It is ignored for single, final or
13737c478bd9Sstevel@tonic-gate * atomic operations.
13747c478bd9Sstevel@tonic-gate */
13757c478bd9Sstevel@tonic-gate skip_notify = (IS_UPDATE_OP(optype) || IS_INIT_OP(optype)) &&
13767c478bd9Sstevel@tonic-gate (!(areq->an_reqarg.cr_flag & CRYPTO_NOTIFY_OPDONE)) &&
13777c478bd9Sstevel@tonic-gate (error == CRYPTO_SUCCESS);
13787c478bd9Sstevel@tonic-gate
13797c478bd9Sstevel@tonic-gate if (!skip_notify) {
13807c478bd9Sstevel@tonic-gate NOTIFY_CLIENT(areq, error);
13817c478bd9Sstevel@tonic-gate }
13827c478bd9Sstevel@tonic-gate
13837c478bd9Sstevel@tonic-gate if (!(areq->an_reqarg.cr_flag & CRYPTO_SKIP_REQID))
13847c478bd9Sstevel@tonic-gate kcf_reqid_delete(areq);
13857c478bd9Sstevel@tonic-gate
13867c478bd9Sstevel@tonic-gate KCF_AREQ_REFRELE(areq);
13877c478bd9Sstevel@tonic-gate }
13887c478bd9Sstevel@tonic-gate
13897c478bd9Sstevel@tonic-gate /*
13906ea3c060SGarrett D'Amore * kcfpool thread spawner. This runs as a process that never exits.
13916ea3c060SGarrett D'Amore * Its a process so that the threads it owns can be manipulated via priocntl.
13927c478bd9Sstevel@tonic-gate */
13937c478bd9Sstevel@tonic-gate static void
kcfpoold(void * arg)13946ea3c060SGarrett D'Amore kcfpoold(void *arg)
13957c478bd9Sstevel@tonic-gate {
13966ea3c060SGarrett D'Amore callb_cpr_t cprinfo;
13976ea3c060SGarrett D'Amore user_t *pu = PTOU(curproc);
13986ea3c060SGarrett D'Amore int cnt;
13996ea3c060SGarrett D'Amore clock_t timeout_val = drv_usectohz(kcf_idlethr_timeout);
14006ea3c060SGarrett D'Amore _NOTE(ARGUNUSED(arg));
14017c478bd9Sstevel@tonic-gate
14026ea3c060SGarrett D'Amore CALLB_CPR_INIT(&cprinfo, &kcfpool->kp_lock,
14036ea3c060SGarrett D'Amore callb_generic_cpr, "kcfpool");
14047c478bd9Sstevel@tonic-gate
14056ea3c060SGarrett D'Amore /* make our process "kcfpoold" */
14066ea3c060SGarrett D'Amore (void) snprintf(pu->u_psargs, sizeof (pu->u_psargs), "kcfpoold");
14076ea3c060SGarrett D'Amore (void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm));
14087c478bd9Sstevel@tonic-gate
14096ea3c060SGarrett D'Amore mutex_enter(&kcfpool->kp_lock);
14107c478bd9Sstevel@tonic-gate
14116ea3c060SGarrett D'Amore /*
14126ea3c060SGarrett D'Amore * Go to sleep, waiting for the signaled flag. Note that as
14136ea3c060SGarrett D'Amore * we always do the same thing, and its always idempotent, we
14146ea3c060SGarrett D'Amore * don't even need to have a real condition to check against.
14156ea3c060SGarrett D'Amore */
14166ea3c060SGarrett D'Amore for (;;) {
14176ea3c060SGarrett D'Amore int rv;
141857a30242SGarrett D'Amore
14196ea3c060SGarrett D'Amore CALLB_CPR_SAFE_BEGIN(&cprinfo);
14206ea3c060SGarrett D'Amore rv = cv_reltimedwait(&kcfpool->kp_cv,
14216ea3c060SGarrett D'Amore &kcfpool->kp_lock, timeout_val, TR_CLOCK_TICK);
14226ea3c060SGarrett D'Amore CALLB_CPR_SAFE_END(&cprinfo, &kcfpool->kp_lock);
14237c478bd9Sstevel@tonic-gate
14247c478bd9Sstevel@tonic-gate switch (rv) {
14257c478bd9Sstevel@tonic-gate case -1:
14267c478bd9Sstevel@tonic-gate /* Timed out. Recalculate the min/max threads */
14277c478bd9Sstevel@tonic-gate compute_min_max_threads();
14287c478bd9Sstevel@tonic-gate break;
14297c478bd9Sstevel@tonic-gate
14307c478bd9Sstevel@tonic-gate default:
14316ea3c060SGarrett D'Amore /* Someone may be looking for a worker thread */
14327c478bd9Sstevel@tonic-gate break;
14337c478bd9Sstevel@tonic-gate }
14346ea3c060SGarrett D'Amore
14356ea3c060SGarrett D'Amore /*
14366ea3c060SGarrett D'Amore * We keep the number of running threads to be at
14376ea3c060SGarrett D'Amore * kcf_minthreads to reduce gs_lock contention.
14386ea3c060SGarrett D'Amore */
14396ea3c060SGarrett D'Amore cnt = kcf_minthreads -
14406ea3c060SGarrett D'Amore (kcfpool->kp_threads - kcfpool->kp_blockedthreads);
14416ea3c060SGarrett D'Amore if (cnt > 0) {
14426ea3c060SGarrett D'Amore /*
14436ea3c060SGarrett D'Amore * The following ensures the number of threads in pool
14446ea3c060SGarrett D'Amore * does not exceed kcf_maxthreads.
14456ea3c060SGarrett D'Amore */
14466ea3c060SGarrett D'Amore cnt = min(cnt, kcf_maxthreads - kcfpool->kp_threads);
14476ea3c060SGarrett D'Amore }
14486ea3c060SGarrett D'Amore
14496ea3c060SGarrett D'Amore for (int i = 0; i < cnt; i++) {
14506ea3c060SGarrett D'Amore (void) lwp_kernel_create(curproc,
14516ea3c060SGarrett D'Amore kcfpool_svc, NULL, TS_RUN, curthread->t_pri);
14526ea3c060SGarrett D'Amore }
14537c478bd9Sstevel@tonic-gate }
14546ea3c060SGarrett D'Amore }
14557c478bd9Sstevel@tonic-gate
14566ea3c060SGarrett D'Amore /*
14576ea3c060SGarrett D'Amore * Allocate the thread pool and initialize all the fields.
14586ea3c060SGarrett D'Amore */
14596ea3c060SGarrett D'Amore static void
kcfpool_alloc(void)14606ea3c060SGarrett D'Amore kcfpool_alloc(void)
14616ea3c060SGarrett D'Amore {
14626ea3c060SGarrett D'Amore kcfpool = kmem_alloc(sizeof (kcf_pool_t), KM_SLEEP);
14636ea3c060SGarrett D'Amore
14646ea3c060SGarrett D'Amore kcfpool->kp_threads = kcfpool->kp_idlethreads = 0;
14656ea3c060SGarrett D'Amore kcfpool->kp_blockedthreads = 0;
14667c478bd9Sstevel@tonic-gate
14676ea3c060SGarrett D'Amore mutex_init(&kcfpool->kp_lock, NULL, MUTEX_DEFAULT, NULL);
14686ea3c060SGarrett D'Amore cv_init(&kcfpool->kp_cv, NULL, CV_DEFAULT, NULL);
14697c478bd9Sstevel@tonic-gate
14706ea3c060SGarrett D'Amore kcf_idlethr_timeout = KCF_DEFAULT_THRTIMEOUT;
14717c478bd9Sstevel@tonic-gate
14726ea3c060SGarrett D'Amore /*
14736ea3c060SGarrett D'Amore * Create the daemon thread.
14746ea3c060SGarrett D'Amore */
14756ea3c060SGarrett D'Amore if (newproc(kcfpoold, NULL, syscid, minclsyspri,
14766ea3c060SGarrett D'Amore NULL, 0) != 0) {
14776ea3c060SGarrett D'Amore cmn_err(CE_PANIC, "unable to fork kcfpoold()");
14786ea3c060SGarrett D'Amore }
14796ea3c060SGarrett D'Amore }
14807c478bd9Sstevel@tonic-gate
14817c478bd9Sstevel@tonic-gate /*
14827c478bd9Sstevel@tonic-gate * This routine introduces a locking order for gswq->gs_lock followed
14837c478bd9Sstevel@tonic-gate * by cpu_lock.
14847c478bd9Sstevel@tonic-gate * This means that no consumer of the k-api should hold cpu_lock when calling
14857c478bd9Sstevel@tonic-gate * k-api routines.
14867c478bd9Sstevel@tonic-gate */
14877c478bd9Sstevel@tonic-gate static void
compute_min_max_threads(void)14886ea3c060SGarrett D'Amore compute_min_max_threads(void)
14897c478bd9Sstevel@tonic-gate {
14907c478bd9Sstevel@tonic-gate mutex_enter(&gswq->gs_lock);
1491455859fbSDarren Moffat mutex_enter(&cpu_lock);
1492455859fbSDarren Moffat kcf_minthreads = curthread->t_cpupart->cp_ncpus;
1493455859fbSDarren Moffat mutex_exit(&cpu_lock);
14947c478bd9Sstevel@tonic-gate kcf_maxthreads = kcf_thr_multiple * kcf_minthreads;
14957c478bd9Sstevel@tonic-gate gswq->gs_maxjobs = kcf_maxthreads * crypto_taskq_maxalloc;
14967c478bd9Sstevel@tonic-gate mutex_exit(&gswq->gs_lock);
14977c478bd9Sstevel@tonic-gate }
14987c478bd9Sstevel@tonic-gate
14997c478bd9Sstevel@tonic-gate /*
15007c478bd9Sstevel@tonic-gate * Insert the async request in the hash table after assigning it
15017c478bd9Sstevel@tonic-gate * an ID. Returns the ID.
15027c478bd9Sstevel@tonic-gate *
15037c478bd9Sstevel@tonic-gate * The ID is used by the caller to pass as an argument to a
15047c478bd9Sstevel@tonic-gate * cancel_req() routine later.
15057c478bd9Sstevel@tonic-gate */
15067c478bd9Sstevel@tonic-gate static crypto_req_id_t
kcf_reqid_insert(kcf_areq_node_t * areq)15077c478bd9Sstevel@tonic-gate kcf_reqid_insert(kcf_areq_node_t *areq)
15087c478bd9Sstevel@tonic-gate {
15097c478bd9Sstevel@tonic-gate int indx;
15107c478bd9Sstevel@tonic-gate crypto_req_id_t id;
15117c478bd9Sstevel@tonic-gate kcf_areq_node_t *headp;
15127c478bd9Sstevel@tonic-gate kcf_reqid_table_t *rt =
15137c478bd9Sstevel@tonic-gate kcf_reqid_table[CPU->cpu_seqid & REQID_TABLE_MASK];
15147c478bd9Sstevel@tonic-gate
15157c478bd9Sstevel@tonic-gate mutex_enter(&rt->rt_lock);
15167c478bd9Sstevel@tonic-gate
15177c478bd9Sstevel@tonic-gate rt->rt_curid = id =
15187c478bd9Sstevel@tonic-gate (rt->rt_curid - REQID_COUNTER_LOW) | REQID_COUNTER_HIGH;
15197c478bd9Sstevel@tonic-gate SET_REQID(areq, id);
15207c478bd9Sstevel@tonic-gate indx = REQID_HASH(id);
15217c478bd9Sstevel@tonic-gate headp = areq->an_idnext = rt->rt_idhash[indx];
15227c478bd9Sstevel@tonic-gate areq->an_idprev = NULL;
15237c478bd9Sstevel@tonic-gate if (headp != NULL)
15247c478bd9Sstevel@tonic-gate headp->an_idprev = areq;
15257c478bd9Sstevel@tonic-gate
15267c478bd9Sstevel@tonic-gate rt->rt_idhash[indx] = areq;
15277c478bd9Sstevel@tonic-gate mutex_exit(&rt->rt_lock);
15287c478bd9Sstevel@tonic-gate
15297c478bd9Sstevel@tonic-gate return (id);
15307c478bd9Sstevel@tonic-gate }
15317c478bd9Sstevel@tonic-gate
15327c478bd9Sstevel@tonic-gate /*
15337c478bd9Sstevel@tonic-gate * Delete the async request from the hash table.
15347c478bd9Sstevel@tonic-gate */
15357c478bd9Sstevel@tonic-gate static void
kcf_reqid_delete(kcf_areq_node_t * areq)15367c478bd9Sstevel@tonic-gate kcf_reqid_delete(kcf_areq_node_t *areq)
15377c478bd9Sstevel@tonic-gate {
15387c478bd9Sstevel@tonic-gate int indx;
15397c478bd9Sstevel@tonic-gate kcf_areq_node_t *nextp, *prevp;
15407c478bd9Sstevel@tonic-gate crypto_req_id_t id = GET_REQID(areq);
15417c478bd9Sstevel@tonic-gate kcf_reqid_table_t *rt;
15427c478bd9Sstevel@tonic-gate
15437c478bd9Sstevel@tonic-gate rt = kcf_reqid_table[id & REQID_TABLE_MASK];
15447c478bd9Sstevel@tonic-gate indx = REQID_HASH(id);
15457c478bd9Sstevel@tonic-gate
15467c478bd9Sstevel@tonic-gate mutex_enter(&rt->rt_lock);
15477c478bd9Sstevel@tonic-gate
15487c478bd9Sstevel@tonic-gate nextp = areq->an_idnext;
15497c478bd9Sstevel@tonic-gate prevp = areq->an_idprev;
15507c478bd9Sstevel@tonic-gate if (nextp != NULL)
15517c478bd9Sstevel@tonic-gate nextp->an_idprev = prevp;
15527c478bd9Sstevel@tonic-gate if (prevp != NULL)
15537c478bd9Sstevel@tonic-gate prevp->an_idnext = nextp;
15547c478bd9Sstevel@tonic-gate else
15557c478bd9Sstevel@tonic-gate rt->rt_idhash[indx] = nextp;
15567c478bd9Sstevel@tonic-gate
15577c478bd9Sstevel@tonic-gate SET_REQID(areq, 0);
15587c478bd9Sstevel@tonic-gate cv_broadcast(&areq->an_done);
15597c478bd9Sstevel@tonic-gate
15607c478bd9Sstevel@tonic-gate mutex_exit(&rt->rt_lock);
15617c478bd9Sstevel@tonic-gate }
15627c478bd9Sstevel@tonic-gate
15637c478bd9Sstevel@tonic-gate /*
15647c478bd9Sstevel@tonic-gate * Cancel a single asynchronous request.
15657c478bd9Sstevel@tonic-gate *
15667c478bd9Sstevel@tonic-gate * We guarantee that no problems will result from calling
15677c478bd9Sstevel@tonic-gate * crypto_cancel_req() for a request which is either running, or
15687c478bd9Sstevel@tonic-gate * has already completed. We remove the request from any queues
15697c478bd9Sstevel@tonic-gate * if it is possible. We wait for request completion if the
15707c478bd9Sstevel@tonic-gate * request is dispatched to a provider.
15717c478bd9Sstevel@tonic-gate *
15727c478bd9Sstevel@tonic-gate * Calling context:
15737c478bd9Sstevel@tonic-gate * Can be called from user context only.
15747c478bd9Sstevel@tonic-gate *
15757c478bd9Sstevel@tonic-gate * NOTE: We acquire the following locks in this routine (in order):
15767c478bd9Sstevel@tonic-gate * - rt_lock (kcf_reqid_table_t)
15777c478bd9Sstevel@tonic-gate * - gswq->gs_lock
15787c478bd9Sstevel@tonic-gate * - areq->an_lock
15797c478bd9Sstevel@tonic-gate * - ictx->kc_in_use_lock (from kcf_removereq_in_ctxchain())
15807c478bd9Sstevel@tonic-gate *
15817c478bd9Sstevel@tonic-gate * This locking order MUST be maintained in code every where else.
15827c478bd9Sstevel@tonic-gate */
15837c478bd9Sstevel@tonic-gate void
crypto_cancel_req(crypto_req_id_t id)15847c478bd9Sstevel@tonic-gate crypto_cancel_req(crypto_req_id_t id)
15857c478bd9Sstevel@tonic-gate {
15867c478bd9Sstevel@tonic-gate int indx;
15877c478bd9Sstevel@tonic-gate kcf_areq_node_t *areq;
15887c478bd9Sstevel@tonic-gate kcf_provider_desc_t *pd;
15897c478bd9Sstevel@tonic-gate kcf_context_t *ictx;
15907c478bd9Sstevel@tonic-gate kcf_reqid_table_t *rt;
15917c478bd9Sstevel@tonic-gate
15927c478bd9Sstevel@tonic-gate rt = kcf_reqid_table[id & REQID_TABLE_MASK];
15937c478bd9Sstevel@tonic-gate indx = REQID_HASH(id);
15947c478bd9Sstevel@tonic-gate
15957c478bd9Sstevel@tonic-gate mutex_enter(&rt->rt_lock);
15967c478bd9Sstevel@tonic-gate for (areq = rt->rt_idhash[indx]; areq; areq = areq->an_idnext) {
15977c478bd9Sstevel@tonic-gate if (GET_REQID(areq) == id) {
15987c478bd9Sstevel@tonic-gate /*
15997c478bd9Sstevel@tonic-gate * We found the request. It is either still waiting
16007c478bd9Sstevel@tonic-gate * in the framework queues or running at the provider.
16017c478bd9Sstevel@tonic-gate */
16027c478bd9Sstevel@tonic-gate pd = areq->an_provider;
16037c478bd9Sstevel@tonic-gate ASSERT(pd != NULL);
16047c478bd9Sstevel@tonic-gate
16057c478bd9Sstevel@tonic-gate switch (pd->pd_prov_type) {
16067c478bd9Sstevel@tonic-gate case CRYPTO_SW_PROVIDER:
16077c478bd9Sstevel@tonic-gate mutex_enter(&gswq->gs_lock);
16087c478bd9Sstevel@tonic-gate mutex_enter(&areq->an_lock);
16097c478bd9Sstevel@tonic-gate
16107c478bd9Sstevel@tonic-gate /* This request can be safely canceled. */
16117c478bd9Sstevel@tonic-gate if (areq->an_state <= REQ_WAITING) {
16127c478bd9Sstevel@tonic-gate /* Remove from gswq, global software queue. */
16137c478bd9Sstevel@tonic-gate kcf_remove_node(areq);
16147c478bd9Sstevel@tonic-gate if ((ictx = areq->an_context) != NULL)
16157c478bd9Sstevel@tonic-gate kcf_removereq_in_ctxchain(ictx, areq);
16167c478bd9Sstevel@tonic-gate
16177c478bd9Sstevel@tonic-gate mutex_exit(&areq->an_lock);
16187c478bd9Sstevel@tonic-gate mutex_exit(&gswq->gs_lock);
16197c478bd9Sstevel@tonic-gate mutex_exit(&rt->rt_lock);
16207c478bd9Sstevel@tonic-gate
16217c478bd9Sstevel@tonic-gate /* Remove areq from hash table and free it. */
16227c478bd9Sstevel@tonic-gate kcf_reqid_delete(areq);
16237c478bd9Sstevel@tonic-gate KCF_AREQ_REFRELE(areq);
16247c478bd9Sstevel@tonic-gate return;
16257c478bd9Sstevel@tonic-gate }
16267c478bd9Sstevel@tonic-gate
16277c478bd9Sstevel@tonic-gate mutex_exit(&areq->an_lock);
16287c478bd9Sstevel@tonic-gate mutex_exit(&gswq->gs_lock);
16297c478bd9Sstevel@tonic-gate break;
16307c478bd9Sstevel@tonic-gate
16317c478bd9Sstevel@tonic-gate case CRYPTO_HW_PROVIDER:
16327c478bd9Sstevel@tonic-gate /*
16337c478bd9Sstevel@tonic-gate * There is no interface to remove an entry
16347c478bd9Sstevel@tonic-gate * once it is on the taskq. So, we do not do
16357c478bd9Sstevel@tonic-gate * any thing for a hardware provider.
16367c478bd9Sstevel@tonic-gate */
16377c478bd9Sstevel@tonic-gate break;
16387c478bd9Sstevel@tonic-gate }
16397c478bd9Sstevel@tonic-gate
16407c478bd9Sstevel@tonic-gate /*
16417c478bd9Sstevel@tonic-gate * The request is running. Wait for the request completion
16427c478bd9Sstevel@tonic-gate * to notify us.
16437c478bd9Sstevel@tonic-gate */
16447c478bd9Sstevel@tonic-gate KCF_AREQ_REFHOLD(areq);
16457c478bd9Sstevel@tonic-gate while (GET_REQID(areq) == id)
16467c478bd9Sstevel@tonic-gate cv_wait(&areq->an_done, &rt->rt_lock);
16477c478bd9Sstevel@tonic-gate KCF_AREQ_REFRELE(areq);
16487c478bd9Sstevel@tonic-gate break;
16497c478bd9Sstevel@tonic-gate }
16507c478bd9Sstevel@tonic-gate }
16517c478bd9Sstevel@tonic-gate
16527c478bd9Sstevel@tonic-gate mutex_exit(&rt->rt_lock);
16537c478bd9Sstevel@tonic-gate }
16547c478bd9Sstevel@tonic-gate
16557c478bd9Sstevel@tonic-gate /*
16567c478bd9Sstevel@tonic-gate * Cancel all asynchronous requests associated with the
16577c478bd9Sstevel@tonic-gate * passed in crypto context and free it.
16587c478bd9Sstevel@tonic-gate *
16597c478bd9Sstevel@tonic-gate * A client SHOULD NOT call this routine after calling a crypto_*_final
16607c478bd9Sstevel@tonic-gate * routine. This routine is called only during intermediate operations.
16617c478bd9Sstevel@tonic-gate * The client should not use the crypto context after this function returns
16627c478bd9Sstevel@tonic-gate * since we destroy it.
16637c478bd9Sstevel@tonic-gate *
16647c478bd9Sstevel@tonic-gate * Calling context:
16657c478bd9Sstevel@tonic-gate * Can be called from user context only.
16667c478bd9Sstevel@tonic-gate */
16677c478bd9Sstevel@tonic-gate void
crypto_cancel_ctx(crypto_context_t ctx)16687c478bd9Sstevel@tonic-gate crypto_cancel_ctx(crypto_context_t ctx)
16697c478bd9Sstevel@tonic-gate {
16707c478bd9Sstevel@tonic-gate kcf_context_t *ictx;
16717c478bd9Sstevel@tonic-gate kcf_areq_node_t *areq;
16727c478bd9Sstevel@tonic-gate
16737c478bd9Sstevel@tonic-gate if (ctx == NULL)
16747c478bd9Sstevel@tonic-gate return;
16757c478bd9Sstevel@tonic-gate
16767c478bd9Sstevel@tonic-gate ictx = (kcf_context_t *)((crypto_ctx_t *)ctx)->cc_framework_private;
16777c478bd9Sstevel@tonic-gate
16787c478bd9Sstevel@tonic-gate mutex_enter(&ictx->kc_in_use_lock);
16797c478bd9Sstevel@tonic-gate
16807c478bd9Sstevel@tonic-gate /* Walk the chain and cancel each request */
16817c478bd9Sstevel@tonic-gate while ((areq = ictx->kc_req_chain_first) != NULL) {
16827c478bd9Sstevel@tonic-gate /*
16837c478bd9Sstevel@tonic-gate * We have to drop the lock here as we may have
16847c478bd9Sstevel@tonic-gate * to wait for request completion. We hold the
16857c478bd9Sstevel@tonic-gate * request before dropping the lock though, so that it
16867c478bd9Sstevel@tonic-gate * won't be freed underneath us.
16877c478bd9Sstevel@tonic-gate */
16887c478bd9Sstevel@tonic-gate KCF_AREQ_REFHOLD(areq);
16897c478bd9Sstevel@tonic-gate mutex_exit(&ictx->kc_in_use_lock);
16907c478bd9Sstevel@tonic-gate
16917c478bd9Sstevel@tonic-gate crypto_cancel_req(GET_REQID(areq));
16927c478bd9Sstevel@tonic-gate KCF_AREQ_REFRELE(areq);
16937c478bd9Sstevel@tonic-gate
16947c478bd9Sstevel@tonic-gate mutex_enter(&ictx->kc_in_use_lock);
16957c478bd9Sstevel@tonic-gate }
16967c478bd9Sstevel@tonic-gate
16977c478bd9Sstevel@tonic-gate mutex_exit(&ictx->kc_in_use_lock);
16987c478bd9Sstevel@tonic-gate KCF_CONTEXT_REFRELE(ictx);
16997c478bd9Sstevel@tonic-gate }
17007c478bd9Sstevel@tonic-gate
17017c478bd9Sstevel@tonic-gate /*
17027c478bd9Sstevel@tonic-gate * Update kstats.
17037c478bd9Sstevel@tonic-gate */
17047c478bd9Sstevel@tonic-gate static int
kcf_misc_kstat_update(kstat_t * ksp,int rw)17057c478bd9Sstevel@tonic-gate kcf_misc_kstat_update(kstat_t *ksp, int rw)
17067c478bd9Sstevel@tonic-gate {
17077c478bd9Sstevel@tonic-gate kcf_stats_t *ks_data;
17087c478bd9Sstevel@tonic-gate
17097c478bd9Sstevel@tonic-gate if (rw == KSTAT_WRITE)
17107c478bd9Sstevel@tonic-gate return (EACCES);
17117c478bd9Sstevel@tonic-gate
17127c478bd9Sstevel@tonic-gate ks_data = ksp->ks_data;
17137c478bd9Sstevel@tonic-gate
17147c478bd9Sstevel@tonic-gate ks_data->ks_thrs_in_pool.value.ui32 = kcfpool->kp_threads;
17156ea3c060SGarrett D'Amore ks_data->ks_idle_thrs.value.ui32 = kcfpool->kp_idlethreads;
17167c478bd9Sstevel@tonic-gate ks_data->ks_minthrs.value.ui32 = kcf_minthreads;
17177c478bd9Sstevel@tonic-gate ks_data->ks_maxthrs.value.ui32 = kcf_maxthreads;
17187c478bd9Sstevel@tonic-gate ks_data->ks_swq_njobs.value.ui32 = gswq->gs_njobs;
17197c478bd9Sstevel@tonic-gate ks_data->ks_swq_maxjobs.value.ui32 = gswq->gs_maxjobs;
1720c41e7ccaSkrishna ks_data->ks_taskq_threads.value.ui32 = crypto_taskq_threads;
17217c478bd9Sstevel@tonic-gate ks_data->ks_taskq_minalloc.value.ui32 = crypto_taskq_minalloc;
17227c478bd9Sstevel@tonic-gate ks_data->ks_taskq_maxalloc.value.ui32 = crypto_taskq_maxalloc;
17237c478bd9Sstevel@tonic-gate
17247c478bd9Sstevel@tonic-gate return (0);
17257c478bd9Sstevel@tonic-gate }
17267c478bd9Sstevel@tonic-gate
17277c478bd9Sstevel@tonic-gate /*
17287c478bd9Sstevel@tonic-gate * Allocate and initiatize a kcf_dual_req, used for saving the arguments of
17297c478bd9Sstevel@tonic-gate * a dual operation or an atomic operation that has to be internally
17307c478bd9Sstevel@tonic-gate * simulated with multiple single steps.
17317c478bd9Sstevel@tonic-gate * crq determines the memory allocation flags.
17327c478bd9Sstevel@tonic-gate */
17337c478bd9Sstevel@tonic-gate
17347c478bd9Sstevel@tonic-gate kcf_dual_req_t *
kcf_alloc_req(crypto_call_req_t * crq)17357c478bd9Sstevel@tonic-gate kcf_alloc_req(crypto_call_req_t *crq)
17367c478bd9Sstevel@tonic-gate {
17377c478bd9Sstevel@tonic-gate kcf_dual_req_t *kcr;
17387c478bd9Sstevel@tonic-gate
17397c478bd9Sstevel@tonic-gate kcr = kmem_alloc(sizeof (kcf_dual_req_t), KCF_KMFLAG(crq));
17407c478bd9Sstevel@tonic-gate
17417c478bd9Sstevel@tonic-gate if (kcr == NULL)
17427c478bd9Sstevel@tonic-gate return (NULL);
17437c478bd9Sstevel@tonic-gate
174495014fbbSDan OpenSolaris Anderson /* Copy the whole crypto_call_req struct, as it isn't persistent */
17457c478bd9Sstevel@tonic-gate if (crq != NULL)
17467c478bd9Sstevel@tonic-gate kcr->kr_callreq = *crq;
17477c478bd9Sstevel@tonic-gate else
17487c478bd9Sstevel@tonic-gate bzero(&(kcr->kr_callreq), sizeof (crypto_call_req_t));
17497c478bd9Sstevel@tonic-gate kcr->kr_areq = NULL;
17507c478bd9Sstevel@tonic-gate kcr->kr_saveoffset = 0;
17517c478bd9Sstevel@tonic-gate kcr->kr_savelen = 0;
17527c478bd9Sstevel@tonic-gate
17537c478bd9Sstevel@tonic-gate return (kcr);
17547c478bd9Sstevel@tonic-gate }
17557c478bd9Sstevel@tonic-gate
17567c478bd9Sstevel@tonic-gate /*
17577c478bd9Sstevel@tonic-gate * Callback routine for the next part of a simulated dual part.
17587c478bd9Sstevel@tonic-gate * Schedules the next step.
17597c478bd9Sstevel@tonic-gate *
17607c478bd9Sstevel@tonic-gate * This routine can be called from interrupt context.
17617c478bd9Sstevel@tonic-gate */
17627c478bd9Sstevel@tonic-gate void
kcf_next_req(void * next_req_arg,int status)17637c478bd9Sstevel@tonic-gate kcf_next_req(void *next_req_arg, int status)
17647c478bd9Sstevel@tonic-gate {
17657c478bd9Sstevel@tonic-gate kcf_dual_req_t *next_req = (kcf_dual_req_t *)next_req_arg;
17667c478bd9Sstevel@tonic-gate kcf_req_params_t *params = &(next_req->kr_params);
17677c478bd9Sstevel@tonic-gate kcf_areq_node_t *areq = next_req->kr_areq;
17687c478bd9Sstevel@tonic-gate int error = status;
17697c478bd9Sstevel@tonic-gate kcf_provider_desc_t *pd;
17707c478bd9Sstevel@tonic-gate crypto_dual_data_t *ct;
17717c478bd9Sstevel@tonic-gate
177295014fbbSDan OpenSolaris Anderson /* Stop the processing if an error occurred at this step */
17737c478bd9Sstevel@tonic-gate if (error != CRYPTO_SUCCESS) {
17747c478bd9Sstevel@tonic-gate out:
17757c478bd9Sstevel@tonic-gate areq->an_reqarg = next_req->kr_callreq;
17767c478bd9Sstevel@tonic-gate KCF_AREQ_REFRELE(areq);
17777c478bd9Sstevel@tonic-gate kmem_free(next_req, sizeof (kcf_dual_req_t));
17787c478bd9Sstevel@tonic-gate areq->an_isdual = B_FALSE;
17797c478bd9Sstevel@tonic-gate kcf_aop_done(areq, error);
17807c478bd9Sstevel@tonic-gate return;
17817c478bd9Sstevel@tonic-gate }
17827c478bd9Sstevel@tonic-gate
17837c478bd9Sstevel@tonic-gate switch (params->rp_opgrp) {
17847c478bd9Sstevel@tonic-gate case KCF_OG_MAC: {
17857c478bd9Sstevel@tonic-gate
17867c478bd9Sstevel@tonic-gate /*
17877c478bd9Sstevel@tonic-gate * The next req is submitted with the same reqid as the
17887c478bd9Sstevel@tonic-gate * first part. The consumer only got back that reqid, and
17897c478bd9Sstevel@tonic-gate * should still be able to cancel the operation during its
17907c478bd9Sstevel@tonic-gate * second step.
17917c478bd9Sstevel@tonic-gate */
17927c478bd9Sstevel@tonic-gate kcf_mac_ops_params_t *mops = &(params->rp_u.mac_params);
17937c478bd9Sstevel@tonic-gate crypto_ctx_template_t mac_tmpl;
17947c478bd9Sstevel@tonic-gate kcf_mech_entry_t *me;
17957c478bd9Sstevel@tonic-gate
17967c478bd9Sstevel@tonic-gate ct = (crypto_dual_data_t *)mops->mo_data;
17977c478bd9Sstevel@tonic-gate mac_tmpl = (crypto_ctx_template_t)mops->mo_templ;
17987c478bd9Sstevel@tonic-gate
17997c478bd9Sstevel@tonic-gate /* No expected recoverable failures, so no retry list */
1800436935a1SVladimir Kotal pd = kcf_get_mech_provider(mops->mo_framework_mechtype, NULL,
18019b009fc1SValerie Bubb Fenwick &me, &error, NULL, CRYPTO_FG_MAC_ATOMIC, ct->dd_len2);
18027c478bd9Sstevel@tonic-gate
18037c478bd9Sstevel@tonic-gate if (pd == NULL) {
18047c478bd9Sstevel@tonic-gate error = CRYPTO_MECH_NOT_SUPPORTED;
18057c478bd9Sstevel@tonic-gate goto out;
18067c478bd9Sstevel@tonic-gate }
18077c478bd9Sstevel@tonic-gate /* Validate the MAC context template here */
18087c478bd9Sstevel@tonic-gate if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) &&
18097c478bd9Sstevel@tonic-gate (mac_tmpl != NULL)) {
18107c478bd9Sstevel@tonic-gate kcf_ctx_template_t *ctx_mac_tmpl;
18117c478bd9Sstevel@tonic-gate
18127c478bd9Sstevel@tonic-gate ctx_mac_tmpl = (kcf_ctx_template_t *)mac_tmpl;
18137c478bd9Sstevel@tonic-gate
18147c478bd9Sstevel@tonic-gate if (ctx_mac_tmpl->ct_generation != me->me_gen_swprov) {
18157c478bd9Sstevel@tonic-gate KCF_PROV_REFRELE(pd);
18167c478bd9Sstevel@tonic-gate error = CRYPTO_OLD_CTX_TEMPLATE;
18177c478bd9Sstevel@tonic-gate goto out;
18187c478bd9Sstevel@tonic-gate }
18197c478bd9Sstevel@tonic-gate mops->mo_templ = ctx_mac_tmpl->ct_prov_tmpl;
18207c478bd9Sstevel@tonic-gate }
18217c478bd9Sstevel@tonic-gate
18227c478bd9Sstevel@tonic-gate break;
18237c478bd9Sstevel@tonic-gate }
18247c478bd9Sstevel@tonic-gate case KCF_OG_DECRYPT: {
18257c478bd9Sstevel@tonic-gate kcf_decrypt_ops_params_t *dcrops =
18267c478bd9Sstevel@tonic-gate &(params->rp_u.decrypt_params);
18277c478bd9Sstevel@tonic-gate
18287c478bd9Sstevel@tonic-gate ct = (crypto_dual_data_t *)dcrops->dop_ciphertext;
18297c478bd9Sstevel@tonic-gate /* No expected recoverable failures, so no retry list */
18307c478bd9Sstevel@tonic-gate pd = kcf_get_mech_provider(dcrops->dop_framework_mechtype,
1831436935a1SVladimir Kotal NULL, NULL, &error, NULL, CRYPTO_FG_DECRYPT_ATOMIC,
18329b009fc1SValerie Bubb Fenwick ct->dd_len1);
18337c478bd9Sstevel@tonic-gate
18347c478bd9Sstevel@tonic-gate if (pd == NULL) {
18357c478bd9Sstevel@tonic-gate error = CRYPTO_MECH_NOT_SUPPORTED;
18367c478bd9Sstevel@tonic-gate goto out;
18377c478bd9Sstevel@tonic-gate }
18387c478bd9Sstevel@tonic-gate break;
18397c478bd9Sstevel@tonic-gate }
18407c478bd9Sstevel@tonic-gate }
18417c478bd9Sstevel@tonic-gate
18427c478bd9Sstevel@tonic-gate /* The second step uses len2 and offset2 of the dual_data */
18437c478bd9Sstevel@tonic-gate next_req->kr_saveoffset = ct->dd_offset1;
18447c478bd9Sstevel@tonic-gate next_req->kr_savelen = ct->dd_len1;
18457c478bd9Sstevel@tonic-gate ct->dd_offset1 = ct->dd_offset2;
18467c478bd9Sstevel@tonic-gate ct->dd_len1 = ct->dd_len2;
18477c478bd9Sstevel@tonic-gate
18489b009fc1SValerie Bubb Fenwick areq->an_reqarg.cr_flag = 0;
18497c478bd9Sstevel@tonic-gate
18507c478bd9Sstevel@tonic-gate areq->an_reqarg.cr_callback_func = kcf_last_req;
18517c478bd9Sstevel@tonic-gate areq->an_reqarg.cr_callback_arg = next_req;
18527c478bd9Sstevel@tonic-gate areq->an_isdual = B_TRUE;
18537c478bd9Sstevel@tonic-gate
18547c478bd9Sstevel@tonic-gate /*
18557c478bd9Sstevel@tonic-gate * We would like to call kcf_submit_request() here. But,
18567c478bd9Sstevel@tonic-gate * that is not possible as that routine allocates a new
18577c478bd9Sstevel@tonic-gate * kcf_areq_node_t request structure, while we need to
18587c478bd9Sstevel@tonic-gate * reuse the existing request structure.
18597c478bd9Sstevel@tonic-gate */
18607c478bd9Sstevel@tonic-gate switch (pd->pd_prov_type) {
18617c478bd9Sstevel@tonic-gate case CRYPTO_SW_PROVIDER:
18627c478bd9Sstevel@tonic-gate error = common_submit_request(pd, NULL, params,
18637c478bd9Sstevel@tonic-gate KCF_RHNDL(KM_NOSLEEP));
18647c478bd9Sstevel@tonic-gate break;
18657c478bd9Sstevel@tonic-gate
18667c478bd9Sstevel@tonic-gate case CRYPTO_HW_PROVIDER: {
18677c478bd9Sstevel@tonic-gate kcf_provider_desc_t *old_pd;
1868ef56a3c5SKrishna Yenduri taskq_t *taskq = pd->pd_taskq;
18697c478bd9Sstevel@tonic-gate
18707c478bd9Sstevel@tonic-gate /*
18717c478bd9Sstevel@tonic-gate * Set the params for the second step in the
18727c478bd9Sstevel@tonic-gate * dual-ops.
18737c478bd9Sstevel@tonic-gate */
18747c478bd9Sstevel@tonic-gate areq->an_params = *params;
18757c478bd9Sstevel@tonic-gate old_pd = areq->an_provider;
18767c478bd9Sstevel@tonic-gate KCF_PROV_REFRELE(old_pd);
18777c478bd9Sstevel@tonic-gate KCF_PROV_REFHOLD(pd);
18787c478bd9Sstevel@tonic-gate areq->an_provider = pd;
18797c478bd9Sstevel@tonic-gate
18807c478bd9Sstevel@tonic-gate /*
18817c478bd9Sstevel@tonic-gate * Note that we have to do a taskq_dispatch()
18827c478bd9Sstevel@tonic-gate * here as we may be in interrupt context.
18837c478bd9Sstevel@tonic-gate */
18847c478bd9Sstevel@tonic-gate if (taskq_dispatch(taskq, process_req_hwp, areq,
1885*fc8ae2ecSToomas Soome TQ_NOSLEEP) == TASKQID_INVALID) {
18867c478bd9Sstevel@tonic-gate error = CRYPTO_HOST_MEMORY;
18877c478bd9Sstevel@tonic-gate } else {
18887c478bd9Sstevel@tonic-gate error = CRYPTO_QUEUED;
18897c478bd9Sstevel@tonic-gate }
18907c478bd9Sstevel@tonic-gate break;
18917c478bd9Sstevel@tonic-gate }
18927c478bd9Sstevel@tonic-gate }
18937c478bd9Sstevel@tonic-gate
18947c478bd9Sstevel@tonic-gate /*
18957c478bd9Sstevel@tonic-gate * We have to release the holds on the request and the provider
18967c478bd9Sstevel@tonic-gate * in all cases.
18977c478bd9Sstevel@tonic-gate */
18987c478bd9Sstevel@tonic-gate KCF_AREQ_REFRELE(areq);
18997c478bd9Sstevel@tonic-gate KCF_PROV_REFRELE(pd);
19007c478bd9Sstevel@tonic-gate
19017c478bd9Sstevel@tonic-gate if (error != CRYPTO_QUEUED) {
19027c478bd9Sstevel@tonic-gate /* restore, clean up, and invoke the client's callback */
19037c478bd9Sstevel@tonic-gate
19047c478bd9Sstevel@tonic-gate ct->dd_offset1 = next_req->kr_saveoffset;
19057c478bd9Sstevel@tonic-gate ct->dd_len1 = next_req->kr_savelen;
19067c478bd9Sstevel@tonic-gate areq->an_reqarg = next_req->kr_callreq;
19077c478bd9Sstevel@tonic-gate kmem_free(next_req, sizeof (kcf_dual_req_t));
19087c478bd9Sstevel@tonic-gate areq->an_isdual = B_FALSE;
19097c478bd9Sstevel@tonic-gate kcf_aop_done(areq, error);
19107c478bd9Sstevel@tonic-gate }
19117c478bd9Sstevel@tonic-gate }
19127c478bd9Sstevel@tonic-gate
19137c478bd9Sstevel@tonic-gate /*
19147c478bd9Sstevel@tonic-gate * Last part of an emulated dual operation.
19157c478bd9Sstevel@tonic-gate * Clean up and restore ...
19167c478bd9Sstevel@tonic-gate */
19177c478bd9Sstevel@tonic-gate void
kcf_last_req(void * last_req_arg,int status)19187c478bd9Sstevel@tonic-gate kcf_last_req(void *last_req_arg, int status)
19197c478bd9Sstevel@tonic-gate {
19207c478bd9Sstevel@tonic-gate kcf_dual_req_t *last_req = (kcf_dual_req_t *)last_req_arg;
19217c478bd9Sstevel@tonic-gate
19227c478bd9Sstevel@tonic-gate kcf_req_params_t *params = &(last_req->kr_params);
19237c478bd9Sstevel@tonic-gate kcf_areq_node_t *areq = last_req->kr_areq;
19247c478bd9Sstevel@tonic-gate crypto_dual_data_t *ct;
19257c478bd9Sstevel@tonic-gate
19267c478bd9Sstevel@tonic-gate switch (params->rp_opgrp) {
19277c478bd9Sstevel@tonic-gate case KCF_OG_MAC: {
19287c478bd9Sstevel@tonic-gate kcf_mac_ops_params_t *mops = &(params->rp_u.mac_params);
19297c478bd9Sstevel@tonic-gate
19307c478bd9Sstevel@tonic-gate ct = (crypto_dual_data_t *)mops->mo_data;
19317c478bd9Sstevel@tonic-gate break;
19327c478bd9Sstevel@tonic-gate }
19337c478bd9Sstevel@tonic-gate case KCF_OG_DECRYPT: {
19347c478bd9Sstevel@tonic-gate kcf_decrypt_ops_params_t *dcrops =
19357c478bd9Sstevel@tonic-gate &(params->rp_u.decrypt_params);
19367c478bd9Sstevel@tonic-gate
19377c478bd9Sstevel@tonic-gate ct = (crypto_dual_data_t *)dcrops->dop_ciphertext;
19387c478bd9Sstevel@tonic-gate break;
19397c478bd9Sstevel@tonic-gate }
19407c478bd9Sstevel@tonic-gate }
19417c478bd9Sstevel@tonic-gate ct->dd_offset1 = last_req->kr_saveoffset;
19427c478bd9Sstevel@tonic-gate ct->dd_len1 = last_req->kr_savelen;
19437c478bd9Sstevel@tonic-gate
19447c478bd9Sstevel@tonic-gate /* The submitter used kcf_last_req as its callback */
19457c478bd9Sstevel@tonic-gate
19467c478bd9Sstevel@tonic-gate if (areq == NULL) {
19477c478bd9Sstevel@tonic-gate crypto_call_req_t *cr = &last_req->kr_callreq;
19487c478bd9Sstevel@tonic-gate
19497c478bd9Sstevel@tonic-gate (*(cr->cr_callback_func))(cr->cr_callback_arg, status);
19507c478bd9Sstevel@tonic-gate kmem_free(last_req, sizeof (kcf_dual_req_t));
19517c478bd9Sstevel@tonic-gate return;
19527c478bd9Sstevel@tonic-gate }
19537c478bd9Sstevel@tonic-gate areq->an_reqarg = last_req->kr_callreq;
19547c478bd9Sstevel@tonic-gate KCF_AREQ_REFRELE(areq);
19557c478bd9Sstevel@tonic-gate kmem_free(last_req, sizeof (kcf_dual_req_t));
19567c478bd9Sstevel@tonic-gate areq->an_isdual = B_FALSE;
19577c478bd9Sstevel@tonic-gate kcf_aop_done(areq, status);
19587c478bd9Sstevel@tonic-gate }
1959