17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 572eff6e2Smcpowers * Common Development and Distribution License (the "License"). 672eff6e2Smcpowers * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 229b009fc1SValerie Bubb Fenwick * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 237c478bd9Sstevel@tonic-gate */ 247c478bd9Sstevel@tonic-gate 256ea3c060SGarrett D'Amore /* 266ea3c060SGarrett D'Amore * Copyright 2010 Nexenta Systems, Inc. All rights reserved. 276ea3c060SGarrett D'Amore */ 286ea3c060SGarrett D'Amore 297c478bd9Sstevel@tonic-gate #ifndef _SYS_CRYPTO_SCHED_IMPL_H 307c478bd9Sstevel@tonic-gate #define _SYS_CRYPTO_SCHED_IMPL_H 317c478bd9Sstevel@tonic-gate 327c478bd9Sstevel@tonic-gate /* 337c478bd9Sstevel@tonic-gate * Scheduler internal structures. 347c478bd9Sstevel@tonic-gate */ 357c478bd9Sstevel@tonic-gate 367c478bd9Sstevel@tonic-gate #ifdef __cplusplus 377c478bd9Sstevel@tonic-gate extern "C" { 387c478bd9Sstevel@tonic-gate #endif 397c478bd9Sstevel@tonic-gate 407c478bd9Sstevel@tonic-gate #include <sys/types.h> 417c478bd9Sstevel@tonic-gate #include <sys/mutex.h> 427c478bd9Sstevel@tonic-gate #include <sys/condvar.h> 437c478bd9Sstevel@tonic-gate #include <sys/door.h> 447c478bd9Sstevel@tonic-gate #include <sys/crypto/api.h> 457c478bd9Sstevel@tonic-gate #include <sys/crypto/spi.h> 467c478bd9Sstevel@tonic-gate #include <sys/crypto/impl.h> 477c478bd9Sstevel@tonic-gate #include <sys/crypto/common.h> 487c478bd9Sstevel@tonic-gate #include <sys/crypto/ops_impl.h> 497c478bd9Sstevel@tonic-gate 507c478bd9Sstevel@tonic-gate typedef void (kcf_func_t)(void *, int); 517c478bd9Sstevel@tonic-gate 527c478bd9Sstevel@tonic-gate typedef enum kcf_req_status { 537c478bd9Sstevel@tonic-gate REQ_ALLOCATED = 1, 547c478bd9Sstevel@tonic-gate REQ_WAITING, /* At the framework level */ 557c478bd9Sstevel@tonic-gate REQ_INPROGRESS, /* At the provider level */ 567c478bd9Sstevel@tonic-gate REQ_DONE, 577c478bd9Sstevel@tonic-gate REQ_CANCELED 587c478bd9Sstevel@tonic-gate } kcf_req_status_t; 597c478bd9Sstevel@tonic-gate 607c478bd9Sstevel@tonic-gate typedef enum kcf_call_type { 617c478bd9Sstevel@tonic-gate CRYPTO_SYNCH = 1, 627c478bd9Sstevel@tonic-gate CRYPTO_ASYNCH 637c478bd9Sstevel@tonic-gate } kcf_call_type_t; 647c478bd9Sstevel@tonic-gate 657c478bd9Sstevel@tonic-gate #define CHECK_FASTPATH(crq, pd) ((crq) == NULL || \ 667c478bd9Sstevel@tonic-gate !((crq)->cr_flag & CRYPTO_ALWAYS_QUEUE)) && \ 677c478bd9Sstevel@tonic-gate (pd)->pd_prov_type == CRYPTO_SW_PROVIDER 687c478bd9Sstevel@tonic-gate 697c478bd9Sstevel@tonic-gate #define KCF_KMFLAG(crq) (((crq) == NULL) ? KM_SLEEP : KM_NOSLEEP) 707c478bd9Sstevel@tonic-gate 717c478bd9Sstevel@tonic-gate /* 727c478bd9Sstevel@tonic-gate * The framework keeps an internal handle to use in the adaptive 737c478bd9Sstevel@tonic-gate * asynchronous case. This is the case when a client has the 747c478bd9Sstevel@tonic-gate * CRYPTO_ALWAYS_QUEUE bit clear and a software provider is used for 757c478bd9Sstevel@tonic-gate * the request. The request is completed in the context of the calling 767c478bd9Sstevel@tonic-gate * thread and kernel memory must be allocated with KM_NOSLEEP. 777c478bd9Sstevel@tonic-gate * 787c478bd9Sstevel@tonic-gate * The framework passes a pointer to the handle in crypto_req_handle_t 797c478bd9Sstevel@tonic-gate * argument when it calls the SPI of the software provider. The macros 807c478bd9Sstevel@tonic-gate * KCF_RHNDL() and KCF_SWFP_RHNDL() are used to do this. 817c478bd9Sstevel@tonic-gate * 827c478bd9Sstevel@tonic-gate * When a provider asks the framework for kmflag value via 837c478bd9Sstevel@tonic-gate * crypto_kmflag(9S) we use REQHNDL2_KMFLAG() macro. 847c478bd9Sstevel@tonic-gate */ 857c478bd9Sstevel@tonic-gate extern ulong_t kcf_swprov_hndl; 867c478bd9Sstevel@tonic-gate #define KCF_RHNDL(kmflag) (((kmflag) == KM_SLEEP) ? NULL : &kcf_swprov_hndl) 877c478bd9Sstevel@tonic-gate #define KCF_SWFP_RHNDL(crq) (((crq) == NULL) ? NULL : &kcf_swprov_hndl) 887c478bd9Sstevel@tonic-gate #define REQHNDL2_KMFLAG(rhndl) \ 897c478bd9Sstevel@tonic-gate ((rhndl == &kcf_swprov_hndl) ? KM_NOSLEEP : KM_SLEEP) 907c478bd9Sstevel@tonic-gate 917c478bd9Sstevel@tonic-gate /* Internal call_req flags. They start after the public ones in api.h */ 927c478bd9Sstevel@tonic-gate 937c478bd9Sstevel@tonic-gate #define CRYPTO_SETDUAL 0x00001000 /* Set the 'cont' boolean before */ 947c478bd9Sstevel@tonic-gate /* submitting the request */ 957c478bd9Sstevel@tonic-gate #define KCF_ISDUALREQ(crq) \ 967c478bd9Sstevel@tonic-gate (((crq) == NULL) ? B_FALSE : (crq->cr_flag & CRYPTO_SETDUAL)) 977c478bd9Sstevel@tonic-gate 987c478bd9Sstevel@tonic-gate typedef struct kcf_prov_tried { 997c478bd9Sstevel@tonic-gate kcf_provider_desc_t *pt_pd; 1007c478bd9Sstevel@tonic-gate struct kcf_prov_tried *pt_next; 1017c478bd9Sstevel@tonic-gate } kcf_prov_tried_t; 1027c478bd9Sstevel@tonic-gate 103ef56a3c5SKrishna Yenduri /* Must be different from KM_SLEEP and KM_NOSLEEP */ 104ef56a3c5SKrishna Yenduri #define KCF_HOLD_PROV 0x1000 105ef56a3c5SKrishna Yenduri 1067c478bd9Sstevel@tonic-gate #define IS_FG_SUPPORTED(mdesc, fg) \ 1077c478bd9Sstevel@tonic-gate (((mdesc)->pm_mech_info.cm_func_group_mask & (fg)) != 0) 1087c478bd9Sstevel@tonic-gate 1097c478bd9Sstevel@tonic-gate #define IS_PROVIDER_TRIED(pd, tlist) \ 1107c478bd9Sstevel@tonic-gate (tlist != NULL && is_in_triedlist(pd, tlist)) 1117c478bd9Sstevel@tonic-gate 1127c478bd9Sstevel@tonic-gate #define IS_RECOVERABLE(error) \ 1137c478bd9Sstevel@tonic-gate (error == CRYPTO_BUFFER_TOO_BIG || \ 1147c478bd9Sstevel@tonic-gate error == CRYPTO_BUSY || \ 1157c478bd9Sstevel@tonic-gate error == CRYPTO_DEVICE_ERROR || \ 1167c478bd9Sstevel@tonic-gate error == CRYPTO_DEVICE_MEMORY || \ 1177c478bd9Sstevel@tonic-gate error == CRYPTO_KEY_SIZE_RANGE || \ 1187c478bd9Sstevel@tonic-gate error == CRYPTO_NO_PERMISSION) 1197c478bd9Sstevel@tonic-gate 120*1a5e258fSJosef 'Jeff' Sipek #define KCF_ATOMIC_INCR(x) atomic_inc_32(&(x)) 121*1a5e258fSJosef 'Jeff' Sipek #define KCF_ATOMIC_DECR(x) atomic_dec_32(&(x)) 1227c478bd9Sstevel@tonic-gate 1237c478bd9Sstevel@tonic-gate /* 1247c478bd9Sstevel@tonic-gate * Node structure for synchronous requests. 1257c478bd9Sstevel@tonic-gate */ 1267c478bd9Sstevel@tonic-gate typedef struct kcf_sreq_node { 1277c478bd9Sstevel@tonic-gate /* Should always be the first field in this structure */ 1287c478bd9Sstevel@tonic-gate kcf_call_type_t sn_type; 1297c478bd9Sstevel@tonic-gate /* 1307c478bd9Sstevel@tonic-gate * sn_cv and sr_lock are used to wait for the 1317c478bd9Sstevel@tonic-gate * operation to complete. sn_lock also protects 1327c478bd9Sstevel@tonic-gate * the sn_state field. 1337c478bd9Sstevel@tonic-gate */ 1347c478bd9Sstevel@tonic-gate kcondvar_t sn_cv; 1357c478bd9Sstevel@tonic-gate kmutex_t sn_lock; 1367c478bd9Sstevel@tonic-gate kcf_req_status_t sn_state; 1377c478bd9Sstevel@tonic-gate 1387c478bd9Sstevel@tonic-gate /* 1397c478bd9Sstevel@tonic-gate * Return value from the operation. This will be 1407c478bd9Sstevel@tonic-gate * one of the CRYPTO_* errors defined in common.h. 1417c478bd9Sstevel@tonic-gate */ 1427c478bd9Sstevel@tonic-gate int sn_rv; 1437c478bd9Sstevel@tonic-gate 1447c478bd9Sstevel@tonic-gate /* 1457c478bd9Sstevel@tonic-gate * parameters to call the SPI with. This can be 1467c478bd9Sstevel@tonic-gate * a pointer as we know the caller context/stack stays. 1477c478bd9Sstevel@tonic-gate */ 1487c478bd9Sstevel@tonic-gate struct kcf_req_params *sn_params; 1497c478bd9Sstevel@tonic-gate 1507c478bd9Sstevel@tonic-gate /* Internal context for this request */ 1517c478bd9Sstevel@tonic-gate struct kcf_context *sn_context; 1527c478bd9Sstevel@tonic-gate 1537c478bd9Sstevel@tonic-gate /* Provider handling this request */ 1547c478bd9Sstevel@tonic-gate kcf_provider_desc_t *sn_provider; 155ef56a3c5SKrishna Yenduri 156ef56a3c5SKrishna Yenduri kcf_prov_cpu_t *sn_mp; 1577c478bd9Sstevel@tonic-gate } kcf_sreq_node_t; 1587c478bd9Sstevel@tonic-gate 1597c478bd9Sstevel@tonic-gate /* 1607c478bd9Sstevel@tonic-gate * Node structure for asynchronous requests. A node can be on 1617c478bd9Sstevel@tonic-gate * on a chain of requests hanging of the internal context 1627c478bd9Sstevel@tonic-gate * structure and can be in the global software provider queue. 1637c478bd9Sstevel@tonic-gate */ 1647c478bd9Sstevel@tonic-gate typedef struct kcf_areq_node { 1657c478bd9Sstevel@tonic-gate /* Should always be the first field in this structure */ 1667c478bd9Sstevel@tonic-gate kcf_call_type_t an_type; 1677c478bd9Sstevel@tonic-gate 1687c478bd9Sstevel@tonic-gate /* an_lock protects the field an_state */ 1697c478bd9Sstevel@tonic-gate kmutex_t an_lock; 1707c478bd9Sstevel@tonic-gate kcf_req_status_t an_state; 1717c478bd9Sstevel@tonic-gate crypto_call_req_t an_reqarg; 1727c478bd9Sstevel@tonic-gate 1737c478bd9Sstevel@tonic-gate /* 1747c478bd9Sstevel@tonic-gate * parameters to call the SPI with. We need to 1757c478bd9Sstevel@tonic-gate * save the params since the caller stack can go away. 1767c478bd9Sstevel@tonic-gate */ 1777c478bd9Sstevel@tonic-gate struct kcf_req_params an_params; 1787c478bd9Sstevel@tonic-gate 1797c478bd9Sstevel@tonic-gate /* 1807c478bd9Sstevel@tonic-gate * The next two fields should be NULL for operations that 1817c478bd9Sstevel@tonic-gate * don't need a context. 1827c478bd9Sstevel@tonic-gate */ 1837c478bd9Sstevel@tonic-gate /* Internal context for this request */ 1847c478bd9Sstevel@tonic-gate struct kcf_context *an_context; 1857c478bd9Sstevel@tonic-gate 1867c478bd9Sstevel@tonic-gate /* next in chain of requests for context */ 1877c478bd9Sstevel@tonic-gate struct kcf_areq_node *an_ctxchain_next; 1887c478bd9Sstevel@tonic-gate 189c41e7ccaSkrishna kcondvar_t an_turn_cv; 1907c478bd9Sstevel@tonic-gate boolean_t an_is_my_turn; 1917c478bd9Sstevel@tonic-gate boolean_t an_isdual; /* for internal reuse */ 1927c478bd9Sstevel@tonic-gate 1937c478bd9Sstevel@tonic-gate /* 1947c478bd9Sstevel@tonic-gate * Next and previous nodes in the global software 1957c478bd9Sstevel@tonic-gate * queue. These fields are NULL for a hardware 1967c478bd9Sstevel@tonic-gate * provider since we use a taskq there. 1977c478bd9Sstevel@tonic-gate */ 1987c478bd9Sstevel@tonic-gate struct kcf_areq_node *an_next; 1997c478bd9Sstevel@tonic-gate struct kcf_areq_node *an_prev; 2007c478bd9Sstevel@tonic-gate 2017c478bd9Sstevel@tonic-gate /* Provider handling this request */ 2027c478bd9Sstevel@tonic-gate kcf_provider_desc_t *an_provider; 203ef56a3c5SKrishna Yenduri kcf_prov_cpu_t *an_mp; 2047c478bd9Sstevel@tonic-gate kcf_prov_tried_t *an_tried_plist; 2057c478bd9Sstevel@tonic-gate 2067c478bd9Sstevel@tonic-gate struct kcf_areq_node *an_idnext; /* Next in ID hash */ 2077c478bd9Sstevel@tonic-gate struct kcf_areq_node *an_idprev; /* Prev in ID hash */ 2087c478bd9Sstevel@tonic-gate kcondvar_t an_done; /* Signal request completion */ 2097c478bd9Sstevel@tonic-gate uint_t an_refcnt; 2107c478bd9Sstevel@tonic-gate } kcf_areq_node_t; 2117c478bd9Sstevel@tonic-gate 2127c478bd9Sstevel@tonic-gate #define KCF_AREQ_REFHOLD(areq) { \ 213*1a5e258fSJosef 'Jeff' Sipek atomic_inc_32(&(areq)->an_refcnt); \ 2147c478bd9Sstevel@tonic-gate ASSERT((areq)->an_refcnt != 0); \ 2157c478bd9Sstevel@tonic-gate } 2167c478bd9Sstevel@tonic-gate 2177c478bd9Sstevel@tonic-gate #define KCF_AREQ_REFRELE(areq) { \ 2187c478bd9Sstevel@tonic-gate ASSERT((areq)->an_refcnt != 0); \ 2197c478bd9Sstevel@tonic-gate membar_exit(); \ 220*1a5e258fSJosef 'Jeff' Sipek if (atomic_dec_32_nv(&(areq)->an_refcnt) == 0) \ 2217c478bd9Sstevel@tonic-gate kcf_free_req(areq); \ 2227c478bd9Sstevel@tonic-gate } 2237c478bd9Sstevel@tonic-gate 2247c478bd9Sstevel@tonic-gate #define GET_REQ_TYPE(arg) *((kcf_call_type_t *)(arg)) 2257c478bd9Sstevel@tonic-gate 2267c478bd9Sstevel@tonic-gate #define NOTIFY_CLIENT(areq, err) (*(areq)->an_reqarg.cr_callback_func)(\ 2277c478bd9Sstevel@tonic-gate (areq)->an_reqarg.cr_callback_arg, err); 2287c478bd9Sstevel@tonic-gate 2297c478bd9Sstevel@tonic-gate /* For internally generated call requests for dual operations */ 2307c478bd9Sstevel@tonic-gate typedef struct kcf_call_req { 2317c478bd9Sstevel@tonic-gate crypto_call_req_t kr_callreq; /* external client call req */ 2327c478bd9Sstevel@tonic-gate kcf_req_params_t kr_params; /* Params saved for next call */ 2337c478bd9Sstevel@tonic-gate kcf_areq_node_t *kr_areq; /* Use this areq */ 2347c478bd9Sstevel@tonic-gate off_t kr_saveoffset; 2357c478bd9Sstevel@tonic-gate size_t kr_savelen; 2367c478bd9Sstevel@tonic-gate } kcf_dual_req_t; 2377c478bd9Sstevel@tonic-gate 2387c478bd9Sstevel@tonic-gate /* 2397c478bd9Sstevel@tonic-gate * The following are some what similar to macros in callo.h, which implement 2407c478bd9Sstevel@tonic-gate * callout tables. 2417c478bd9Sstevel@tonic-gate * 2427c478bd9Sstevel@tonic-gate * The lower four bits of the ID are used to encode the table ID to 2437c478bd9Sstevel@tonic-gate * index in to. The REQID_COUNTER_HIGH bit is used to avoid any check for 2447c478bd9Sstevel@tonic-gate * wrap around when generating ID. We assume that there won't be a request 2457c478bd9Sstevel@tonic-gate * which takes more time than 2^^(sizeof (long) - 5) other requests submitted 2467c478bd9Sstevel@tonic-gate * after it. This ensures there won't be any ID collision. 2477c478bd9Sstevel@tonic-gate */ 2487c478bd9Sstevel@tonic-gate #define REQID_COUNTER_HIGH (1UL << (8 * sizeof (long) - 1)) 2497c478bd9Sstevel@tonic-gate #define REQID_COUNTER_SHIFT 4 2507c478bd9Sstevel@tonic-gate #define REQID_COUNTER_LOW (1 << REQID_COUNTER_SHIFT) 2517c478bd9Sstevel@tonic-gate #define REQID_TABLES 16 2527c478bd9Sstevel@tonic-gate #define REQID_TABLE_MASK (REQID_TABLES - 1) 2537c478bd9Sstevel@tonic-gate 2547c478bd9Sstevel@tonic-gate #define REQID_BUCKETS 512 2557c478bd9Sstevel@tonic-gate #define REQID_BUCKET_MASK (REQID_BUCKETS - 1) 2567c478bd9Sstevel@tonic-gate #define REQID_HASH(id) (((id) >> REQID_COUNTER_SHIFT) & REQID_BUCKET_MASK) 2577c478bd9Sstevel@tonic-gate 2587c478bd9Sstevel@tonic-gate #define GET_REQID(areq) (areq)->an_reqarg.cr_reqid 2597c478bd9Sstevel@tonic-gate #define SET_REQID(areq, val) GET_REQID(areq) = val 2607c478bd9Sstevel@tonic-gate 2617c478bd9Sstevel@tonic-gate /* 2627c478bd9Sstevel@tonic-gate * Hash table for async requests. 2637c478bd9Sstevel@tonic-gate */ 2647c478bd9Sstevel@tonic-gate typedef struct kcf_reqid_table { 2657c478bd9Sstevel@tonic-gate kmutex_t rt_lock; 2667c478bd9Sstevel@tonic-gate crypto_req_id_t rt_curid; 2677c478bd9Sstevel@tonic-gate kcf_areq_node_t *rt_idhash[REQID_BUCKETS]; 2687c478bd9Sstevel@tonic-gate } kcf_reqid_table_t; 2697c478bd9Sstevel@tonic-gate 2707c478bd9Sstevel@tonic-gate /* 2717c478bd9Sstevel@tonic-gate * Global software provider queue structure. Requests to be 2727c478bd9Sstevel@tonic-gate * handled by a SW provider and have the ALWAYS_QUEUE flag set 2737c478bd9Sstevel@tonic-gate * get queued here. 2747c478bd9Sstevel@tonic-gate */ 2757c478bd9Sstevel@tonic-gate typedef struct kcf_global_swq { 2767c478bd9Sstevel@tonic-gate /* 2777c478bd9Sstevel@tonic-gate * gs_cv and gs_lock are used to wait for new requests. 2787c478bd9Sstevel@tonic-gate * gs_lock protects the changes to the queue. 2797c478bd9Sstevel@tonic-gate */ 2807c478bd9Sstevel@tonic-gate kcondvar_t gs_cv; 2817c478bd9Sstevel@tonic-gate kmutex_t gs_lock; 2827c478bd9Sstevel@tonic-gate uint_t gs_njobs; 2837c478bd9Sstevel@tonic-gate uint_t gs_maxjobs; 2847c478bd9Sstevel@tonic-gate kcf_areq_node_t *gs_first; 2857c478bd9Sstevel@tonic-gate kcf_areq_node_t *gs_last; 2867c478bd9Sstevel@tonic-gate } kcf_global_swq_t; 2877c478bd9Sstevel@tonic-gate 2887c478bd9Sstevel@tonic-gate 2897c478bd9Sstevel@tonic-gate /* 2907c478bd9Sstevel@tonic-gate * Internal representation of a canonical context. We contain crypto_ctx_t 2917c478bd9Sstevel@tonic-gate * structure in order to have just one memory allocation. The SPI 2927c478bd9Sstevel@tonic-gate * ((crypto_ctx_t *)ctx)->cc_framework_private maps to this structure. 2937c478bd9Sstevel@tonic-gate */ 2947c478bd9Sstevel@tonic-gate typedef struct kcf_context { 2957c478bd9Sstevel@tonic-gate crypto_ctx_t kc_glbl_ctx; 2967c478bd9Sstevel@tonic-gate uint_t kc_refcnt; 2977c478bd9Sstevel@tonic-gate kmutex_t kc_in_use_lock; 2987c478bd9Sstevel@tonic-gate /* 2997c478bd9Sstevel@tonic-gate * kc_req_chain_first and kc_req_chain_last are used to chain 3007c478bd9Sstevel@tonic-gate * multiple async requests using the same context. They should be 3017c478bd9Sstevel@tonic-gate * NULL for sync requests. 3027c478bd9Sstevel@tonic-gate */ 3037c478bd9Sstevel@tonic-gate kcf_areq_node_t *kc_req_chain_first; 3047c478bd9Sstevel@tonic-gate kcf_areq_node_t *kc_req_chain_last; 3057c478bd9Sstevel@tonic-gate kcf_provider_desc_t *kc_prov_desc; /* Prov. descriptor */ 3066a1073f8Skrishna kcf_provider_desc_t *kc_sw_prov_desc; /* Prov. descriptor */ 3076a1073f8Skrishna kcf_mech_entry_t *kc_mech; 3087c478bd9Sstevel@tonic-gate struct kcf_context *kc_secondctx; /* for dual contexts */ 3097c478bd9Sstevel@tonic-gate } kcf_context_t; 3107c478bd9Sstevel@tonic-gate 3117c478bd9Sstevel@tonic-gate /* 3127c478bd9Sstevel@tonic-gate * Bump up the reference count on the framework private context. A 3137c478bd9Sstevel@tonic-gate * global context or a request that references this structure should 3147c478bd9Sstevel@tonic-gate * do a hold. 3157c478bd9Sstevel@tonic-gate */ 3167c478bd9Sstevel@tonic-gate #define KCF_CONTEXT_REFHOLD(ictx) { \ 317*1a5e258fSJosef 'Jeff' Sipek atomic_inc_32(&(ictx)->kc_refcnt); \ 3187c478bd9Sstevel@tonic-gate ASSERT((ictx)->kc_refcnt != 0); \ 3197c478bd9Sstevel@tonic-gate } 3207c478bd9Sstevel@tonic-gate 3217c478bd9Sstevel@tonic-gate /* 3227c478bd9Sstevel@tonic-gate * Decrement the reference count on the framework private context. 3237c478bd9Sstevel@tonic-gate * When the last reference is released, the framework private 3247c478bd9Sstevel@tonic-gate * context structure is freed along with the global context. 3257c478bd9Sstevel@tonic-gate */ 3267c478bd9Sstevel@tonic-gate #define KCF_CONTEXT_REFRELE(ictx) { \ 3277c478bd9Sstevel@tonic-gate ASSERT((ictx)->kc_refcnt != 0); \ 3287c478bd9Sstevel@tonic-gate membar_exit(); \ 329*1a5e258fSJosef 'Jeff' Sipek if (atomic_dec_32_nv(&(ictx)->kc_refcnt) == 0) \ 3307c478bd9Sstevel@tonic-gate kcf_free_context(ictx); \ 3317c478bd9Sstevel@tonic-gate } 3327c478bd9Sstevel@tonic-gate 3337c478bd9Sstevel@tonic-gate /* 3347c478bd9Sstevel@tonic-gate * Check if we can release the context now. In case of CRYPTO_QUEUED 3357c478bd9Sstevel@tonic-gate * we do not release it as we can do it only after the provider notified 3367c478bd9Sstevel@tonic-gate * us. In case of CRYPTO_BUSY, the client can retry the request using 3377c478bd9Sstevel@tonic-gate * the context, so we do not release the context. 3387c478bd9Sstevel@tonic-gate * 3397c478bd9Sstevel@tonic-gate * This macro should be called only from the final routine in 3407c478bd9Sstevel@tonic-gate * an init/update/final sequence. We do not release the context in case 3417c478bd9Sstevel@tonic-gate * of update operations. We require the consumer to free it 3427c478bd9Sstevel@tonic-gate * explicitly, in case it wants to abandon the operation. This is done 3437c478bd9Sstevel@tonic-gate * as there may be mechanisms in ECB mode that can continue even if 3447c478bd9Sstevel@tonic-gate * an operation on a block fails. 3457c478bd9Sstevel@tonic-gate */ 3467c478bd9Sstevel@tonic-gate #define KCF_CONTEXT_COND_RELEASE(rv, kcf_ctx) { \ 3477c478bd9Sstevel@tonic-gate if (KCF_CONTEXT_DONE(rv)) \ 3487c478bd9Sstevel@tonic-gate KCF_CONTEXT_REFRELE(kcf_ctx); \ 3497c478bd9Sstevel@tonic-gate } 3507c478bd9Sstevel@tonic-gate 3517c478bd9Sstevel@tonic-gate /* 3527c478bd9Sstevel@tonic-gate * This macro determines whether we're done with a context. 3537c478bd9Sstevel@tonic-gate */ 3547c478bd9Sstevel@tonic-gate #define KCF_CONTEXT_DONE(rv) \ 3557c478bd9Sstevel@tonic-gate ((rv) != CRYPTO_QUEUED && (rv) != CRYPTO_BUSY && \ 3567c478bd9Sstevel@tonic-gate (rv) != CRYPTO_BUFFER_TOO_SMALL) 3577c478bd9Sstevel@tonic-gate 3587c478bd9Sstevel@tonic-gate /* 3597c478bd9Sstevel@tonic-gate * A crypto_ctx_template_t is internally a pointer to this struct 3607c478bd9Sstevel@tonic-gate */ 3617c478bd9Sstevel@tonic-gate typedef struct kcf_ctx_template { 3627c478bd9Sstevel@tonic-gate crypto_kcf_provider_handle_t ct_prov_handle; /* provider handle */ 3637c478bd9Sstevel@tonic-gate uint_t ct_generation; /* generation # */ 3647c478bd9Sstevel@tonic-gate size_t ct_size; /* for freeing */ 3657c478bd9Sstevel@tonic-gate crypto_spi_ctx_template_t ct_prov_tmpl; /* context template */ 3667c478bd9Sstevel@tonic-gate /* from the SW prov */ 3677c478bd9Sstevel@tonic-gate } kcf_ctx_template_t; 3687c478bd9Sstevel@tonic-gate 3697c478bd9Sstevel@tonic-gate /* 3707c478bd9Sstevel@tonic-gate * Structure for pool of threads working on global software queue. 3717c478bd9Sstevel@tonic-gate */ 3727c478bd9Sstevel@tonic-gate typedef struct kcf_pool { 3737c478bd9Sstevel@tonic-gate uint32_t kp_threads; /* Number of threads in pool */ 3747c478bd9Sstevel@tonic-gate uint32_t kp_idlethreads; /* Idle threads in pool */ 3757c478bd9Sstevel@tonic-gate uint32_t kp_blockedthreads; /* Blocked threads in pool */ 3767c478bd9Sstevel@tonic-gate 3777c478bd9Sstevel@tonic-gate /* 3786ea3c060SGarrett D'Amore * cv & lock for the condition where more threads need to be created. 3797c478bd9Sstevel@tonic-gate */ 3806ea3c060SGarrett D'Amore kcondvar_t kp_cv; /* Creator cond. variable */ 3816ea3c060SGarrett D'Amore kmutex_t kp_lock; /* Creator lock */ 3827c478bd9Sstevel@tonic-gate 3837c478bd9Sstevel@tonic-gate } kcf_pool_t; 3847c478bd9Sstevel@tonic-gate 3857c478bd9Sstevel@tonic-gate 3867c478bd9Sstevel@tonic-gate /* 3877c478bd9Sstevel@tonic-gate * State of a crypto bufcall element. 3887c478bd9Sstevel@tonic-gate */ 3897c478bd9Sstevel@tonic-gate typedef enum cbuf_state { 3907c478bd9Sstevel@tonic-gate CBUF_FREE = 1, 3917c478bd9Sstevel@tonic-gate CBUF_WAITING, 3927c478bd9Sstevel@tonic-gate CBUF_RUNNING 3937c478bd9Sstevel@tonic-gate } cbuf_state_t; 3947c478bd9Sstevel@tonic-gate 3957c478bd9Sstevel@tonic-gate /* 3967c478bd9Sstevel@tonic-gate * Structure of a crypto bufcall element. 3977c478bd9Sstevel@tonic-gate */ 3987c478bd9Sstevel@tonic-gate typedef struct kcf_cbuf_elem { 3997c478bd9Sstevel@tonic-gate /* 4007c478bd9Sstevel@tonic-gate * lock and cv to wait for CBUF_RUNNING to be done 4017c478bd9Sstevel@tonic-gate * kc_lock also protects kc_state. 4027c478bd9Sstevel@tonic-gate */ 4037c478bd9Sstevel@tonic-gate kmutex_t kc_lock; 4047c478bd9Sstevel@tonic-gate kcondvar_t kc_cv; 4057c478bd9Sstevel@tonic-gate cbuf_state_t kc_state; 4067c478bd9Sstevel@tonic-gate 4077c478bd9Sstevel@tonic-gate struct kcf_cbuf_elem *kc_next; 4087c478bd9Sstevel@tonic-gate struct kcf_cbuf_elem *kc_prev; 4097c478bd9Sstevel@tonic-gate 4107c478bd9Sstevel@tonic-gate void (*kc_func)(void *arg); 4117c478bd9Sstevel@tonic-gate void *kc_arg; 4127c478bd9Sstevel@tonic-gate } kcf_cbuf_elem_t; 4137c478bd9Sstevel@tonic-gate 4147c478bd9Sstevel@tonic-gate /* 4157c478bd9Sstevel@tonic-gate * State of a notify element. 4167c478bd9Sstevel@tonic-gate */ 4177c478bd9Sstevel@tonic-gate typedef enum ntfy_elem_state { 4187c478bd9Sstevel@tonic-gate NTFY_WAITING = 1, 4197c478bd9Sstevel@tonic-gate NTFY_RUNNING 4207c478bd9Sstevel@tonic-gate } ntfy_elem_state_t; 4217c478bd9Sstevel@tonic-gate 4227c478bd9Sstevel@tonic-gate /* 4237c478bd9Sstevel@tonic-gate * Structure of a notify list element. 4247c478bd9Sstevel@tonic-gate */ 4257c478bd9Sstevel@tonic-gate typedef struct kcf_ntfy_elem { 4267c478bd9Sstevel@tonic-gate /* 4277c478bd9Sstevel@tonic-gate * lock and cv to wait for NTFY_RUNNING to be done. 4287c478bd9Sstevel@tonic-gate * kn_lock also protects kn_state. 4297c478bd9Sstevel@tonic-gate */ 4307c478bd9Sstevel@tonic-gate kmutex_t kn_lock; 4317c478bd9Sstevel@tonic-gate kcondvar_t kn_cv; 4327c478bd9Sstevel@tonic-gate ntfy_elem_state_t kn_state; 4337c478bd9Sstevel@tonic-gate 4347c478bd9Sstevel@tonic-gate struct kcf_ntfy_elem *kn_next; 4357c478bd9Sstevel@tonic-gate struct kcf_ntfy_elem *kn_prev; 4367c478bd9Sstevel@tonic-gate 4377c478bd9Sstevel@tonic-gate crypto_notify_callback_t kn_func; 4387c478bd9Sstevel@tonic-gate uint32_t kn_event_mask; 4397c478bd9Sstevel@tonic-gate } kcf_ntfy_elem_t; 4407c478bd9Sstevel@tonic-gate 4417c478bd9Sstevel@tonic-gate 4427c478bd9Sstevel@tonic-gate /* 4437c478bd9Sstevel@tonic-gate * The following values are based on the assumption that it would 4447c478bd9Sstevel@tonic-gate * take around eight cpus to load a hardware provider (This is true for 4457c478bd9Sstevel@tonic-gate * at least one product) and a kernel client may come from different 4467c478bd9Sstevel@tonic-gate * low-priority interrupt levels. We will have CYRPTO_TASKQ_MIN number 447c41e7ccaSkrishna * of cached taskq entries. The CRYPTO_TASKQ_MAX number is based on 448c41e7ccaSkrishna * a throughput of 1GB/s using 512-byte buffers. These are just 449c41e7ccaSkrishna * reasonable estimates and might need to change in future. 4507c478bd9Sstevel@tonic-gate */ 451c41e7ccaSkrishna #define CRYPTO_TASKQ_THREADS 8 4527c478bd9Sstevel@tonic-gate #define CYRPTO_TASKQ_MIN 64 453c41e7ccaSkrishna #define CRYPTO_TASKQ_MAX 2 * 1024 * 1024 4547c478bd9Sstevel@tonic-gate 455c41e7ccaSkrishna extern int crypto_taskq_threads; 4567c478bd9Sstevel@tonic-gate extern int crypto_taskq_minalloc; 4577c478bd9Sstevel@tonic-gate extern int crypto_taskq_maxalloc; 4587c478bd9Sstevel@tonic-gate extern kcf_global_swq_t *gswq; 4597c478bd9Sstevel@tonic-gate extern int kcf_maxthreads; 4607c478bd9Sstevel@tonic-gate extern int kcf_minthreads; 4617c478bd9Sstevel@tonic-gate 4627c478bd9Sstevel@tonic-gate /* 4637c478bd9Sstevel@tonic-gate * All pending crypto bufcalls are put on a list. cbuf_list_lock 4647c478bd9Sstevel@tonic-gate * protects changes to this list. 4657c478bd9Sstevel@tonic-gate */ 4667c478bd9Sstevel@tonic-gate extern kmutex_t cbuf_list_lock; 4677c478bd9Sstevel@tonic-gate extern kcondvar_t cbuf_list_cv; 4687c478bd9Sstevel@tonic-gate 4697c478bd9Sstevel@tonic-gate /* 4707c478bd9Sstevel@tonic-gate * All event subscribers are put on a list. kcf_notify_list_lock 4717c478bd9Sstevel@tonic-gate * protects changes to this list. 4727c478bd9Sstevel@tonic-gate */ 4737c478bd9Sstevel@tonic-gate extern kmutex_t ntfy_list_lock; 4747c478bd9Sstevel@tonic-gate extern kcondvar_t ntfy_list_cv; 4757c478bd9Sstevel@tonic-gate 4767c478bd9Sstevel@tonic-gate boolean_t kcf_get_next_logical_provider_member(kcf_provider_desc_t *, 4777c478bd9Sstevel@tonic-gate kcf_provider_desc_t *, kcf_provider_desc_t **); 478436935a1SVladimir Kotal extern int kcf_get_hardware_provider(crypto_mech_type_t, crypto_key_t *, 479436935a1SVladimir Kotal crypto_mech_type_t, crypto_key_t *, 4809b009fc1SValerie Bubb Fenwick kcf_provider_desc_t *, kcf_provider_desc_t **, 48172eff6e2Smcpowers crypto_func_group_t); 4827c478bd9Sstevel@tonic-gate extern int kcf_get_hardware_provider_nomech(offset_t, offset_t, 4839b009fc1SValerie Bubb Fenwick kcf_provider_desc_t *, kcf_provider_desc_t **); 4847c478bd9Sstevel@tonic-gate extern void kcf_free_triedlist(kcf_prov_tried_t *); 4857c478bd9Sstevel@tonic-gate extern kcf_prov_tried_t *kcf_insert_triedlist(kcf_prov_tried_t **, 4867c478bd9Sstevel@tonic-gate kcf_provider_desc_t *, int); 4877c478bd9Sstevel@tonic-gate extern kcf_provider_desc_t *kcf_get_mech_provider(crypto_mech_type_t, 488436935a1SVladimir Kotal crypto_key_t *, kcf_mech_entry_t **, int *, kcf_prov_tried_t *, 4899b009fc1SValerie Bubb Fenwick crypto_func_group_t, size_t); 4907c478bd9Sstevel@tonic-gate extern kcf_provider_desc_t *kcf_get_dual_provider(crypto_mechanism_t *, 491436935a1SVladimir Kotal crypto_key_t *, crypto_mechanism_t *, crypto_key_t *, 492436935a1SVladimir Kotal kcf_mech_entry_t **, crypto_mech_type_t *, 4937c478bd9Sstevel@tonic-gate crypto_mech_type_t *, int *, kcf_prov_tried_t *, 4949b009fc1SValerie Bubb Fenwick crypto_func_group_t, crypto_func_group_t, size_t); 4957c478bd9Sstevel@tonic-gate extern crypto_ctx_t *kcf_new_ctx(crypto_call_req_t *, kcf_provider_desc_t *, 4967c478bd9Sstevel@tonic-gate crypto_session_id_t); 4977c478bd9Sstevel@tonic-gate extern int kcf_submit_request(kcf_provider_desc_t *, crypto_ctx_t *, 4987c478bd9Sstevel@tonic-gate crypto_call_req_t *, kcf_req_params_t *, boolean_t); 4997c478bd9Sstevel@tonic-gate extern void kcf_sched_init(void); 5007c478bd9Sstevel@tonic-gate extern void kcf_sched_start(void); 5017c478bd9Sstevel@tonic-gate extern void kcf_sop_done(kcf_sreq_node_t *, int); 5027c478bd9Sstevel@tonic-gate extern void kcf_aop_done(kcf_areq_node_t *, int); 5037c478bd9Sstevel@tonic-gate extern int common_submit_request(kcf_provider_desc_t *, 5047c478bd9Sstevel@tonic-gate crypto_ctx_t *, kcf_req_params_t *, crypto_req_handle_t); 5057c478bd9Sstevel@tonic-gate extern void kcf_free_context(kcf_context_t *); 5067c478bd9Sstevel@tonic-gate 5077c478bd9Sstevel@tonic-gate extern struct modctl *kcf_get_modctl(crypto_provider_info_t *); 5087c478bd9Sstevel@tonic-gate extern void kcf_free_req(kcf_areq_node_t *areq); 5097c478bd9Sstevel@tonic-gate extern void crypto_bufcall_service(void); 5107c478bd9Sstevel@tonic-gate 5117c478bd9Sstevel@tonic-gate extern void kcf_walk_ntfylist(uint32_t, void *); 512fe2f7468Skrishna extern void kcf_do_notify(kcf_provider_desc_t *, boolean_t); 5137c478bd9Sstevel@tonic-gate 5147c478bd9Sstevel@tonic-gate extern kcf_dual_req_t *kcf_alloc_req(crypto_call_req_t *); 5157c478bd9Sstevel@tonic-gate extern void kcf_next_req(void *, int); 5167c478bd9Sstevel@tonic-gate extern void kcf_last_req(void *, int); 5177c478bd9Sstevel@tonic-gate 5187c478bd9Sstevel@tonic-gate #ifdef __cplusplus 5197c478bd9Sstevel@tonic-gate } 5207c478bd9Sstevel@tonic-gate #endif 5217c478bd9Sstevel@tonic-gate 5227c478bd9Sstevel@tonic-gate #endif /* _SYS_CRYPTO_SCHED_IMPL_H */ 523