/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2012 Garrett D'Amore . All rights reserved. * Copyright 2012 Alexey Zaytsev All rights reserved. * Copyright 2017 The MathWorks, Inc. All rights reserved. * Copyright 2020 Joyent, Inc. * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. * Copyright 2022 Tintri by DDN, Inc. All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* for DTYPE_DIRECT */ #include #include #include #include #include #include #include #include #include #include #include #include /* * blkdev is a driver which provides a lot of the common functionality * a block device driver may need and helps by removing code which * is frequently duplicated in block device drivers. * * Within this driver all the struct cb_ops functions required for a * block device driver are written with appropriate call back functions * to be provided by the parent driver. * * To use blkdev, a driver needs to: * 1. Create a bd_ops_t structure which has the call back operations * blkdev will use. * 2. Create a handle by calling bd_alloc_handle(). One of the * arguments to this function is the bd_ops_t. * 3. Call bd_attach_handle(). This will instantiate a blkdev device * as a child device node of the calling driver. * * A parent driver is not restricted to just allocating and attaching a * single instance, it may attach as many as it wishes. For each handle * attached, appropriate entries in /dev/[r]dsk are created. * * The bd_ops_t routines that a parent of blkdev need to provide are: * * o_drive_info: Provide information to blkdev such as how many I/O queues * to create and the size of those queues. Also some device * specifics such as EUI, vendor, product, model, serial * number .... * * o_media_info: Provide information about the media. Eg size and block size. * * o_devid_init: Creates and initializes the device id. Typically calls * ddi_devid_init(). * * o_sync_cache: Issues a device appropriate command to flush any write * caches. * * o_read: Read data as described by bd_xfer_t argument. * * o_write: Write data as described by bd_xfer_t argument. * * o_free_space: Free the space described by bd_xfer_t argument (optional). * * Queues * ------ * Part of the drive_info data is a queue count. blkdev will create * "queue count" number of waitq/runq pairs. Each waitq/runq pair * operates independently. As an I/O is scheduled up to the parent * driver via o_read or o_write its queue number is given. If the * parent driver supports multiple hardware queues it can then select * where to submit the I/O request. * * Currently blkdev uses a simplistic round-robin queue selection method. * It has the advantage that it is lockless. In the future it will be * worthwhile reviewing this strategy for something which prioritizes queues * depending on how busy they are. * * Each waitq/runq pair is protected by its mutex (q_iomutex). Incoming * I/O requests are initially added to the waitq. They are taken off the * waitq, added to the runq and submitted, providing the runq is less * than the qsize as specified in the drive_info. As an I/O request * completes, the parent driver is required to call bd_xfer_done(), which * will remove the I/O request from the runq and pass I/O completion * status up the stack. * * Locks * ----- * There are 5 instance global locks d_ocmutex, d_ksmutex, d_errmutex, * d_statemutex and d_dle_mutex. As well a q_iomutex per waitq/runq pair. * * Lock Hierarchy * -------------- * The only two locks which may be held simultaneously are q_iomutex and * d_ksmutex. In all cases q_iomutex must be acquired before d_ksmutex. */ #define BD_MAXPART 64 #define BDINST(dev) (getminor(dev) / BD_MAXPART) #define BDPART(dev) (getminor(dev) % BD_MAXPART) typedef struct bd bd_t; typedef struct bd_xfer_impl bd_xfer_impl_t; typedef struct bd_queue bd_queue_t; typedef enum { BD_DLE_PENDING = 1 << 0, BD_DLE_RUNNING = 1 << 1 } bd_dle_state_t; struct bd { void *d_private; dev_info_t *d_dip; kmutex_t d_ocmutex; /* open/close */ kmutex_t d_ksmutex; /* kstat */ kmutex_t d_errmutex; kmutex_t d_statemutex; kcondvar_t d_statecv; enum dkio_state d_state; cmlb_handle_t d_cmlbh; unsigned d_open_lyr[BD_MAXPART]; /* open count */ uint64_t d_open_excl; /* bit mask indexed by partition */ uint64_t d_open_reg[OTYPCNT]; /* bit mask */ uint64_t d_io_counter; uint32_t d_qcount; uint32_t d_qactive; uint32_t d_maxxfer; uint32_t d_blkshift; uint32_t d_pblkshift; uint64_t d_numblks; ddi_devid_t d_devid; uint64_t d_max_free_seg; uint64_t d_max_free_blks; uint64_t d_max_free_seg_blks; uint64_t d_free_align; kmem_cache_t *d_cache; bd_queue_t *d_queues; kstat_t *d_ksp; kstat_io_t *d_kiop; kstat_t *d_errstats; struct bd_errstats *d_kerr; boolean_t d_rdonly; boolean_t d_ssd; boolean_t d_removable; boolean_t d_hotpluggable; boolean_t d_use_dma; ddi_dma_attr_t d_dma; bd_ops_t d_ops; bd_handle_t d_handle; kmutex_t d_dle_mutex; taskq_ent_t d_dle_ent; bd_dle_state_t d_dle_state; }; struct bd_handle { bd_ops_t h_ops; ddi_dma_attr_t *h_dma; dev_info_t *h_parent; dev_info_t *h_child; void *h_private; bd_t *h_bd; char *h_name; char h_addr[50]; /* enough for w%0.32x,%X */ }; struct bd_xfer_impl { bd_xfer_t i_public; list_node_t i_linkage; bd_t *i_bd; buf_t *i_bp; bd_queue_t *i_bq; uint_t i_num_win; uint_t i_cur_win; off_t i_offset; int (*i_func)(void *, bd_xfer_t *); uint32_t i_blkshift; size_t i_len; size_t i_resid; }; struct bd_queue { kmutex_t q_iomutex; uint32_t q_qsize; uint32_t q_qactive; list_t q_runq; list_t q_waitq; }; #define i_dmah i_public.x_dmah #define i_dmac i_public.x_dmac #define i_ndmac i_public.x_ndmac #define i_kaddr i_public.x_kaddr #define i_nblks i_public.x_nblks #define i_blkno i_public.x_blkno #define i_flags i_public.x_flags #define i_qnum i_public.x_qnum #define i_dfl i_public.x_dfl #define CAN_FREESPACE(bd) \ (((bd)->d_ops.o_free_space == NULL) ? B_FALSE : B_TRUE) /* * Private prototypes. */ static void bd_prop_update_inqstring(dev_info_t *, char *, char *, size_t); static void bd_create_inquiry_props(dev_info_t *, bd_drive_t *); static void bd_create_errstats(bd_t *, int, bd_drive_t *); static void bd_destroy_errstats(bd_t *); static void bd_errstats_setstr(kstat_named_t *, char *, size_t, char *); static void bd_init_errstats(bd_t *, bd_drive_t *); static void bd_fini_errstats(bd_t *); static int bd_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); static int bd_attach(dev_info_t *, ddi_attach_cmd_t); static int bd_detach(dev_info_t *, ddi_detach_cmd_t); static int bd_open(dev_t *, int, int, cred_t *); static int bd_close(dev_t, int, int, cred_t *); static int bd_strategy(struct buf *); static int bd_ioctl(dev_t, int, intptr_t, int, cred_t *, int *); static int bd_dump(dev_t, caddr_t, daddr_t, int); static int bd_read(dev_t, struct uio *, cred_t *); static int bd_write(dev_t, struct uio *, cred_t *); static int bd_aread(dev_t, struct aio_req *, cred_t *); static int bd_awrite(dev_t, struct aio_req *, cred_t *); static int bd_prop_op(dev_t, dev_info_t *, ddi_prop_op_t, int, char *, caddr_t, int *); static int bd_tg_rdwr(dev_info_t *, uchar_t, void *, diskaddr_t, size_t, void *); static int bd_tg_getinfo(dev_info_t *, int, void *, void *); static int bd_xfer_ctor(void *, void *, int); static void bd_xfer_dtor(void *, void *); static void bd_sched(bd_t *, bd_queue_t *); static void bd_submit(bd_t *, bd_xfer_impl_t *); static void bd_runq_exit(bd_xfer_impl_t *, int); static void bd_update_state(bd_t *); static int bd_check_state(bd_t *, enum dkio_state *); static int bd_flush_write_cache(bd_t *, struct dk_callback *); static int bd_check_uio(dev_t, struct uio *); static int bd_free_space(dev_t, bd_t *, dkioc_free_list_t *); struct cmlb_tg_ops bd_tg_ops = { TG_DK_OPS_VERSION_1, bd_tg_rdwr, bd_tg_getinfo, }; static struct cb_ops bd_cb_ops = { bd_open, /* open */ bd_close, /* close */ bd_strategy, /* strategy */ nodev, /* print */ bd_dump, /* dump */ bd_read, /* read */ bd_write, /* write */ bd_ioctl, /* ioctl */ nodev, /* devmap */ nodev, /* mmap */ nodev, /* segmap */ nochpoll, /* poll */ bd_prop_op, /* cb_prop_op */ 0, /* streamtab */ D_64BIT | D_MP, /* Driver comaptibility flag */ CB_REV, /* cb_rev */ bd_aread, /* async read */ bd_awrite /* async write */ }; struct dev_ops bd_dev_ops = { DEVO_REV, /* devo_rev, */ 0, /* refcnt */ bd_getinfo, /* getinfo */ nulldev, /* identify */ nulldev, /* probe */ bd_attach, /* attach */ bd_detach, /* detach */ nodev, /* reset */ &bd_cb_ops, /* driver operations */ NULL, /* bus operations */ NULL, /* power */ ddi_quiesce_not_needed, /* quiesce */ }; static struct modldrv modldrv = { &mod_driverops, "Generic Block Device", &bd_dev_ops, }; static struct modlinkage modlinkage = { MODREV_1, { &modldrv, NULL } }; static void *bd_state; static krwlock_t bd_lock; static taskq_t *bd_taskq; int _init(void) { char taskq_name[TASKQ_NAMELEN]; const char *name; int rv; rv = ddi_soft_state_init(&bd_state, sizeof (struct bd), 2); if (rv != DDI_SUCCESS) return (rv); name = mod_modname(&modlinkage); (void) snprintf(taskq_name, sizeof (taskq_name), "%s_taskq", name); bd_taskq = taskq_create(taskq_name, 1, minclsyspri, 0, 0, 0); if (bd_taskq == NULL) { cmn_err(CE_WARN, "%s: unable to create %s", name, taskq_name); ddi_soft_state_fini(&bd_state); return (DDI_FAILURE); } rw_init(&bd_lock, NULL, RW_DRIVER, NULL); rv = mod_install(&modlinkage); if (rv != DDI_SUCCESS) { rw_destroy(&bd_lock); taskq_destroy(bd_taskq); ddi_soft_state_fini(&bd_state); } return (rv); } int _fini(void) { int rv; rv = mod_remove(&modlinkage); if (rv == DDI_SUCCESS) { rw_destroy(&bd_lock); taskq_destroy(bd_taskq); ddi_soft_state_fini(&bd_state); } return (rv); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } static int bd_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **resultp) { bd_t *bd; minor_t inst; _NOTE(ARGUNUSED(dip)); inst = BDINST((dev_t)arg); switch (cmd) { case DDI_INFO_DEVT2DEVINFO: bd = ddi_get_soft_state(bd_state, inst); if (bd == NULL) { return (DDI_FAILURE); } *resultp = (void *)bd->d_dip; break; case DDI_INFO_DEVT2INSTANCE: *resultp = (void *)(intptr_t)inst; break; default: return (DDI_FAILURE); } return (DDI_SUCCESS); } static void bd_prop_update_inqstring(dev_info_t *dip, char *name, char *data, size_t len) { int ilen; char *data_string; ilen = scsi_ascii_inquiry_len(data, len); ASSERT3U(ilen, <=, len); if (ilen <= 0) return; /* ensure null termination */ data_string = kmem_zalloc(ilen + 1, KM_SLEEP); bcopy(data, data_string, ilen); (void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, name, data_string); kmem_free(data_string, ilen + 1); } static void bd_create_inquiry_props(dev_info_t *dip, bd_drive_t *drive) { if (drive->d_vendor_len > 0) bd_prop_update_inqstring(dip, INQUIRY_VENDOR_ID, drive->d_vendor, drive->d_vendor_len); if (drive->d_product_len > 0) bd_prop_update_inqstring(dip, INQUIRY_PRODUCT_ID, drive->d_product, drive->d_product_len); if (drive->d_serial_len > 0) bd_prop_update_inqstring(dip, INQUIRY_SERIAL_NO, drive->d_serial, drive->d_serial_len); if (drive->d_revision_len > 0) bd_prop_update_inqstring(dip, INQUIRY_REVISION_ID, drive->d_revision, drive->d_revision_len); } static void bd_create_errstats(bd_t *bd, int inst, bd_drive_t *drive) { char ks_module[KSTAT_STRLEN]; char ks_name[KSTAT_STRLEN]; int ndata = sizeof (struct bd_errstats) / sizeof (kstat_named_t); if (bd->d_errstats != NULL) return; (void) snprintf(ks_module, sizeof (ks_module), "%serr", ddi_driver_name(bd->d_dip)); (void) snprintf(ks_name, sizeof (ks_name), "%s%d,err", ddi_driver_name(bd->d_dip), inst); bd->d_errstats = kstat_create(ks_module, inst, ks_name, "device_error", KSTAT_TYPE_NAMED, ndata, KSTAT_FLAG_PERSISTENT); mutex_init(&bd->d_errmutex, NULL, MUTEX_DRIVER, NULL); if (bd->d_errstats == NULL) { /* * Even if we cannot create the kstat, we create a * scratch kstat. The reason for this is to ensure * that we can update the kstat all of the time, * without adding an extra branch instruction. */ bd->d_kerr = kmem_zalloc(sizeof (struct bd_errstats), KM_SLEEP); } else { bd->d_errstats->ks_lock = &bd->d_errmutex; bd->d_kerr = (struct bd_errstats *)bd->d_errstats->ks_data; } kstat_named_init(&bd->d_kerr->bd_softerrs, "Soft Errors", KSTAT_DATA_UINT32); kstat_named_init(&bd->d_kerr->bd_harderrs, "Hard Errors", KSTAT_DATA_UINT32); kstat_named_init(&bd->d_kerr->bd_transerrs, "Transport Errors", KSTAT_DATA_UINT32); if (drive->d_model_len > 0) { kstat_named_init(&bd->d_kerr->bd_model, "Model", KSTAT_DATA_STRING); } else { kstat_named_init(&bd->d_kerr->bd_vid, "Vendor", KSTAT_DATA_STRING); kstat_named_init(&bd->d_kerr->bd_pid, "Product", KSTAT_DATA_STRING); } kstat_named_init(&bd->d_kerr->bd_revision, "Revision", KSTAT_DATA_STRING); kstat_named_init(&bd->d_kerr->bd_serial, "Serial No", KSTAT_DATA_STRING); kstat_named_init(&bd->d_kerr->bd_capacity, "Size", KSTAT_DATA_ULONGLONG); kstat_named_init(&bd->d_kerr->bd_rq_media_err, "Media Error", KSTAT_DATA_UINT32); kstat_named_init(&bd->d_kerr->bd_rq_ntrdy_err, "Device Not Ready", KSTAT_DATA_UINT32); kstat_named_init(&bd->d_kerr->bd_rq_nodev_err, "No Device", KSTAT_DATA_UINT32); kstat_named_init(&bd->d_kerr->bd_rq_recov_err, "Recoverable", KSTAT_DATA_UINT32); kstat_named_init(&bd->d_kerr->bd_rq_illrq_err, "Illegal Request", KSTAT_DATA_UINT32); kstat_named_init(&bd->d_kerr->bd_rq_pfa_err, "Predictive Failure Analysis", KSTAT_DATA_UINT32); bd->d_errstats->ks_private = bd; kstat_install(bd->d_errstats); bd_init_errstats(bd, drive); } static void bd_destroy_errstats(bd_t *bd) { if (bd->d_errstats != NULL) { bd_fini_errstats(bd); kstat_delete(bd->d_errstats); bd->d_errstats = NULL; } else { kmem_free(bd->d_kerr, sizeof (struct bd_errstats)); bd->d_kerr = NULL; mutex_destroy(&bd->d_errmutex); } } static void bd_errstats_setstr(kstat_named_t *k, char *str, size_t len, char *alt) { char *tmp; size_t km_len; if (KSTAT_NAMED_STR_PTR(k) == NULL) { if (len > 0) km_len = strnlen(str, len); else if (alt != NULL) km_len = strlen(alt); else return; tmp = kmem_alloc(km_len + 1, KM_SLEEP); bcopy(len > 0 ? str : alt, tmp, km_len); tmp[km_len] = '\0'; kstat_named_setstr(k, tmp); } } static void bd_errstats_clrstr(kstat_named_t *k) { if (KSTAT_NAMED_STR_PTR(k) == NULL) return; kmem_free(KSTAT_NAMED_STR_PTR(k), KSTAT_NAMED_STR_BUFLEN(k)); kstat_named_setstr(k, NULL); } static void bd_init_errstats(bd_t *bd, bd_drive_t *drive) { struct bd_errstats *est = bd->d_kerr; mutex_enter(&bd->d_errmutex); if (drive->d_model_len > 0 && KSTAT_NAMED_STR_PTR(&est->bd_model) == NULL) { bd_errstats_setstr(&est->bd_model, drive->d_model, drive->d_model_len, NULL); } else { bd_errstats_setstr(&est->bd_vid, drive->d_vendor, drive->d_vendor_len, "Unknown "); bd_errstats_setstr(&est->bd_pid, drive->d_product, drive->d_product_len, "Unknown "); } bd_errstats_setstr(&est->bd_revision, drive->d_revision, drive->d_revision_len, "0001"); bd_errstats_setstr(&est->bd_serial, drive->d_serial, drive->d_serial_len, "0 "); mutex_exit(&bd->d_errmutex); } static void bd_fini_errstats(bd_t *bd) { struct bd_errstats *est = bd->d_kerr; mutex_enter(&bd->d_errmutex); bd_errstats_clrstr(&est->bd_model); bd_errstats_clrstr(&est->bd_vid); bd_errstats_clrstr(&est->bd_pid); bd_errstats_clrstr(&est->bd_revision); bd_errstats_clrstr(&est->bd_serial); mutex_exit(&bd->d_errmutex); } static void bd_queues_free(bd_t *bd) { uint32_t i; for (i = 0; i < bd->d_qcount; i++) { bd_queue_t *bq = &bd->d_queues[i]; mutex_destroy(&bq->q_iomutex); list_destroy(&bq->q_waitq); list_destroy(&bq->q_runq); } kmem_free(bd->d_queues, sizeof (*bd->d_queues) * bd->d_qcount); } static int bd_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { int inst; bd_handle_t hdl; bd_t *bd; bd_drive_t drive; uint32_t i; int rv; char name[16]; char kcache[32]; char *node_type; switch (cmd) { case DDI_ATTACH: break; case DDI_RESUME: /* We don't do anything native for suspend/resume */ return (DDI_SUCCESS); default: return (DDI_FAILURE); } inst = ddi_get_instance(dip); hdl = ddi_get_parent_data(dip); (void) snprintf(name, sizeof (name), "%s%d", ddi_driver_name(dip), ddi_get_instance(dip)); (void) snprintf(kcache, sizeof (kcache), "%s_xfer", name); if (hdl == NULL) { cmn_err(CE_WARN, "%s: missing parent data!", name); return (DDI_FAILURE); } if (ddi_soft_state_zalloc(bd_state, inst) != DDI_SUCCESS) { cmn_err(CE_WARN, "%s: unable to zalloc soft state!", name); return (DDI_FAILURE); } bd = ddi_get_soft_state(bd_state, inst); if (hdl->h_dma) { bd->d_dma = *(hdl->h_dma); bd->d_dma.dma_attr_granular = max(DEV_BSIZE, bd->d_dma.dma_attr_granular); bd->d_use_dma = B_TRUE; if (bd->d_maxxfer && (bd->d_maxxfer != bd->d_dma.dma_attr_maxxfer)) { cmn_err(CE_WARN, "%s: inconsistent maximum transfer size!", name); /* We force it */ bd->d_maxxfer = bd->d_dma.dma_attr_maxxfer; } else { bd->d_maxxfer = bd->d_dma.dma_attr_maxxfer; } } else { bd->d_use_dma = B_FALSE; if (bd->d_maxxfer == 0) { bd->d_maxxfer = 1024 * 1024; } } bd->d_ops = hdl->h_ops; bd->d_private = hdl->h_private; bd->d_blkshift = DEV_BSHIFT; /* 512 bytes, to start */ if (bd->d_maxxfer % DEV_BSIZE) { cmn_err(CE_WARN, "%s: maximum transfer misaligned!", name); bd->d_maxxfer &= ~(DEV_BSIZE - 1); } if (bd->d_maxxfer < DEV_BSIZE) { cmn_err(CE_WARN, "%s: maximum transfer size too small!", name); ddi_soft_state_free(bd_state, inst); return (DDI_FAILURE); } bd->d_dip = dip; bd->d_handle = hdl; ddi_set_driver_private(dip, bd); mutex_init(&bd->d_ksmutex, NULL, MUTEX_DRIVER, NULL); mutex_init(&bd->d_ocmutex, NULL, MUTEX_DRIVER, NULL); mutex_init(&bd->d_statemutex, NULL, MUTEX_DRIVER, NULL); cv_init(&bd->d_statecv, NULL, CV_DRIVER, NULL); mutex_init(&bd->d_dle_mutex, NULL, MUTEX_DRIVER, NULL); bd->d_dle_state = 0; bd->d_cache = kmem_cache_create(kcache, sizeof (bd_xfer_impl_t), 8, bd_xfer_ctor, bd_xfer_dtor, NULL, bd, NULL, 0); bd->d_ksp = kstat_create(ddi_driver_name(dip), inst, NULL, "disk", KSTAT_TYPE_IO, 1, KSTAT_FLAG_PERSISTENT); if (bd->d_ksp != NULL) { bd->d_ksp->ks_lock = &bd->d_ksmutex; kstat_install(bd->d_ksp); bd->d_kiop = bd->d_ksp->ks_data; } else { /* * Even if we cannot create the kstat, we create a * scratch kstat. The reason for this is to ensure * that we can update the kstat all of the time, * without adding an extra branch instruction. */ bd->d_kiop = kmem_zalloc(sizeof (kstat_io_t), KM_SLEEP); } cmlb_alloc_handle(&bd->d_cmlbh); bd->d_state = DKIO_NONE; bzero(&drive, sizeof (drive)); /* * Default to one queue, and no restrictions on free space requests * (if driver provides method) parent driver can override. */ drive.d_qcount = 1; drive.d_free_align = 1; bd->d_ops.o_drive_info(bd->d_private, &drive); /* * Several checks to make sure o_drive_info() didn't return bad * values: * * There must be at least one queue */ if (drive.d_qcount == 0) goto fail_drive_info; /* FREE/UNMAP/TRIM alignment needs to be at least 1 block */ if (drive.d_free_align == 0) goto fail_drive_info; /* * If d_max_free_blks is not unlimited (not 0), then we cannot allow * an unlimited segment size. It is however permissible to not impose * a limit on the total number of blocks freed while limiting the * amount allowed in an individual segment. */ if ((drive.d_max_free_blks > 0 && drive.d_max_free_seg_blks == 0)) goto fail_drive_info; /* * If a limit is set on d_max_free_blks (by the above check, we know * if there's a limit on d_max_free_blks, d_max_free_seg_blks cannot * be unlimited), it cannot be smaller than the limit on an individual * segment. */ if ((drive.d_max_free_blks > 0 && drive.d_max_free_seg_blks > drive.d_max_free_blks)) { goto fail_drive_info; } bd->d_qcount = drive.d_qcount; bd->d_removable = drive.d_removable; bd->d_hotpluggable = drive.d_hotpluggable; if (drive.d_maxxfer && drive.d_maxxfer < bd->d_maxxfer) bd->d_maxxfer = drive.d_maxxfer; bd->d_free_align = drive.d_free_align; bd->d_max_free_seg = drive.d_max_free_seg; bd->d_max_free_blks = drive.d_max_free_blks; bd->d_max_free_seg_blks = drive.d_max_free_seg_blks; bd_create_inquiry_props(dip, &drive); bd_create_errstats(bd, inst, &drive); bd_update_state(bd); bd->d_queues = kmem_alloc(sizeof (*bd->d_queues) * bd->d_qcount, KM_SLEEP); for (i = 0; i < bd->d_qcount; i++) { bd_queue_t *bq = &bd->d_queues[i]; bq->q_qsize = drive.d_qsize; bq->q_qactive = 0; mutex_init(&bq->q_iomutex, NULL, MUTEX_DRIVER, NULL); list_create(&bq->q_waitq, sizeof (bd_xfer_impl_t), offsetof(struct bd_xfer_impl, i_linkage)); list_create(&bq->q_runq, sizeof (bd_xfer_impl_t), offsetof(struct bd_xfer_impl, i_linkage)); } if (*(uint64_t *)drive.d_eui64 != 0 || *(uint64_t *)drive.d_guid != 0 || *((uint64_t *)drive.d_guid + 1) != 0) node_type = DDI_NT_BLOCK_BLKDEV; else if (drive.d_lun >= 0) node_type = DDI_NT_BLOCK_CHAN; else node_type = DDI_NT_BLOCK; rv = cmlb_attach(dip, &bd_tg_ops, DTYPE_DIRECT, bd->d_removable, bd->d_hotpluggable, node_type, CMLB_FAKE_LABEL_ONE_PARTITION, bd->d_cmlbh, 0); if (rv != 0) { goto fail_cmlb_attach; } if (bd->d_ops.o_devid_init != NULL) { rv = bd->d_ops.o_devid_init(bd->d_private, dip, &bd->d_devid); if (rv == DDI_SUCCESS) { if (ddi_devid_register(dip, bd->d_devid) != DDI_SUCCESS) { cmn_err(CE_WARN, "%s: unable to register devid", name); } } } /* * Add a zero-length attribute to tell the world we support * kernel ioctls (for layered drivers). Also set up properties * used by HAL to identify removable media. */ (void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP, DDI_KERNEL_IOCTL, NULL, 0); if (bd->d_removable) { (void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP, "removable-media", NULL, 0); } if (bd->d_hotpluggable) { (void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP, "hotpluggable", NULL, 0); } hdl->h_bd = bd; ddi_report_dev(dip); return (DDI_SUCCESS); fail_cmlb_attach: bd_queues_free(bd); bd_destroy_errstats(bd); fail_drive_info: cmlb_free_handle(&bd->d_cmlbh); if (bd->d_ksp != NULL) { kstat_delete(bd->d_ksp); bd->d_ksp = NULL; } else { kmem_free(bd->d_kiop, sizeof (kstat_io_t)); } kmem_cache_destroy(bd->d_cache); cv_destroy(&bd->d_statecv); mutex_destroy(&bd->d_statemutex); mutex_destroy(&bd->d_ocmutex); mutex_destroy(&bd->d_ksmutex); mutex_destroy(&bd->d_dle_mutex); ddi_soft_state_free(bd_state, inst); return (DDI_FAILURE); } static int bd_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { bd_handle_t hdl; bd_t *bd; bd = ddi_get_driver_private(dip); hdl = ddi_get_parent_data(dip); switch (cmd) { case DDI_DETACH: break; case DDI_SUSPEND: /* We don't suspend, but our parent does */ return (DDI_SUCCESS); default: return (DDI_FAILURE); } hdl->h_bd = NULL; if (bd->d_ksp != NULL) { kstat_delete(bd->d_ksp); bd->d_ksp = NULL; } else { kmem_free(bd->d_kiop, sizeof (kstat_io_t)); } bd_destroy_errstats(bd); cmlb_detach(bd->d_cmlbh, 0); cmlb_free_handle(&bd->d_cmlbh); if (bd->d_devid) ddi_devid_free(bd->d_devid); kmem_cache_destroy(bd->d_cache); mutex_destroy(&bd->d_ksmutex); mutex_destroy(&bd->d_ocmutex); mutex_destroy(&bd->d_statemutex); cv_destroy(&bd->d_statecv); mutex_destroy(&bd->d_dle_mutex); bd_queues_free(bd); ddi_soft_state_free(bd_state, ddi_get_instance(dip)); return (DDI_SUCCESS); } static int bd_xfer_ctor(void *buf, void *arg, int kmflag) { bd_xfer_impl_t *xi; bd_t *bd = arg; int (*dcb)(caddr_t); if (kmflag == KM_PUSHPAGE || kmflag == KM_SLEEP) { dcb = DDI_DMA_SLEEP; } else { dcb = DDI_DMA_DONTWAIT; } xi = buf; bzero(xi, sizeof (*xi)); xi->i_bd = bd; if (bd->d_use_dma) { if (ddi_dma_alloc_handle(bd->d_dip, &bd->d_dma, dcb, NULL, &xi->i_dmah) != DDI_SUCCESS) { return (-1); } } return (0); } static void bd_xfer_dtor(void *buf, void *arg) { bd_xfer_impl_t *xi = buf; _NOTE(ARGUNUSED(arg)); if (xi->i_dmah) ddi_dma_free_handle(&xi->i_dmah); xi->i_dmah = NULL; } static bd_xfer_impl_t * bd_xfer_alloc(bd_t *bd, struct buf *bp, int (*func)(void *, bd_xfer_t *), int kmflag) { bd_xfer_impl_t *xi; int rv = 0; int status; unsigned dir; int (*cb)(caddr_t); size_t len; uint32_t shift; if (kmflag == KM_SLEEP) { cb = DDI_DMA_SLEEP; } else { cb = DDI_DMA_DONTWAIT; } xi = kmem_cache_alloc(bd->d_cache, kmflag); if (xi == NULL) { bioerror(bp, ENOMEM); return (NULL); } ASSERT(bp); xi->i_bp = bp; xi->i_func = func; xi->i_blkno = bp->b_lblkno >> (bd->d_blkshift - DEV_BSHIFT); if (bp->b_bcount == 0) { xi->i_len = 0; xi->i_nblks = 0; xi->i_kaddr = NULL; xi->i_resid = 0; xi->i_num_win = 0; goto done; } if (bp->b_flags & B_READ) { dir = DDI_DMA_READ; xi->i_func = bd->d_ops.o_read; } else { dir = DDI_DMA_WRITE; xi->i_func = bd->d_ops.o_write; } shift = bd->d_blkshift; xi->i_blkshift = shift; if (!bd->d_use_dma) { bp_mapin(bp); rv = 0; xi->i_offset = 0; xi->i_num_win = (bp->b_bcount + (bd->d_maxxfer - 1)) / bd->d_maxxfer; xi->i_cur_win = 0; xi->i_len = min(bp->b_bcount, bd->d_maxxfer); xi->i_nblks = xi->i_len >> shift; xi->i_kaddr = bp->b_un.b_addr; xi->i_resid = bp->b_bcount; } else { /* * We have to use consistent DMA if the address is misaligned. */ if (((bp->b_flags & (B_PAGEIO | B_REMAPPED)) != B_PAGEIO) && ((uintptr_t)bp->b_un.b_addr & 0x7)) { dir |= DDI_DMA_CONSISTENT | DDI_DMA_PARTIAL; } else { dir |= DDI_DMA_STREAMING | DDI_DMA_PARTIAL; } status = ddi_dma_buf_bind_handle(xi->i_dmah, bp, dir, cb, NULL, &xi->i_dmac, &xi->i_ndmac); switch (status) { case DDI_DMA_MAPPED: xi->i_num_win = 1; xi->i_cur_win = 0; xi->i_offset = 0; xi->i_len = bp->b_bcount; xi->i_nblks = xi->i_len >> shift; xi->i_resid = bp->b_bcount; rv = 0; break; case DDI_DMA_PARTIAL_MAP: xi->i_cur_win = 0; if ((ddi_dma_numwin(xi->i_dmah, &xi->i_num_win) != DDI_SUCCESS) || (ddi_dma_getwin(xi->i_dmah, 0, &xi->i_offset, &len, &xi->i_dmac, &xi->i_ndmac) != DDI_SUCCESS) || (P2PHASE(len, (1U << shift)) != 0)) { (void) ddi_dma_unbind_handle(xi->i_dmah); rv = EFAULT; goto done; } xi->i_len = len; xi->i_nblks = xi->i_len >> shift; xi->i_resid = bp->b_bcount; rv = 0; break; case DDI_DMA_NORESOURCES: rv = EAGAIN; goto done; case DDI_DMA_TOOBIG: rv = EINVAL; goto done; case DDI_DMA_NOMAPPING: case DDI_DMA_INUSE: default: rv = EFAULT; goto done; } } done: if (rv != 0) { kmem_cache_free(bd->d_cache, xi); bioerror(bp, rv); return (NULL); } return (xi); } static void bd_xfer_free(bd_xfer_impl_t *xi) { if (xi->i_dmah) { (void) ddi_dma_unbind_handle(xi->i_dmah); } if (xi->i_dfl != NULL) { dfl_free((dkioc_free_list_t *)xi->i_dfl); xi->i_dfl = NULL; } kmem_cache_free(xi->i_bd->d_cache, xi); } static int bd_open(dev_t *devp, int flag, int otyp, cred_t *credp) { dev_t dev = *devp; bd_t *bd; minor_t part; minor_t inst; uint64_t mask; boolean_t ndelay; int rv; diskaddr_t nblks; diskaddr_t lba; _NOTE(ARGUNUSED(credp)); part = BDPART(dev); inst = BDINST(dev); if (otyp >= OTYPCNT) return (EINVAL); ndelay = (flag & (FNDELAY | FNONBLOCK)) ? B_TRUE : B_FALSE; /* * Block any DR events from changing the set of registered * devices while we function. */ rw_enter(&bd_lock, RW_READER); if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) { rw_exit(&bd_lock); return (ENXIO); } mutex_enter(&bd->d_ocmutex); ASSERT(part < 64); mask = (1U << part); bd_update_state(bd); if (cmlb_validate(bd->d_cmlbh, 0, 0) != 0) { /* non-blocking opens are allowed to succeed */ if (!ndelay) { rv = ENXIO; goto done; } } else if (cmlb_partinfo(bd->d_cmlbh, part, &nblks, &lba, NULL, NULL, 0) == 0) { /* * We read the partinfo, verify valid ranges. If the * partition is invalid, and we aren't blocking or * doing a raw access, then fail. (Non-blocking and * raw accesses can still succeed to allow a disk with * bad partition data to opened by format and fdisk.) */ if ((!nblks) && ((!ndelay) || (otyp != OTYP_CHR))) { rv = ENXIO; goto done; } } else if (!ndelay) { /* * cmlb_partinfo failed -- invalid partition or no * disk label. */ rv = ENXIO; goto done; } if ((flag & FWRITE) && bd->d_rdonly) { rv = EROFS; goto done; } if ((bd->d_open_excl) & (mask)) { rv = EBUSY; goto done; } if (flag & FEXCL) { if (bd->d_open_lyr[part]) { rv = EBUSY; goto done; } for (int i = 0; i < OTYP_LYR; i++) { if (bd->d_open_reg[i] & mask) { rv = EBUSY; goto done; } } } if (otyp == OTYP_LYR) { bd->d_open_lyr[part]++; } else { bd->d_open_reg[otyp] |= mask; } if (flag & FEXCL) { bd->d_open_excl |= mask; } rv = 0; done: mutex_exit(&bd->d_ocmutex); rw_exit(&bd_lock); return (rv); } static int bd_close(dev_t dev, int flag, int otyp, cred_t *credp) { bd_t *bd; minor_t inst; minor_t part; uint64_t mask; boolean_t last = B_TRUE; _NOTE(ARGUNUSED(flag)); _NOTE(ARGUNUSED(credp)); part = BDPART(dev); inst = BDINST(dev); ASSERT(part < 64); mask = (1U << part); rw_enter(&bd_lock, RW_READER); if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) { rw_exit(&bd_lock); return (ENXIO); } mutex_enter(&bd->d_ocmutex); if (bd->d_open_excl & mask) { bd->d_open_excl &= ~mask; } if (otyp == OTYP_LYR) { bd->d_open_lyr[part]--; } else { bd->d_open_reg[otyp] &= ~mask; } for (int i = 0; i < 64; i++) { if (bd->d_open_lyr[part]) { last = B_FALSE; } } for (int i = 0; last && (i < OTYP_LYR); i++) { if (bd->d_open_reg[i]) { last = B_FALSE; } } mutex_exit(&bd->d_ocmutex); if (last) { cmlb_invalidate(bd->d_cmlbh, 0); } rw_exit(&bd_lock); return (0); } static int bd_dump(dev_t dev, caddr_t caddr, daddr_t blkno, int nblk) { minor_t inst; minor_t part; diskaddr_t pstart; diskaddr_t psize; bd_t *bd; bd_xfer_impl_t *xi; buf_t *bp; int rv; uint32_t shift; daddr_t d_blkno; int d_nblk; rw_enter(&bd_lock, RW_READER); part = BDPART(dev); inst = BDINST(dev); if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) { rw_exit(&bd_lock); return (ENXIO); } shift = bd->d_blkshift; d_blkno = blkno >> (shift - DEV_BSHIFT); d_nblk = nblk >> (shift - DEV_BSHIFT); /* * do cmlb, but do it synchronously unless we already have the * partition (which we probably should.) */ if (cmlb_partinfo(bd->d_cmlbh, part, &psize, &pstart, NULL, NULL, (void *)1)) { rw_exit(&bd_lock); return (ENXIO); } if ((d_blkno + d_nblk) > psize) { rw_exit(&bd_lock); return (EINVAL); } bp = getrbuf(KM_NOSLEEP); if (bp == NULL) { rw_exit(&bd_lock); return (ENOMEM); } bp->b_bcount = nblk << DEV_BSHIFT; bp->b_resid = bp->b_bcount; bp->b_lblkno = blkno; bp->b_un.b_addr = caddr; xi = bd_xfer_alloc(bd, bp, bd->d_ops.o_write, KM_NOSLEEP); if (xi == NULL) { rw_exit(&bd_lock); freerbuf(bp); return (ENOMEM); } xi->i_blkno = d_blkno + pstart; xi->i_flags = BD_XFER_POLL; bd_submit(bd, xi); rw_exit(&bd_lock); /* * Generally, we should have run this entirely synchronously * at this point and the biowait call should be a no-op. If * it didn't happen this way, it's a bug in the underlying * driver not honoring BD_XFER_POLL. */ (void) biowait(bp); rv = geterror(bp); freerbuf(bp); return (rv); } void bd_minphys(struct buf *bp) { minor_t inst; bd_t *bd; inst = BDINST(bp->b_edev); bd = ddi_get_soft_state(bd_state, inst); /* * In a non-debug kernel, bd_strategy will catch !bd as * well, and will fail nicely. */ ASSERT(bd); if (bp->b_bcount > bd->d_maxxfer) bp->b_bcount = bd->d_maxxfer; } static int bd_check_uio(dev_t dev, struct uio *uio) { bd_t *bd; uint32_t shift; if ((bd = ddi_get_soft_state(bd_state, BDINST(dev))) == NULL) { return (ENXIO); } shift = bd->d_blkshift; if ((P2PHASE(uio->uio_loffset, (1U << shift)) != 0) || (P2PHASE(uio->uio_iov->iov_len, (1U << shift)) != 0)) { return (EINVAL); } return (0); } static int bd_read(dev_t dev, struct uio *uio, cred_t *credp) { _NOTE(ARGUNUSED(credp)); int ret = bd_check_uio(dev, uio); if (ret != 0) { return (ret); } return (physio(bd_strategy, NULL, dev, B_READ, bd_minphys, uio)); } static int bd_write(dev_t dev, struct uio *uio, cred_t *credp) { _NOTE(ARGUNUSED(credp)); int ret = bd_check_uio(dev, uio); if (ret != 0) { return (ret); } return (physio(bd_strategy, NULL, dev, B_WRITE, bd_minphys, uio)); } static int bd_aread(dev_t dev, struct aio_req *aio, cred_t *credp) { _NOTE(ARGUNUSED(credp)); int ret = bd_check_uio(dev, aio->aio_uio); if (ret != 0) { return (ret); } return (aphysio(bd_strategy, anocancel, dev, B_READ, bd_minphys, aio)); } static int bd_awrite(dev_t dev, struct aio_req *aio, cred_t *credp) { _NOTE(ARGUNUSED(credp)); int ret = bd_check_uio(dev, aio->aio_uio); if (ret != 0) { return (ret); } return (aphysio(bd_strategy, anocancel, dev, B_WRITE, bd_minphys, aio)); } static int bd_strategy(struct buf *bp) { minor_t inst; minor_t part; bd_t *bd; diskaddr_t p_lba; diskaddr_t p_nblks; diskaddr_t b_nblks; bd_xfer_impl_t *xi; uint32_t shift; int (*func)(void *, bd_xfer_t *); diskaddr_t lblkno; part = BDPART(bp->b_edev); inst = BDINST(bp->b_edev); ASSERT(bp); bp->b_resid = bp->b_bcount; if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) { bioerror(bp, ENXIO); biodone(bp); return (0); } if (cmlb_partinfo(bd->d_cmlbh, part, &p_nblks, &p_lba, NULL, NULL, 0)) { bioerror(bp, ENXIO); biodone(bp); return (0); } shift = bd->d_blkshift; lblkno = bp->b_lblkno >> (shift - DEV_BSHIFT); if ((P2PHASE(bp->b_lblkno, (1U << (shift - DEV_BSHIFT))) != 0) || (P2PHASE(bp->b_bcount, (1U << shift)) != 0) || (lblkno > p_nblks)) { bioerror(bp, EINVAL); biodone(bp); return (0); } b_nblks = bp->b_bcount >> shift; if ((lblkno == p_nblks) || (bp->b_bcount == 0)) { biodone(bp); return (0); } if ((b_nblks + lblkno) > p_nblks) { bp->b_resid = ((lblkno + b_nblks - p_nblks) << shift); bp->b_bcount -= bp->b_resid; } else { bp->b_resid = 0; } func = (bp->b_flags & B_READ) ? bd->d_ops.o_read : bd->d_ops.o_write; xi = bd_xfer_alloc(bd, bp, func, KM_NOSLEEP); if (xi == NULL) { xi = bd_xfer_alloc(bd, bp, func, KM_PUSHPAGE); } if (xi == NULL) { /* bd_request_alloc will have done bioerror */ biodone(bp); return (0); } xi->i_blkno = lblkno + p_lba; bd_submit(bd, xi); return (0); } static int bd_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *credp, int *rvalp) { minor_t inst; uint16_t part; bd_t *bd; void *ptr = (void *)arg; int rv; part = BDPART(dev); inst = BDINST(dev); if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) { return (ENXIO); } rv = cmlb_ioctl(bd->d_cmlbh, dev, cmd, arg, flag, credp, rvalp, 0); if (rv != ENOTTY) return (rv); if (rvalp != NULL) { /* the return value of the ioctl is 0 by default */ *rvalp = 0; } switch (cmd) { case DKIOCGMEDIAINFO: { struct dk_minfo minfo; /* make sure our state information is current */ bd_update_state(bd); bzero(&minfo, sizeof (minfo)); minfo.dki_media_type = DK_FIXED_DISK; minfo.dki_lbsize = (1U << bd->d_blkshift); minfo.dki_capacity = bd->d_numblks; if (ddi_copyout(&minfo, ptr, sizeof (minfo), flag)) { return (EFAULT); } return (0); } case DKIOCGMEDIAINFOEXT: { struct dk_minfo_ext miext; size_t len; /* make sure our state information is current */ bd_update_state(bd); bzero(&miext, sizeof (miext)); miext.dki_media_type = DK_FIXED_DISK; miext.dki_lbsize = (1U << bd->d_blkshift); miext.dki_pbsize = (1U << bd->d_pblkshift); miext.dki_capacity = bd->d_numblks; switch (ddi_model_convert_from(flag & FMODELS)) { case DDI_MODEL_ILP32: len = sizeof (struct dk_minfo_ext32); break; default: len = sizeof (struct dk_minfo_ext); break; } if (ddi_copyout(&miext, ptr, len, flag)) { return (EFAULT); } return (0); } case DKIOCINFO: { struct dk_cinfo cinfo; bzero(&cinfo, sizeof (cinfo)); cinfo.dki_ctype = DKC_BLKDEV; cinfo.dki_cnum = ddi_get_instance(ddi_get_parent(bd->d_dip)); (void) snprintf(cinfo.dki_cname, sizeof (cinfo.dki_cname), "%s", ddi_driver_name(ddi_get_parent(bd->d_dip))); (void) snprintf(cinfo.dki_dname, sizeof (cinfo.dki_dname), "%s", ddi_driver_name(bd->d_dip)); cinfo.dki_unit = inst; cinfo.dki_flags = DKI_FMTVOL; cinfo.dki_partition = part; cinfo.dki_maxtransfer = bd->d_maxxfer / DEV_BSIZE; cinfo.dki_addr = 0; cinfo.dki_slave = 0; cinfo.dki_space = 0; cinfo.dki_prio = 0; cinfo.dki_vec = 0; if (ddi_copyout(&cinfo, ptr, sizeof (cinfo), flag)) { return (EFAULT); } return (0); } case DKIOCREMOVABLE: { int i; i = bd->d_removable ? 1 : 0; if (ddi_copyout(&i, ptr, sizeof (i), flag)) { return (EFAULT); } return (0); } case DKIOCHOTPLUGGABLE: { int i; i = bd->d_hotpluggable ? 1 : 0; if (ddi_copyout(&i, ptr, sizeof (i), flag)) { return (EFAULT); } return (0); } case DKIOCREADONLY: { int i; i = bd->d_rdonly ? 1 : 0; if (ddi_copyout(&i, ptr, sizeof (i), flag)) { return (EFAULT); } return (0); } case DKIOCSOLIDSTATE: { int i; i = bd->d_ssd ? 1 : 0; if (ddi_copyout(&i, ptr, sizeof (i), flag)) { return (EFAULT); } return (0); } case DKIOCSTATE: { enum dkio_state state; if (ddi_copyin(ptr, &state, sizeof (state), flag)) { return (EFAULT); } if ((rv = bd_check_state(bd, &state)) != 0) { return (rv); } if (ddi_copyout(&state, ptr, sizeof (state), flag)) { return (EFAULT); } return (0); } case DKIOCFLUSHWRITECACHE: { struct dk_callback *dkc = NULL; if (flag & FKIOCTL) dkc = (void *)arg; rv = bd_flush_write_cache(bd, dkc); return (rv); } case DKIOCFREE: { dkioc_free_list_t *dfl = NULL; /* * Check free space support early to avoid copyin/allocation * when unnecessary. */ if (!CAN_FREESPACE(bd)) return (ENOTSUP); rv = dfl_copyin(ptr, &dfl, flag, KM_SLEEP); if (rv != 0) return (rv); /* * bd_free_space() consumes 'dfl'. bd_free_space() will * call dfl_iter() which will normally try to pass dfl through * to bd_free_space_cb() which attaches dfl to the bd_xfer_t * that is then queued for the underlying driver. Once the * driver processes the request, the bd_xfer_t instance is * disposed of, including any attached dkioc_free_list_t. * * If dfl cannot be processed by the underlying driver due to * size or alignment requirements of the driver, dfl_iter() * will replace dfl with one or more new dkioc_free_list_t * instances with the correct alignment and sizes for the driver * (and free the original dkioc_free_list_t). */ rv = bd_free_space(dev, bd, dfl); return (rv); } case DKIOC_CANFREE: { boolean_t supported = CAN_FREESPACE(bd); if (ddi_copyout(&supported, (void *)arg, sizeof (supported), flag) != 0) { return (EFAULT); } return (0); } default: break; } return (ENOTTY); } static int bd_prop_op(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op, int mod_flags, char *name, caddr_t valuep, int *lengthp) { bd_t *bd; bd = ddi_get_soft_state(bd_state, ddi_get_instance(dip)); if (bd == NULL) return (ddi_prop_op(dev, dip, prop_op, mod_flags, name, valuep, lengthp)); return (cmlb_prop_op(bd->d_cmlbh, dev, dip, prop_op, mod_flags, name, valuep, lengthp, BDPART(dev), 0)); } static int bd_tg_rdwr(dev_info_t *dip, uchar_t cmd, void *bufaddr, diskaddr_t start, size_t length, void *tg_cookie) { bd_t *bd; buf_t *bp; bd_xfer_impl_t *xi; int rv; int (*func)(void *, bd_xfer_t *); int kmflag; /* * If we are running in polled mode (such as during dump(9e) * execution), then we cannot sleep for kernel allocations. */ kmflag = tg_cookie ? KM_NOSLEEP : KM_SLEEP; bd = ddi_get_soft_state(bd_state, ddi_get_instance(dip)); if (P2PHASE(length, (1U << bd->d_blkshift)) != 0) { /* We can only transfer whole blocks at a time! */ return (EINVAL); } if ((bp = getrbuf(kmflag)) == NULL) { return (ENOMEM); } switch (cmd) { case TG_READ: bp->b_flags = B_READ; func = bd->d_ops.o_read; break; case TG_WRITE: bp->b_flags = B_WRITE; func = bd->d_ops.o_write; break; default: freerbuf(bp); return (EINVAL); } bp->b_un.b_addr = bufaddr; bp->b_bcount = length; xi = bd_xfer_alloc(bd, bp, func, kmflag); if (xi == NULL) { rv = geterror(bp); freerbuf(bp); return (rv); } xi->i_flags = tg_cookie ? BD_XFER_POLL : 0; xi->i_blkno = start; bd_submit(bd, xi); (void) biowait(bp); rv = geterror(bp); freerbuf(bp); return (rv); } static int bd_tg_getinfo(dev_info_t *dip, int cmd, void *arg, void *tg_cookie) { bd_t *bd; _NOTE(ARGUNUSED(tg_cookie)); bd = ddi_get_soft_state(bd_state, ddi_get_instance(dip)); switch (cmd) { case TG_GETPHYGEOM: case TG_GETVIRTGEOM: /* * We don't have any "geometry" as such, let cmlb * fabricate something. */ return (ENOTTY); case TG_GETCAPACITY: bd_update_state(bd); *(diskaddr_t *)arg = bd->d_numblks; return (0); case TG_GETBLOCKSIZE: *(uint32_t *)arg = (1U << bd->d_blkshift); return (0); case TG_GETATTR: /* * It turns out that cmlb really doesn't do much for * non-writable media, but lets make the information * available for it in case it does more in the * future. (The value is currently used for * triggering special behavior for CD-ROMs.) */ bd_update_state(bd); ((tg_attribute_t *)arg)->media_is_writable = bd->d_rdonly ? B_FALSE : B_TRUE; ((tg_attribute_t *)arg)->media_is_solid_state = bd->d_ssd; ((tg_attribute_t *)arg)->media_is_rotational = B_FALSE; return (0); default: return (EINVAL); } } static void bd_sched(bd_t *bd, bd_queue_t *bq) { bd_xfer_impl_t *xi; struct buf *bp; int rv; mutex_enter(&bq->q_iomutex); while ((bq->q_qactive < bq->q_qsize) && ((xi = list_remove_head(&bq->q_waitq)) != NULL)) { mutex_enter(&bd->d_ksmutex); kstat_waitq_to_runq(bd->d_kiop); mutex_exit(&bd->d_ksmutex); bq->q_qactive++; list_insert_tail(&bq->q_runq, xi); /* * Submit the job to the driver. We drop the I/O mutex * so that we can deal with the case where the driver * completion routine calls back into us synchronously. */ mutex_exit(&bq->q_iomutex); rv = xi->i_func(bd->d_private, &xi->i_public); if (rv != 0) { bp = xi->i_bp; bioerror(bp, rv); biodone(bp); atomic_inc_32(&bd->d_kerr->bd_transerrs.value.ui32); mutex_enter(&bq->q_iomutex); mutex_enter(&bd->d_ksmutex); kstat_runq_exit(bd->d_kiop); mutex_exit(&bd->d_ksmutex); bq->q_qactive--; list_remove(&bq->q_runq, xi); bd_xfer_free(xi); } else { mutex_enter(&bq->q_iomutex); } } mutex_exit(&bq->q_iomutex); } static void bd_submit(bd_t *bd, bd_xfer_impl_t *xi) { uint64_t nv = atomic_inc_64_nv(&bd->d_io_counter); unsigned q = nv % bd->d_qcount; bd_queue_t *bq = &bd->d_queues[q]; xi->i_bq = bq; xi->i_qnum = q; mutex_enter(&bq->q_iomutex); list_insert_tail(&bq->q_waitq, xi); mutex_enter(&bd->d_ksmutex); kstat_waitq_enter(bd->d_kiop); mutex_exit(&bd->d_ksmutex); mutex_exit(&bq->q_iomutex); bd_sched(bd, bq); } static void bd_runq_exit(bd_xfer_impl_t *xi, int err) { bd_t *bd = xi->i_bd; buf_t *bp = xi->i_bp; bd_queue_t *bq = xi->i_bq; mutex_enter(&bq->q_iomutex); bq->q_qactive--; mutex_enter(&bd->d_ksmutex); kstat_runq_exit(bd->d_kiop); mutex_exit(&bd->d_ksmutex); list_remove(&bq->q_runq, xi); mutex_exit(&bq->q_iomutex); if (err == 0) { if (bp->b_flags & B_READ) { atomic_inc_uint(&bd->d_kiop->reads); atomic_add_64((uint64_t *)&bd->d_kiop->nread, bp->b_bcount - xi->i_resid); } else { atomic_inc_uint(&bd->d_kiop->writes); atomic_add_64((uint64_t *)&bd->d_kiop->nwritten, bp->b_bcount - xi->i_resid); } } bd_sched(bd, bq); } static void bd_dle_sysevent_task(void *arg) { nvlist_t *attr = NULL; char *path = NULL; bd_t *bd = arg; dev_info_t *dip = bd->d_dip; size_t n; mutex_enter(&bd->d_dle_mutex); bd->d_dle_state &= ~BD_DLE_PENDING; bd->d_dle_state |= BD_DLE_RUNNING; mutex_exit(&bd->d_dle_mutex); dev_err(dip, CE_NOTE, "!dynamic LUN expansion"); if (nvlist_alloc(&attr, NV_UNIQUE_NAME_TYPE, KM_SLEEP) != 0) { mutex_enter(&bd->d_dle_mutex); bd->d_dle_state &= ~(BD_DLE_RUNNING|BD_DLE_PENDING); mutex_exit(&bd->d_dle_mutex); return; } path = kmem_zalloc(MAXPATHLEN, KM_SLEEP); n = snprintf(path, MAXPATHLEN, "/devices"); (void) ddi_pathname(dip, path + n); n = strlen(path); n += snprintf(path + n, MAXPATHLEN - n, ":x"); for (;;) { /* * On receipt of this event, the ZFS sysevent module will scan * active zpools for child vdevs matching this physical path. * In order to catch both whole disk pools and those with an * EFI boot partition, generate separate sysevents for minor * node 'a' and 'b'. */ for (char c = 'a'; c < 'c'; c++) { path[n - 1] = c; if (nvlist_add_string(attr, DEV_PHYS_PATH, path) != 0) break; (void) ddi_log_sysevent(dip, DDI_VENDOR_SUNW, EC_DEV_STATUS, ESC_DEV_DLE, attr, NULL, DDI_SLEEP); } mutex_enter(&bd->d_dle_mutex); if ((bd->d_dle_state & BD_DLE_PENDING) == 0) { bd->d_dle_state &= ~BD_DLE_RUNNING; mutex_exit(&bd->d_dle_mutex); break; } bd->d_dle_state &= ~BD_DLE_PENDING; mutex_exit(&bd->d_dle_mutex); } nvlist_free(attr); kmem_free(path, MAXPATHLEN); } static void bd_update_state(bd_t *bd) { enum dkio_state state = DKIO_INSERTED; boolean_t docmlb = B_FALSE; bd_media_t media; bzero(&media, sizeof (media)); mutex_enter(&bd->d_statemutex); if (bd->d_ops.o_media_info(bd->d_private, &media) != 0) { bd->d_numblks = 0; state = DKIO_EJECTED; goto done; } if ((media.m_blksize < 512) || (!ISP2(media.m_blksize)) || (P2PHASE(bd->d_maxxfer, media.m_blksize))) { dev_err(bd->d_dip, CE_WARN, "Invalid media block size (%d)", media.m_blksize); /* * We can't use the media, treat it as not present. */ state = DKIO_EJECTED; bd->d_numblks = 0; goto done; } if (((1U << bd->d_blkshift) != media.m_blksize) || (bd->d_numblks != media.m_nblks)) { /* Device size changed */ docmlb = B_TRUE; } bd->d_blkshift = ddi_ffs(media.m_blksize) - 1; bd->d_pblkshift = bd->d_blkshift; bd->d_numblks = media.m_nblks; bd->d_rdonly = media.m_readonly; bd->d_ssd = media.m_solidstate; /* * Only use the supplied physical block size if it is non-zero, * greater or equal to the block size, and a power of 2. Ignore it * if not, it's just informational and we can still use the media. */ if ((media.m_pblksize != 0) && (media.m_pblksize >= media.m_blksize) && (ISP2(media.m_pblksize))) bd->d_pblkshift = ddi_ffs(media.m_pblksize) - 1; done: if (state != bd->d_state) { bd->d_state = state; cv_broadcast(&bd->d_statecv); docmlb = B_TRUE; } mutex_exit(&bd->d_statemutex); bd->d_kerr->bd_capacity.value.ui64 = bd->d_numblks << bd->d_blkshift; if (docmlb) { if (state == DKIO_INSERTED) { (void) cmlb_validate(bd->d_cmlbh, 0, 0); mutex_enter(&bd->d_dle_mutex); /* * If there is already an event pending, there's * nothing to do; we coalesce multiple events. */ if ((bd->d_dle_state & BD_DLE_PENDING) == 0) { if ((bd->d_dle_state & BD_DLE_RUNNING) == 0) { taskq_dispatch_ent(bd_taskq, bd_dle_sysevent_task, bd, 0, &bd->d_dle_ent); } bd->d_dle_state |= BD_DLE_PENDING; } mutex_exit(&bd->d_dle_mutex); } else { cmlb_invalidate(bd->d_cmlbh, 0); } } } static int bd_check_state(bd_t *bd, enum dkio_state *state) { clock_t when; for (;;) { bd_update_state(bd); mutex_enter(&bd->d_statemutex); if (bd->d_state != *state) { *state = bd->d_state; mutex_exit(&bd->d_statemutex); break; } when = drv_usectohz(1000000); if (cv_reltimedwait_sig(&bd->d_statecv, &bd->d_statemutex, when, TR_CLOCK_TICK) == 0) { mutex_exit(&bd->d_statemutex); return (EINTR); } mutex_exit(&bd->d_statemutex); } return (0); } static int bd_flush_write_cache_done(struct buf *bp) { struct dk_callback *dc = (void *)bp->b_private; (*dc->dkc_callback)(dc->dkc_cookie, geterror(bp)); kmem_free(dc, sizeof (*dc)); freerbuf(bp); return (0); } static int bd_flush_write_cache(bd_t *bd, struct dk_callback *dkc) { buf_t *bp; struct dk_callback *dc; bd_xfer_impl_t *xi; int rv; if (bd->d_ops.o_sync_cache == NULL) { return (ENOTSUP); } if ((bp = getrbuf(KM_SLEEP)) == NULL) { return (ENOMEM); } bp->b_resid = 0; bp->b_bcount = 0; xi = bd_xfer_alloc(bd, bp, bd->d_ops.o_sync_cache, KM_SLEEP); if (xi == NULL) { rv = geterror(bp); freerbuf(bp); return (rv); } /* Make an asynchronous flush, but only if there is a callback */ if (dkc != NULL && dkc->dkc_callback != NULL) { /* Make a private copy of the callback structure */ dc = kmem_alloc(sizeof (*dc), KM_SLEEP); *dc = *dkc; bp->b_private = dc; bp->b_iodone = bd_flush_write_cache_done; bd_submit(bd, xi); return (0); } /* In case there is no callback, perform a synchronous flush */ bd_submit(bd, xi); (void) biowait(bp); rv = geterror(bp); freerbuf(bp); return (rv); } static int bd_free_space_done(struct buf *bp) { freerbuf(bp); return (0); } static int bd_free_space_cb(dkioc_free_list_t *dfl, void *arg, int kmflag) { bd_t *bd = arg; buf_t *bp = NULL; bd_xfer_impl_t *xi = NULL; boolean_t sync = DFL_ISSYNC(dfl) ? B_TRUE : B_FALSE; int rv = 0; bp = getrbuf(KM_SLEEP); bp->b_resid = 0; bp->b_bcount = 0; bp->b_lblkno = 0; xi = bd_xfer_alloc(bd, bp, bd->d_ops.o_free_space, kmflag); xi->i_dfl = dfl; if (!sync) { bp->b_iodone = bd_free_space_done; bd_submit(bd, xi); return (0); } xi->i_flags |= BD_XFER_POLL; bd_submit(bd, xi); (void) biowait(bp); rv = geterror(bp); freerbuf(bp); return (rv); } static int bd_free_space(dev_t dev, bd_t *bd, dkioc_free_list_t *dfl) { diskaddr_t p_len, p_offset; uint64_t offset_bytes, len_bytes; minor_t part = BDPART(dev); const uint_t bshift = bd->d_blkshift; dkioc_free_info_t dfi = { .dfi_bshift = bshift, .dfi_align = bd->d_free_align << bshift, .dfi_max_bytes = bd->d_max_free_blks << bshift, .dfi_max_ext = bd->d_max_free_seg, .dfi_max_ext_bytes = bd->d_max_free_seg_blks << bshift, }; if (cmlb_partinfo(bd->d_cmlbh, part, &p_len, &p_offset, NULL, NULL, 0) != 0) { dfl_free(dfl); return (ENXIO); } /* * bd_ioctl created our own copy of dfl, so we can modify as * necessary */ offset_bytes = (uint64_t)p_offset << bshift; len_bytes = (uint64_t)p_len << bshift; dfl->dfl_offset += offset_bytes; if (dfl->dfl_offset < offset_bytes) { dfl_free(dfl); return (EOVERFLOW); } return (dfl_iter(dfl, &dfi, offset_bytes + len_bytes, bd_free_space_cb, bd, KM_SLEEP)); } /* * Nexus support. */ int bd_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, void *arg, void *result) { bd_handle_t hdl; switch (ctlop) { case DDI_CTLOPS_REPORTDEV: cmn_err(CE_CONT, "?Block device: %s@%s, %s%d\n", ddi_node_name(rdip), ddi_get_name_addr(rdip), ddi_driver_name(rdip), ddi_get_instance(rdip)); return (DDI_SUCCESS); case DDI_CTLOPS_INITCHILD: hdl = ddi_get_parent_data((dev_info_t *)arg); if (hdl == NULL) { return (DDI_NOT_WELL_FORMED); } ddi_set_name_addr((dev_info_t *)arg, hdl->h_addr); return (DDI_SUCCESS); case DDI_CTLOPS_UNINITCHILD: ddi_set_name_addr((dev_info_t *)arg, NULL); ndi_prop_remove_all((dev_info_t *)arg); return (DDI_SUCCESS); default: return (ddi_ctlops(dip, rdip, ctlop, arg, result)); } } /* * Functions for device drivers. */ bd_handle_t bd_alloc_handle(void *private, bd_ops_t *ops, ddi_dma_attr_t *dma, int kmflag) { bd_handle_t hdl; switch (ops->o_version) { case BD_OPS_VERSION_0: case BD_OPS_VERSION_1: case BD_OPS_VERSION_2: break; default: /* Unsupported version */ return (NULL); } hdl = kmem_zalloc(sizeof (*hdl), kmflag); if (hdl == NULL) { return (NULL); } switch (ops->o_version) { case BD_OPS_VERSION_2: hdl->h_ops.o_free_space = ops->o_free_space; /*FALLTHRU*/ case BD_OPS_VERSION_1: case BD_OPS_VERSION_0: hdl->h_ops.o_drive_info = ops->o_drive_info; hdl->h_ops.o_media_info = ops->o_media_info; hdl->h_ops.o_devid_init = ops->o_devid_init; hdl->h_ops.o_sync_cache = ops->o_sync_cache; hdl->h_ops.o_read = ops->o_read; hdl->h_ops.o_write = ops->o_write; break; } hdl->h_dma = dma; hdl->h_private = private; return (hdl); } void bd_free_handle(bd_handle_t hdl) { kmem_free(hdl, sizeof (*hdl)); } int bd_attach_handle(dev_info_t *dip, bd_handle_t hdl) { bd_drive_t drive = { 0 }; dev_info_t *child; size_t len; /* * It's not an error if bd_attach_handle() is called on a handle that * already is attached. We just ignore the request to attach and return. * This way drivers using blkdev don't have to keep track about blkdev * state, they can just call this function to make sure it attached. */ if (hdl->h_child != NULL) { return (DDI_SUCCESS); } /* if drivers don't override this, make it assume none */ drive.d_lun = -1; hdl->h_ops.o_drive_info(hdl->h_private, &drive); hdl->h_parent = dip; hdl->h_name = "blkdev"; /* * Prefer the GUID over the EUI64. */ if (*(uint64_t *)drive.d_guid != 0 || *((uint64_t *)drive.d_guid + 1) != 0) { len = snprintf(hdl->h_addr, sizeof (hdl->h_addr), "w%02X%02X%02X%02X%02X%02X%02X%02X" "%02X%02X%02X%02X%02X%02X%02X%02X", drive.d_guid[0], drive.d_guid[1], drive.d_guid[2], drive.d_guid[3], drive.d_guid[4], drive.d_guid[5], drive.d_guid[6], drive.d_guid[7], drive.d_guid[8], drive.d_guid[9], drive.d_guid[10], drive.d_guid[11], drive.d_guid[12], drive.d_guid[13], drive.d_guid[14], drive.d_guid[15]); } else if (*(uint64_t *)drive.d_eui64 != 0) { len = snprintf(hdl->h_addr, sizeof (hdl->h_addr), "w%02X%02X%02X%02X%02X%02X%02X%02X", drive.d_eui64[0], drive.d_eui64[1], drive.d_eui64[2], drive.d_eui64[3], drive.d_eui64[4], drive.d_eui64[5], drive.d_eui64[6], drive.d_eui64[7]); } else { len = snprintf(hdl->h_addr, sizeof (hdl->h_addr), "%X", drive.d_target); } VERIFY(len <= sizeof (hdl->h_addr)); if (drive.d_lun >= 0) { (void) snprintf(hdl->h_addr + len, sizeof (hdl->h_addr) - len, ",%X", drive.d_lun); } if (ndi_devi_alloc(dip, hdl->h_name, (pnode_t)DEVI_SID_NODEID, &child) != NDI_SUCCESS) { cmn_err(CE_WARN, "%s%d: unable to allocate node %s@%s", ddi_driver_name(dip), ddi_get_instance(dip), "blkdev", hdl->h_addr); return (DDI_FAILURE); } ddi_set_parent_data(child, hdl); hdl->h_child = child; if (ndi_devi_online(child, 0) != NDI_SUCCESS) { cmn_err(CE_WARN, "%s%d: failed bringing node %s@%s online", ddi_driver_name(dip), ddi_get_instance(dip), hdl->h_name, hdl->h_addr); (void) ndi_devi_free(child); hdl->h_child = NULL; return (DDI_FAILURE); } return (DDI_SUCCESS); } int bd_detach_handle(bd_handle_t hdl) { int circ; int rv; char *devnm; /* * It's not an error if bd_detach_handle() is called on a handle that * already is detached. We just ignore the request to detach and return. * This way drivers using blkdev don't have to keep track about blkdev * state, they can just call this function to make sure it detached. */ if (hdl->h_child == NULL) { return (DDI_SUCCESS); } ndi_devi_enter(hdl->h_parent, &circ); if (i_ddi_node_state(hdl->h_child) < DS_INITIALIZED) { rv = ddi_remove_child(hdl->h_child, 0); } else { devnm = kmem_alloc(MAXNAMELEN + 1, KM_SLEEP); (void) ddi_deviname(hdl->h_child, devnm); (void) devfs_clean(hdl->h_parent, devnm + 1, DV_CLEAN_FORCE); rv = ndi_devi_unconfig_one(hdl->h_parent, devnm + 1, NULL, NDI_DEVI_REMOVE | NDI_UNCONFIG); kmem_free(devnm, MAXNAMELEN + 1); } if (rv == 0) { hdl->h_child = NULL; } ndi_devi_exit(hdl->h_parent, circ); return (rv == NDI_SUCCESS ? DDI_SUCCESS : DDI_FAILURE); } void bd_xfer_done(bd_xfer_t *xfer, int err) { bd_xfer_impl_t *xi = (void *)xfer; buf_t *bp = xi->i_bp; int rv = DDI_SUCCESS; bd_t *bd = xi->i_bd; size_t len; if (err != 0) { bd_runq_exit(xi, err); atomic_inc_32(&bd->d_kerr->bd_harderrs.value.ui32); bp->b_resid += xi->i_resid; bd_xfer_free(xi); bioerror(bp, err); biodone(bp); return; } xi->i_cur_win++; xi->i_resid -= xi->i_len; if (xi->i_resid == 0) { /* Job completed succcessfully! */ bd_runq_exit(xi, 0); bd_xfer_free(xi); biodone(bp); return; } xi->i_blkno += xi->i_nblks; if (bd->d_use_dma) { /* More transfer still pending... advance to next DMA window. */ rv = ddi_dma_getwin(xi->i_dmah, xi->i_cur_win, &xi->i_offset, &len, &xi->i_dmac, &xi->i_ndmac); } else { /* Advance memory window. */ xi->i_kaddr += xi->i_len; xi->i_offset += xi->i_len; len = min(bp->b_bcount - xi->i_offset, bd->d_maxxfer); } if ((rv != DDI_SUCCESS) || (P2PHASE(len, (1U << xi->i_blkshift)) != 0)) { bd_runq_exit(xi, EFAULT); bp->b_resid += xi->i_resid; bd_xfer_free(xi); bioerror(bp, EFAULT); biodone(bp); return; } xi->i_len = len; xi->i_nblks = len >> xi->i_blkshift; /* Submit next window to hardware. */ rv = xi->i_func(bd->d_private, &xi->i_public); if (rv != 0) { bd_runq_exit(xi, rv); atomic_inc_32(&bd->d_kerr->bd_transerrs.value.ui32); bp->b_resid += xi->i_resid; bd_xfer_free(xi); bioerror(bp, rv); biodone(bp); } } void bd_error(bd_xfer_t *xfer, int error) { bd_xfer_impl_t *xi = (void *)xfer; bd_t *bd = xi->i_bd; switch (error) { case BD_ERR_MEDIA: atomic_inc_32(&bd->d_kerr->bd_rq_media_err.value.ui32); break; case BD_ERR_NTRDY: atomic_inc_32(&bd->d_kerr->bd_rq_ntrdy_err.value.ui32); break; case BD_ERR_NODEV: atomic_inc_32(&bd->d_kerr->bd_rq_nodev_err.value.ui32); break; case BD_ERR_RECOV: atomic_inc_32(&bd->d_kerr->bd_rq_recov_err.value.ui32); break; case BD_ERR_ILLRQ: atomic_inc_32(&bd->d_kerr->bd_rq_illrq_err.value.ui32); break; case BD_ERR_PFA: atomic_inc_32(&bd->d_kerr->bd_rq_pfa_err.value.ui32); break; default: cmn_err(CE_PANIC, "bd_error: unknown error type %d", error); break; } } void bd_state_change(bd_handle_t hdl) { bd_t *bd; if ((bd = hdl->h_bd) != NULL) { bd_update_state(bd); } } void bd_mod_init(struct dev_ops *devops) { static struct bus_ops bd_bus_ops = { BUSO_REV, /* busops_rev */ nullbusmap, /* bus_map */ NULL, /* bus_get_intrspec (OBSOLETE) */ NULL, /* bus_add_intrspec (OBSOLETE) */ NULL, /* bus_remove_intrspec (OBSOLETE) */ i_ddi_map_fault, /* bus_map_fault */ NULL, /* bus_dma_map (OBSOLETE) */ ddi_dma_allochdl, /* bus_dma_allochdl */ ddi_dma_freehdl, /* bus_dma_freehdl */ ddi_dma_bindhdl, /* bus_dma_bindhdl */ ddi_dma_unbindhdl, /* bus_dma_unbindhdl */ ddi_dma_flush, /* bus_dma_flush */ ddi_dma_win, /* bus_dma_win */ ddi_dma_mctl, /* bus_dma_ctl */ bd_bus_ctl, /* bus_ctl */ ddi_bus_prop_op, /* bus_prop_op */ NULL, /* bus_get_eventcookie */ NULL, /* bus_add_eventcall */ NULL, /* bus_remove_eventcall */ NULL, /* bus_post_event */ NULL, /* bus_intr_ctl (OBSOLETE) */ NULL, /* bus_config */ NULL, /* bus_unconfig */ NULL, /* bus_fm_init */ NULL, /* bus_fm_fini */ NULL, /* bus_fm_access_enter */ NULL, /* bus_fm_access_exit */ NULL, /* bus_power */ NULL, /* bus_intr_op */ }; devops->devo_bus_ops = &bd_bus_ops; /* * NB: The device driver is free to supply its own * character entry device support. */ } void bd_mod_fini(struct dev_ops *devops) { devops->devo_bus_ops = NULL; }