1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2024 Racktop Systems, Inc.
14 */
15
16 /*
17 * This file implementes the iport and tgtmap for physical devices on lmrc.
18 *
19 * When the phys iport is attached, a FULLSET tgtmap is created for physical
20 * devices (PDs).
21 *
22 * During attach or as a result of an async event received from the hardware,
23 * we'll get the PD list from the HBA and populate the tgtmap with what we have
24 * found. The PD list includes the SAS WWN of each device found, which we will
25 * use for the unit address.
26 *
27 * In the target activation callback, we'll retrieve the PD info from the HBA
28 * and pass it to lmrc_tgt_init(). This contains additional information such as
29 * the device and interconnect types.
30 */
31
32 #include <sys/ddi.h>
33 #include <sys/sunddi.h>
34
35 #include "lmrc.h"
36 #include "lmrc_reg.h"
37 #include "lmrc_raid.h"
38 #include "lmrc_phys.h"
39
40 static int lmrc_get_pdmap(lmrc_t *, lmrc_pd_map_t **);
41 static int lmrc_sync_pdmap(lmrc_t *, size_t);
42 static void lmrc_complete_sync_pdmap(lmrc_t *, lmrc_mfi_cmd_t *);
43
44 static lmrc_pd_info_t *lmrc_get_pd_info(lmrc_t *, uint16_t);
45 static void lmrc_phys_tgt_activate_cb(void *, char *, scsi_tgtmap_tgt_type_t,
46 void **);
47 static boolean_t lmrc_phys_tgt_deactivate_cb(void *, char *,
48 scsi_tgtmap_tgt_type_t, void *, scsi_tgtmap_deact_rsn_t);
49 static int lmrc_phys_update_tgtmap(lmrc_t *, lmrc_pd_list_t *);
50
51 /*
52 * lmrc_get_pdmap
53 *
54 * Get the physical device map from the firmware. Return a minimally sized copy.
55 */
56 static int
lmrc_get_pdmap(lmrc_t * lmrc,lmrc_pd_map_t ** pdmap)57 lmrc_get_pdmap(lmrc_t *lmrc, lmrc_pd_map_t **pdmap)
58 {
59 uint32_t pdmap_sz = sizeof (lmrc_pd_map_t) +
60 sizeof (lmrc_pd_cfg_t) * LMRC_MAX_PHYS_DEV;
61 lmrc_mfi_cmd_t *mfi;
62 lmrc_pd_map_t *pm;
63 int ret;
64
65 mfi = lmrc_get_dcmd(lmrc, MFI_FRAME_DIR_READ,
66 LMRC_DCMD_SYSTEM_PD_MAP_GET_INFO, pdmap_sz, 4);
67
68 if (mfi == NULL)
69 return (DDI_FAILURE);
70
71 ret = lmrc_issue_blocked_mfi(lmrc, mfi);
72
73 if (ret != DDI_SUCCESS)
74 goto out;
75
76 pm = mfi->mfi_data_dma.ld_buf;
77
78 if (pm->pm_count > LMRC_MAX_PHYS_DEV) {
79 dev_err(lmrc->l_dip, CE_WARN,
80 "!FW reports too many PDs: %d", pm->pm_count);
81 ret = DDI_FAILURE;
82 goto out;
83 }
84
85 pdmap_sz = sizeof (lmrc_pd_map_t) +
86 pm->pm_count * sizeof (lmrc_pd_cfg_t);
87 *pdmap = kmem_zalloc(pdmap_sz, KM_SLEEP);
88 bcopy(pm, *pdmap, pdmap_sz);
89
90 out:
91 lmrc_put_dcmd(lmrc, mfi);
92 return (ret);
93 }
94
95 /*
96 * lmrc_sync_pdmap
97 *
98 * Get the physical device map to the firmware. The command will complete
99 * when the firmware detects a change.
100 *
101 * mbox byte values:
102 * [0]: PEND_FLAG, delay completion until a config change pending
103 */
104 static int
lmrc_sync_pdmap(lmrc_t * lmrc,size_t pd_count)105 lmrc_sync_pdmap(lmrc_t *lmrc, size_t pd_count)
106 {
107 uint32_t pdmap_sz = sizeof (lmrc_pd_map_t) +
108 pd_count * sizeof (lmrc_pd_cfg_t);
109 lmrc_mfi_dcmd_payload_t *dcmd;
110 lmrc_mfi_cmd_t *mfi;
111
112 mfi = lmrc_get_dcmd(lmrc, MFI_FRAME_DIR_WRITE,
113 LMRC_DCMD_SYSTEM_PD_MAP_GET_INFO, pdmap_sz, 4);
114
115 if (mfi == NULL)
116 return (DDI_FAILURE);
117
118 dcmd = &mfi->mfi_frame->mf_dcmd;
119 dcmd->md_mbox_8[0] = LMRC_DCMD_MBOX_PEND_FLAG;
120
121 mutex_enter(&mfi->mfi_lock);
122 lmrc_issue_mfi(lmrc, mfi, lmrc_complete_sync_pdmap);
123 mutex_exit(&mfi->mfi_lock);
124
125 return (DDI_SUCCESS);
126 }
127
128 /*
129 * lmrc_complete_sync_pdmap
130 *
131 * The PDMAP GET INFO command completed, most likely due to the hardware
132 * detecting a change and informing us.
133 */
134 static void
lmrc_complete_sync_pdmap(lmrc_t * lmrc,lmrc_mfi_cmd_t * mfi)135 lmrc_complete_sync_pdmap(lmrc_t *lmrc, lmrc_mfi_cmd_t *mfi)
136 {
137 lmrc_mfi_header_t *hdr = &mfi->mfi_frame->mf_hdr;
138 lmrc_dma_t *dma = &mfi->mfi_data_dma;
139 lmrc_pd_map_t *pm = dma->ld_buf;
140 uint32_t pdmap_sz = sizeof (lmrc_pd_map_t) +
141 lmrc->l_pdmap->pm_count * sizeof (lmrc_pd_cfg_t);
142
143 ASSERT(mutex_owned(&mfi->mfi_lock));
144
145 if (hdr->mh_cmd_status != MFI_STAT_OK) {
146 /* Was the command aborted? */
147 if (hdr->mh_cmd_status == MFI_STAT_NOT_FOUND)
148 return;
149
150 /*
151 * In the case of any other error, log the error and schedule
152 * a taskq to clean up the command.
153 */
154 dev_err(lmrc->l_dip, CE_WARN,
155 "!PD map sync failed, status = %d",
156 hdr->mh_cmd_status);
157 lmrc->l_use_seqnum_jbod_fp = B_FALSE;
158 taskq_dispatch_ent(lmrc->l_taskq, (task_func_t *)lmrc_put_mfi,
159 mfi, TQ_NOSLEEP, &mfi->mfi_tqent);
160 return;
161 }
162
163 VERIFY3U(pdmap_sz, ==, dma->ld_len);
164
165 /* Update our copy of the pdmap and restart the command. */
166 rw_enter(&lmrc->l_pdmap_lock, RW_WRITER);
167 bcopy(pm, lmrc->l_pdmap, pdmap_sz);
168 rw_exit(&lmrc->l_pdmap_lock);
169 bzero(pm, pdmap_sz);
170 lmrc_issue_mfi(lmrc, mfi, lmrc_complete_sync_pdmap);
171 }
172
173 /*
174 * lmrc_setup_pdmap
175 *
176 * Get the physical device map from the firmware, and sync it back.
177 * Replace the copy in the soft state if successful.
178 */
179 int
lmrc_setup_pdmap(lmrc_t * lmrc)180 lmrc_setup_pdmap(lmrc_t *lmrc)
181 {
182 lmrc_pd_map_t *pdmap = NULL;
183 int ret;
184
185 ret = lmrc_get_pdmap(lmrc, &pdmap);
186 if (ret != DDI_SUCCESS)
187 return (ret);
188
189 rw_enter(&lmrc->l_pdmap_lock, RW_WRITER);
190 ASSERT(lmrc->l_pdmap == NULL);
191 lmrc->l_pdmap = pdmap;
192 rw_exit(&lmrc->l_pdmap_lock);
193
194 ret = lmrc_sync_pdmap(lmrc, pdmap->pm_count);
195 return (ret);
196 }
197
198 /*
199 * lmrc_free_pdmap
200 *
201 * Free the buffer used to hold the physical device map.
202 */
203 void
lmrc_free_pdmap(lmrc_t * lmrc)204 lmrc_free_pdmap(lmrc_t *lmrc)
205 {
206 if (lmrc->l_pdmap != NULL) {
207 uint32_t pdmap_sz = sizeof (lmrc_pd_map_t) +
208 lmrc->l_pdmap->pm_count * sizeof (lmrc_pd_cfg_t);
209 kmem_free(lmrc->l_pdmap, pdmap_sz);
210 lmrc->l_pdmap = NULL;
211 }
212 }
213
214 /*
215 * lmrc_pd_tm_capable
216 *
217 * Determine whether a PD can be sent TASK MGMT requests. By default we assume
218 * it can't, unless the the PD map indicates otherwise.
219 */
220 boolean_t
lmrc_pd_tm_capable(lmrc_t * lmrc,uint16_t tgtid)221 lmrc_pd_tm_capable(lmrc_t *lmrc, uint16_t tgtid)
222 {
223 boolean_t tm_capable = B_FALSE;
224
225 rw_enter(&lmrc->l_pdmap_lock, RW_READER);
226 if (lmrc->l_pdmap != NULL &&
227 lmrc->l_pdmap->pm_pdcfg[tgtid].pd_tgtid != LMRC_DEVHDL_INVALID &&
228 lmrc->l_pdmap->pm_pdcfg[tgtid].pd_tm_capable != 0)
229 tm_capable = B_TRUE;
230 rw_exit(&lmrc->l_pdmap_lock);
231
232 return (tm_capable);
233 }
234
235 /*
236 * lmrc_get_pd_info
237 *
238 * Get physical drive info from FW.
239 */
240 static lmrc_pd_info_t *
lmrc_get_pd_info(lmrc_t * lmrc,uint16_t dev_id)241 lmrc_get_pd_info(lmrc_t *lmrc, uint16_t dev_id)
242 {
243 lmrc_pd_info_t *pdinfo = NULL;
244 lmrc_mfi_cmd_t *mfi;
245 lmrc_mfi_dcmd_payload_t *dcmd;
246 int ret;
247
248 mfi = lmrc_get_dcmd(lmrc, MFI_FRAME_DIR_READ, LMRC_DCMD_PD_GET_INFO,
249 sizeof (lmrc_pd_info_t), 1);
250
251 if (mfi == NULL)
252 return (NULL);
253
254 dcmd = &mfi->mfi_frame->mf_dcmd;
255 dcmd->md_mbox_16[0] = dev_id;
256
257 ret = lmrc_issue_blocked_mfi(lmrc, mfi);
258
259 if (ret != DDI_SUCCESS)
260 goto out;
261
262 pdinfo = kmem_zalloc(sizeof (lmrc_pd_info_t), KM_SLEEP);
263 bcopy(mfi->mfi_data_dma.ld_buf, pdinfo, sizeof (lmrc_pd_info_t));
264
265 out:
266 lmrc_put_dcmd(lmrc, mfi);
267 return (pdinfo);
268 }
269
270 /*
271 * lmrc_phys_tgt_activate_cb
272 *
273 * Set up a tgt structure for a newly discovered PD.
274 */
275 static void
lmrc_phys_tgt_activate_cb(void * tgtmap_priv,char * tgt_addr,scsi_tgtmap_tgt_type_t type,void ** tgt_privp)276 lmrc_phys_tgt_activate_cb(void *tgtmap_priv, char *tgt_addr,
277 scsi_tgtmap_tgt_type_t type, void **tgt_privp)
278 {
279 lmrc_t *lmrc = tgtmap_priv;
280 lmrc_tgt_t *tgt = *tgt_privp;
281 uint16_t dev_id = tgt - lmrc->l_targets;
282 lmrc_pd_info_t *pd_info;
283
284 VERIFY(lmrc == tgt->tgt_lmrc);
285
286 dev_id -= LMRC_MAX_LD;
287
288 VERIFY3U(dev_id, <, LMRC_MAX_PD);
289
290 pd_info = lmrc_get_pd_info(lmrc, dev_id);
291 if (pd_info == NULL)
292 return;
293
294 lmrc_tgt_init(tgt, dev_id, tgt_addr, pd_info);
295 }
296
297 /*
298 * lmrc_phys_tgt_deactivate_cb
299 *
300 * Tear down the tgt structure of a PD that is no longer present.
301 */
302 static boolean_t
lmrc_phys_tgt_deactivate_cb(void * tgtmap_priv,char * tgt_addr,scsi_tgtmap_tgt_type_t type,void * tgt_priv,scsi_tgtmap_deact_rsn_t deact)303 lmrc_phys_tgt_deactivate_cb(void *tgtmap_priv, char *tgt_addr,
304 scsi_tgtmap_tgt_type_t type, void *tgt_priv, scsi_tgtmap_deact_rsn_t deact)
305 {
306 lmrc_t *lmrc = tgtmap_priv;
307 lmrc_tgt_t *tgt = tgt_priv;
308
309 VERIFY(lmrc == tgt->tgt_lmrc);
310
311 lmrc_tgt_clear(tgt);
312
313 return (B_FALSE);
314 }
315
316 /*
317 * lmrc_phys_update_tgtmap
318 *
319 * Feed the PD list into the target map.
320 */
321 static int
lmrc_phys_update_tgtmap(lmrc_t * lmrc,lmrc_pd_list_t * pd_list)322 lmrc_phys_update_tgtmap(lmrc_t *lmrc, lmrc_pd_list_t *pd_list)
323 {
324 int ret;
325 int i;
326
327 if (pd_list->pl_count > LMRC_MAX_PD)
328 return (DDI_FAILURE);
329
330 ret = scsi_hba_tgtmap_set_begin(lmrc->l_phys_tgtmap);
331 if (ret != DDI_SUCCESS)
332 return (ret);
333
334 for (i = 0; i < pd_list->pl_count; i++) {
335 lmrc_pd_addr_t *pa = &pd_list->pl_addr[i];
336 char name[SCSI_WWN_BUFLEN];
337
338 if (pa->pa_dev_id > LMRC_MAX_PHYS_DEV) {
339 dev_err(lmrc->l_dip, CE_WARN,
340 "!%s: invalid PD dev id %d", __func__,
341 pa->pa_dev_id);
342 goto fail;
343 }
344
345 if (scsi_wwn_to_wwnstr(pa->pa_sas_addr[0], 1, name) == NULL)
346 goto fail;
347
348 ret = scsi_hba_tgtmap_set_add(lmrc->l_phys_tgtmap,
349 SCSI_TGT_SCSI_DEVICE, name,
350 &lmrc->l_targets[pa->pa_dev_id + LMRC_MAX_LD]);
351
352 if (ret != DDI_SUCCESS)
353 goto fail;
354 }
355
356 return (scsi_hba_tgtmap_set_end(lmrc->l_phys_tgtmap, 0));
357
358 fail:
359 (void) scsi_hba_tgtmap_set_flush(lmrc->l_raid_tgtmap);
360 return (DDI_FAILURE);
361 }
362
363 /*
364 * lmrc_get_pd_list
365 *
366 * Get the list of physical devices from the firmware and update the target map.
367 */
368 int
lmrc_get_pd_list(lmrc_t * lmrc)369 lmrc_get_pd_list(lmrc_t *lmrc)
370 {
371 lmrc_mfi_cmd_t *mfi;
372 lmrc_mfi_dcmd_payload_t *dcmd;
373 int ret;
374
375 /* If the phys iport isn't attached yet, just return success. */
376 if (!INITLEVEL_ACTIVE(lmrc, LMRC_INITLEVEL_PHYS))
377 return (DDI_SUCCESS);
378
379 mfi = lmrc_get_dcmd(lmrc, MFI_FRAME_DIR_READ, LMRC_DCMD_PD_LIST_QUERY,
380 sizeof (lmrc_pd_list_t) + sizeof (lmrc_pd_addr_t) * LMRC_MAX_PD, 1);
381
382 if (mfi == NULL)
383 return (DDI_FAILURE);
384
385 dcmd = &mfi->mfi_frame->mf_dcmd;
386 dcmd->md_mbox_8[0] = LMRC_PD_QUERY_TYPE_EXPOSED_TO_HOST;
387
388 ret = lmrc_issue_blocked_mfi(lmrc, mfi);
389
390 if (ret != DDI_SUCCESS)
391 goto out;
392
393 ret = lmrc_phys_update_tgtmap(lmrc, mfi->mfi_data_dma.ld_buf);
394
395 out:
396 lmrc_put_dcmd(lmrc, mfi);
397 return (ret);
398 }
399
400 /*
401 * lmrc_phys_aen_handler
402 *
403 * Handle AENs with locale code LMRC_EVT_LOCALE_PD. If the PD configuration
404 * changed, update the PD list and target map.
405 */
406
407 int
lmrc_phys_aen_handler(lmrc_t * lmrc,lmrc_evt_t * evt)408 lmrc_phys_aen_handler(lmrc_t *lmrc, lmrc_evt_t *evt)
409 {
410 int ret = DDI_SUCCESS;
411
412 switch (evt->evt_code) {
413 case LMRC_EVT_PD_INSERTED:
414 case LMRC_EVT_PD_REMOVED:
415 case LMRC_EVT_PD_CHANGED:
416 /*
417 * For any change w.r.t. the PDs, refresh the PD list.
418 */
419 ret = lmrc_get_pd_list(lmrc);
420 break;
421
422 case LMRC_EVT_PD_PATROL_READ_PROGRESS:
423 case LMRC_EVT_PD_RESET:
424 break;
425
426 default:
427 ret = DDI_FAILURE;
428 }
429
430 return (ret);
431 }
432
433 int
lmrc_phys_attach(dev_info_t * dip)434 lmrc_phys_attach(dev_info_t *dip)
435 {
436 scsi_hba_tran_t *tran = ddi_get_driver_private(dip);
437 dev_info_t *pdip = ddi_get_parent(dip);
438 lmrc_t *lmrc = ddi_get_soft_state(lmrc_state, ddi_get_instance(pdip));
439 int ret;
440
441 VERIFY(tran != NULL);
442 VERIFY(lmrc != NULL);
443
444 if (lmrc->l_fw_fault)
445 return (DDI_FAILURE);
446
447 tran->tran_hba_private = lmrc;
448 lmrc->l_phys_dip = dip;
449
450 ret = scsi_hba_tgtmap_create(dip, SCSI_TM_FULLSET, MICROSEC,
451 2 * MICROSEC, lmrc, lmrc_phys_tgt_activate_cb,
452 lmrc_phys_tgt_deactivate_cb, &lmrc->l_phys_tgtmap);
453 if (ret != DDI_SUCCESS)
454 return (DDI_FAILURE);
455
456 if (lmrc->l_use_seqnum_jbod_fp)
457 if (lmrc_setup_pdmap(lmrc) != DDI_SUCCESS)
458 lmrc->l_use_seqnum_jbod_fp = B_FALSE;
459
460 INITLEVEL_SET(lmrc, LMRC_INITLEVEL_PHYS);
461
462 ret = lmrc_get_pd_list(lmrc);
463 if (ret != DDI_SUCCESS) {
464 dev_err(lmrc->l_dip, CE_WARN, "!Failed to get PD list.");
465 return (ret);
466 }
467
468 return (DDI_SUCCESS);
469 }
470
471 int
lmrc_phys_detach(dev_info_t * dip)472 lmrc_phys_detach(dev_info_t *dip)
473 {
474 dev_info_t *pdip = ddi_get_parent(dip);
475 lmrc_t *lmrc = ddi_get_soft_state(lmrc_state, ddi_get_instance(pdip));
476
477 VERIFY(lmrc != NULL);
478 INITLEVEL_CLEAR(lmrc, LMRC_INITLEVEL_PHYS);
479
480 if (lmrc->l_phys_tgtmap != NULL) {
481 scsi_hba_tgtmap_destroy(lmrc->l_phys_tgtmap);
482 lmrc->l_phys_tgtmap = NULL;
483 }
484
485 lmrc->l_phys_dip = NULL;
486
487 return (DDI_SUCCESS);
488 }
489