1*a23fd118Syl /* 2*a23fd118Syl * CDDL HEADER START 3*a23fd118Syl * 4*a23fd118Syl * The contents of this file are subject to the terms of the 5*a23fd118Syl * Common Development and Distribution License (the "License"). 6*a23fd118Syl * You may not use this file except in compliance with the License. 7*a23fd118Syl * 8*a23fd118Syl * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9*a23fd118Syl * or http://www.opensolaris.org/os/licensing. 10*a23fd118Syl * See the License for the specific language governing permissions 11*a23fd118Syl * and limitations under the License. 12*a23fd118Syl * 13*a23fd118Syl * When distributing Covered Code, include this CDDL HEADER in each 14*a23fd118Syl * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15*a23fd118Syl * If applicable, add the following below this CDDL HEADER, with the 16*a23fd118Syl * fields enclosed by brackets "[]" replaced with your own identifying 17*a23fd118Syl * information: Portions Copyright [yyyy] [name of copyright owner] 18*a23fd118Syl * 19*a23fd118Syl * CDDL HEADER END 20*a23fd118Syl */ 21*a23fd118Syl 22*a23fd118Syl /* 23*a23fd118Syl * Copyright (c) 2002-2005 Neterion, Inc. 24*a23fd118Syl * All right Reserved. 25*a23fd118Syl * 26*a23fd118Syl * FileName : hal-mm.c 27*a23fd118Syl * 28*a23fd118Syl * Description: chipset memory pool object implementation 29*a23fd118Syl * 30*a23fd118Syl * Created: 10 May 2004 31*a23fd118Syl */ 32*a23fd118Syl 33*a23fd118Syl #include "xge-os-pal.h" 34*a23fd118Syl #include "xgehal-mm.h" 35*a23fd118Syl #include "xge-debug.h" 36*a23fd118Syl 37*a23fd118Syl /* 38*a23fd118Syl * __hal_mempool_grow 39*a23fd118Syl * 40*a23fd118Syl * Will resize mempool up to %num_allocate value. 41*a23fd118Syl */ 42*a23fd118Syl xge_hal_status_e 43*a23fd118Syl __hal_mempool_grow(xge_hal_mempool_t *mempool, int num_allocate, 44*a23fd118Syl int *num_allocated) 45*a23fd118Syl { 46*a23fd118Syl int i, first_time = mempool->memblocks_allocated == 0 ? 1 : 0; 47*a23fd118Syl int n_items = mempool->items_per_memblock; 48*a23fd118Syl 49*a23fd118Syl *num_allocated = 0; 50*a23fd118Syl 51*a23fd118Syl if ((mempool->memblocks_allocated + num_allocate) > 52*a23fd118Syl mempool->memblocks_max) { 53*a23fd118Syl xge_debug_mm(XGE_ERR, "%s", 54*a23fd118Syl "__hal_mempool_grow: can grow anymore"); 55*a23fd118Syl return XGE_HAL_ERR_OUT_OF_MEMORY; 56*a23fd118Syl } 57*a23fd118Syl 58*a23fd118Syl for (i = mempool->memblocks_allocated; 59*a23fd118Syl i < mempool->memblocks_allocated + num_allocate; i++) { 60*a23fd118Syl int j; 61*a23fd118Syl int is_last = 62*a23fd118Syl ((mempool->memblocks_allocated+num_allocate-1) == i); 63*a23fd118Syl xge_hal_mempool_dma_t *dma_object = 64*a23fd118Syl mempool->memblocks_dma_arr + i; 65*a23fd118Syl void *the_memblock; 66*a23fd118Syl int dma_flags; 67*a23fd118Syl 68*a23fd118Syl dma_flags = XGE_OS_DMA_CACHELINE_ALIGNED; 69*a23fd118Syl #ifdef XGE_HAL_DMA_DTR_CONSISTENT 70*a23fd118Syl dma_flags |= XGE_OS_DMA_CONSISTENT; 71*a23fd118Syl #else 72*a23fd118Syl dma_flags |= XGE_OS_DMA_STREAMING; 73*a23fd118Syl #endif 74*a23fd118Syl 75*a23fd118Syl /* allocate DMA-capable memblock */ 76*a23fd118Syl mempool->memblocks_arr[i] = xge_os_dma_malloc(mempool->pdev, 77*a23fd118Syl mempool->memblock_size, 78*a23fd118Syl dma_flags, 79*a23fd118Syl &dma_object->handle, 80*a23fd118Syl &dma_object->acc_handle); 81*a23fd118Syl if (mempool->memblocks_arr[i] == NULL) { 82*a23fd118Syl xge_debug_mm(XGE_ERR, 83*a23fd118Syl "memblock[%d]: out of DMA memory", i); 84*a23fd118Syl return XGE_HAL_ERR_OUT_OF_MEMORY; 85*a23fd118Syl } 86*a23fd118Syl xge_os_memzero(mempool->memblocks_arr[i], 87*a23fd118Syl mempool->memblock_size); 88*a23fd118Syl the_memblock = mempool->memblocks_arr[i]; 89*a23fd118Syl 90*a23fd118Syl /* allocate memblock's private part. Each DMA memblock 91*a23fd118Syl * has a space allocated for item's private usage upon 92*a23fd118Syl * mempool's user request. Each time mempool grows, it will 93*a23fd118Syl * allocate new memblock and its private part at once. 94*a23fd118Syl * This helps to minimize memory usage a lot. */ 95*a23fd118Syl mempool->memblocks_priv_arr[i] = xge_os_malloc(mempool->pdev, 96*a23fd118Syl mempool->items_priv_size * n_items); 97*a23fd118Syl if (mempool->memblocks_priv_arr[i] == NULL) { 98*a23fd118Syl xge_os_dma_free(mempool->pdev, 99*a23fd118Syl the_memblock, 100*a23fd118Syl mempool->memblock_size, 101*a23fd118Syl &dma_object->acc_handle, 102*a23fd118Syl &dma_object->handle); 103*a23fd118Syl xge_debug_mm(XGE_ERR, 104*a23fd118Syl "memblock_priv[%d]: out of virtual memory, " 105*a23fd118Syl "requested %d(%d:%d) bytes", i, 106*a23fd118Syl mempool->items_priv_size * n_items, 107*a23fd118Syl mempool->items_priv_size, n_items); 108*a23fd118Syl return XGE_HAL_ERR_OUT_OF_MEMORY; 109*a23fd118Syl } 110*a23fd118Syl xge_os_memzero(mempool->memblocks_priv_arr[i], 111*a23fd118Syl mempool->items_priv_size * n_items); 112*a23fd118Syl 113*a23fd118Syl /* map memblock to physical memory */ 114*a23fd118Syl dma_object->addr = xge_os_dma_map(mempool->pdev, 115*a23fd118Syl dma_object->handle, 116*a23fd118Syl the_memblock, 117*a23fd118Syl mempool->memblock_size, 118*a23fd118Syl XGE_OS_DMA_DIR_BIDIRECTIONAL, 119*a23fd118Syl #ifdef XGE_HAL_DMA_DTR_CONSISTENT 120*a23fd118Syl XGE_OS_DMA_CONSISTENT 121*a23fd118Syl #else 122*a23fd118Syl XGE_OS_DMA_STREAMING 123*a23fd118Syl #endif 124*a23fd118Syl ); 125*a23fd118Syl if (dma_object->addr == XGE_OS_INVALID_DMA_ADDR) { 126*a23fd118Syl xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i], 127*a23fd118Syl mempool->items_priv_size * 128*a23fd118Syl n_items); 129*a23fd118Syl xge_os_dma_free(mempool->pdev, 130*a23fd118Syl the_memblock, 131*a23fd118Syl mempool->memblock_size, 132*a23fd118Syl &dma_object->acc_handle, 133*a23fd118Syl &dma_object->handle); 134*a23fd118Syl return XGE_HAL_ERR_OUT_OF_MAPPING; 135*a23fd118Syl } 136*a23fd118Syl 137*a23fd118Syl /* fill the items hash array */ 138*a23fd118Syl for (j=0; j<n_items; j++) { 139*a23fd118Syl int index = i*n_items + j; 140*a23fd118Syl 141*a23fd118Syl if (first_time && index >= mempool->items_initial) { 142*a23fd118Syl break; 143*a23fd118Syl } 144*a23fd118Syl 145*a23fd118Syl mempool->items_arr[index] = 146*a23fd118Syl ((char *)the_memblock + j*mempool->item_size); 147*a23fd118Syl 148*a23fd118Syl /* let caller to do more job on each item */ 149*a23fd118Syl if (mempool->item_func_alloc != NULL) { 150*a23fd118Syl xge_hal_status_e status; 151*a23fd118Syl 152*a23fd118Syl if ((status = mempool->item_func_alloc( 153*a23fd118Syl mempool, 154*a23fd118Syl the_memblock, 155*a23fd118Syl i, 156*a23fd118Syl dma_object, 157*a23fd118Syl mempool->items_arr[index], 158*a23fd118Syl index, 159*a23fd118Syl is_last, 160*a23fd118Syl mempool->userdata)) != XGE_HAL_OK) { 161*a23fd118Syl 162*a23fd118Syl if (mempool->item_func_free != NULL) { 163*a23fd118Syl int k; 164*a23fd118Syl 165*a23fd118Syl for (k=0; k<j; k++) { 166*a23fd118Syl 167*a23fd118Syl index =i*n_items + k; 168*a23fd118Syl 169*a23fd118Syl (void)mempool->item_func_free( 170*a23fd118Syl mempool, the_memblock, 171*a23fd118Syl i, dma_object, 172*a23fd118Syl mempool->items_arr[index], 173*a23fd118Syl index, is_last, 174*a23fd118Syl mempool->userdata); 175*a23fd118Syl } 176*a23fd118Syl } 177*a23fd118Syl 178*a23fd118Syl xge_os_free(mempool->pdev, 179*a23fd118Syl mempool->memblocks_priv_arr[i], 180*a23fd118Syl mempool->items_priv_size * 181*a23fd118Syl n_items); 182*a23fd118Syl xge_os_dma_unmap(mempool->pdev, 183*a23fd118Syl dma_object->handle, 184*a23fd118Syl dma_object->addr, 185*a23fd118Syl mempool->memblock_size, 186*a23fd118Syl XGE_OS_DMA_DIR_BIDIRECTIONAL); 187*a23fd118Syl xge_os_dma_free(mempool->pdev, 188*a23fd118Syl the_memblock, 189*a23fd118Syl mempool->memblock_size, 190*a23fd118Syl &dma_object->acc_handle, 191*a23fd118Syl &dma_object->handle); 192*a23fd118Syl return status; 193*a23fd118Syl } 194*a23fd118Syl } 195*a23fd118Syl 196*a23fd118Syl mempool->items_current = index + 1; 197*a23fd118Syl } 198*a23fd118Syl 199*a23fd118Syl xge_debug_mm(XGE_TRACE, 200*a23fd118Syl "memblock%d: allocated %dk, vaddr 0x%llx, " 201*a23fd118Syl "dma_addr 0x%llx", i, mempool->memblock_size / 1024, 202*a23fd118Syl (unsigned long long)(ulong_t)mempool->memblocks_arr[i], 203*a23fd118Syl (unsigned long long)dma_object->addr); 204*a23fd118Syl 205*a23fd118Syl (*num_allocated)++; 206*a23fd118Syl 207*a23fd118Syl if (first_time && mempool->items_current == 208*a23fd118Syl mempool->items_initial) { 209*a23fd118Syl break; 210*a23fd118Syl } 211*a23fd118Syl } 212*a23fd118Syl 213*a23fd118Syl /* increment actual number of allocated memblocks */ 214*a23fd118Syl mempool->memblocks_allocated += *num_allocated; 215*a23fd118Syl 216*a23fd118Syl return XGE_HAL_OK; 217*a23fd118Syl } 218*a23fd118Syl 219*a23fd118Syl /* 220*a23fd118Syl * xge_hal_mempool_create 221*a23fd118Syl * @memblock_size: 222*a23fd118Syl * @items_initial: 223*a23fd118Syl * @items_max: 224*a23fd118Syl * @item_size: 225*a23fd118Syl * @item_func: 226*a23fd118Syl * 227*a23fd118Syl * This function will create memory pool object. Pool may grow but will 228*a23fd118Syl * never shrink. Pool consists of number of dynamically allocated blocks 229*a23fd118Syl * with size enough to hold %items_initial number of items. Memory is 230*a23fd118Syl * DMA-able but client must map/unmap before interoperating with the device. 231*a23fd118Syl * See also: xge_os_dma_map(), xge_hal_dma_unmap(), xge_hal_status_e{}. 232*a23fd118Syl */ 233*a23fd118Syl xge_hal_mempool_t* 234*a23fd118Syl __hal_mempool_create(pci_dev_h pdev, int memblock_size, int item_size, 235*a23fd118Syl int items_priv_size, int items_initial, int items_max, 236*a23fd118Syl xge_hal_mempool_item_f item_func_alloc, 237*a23fd118Syl xge_hal_mempool_item_f item_func_free, void *userdata) 238*a23fd118Syl { 239*a23fd118Syl xge_hal_status_e status; 240*a23fd118Syl int memblocks_to_allocate; 241*a23fd118Syl xge_hal_mempool_t *mempool; 242*a23fd118Syl int allocated; 243*a23fd118Syl 244*a23fd118Syl if (memblock_size < item_size) { 245*a23fd118Syl xge_debug_mm(XGE_ERR, 246*a23fd118Syl "memblock_size %d < item_size %d: misconfiguration", 247*a23fd118Syl memblock_size, item_size); 248*a23fd118Syl return NULL; 249*a23fd118Syl } 250*a23fd118Syl 251*a23fd118Syl mempool = xge_os_malloc(pdev, sizeof(xge_hal_mempool_t)); 252*a23fd118Syl if (mempool == NULL) { 253*a23fd118Syl xge_debug_mm(XGE_ERR, "mempool allocation failure"); 254*a23fd118Syl return NULL; 255*a23fd118Syl } 256*a23fd118Syl xge_os_memzero(mempool, sizeof(xge_hal_mempool_t)); 257*a23fd118Syl 258*a23fd118Syl mempool->pdev = pdev; 259*a23fd118Syl mempool->memblock_size = memblock_size; 260*a23fd118Syl mempool->items_max = items_max; 261*a23fd118Syl mempool->items_initial = items_initial; 262*a23fd118Syl mempool->item_size = item_size; 263*a23fd118Syl mempool->items_priv_size = items_priv_size; 264*a23fd118Syl mempool->item_func_alloc = item_func_alloc; 265*a23fd118Syl mempool->item_func_free = item_func_free; 266*a23fd118Syl mempool->userdata = userdata; 267*a23fd118Syl 268*a23fd118Syl mempool->memblocks_allocated = 0; 269*a23fd118Syl 270*a23fd118Syl mempool->items_per_memblock = memblock_size / item_size; 271*a23fd118Syl 272*a23fd118Syl mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) / 273*a23fd118Syl mempool->items_per_memblock; 274*a23fd118Syl 275*a23fd118Syl /* allocate array of memblocks */ 276*a23fd118Syl mempool->memblocks_arr = xge_os_malloc(mempool->pdev, 277*a23fd118Syl sizeof(void*) * mempool->memblocks_max); 278*a23fd118Syl if (mempool->memblocks_arr == NULL) { 279*a23fd118Syl xge_debug_mm(XGE_ERR, "memblocks_arr allocation failure"); 280*a23fd118Syl __hal_mempool_destroy(mempool); 281*a23fd118Syl return NULL; 282*a23fd118Syl } 283*a23fd118Syl xge_os_memzero(mempool->memblocks_arr, 284*a23fd118Syl sizeof(void*) * mempool->memblocks_max); 285*a23fd118Syl 286*a23fd118Syl /* allocate array of private parts of items per memblocks */ 287*a23fd118Syl mempool->memblocks_priv_arr = xge_os_malloc(mempool->pdev, 288*a23fd118Syl sizeof(void*) * mempool->memblocks_max); 289*a23fd118Syl if (mempool->memblocks_priv_arr == NULL) { 290*a23fd118Syl xge_debug_mm(XGE_ERR, "memblocks_priv_arr allocation failure"); 291*a23fd118Syl __hal_mempool_destroy(mempool); 292*a23fd118Syl return NULL; 293*a23fd118Syl } 294*a23fd118Syl xge_os_memzero(mempool->memblocks_priv_arr, 295*a23fd118Syl sizeof(void*) * mempool->memblocks_max); 296*a23fd118Syl 297*a23fd118Syl /* allocate array of memblocks DMA objects */ 298*a23fd118Syl mempool->memblocks_dma_arr = xge_os_malloc(mempool->pdev, 299*a23fd118Syl sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max); 300*a23fd118Syl if (mempool->memblocks_dma_arr == NULL) { 301*a23fd118Syl xge_debug_mm(XGE_ERR, "memblocks_dma_arr allocation failure"); 302*a23fd118Syl __hal_mempool_destroy(mempool); 303*a23fd118Syl return NULL; 304*a23fd118Syl } 305*a23fd118Syl xge_os_memzero(mempool->memblocks_dma_arr, 306*a23fd118Syl sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max); 307*a23fd118Syl 308*a23fd118Syl /* allocate hash array of items */ 309*a23fd118Syl mempool->items_arr = xge_os_malloc(mempool->pdev, 310*a23fd118Syl sizeof(void*) * mempool->items_max); 311*a23fd118Syl if (mempool->items_arr == NULL) { 312*a23fd118Syl xge_debug_mm(XGE_ERR, "items_arr allocation failure"); 313*a23fd118Syl __hal_mempool_destroy(mempool); 314*a23fd118Syl return NULL; 315*a23fd118Syl } 316*a23fd118Syl xge_os_memzero(mempool->items_arr, sizeof(void *) * mempool->items_max); 317*a23fd118Syl 318*a23fd118Syl mempool->shadow_items_arr = xge_os_malloc(mempool->pdev,sizeof(void*) * 319*a23fd118Syl mempool->items_max); 320*a23fd118Syl if (mempool->shadow_items_arr == NULL) { 321*a23fd118Syl xge_debug_mm(XGE_ERR, "shadow_items_arr allocation failure"); 322*a23fd118Syl __hal_mempool_destroy(mempool); 323*a23fd118Syl return NULL; 324*a23fd118Syl } 325*a23fd118Syl xge_os_memzero(mempool->shadow_items_arr, 326*a23fd118Syl sizeof(void *) * mempool->items_max); 327*a23fd118Syl 328*a23fd118Syl /* calculate initial number of memblocks */ 329*a23fd118Syl memblocks_to_allocate = (mempool->items_initial + 330*a23fd118Syl mempool->items_per_memblock - 1) / 331*a23fd118Syl mempool->items_per_memblock; 332*a23fd118Syl 333*a23fd118Syl xge_debug_mm(XGE_TRACE, "allocating %d memblocks, " 334*a23fd118Syl "%d items per memblock", memblocks_to_allocate, 335*a23fd118Syl mempool->items_per_memblock); 336*a23fd118Syl 337*a23fd118Syl /* pre-allocate the mempool */ 338*a23fd118Syl status = __hal_mempool_grow(mempool, memblocks_to_allocate, &allocated); 339*a23fd118Syl xge_os_memcpy(mempool->shadow_items_arr, mempool->items_arr, 340*a23fd118Syl sizeof(void*) * mempool->items_max); 341*a23fd118Syl if (status != XGE_HAL_OK) { 342*a23fd118Syl xge_debug_mm(XGE_ERR, "mempool_grow failure"); 343*a23fd118Syl __hal_mempool_destroy(mempool); 344*a23fd118Syl return NULL; 345*a23fd118Syl } 346*a23fd118Syl 347*a23fd118Syl xge_debug_mm(XGE_TRACE, 348*a23fd118Syl "total: allocated %dk of DMA-capable memory", 349*a23fd118Syl mempool->memblock_size * allocated / 1024); 350*a23fd118Syl 351*a23fd118Syl return mempool; 352*a23fd118Syl } 353*a23fd118Syl 354*a23fd118Syl /* 355*a23fd118Syl * xge_hal_mempool_destroy 356*a23fd118Syl */ 357*a23fd118Syl void 358*a23fd118Syl __hal_mempool_destroy(xge_hal_mempool_t *mempool) 359*a23fd118Syl { 360*a23fd118Syl int i, j; 361*a23fd118Syl 362*a23fd118Syl for (i=0; i<mempool->memblocks_allocated; i++) { 363*a23fd118Syl xge_hal_mempool_dma_t *dma_object; 364*a23fd118Syl 365*a23fd118Syl xge_assert(mempool->memblocks_arr[i]); 366*a23fd118Syl xge_assert(mempool->memblocks_dma_arr + i); 367*a23fd118Syl 368*a23fd118Syl dma_object = mempool->memblocks_dma_arr + i; 369*a23fd118Syl 370*a23fd118Syl for (j=0; j<mempool->items_per_memblock; j++) { 371*a23fd118Syl int index = i*mempool->items_per_memblock + j; 372*a23fd118Syl 373*a23fd118Syl /* to skip last partially filled(if any) memblock */ 374*a23fd118Syl if (index >= mempool->items_current) { 375*a23fd118Syl break; 376*a23fd118Syl } 377*a23fd118Syl 378*a23fd118Syl /* let caller to do more job on each item */ 379*a23fd118Syl if (mempool->item_func_free != NULL) { 380*a23fd118Syl 381*a23fd118Syl mempool->item_func_free(mempool, 382*a23fd118Syl mempool->memblocks_arr[i], 383*a23fd118Syl i, dma_object, 384*a23fd118Syl mempool->shadow_items_arr[index], 385*a23fd118Syl index, /* unused */ -1, 386*a23fd118Syl mempool->userdata); 387*a23fd118Syl } 388*a23fd118Syl } 389*a23fd118Syl 390*a23fd118Syl xge_os_dma_unmap(mempool->pdev, 391*a23fd118Syl dma_object->handle, dma_object->addr, 392*a23fd118Syl mempool->memblock_size, XGE_OS_DMA_DIR_BIDIRECTIONAL); 393*a23fd118Syl 394*a23fd118Syl xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i], 395*a23fd118Syl mempool->items_priv_size * mempool->items_per_memblock); 396*a23fd118Syl 397*a23fd118Syl xge_os_dma_free(mempool->pdev, mempool->memblocks_arr[i], 398*a23fd118Syl mempool->memblock_size, &dma_object->acc_handle, 399*a23fd118Syl &dma_object->handle); 400*a23fd118Syl } 401*a23fd118Syl 402*a23fd118Syl if (mempool->items_arr) { 403*a23fd118Syl xge_os_free(mempool->pdev, mempool->items_arr, sizeof(void*) * 404*a23fd118Syl mempool->items_max); 405*a23fd118Syl } 406*a23fd118Syl 407*a23fd118Syl if (mempool->shadow_items_arr) { 408*a23fd118Syl xge_os_free(mempool->pdev, mempool->shadow_items_arr, 409*a23fd118Syl sizeof(void*) * mempool->items_max); 410*a23fd118Syl } 411*a23fd118Syl 412*a23fd118Syl if (mempool->memblocks_dma_arr) { 413*a23fd118Syl xge_os_free(mempool->pdev, mempool->memblocks_dma_arr, 414*a23fd118Syl sizeof(xge_hal_mempool_dma_t) * 415*a23fd118Syl mempool->memblocks_max); 416*a23fd118Syl } 417*a23fd118Syl 418*a23fd118Syl if (mempool->memblocks_priv_arr) { 419*a23fd118Syl xge_os_free(mempool->pdev, mempool->memblocks_priv_arr, 420*a23fd118Syl sizeof(void*) * mempool->memblocks_max); 421*a23fd118Syl } 422*a23fd118Syl 423*a23fd118Syl if (mempool->memblocks_arr) { 424*a23fd118Syl xge_os_free(mempool->pdev, mempool->memblocks_arr, 425*a23fd118Syl sizeof(void*) * mempool->memblocks_max); 426*a23fd118Syl } 427*a23fd118Syl 428*a23fd118Syl xge_os_free(mempool->pdev, mempool, sizeof(xge_hal_mempool_t)); 429*a23fd118Syl } 430