1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  *  Copyright (c) 2002-2005 Neterion, Inc.
24  *  All right Reserved.
25  *
26  *  FileName :    hal-mm.c
27  *
28  *  Description:  chipset memory pool object implementation
29  *
30  *  Created:      10 May 2004
31  */
32 
33 #include "xge-os-pal.h"
34 #include "xgehal-mm.h"
35 #include "xge-debug.h"
36 
37 /*
38  * __hal_mempool_grow
39  *
40  * Will resize mempool up to %num_allocate value.
41  */
42 xge_hal_status_e
43 __hal_mempool_grow(xge_hal_mempool_t *mempool, int num_allocate,
44 		int *num_allocated)
45 {
46 	int i, first_time = mempool->memblocks_allocated == 0 ? 1 : 0;
47 	int n_items = mempool->items_per_memblock;
48 
49 	*num_allocated = 0;
50 
51 	if ((mempool->memblocks_allocated + num_allocate) >
52 						mempool->memblocks_max) {
53 		xge_debug_mm(XGE_ERR, "%s",
54 			      "__hal_mempool_grow: can grow anymore");
55 		return XGE_HAL_ERR_OUT_OF_MEMORY;
56 	}
57 
58 	for (i = mempool->memblocks_allocated;
59 	     i < mempool->memblocks_allocated + num_allocate; i++) {
60 		int j;
61 		int is_last =
62 			((mempool->memblocks_allocated+num_allocate-1) == i);
63 		xge_hal_mempool_dma_t *dma_object =
64 			mempool->memblocks_dma_arr + i;
65 		void *the_memblock;
66 		int dma_flags;
67 
68 		dma_flags = XGE_OS_DMA_CACHELINE_ALIGNED;
69 #ifdef XGE_HAL_DMA_DTR_CONSISTENT
70 		dma_flags |= XGE_OS_DMA_CONSISTENT;
71 #else
72 		dma_flags |= XGE_OS_DMA_STREAMING;
73 #endif
74 
75 		/* allocate DMA-capable memblock */
76 		mempool->memblocks_arr[i] = xge_os_dma_malloc(mempool->pdev,
77 					        mempool->memblock_size,
78 						dma_flags,
79 					        &dma_object->handle,
80 					        &dma_object->acc_handle);
81 		if (mempool->memblocks_arr[i] == NULL) {
82 			xge_debug_mm(XGE_ERR,
83 				      "memblock[%d]: out of DMA memory", i);
84 			return XGE_HAL_ERR_OUT_OF_MEMORY;
85 		}
86 		xge_os_memzero(mempool->memblocks_arr[i],
87 		mempool->memblock_size);
88 		the_memblock = mempool->memblocks_arr[i];
89 
90 		/* allocate memblock's private part. Each DMA memblock
91 		 * has a space allocated for item's private usage upon
92 		 * mempool's user request. Each time mempool grows, it will
93 		 * allocate new memblock and its private part at once.
94 		 * This helps to minimize memory usage a lot. */
95 		mempool->memblocks_priv_arr[i] = xge_os_malloc(mempool->pdev,
96 					mempool->items_priv_size * n_items);
97 		if (mempool->memblocks_priv_arr[i] == NULL) {
98 			xge_os_dma_free(mempool->pdev,
99 				      the_memblock,
100 				      mempool->memblock_size,
101 				      &dma_object->acc_handle,
102 				      &dma_object->handle);
103 			xge_debug_mm(XGE_ERR,
104 			        "memblock_priv[%d]: out of virtual memory, "
105 			        "requested %d(%d:%d) bytes", i,
106 				mempool->items_priv_size * n_items,
107 				mempool->items_priv_size, n_items);
108 			return XGE_HAL_ERR_OUT_OF_MEMORY;
109 		}
110 		xge_os_memzero(mempool->memblocks_priv_arr[i],
111 			     mempool->items_priv_size * n_items);
112 
113 		/* map memblock to physical memory */
114 		dma_object->addr = xge_os_dma_map(mempool->pdev,
115 		                                dma_object->handle,
116 						the_memblock,
117 						mempool->memblock_size,
118 						XGE_OS_DMA_DIR_BIDIRECTIONAL,
119 #ifdef XGE_HAL_DMA_DTR_CONSISTENT
120 					        XGE_OS_DMA_CONSISTENT
121 #else
122 					        XGE_OS_DMA_STREAMING
123 #endif
124                                                 );
125 		if (dma_object->addr == XGE_OS_INVALID_DMA_ADDR) {
126 			xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i],
127 				  mempool->items_priv_size *
128 					n_items);
129 			xge_os_dma_free(mempool->pdev,
130 				      the_memblock,
131 				      mempool->memblock_size,
132 				      &dma_object->acc_handle,
133 				      &dma_object->handle);
134 			return XGE_HAL_ERR_OUT_OF_MAPPING;
135 		}
136 
137 		/* fill the items hash array */
138 		for (j=0; j<n_items; j++) {
139 			int index = i*n_items + j;
140 
141 			if (first_time && index >= mempool->items_initial) {
142 				break;
143 			}
144 
145 			mempool->items_arr[index] =
146 				((char *)the_memblock + j*mempool->item_size);
147 
148 			/* let caller to do more job on each item */
149 			if (mempool->item_func_alloc != NULL) {
150 				xge_hal_status_e status;
151 
152 				if ((status = mempool->item_func_alloc(
153 					mempool,
154 					the_memblock,
155 					i,
156 					dma_object,
157 					mempool->items_arr[index],
158 					index,
159 					is_last,
160 					mempool->userdata)) != XGE_HAL_OK) {
161 
162 					if (mempool->item_func_free != NULL) {
163 						int k;
164 
165 						for (k=0; k<j; k++) {
166 
167 						    index =i*n_items + k;
168 
169 						  (void)mempool->item_func_free(
170 						     mempool, the_memblock,
171 						     i, dma_object,
172 						     mempool->items_arr[index],
173 						     index, is_last,
174 						     mempool->userdata);
175 						}
176 					}
177 
178 					xge_os_free(mempool->pdev,
179 					     mempool->memblocks_priv_arr[i],
180 					     mempool->items_priv_size *
181 					     n_items);
182 					xge_os_dma_unmap(mempool->pdev,
183 					     dma_object->handle,
184 					     dma_object->addr,
185 					     mempool->memblock_size,
186 					     XGE_OS_DMA_DIR_BIDIRECTIONAL);
187 					xge_os_dma_free(mempool->pdev,
188 					     the_memblock,
189 					     mempool->memblock_size,
190 					     &dma_object->acc_handle,
191 					     &dma_object->handle);
192 					return status;
193 				}
194 			}
195 
196 			mempool->items_current = index + 1;
197 		}
198 
199 		xge_debug_mm(XGE_TRACE,
200 			"memblock%d: allocated %dk, vaddr 0x%llx, "
201 			"dma_addr 0x%llx", i, mempool->memblock_size / 1024,
202 			(unsigned long long)(ulong_t)mempool->memblocks_arr[i],
203 			(unsigned long long)dma_object->addr);
204 
205 		(*num_allocated)++;
206 
207 		if (first_time && mempool->items_current ==
208 						mempool->items_initial) {
209 			break;
210 		}
211 	}
212 
213 	/* increment actual number of allocated memblocks */
214 	mempool->memblocks_allocated += *num_allocated;
215 
216 	return XGE_HAL_OK;
217 }
218 
219 /*
220  * xge_hal_mempool_create
221  * @memblock_size:
222  * @items_initial:
223  * @items_max:
224  * @item_size:
225  * @item_func:
226  *
227  * This function will create memory pool object. Pool may grow but will
228  * never shrink. Pool consists of number of dynamically allocated blocks
229  * with size enough to hold %items_initial number of items. Memory is
230  * DMA-able but client must map/unmap before interoperating with the device.
231  * See also: xge_os_dma_map(), xge_hal_dma_unmap(), xge_hal_status_e{}.
232  */
233 xge_hal_mempool_t*
234 __hal_mempool_create(pci_dev_h pdev, int memblock_size, int item_size,
235 		int items_priv_size, int items_initial, int items_max,
236 		xge_hal_mempool_item_f item_func_alloc,
237 		xge_hal_mempool_item_f item_func_free, void *userdata)
238 {
239 	xge_hal_status_e status;
240 	int memblocks_to_allocate;
241 	xge_hal_mempool_t *mempool;
242 	int allocated;
243 
244 	if (memblock_size < item_size) {
245 		xge_debug_mm(XGE_ERR,
246 			"memblock_size %d < item_size %d: misconfiguration",
247 			memblock_size, item_size);
248 		return NULL;
249 	}
250 
251 	mempool = xge_os_malloc(pdev, sizeof(xge_hal_mempool_t));
252 	if (mempool == NULL) {
253 		xge_debug_mm(XGE_ERR, "mempool allocation failure");
254 		return NULL;
255 	}
256 	xge_os_memzero(mempool, sizeof(xge_hal_mempool_t));
257 
258 	mempool->pdev			= pdev;
259 	mempool->memblock_size		= memblock_size;
260 	mempool->items_max		= items_max;
261 	mempool->items_initial		= items_initial;
262 	mempool->item_size		= item_size;
263 	mempool->items_priv_size	= items_priv_size;
264 	mempool->item_func_alloc	= item_func_alloc;
265 	mempool->item_func_free		= item_func_free;
266 	mempool->userdata		= userdata;
267 
268 	mempool->memblocks_allocated = 0;
269 
270 	mempool->items_per_memblock = memblock_size / item_size;
271 
272 	mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) /
273 					mempool->items_per_memblock;
274 
275 	/* allocate array of memblocks */
276 	mempool->memblocks_arr = xge_os_malloc(mempool->pdev,
277 					sizeof(void*) * mempool->memblocks_max);
278 	if (mempool->memblocks_arr == NULL) {
279 		xge_debug_mm(XGE_ERR, "memblocks_arr allocation failure");
280 		__hal_mempool_destroy(mempool);
281 		return NULL;
282 	}
283 	xge_os_memzero(mempool->memblocks_arr,
284 		    sizeof(void*) * mempool->memblocks_max);
285 
286 	/* allocate array of private parts of items per memblocks */
287 	mempool->memblocks_priv_arr = xge_os_malloc(mempool->pdev,
288 					sizeof(void*) * mempool->memblocks_max);
289 	if (mempool->memblocks_priv_arr == NULL) {
290 		xge_debug_mm(XGE_ERR, "memblocks_priv_arr allocation failure");
291 		__hal_mempool_destroy(mempool);
292 		return NULL;
293 	}
294 	xge_os_memzero(mempool->memblocks_priv_arr,
295 		    sizeof(void*) * mempool->memblocks_max);
296 
297 	/* allocate array of memblocks DMA objects */
298 	mempool->memblocks_dma_arr = xge_os_malloc(mempool->pdev,
299 			sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max);
300 	if (mempool->memblocks_dma_arr == NULL) {
301 		xge_debug_mm(XGE_ERR, "memblocks_dma_arr allocation failure");
302 		__hal_mempool_destroy(mempool);
303 		return NULL;
304 	}
305 	xge_os_memzero(mempool->memblocks_dma_arr,
306 		     sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max);
307 
308 	/* allocate hash array of items */
309 	mempool->items_arr = xge_os_malloc(mempool->pdev,
310 				 sizeof(void*) * mempool->items_max);
311 	if (mempool->items_arr == NULL) {
312 		xge_debug_mm(XGE_ERR, "items_arr allocation failure");
313 		__hal_mempool_destroy(mempool);
314 		return NULL;
315 	}
316 	xge_os_memzero(mempool->items_arr, sizeof(void *) * mempool->items_max);
317 
318 	mempool->shadow_items_arr = xge_os_malloc(mempool->pdev,sizeof(void*) *
319 							    mempool->items_max);
320 	if (mempool->shadow_items_arr == NULL) {
321 		xge_debug_mm(XGE_ERR, "shadow_items_arr allocation failure");
322 		__hal_mempool_destroy(mempool);
323 		return NULL;
324 	}
325 	xge_os_memzero(mempool->shadow_items_arr,
326 		     sizeof(void *) * mempool->items_max);
327 
328 	/* calculate initial number of memblocks */
329 	memblocks_to_allocate = (mempool->items_initial +
330 				 mempool->items_per_memblock - 1) /
331 						mempool->items_per_memblock;
332 
333 	xge_debug_mm(XGE_TRACE, "allocating %d memblocks, "
334 			"%d items per memblock", memblocks_to_allocate,
335 			mempool->items_per_memblock);
336 
337 	/* pre-allocate the mempool */
338 	status = __hal_mempool_grow(mempool, memblocks_to_allocate, &allocated);
339 	xge_os_memcpy(mempool->shadow_items_arr, mempool->items_arr,
340 		    sizeof(void*) * mempool->items_max);
341 	if (status != XGE_HAL_OK) {
342 		xge_debug_mm(XGE_ERR, "mempool_grow failure");
343 		__hal_mempool_destroy(mempool);
344 		return NULL;
345 	}
346 
347 	xge_debug_mm(XGE_TRACE,
348 		"total: allocated %dk of DMA-capable memory",
349 		mempool->memblock_size * allocated / 1024);
350 
351 	return mempool;
352 }
353 
354 /*
355  * xge_hal_mempool_destroy
356  */
357 void
358 __hal_mempool_destroy(xge_hal_mempool_t *mempool)
359 {
360 	int i, j;
361 
362 	for (i=0; i<mempool->memblocks_allocated; i++) {
363 		xge_hal_mempool_dma_t *dma_object;
364 
365 		xge_assert(mempool->memblocks_arr[i]);
366 		xge_assert(mempool->memblocks_dma_arr + i);
367 
368 		dma_object = mempool->memblocks_dma_arr + i;
369 
370 		for (j=0; j<mempool->items_per_memblock; j++) {
371 			int index = i*mempool->items_per_memblock + j;
372 
373 			/* to skip last partially filled(if any) memblock */
374 			if (index >= mempool->items_current) {
375 				break;
376 			}
377 
378 			/* let caller to do more job on each item */
379 			if (mempool->item_func_free != NULL) {
380 
381 				mempool->item_func_free(mempool,
382 					mempool->memblocks_arr[i],
383 					i, dma_object,
384 					mempool->shadow_items_arr[index],
385 					index, /* unused */ -1,
386 					mempool->userdata);
387 			}
388 		}
389 
390 		xge_os_dma_unmap(mempool->pdev,
391 	               dma_object->handle, dma_object->addr,
392 		       mempool->memblock_size, XGE_OS_DMA_DIR_BIDIRECTIONAL);
393 
394 		xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i],
395 			mempool->items_priv_size * mempool->items_per_memblock);
396 
397 		xge_os_dma_free(mempool->pdev, mempool->memblocks_arr[i],
398 			      mempool->memblock_size, &dma_object->acc_handle,
399 			      &dma_object->handle);
400 	}
401 
402 	if (mempool->items_arr) {
403 		xge_os_free(mempool->pdev, mempool->items_arr, sizeof(void*) *
404 		          mempool->items_max);
405 	}
406 
407 	if (mempool->shadow_items_arr) {
408 		xge_os_free(mempool->pdev, mempool->shadow_items_arr,
409 			  sizeof(void*) * mempool->items_max);
410 	}
411 
412 	if (mempool->memblocks_dma_arr) {
413 		xge_os_free(mempool->pdev, mempool->memblocks_dma_arr,
414 		          sizeof(xge_hal_mempool_dma_t) *
415 			     mempool->memblocks_max);
416 	}
417 
418 	if (mempool->memblocks_priv_arr) {
419 		xge_os_free(mempool->pdev, mempool->memblocks_priv_arr,
420 		          sizeof(void*) * mempool->memblocks_max);
421 	}
422 
423 	if (mempool->memblocks_arr) {
424 		xge_os_free(mempool->pdev, mempool->memblocks_arr,
425 		          sizeof(void*) * mempool->memblocks_max);
426 	}
427 
428 	xge_os_free(mempool->pdev, mempool, sizeof(xge_hal_mempool_t));
429 }
430