1a23fd118Syl /*
2a23fd118Syl * CDDL HEADER START
3a23fd118Syl *
4a23fd118Syl * The contents of this file are subject to the terms of the
5a23fd118Syl * Common Development and Distribution License (the "License").
6a23fd118Syl * You may not use this file except in compliance with the License.
7a23fd118Syl *
8a23fd118Syl * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9a23fd118Syl * or http://www.opensolaris.org/os/licensing.
10a23fd118Syl * See the License for the specific language governing permissions
11a23fd118Syl * and limitations under the License.
12a23fd118Syl *
13a23fd118Syl * When distributing Covered Code, include this CDDL HEADER in each
14a23fd118Syl * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15a23fd118Syl * If applicable, add the following below this CDDL HEADER, with the
16a23fd118Syl * fields enclosed by brackets "[]" replaced with your own identifying
17a23fd118Syl * information: Portions Copyright [yyyy] [name of copyright owner]
18a23fd118Syl *
19a23fd118Syl * CDDL HEADER END
20a23fd118Syl *
21*8347601bSyl * Copyright (c) 2002-2006 Neterion, Inc.
22a23fd118Syl */
23a23fd118Syl
24a23fd118Syl #include "xge-os-pal.h"
25a23fd118Syl #include "xgehal-mm.h"
26a23fd118Syl #include "xge-debug.h"
27a23fd118Syl
28a23fd118Syl /*
29a23fd118Syl * __hal_mempool_grow
30a23fd118Syl *
31a23fd118Syl * Will resize mempool up to %num_allocate value.
32a23fd118Syl */
33a23fd118Syl xge_hal_status_e
__hal_mempool_grow(xge_hal_mempool_t * mempool,int num_allocate,int * num_allocated)34a23fd118Syl __hal_mempool_grow(xge_hal_mempool_t *mempool, int num_allocate,
35a23fd118Syl int *num_allocated)
36a23fd118Syl {
37a23fd118Syl int i, first_time = mempool->memblocks_allocated == 0 ? 1 : 0;
38a23fd118Syl int n_items = mempool->items_per_memblock;
39a23fd118Syl
40a23fd118Syl *num_allocated = 0;
41a23fd118Syl
42a23fd118Syl if ((mempool->memblocks_allocated + num_allocate) >
43a23fd118Syl mempool->memblocks_max) {
44a23fd118Syl xge_debug_mm(XGE_ERR, "%s",
45a23fd118Syl "__hal_mempool_grow: can grow anymore");
46a23fd118Syl return XGE_HAL_ERR_OUT_OF_MEMORY;
47a23fd118Syl }
48a23fd118Syl
49a23fd118Syl for (i = mempool->memblocks_allocated;
50a23fd118Syl i < mempool->memblocks_allocated + num_allocate; i++) {
51a23fd118Syl int j;
52a23fd118Syl int is_last =
53a23fd118Syl ((mempool->memblocks_allocated+num_allocate-1) == i);
54a23fd118Syl xge_hal_mempool_dma_t *dma_object =
55a23fd118Syl mempool->memblocks_dma_arr + i;
56a23fd118Syl void *the_memblock;
57a23fd118Syl int dma_flags;
58a23fd118Syl
59a23fd118Syl dma_flags = XGE_OS_DMA_CACHELINE_ALIGNED;
60a23fd118Syl #ifdef XGE_HAL_DMA_DTR_CONSISTENT
61a23fd118Syl dma_flags |= XGE_OS_DMA_CONSISTENT;
62a23fd118Syl #else
63a23fd118Syl dma_flags |= XGE_OS_DMA_STREAMING;
64a23fd118Syl #endif
65a23fd118Syl
66a23fd118Syl /* allocate DMA-capable memblock */
67a23fd118Syl mempool->memblocks_arr[i] = xge_os_dma_malloc(mempool->pdev,
68a23fd118Syl mempool->memblock_size,
69a23fd118Syl dma_flags,
70a23fd118Syl &dma_object->handle,
71a23fd118Syl &dma_object->acc_handle);
72a23fd118Syl if (mempool->memblocks_arr[i] == NULL) {
73a23fd118Syl xge_debug_mm(XGE_ERR,
74a23fd118Syl "memblock[%d]: out of DMA memory", i);
75a23fd118Syl return XGE_HAL_ERR_OUT_OF_MEMORY;
76a23fd118Syl }
77a23fd118Syl xge_os_memzero(mempool->memblocks_arr[i],
78a23fd118Syl mempool->memblock_size);
79a23fd118Syl the_memblock = mempool->memblocks_arr[i];
80a23fd118Syl
81a23fd118Syl /* allocate memblock's private part. Each DMA memblock
82a23fd118Syl * has a space allocated for item's private usage upon
83a23fd118Syl * mempool's user request. Each time mempool grows, it will
84a23fd118Syl * allocate new memblock and its private part at once.
85a23fd118Syl * This helps to minimize memory usage a lot. */
86a23fd118Syl mempool->memblocks_priv_arr[i] = xge_os_malloc(mempool->pdev,
87a23fd118Syl mempool->items_priv_size * n_items);
88a23fd118Syl if (mempool->memblocks_priv_arr[i] == NULL) {
89a23fd118Syl xge_os_dma_free(mempool->pdev,
90a23fd118Syl the_memblock,
91a23fd118Syl mempool->memblock_size,
92a23fd118Syl &dma_object->acc_handle,
93a23fd118Syl &dma_object->handle);
94a23fd118Syl xge_debug_mm(XGE_ERR,
95a23fd118Syl "memblock_priv[%d]: out of virtual memory, "
96a23fd118Syl "requested %d(%d:%d) bytes", i,
97a23fd118Syl mempool->items_priv_size * n_items,
98a23fd118Syl mempool->items_priv_size, n_items);
99a23fd118Syl return XGE_HAL_ERR_OUT_OF_MEMORY;
100a23fd118Syl }
101a23fd118Syl xge_os_memzero(mempool->memblocks_priv_arr[i],
102a23fd118Syl mempool->items_priv_size * n_items);
103a23fd118Syl
104a23fd118Syl /* map memblock to physical memory */
105a23fd118Syl dma_object->addr = xge_os_dma_map(mempool->pdev,
106a23fd118Syl dma_object->handle,
107a23fd118Syl the_memblock,
108a23fd118Syl mempool->memblock_size,
109a23fd118Syl XGE_OS_DMA_DIR_BIDIRECTIONAL,
110a23fd118Syl #ifdef XGE_HAL_DMA_DTR_CONSISTENT
111a23fd118Syl XGE_OS_DMA_CONSISTENT
112a23fd118Syl #else
113a23fd118Syl XGE_OS_DMA_STREAMING
114a23fd118Syl #endif
115a23fd118Syl );
116a23fd118Syl if (dma_object->addr == XGE_OS_INVALID_DMA_ADDR) {
117a23fd118Syl xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i],
118a23fd118Syl mempool->items_priv_size *
119a23fd118Syl n_items);
120a23fd118Syl xge_os_dma_free(mempool->pdev,
121a23fd118Syl the_memblock,
122a23fd118Syl mempool->memblock_size,
123a23fd118Syl &dma_object->acc_handle,
124a23fd118Syl &dma_object->handle);
125a23fd118Syl return XGE_HAL_ERR_OUT_OF_MAPPING;
126a23fd118Syl }
127a23fd118Syl
128a23fd118Syl /* fill the items hash array */
129a23fd118Syl for (j=0; j<n_items; j++) {
130a23fd118Syl int index = i*n_items + j;
131a23fd118Syl
132a23fd118Syl if (first_time && index >= mempool->items_initial) {
133a23fd118Syl break;
134a23fd118Syl }
135a23fd118Syl
136a23fd118Syl mempool->items_arr[index] =
137a23fd118Syl ((char *)the_memblock + j*mempool->item_size);
138a23fd118Syl
139a23fd118Syl /* let caller to do more job on each item */
140a23fd118Syl if (mempool->item_func_alloc != NULL) {
141a23fd118Syl xge_hal_status_e status;
142a23fd118Syl
143a23fd118Syl if ((status = mempool->item_func_alloc(
144a23fd118Syl mempool,
145a23fd118Syl the_memblock,
146a23fd118Syl i,
147a23fd118Syl dma_object,
148a23fd118Syl mempool->items_arr[index],
149a23fd118Syl index,
150a23fd118Syl is_last,
151a23fd118Syl mempool->userdata)) != XGE_HAL_OK) {
152a23fd118Syl
153a23fd118Syl if (mempool->item_func_free != NULL) {
154a23fd118Syl int k;
155a23fd118Syl
156a23fd118Syl for (k=0; k<j; k++) {
157a23fd118Syl
158a23fd118Syl index =i*n_items + k;
159a23fd118Syl
160a23fd118Syl (void)mempool->item_func_free(
161a23fd118Syl mempool, the_memblock,
162a23fd118Syl i, dma_object,
163a23fd118Syl mempool->items_arr[index],
164a23fd118Syl index, is_last,
165a23fd118Syl mempool->userdata);
166a23fd118Syl }
167a23fd118Syl }
168a23fd118Syl
169a23fd118Syl xge_os_free(mempool->pdev,
170a23fd118Syl mempool->memblocks_priv_arr[i],
171a23fd118Syl mempool->items_priv_size *
172a23fd118Syl n_items);
173a23fd118Syl xge_os_dma_unmap(mempool->pdev,
174a23fd118Syl dma_object->handle,
175a23fd118Syl dma_object->addr,
176a23fd118Syl mempool->memblock_size,
177a23fd118Syl XGE_OS_DMA_DIR_BIDIRECTIONAL);
178a23fd118Syl xge_os_dma_free(mempool->pdev,
179a23fd118Syl the_memblock,
180a23fd118Syl mempool->memblock_size,
181a23fd118Syl &dma_object->acc_handle,
182a23fd118Syl &dma_object->handle);
183a23fd118Syl return status;
184a23fd118Syl }
185a23fd118Syl }
186a23fd118Syl
187a23fd118Syl mempool->items_current = index + 1;
188a23fd118Syl }
189a23fd118Syl
190a23fd118Syl xge_debug_mm(XGE_TRACE,
191*8347601bSyl "memblock%d: allocated %dk, vaddr 0x"XGE_OS_LLXFMT", "
192*8347601bSyl "dma_addr 0x"XGE_OS_LLXFMT, i, mempool->memblock_size / 1024,
193a23fd118Syl (unsigned long long)(ulong_t)mempool->memblocks_arr[i],
194a23fd118Syl (unsigned long long)dma_object->addr);
195a23fd118Syl
196a23fd118Syl (*num_allocated)++;
197a23fd118Syl
198a23fd118Syl if (first_time && mempool->items_current ==
199a23fd118Syl mempool->items_initial) {
200a23fd118Syl break;
201a23fd118Syl }
202a23fd118Syl }
203a23fd118Syl
204a23fd118Syl /* increment actual number of allocated memblocks */
205a23fd118Syl mempool->memblocks_allocated += *num_allocated;
206a23fd118Syl
207a23fd118Syl return XGE_HAL_OK;
208a23fd118Syl }
209a23fd118Syl
210a23fd118Syl /*
211a23fd118Syl * xge_hal_mempool_create
212a23fd118Syl * @memblock_size:
213a23fd118Syl * @items_initial:
214a23fd118Syl * @items_max:
215a23fd118Syl * @item_size:
216a23fd118Syl * @item_func:
217a23fd118Syl *
218a23fd118Syl * This function will create memory pool object. Pool may grow but will
219a23fd118Syl * never shrink. Pool consists of number of dynamically allocated blocks
220a23fd118Syl * with size enough to hold %items_initial number of items. Memory is
221a23fd118Syl * DMA-able but client must map/unmap before interoperating with the device.
222a23fd118Syl * See also: xge_os_dma_map(), xge_hal_dma_unmap(), xge_hal_status_e{}.
223a23fd118Syl */
224a23fd118Syl xge_hal_mempool_t*
__hal_mempool_create(pci_dev_h pdev,int memblock_size,int item_size,int items_priv_size,int items_initial,int items_max,xge_hal_mempool_item_f item_func_alloc,xge_hal_mempool_item_f item_func_free,void * userdata)225a23fd118Syl __hal_mempool_create(pci_dev_h pdev, int memblock_size, int item_size,
226a23fd118Syl int items_priv_size, int items_initial, int items_max,
227a23fd118Syl xge_hal_mempool_item_f item_func_alloc,
228a23fd118Syl xge_hal_mempool_item_f item_func_free, void *userdata)
229a23fd118Syl {
230a23fd118Syl xge_hal_status_e status;
231a23fd118Syl int memblocks_to_allocate;
232a23fd118Syl xge_hal_mempool_t *mempool;
233a23fd118Syl int allocated;
234a23fd118Syl
235a23fd118Syl if (memblock_size < item_size) {
236a23fd118Syl xge_debug_mm(XGE_ERR,
237a23fd118Syl "memblock_size %d < item_size %d: misconfiguration",
238a23fd118Syl memblock_size, item_size);
239a23fd118Syl return NULL;
240a23fd118Syl }
241a23fd118Syl
242*8347601bSyl mempool = (xge_hal_mempool_t *) \
243*8347601bSyl xge_os_malloc(pdev, sizeof(xge_hal_mempool_t));
244a23fd118Syl if (mempool == NULL) {
245a23fd118Syl xge_debug_mm(XGE_ERR, "mempool allocation failure");
246a23fd118Syl return NULL;
247a23fd118Syl }
248a23fd118Syl xge_os_memzero(mempool, sizeof(xge_hal_mempool_t));
249a23fd118Syl
250a23fd118Syl mempool->pdev = pdev;
251a23fd118Syl mempool->memblock_size = memblock_size;
252a23fd118Syl mempool->items_max = items_max;
253a23fd118Syl mempool->items_initial = items_initial;
254a23fd118Syl mempool->item_size = item_size;
255a23fd118Syl mempool->items_priv_size = items_priv_size;
256a23fd118Syl mempool->item_func_alloc = item_func_alloc;
257a23fd118Syl mempool->item_func_free = item_func_free;
258a23fd118Syl mempool->userdata = userdata;
259a23fd118Syl
260a23fd118Syl mempool->memblocks_allocated = 0;
261a23fd118Syl
262a23fd118Syl mempool->items_per_memblock = memblock_size / item_size;
263a23fd118Syl
264a23fd118Syl mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) /
265a23fd118Syl mempool->items_per_memblock;
266a23fd118Syl
267a23fd118Syl /* allocate array of memblocks */
268*8347601bSyl mempool->memblocks_arr = (void ** ) xge_os_malloc(mempool->pdev,
269a23fd118Syl sizeof(void*) * mempool->memblocks_max);
270a23fd118Syl if (mempool->memblocks_arr == NULL) {
271a23fd118Syl xge_debug_mm(XGE_ERR, "memblocks_arr allocation failure");
272a23fd118Syl __hal_mempool_destroy(mempool);
273a23fd118Syl return NULL;
274a23fd118Syl }
275a23fd118Syl xge_os_memzero(mempool->memblocks_arr,
276a23fd118Syl sizeof(void*) * mempool->memblocks_max);
277a23fd118Syl
278a23fd118Syl /* allocate array of private parts of items per memblocks */
279*8347601bSyl mempool->memblocks_priv_arr = (void **) xge_os_malloc(mempool->pdev,
280a23fd118Syl sizeof(void*) * mempool->memblocks_max);
281a23fd118Syl if (mempool->memblocks_priv_arr == NULL) {
282a23fd118Syl xge_debug_mm(XGE_ERR, "memblocks_priv_arr allocation failure");
283a23fd118Syl __hal_mempool_destroy(mempool);
284a23fd118Syl return NULL;
285a23fd118Syl }
286a23fd118Syl xge_os_memzero(mempool->memblocks_priv_arr,
287a23fd118Syl sizeof(void*) * mempool->memblocks_max);
288a23fd118Syl
289a23fd118Syl /* allocate array of memblocks DMA objects */
290*8347601bSyl mempool->memblocks_dma_arr =
291*8347601bSyl (xge_hal_mempool_dma_t *) xge_os_malloc(mempool->pdev,
292*8347601bSyl sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max);
293*8347601bSyl
294a23fd118Syl if (mempool->memblocks_dma_arr == NULL) {
295a23fd118Syl xge_debug_mm(XGE_ERR, "memblocks_dma_arr allocation failure");
296a23fd118Syl __hal_mempool_destroy(mempool);
297a23fd118Syl return NULL;
298a23fd118Syl }
299a23fd118Syl xge_os_memzero(mempool->memblocks_dma_arr,
300a23fd118Syl sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max);
301a23fd118Syl
302a23fd118Syl /* allocate hash array of items */
303*8347601bSyl mempool->items_arr = (void **) xge_os_malloc(mempool->pdev,
304a23fd118Syl sizeof(void*) * mempool->items_max);
305a23fd118Syl if (mempool->items_arr == NULL) {
306a23fd118Syl xge_debug_mm(XGE_ERR, "items_arr allocation failure");
307a23fd118Syl __hal_mempool_destroy(mempool);
308a23fd118Syl return NULL;
309a23fd118Syl }
310a23fd118Syl xge_os_memzero(mempool->items_arr, sizeof(void *) * mempool->items_max);
311a23fd118Syl
312*8347601bSyl mempool->shadow_items_arr = (void **) xge_os_malloc(mempool->pdev,
313*8347601bSyl sizeof(void*) * mempool->items_max);
314a23fd118Syl if (mempool->shadow_items_arr == NULL) {
315a23fd118Syl xge_debug_mm(XGE_ERR, "shadow_items_arr allocation failure");
316a23fd118Syl __hal_mempool_destroy(mempool);
317a23fd118Syl return NULL;
318a23fd118Syl }
319a23fd118Syl xge_os_memzero(mempool->shadow_items_arr,
320a23fd118Syl sizeof(void *) * mempool->items_max);
321a23fd118Syl
322a23fd118Syl /* calculate initial number of memblocks */
323a23fd118Syl memblocks_to_allocate = (mempool->items_initial +
324a23fd118Syl mempool->items_per_memblock - 1) /
325a23fd118Syl mempool->items_per_memblock;
326a23fd118Syl
327a23fd118Syl xge_debug_mm(XGE_TRACE, "allocating %d memblocks, "
328a23fd118Syl "%d items per memblock", memblocks_to_allocate,
329a23fd118Syl mempool->items_per_memblock);
330a23fd118Syl
331a23fd118Syl /* pre-allocate the mempool */
332a23fd118Syl status = __hal_mempool_grow(mempool, memblocks_to_allocate, &allocated);
333a23fd118Syl xge_os_memcpy(mempool->shadow_items_arr, mempool->items_arr,
334a23fd118Syl sizeof(void*) * mempool->items_max);
335a23fd118Syl if (status != XGE_HAL_OK) {
336a23fd118Syl xge_debug_mm(XGE_ERR, "mempool_grow failure");
337a23fd118Syl __hal_mempool_destroy(mempool);
338a23fd118Syl return NULL;
339a23fd118Syl }
340a23fd118Syl
341a23fd118Syl xge_debug_mm(XGE_TRACE,
342a23fd118Syl "total: allocated %dk of DMA-capable memory",
343a23fd118Syl mempool->memblock_size * allocated / 1024);
344a23fd118Syl
345a23fd118Syl return mempool;
346a23fd118Syl }
347a23fd118Syl
348a23fd118Syl /*
349a23fd118Syl * xge_hal_mempool_destroy
350a23fd118Syl */
351a23fd118Syl void
__hal_mempool_destroy(xge_hal_mempool_t * mempool)352a23fd118Syl __hal_mempool_destroy(xge_hal_mempool_t *mempool)
353a23fd118Syl {
354a23fd118Syl int i, j;
355a23fd118Syl
356a23fd118Syl for (i=0; i<mempool->memblocks_allocated; i++) {
357a23fd118Syl xge_hal_mempool_dma_t *dma_object;
358a23fd118Syl
359a23fd118Syl xge_assert(mempool->memblocks_arr[i]);
360a23fd118Syl xge_assert(mempool->memblocks_dma_arr + i);
361a23fd118Syl
362a23fd118Syl dma_object = mempool->memblocks_dma_arr + i;
363a23fd118Syl
364a23fd118Syl for (j=0; j<mempool->items_per_memblock; j++) {
365a23fd118Syl int index = i*mempool->items_per_memblock + j;
366a23fd118Syl
367a23fd118Syl /* to skip last partially filled(if any) memblock */
368a23fd118Syl if (index >= mempool->items_current) {
369a23fd118Syl break;
370a23fd118Syl }
371a23fd118Syl
372a23fd118Syl /* let caller to do more job on each item */
373a23fd118Syl if (mempool->item_func_free != NULL) {
374a23fd118Syl
375a23fd118Syl mempool->item_func_free(mempool,
376a23fd118Syl mempool->memblocks_arr[i],
377a23fd118Syl i, dma_object,
378a23fd118Syl mempool->shadow_items_arr[index],
379a23fd118Syl index, /* unused */ -1,
380a23fd118Syl mempool->userdata);
381a23fd118Syl }
382a23fd118Syl }
383a23fd118Syl
384a23fd118Syl xge_os_dma_unmap(mempool->pdev,
385a23fd118Syl dma_object->handle, dma_object->addr,
386a23fd118Syl mempool->memblock_size, XGE_OS_DMA_DIR_BIDIRECTIONAL);
387a23fd118Syl
388a23fd118Syl xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i],
389a23fd118Syl mempool->items_priv_size * mempool->items_per_memblock);
390a23fd118Syl
391a23fd118Syl xge_os_dma_free(mempool->pdev, mempool->memblocks_arr[i],
392a23fd118Syl mempool->memblock_size, &dma_object->acc_handle,
393a23fd118Syl &dma_object->handle);
394a23fd118Syl }
395a23fd118Syl
396a23fd118Syl if (mempool->items_arr) {
397a23fd118Syl xge_os_free(mempool->pdev, mempool->items_arr, sizeof(void*) *
398a23fd118Syl mempool->items_max);
399a23fd118Syl }
400a23fd118Syl
401a23fd118Syl if (mempool->shadow_items_arr) {
402a23fd118Syl xge_os_free(mempool->pdev, mempool->shadow_items_arr,
403a23fd118Syl sizeof(void*) * mempool->items_max);
404a23fd118Syl }
405a23fd118Syl
406a23fd118Syl if (mempool->memblocks_dma_arr) {
407a23fd118Syl xge_os_free(mempool->pdev, mempool->memblocks_dma_arr,
408a23fd118Syl sizeof(xge_hal_mempool_dma_t) *
409a23fd118Syl mempool->memblocks_max);
410a23fd118Syl }
411a23fd118Syl
412a23fd118Syl if (mempool->memblocks_priv_arr) {
413a23fd118Syl xge_os_free(mempool->pdev, mempool->memblocks_priv_arr,
414a23fd118Syl sizeof(void*) * mempool->memblocks_max);
415a23fd118Syl }
416a23fd118Syl
417a23fd118Syl if (mempool->memblocks_arr) {
418a23fd118Syl xge_os_free(mempool->pdev, mempool->memblocks_arr,
419a23fd118Syl sizeof(void*) * mempool->memblocks_max);
420a23fd118Syl }
421a23fd118Syl
422a23fd118Syl xge_os_free(mempool->pdev, mempool, sizeof(xge_hal_mempool_t));
423a23fd118Syl }
424