util/u_linkage.c \ | util/u_linkage.c \ | ||||
util/u_network.c \ | util/u_network.c \ | ||||
util/u_math.c \ | util/u_math.c \ | ||||
util/u_mempool.c \ | |||||
util/u_mm.c \ | util/u_mm.c \ | ||||
util/u_rect.c \ | util/u_rect.c \ | ||||
util/u_ringbuffer.c \ | util/u_ringbuffer.c \ | ||||
util/u_sampler.c \ | util/u_sampler.c \ | ||||
util/u_simple_shaders.c \ | util/u_simple_shaders.c \ | ||||
util/u_slab.c \ | |||||
util/u_snprintf.c \ | util/u_snprintf.c \ | ||||
util/u_staging.c \ | util/u_staging.c \ | ||||
util/u_surface.c \ | util/u_surface.c \ |
'util/u_linkage.c', | 'util/u_linkage.c', | ||||
'util/u_network.c', | 'util/u_network.c', | ||||
'util/u_math.c', | 'util/u_math.c', | ||||
'util/u_mempool.c', | |||||
'util/u_mm.c', | 'util/u_mm.c', | ||||
'util/u_rect.c', | 'util/u_rect.c', | ||||
'util/u_resource.c', | 'util/u_resource.c', | ||||
'util/u_ringbuffer.c', | 'util/u_ringbuffer.c', | ||||
'util/u_sampler.c', | 'util/u_sampler.c', | ||||
'util/u_simple_shaders.c', | 'util/u_simple_shaders.c', | ||||
'util/u_slab.c', | |||||
'util/u_snprintf.c', | 'util/u_snprintf.c', | ||||
'util/u_staging.c', | 'util/u_staging.c', | ||||
'util/u_surface.c', | 'util/u_surface.c', |
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | ||||
* USE OR OTHER DEALINGS IN THE SOFTWARE. */ | * USE OR OTHER DEALINGS IN THE SOFTWARE. */ | ||||
#include "util/u_mempool.h" | |||||
#include "util/u_slab.h" | |||||
#include "util/u_math.h" | #include "util/u_math.h" | ||||
#include "util/u_memory.h" | #include "util/u_memory.h" | ||||
#include <stdio.h> | #include <stdio.h> | ||||
#define UTIL_MEMPOOL_MAGIC 0xcafe4321 | |||||
#define UTIL_SLAB_MAGIC 0xcafe4321 | |||||
/* The block is either allocated memory or free space. */ | /* The block is either allocated memory or free space. */ | ||||
struct util_mempool_block { | |||||
struct util_slab_block { | |||||
/* The header. */ | /* The header. */ | ||||
/* The first next free block. */ | /* The first next free block. */ | ||||
struct util_mempool_block *next_free; | |||||
struct util_slab_block *next_free; | |||||
intptr_t magic; | intptr_t magic; | ||||
* The allocated size is always larger than this structure. */ | * The allocated size is always larger than this structure. */ | ||||
}; | }; | ||||
static struct util_mempool_block * | |||||
util_mempool_get_block(struct util_mempool *pool, | |||||
struct util_mempool_page *page, unsigned index) | |||||
static struct util_slab_block * | |||||
util_slab_get_block(struct util_slab_mempool *pool, | |||||
struct util_slab_page *page, unsigned index) | |||||
{ | { | ||||
return (struct util_mempool_block*) | |||||
((uint8_t*)page + sizeof(struct util_mempool_page) + | |||||
return (struct util_slab_block*) | |||||
((uint8_t*)page + sizeof(struct util_slab_page) + | |||||
(pool->block_size * index)); | (pool->block_size * index)); | ||||
} | } | ||||
static void util_mempool_add_new_page(struct util_mempool *pool) | |||||
static void util_slab_add_new_page(struct util_slab_mempool *pool) | |||||
{ | { | ||||
struct util_mempool_page *page; | |||||
struct util_mempool_block *block; | |||||
struct util_slab_page *page; | |||||
struct util_slab_block *block; | |||||
int i; | int i; | ||||
page = MALLOC(pool->page_size); | page = MALLOC(pool->page_size); | ||||
/* Mark all blocks as free. */ | /* Mark all blocks as free. */ | ||||
for (i = 0; i < pool->num_blocks-1; i++) { | for (i = 0; i < pool->num_blocks-1; i++) { | ||||
block = util_mempool_get_block(pool, page, i); | |||||
block->next_free = util_mempool_get_block(pool, page, i+1); | |||||
block->magic = UTIL_MEMPOOL_MAGIC; | |||||
block = util_slab_get_block(pool, page, i); | |||||
block->next_free = util_slab_get_block(pool, page, i+1); | |||||
block->magic = UTIL_SLAB_MAGIC; | |||||
} | } | ||||
block = util_mempool_get_block(pool, page, pool->num_blocks-1); | |||||
block = util_slab_get_block(pool, page, pool->num_blocks-1); | |||||
block->next_free = pool->first_free; | block->next_free = pool->first_free; | ||||
block->magic = UTIL_MEMPOOL_MAGIC; | |||||
pool->first_free = util_mempool_get_block(pool, page, 0); | |||||
block->magic = UTIL_SLAB_MAGIC; | |||||
pool->first_free = util_slab_get_block(pool, page, 0); | |||||
pool->num_pages++; | pool->num_pages++; | ||||
#if 0 | #if 0 | ||||
#endif | #endif | ||||
} | } | ||||
static void *util_mempool_malloc_st(struct util_mempool *pool) | |||||
static void *util_slab_alloc_st(struct util_slab_mempool *pool) | |||||
{ | { | ||||
struct util_mempool_block *block; | |||||
struct util_slab_block *block; | |||||
if (!pool->first_free) | if (!pool->first_free) | ||||
util_mempool_add_new_page(pool); | |||||
util_slab_add_new_page(pool); | |||||
block = pool->first_free; | block = pool->first_free; | ||||
assert(block->magic == UTIL_MEMPOOL_MAGIC); | |||||
assert(block->magic == UTIL_SLAB_MAGIC); | |||||
pool->first_free = block->next_free; | pool->first_free = block->next_free; | ||||
return (uint8_t*)block + sizeof(struct util_mempool_block); | |||||
return (uint8_t*)block + sizeof(struct util_slab_block); | |||||
} | } | ||||
static void util_mempool_free_st(struct util_mempool *pool, void *ptr) | |||||
static void util_slab_free_st(struct util_slab_mempool *pool, void *ptr) | |||||
{ | { | ||||
struct util_mempool_block *block = | |||||
(struct util_mempool_block*) | |||||
((uint8_t*)ptr - sizeof(struct util_mempool_block)); | |||||
struct util_slab_block *block = | |||||
(struct util_slab_block*) | |||||
((uint8_t*)ptr - sizeof(struct util_slab_block)); | |||||
assert(block->magic == UTIL_MEMPOOL_MAGIC); | |||||
assert(block->magic == UTIL_SLAB_MAGIC); | |||||
block->next_free = pool->first_free; | block->next_free = pool->first_free; | ||||
pool->first_free = block; | pool->first_free = block; | ||||
} | } | ||||
static void *util_mempool_malloc_mt(struct util_mempool *pool) | |||||
static void *util_slab_alloc_mt(struct util_slab_mempool *pool) | |||||
{ | { | ||||
void *mem; | void *mem; | ||||
pipe_mutex_lock(pool->mutex); | pipe_mutex_lock(pool->mutex); | ||||
mem = util_mempool_malloc_st(pool); | |||||
mem = util_slab_alloc_st(pool); | |||||
pipe_mutex_unlock(pool->mutex); | pipe_mutex_unlock(pool->mutex); | ||||
return mem; | return mem; | ||||
} | } | ||||
static void util_mempool_free_mt(struct util_mempool *pool, void *ptr) | |||||
static void util_slab_free_mt(struct util_slab_mempool *pool, void *ptr) | |||||
{ | { | ||||
pipe_mutex_lock(pool->mutex); | pipe_mutex_lock(pool->mutex); | ||||
util_mempool_free_st(pool, ptr); | |||||
util_slab_free_st(pool, ptr); | |||||
pipe_mutex_unlock(pool->mutex); | pipe_mutex_unlock(pool->mutex); | ||||
} | } | ||||
void util_mempool_set_thread_safety(struct util_mempool *pool, | |||||
enum util_mempool_threading threading) | |||||
void util_slab_set_thread_safety(struct util_slab_mempool *pool, | |||||
enum util_slab_threading threading) | |||||
{ | { | ||||
pool->threading = threading; | pool->threading = threading; | ||||
if (threading) { | if (threading) { | ||||
pool->malloc = util_mempool_malloc_mt; | |||||
pool->free = util_mempool_free_mt; | |||||
pool->alloc = util_slab_alloc_mt; | |||||
pool->free = util_slab_free_mt; | |||||
} else { | } else { | ||||
pool->malloc = util_mempool_malloc_st; | |||||
pool->free = util_mempool_free_st; | |||||
pool->alloc = util_slab_alloc_st; | |||||
pool->free = util_slab_free_st; | |||||
} | } | ||||
} | } | ||||
void util_mempool_create(struct util_mempool *pool, | |||||
unsigned item_size, | |||||
unsigned num_blocks, | |||||
enum util_mempool_threading threading) | |||||
void util_slab_create(struct util_slab_mempool *pool, | |||||
unsigned item_size, | |||||
unsigned num_blocks, | |||||
enum util_slab_threading threading) | |||||
{ | { | ||||
item_size = align(item_size, sizeof(intptr_t)); | item_size = align(item_size, sizeof(intptr_t)); | ||||
pool->num_pages = 0; | pool->num_pages = 0; | ||||
pool->num_blocks = num_blocks; | pool->num_blocks = num_blocks; | ||||
pool->block_size = sizeof(struct util_mempool_block) + item_size; | |||||
pool->block_size = sizeof(struct util_slab_block) + item_size; | |||||
pool->block_size = align(pool->block_size, sizeof(intptr_t)); | pool->block_size = align(pool->block_size, sizeof(intptr_t)); | ||||
pool->page_size = sizeof(struct util_mempool_page) + | |||||
pool->page_size = sizeof(struct util_slab_page) + | |||||
num_blocks * pool->block_size; | num_blocks * pool->block_size; | ||||
pool->first_free = NULL; | pool->first_free = NULL; | ||||
pipe_mutex_init(pool->mutex); | pipe_mutex_init(pool->mutex); | ||||
util_mempool_set_thread_safety(pool, threading); | |||||
util_slab_set_thread_safety(pool, threading); | |||||
} | } | ||||
void util_mempool_destroy(struct util_mempool *pool) | |||||
void util_slab_destroy(struct util_slab_mempool *pool) | |||||
{ | { | ||||
struct util_mempool_page *page, *temp; | |||||
struct util_slab_page *page, *temp; | |||||
foreach_s(page, temp, &pool->list) { | foreach_s(page, temp, &pool->list) { | ||||
remove_from_list(page); | remove_from_list(page); |
/** | /** | ||||
* @file | * @file | ||||
* Simple memory pool for equally sized memory allocations. | |||||
* util_mempool_malloc and util_mempool_free are in O(1). | |||||
* Simple slab allocator for equally sized memory allocations. | |||||
* util_slab_alloc and util_slab_free have time complexity in O(1). | |||||
* | * | ||||
* Good for allocations which have very low lifetime and are allocated | * Good for allocations which have very low lifetime and are allocated | ||||
* and freed very often. Use a profiler first! | |||||
* and freed very often. Use a profiler first to know if it's worth using it! | |||||
* | * | ||||
* Candidates: get_transfer, user_buffer_create | * Candidates: get_transfer, user_buffer_create | ||||
* | * | ||||
* @author Marek Olšák | * @author Marek Olšák | ||||
*/ | */ | ||||
#ifndef U_MEMPOOL_H | |||||
#define U_MEMPOOL_H | |||||
#ifndef U_SLAB_H | |||||
#define U_SLAB_H | |||||
#include "os/os_thread.h" | #include "os/os_thread.h" | ||||
enum util_mempool_threading { | |||||
UTIL_MEMPOOL_SINGLETHREADED = FALSE, | |||||
UTIL_MEMPOOL_MULTITHREADED = TRUE | |||||
enum util_slab_threading { | |||||
UTIL_SLAB_SINGLETHREADED = FALSE, | |||||
UTIL_SLAB_MULTITHREADED = TRUE | |||||
}; | }; | ||||
/* The page is an array of blocks (allocations). */ | /* The page is an array of blocks (allocations). */ | ||||
struct util_mempool_page { | |||||
struct util_slab_page { | |||||
/* The header (linked-list pointers). */ | /* The header (linked-list pointers). */ | ||||
struct util_mempool_page *prev, *next; | |||||
struct util_slab_page *prev, *next; | |||||
/* Memory after the last member is dedicated to the page itself. | /* Memory after the last member is dedicated to the page itself. | ||||
* The allocated size is always larger than this structure. */ | * The allocated size is always larger than this structure. */ | ||||
}; | }; | ||||
struct util_mempool { | |||||
struct util_slab_mempool { | |||||
/* Public members. */ | /* Public members. */ | ||||
void *(*malloc)(struct util_mempool *pool); | |||||
void (*free)(struct util_mempool *pool, void *ptr); | |||||
void *(*alloc)(struct util_slab_mempool *pool); | |||||
void (*free)(struct util_slab_mempool *pool, void *ptr); | |||||
/* Private members. */ | /* Private members. */ | ||||
struct util_mempool_block *first_free; | |||||
struct util_slab_block *first_free; | |||||
struct util_mempool_page list; | |||||
struct util_slab_page list; | |||||
unsigned block_size; | unsigned block_size; | ||||
unsigned page_size; | unsigned page_size; | ||||
unsigned num_blocks; | unsigned num_blocks; | ||||
unsigned num_pages; | unsigned num_pages; | ||||
enum util_mempool_threading threading; | |||||
enum util_slab_threading threading; | |||||
pipe_mutex mutex; | pipe_mutex mutex; | ||||
}; | }; | ||||
void util_mempool_create(struct util_mempool *pool, | |||||
unsigned item_size, | |||||
unsigned num_blocks, | |||||
enum util_mempool_threading threading); | |||||
void util_slab_create(struct util_slab_mempool *pool, | |||||
unsigned item_size, | |||||
unsigned num_blocks, | |||||
enum util_slab_threading threading); | |||||
void util_mempool_destroy(struct util_mempool *pool); | |||||
void util_slab_destroy(struct util_slab_mempool *pool); | |||||
void util_mempool_set_thread_safety(struct util_mempool *pool, | |||||
enum util_mempool_threading threading); | |||||
void util_slab_set_thread_safety(struct util_slab_mempool *pool, | |||||
enum util_slab_threading threading); | |||||
#define util_mempool_malloc(pool) (pool)->malloc(pool) | |||||
#define util_mempool_free(pool, ptr) (pool)->free(pool, ptr) | |||||
#define util_slab_alloc(pool) (pool)->alloc(pool) | |||||
#define util_slab_free(pool, ptr) (pool)->free(pool, ptr) | |||||
#endif | #endif |
p_atomic_inc(&r300screen->num_contexts); | p_atomic_inc(&r300screen->num_contexts); | ||||
if (r300screen->num_contexts > 1) | if (r300screen->num_contexts > 1) | ||||
util_mempool_set_thread_safety(&r300screen->pool_buffers, | |||||
UTIL_MEMPOOL_MULTITHREADED); | |||||
util_slab_set_thread_safety(&r300screen->pool_buffers, | |||||
UTIL_SLAB_MULTITHREADED); | |||||
} else { | } else { | ||||
p_atomic_dec(&r300screen->num_contexts); | p_atomic_dec(&r300screen->num_contexts); | ||||
if (r300screen->num_contexts <= 1) | if (r300screen->num_contexts <= 1) | ||||
util_mempool_set_thread_safety(&r300screen->pool_buffers, | |||||
UTIL_MEMPOOL_SINGLETHREADED); | |||||
util_slab_set_thread_safety(&r300screen->pool_buffers, | |||||
UTIL_SLAB_SINGLETHREADED); | |||||
} | } | ||||
} | } | ||||
r300->rws->cs_destroy(r300->cs); | r300->rws->cs_destroy(r300->cs); | ||||
/* XXX: No way to tell if this was initialized or not? */ | /* XXX: No way to tell if this was initialized or not? */ | ||||
util_mempool_destroy(&r300->pool_transfers); | |||||
util_slab_destroy(&r300->pool_transfers); | |||||
r300_update_num_contexts(r300->screen, -1); | r300_update_num_contexts(r300->screen, -1); | ||||
make_empty_list(&r300->query_list); | make_empty_list(&r300->query_list); | ||||
util_mempool_create(&r300->pool_transfers, | |||||
sizeof(struct pipe_transfer), 64, | |||||
UTIL_MEMPOOL_SINGLETHREADED); | |||||
util_slab_create(&r300->pool_transfers, | |||||
sizeof(struct pipe_transfer), 64, | |||||
UTIL_SLAB_SINGLETHREADED); | |||||
r300->cs = rws->cs_create(rws); | r300->cs = rws->cs_create(rws); | ||||
if (r300->cs == NULL) | if (r300->cs == NULL) |
struct u_upload_mgr *upload_vb; | struct u_upload_mgr *upload_vb; | ||||
struct u_upload_mgr *upload_ib; | struct u_upload_mgr *upload_ib; | ||||
struct util_mempool pool_transfers; | |||||
struct util_slab_mempool pool_transfers; | |||||
/* Stat counter. */ | /* Stat counter. */ | ||||
uint64_t flush_counter; | uint64_t flush_counter; |
struct r300_screen* r300screen = r300_screen(pscreen); | struct r300_screen* r300screen = r300_screen(pscreen); | ||||
struct r300_winsys_screen *rws = r300_winsys_screen(pscreen); | struct r300_winsys_screen *rws = r300_winsys_screen(pscreen); | ||||
util_mempool_destroy(&r300screen->pool_buffers); | |||||
util_slab_destroy(&r300screen->pool_buffers); | |||||
if (rws) | if (rws) | ||||
rws->destroy(rws); | rws->destroy(rws); | ||||
r300_init_debug(r300screen); | r300_init_debug(r300screen); | ||||
r300_parse_chipset(&r300screen->caps); | r300_parse_chipset(&r300screen->caps); | ||||
util_mempool_create(&r300screen->pool_buffers, | |||||
sizeof(struct r300_buffer), 64, | |||||
UTIL_MEMPOOL_SINGLETHREADED); | |||||
util_slab_create(&r300screen->pool_buffers, | |||||
sizeof(struct r300_buffer), 64, | |||||
UTIL_SLAB_SINGLETHREADED); | |||||
r300screen->rws = rws; | r300screen->rws = rws; | ||||
r300screen->screen.winsys = (struct pipe_winsys*)rws; | r300screen->screen.winsys = (struct pipe_winsys*)rws; |
#include "r300_chipset.h" | #include "r300_chipset.h" | ||||
#include "util/u_mempool.h" | |||||
#include "util/u_slab.h" | |||||
#include <stdio.h> | #include <stdio.h> | ||||
struct r300_capabilities caps; | struct r300_capabilities caps; | ||||
/* Memory pools. */ | /* Memory pools. */ | ||||
struct util_mempool pool_buffers; | |||||
struct util_slab_mempool pool_buffers; | |||||
/** Combination of DBG_xxx flags */ | /** Combination of DBG_xxx flags */ | ||||
unsigned debug; | unsigned debug; |
if (rbuf->buf) | if (rbuf->buf) | ||||
rws->buffer_reference(rws, &rbuf->buf, NULL); | rws->buffer_reference(rws, &rbuf->buf, NULL); | ||||
util_mempool_free(&r300screen->pool_buffers, rbuf); | |||||
util_slab_free(&r300screen->pool_buffers, rbuf); | |||||
} | } | ||||
static struct pipe_transfer* | static struct pipe_transfer* | ||||
{ | { | ||||
struct r300_context *r300 = r300_context(context); | struct r300_context *r300 = r300_context(context); | ||||
struct pipe_transfer *transfer = | struct pipe_transfer *transfer = | ||||
util_mempool_malloc(&r300->pool_transfers); | |||||
util_slab_alloc(&r300->pool_transfers); | |||||
transfer->resource = resource; | transfer->resource = resource; | ||||
transfer->sr = sr; | transfer->sr = sr; | ||||
struct pipe_transfer *transfer) | struct pipe_transfer *transfer) | ||||
{ | { | ||||
struct r300_context *r300 = r300_context(pipe); | struct r300_context *r300 = r300_context(pipe); | ||||
util_mempool_free(&r300->pool_transfers, transfer); | |||||
util_slab_free(&r300->pool_transfers, transfer); | |||||
} | } | ||||
static void * | static void * | ||||
struct r300_buffer *rbuf; | struct r300_buffer *rbuf; | ||||
unsigned alignment = 16; | unsigned alignment = 16; | ||||
rbuf = util_mempool_malloc(&r300screen->pool_buffers); | |||||
rbuf = util_slab_alloc(&r300screen->pool_buffers); | |||||
rbuf->magic = R300_BUFFER_MAGIC; | rbuf->magic = R300_BUFFER_MAGIC; | ||||
rbuf->domain); | rbuf->domain); | ||||
if (!rbuf->buf) { | if (!rbuf->buf) { | ||||
util_mempool_free(&r300screen->pool_buffers, rbuf); | |||||
util_slab_free(&r300screen->pool_buffers, rbuf); | |||||
return NULL; | return NULL; | ||||
} | } | ||||
struct r300_screen *r300screen = r300_screen(screen); | struct r300_screen *r300screen = r300_screen(screen); | ||||
struct r300_buffer *rbuf; | struct r300_buffer *rbuf; | ||||
rbuf = util_mempool_malloc(&r300screen->pool_buffers); | |||||
rbuf = util_slab_alloc(&r300screen->pool_buffers); | |||||
rbuf->magic = R300_BUFFER_MAGIC; | rbuf->magic = R300_BUFFER_MAGIC; | ||||