Commit ffd9084d authored by Daniel Salzman's avatar Daniel Salzman
Browse files

Merge branch 'mempool-asan' into 'master'

Mempool metadata sanitizing

See merge request !781
parents 6286b8a5 744bee01
Pipeline #12296 passed with stages
in 10 minutes and 7 seconds
......@@ -3,6 +3,7 @@
*
* (c) 1997--2001 Martin Mares <mj@ucw.cz>
* (c) 2007 Pavel Charvat <pchar@ucw.cz>
* (c) 2015, 2017 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
*
* This software may be freely distributed and used according to the terms
* of the GNU Lesser General Public License.
......@@ -34,86 +35,90 @@
static void *
page_alloc(uint64_t len)
{
if (!len)
return NULL;
if (len > SIZE_MAX)
return NULL;
assert(!(len & (CPU_PAGE_SIZE-1)));
uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
if (p == (uint8_t*) MAP_FAILED)
return NULL;
return p;
if (!len) {
return NULL;
}
if (len > SIZE_MAX) {
return NULL;
}
assert(!(len & (CPU_PAGE_SIZE-1)));
uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
if (p == (uint8_t*) MAP_FAILED) {
return NULL;
}
return p;
}
static void
page_free(void *start, uint64_t len)
{
assert(!(len & (CPU_PAGE_SIZE-1)));
assert(!((uintptr_t) start & (CPU_PAGE_SIZE-1)));
munmap(start, len);
assert(!(len & (CPU_PAGE_SIZE-1)));
assert(!((uintptr_t) start & (CPU_PAGE_SIZE-1)));
munmap(start, len);
}
#endif
struct mempool_chunk {
struct mempool_chunk *next;
unsigned size;
struct mempool_chunk *next;
unsigned size;
};
static unsigned
mp_align_size(unsigned size)
{
#ifdef CONFIG_UCW_POOL_IS_MMAP
return ALIGN_TO(size + MP_CHUNK_TAIL, CPU_PAGE_SIZE) - MP_CHUNK_TAIL;
return ALIGN_TO(size + MP_CHUNK_TAIL, CPU_PAGE_SIZE) - MP_CHUNK_TAIL;
#else
return ALIGN_TO(size, CPU_STRUCT_ALIGN);
return ALIGN_TO(size, CPU_STRUCT_ALIGN);
#endif
}
void
mp_init(struct mempool *pool, unsigned chunk_size)
{
chunk_size = mp_align_size(MAX(sizeof(struct mempool), chunk_size));
*pool = (struct mempool) {
.chunk_size = chunk_size,
.threshold = chunk_size >> 1,
.last_big = &pool->last_big };
chunk_size = mp_align_size(MAX(sizeof(struct mempool), chunk_size));
*pool = (struct mempool) {
.chunk_size = chunk_size,
.threshold = chunk_size >> 1,
.last_big = &pool->last_big
};
}
static void *
mp_new_big_chunk(unsigned size)
{
uint8_t *data = malloc(size + MP_CHUNK_TAIL);
if (!data) {
return NULL;
}
ASAN_POISON_MEMORY_REGION(data, size);
struct mempool_chunk *chunk = (struct mempool_chunk *)(data + size);
chunk->size = size;
return chunk;
uint8_t *data = malloc(size + MP_CHUNK_TAIL);
if (!data) {
return NULL;
}
ASAN_POISON_MEMORY_REGION(data, size);
struct mempool_chunk *chunk = (struct mempool_chunk *)(data + size);
chunk->size = size;
return chunk;
}
static void
mp_free_big_chunk(struct mempool_chunk *chunk)
{
void *ptr = (void*)chunk - chunk->size;
ASAN_UNPOISON_MEMORY_REGION(ptr, chunk->size);
free(ptr);
void *ptr = (void*)chunk - chunk->size;
ASAN_UNPOISON_MEMORY_REGION(ptr, chunk->size);
free(ptr);
}
static void *
mp_new_chunk(unsigned size)
{
#ifdef CONFIG_UCW_POOL_IS_MMAP
uint8_t *data = page_alloc(size + MP_CHUNK_TAIL);
if (!data) {
return NULL;
}
ASAN_POISON_MEMORY_REGION(data, size);
struct mempool_chunk *chunk = (struct mempool_chunk *)(data + size);
chunk->size = size;
return chunk;
uint8_t *data = page_alloc(size + MP_CHUNK_TAIL);
if (!data) {
return NULL;
}
ASAN_POISON_MEMORY_REGION(data, size);
struct mempool_chunk *chunk = (struct mempool_chunk *)(data + size);
chunk->size = size;
return chunk;
#else
return mp_new_big_chunk(size);
return mp_new_big_chunk(size);
#endif
}
......@@ -121,186 +126,194 @@ static void
mp_free_chunk(struct mempool_chunk *chunk)
{
#ifdef CONFIG_UCW_POOL_IS_MMAP
uint8_t *data = (void *)chunk - chunk->size;
ASAN_UNPOISON_MEMORY_REGION(data, chunk->size);
page_free(data, chunk->size + MP_CHUNK_TAIL);
uint8_t *data = (void *)chunk - chunk->size;
ASAN_UNPOISON_MEMORY_REGION(data, chunk->size);
page_free(data, chunk->size + MP_CHUNK_TAIL);
#else
mp_free_big_chunk(chunk);
mp_free_big_chunk(chunk);
#endif
}
struct mempool *
mp_new(unsigned chunk_size)
{
chunk_size = mp_align_size(MAX(sizeof(struct mempool), chunk_size));
struct mempool_chunk *chunk = mp_new_chunk(chunk_size);
struct mempool *pool = (void *)chunk - chunk_size;
ASAN_UNPOISON_MEMORY_REGION(pool, sizeof(*pool));
DBG("Creating mempool %p with %u bytes long chunks", pool, chunk_size);
chunk->next = NULL;
*pool = (struct mempool) {
.state = { .free = { chunk_size - sizeof(*pool) }, .last = { chunk } },
.chunk_size = chunk_size,
.threshold = chunk_size >> 1,
.last_big = &pool->last_big };
return pool;
chunk_size = mp_align_size(MAX(sizeof(struct mempool), chunk_size));
struct mempool_chunk *chunk = mp_new_chunk(chunk_size);
struct mempool *pool = (void *)chunk - chunk_size;
ASAN_UNPOISON_MEMORY_REGION(pool, sizeof(*pool));
DBG("Creating mempool %p with %u bytes long chunks", pool, chunk_size);
chunk->next = NULL;
ASAN_POISON_MEMORY_REGION(chunk, sizeof(struct mempool_chunk));
*pool = (struct mempool) {
.state = { .free = { chunk_size - sizeof(*pool) }, .last = { chunk } },
.chunk_size = chunk_size,
.threshold = chunk_size >> 1,
.last_big = &pool->last_big
};
return pool;
}
static void
mp_free_chain(struct mempool_chunk *chunk)
{
while (chunk)
{
struct mempool_chunk *next = chunk->next;
mp_free_chunk(chunk);
chunk = next;
}
while (chunk) {
ASAN_UNPOISON_MEMORY_REGION(chunk, sizeof(struct mempool_chunk));
struct mempool_chunk *next = chunk->next;
mp_free_chunk(chunk);
chunk = next;
}
}
static void
mp_free_big_chain(struct mempool_chunk *chunk)
{
while (chunk)
{
struct mempool_chunk *next = chunk->next;
mp_free_big_chunk(chunk);
chunk = next;
}
while (chunk) {
ASAN_UNPOISON_MEMORY_REGION(chunk, sizeof(struct mempool_chunk));
struct mempool_chunk *next = chunk->next;
mp_free_big_chunk(chunk);
chunk = next;
}
}
void
mp_delete(struct mempool *pool)
{
DBG("Deleting mempool %p", pool);
mp_free_big_chain(pool->state.last[1]);
mp_free_chain(pool->unused);
mp_free_chain(pool->state.last[0]); // can contain the mempool structure
DBG("Deleting mempool %p", pool);
mp_free_big_chain(pool->state.last[1]);
mp_free_chain(pool->unused);
mp_free_chain(pool->state.last[0]); // can contain the mempool structure
}
void
mp_flush(struct mempool *pool)
{
mp_free_big_chain(pool->state.last[1]);
struct mempool_chunk *chunk, *next;
for (chunk = pool->state.last[0]; chunk && (void *)chunk - chunk->size != pool; chunk = next)
{
next = chunk->next;
chunk->next = pool->unused;
pool->unused = chunk;
}
pool->state.last[0] = chunk;
pool->state.free[0] = chunk ? chunk->size - sizeof(*pool) : 0;
pool->state.last[1] = NULL;
pool->state.free[1] = 0;
pool->state.next = NULL;
pool->last_big = &pool->last_big;
mp_free_big_chain(pool->state.last[1]);
struct mempool_chunk *chunk = pool->state.last[0], *next;
while (chunk) {
ASAN_UNPOISON_MEMORY_REGION(chunk, sizeof(struct mempool_chunk));
if ((void *)chunk - chunk->size == pool) {
break;
}
next = chunk->next;
chunk->next = pool->unused;
ASAN_POISON_MEMORY_REGION(chunk, sizeof(struct mempool_chunk));
pool->unused = chunk;
chunk = next;
}
pool->state.last[0] = chunk;
if (chunk) {
pool->state.free[0] = chunk->size - sizeof(*pool);
ASAN_POISON_MEMORY_REGION(chunk, sizeof(struct mempool_chunk));
} else {
pool->state.free[0] = 0;
}
pool->state.last[1] = NULL;
pool->state.free[1] = 0;
pool->last_big = &pool->last_big;
}
static void
mp_stats_chain(struct mempool_chunk *chunk, struct mempool_stats *stats, unsigned idx)
{
while (chunk)
{
stats->chain_size[idx] += chunk->size + sizeof(*chunk);
stats->chain_count[idx]++;
chunk = chunk->next;
}
stats->total_size += stats->chain_size[idx];
struct mempool_chunk *next;
while (chunk) {
ASAN_UNPOISON_MEMORY_REGION(chunk, sizeof(struct mempool_chunk));
stats->chain_size[idx] += chunk->size + sizeof(*chunk);
stats->chain_count[idx]++;
next = chunk->next;
ASAN_POISON_MEMORY_REGION(chunk, sizeof(struct mempool_chunk));
chunk = next;
}
stats->total_size += stats->chain_size[idx];
}
void
mp_stats(struct mempool *pool, struct mempool_stats *stats)
{
bzero(stats, sizeof(*stats));
mp_stats_chain(pool->state.last[0], stats, 0);
mp_stats_chain(pool->state.last[1], stats, 1);
mp_stats_chain(pool->unused, stats, 2);
bzero(stats, sizeof(*stats));
mp_stats_chain(pool->state.last[0], stats, 0);
mp_stats_chain(pool->state.last[1], stats, 1);
mp_stats_chain(pool->unused, stats, 2);
}
uint64_t
mp_total_size(struct mempool *pool)
{
struct mempool_stats stats;
mp_stats(pool, &stats);
return stats.total_size;
struct mempool_stats stats;
mp_stats(pool, &stats);
return stats.total_size;
}
static void *
mp_alloc_internal(struct mempool *pool, unsigned size)
{
struct mempool_chunk *chunk;
if (size <= pool->threshold)
{
pool->idx = 0;
if (pool->unused)
{
chunk = pool->unused;
pool->unused = chunk->next;
struct mempool_chunk *chunk;
if (size <= pool->threshold) {
pool->idx = 0;
if (pool->unused) {
chunk = pool->unused;
ASAN_UNPOISON_MEMORY_REGION(chunk, sizeof(struct mempool_chunk));
pool->unused = chunk->next;
} else {
chunk = mp_new_chunk(pool->chunk_size);
}
chunk->next = pool->state.last[0];
ASAN_POISON_MEMORY_REGION(chunk, sizeof(struct mempool_chunk));
pool->state.last[0] = chunk;
pool->state.free[0] = pool->chunk_size - size;
return (void *)chunk - pool->chunk_size;
} else if (size <= MP_SIZE_MAX) {
pool->idx = 1;
unsigned aligned = ALIGN_TO(size, CPU_STRUCT_ALIGN);
chunk = mp_new_big_chunk(aligned);
if (!chunk) {
return NULL;
}
chunk->next = pool->state.last[1];
ASAN_POISON_MEMORY_REGION(chunk, sizeof(struct mempool_chunk));
pool->state.last[1] = chunk;
pool->state.free[1] = aligned - size;
return pool->last_big = (void *)chunk - aligned;
} else {
fprintf(stderr, "Cannot allocate %u bytes from a mempool", size);
assert(0);
return NULL;
}
else
chunk = mp_new_chunk(pool->chunk_size);
chunk->next = pool->state.last[0];
pool->state.last[0] = chunk;
pool->state.free[0] = pool->chunk_size - size;
return (void *)chunk - pool->chunk_size;
}
else if (size <= MP_SIZE_MAX)
{
pool->idx = 1;
unsigned aligned = ALIGN_TO(size, CPU_STRUCT_ALIGN);
chunk = mp_new_big_chunk(aligned);
if (!chunk) {
return NULL;
}
chunk->next = pool->state.last[1];
pool->state.last[1] = chunk;
pool->state.free[1] = aligned - size;
return pool->last_big = (void *)chunk - aligned;
}
else {
fprintf(stderr, "Cannot allocate %u bytes from a mempool", size);
assert(0);
return NULL;
}
}
void *
mp_alloc(struct mempool *pool, unsigned size)
{
unsigned avail = pool->state.free[0] & ~(CPU_STRUCT_ALIGN - 1);
void *ptr = NULL;
if (size <= avail)
{
pool->state.free[0] = avail - size;
ptr = (uint8_t*)pool->state.last[0] - avail;
}
else
ptr = mp_alloc_internal(pool, size);
ASAN_UNPOISON_MEMORY_REGION(ptr, size);
return ptr;
unsigned avail = pool->state.free[0] & ~(CPU_STRUCT_ALIGN - 1);
void *ptr = NULL;
if (size <= avail) {
pool->state.free[0] = avail - size;
ptr = (uint8_t*)pool->state.last[0] - avail;
} else {
ptr = mp_alloc_internal(pool, size);
}
ASAN_UNPOISON_MEMORY_REGION(ptr, size);
return ptr;
}
void *
mp_alloc_noalign(struct mempool *pool, unsigned size)
{
void *ptr = NULL;
if (size <= pool->state.free[0])
{
ptr = (uint8_t*)pool->state.last[0] - pool->state.free[0];
pool->state.free[0] -= size;
}
else
ptr = mp_alloc_internal(pool, size);
ASAN_UNPOISON_MEMORY_REGION(ptr, size);
return ptr;
void *ptr = NULL;
if (size <= pool->state.free[0]) {
ptr = (uint8_t*)pool->state.last[0] - pool->state.free[0];
pool->state.free[0] -= size;
} else {
ptr = mp_alloc_internal(pool, size);
}
ASAN_UNPOISON_MEMORY_REGION(ptr, size);
return ptr;
}
void *
mp_alloc_zero(struct mempool *pool, unsigned size)
{
void *ptr = mp_alloc(pool, size);
bzero(ptr, size);
return ptr;
void *ptr = mp_alloc(pool, size);
bzero(ptr, size);
return ptr;
}
......@@ -3,6 +3,7 @@
*
* (c) 1997--2005 Martin Mares <mj@ucw.cz>
* (c) 2007 Pavel Charvat <pchar@ucw.cz>
* (c) 2015, 2017 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
*
* This software may be freely distributed and used according to the terms
* of the GNU Lesser General Public License.
......@@ -26,9 +27,8 @@
* You should use this one as an opaque handle only, the insides are internal.
**/
struct mempool_state {
unsigned free[2];
void *last[2];
struct mempool_state *next;
unsigned free[2];
void *last[2];
};
/**
......@@ -36,15 +36,15 @@ struct mempool_state {
* You should use this one as an opaque handle only, the insides are internal.
**/
struct mempool {
struct mempool_state state;
void *unused, *last_big;
unsigned chunk_size, threshold, idx;
struct mempool_state state;
void *unused, *last_big;
unsigned chunk_size, threshold, idx;
};
struct mempool_stats { /** Mempool statistics. See @mp_stats(). **/
uint64_t total_size; /* Real allocated size in bytes */
unsigned chain_count[3]; /* Number of allocated chunks in small/big/unused chains */
unsigned chain_size[3]; /* Size of allocated chunks in small/big/unused chains */
uint64_t total_size; /** Real allocated size in bytes. */
unsigned chain_count[3]; /** Number of allocated chunks in small/big/unused chains. */
unsigned chain_size[3]; /** Size of allocated chunks in small/big/unused chains. */
};
/***
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment