diff --git a/lib/c.zig b/lib/c.zig index c25293b84d..8b181dc118 100644 --- a/lib/c.zig +++ b/lib/c.zig @@ -47,19 +47,22 @@ pub fn errno(syscall_return_value: usize) c_int { } comptime { - _ = @import("c/inttypes.zig"); _ = @import("c/ctype.zig"); - _ = @import("c/stdlib.zig"); + _ = @import("c/inttypes.zig"); + if (!builtin.target.isMinGW()) { + _ = @import("c/malloc.zig"); + } _ = @import("c/math.zig"); + _ = @import("c/stdlib.zig"); _ = @import("c/string.zig"); _ = @import("c/strings.zig"); - _ = @import("c/wchar.zig"); - _ = @import("c/sys/mman.zig"); - _ = @import("c/sys/file.zig"); - _ = @import("c/sys/reboot.zig"); _ = @import("c/sys/capability.zig"); + _ = @import("c/sys/file.zig"); + _ = @import("c/sys/mman.zig"); + _ = @import("c/sys/reboot.zig"); _ = @import("c/sys/utsname.zig"); _ = @import("c/unistd.zig"); + _ = @import("c/wchar.zig"); } diff --git a/lib/c/malloc.zig b/lib/c/malloc.zig new file mode 100644 index 0000000000..72e7621b61 --- /dev/null +++ b/lib/c/malloc.zig @@ -0,0 +1,182 @@ +//! Based on wrapping a stateless Zig Allocator implementation, apropriate for: +//! - ReleaseFast and ReleaseSmall optimization modes, with multi-threading +//! enabled. +//! - WebAssembly in single-threaded mode. +//! +//! Because the libc APIs don't have client alignment and size tracking, in +//! order to take advantage of Zig allocator implementations, additional +//! metadata must be stored in the allocations. +//! +//! This implementation stores the metadata just before the pointer returned +//! from `malloc`, just like many libc malloc implementations do, including +//! musl. This has the downside of causing fragmentation for allocations with +//! higher alignment, however most of that memory can be recovered by +//! preemptively putting the gap onto the freelist. +const builtin = @import("builtin"); + +const std = @import("std"); +const assert = std.debug.assert; +const Alignment = std.mem.Alignment; +const alignment_bytes = @max(@alignOf(std.c.max_align_t), @sizeOf(Header)); +const alignment: Alignment = .fromByteUnits(alignment_bytes); + +const symbol = @import("../c.zig").symbol; + +comptime { + symbol(&malloc, "malloc"); + symbol(&aligned_alloc, "aligned_alloc"); + symbol(&posix_memalign, "posix_memalign"); + symbol(&calloc, "calloc"); + symbol(&realloc, "realloc"); + symbol(&reallocarray, "reallocarray"); + symbol(&free, "free"); + symbol(&malloc_usable_size, "malloc_usable_size"); + + symbol(&valloc, "valloc"); + symbol(&memalign, "memalign"); +} + +const no_context: *anyopaque = undefined; +const no_ra: usize = undefined; +const vtable = switch (builtin.cpu.arch) { + .wasm32, .wasm64 => std.heap.WasmAllocator.vtable, + else => std.heap.SmpAllocator.vtable, +}; + +/// Needed because libc memory allocators don't provide old alignment and size +/// which are required by Zig memory allocators. +const Header = packed struct(u64) { + alignment: Alignment, + /// Does not include the extra alignment bytes added. + size: Size, + padding: Padding = 0, + + comptime { + assert(@sizeOf(Header) <= alignment_bytes); + } + + const Size = @Int(.unsigned, @min(64 - @bitSizeOf(Alignment), @bitSizeOf(usize))); + const Padding = @Int(.unsigned, 64 - @bitSizeOf(Alignment) - @bitSizeOf(Size)); + + fn fromBase(base: [*]align(alignment_bytes) u8) *Header { + return @ptrCast(base - @sizeOf(Header)); + } +}; + +fn malloc(n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + const size = std.math.cast(Header.Size, n) orelse return null; + const ptr: [*]align(alignment_bytes) u8 = @alignCast( + vtable.alloc(no_context, n + alignment_bytes, alignment, no_ra) orelse return null, + ); + const base = ptr + alignment_bytes; + const header: *Header = .fromBase(base); + header.* = .{ + .alignment = alignment, + .size = size, + }; + return base; +} + +fn aligned_alloc(alloc_alignment: usize, n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + const size = std.math.cast(Header.Size, n) orelse return null; + const max_align = alignment.max(.fromByteUnits(alloc_alignment)); + const max_align_bytes = max_align.toByteUnits(); + const ptr: [*]align(alignment_bytes) u8 = @alignCast( + vtable.alloc(no_context, n + max_align_bytes, max_align, no_ra) orelse return null, + ); + const base: [*]align(alignment_bytes) u8 = @alignCast(ptr + max_align_bytes); + const header: *Header = .fromBase(base); + header.* = .{ + .alignment = max_align, + .size = size, + }; + return base; +} + +fn calloc(elems: usize, len: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + const n = std.math.mul(usize, elems, len) catch return null; + const base = malloc(n) orelse return null; + @memset(base[0..n], 0); + return base; +} + +fn realloc(opt_old_base: ?[*]align(alignment_bytes) u8, n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + if (n == 0) { + free(opt_old_base); + return null; + } + const old_base = opt_old_base orelse return malloc(n); + const new_size = std.math.cast(Header.Size, n) orelse return null; + const old_header: *Header = .fromBase(old_base); + assert(old_header.padding == 0); + const old_size = old_header.size; + const old_alignment = old_header.alignment; + const old_alignment_bytes = old_alignment.toByteUnits(); + const old_ptr = old_base - old_alignment_bytes; + const old_slice = old_ptr[0 .. old_size + old_alignment_bytes]; + const new_base: [*]align(alignment_bytes) u8 = if (vtable.remap( + no_context, + old_slice, + old_alignment, + n + old_alignment_bytes, + no_ra, + )) |new_ptr| @alignCast(new_ptr + old_alignment_bytes) else b: { + const new_ptr: [*]align(alignment_bytes) u8 = @alignCast( + vtable.alloc(no_context, n + old_alignment_bytes, old_alignment, no_ra) orelse return null, + ); + const new_base: [*]align(alignment_bytes) u8 = @alignCast(new_ptr + old_alignment_bytes); + const copy_len = @min(new_size, old_size); + @memcpy(new_base[0..copy_len], old_base[0..copy_len]); + vtable.free(no_context, old_slice, old_alignment, no_ra); + break :b new_base; + }; + const new_header: *Header = .fromBase(new_base); + new_header.* = .{ + .alignment = old_alignment, + .size = new_size, + }; + return new_base; +} + +fn reallocarray(opt_base: ?[*]align(alignment_bytes) u8, elems: usize, len: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + const n = std.math.mul(usize, elems, len) catch return null; + return realloc(opt_base, n); +} + +fn free(opt_old_base: ?[*]align(alignment_bytes) u8) callconv(.c) void { + const old_base = opt_old_base orelse return; + const old_header: *Header = .fromBase(old_base); + assert(old_header.padding == 0); + const old_size = old_header.size; + const old_alignment = old_header.alignment; + const old_alignment_bytes = old_alignment.toByteUnits(); + const old_ptr = old_base - old_alignment_bytes; + const old_slice = old_ptr[0 .. old_size + old_alignment_bytes]; + vtable.free(no_context, old_slice, old_alignment, no_ra); +} + +fn malloc_usable_size(opt_old_base: ?[*]align(alignment_bytes) u8) callconv(.c) usize { + const old_base = opt_old_base orelse return 0; + const old_header: *Header = .fromBase(old_base); + assert(old_header.padding == 0); + const old_size = old_header.size; + return old_size; +} + +fn valloc(n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + return aligned_alloc(std.heap.pageSize(), n); +} + +fn memalign(alloc_alignment: usize, n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + return aligned_alloc(alloc_alignment, n); +} + +fn posix_memalign(result: *?[*]align(alignment_bytes) u8, alloc_alignment: usize, n: usize) callconv(.c) c_int { + if (alloc_alignment < @sizeOf(*anyopaque)) return @intFromEnum(std.c.E.INVAL); + if (n == 0) { + result.* = null; + } else { + result.* = aligned_alloc(alloc_alignment, n) orelse return @intFromEnum(std.c.E.NOMEM); + } + return 0; +} diff --git a/lib/libc/musl/src/malloc/calloc.c b/lib/libc/musl/src/malloc/calloc.c deleted file mode 100644 index bf6bddca3f..0000000000 --- a/lib/libc/musl/src/malloc/calloc.c +++ /dev/null @@ -1,45 +0,0 @@ -#include -#include -#include -#include -#include "dynlink.h" - -static size_t mal0_clear(char *p, size_t n) -{ - const size_t pagesz = 4096; /* arbitrary */ - if (n < pagesz) return n; -#ifdef __GNUC__ - typedef uint64_t __attribute__((__may_alias__)) T; -#else - typedef unsigned char T; -#endif - char *pp = p + n; - size_t i = (uintptr_t)pp & (pagesz - 1); - for (;;) { - pp = memset(pp - i, 0, i); - if (pp - p < pagesz) return pp - p; - for (i = pagesz; i; i -= 2*sizeof(T), pp -= 2*sizeof(T)) - if (((T *)pp)[-1] | ((T *)pp)[-2]) - break; - } -} - -static int allzerop(void *p) -{ - return 0; -} -weak_alias(allzerop, __malloc_allzerop); - -void *calloc(size_t m, size_t n) -{ - if (n && m > (size_t)-1/n) { - errno = ENOMEM; - return 0; - } - n *= m; - void *p = malloc(n); - if (!p || (!__malloc_replaced && __malloc_allzerop(p))) - return p; - n = mal0_clear(p, n); - return memset(p, 0, n); -} diff --git a/lib/libc/musl/src/malloc/free.c b/lib/libc/musl/src/malloc/free.c deleted file mode 100644 index 3944f7b28f..0000000000 --- a/lib/libc/musl/src/malloc/free.c +++ /dev/null @@ -1,6 +0,0 @@ -#include - -void free(void *p) -{ - __libc_free(p); -} diff --git a/lib/libc/musl/src/malloc/libc_calloc.c b/lib/libc/musl/src/malloc/libc_calloc.c deleted file mode 100644 index d25eabea47..0000000000 --- a/lib/libc/musl/src/malloc/libc_calloc.c +++ /dev/null @@ -1,4 +0,0 @@ -#define calloc __libc_calloc -#define malloc __libc_malloc - -#include "calloc.c" diff --git a/lib/libc/musl/src/malloc/lite_malloc.c b/lib/libc/musl/src/malloc/lite_malloc.c deleted file mode 100644 index 43a988fbb8..0000000000 --- a/lib/libc/musl/src/malloc/lite_malloc.c +++ /dev/null @@ -1,118 +0,0 @@ -#include -#include -#include -#include -#include -#include "libc.h" -#include "lock.h" -#include "syscall.h" -#include "fork_impl.h" - -#define ALIGN 16 - -/* This function returns true if the interval [old,new] - * intersects the 'len'-sized interval below &libc.auxv - * (interpreted as the main-thread stack) or below &b - * (the current stack). It is used to defend against - * buggy brk implementations that can cross the stack. */ - -static int traverses_stack_p(uintptr_t old, uintptr_t new) -{ - const uintptr_t len = 8<<20; - uintptr_t a, b; - - b = (uintptr_t)libc.auxv; - a = b > len ? b-len : 0; - if (new>a && old len ? b-len : 0; - if (new>a && old SIZE_MAX/2) { - errno = ENOMEM; - return 0; - } - - if (!n) n++; - while (align end-cur) { - size_t req = n - (end-cur) + PAGE_SIZE-1 & -PAGE_SIZE; - - if (!cur) { - brk = __syscall(SYS_brk, 0); - brk += -brk & PAGE_SIZE-1; - cur = end = brk; - } - - if (brk == end && req < SIZE_MAX-brk - && !traverses_stack_p(brk, brk+req) - && __syscall(SYS_brk, brk+req)==brk+req) { - brk = end += req; - } else { - int new_area = 0; - req = n + PAGE_SIZE-1 & -PAGE_SIZE; - /* Only make a new area rather than individual mmap - * if wasted space would be over 1/8 of the map. */ - if (req-n > req/8) { - /* Geometric area size growth up to 64 pages, - * bounding waste by 1/8 of the area. */ - size_t min = PAGE_SIZE<<(mmap_step/2); - if (min-n > end-cur) { - if (req < min) { - req = min; - if (mmap_step < 12) - mmap_step++; - } - new_area = 1; - } - } - void *mem = __mmap(0, req, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (mem == MAP_FAILED || !new_area) { - UNLOCK(lock); - return mem==MAP_FAILED ? 0 : mem; - } - cur = (uintptr_t)mem; - end = cur + req; - } - } - - p = (void *)cur; - cur += n; - UNLOCK(lock); - return p; -} - -weak_alias(__simple_malloc, __libc_malloc_impl); - -void *__libc_malloc(size_t n) -{ - return __libc_malloc_impl(n); -} - -static void *default_malloc(size_t n) -{ - return __libc_malloc_impl(n); -} - -weak_alias(default_malloc, malloc); diff --git a/lib/libc/musl/src/malloc/mallocng/aligned_alloc.c b/lib/libc/musl/src/malloc/mallocng/aligned_alloc.c deleted file mode 100644 index e0862a83ae..0000000000 --- a/lib/libc/musl/src/malloc/mallocng/aligned_alloc.c +++ /dev/null @@ -1,60 +0,0 @@ -#include -#include -#include "meta.h" - -void *aligned_alloc(size_t align, size_t len) -{ - if ((align & -align) != align) { - errno = EINVAL; - return 0; - } - - if (len > SIZE_MAX - align || align >= (1ULL<<31)*UNIT) { - errno = ENOMEM; - return 0; - } - - if (DISABLE_ALIGNED_ALLOC) { - errno = ENOMEM; - return 0; - } - - if (align <= UNIT) align = UNIT; - - unsigned char *p = malloc(len + align - UNIT); - if (!p) - return 0; - - struct meta *g = get_meta(p); - int idx = get_slot_index(p); - size_t stride = get_stride(g); - unsigned char *start = g->mem->storage + stride*idx; - unsigned char *end = g->mem->storage + stride*(idx+1) - IB; - size_t adj = -(uintptr_t)p & (align-1); - - if (!adj) { - set_size(p, end, len); - return p; - } - p += adj; - uint32_t offset = (size_t)(p-g->mem->storage)/UNIT; - if (offset <= 0xffff) { - *(uint16_t *)(p-2) = offset; - p[-4] = 0; - } else { - // use a 32-bit offset if 16-bit doesn't fit. for this, - // 16-bit field must be zero, [-4] byte nonzero. - *(uint16_t *)(p-2) = 0; - *(uint32_t *)(p-8) = offset; - p[-4] = 1; - } - p[-3] = idx; - set_size(p, end, len); - // store offset to aligned enframing. this facilitates cycling - // offset and also iteration of heap for debugging/measurement. - // for extreme overalignment it won't fit but these are classless - // allocations anyway. - *(uint16_t *)(start - 2) = (size_t)(p-start)/UNIT; - start[-3] = 7<<5; - return p; -} diff --git a/lib/libc/musl/src/malloc/mallocng/donate.c b/lib/libc/musl/src/malloc/mallocng/donate.c deleted file mode 100644 index 41d850f357..0000000000 --- a/lib/libc/musl/src/malloc/mallocng/donate.c +++ /dev/null @@ -1,39 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "meta.h" - -static void donate(unsigned char *base, size_t len) -{ - uintptr_t a = (uintptr_t)base; - uintptr_t b = a + len; - a += -a & (UNIT-1); - b -= b & (UNIT-1); - memset(base, 0, len); - for (int sc=47; sc>0 && b>a; sc-=4) { - if (b-a < (size_classes[sc]+1)*UNIT) continue; - struct meta *m = alloc_meta(); - m->avail_mask = 0; - m->freed_mask = 1; - m->mem = (void *)a; - m->mem->meta = m; - m->last_idx = 0; - m->freeable = 0; - m->sizeclass = sc; - m->maplen = 0; - *((unsigned char *)m->mem+UNIT-4) = 0; - *((unsigned char *)m->mem+UNIT-3) = 255; - m->mem->storage[size_classes[sc]*UNIT-4] = 0; - queue(&ctx.active[sc], m); - a += (size_classes[sc]+1)*UNIT; - } -} - -void __malloc_donate(char *start, char *end) -{ - donate((void *)start, end-start); -} diff --git a/lib/libc/musl/src/malloc/mallocng/free.c b/lib/libc/musl/src/malloc/mallocng/free.c deleted file mode 100644 index 43f32aade8..0000000000 --- a/lib/libc/musl/src/malloc/mallocng/free.c +++ /dev/null @@ -1,151 +0,0 @@ -#define _BSD_SOURCE -#include -#include - -#include "meta.h" - -struct mapinfo { - void *base; - size_t len; -}; - -static struct mapinfo nontrivial_free(struct meta *, int); - -static struct mapinfo free_group(struct meta *g) -{ - struct mapinfo mi = { 0 }; - int sc = g->sizeclass; - if (sc < 48) { - ctx.usage_by_class[sc] -= g->last_idx+1; - } - if (g->maplen) { - step_seq(); - record_seq(sc); - mi.base = g->mem; - mi.len = g->maplen*4096UL; - } else { - void *p = g->mem; - struct meta *m = get_meta(p); - int idx = get_slot_index(p); - g->mem->meta = 0; - // not checking size/reserved here; it's intentionally invalid - mi = nontrivial_free(m, idx); - } - free_meta(g); - return mi; -} - -static int okay_to_free(struct meta *g) -{ - int sc = g->sizeclass; - - if (!g->freeable) return 0; - - // always free individual mmaps not suitable for reuse - if (sc >= 48 || get_stride(g) < UNIT*size_classes[sc]) - return 1; - - // always free groups allocated inside another group's slot - // since recreating them should not be expensive and they - // might be blocking freeing of a much larger group. - if (!g->maplen) return 1; - - // if there is another non-full group, free this one to - // consolidate future allocations, reduce fragmentation. - if (g->next != g) return 1; - - // free any group in a size class that's not bouncing - if (!is_bouncing(sc)) return 1; - - size_t cnt = g->last_idx+1; - size_t usage = ctx.usage_by_class[sc]; - - // if usage is high enough that a larger count should be - // used, free the low-count group so a new one will be made. - if (9*cnt <= usage && cnt < 20) - return 1; - - // otherwise, keep the last group in a bouncing class. - return 0; -} - -static struct mapinfo nontrivial_free(struct meta *g, int i) -{ - uint32_t self = 1u<sizeclass; - uint32_t mask = g->freed_mask | g->avail_mask; - - if (mask+self == (2u<last_idx)-1 && okay_to_free(g)) { - // any multi-slot group is necessarily on an active list - // here, but single-slot groups might or might not be. - if (g->next) { - assert(sc < 48); - int activate_new = (ctx.active[sc]==g); - dequeue(&ctx.active[sc], g); - if (activate_new && ctx.active[sc]) - activate_group(ctx.active[sc]); - } - return free_group(g); - } else if (!mask) { - assert(sc < 48); - // might still be active if there were no allocations - // after last available slot was taken. - if (ctx.active[sc] != g) { - queue(&ctx.active[sc], g); - } - } - a_or(&g->freed_mask, self); - return (struct mapinfo){ 0 }; -} - -void free(void *p) -{ - if (!p) return; - - struct meta *g = get_meta(p); - int idx = get_slot_index(p); - size_t stride = get_stride(g); - unsigned char *start = g->mem->storage + stride*idx; - unsigned char *end = start + stride - IB; - get_nominal_size(p, end); - uint32_t self = 1u<last_idx)-1; - ((unsigned char *)p)[-3] = 255; - // invalidate offset to group header, and cycle offset of - // used region within slot if current offset is zero. - *(uint16_t *)((char *)p-2) = 0; - - // release any whole pages contained in the slot to be freed - // unless it's a single-slot group that will be unmapped. - if (((uintptr_t)(start-1) ^ (uintptr_t)end) >= 2*PGSZ && g->last_idx) { - unsigned char *base = start + (-(uintptr_t)start & (PGSZ-1)); - size_t len = (end-base) & -PGSZ; - if (len && USE_MADV_FREE) { - int e = errno; - madvise(base, len, MADV_FREE); - errno = e; - } - } - - // atomic free without locking if this is neither first or last slot - for (;;) { - uint32_t freed = g->freed_mask; - uint32_t avail = g->avail_mask; - uint32_t mask = freed | avail; - assert(!(mask&self)); - if (!freed || mask+self==all) break; - if (!MT) - g->freed_mask = freed+self; - else if (a_cas(&g->freed_mask, freed, freed+self)!=freed) - continue; - return; - } - - wrlock(); - struct mapinfo mi = nontrivial_free(g, idx); - unlock(); - if (mi.len) { - int e = errno; - munmap(mi.base, mi.len); - errno = e; - } -} diff --git a/lib/libc/musl/src/malloc/mallocng/glue.h b/lib/libc/musl/src/malloc/mallocng/glue.h deleted file mode 100644 index 77f4c812b2..0000000000 --- a/lib/libc/musl/src/malloc/mallocng/glue.h +++ /dev/null @@ -1,95 +0,0 @@ -#ifndef MALLOC_GLUE_H -#define MALLOC_GLUE_H - -#include -#include -#include -#include -#include -#include -#include "atomic.h" -#include "syscall.h" -#include "libc.h" -#include "lock.h" -#include "dynlink.h" - -// use macros to appropriately namespace these. -#define size_classes __malloc_size_classes -#define ctx __malloc_context -#define alloc_meta __malloc_alloc_meta -#define is_allzero __malloc_allzerop -#define dump_heap __dump_heap - -#define malloc __libc_malloc_impl -#define realloc __libc_realloc -#define free __libc_free - -#define USE_MADV_FREE 0 - -#if USE_REAL_ASSERT -#include -#else -#undef assert -#define assert(x) do { if (!(x)) a_crash(); } while(0) -#endif - -#define brk(p) ((uintptr_t)__syscall(SYS_brk, p)) - -#define mmap __mmap -#define madvise __madvise -#define mremap __mremap - -#define DISABLE_ALIGNED_ALLOC (__malloc_replaced && !__aligned_alloc_replaced) - -static inline uint64_t get_random_secret() -{ - uint64_t secret = (uintptr_t)&secret * 1103515245; - for (size_t i=0; libc.auxv[i]; i+=2) - if (libc.auxv[i]==AT_RANDOM) - memcpy(&secret, (char *)libc.auxv[i+1]+8, sizeof secret); - return secret; -} - -#ifndef PAGESIZE -#define PAGESIZE PAGE_SIZE -#endif - -#define MT (libc.need_locks) - -#define RDLOCK_IS_EXCLUSIVE 1 - -__attribute__((__visibility__("hidden"))) -extern int __malloc_lock[1]; - -#define LOCK_OBJ_DEF \ -int __malloc_lock[1]; \ -void __malloc_atfork(int who) { malloc_atfork(who); } - -static inline void rdlock() -{ - if (MT) LOCK(__malloc_lock); -} -static inline void wrlock() -{ - if (MT) LOCK(__malloc_lock); -} -static inline void unlock() -{ - UNLOCK(__malloc_lock); -} -static inline void upgradelock() -{ -} -static inline void resetlock() -{ - __malloc_lock[0] = 0; -} - -static inline void malloc_atfork(int who) -{ - if (who<0) rdlock(); - else if (who>0) resetlock(); - else unlock(); -} - -#endif diff --git a/lib/libc/musl/src/malloc/mallocng/malloc.c b/lib/libc/musl/src/malloc/mallocng/malloc.c deleted file mode 100644 index d695ab8ec9..0000000000 --- a/lib/libc/musl/src/malloc/mallocng/malloc.c +++ /dev/null @@ -1,387 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "meta.h" - -LOCK_OBJ_DEF; - -const uint16_t size_classes[] = { - 1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 12, 15, - 18, 20, 25, 31, - 36, 42, 50, 63, - 72, 84, 102, 127, - 146, 170, 204, 255, - 292, 340, 409, 511, - 584, 682, 818, 1023, - 1169, 1364, 1637, 2047, - 2340, 2730, 3276, 4095, - 4680, 5460, 6552, 8191, -}; - -static const uint8_t small_cnt_tab[][3] = { - { 30, 30, 30 }, - { 31, 15, 15 }, - { 20, 10, 10 }, - { 31, 15, 7 }, - { 25, 12, 6 }, - { 21, 10, 5 }, - { 18, 8, 4 }, - { 31, 15, 7 }, - { 28, 14, 6 }, -}; - -static const uint8_t med_cnt_tab[4] = { 28, 24, 20, 32 }; - -struct malloc_context ctx = { 0 }; - -struct meta *alloc_meta(void) -{ - struct meta *m; - unsigned char *p; - if (!ctx.init_done) { -#ifndef PAGESIZE - ctx.pagesize = get_page_size(); -#endif - ctx.secret = get_random_secret(); - ctx.init_done = 1; - } - size_t pagesize = PGSZ; - if (pagesize < 4096) pagesize = 4096; - if ((m = dequeue_head(&ctx.free_meta_head))) return m; - if (!ctx.avail_meta_count) { - int need_unprotect = 1; - if (!ctx.avail_meta_area_count && ctx.brk!=-1) { - uintptr_t new = ctx.brk + pagesize; - int need_guard = 0; - if (!ctx.brk) { - need_guard = 1; - ctx.brk = brk(0); - // some ancient kernels returned _ebss - // instead of next page as initial brk. - ctx.brk += -ctx.brk & (pagesize-1); - new = ctx.brk + 2*pagesize; - } - if (brk(new) != new) { - ctx.brk = -1; - } else { - if (need_guard) mmap((void *)ctx.brk, pagesize, - PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); - ctx.brk = new; - ctx.avail_meta_areas = (void *)(new - pagesize); - ctx.avail_meta_area_count = pagesize>>12; - need_unprotect = 0; - } - } - if (!ctx.avail_meta_area_count) { - size_t n = 2UL << ctx.meta_alloc_shift; - p = mmap(0, n*pagesize, PROT_NONE, - MAP_PRIVATE|MAP_ANON, -1, 0); - if (p==MAP_FAILED) return 0; - ctx.avail_meta_areas = p + pagesize; - ctx.avail_meta_area_count = (n-1)*(pagesize>>12); - ctx.meta_alloc_shift++; - } - p = ctx.avail_meta_areas; - if ((uintptr_t)p & (pagesize-1)) need_unprotect = 0; - if (need_unprotect) - if (mprotect(p, pagesize, PROT_READ|PROT_WRITE) - && errno != ENOSYS) - return 0; - ctx.avail_meta_area_count--; - ctx.avail_meta_areas = p + 4096; - if (ctx.meta_area_tail) { - ctx.meta_area_tail->next = (void *)p; - } else { - ctx.meta_area_head = (void *)p; - } - ctx.meta_area_tail = (void *)p; - ctx.meta_area_tail->check = ctx.secret; - ctx.avail_meta_count = ctx.meta_area_tail->nslots - = (4096-sizeof(struct meta_area))/sizeof *m; - ctx.avail_meta = ctx.meta_area_tail->slots; - } - ctx.avail_meta_count--; - m = ctx.avail_meta++; - m->prev = m->next = 0; - return m; -} - -static uint32_t try_avail(struct meta **pm) -{ - struct meta *m = *pm; - uint32_t first; - if (!m) return 0; - uint32_t mask = m->avail_mask; - if (!mask) { - if (!m) return 0; - if (!m->freed_mask) { - dequeue(pm, m); - m = *pm; - if (!m) return 0; - } else { - m = m->next; - *pm = m; - } - - mask = m->freed_mask; - - // skip fully-free group unless it's the only one - // or it's a permanently non-freeable group - if (mask == (2u<last_idx)-1 && m->freeable) { - m = m->next; - *pm = m; - mask = m->freed_mask; - } - - // activate more slots in a not-fully-active group - // if needed, but only as a last resort. prefer using - // any other group with free slots. this avoids - // touching & dirtying as-yet-unused pages. - if (!(mask & ((2u<mem->active_idx)-1))) { - if (m->next != m) { - m = m->next; - *pm = m; - } else { - int cnt = m->mem->active_idx + 2; - int size = size_classes[m->sizeclass]*UNIT; - int span = UNIT + size*cnt; - // activate up to next 4k boundary - while ((span^(span+size-1)) < 4096) { - cnt++; - span += size; - } - if (cnt > m->last_idx+1) - cnt = m->last_idx+1; - m->mem->active_idx = cnt-1; - } - } - mask = activate_group(m); - assert(mask); - decay_bounces(m->sizeclass); - } - first = mask&-mask; - m->avail_mask = mask-first; - return first; -} - -static int alloc_slot(int, size_t); - -static struct meta *alloc_group(int sc, size_t req) -{ - size_t size = UNIT*size_classes[sc]; - int i = 0, cnt; - unsigned char *p; - struct meta *m = alloc_meta(); - if (!m) return 0; - size_t usage = ctx.usage_by_class[sc]; - size_t pagesize = PGSZ; - int active_idx; - if (sc < 9) { - while (i<2 && 4*small_cnt_tab[sc][i] > usage) - i++; - cnt = small_cnt_tab[sc][i]; - } else { - // lookup max number of slots fitting in power-of-two size - // from a table, along with number of factors of two we - // can divide out without a remainder or reaching 1. - cnt = med_cnt_tab[sc&3]; - - // reduce cnt to avoid excessive eagar allocation. - while (!(cnt&1) && 4*cnt > usage) - cnt >>= 1; - - // data structures don't support groups whose slot offsets - // in units don't fit in 16 bits. - while (size*cnt >= 65536*UNIT) - cnt >>= 1; - } - - // If we selected a count of 1 above but it's not sufficient to use - // mmap, increase to 2. Then it might be; if not it will nest. - if (cnt==1 && size*cnt+UNIT <= pagesize/2) cnt = 2; - - // All choices of size*cnt are "just below" a power of two, so anything - // larger than half the page size should be allocated as whole pages. - if (size*cnt+UNIT > pagesize/2) { - // check/update bounce counter to start/increase retention - // of freed maps, and inhibit use of low-count, odd-size - // small mappings and single-slot groups if activated. - int nosmall = is_bouncing(sc); - account_bounce(sc); - step_seq(); - - // since the following count reduction opportunities have - // an absolute memory usage cost, don't overdo them. count - // coarse usage as part of usage. - if (!(sc&1) && sc<32) usage += ctx.usage_by_class[sc+1]; - - // try to drop to a lower count if the one found above - // increases usage by more than 25%. these reduced counts - // roughly fill an integral number of pages, just not a - // power of two, limiting amount of unusable space. - if (4*cnt > usage && !nosmall) { - if (0); - else if ((sc&3)==1 && size*cnt>8*pagesize) cnt = 2; - else if ((sc&3)==2 && size*cnt>4*pagesize) cnt = 3; - else if ((sc&3)==0 && size*cnt>8*pagesize) cnt = 3; - else if ((sc&3)==0 && size*cnt>2*pagesize) cnt = 5; - } - size_t needed = size*cnt + UNIT; - needed += -needed & (pagesize-1); - - // produce an individually-mmapped allocation if usage is low, - // bounce counter hasn't triggered, and either it saves memory - // or it avoids eagar slot allocation without wasting too much. - if (!nosmall && cnt<=7) { - req += IB + UNIT; - req += -req & (pagesize-1); - if (req=4*pagesize && 2*cnt>usage)) { - cnt = 1; - needed = req; - } - } - - p = mmap(0, needed, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0); - if (p==MAP_FAILED) { - free_meta(m); - return 0; - } - m->maplen = needed>>12; - ctx.mmap_counter++; - active_idx = (4096-UNIT)/size-1; - if (active_idx > cnt-1) active_idx = cnt-1; - if (active_idx < 0) active_idx = 0; - } else { - int j = size_to_class(UNIT+cnt*size-IB); - int idx = alloc_slot(j, UNIT+cnt*size-IB); - if (idx < 0) { - free_meta(m); - return 0; - } - struct meta *g = ctx.active[j]; - p = enframe(g, idx, UNIT*size_classes[j]-IB, ctx.mmap_counter); - m->maplen = 0; - p[-3] = (p[-3]&31) | (6<<5); - for (int i=0; i<=cnt; i++) - p[UNIT+i*size-4] = 0; - active_idx = cnt-1; - } - ctx.usage_by_class[sc] += cnt; - m->avail_mask = (2u<freed_mask = (2u<<(cnt-1))-1 - m->avail_mask; - m->mem = (void *)p; - m->mem->meta = m; - m->mem->active_idx = active_idx; - m->last_idx = cnt-1; - m->freeable = 1; - m->sizeclass = sc; - return m; -} - -static int alloc_slot(int sc, size_t req) -{ - uint32_t first = try_avail(&ctx.active[sc]); - if (first) return a_ctz_32(first); - - struct meta *g = alloc_group(sc, req); - if (!g) return -1; - - g->avail_mask--; - queue(&ctx.active[sc], g); - return 0; -} - -void *malloc(size_t n) -{ - if (size_overflows(n)) return 0; - struct meta *g; - uint32_t mask, first; - int sc; - int idx; - int ctr; - - if (n >= MMAP_THRESHOLD) { - size_t needed = n + IB + UNIT; - void *p = mmap(0, needed, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANON, -1, 0); - if (p==MAP_FAILED) return 0; - wrlock(); - step_seq(); - g = alloc_meta(); - if (!g) { - unlock(); - munmap(p, needed); - return 0; - } - g->mem = p; - g->mem->meta = g; - g->last_idx = 0; - g->freeable = 1; - g->sizeclass = 63; - g->maplen = (needed+4095)/4096; - g->avail_mask = g->freed_mask = 0; - // use a global counter to cycle offset in - // individually-mmapped allocations. - ctx.mmap_counter++; - idx = 0; - goto success; - } - - sc = size_to_class(n); - - rdlock(); - g = ctx.active[sc]; - - // use coarse size classes initially when there are not yet - // any groups of desired size. this allows counts of 2 or 3 - // to be allocated at first rather than having to start with - // 7 or 5, the min counts for even size classes. - if (!g && sc>=4 && sc<32 && sc!=6 && !(sc&1) && !ctx.usage_by_class[sc]) { - size_t usage = ctx.usage_by_class[sc|1]; - // if a new group may be allocated, count it toward - // usage in deciding if we can use coarse class. - if (!ctx.active[sc|1] || (!ctx.active[sc|1]->avail_mask - && !ctx.active[sc|1]->freed_mask)) - usage += 3; - if (usage <= 12) - sc |= 1; - g = ctx.active[sc]; - } - - for (;;) { - mask = g ? g->avail_mask : 0; - first = mask&-mask; - if (!first) break; - if (RDLOCK_IS_EXCLUSIVE || !MT) - g->avail_mask = mask-first; - else if (a_cas(&g->avail_mask, mask, mask-first)!=mask) - continue; - idx = a_ctz_32(first); - goto success; - } - upgradelock(); - - idx = alloc_slot(sc, n); - if (idx < 0) { - unlock(); - return 0; - } - g = ctx.active[sc]; - -success: - ctr = ctx.mmap_counter; - unlock(); - return enframe(g, idx, n, ctr); -} - -int is_allzero(void *p) -{ - struct meta *g = get_meta(p); - return g->sizeclass >= 48 || - get_stride(g) < UNIT*size_classes[g->sizeclass]; -} diff --git a/lib/libc/musl/src/malloc/mallocng/malloc_usable_size.c b/lib/libc/musl/src/malloc/mallocng/malloc_usable_size.c deleted file mode 100644 index ce6a960c6f..0000000000 --- a/lib/libc/musl/src/malloc/mallocng/malloc_usable_size.c +++ /dev/null @@ -1,13 +0,0 @@ -#include -#include "meta.h" - -size_t malloc_usable_size(void *p) -{ - if (!p) return 0; - struct meta *g = get_meta(p); - int idx = get_slot_index(p); - size_t stride = get_stride(g); - unsigned char *start = g->mem->storage + stride*idx; - unsigned char *end = start + stride - IB; - return get_nominal_size(p, end); -} diff --git a/lib/libc/musl/src/malloc/mallocng/meta.h b/lib/libc/musl/src/malloc/mallocng/meta.h deleted file mode 100644 index 61ec53f9a5..0000000000 --- a/lib/libc/musl/src/malloc/mallocng/meta.h +++ /dev/null @@ -1,288 +0,0 @@ -#ifndef MALLOC_META_H -#define MALLOC_META_H - -#include -#include -#include -#include "glue.h" - -__attribute__((__visibility__("hidden"))) -extern const uint16_t size_classes[]; - -#define MMAP_THRESHOLD 131052 - -#define UNIT 16 -#define IB 4 - -struct group { - struct meta *meta; - unsigned char active_idx:5; - char pad[UNIT - sizeof(struct meta *) - 1]; - unsigned char storage[]; -}; - -struct meta { - struct meta *prev, *next; - struct group *mem; - volatile int avail_mask, freed_mask; - uintptr_t last_idx:5; - uintptr_t freeable:1; - uintptr_t sizeclass:6; - uintptr_t maplen:8*sizeof(uintptr_t)-12; -}; - -struct meta_area { - uint64_t check; - struct meta_area *next; - int nslots; - struct meta slots[]; -}; - -struct malloc_context { - uint64_t secret; -#ifndef PAGESIZE - size_t pagesize; -#endif - int init_done; - unsigned mmap_counter; - struct meta *free_meta_head; - struct meta *avail_meta; - size_t avail_meta_count, avail_meta_area_count, meta_alloc_shift; - struct meta_area *meta_area_head, *meta_area_tail; - unsigned char *avail_meta_areas; - struct meta *active[48]; - size_t usage_by_class[48]; - uint8_t unmap_seq[32], bounces[32]; - uint8_t seq; - uintptr_t brk; -}; - -__attribute__((__visibility__("hidden"))) -extern struct malloc_context ctx; - -#ifdef PAGESIZE -#define PGSZ PAGESIZE -#else -#define PGSZ ctx.pagesize -#endif - -__attribute__((__visibility__("hidden"))) -struct meta *alloc_meta(void); - -__attribute__((__visibility__("hidden"))) -int is_allzero(void *); - -static inline void queue(struct meta **phead, struct meta *m) -{ - assert(!m->next); - assert(!m->prev); - if (*phead) { - struct meta *head = *phead; - m->next = head; - m->prev = head->prev; - m->next->prev = m->prev->next = m; - } else { - m->prev = m->next = m; - *phead = m; - } -} - -static inline void dequeue(struct meta **phead, struct meta *m) -{ - if (m->next != m) { - m->prev->next = m->next; - m->next->prev = m->prev; - if (*phead == m) *phead = m->next; - } else { - *phead = 0; - } - m->prev = m->next = 0; -} - -static inline struct meta *dequeue_head(struct meta **phead) -{ - struct meta *m = *phead; - if (m) dequeue(phead, m); - return m; -} - -static inline void free_meta(struct meta *m) -{ - *m = (struct meta){0}; - queue(&ctx.free_meta_head, m); -} - -static inline uint32_t activate_group(struct meta *m) -{ - assert(!m->avail_mask); - uint32_t mask, act = (2u<mem->active_idx)-1; - do mask = m->freed_mask; - while (a_cas(&m->freed_mask, mask, mask&~act)!=mask); - return m->avail_mask = mask & act; -} - -static inline int get_slot_index(const unsigned char *p) -{ - return p[-3] & 31; -} - -static inline struct meta *get_meta(const unsigned char *p) -{ - assert(!((uintptr_t)p & 15)); - int offset = *(const uint16_t *)(p - 2); - int index = get_slot_index(p); - if (p[-4]) { - assert(!offset); - offset = *(uint32_t *)(p - 8); - assert(offset > 0xffff); - } - const struct group *base = (const void *)(p - UNIT*offset - UNIT); - const struct meta *meta = base->meta; - assert(meta->mem == base); - assert(index <= meta->last_idx); - assert(!(meta->avail_mask & (1u<freed_mask & (1u<check == ctx.secret); - if (meta->sizeclass < 48) { - assert(offset >= size_classes[meta->sizeclass]*index); - assert(offset < size_classes[meta->sizeclass]*(index+1)); - } else { - assert(meta->sizeclass == 63); - } - if (meta->maplen) { - assert(offset <= meta->maplen*4096UL/UNIT - 1); - } - return (struct meta *)meta; -} - -static inline size_t get_nominal_size(const unsigned char *p, const unsigned char *end) -{ - size_t reserved = p[-3] >> 5; - if (reserved >= 5) { - assert(reserved == 5); - reserved = *(const uint32_t *)(end-4); - assert(reserved >= 5); - assert(!end[-5]); - } - assert(reserved <= end-p); - assert(!*(end-reserved)); - // also check the slot's overflow byte - assert(!*end); - return end-reserved-p; -} - -static inline size_t get_stride(const struct meta *g) -{ - if (!g->last_idx && g->maplen) { - return g->maplen*4096UL - UNIT; - } else { - return UNIT*size_classes[g->sizeclass]; - } -} - -static inline void set_size(unsigned char *p, unsigned char *end, size_t n) -{ - int reserved = end-p-n; - if (reserved) end[-reserved] = 0; - if (reserved >= 5) { - *(uint32_t *)(end-4) = reserved; - end[-5] = 0; - reserved = 5; - } - p[-3] = (p[-3]&31) + (reserved<<5); -} - -static inline void *enframe(struct meta *g, int idx, size_t n, int ctr) -{ - size_t stride = get_stride(g); - size_t slack = (stride-IB-n)/UNIT; - unsigned char *p = g->mem->storage + stride*idx; - unsigned char *end = p+stride-IB; - // cycle offset within slot to increase interval to address - // reuse, facilitate trapping double-free. - int off = (p[-3] ? *(uint16_t *)(p-2) + 1 : ctr) & 255; - assert(!p[-4]); - if (off > slack) { - size_t m = slack; - m |= m>>1; m |= m>>2; m |= m>>4; - off &= m; - if (off > slack) off -= slack+1; - assert(off <= slack); - } - if (off) { - // store offset in unused header at offset zero - // if enframing at non-zero offset. - *(uint16_t *)(p-2) = off; - p[-3] = 7<<5; - p += UNIT*off; - // for nonzero offset there is no permanent check - // byte, so make one. - p[-4] = 0; - } - *(uint16_t *)(p-2) = (size_t)(p-g->mem->storage)/UNIT; - p[-3] = idx; - set_size(p, end, n); - return p; -} - -static inline int size_to_class(size_t n) -{ - n = (n+IB-1)>>4; - if (n<10) return n; - n++; - int i = (28-a_clz_32(n))*4 + 8; - if (n>size_classes[i+1]) i+=2; - if (n>size_classes[i]) i++; - return i; -} - -static inline int size_overflows(size_t n) -{ - if (n >= SIZE_MAX/2 - 4096) { - errno = ENOMEM; - return 1; - } - return 0; -} - -static inline void step_seq(void) -{ - if (ctx.seq==255) { - for (int i=0; i<32; i++) ctx.unmap_seq[i] = 0; - ctx.seq = 1; - } else { - ctx.seq++; - } -} - -static inline void record_seq(int sc) -{ - if (sc-7U < 32) ctx.unmap_seq[sc-7] = ctx.seq; -} - -static inline void account_bounce(int sc) -{ - if (sc-7U < 32) { - int seq = ctx.unmap_seq[sc-7]; - if (seq && ctx.seq-seq < 10) { - if (ctx.bounces[sc-7]+1 < 100) - ctx.bounces[sc-7]++; - else - ctx.bounces[sc-7] = 150; - } - } -} - -static inline void decay_bounces(int sc) -{ - if (sc-7U < 32 && ctx.bounces[sc-7]) - ctx.bounces[sc-7]--; -} - -static inline int is_bouncing(int sc) -{ - return (sc-7U < 32 && ctx.bounces[sc-7] >= 100); -} - -#endif diff --git a/lib/libc/musl/src/malloc/mallocng/realloc.c b/lib/libc/musl/src/malloc/mallocng/realloc.c deleted file mode 100644 index 18769f42d8..0000000000 --- a/lib/libc/musl/src/malloc/mallocng/realloc.c +++ /dev/null @@ -1,51 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include "meta.h" - -void *realloc(void *p, size_t n) -{ - if (!p) return malloc(n); - if (size_overflows(n)) return 0; - - struct meta *g = get_meta(p); - int idx = get_slot_index(p); - size_t stride = get_stride(g); - unsigned char *start = g->mem->storage + stride*idx; - unsigned char *end = start + stride - IB; - size_t old_size = get_nominal_size(p, end); - size_t avail_size = end-(unsigned char *)p; - void *new; - - // only resize in-place if size class matches - if (n <= avail_size && n= g->sizeclass) { - set_size(p, end, n); - return p; - } - - // use mremap if old and new size are both mmap-worthy - if (g->sizeclass>=48 && n>=MMAP_THRESHOLD) { - assert(g->sizeclass==63); - size_t base = (unsigned char *)p-start; - size_t needed = (n + base + UNIT + IB + 4095) & -4096; - new = g->maplen*4096UL == needed ? g->mem : - mremap(g->mem, g->maplen*4096UL, needed, MREMAP_MAYMOVE); - if (new!=MAP_FAILED) { - g->mem = new; - g->maplen = needed/4096; - p = g->mem->storage + base; - end = g->mem->storage + (needed - UNIT) - IB; - *end = 0; - set_size(p, end, n); - return p; - } - } - - new = malloc(n); - if (!new) return 0; - memcpy(new, p, n < old_size ? n : old_size); - free(p); - return new; -} diff --git a/lib/libc/musl/src/malloc/memalign.c b/lib/libc/musl/src/malloc/memalign.c deleted file mode 100644 index 32cd87d812..0000000000 --- a/lib/libc/musl/src/malloc/memalign.c +++ /dev/null @@ -1,7 +0,0 @@ -#define _BSD_SOURCE -#include - -void *memalign(size_t align, size_t len) -{ - return aligned_alloc(align, len); -} diff --git a/lib/libc/musl/src/malloc/oldmalloc/aligned_alloc.c b/lib/libc/musl/src/malloc/oldmalloc/aligned_alloc.c deleted file mode 100644 index 4adca3b4f6..0000000000 --- a/lib/libc/musl/src/malloc/oldmalloc/aligned_alloc.c +++ /dev/null @@ -1,53 +0,0 @@ -#include -#include -#include -#include "malloc_impl.h" - -void *aligned_alloc(size_t align, size_t len) -{ - unsigned char *mem, *new; - - if ((align & -align) != align) { - errno = EINVAL; - return 0; - } - - if (len > SIZE_MAX - align || - (__malloc_replaced && !__aligned_alloc_replaced)) { - errno = ENOMEM; - return 0; - } - - if (align <= SIZE_ALIGN) - return malloc(len); - - if (!(mem = malloc(len + align-1))) - return 0; - - new = (void *)((uintptr_t)mem + align-1 & -align); - if (new == mem) return mem; - - struct chunk *c = MEM_TO_CHUNK(mem); - struct chunk *n = MEM_TO_CHUNK(new); - - if (IS_MMAPPED(c)) { - /* Apply difference between aligned and original - * address to the "extra" field of mmapped chunk. */ - n->psize = c->psize + (new-mem); - n->csize = c->csize - (new-mem); - return new; - } - - struct chunk *t = NEXT_CHUNK(c); - - /* Split the allocated chunk into two chunks. The aligned part - * that will be used has the size in its footer reduced by the - * difference between the aligned and original addresses, and - * the resulting size copied to its header. A new header and - * footer are written for the split-off part to be freed. */ - n->psize = c->csize = C_INUSE | (new-mem); - n->csize = t->psize -= new-mem; - - __bin_chunk(c); - return new; -} diff --git a/lib/libc/musl/src/malloc/oldmalloc/malloc.c b/lib/libc/musl/src/malloc/oldmalloc/malloc.c deleted file mode 100644 index 25d00d44de..0000000000 --- a/lib/libc/musl/src/malloc/oldmalloc/malloc.c +++ /dev/null @@ -1,556 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include "libc.h" -#include "atomic.h" -#include "pthread_impl.h" -#include "malloc_impl.h" -#include "fork_impl.h" - -#define malloc __libc_malloc_impl -#define realloc __libc_realloc -#define free __libc_free - -#if defined(__GNUC__) && defined(__PIC__) -#define inline inline __attribute__((always_inline)) -#endif - -static struct { - volatile uint64_t binmap; - struct bin bins[64]; - volatile int split_merge_lock[2]; -} mal; - -/* Synchronization tools */ - -static inline void lock(volatile int *lk) -{ - int need_locks = libc.need_locks; - if (need_locks) { - while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1); - if (need_locks < 0) libc.need_locks = 0; - } -} - -static inline void unlock(volatile int *lk) -{ - if (lk[0]) { - a_store(lk, 0); - if (lk[1]) __wake(lk, 1, 1); - } -} - -static inline void lock_bin(int i) -{ - lock(mal.bins[i].lock); - if (!mal.bins[i].head) - mal.bins[i].head = mal.bins[i].tail = BIN_TO_CHUNK(i); -} - -static inline void unlock_bin(int i) -{ - unlock(mal.bins[i].lock); -} - -static int first_set(uint64_t x) -{ -#if 1 - return a_ctz_64(x); -#else - static const char debruijn64[64] = { - 0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28, - 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11, - 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10, - 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12 - }; - static const char debruijn32[32] = { - 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, - 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14 - }; - if (sizeof(long) < 8) { - uint32_t y = x; - if (!y) { - y = x>>32; - return 32 + debruijn32[(y&-y)*0x076be629 >> 27]; - } - return debruijn32[(y&-y)*0x076be629 >> 27]; - } - return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58]; -#endif -} - -static const unsigned char bin_tab[60] = { - 32,33,34,35,36,36,37,37,38,38,39,39, - 40,40,40,40,41,41,41,41,42,42,42,42,43,43,43,43, - 44,44,44,44,44,44,44,44,45,45,45,45,45,45,45,45, - 46,46,46,46,46,46,46,46,47,47,47,47,47,47,47,47, -}; - -static int bin_index(size_t x) -{ - x = x / SIZE_ALIGN - 1; - if (x <= 32) return x; - if (x < 512) return bin_tab[x/8-4]; - if (x > 0x1c00) return 63; - return bin_tab[x/128-4] + 16; -} - -static int bin_index_up(size_t x) -{ - x = x / SIZE_ALIGN - 1; - if (x <= 32) return x; - x--; - if (x < 512) return bin_tab[x/8-4] + 1; - return bin_tab[x/128-4] + 17; -} - -#if 0 -void __dump_heap(int x) -{ - struct chunk *c; - int i; - for (c = (void *)mal.heap; CHUNK_SIZE(c); c = NEXT_CHUNK(c)) - fprintf(stderr, "base %p size %zu (%d) flags %d/%d\n", - c, CHUNK_SIZE(c), bin_index(CHUNK_SIZE(c)), - c->csize & 15, - NEXT_CHUNK(c)->psize & 15); - for (i=0; i<64; i++) { - if (mal.bins[i].head != BIN_TO_CHUNK(i) && mal.bins[i].head) { - fprintf(stderr, "bin %d: %p\n", i, mal.bins[i].head); - if (!(mal.binmap & 1ULL< len ? b-len : 0; - if (new>a && old len ? b-len : 0; - if (new>a && old SIZE_MAX/2 - PAGE_SIZE) { - errno = ENOMEM; - return 0; - } - n += -n & PAGE_SIZE-1; - - if (!brk) { - brk = __syscall(SYS_brk, 0); - brk += -brk & PAGE_SIZE-1; - } - - if (n < SIZE_MAX-brk && !traverses_stack_p(brk, brk+n) - && __syscall(SYS_brk, brk+n)==brk+n) { - *pn = n; - brk += n; - return (void *)(brk-n); - } - - size_t min = (size_t)PAGE_SIZE << mmap_step/2; - if (n < min) n = min; - void *area = __mmap(0, n, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (area == MAP_FAILED) return 0; - *pn = n; - mmap_step++; - return area; -} - -static struct chunk *expand_heap(size_t n) -{ - static void *end; - void *p; - struct chunk *w; - - /* The argument n already accounts for the caller's chunk - * overhead needs, but if the heap can't be extended in-place, - * we need room for an extra zero-sized sentinel chunk. */ - n += SIZE_ALIGN; - - p = __expand_heap(&n); - if (!p) return 0; - - /* If not just expanding existing space, we need to make a - * new sentinel chunk below the allocated space. */ - if (p != end) { - /* Valid/safe because of the prologue increment. */ - n -= SIZE_ALIGN; - p = (char *)p + SIZE_ALIGN; - w = MEM_TO_CHUNK(p); - w->psize = 0 | C_INUSE; - } - - /* Record new heap end and fill in footer. */ - end = (char *)p + n; - w = MEM_TO_CHUNK(end); - w->psize = n | C_INUSE; - w->csize = 0 | C_INUSE; - - /* Fill in header, which may be new or may be replacing a - * zero-size sentinel header at the old end-of-heap. */ - w = MEM_TO_CHUNK(p); - w->csize = n | C_INUSE; - - return w; -} - -static int adjust_size(size_t *n) -{ - /* Result of pointer difference must fit in ptrdiff_t. */ - if (*n-1 > PTRDIFF_MAX - SIZE_ALIGN - PAGE_SIZE) { - if (*n) { - errno = ENOMEM; - return -1; - } else { - *n = SIZE_ALIGN; - return 0; - } - } - *n = (*n + OVERHEAD + SIZE_ALIGN - 1) & SIZE_MASK; - return 0; -} - -static void unbin(struct chunk *c, int i) -{ - if (c->prev == c->next) - a_and_64(&mal.binmap, ~(1ULL<prev->next = c->next; - c->next->prev = c->prev; - c->csize |= C_INUSE; - NEXT_CHUNK(c)->psize |= C_INUSE; -} - -static void bin_chunk(struct chunk *self, int i) -{ - self->next = BIN_TO_CHUNK(i); - self->prev = mal.bins[i].tail; - self->next->prev = self; - self->prev->next = self; - if (self->prev == BIN_TO_CHUNK(i)) - a_or_64(&mal.binmap, 1ULL<= n1 - DONTCARE) return; - - next = NEXT_CHUNK(self); - split = (void *)((char *)self + n); - - split->psize = n | C_INUSE; - split->csize = n1-n; - next->psize = n1-n; - self->csize = n | C_INUSE; - - int i = bin_index(n1-n); - lock_bin(i); - - bin_chunk(split, i); - - unlock_bin(i); -} - -void *malloc(size_t n) -{ - struct chunk *c; - int i, j; - uint64_t mask; - - if (adjust_size(&n) < 0) return 0; - - if (n > MMAP_THRESHOLD) { - size_t len = n + OVERHEAD + PAGE_SIZE - 1 & -PAGE_SIZE; - char *base = __mmap(0, len, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (base == (void *)-1) return 0; - c = (void *)(base + SIZE_ALIGN - OVERHEAD); - c->csize = len - (SIZE_ALIGN - OVERHEAD); - c->psize = SIZE_ALIGN - OVERHEAD; - return CHUNK_TO_MEM(c); - } - - i = bin_index_up(n); - if (i<63 && (mal.binmap & (1ULL<psize; - char *base = (char *)self - extra; - size_t oldlen = n0 + extra; - size_t newlen = n + extra; - /* Crash on realloc of freed chunk */ - if (extra & 1) a_crash(); - if (newlen < PAGE_SIZE && (new = malloc(n-OVERHEAD))) { - n0 = n; - goto copy_free_ret; - } - newlen = (newlen + PAGE_SIZE-1) & -PAGE_SIZE; - if (oldlen == newlen) return p; - base = __mremap(base, oldlen, newlen, MREMAP_MAYMOVE); - if (base == (void *)-1) - goto copy_realloc; - self = (void *)(base + extra); - self->csize = newlen - extra; - return CHUNK_TO_MEM(self); - } - - next = NEXT_CHUNK(self); - - /* Crash on corrupted footer (likely from buffer overflow) */ - if (next->psize != self->csize) a_crash(); - - if (n < n0) { - int i = bin_index_up(n); - int j = bin_index(n0); - if (icsize = split->psize = n | C_INUSE; - split->csize = next->psize = n0-n | C_INUSE; - __bin_chunk(split); - return CHUNK_TO_MEM(self); - } - - lock(mal.split_merge_lock); - - size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next); - if (n0+nsize >= n) { - int i = bin_index(nsize); - lock_bin(i); - if (!(next->csize & C_INUSE)) { - unbin(next, i); - unlock_bin(i); - next = NEXT_CHUNK(next); - self->csize = next->psize = n0+nsize | C_INUSE; - trim(self, n); - unlock(mal.split_merge_lock); - return CHUNK_TO_MEM(self); - } - unlock_bin(i); - } - unlock(mal.split_merge_lock); - -copy_realloc: - /* As a last resort, allocate a new chunk and copy to it. */ - new = malloc(n-OVERHEAD); - if (!new) return 0; -copy_free_ret: - memcpy(new, p, (npsize != self->csize) a_crash(); - - lock(mal.split_merge_lock); - - size_t osize = CHUNK_SIZE(self), size = osize; - - /* Since we hold split_merge_lock, only transition from free to - * in-use can race; in-use to free is impossible */ - size_t psize = self->psize & C_INUSE ? 0 : CHUNK_PSIZE(self); - size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next); - - if (psize) { - int i = bin_index(psize); - lock_bin(i); - if (!(self->psize & C_INUSE)) { - struct chunk *prev = PREV_CHUNK(self); - unbin(prev, i); - self = prev; - size += psize; - } - unlock_bin(i); - } - if (nsize) { - int i = bin_index(nsize); - lock_bin(i); - if (!(next->csize & C_INUSE)) { - unbin(next, i); - next = NEXT_CHUNK(next); - size += nsize; - } - unlock_bin(i); - } - - int i = bin_index(size); - lock_bin(i); - - self->csize = size; - next->psize = size; - bin_chunk(self, i); - unlock(mal.split_merge_lock); - - /* Replace middle of large chunks with fresh zero pages */ - if (size > RECLAIM && (size^(size-osize)) > size-osize) { - uintptr_t a = (uintptr_t)self + SIZE_ALIGN+PAGE_SIZE-1 & -PAGE_SIZE; - uintptr_t b = (uintptr_t)next - SIZE_ALIGN & -PAGE_SIZE; - int e = errno; -#if 1 - __madvise((void *)a, b-a, MADV_DONTNEED); -#else - __mmap((void *)a, b-a, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0); -#endif - errno = e; - } - - unlock_bin(i); -} - -static void unmap_chunk(struct chunk *self) -{ - size_t extra = self->psize; - char *base = (char *)self - extra; - size_t len = CHUNK_SIZE(self) + extra; - /* Crash on double free */ - if (extra & 1) a_crash(); - int e = errno; - __munmap(base, len); - errno = e; -} - -void free(void *p) -{ - if (!p) return; - - struct chunk *self = MEM_TO_CHUNK(p); - - if (IS_MMAPPED(self)) - unmap_chunk(self); - else - __bin_chunk(self); -} - -void __malloc_donate(char *start, char *end) -{ - size_t align_start_up = (SIZE_ALIGN-1) & (-(uintptr_t)start - OVERHEAD); - size_t align_end_down = (SIZE_ALIGN-1) & (uintptr_t)end; - - /* Getting past this condition ensures that the padding for alignment - * and header overhead will not overflow and will leave a nonzero - * multiple of SIZE_ALIGN bytes between start and end. */ - if (end - start <= OVERHEAD + align_start_up + align_end_down) - return; - start += align_start_up + OVERHEAD; - end -= align_end_down; - - struct chunk *c = MEM_TO_CHUNK(start), *n = MEM_TO_CHUNK(end); - c->psize = n->csize = C_INUSE; - c->csize = n->psize = C_INUSE | (end-start); - __bin_chunk(c); -} - -void __malloc_atfork(int who) -{ - if (who<0) { - lock(mal.split_merge_lock); - for (int i=0; i<64; i++) - lock(mal.bins[i].lock); - } else if (!who) { - for (int i=0; i<64; i++) - unlock(mal.bins[i].lock); - unlock(mal.split_merge_lock); - } else { - for (int i=0; i<64; i++) - mal.bins[i].lock[0] = mal.bins[i].lock[1] = 0; - mal.split_merge_lock[1] = 0; - mal.split_merge_lock[0] = 0; - } -} diff --git a/lib/libc/musl/src/malloc/oldmalloc/malloc_impl.h b/lib/libc/musl/src/malloc/oldmalloc/malloc_impl.h deleted file mode 100644 index e1cf4774c1..0000000000 --- a/lib/libc/musl/src/malloc/oldmalloc/malloc_impl.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef MALLOC_IMPL_H -#define MALLOC_IMPL_H - -#include -#include "dynlink.h" - -struct chunk { - size_t psize, csize; - struct chunk *next, *prev; -}; - -struct bin { - volatile int lock[2]; - struct chunk *head; - struct chunk *tail; -}; - -#define SIZE_ALIGN (4*sizeof(size_t)) -#define SIZE_MASK (-SIZE_ALIGN) -#define OVERHEAD (2*sizeof(size_t)) -#define MMAP_THRESHOLD (0x1c00*SIZE_ALIGN) -#define DONTCARE 16 -#define RECLAIM 163840 - -#define CHUNK_SIZE(c) ((c)->csize & -2) -#define CHUNK_PSIZE(c) ((c)->psize & -2) -#define PREV_CHUNK(c) ((struct chunk *)((char *)(c) - CHUNK_PSIZE(c))) -#define NEXT_CHUNK(c) ((struct chunk *)((char *)(c) + CHUNK_SIZE(c))) -#define MEM_TO_CHUNK(p) (struct chunk *)((char *)(p) - OVERHEAD) -#define CHUNK_TO_MEM(c) (void *)((char *)(c) + OVERHEAD) -#define BIN_TO_CHUNK(i) (MEM_TO_CHUNK(&mal.bins[i].head)) - -#define C_INUSE ((size_t)1) - -#define IS_MMAPPED(c) !((c)->csize & (C_INUSE)) - -hidden void __bin_chunk(struct chunk *); - -#endif diff --git a/lib/libc/musl/src/malloc/oldmalloc/malloc_usable_size.c b/lib/libc/musl/src/malloc/oldmalloc/malloc_usable_size.c deleted file mode 100644 index 672b518ad0..0000000000 --- a/lib/libc/musl/src/malloc/oldmalloc/malloc_usable_size.c +++ /dev/null @@ -1,9 +0,0 @@ -#include -#include "malloc_impl.h" - -hidden void *(*const __realloc_dep)(void *, size_t) = realloc; - -size_t malloc_usable_size(void *p) -{ - return p ? CHUNK_SIZE(MEM_TO_CHUNK(p)) - OVERHEAD : 0; -} diff --git a/lib/libc/musl/src/malloc/posix_memalign.c b/lib/libc/musl/src/malloc/posix_memalign.c deleted file mode 100644 index ad4d8f4730..0000000000 --- a/lib/libc/musl/src/malloc/posix_memalign.c +++ /dev/null @@ -1,11 +0,0 @@ -#include -#include - -int posix_memalign(void **res, size_t align, size_t len) -{ - if (align < sizeof(void *)) return EINVAL; - void *mem = aligned_alloc(align, len); - if (!mem) return errno; - *res = mem; - return 0; -} diff --git a/lib/libc/musl/src/malloc/realloc.c b/lib/libc/musl/src/malloc/realloc.c deleted file mode 100644 index fb0e8b7c47..0000000000 --- a/lib/libc/musl/src/malloc/realloc.c +++ /dev/null @@ -1,6 +0,0 @@ -#include - -void *realloc(void *p, size_t n) -{ - return __libc_realloc(p, n); -} diff --git a/lib/libc/musl/src/malloc/reallocarray.c b/lib/libc/musl/src/malloc/reallocarray.c deleted file mode 100644 index 4a6ebe4604..0000000000 --- a/lib/libc/musl/src/malloc/reallocarray.c +++ /dev/null @@ -1,13 +0,0 @@ -#define _BSD_SOURCE -#include -#include - -void *reallocarray(void *ptr, size_t m, size_t n) -{ - if (n && m > -1 / n) { - errno = ENOMEM; - return 0; - } - - return realloc(ptr, m * n); -} diff --git a/lib/libc/musl/src/malloc/replaced.c b/lib/libc/musl/src/malloc/replaced.c deleted file mode 100644 index 07fce61ec0..0000000000 --- a/lib/libc/musl/src/malloc/replaced.c +++ /dev/null @@ -1,4 +0,0 @@ -#include "dynlink.h" - -int __malloc_replaced; -int __aligned_alloc_replaced; diff --git a/lib/libc/wasi/emmalloc/emmalloc.c b/lib/libc/wasi/emmalloc/emmalloc.c deleted file mode 100644 index e97ef44dc0..0000000000 --- a/lib/libc/wasi/emmalloc/emmalloc.c +++ /dev/null @@ -1,1540 +0,0 @@ -/* - * Copyright 2018 The Emscripten Authors. All rights reserved. - * Emscripten is available under two separate licenses, the MIT license and the - * University of Illinois/NCSA Open Source License. Both these licenses can be - * found in the LICENSE file. - * - * Simple minimalistic but efficient sbrk()-based malloc/free that works in - * singlethreaded and multithreaded builds. - * - * Assumptions: - * - * - sbrk() is used to claim new memory (sbrk handles geometric/linear - * - overallocation growth) - * - sbrk() can be used by other code outside emmalloc. - * - sbrk() is very fast in most cases (internal wasm call). - * - sbrk() returns pointers with an alignment of alignof(max_align_t) - * - * Invariants: - * - * - Per-allocation header overhead is 8 bytes, smallest allocated payload - * amount is 8 bytes, and a multiple of 4 bytes. - * - Acquired memory blocks are subdivided into disjoint regions that lie - * next to each other. - * - A region is either in used or free. - * Used regions may be adjacent, and a used and unused region - * may be adjacent, but not two unused ones - they would be - * merged. - * - Memory allocation takes constant time, unless the alloc needs to sbrk() - * or memory is very close to being exhausted. - * - * Debugging: - * - * - If not NDEBUG, runtime assert()s are in use. - * - If EMMALLOC_MEMVALIDATE is defined, a large amount of extra checks are done. - * - If EMMALLOC_VERBOSE is defined, a lot of operations are logged - * out, in addition to EMMALLOC_MEMVALIDATE. - * - Debugging and logging directly uses console.log via uses EM_ASM, not - * printf etc., to minimize any risk of debugging or logging depending on - * malloc. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __EMSCRIPTEN_TRACING__ -#include -#endif - -// Defind by the linker to have the address of the start of the heap. -extern unsigned char __heap_base; -extern unsigned char __heap_end; - -// Behavior of right shifting a signed integer is compiler implementation defined. -static_assert((((int32_t)0x80000000U) >> 31) == -1, "This malloc implementation requires that right-shifting a signed integer produces a sign-extending (arithmetic) shift!"); - -// Configuration: specifies the minimum alignment that malloc()ed memory outputs. Allocation requests with smaller alignment -// than this will yield an allocation with this much alignment. -#define MALLOC_ALIGNMENT alignof(max_align_t) -static_assert(alignof(max_align_t) == 16, "max_align_t must be correct"); - -#define EMMALLOC_EXPORT __attribute__((weak)) - -#define MIN(x, y) ((x) < (y) ? (x) : (y)) -#define MAX(x, y) ((x) > (y) ? (x) : (y)) - -#define NUM_FREE_BUCKETS 64 -#define BUCKET_BITMASK_T uint64_t - -// Dynamic memory is subdivided into regions, in the format - -// ..... | ..... | ..... | ..... - -// That is, at the bottom and top end of each memory region, the size of that region is stored. That allows traversing the -// memory regions backwards and forwards. Because each allocation must be at least a multiple of 4 bytes, the lowest two bits of -// each size field is unused. Free regions are distinguished by used regions by having the FREE_REGION_FLAG bit present -// in the size field. I.e. for free regions, the size field is odd, and for used regions, the size field reads even. -#define FREE_REGION_FLAG 0x1u - -// Attempts to malloc() more than this many bytes would cause an overflow when calculating the size of a region, -// therefore allocations larger than this are short-circuited immediately on entry. -#define MAX_ALLOC_SIZE 0xFFFFFFC7u - -// A free region has the following structure: -// ... - -typedef struct Region -{ - size_t size; - // Use a circular doubly linked list to represent free region data. - struct Region *prev, *next; - // ... N bytes of free data - size_t _at_the_end_of_this_struct_size; // do not dereference, this is present for convenient struct sizeof() computation only -} Region; - -// Each memory block starts with a RootRegion at the beginning. -// The RootRegion specifies the size of the region block, and forms a linked -// list of all RootRegions in the program, starting with `listOfAllRegions` -// below. -typedef struct RootRegion -{ - uint32_t size; - struct RootRegion *next; - uint8_t* endPtr; -} RootRegion; - -#if defined(__EMSCRIPTEN_PTHREADS__) -// In multithreaded builds, use a simple global spinlock strategy to acquire/release access to the memory allocator. -static volatile uint8_t multithreadingLock = 0; -#define MALLOC_ACQUIRE() while(__sync_lock_test_and_set(&multithreadingLock, 1)) { while(multithreadingLock) { /*nop*/ } } -#define MALLOC_RELEASE() __sync_lock_release(&multithreadingLock) -// Test code to ensure we have tight malloc acquire/release guards in place. -#define ASSERT_MALLOC_IS_ACQUIRED() assert(multithreadingLock == 1) -#else -// In singlethreaded builds, no need for locking. -#define MALLOC_ACQUIRE() ((void)0) -#define MALLOC_RELEASE() ((void)0) -#define ASSERT_MALLOC_IS_ACQUIRED() ((void)0) -#endif - -#define IS_POWER_OF_2(val) (((val) & ((val)-1)) == 0) -#define ALIGN_UP(ptr, alignment) ((uint8_t*)((((uintptr_t)(ptr)) + ((alignment)-1)) & ~((alignment)-1))) -#define HAS_ALIGNMENT(ptr, alignment) ((((uintptr_t)(ptr)) & ((alignment)-1)) == 0) - -static_assert(IS_POWER_OF_2(MALLOC_ALIGNMENT), "MALLOC_ALIGNMENT must be a power of two value!"); -static_assert(MALLOC_ALIGNMENT >= 4, "Smallest possible MALLOC_ALIGNMENT if 4!"); - -// A region that contains as payload a single forward linked list of pointers to -// root regions of each disjoint region blocks. -static RootRegion *listOfAllRegions = NULL; - -// For each of the buckets, maintain a linked list head node. The head node for each -// free region is a sentinel node that does not actually represent any free space, but -// the sentinel is used to avoid awkward testing against (if node == freeRegionHeadNode) -// when adding and removing elements from the linked list, i.e. we are guaranteed that -// the sentinel node is always fixed and there, and the actual free region list elements -// start at freeRegionBuckets[i].next each. -static Region freeRegionBuckets[NUM_FREE_BUCKETS] = { - { .prev = &freeRegionBuckets[0], .next = &freeRegionBuckets[0] }, - { .prev = &freeRegionBuckets[1], .next = &freeRegionBuckets[1] }, - { .prev = &freeRegionBuckets[2], .next = &freeRegionBuckets[2] }, - { .prev = &freeRegionBuckets[3], .next = &freeRegionBuckets[3] }, - { .prev = &freeRegionBuckets[4], .next = &freeRegionBuckets[4] }, - { .prev = &freeRegionBuckets[5], .next = &freeRegionBuckets[5] }, - { .prev = &freeRegionBuckets[6], .next = &freeRegionBuckets[6] }, - { .prev = &freeRegionBuckets[7], .next = &freeRegionBuckets[7] }, - { .prev = &freeRegionBuckets[8], .next = &freeRegionBuckets[8] }, - { .prev = &freeRegionBuckets[9], .next = &freeRegionBuckets[9] }, - { .prev = &freeRegionBuckets[10], .next = &freeRegionBuckets[10] }, - { .prev = &freeRegionBuckets[11], .next = &freeRegionBuckets[11] }, - { .prev = &freeRegionBuckets[12], .next = &freeRegionBuckets[12] }, - { .prev = &freeRegionBuckets[13], .next = &freeRegionBuckets[13] }, - { .prev = &freeRegionBuckets[14], .next = &freeRegionBuckets[14] }, - { .prev = &freeRegionBuckets[15], .next = &freeRegionBuckets[15] }, - { .prev = &freeRegionBuckets[16], .next = &freeRegionBuckets[16] }, - { .prev = &freeRegionBuckets[17], .next = &freeRegionBuckets[17] }, - { .prev = &freeRegionBuckets[18], .next = &freeRegionBuckets[18] }, - { .prev = &freeRegionBuckets[19], .next = &freeRegionBuckets[19] }, - { .prev = &freeRegionBuckets[20], .next = &freeRegionBuckets[20] }, - { .prev = &freeRegionBuckets[21], .next = &freeRegionBuckets[21] }, - { .prev = &freeRegionBuckets[22], .next = &freeRegionBuckets[22] }, - { .prev = &freeRegionBuckets[23], .next = &freeRegionBuckets[23] }, - { .prev = &freeRegionBuckets[24], .next = &freeRegionBuckets[24] }, - { .prev = &freeRegionBuckets[25], .next = &freeRegionBuckets[25] }, - { .prev = &freeRegionBuckets[26], .next = &freeRegionBuckets[26] }, - { .prev = &freeRegionBuckets[27], .next = &freeRegionBuckets[27] }, - { .prev = &freeRegionBuckets[28], .next = &freeRegionBuckets[28] }, - { .prev = &freeRegionBuckets[29], .next = &freeRegionBuckets[29] }, - { .prev = &freeRegionBuckets[30], .next = &freeRegionBuckets[30] }, - { .prev = &freeRegionBuckets[31], .next = &freeRegionBuckets[31] }, - { .prev = &freeRegionBuckets[32], .next = &freeRegionBuckets[32] }, - { .prev = &freeRegionBuckets[33], .next = &freeRegionBuckets[33] }, - { .prev = &freeRegionBuckets[34], .next = &freeRegionBuckets[34] }, - { .prev = &freeRegionBuckets[35], .next = &freeRegionBuckets[35] }, - { .prev = &freeRegionBuckets[36], .next = &freeRegionBuckets[36] }, - { .prev = &freeRegionBuckets[37], .next = &freeRegionBuckets[37] }, - { .prev = &freeRegionBuckets[38], .next = &freeRegionBuckets[38] }, - { .prev = &freeRegionBuckets[39], .next = &freeRegionBuckets[39] }, - { .prev = &freeRegionBuckets[40], .next = &freeRegionBuckets[40] }, - { .prev = &freeRegionBuckets[41], .next = &freeRegionBuckets[41] }, - { .prev = &freeRegionBuckets[42], .next = &freeRegionBuckets[42] }, - { .prev = &freeRegionBuckets[43], .next = &freeRegionBuckets[43] }, - { .prev = &freeRegionBuckets[44], .next = &freeRegionBuckets[44] }, - { .prev = &freeRegionBuckets[45], .next = &freeRegionBuckets[45] }, - { .prev = &freeRegionBuckets[46], .next = &freeRegionBuckets[46] }, - { .prev = &freeRegionBuckets[47], .next = &freeRegionBuckets[47] }, - { .prev = &freeRegionBuckets[48], .next = &freeRegionBuckets[48] }, - { .prev = &freeRegionBuckets[49], .next = &freeRegionBuckets[49] }, - { .prev = &freeRegionBuckets[50], .next = &freeRegionBuckets[50] }, - { .prev = &freeRegionBuckets[51], .next = &freeRegionBuckets[51] }, - { .prev = &freeRegionBuckets[52], .next = &freeRegionBuckets[52] }, - { .prev = &freeRegionBuckets[53], .next = &freeRegionBuckets[53] }, - { .prev = &freeRegionBuckets[54], .next = &freeRegionBuckets[54] }, - { .prev = &freeRegionBuckets[55], .next = &freeRegionBuckets[55] }, - { .prev = &freeRegionBuckets[56], .next = &freeRegionBuckets[56] }, - { .prev = &freeRegionBuckets[57], .next = &freeRegionBuckets[57] }, - { .prev = &freeRegionBuckets[58], .next = &freeRegionBuckets[58] }, - { .prev = &freeRegionBuckets[59], .next = &freeRegionBuckets[59] }, - { .prev = &freeRegionBuckets[60], .next = &freeRegionBuckets[60] }, - { .prev = &freeRegionBuckets[61], .next = &freeRegionBuckets[61] }, - { .prev = &freeRegionBuckets[62], .next = &freeRegionBuckets[62] }, - { .prev = &freeRegionBuckets[63], .next = &freeRegionBuckets[63] }, -}; - -// A bitmask that tracks the population status for each of the 64 distinct memory regions: -// a zero at bit position i means that the free list bucket i is empty. This bitmask is -// used to avoid redundant scanning of the 64 different free region buckets: instead by -// looking at the bitmask we can find in constant time an index to a free region bucket -// that contains free memory of desired size. -static BUCKET_BITMASK_T freeRegionBucketsUsed = 0; - -// Amount of bytes taken up by allocation header data -#define REGION_HEADER_SIZE (2*sizeof(size_t)) - -// Smallest allocation size that is possible is 2*pointer size, since payload of each region must at least contain space -// to store the free region linked list prev and next pointers. An allocation size smaller than this will be rounded up -// to this size. -#define SMALLEST_ALLOCATION_SIZE (2*sizeof(void*)) - -/* Subdivide regions of free space into distinct circular doubly linked lists, where each linked list -represents a range of free space blocks. The following function compute_free_list_bucket() converts -an allocation size to the bucket index that should be looked at. The buckets are grouped as follows: - - Bucket 0: [8, 15], range size=8 - Bucket 1: [16, 23], range size=8 - Bucket 2: [24, 31], range size=8 - Bucket 3: [32, 39], range size=8 - Bucket 4: [40, 47], range size=8 - Bucket 5: [48, 55], range size=8 - Bucket 6: [56, 63], range size=8 - Bucket 7: [64, 71], range size=8 - Bucket 8: [72, 79], range size=8 - Bucket 9: [80, 87], range size=8 - Bucket 10: [88, 95], range size=8 - Bucket 11: [96, 103], range size=8 - Bucket 12: [104, 111], range size=8 - Bucket 13: [112, 119], range size=8 - Bucket 14: [120, 159], range size=40 - Bucket 15: [160, 191], range size=32 - Bucket 16: [192, 223], range size=32 - Bucket 17: [224, 255], range size=32 - Bucket 18: [256, 319], range size=64 - Bucket 19: [320, 383], range size=64 - Bucket 20: [384, 447], range size=64 - Bucket 21: [448, 511], range size=64 - Bucket 22: [512, 639], range size=128 - Bucket 23: [640, 767], range size=128 - Bucket 24: [768, 895], range size=128 - Bucket 25: [896, 1023], range size=128 - Bucket 26: [1024, 1279], range size=256 - Bucket 27: [1280, 1535], range size=256 - Bucket 28: [1536, 1791], range size=256 - Bucket 29: [1792, 2047], range size=256 - Bucket 30: [2048, 2559], range size=512 - Bucket 31: [2560, 3071], range size=512 - Bucket 32: [3072, 3583], range size=512 - Bucket 33: [3584, 6143], range size=2560 - Bucket 34: [6144, 8191], range size=2048 - Bucket 35: [8192, 12287], range size=4096 - Bucket 36: [12288, 16383], range size=4096 - Bucket 37: [16384, 24575], range size=8192 - Bucket 38: [24576, 32767], range size=8192 - Bucket 39: [32768, 49151], range size=16384 - Bucket 40: [49152, 65535], range size=16384 - Bucket 41: [65536, 98303], range size=32768 - Bucket 42: [98304, 131071], range size=32768 - Bucket 43: [131072, 196607], range size=65536 - Bucket 44: [196608, 262143], range size=65536 - Bucket 45: [262144, 393215], range size=131072 - Bucket 46: [393216, 524287], range size=131072 - Bucket 47: [524288, 786431], range size=262144 - Bucket 48: [786432, 1048575], range size=262144 - Bucket 49: [1048576, 1572863], range size=524288 - Bucket 50: [1572864, 2097151], range size=524288 - Bucket 51: [2097152, 3145727], range size=1048576 - Bucket 52: [3145728, 4194303], range size=1048576 - Bucket 53: [4194304, 6291455], range size=2097152 - Bucket 54: [6291456, 8388607], range size=2097152 - Bucket 55: [8388608, 12582911], range size=4194304 - Bucket 56: [12582912, 16777215], range size=4194304 - Bucket 57: [16777216, 25165823], range size=8388608 - Bucket 58: [25165824, 33554431], range size=8388608 - Bucket 59: [33554432, 50331647], range size=16777216 - Bucket 60: [50331648, 67108863], range size=16777216 - Bucket 61: [67108864, 100663295], range size=33554432 - Bucket 62: [100663296, 134217727], range size=33554432 - Bucket 63: 134217728 bytes and larger. */ -static_assert(NUM_FREE_BUCKETS == 64, "Following function is tailored specifically for NUM_FREE_BUCKETS == 64 case"); -static int compute_free_list_bucket(size_t allocSize) -{ - if (allocSize < 128) return (allocSize >> 3) - 1; - int clz = __builtin_clz(allocSize); - int bucketIndex = (clz > 19) ? 110 - (clz<<2) + ((allocSize >> (29-clz)) ^ 4) : MIN(71 - (clz<<1) + ((allocSize >> (30-clz)) ^ 2), NUM_FREE_BUCKETS-1); - assert(bucketIndex >= 0); - assert(bucketIndex < NUM_FREE_BUCKETS); - return bucketIndex; -} - -#define DECODE_CEILING_SIZE(size) ((size_t)((size) & ~FREE_REGION_FLAG)) - -static Region *prev_region(Region *region) -{ - size_t prevRegionSize = ((size_t*)region)[-1]; - prevRegionSize = DECODE_CEILING_SIZE(prevRegionSize); - return (Region*)((uint8_t*)region - prevRegionSize); -} - -static Region *next_region(Region *region) -{ - return (Region*)((uint8_t*)region + region->size); -} - -static size_t region_ceiling_size(Region *region) -{ - return ((size_t*)((uint8_t*)region + region->size))[-1]; -} - -static bool region_is_free(Region *r) -{ - return region_ceiling_size(r) & FREE_REGION_FLAG; -} - -static bool region_is_in_use(Region *r) -{ - return r->size == region_ceiling_size(r); -} - -static size_t size_of_region_from_ceiling(Region *r) -{ - size_t size = region_ceiling_size(r); - return DECODE_CEILING_SIZE(size); -} - -static bool debug_region_is_consistent(Region *r) -{ - assert(r); - size_t sizeAtBottom = r->size; - size_t sizeAtCeiling = size_of_region_from_ceiling(r); - return sizeAtBottom == sizeAtCeiling; -} - -static uint8_t *region_payload_start_ptr(Region *region) -{ - return (uint8_t*)region + sizeof(size_t); -} - -static uint8_t *region_payload_end_ptr(Region *region) -{ - return (uint8_t*)region + region->size - sizeof(size_t); -} - -static void create_used_region(void *ptr, size_t size) -{ - assert(ptr); - assert(HAS_ALIGNMENT(ptr, sizeof(size_t))); - assert(HAS_ALIGNMENT(size, sizeof(size_t))); - assert(size >= sizeof(Region)); - *(size_t*)ptr = size; - ((size_t*)ptr)[(size/sizeof(size_t))-1] = size; -} - -static void create_free_region(void *ptr, size_t size) -{ - assert(ptr); - assert(HAS_ALIGNMENT(ptr, sizeof(size_t))); - assert(HAS_ALIGNMENT(size, sizeof(size_t))); - assert(size >= sizeof(Region)); - Region *freeRegion = (Region*)ptr; - freeRegion->size = size; - ((size_t*)ptr)[(size/sizeof(size_t))-1] = size | FREE_REGION_FLAG; -} - -static void prepend_to_free_list(Region *region, Region *prependTo) -{ - assert(region); - assert(prependTo); - // N.b. the region we are prepending to is always the sentinel node, - // which represents a dummy node that is technically not a free node, so - // region_is_free(prependTo) does not hold. - assert(region_is_free((Region*)region)); - region->next = prependTo; - region->prev = prependTo->prev; - assert(region->prev); - prependTo->prev = region; - region->prev->next = region; -} - -static void unlink_from_free_list(Region *region) -{ - assert(region); - assert(region_is_free((Region*)region)); - assert(region->prev); - assert(region->next); - region->prev->next = region->next; - region->next->prev = region->prev; -} - -static void link_to_free_list(Region *freeRegion) -{ - assert(freeRegion); - assert(freeRegion->size >= sizeof(Region)); - int bucketIndex = compute_free_list_bucket(freeRegion->size-REGION_HEADER_SIZE); - Region *freeListHead = freeRegionBuckets + bucketIndex; - freeRegion->prev = freeListHead; - freeRegion->next = freeListHead->next; - assert(freeRegion->next); - freeListHead->next = freeRegion; - freeRegion->next->prev = freeRegion; - freeRegionBucketsUsed |= ((BUCKET_BITMASK_T)1) << bucketIndex; -} - -#if 0 -static void dump_memory_regions() -{ - ASSERT_MALLOC_IS_ACQUIRED(); - RootRegion *root = listOfAllRegions; - MAIN_THREAD_ASYNC_EM_ASM(console.log('All memory regions:')); - while(root) - { - Region *r = (Region*)root; - assert(debug_region_is_consistent(r)); - uint8_t *lastRegionEnd = root->endPtr; - MAIN_THREAD_ASYNC_EM_ASM(console.log('Region block 0x'+($0>>>0).toString(16)+' - 0x'+($1>>>0).toString(16)+ ' ('+($2>>>0)+' bytes):'), - r, lastRegionEnd, lastRegionEnd-(uint8_t*)r); - while((uint8_t*)r < lastRegionEnd) - { - MAIN_THREAD_ASYNC_EM_ASM(console.log('Region 0x'+($0>>>0).toString(16)+', size: '+($1>>>0)+' ('+($2?"used":"--FREE--")+')'), - r, r->size, region_ceiling_size(r) == r->size); - - assert(debug_region_is_consistent(r)); - size_t sizeFromCeiling = size_of_region_from_ceiling(r); - if (sizeFromCeiling != r->size) - MAIN_THREAD_ASYNC_EM_ASM(console.log('Corrupt region! Size marker at the end of the region does not match: '+($0>>>0)), sizeFromCeiling); - if (r->size == 0) - break; - r = next_region(r); - } - root = root->next; - MAIN_THREAD_ASYNC_EM_ASM(console.log("")); - } - MAIN_THREAD_ASYNC_EM_ASM(console.log('Free regions:')); - for(int i = 0; i < NUM_FREE_BUCKETS; ++i) - { - Region *prev = &freeRegionBuckets[i]; - Region *fr = freeRegionBuckets[i].next; - while(fr != &freeRegionBuckets[i]) - { - MAIN_THREAD_ASYNC_EM_ASM(console.log('In bucket '+$0+', free region 0x'+($1>>>0).toString(16)+', size: ' + ($2>>>0) + ' (size at ceiling: '+($3>>>0)+'), prev: 0x' + ($4>>>0).toString(16) + ', next: 0x' + ($5>>>0).toString(16)), - i, fr, fr->size, size_of_region_from_ceiling(fr), fr->prev, fr->next); - assert(debug_region_is_consistent(fr)); - assert(region_is_free(fr)); - assert(fr->prev == prev); - prev = fr; - assert(fr->next != fr); - assert(fr->prev != fr); - fr = fr->next; - } - } - MAIN_THREAD_ASYNC_EM_ASM(console.log('Free bucket index map: ' + ($0>>>0).toString(2) + ' ' + ($1>>>0).toString(2)), (uint32_t)(freeRegionBucketsUsed >> 32), (uint32_t)freeRegionBucketsUsed); - MAIN_THREAD_ASYNC_EM_ASM(console.log("")); -} - -void emmalloc_dump_memory_regions() -{ - MALLOC_ACQUIRE(); - dump_memory_regions(); - MALLOC_RELEASE(); -} - -static int validate_memory_regions() -{ - ASSERT_MALLOC_IS_ACQUIRED(); - RootRegion *root = listOfAllRegions; - while(root) - { - Region *r = (Region*)root; - if (!debug_region_is_consistent(r)) - { - MAIN_THREAD_ASYNC_EM_ASM(console.error('Used region 0x'+($0>>>0).toString(16)+', size: '+($1>>>0)+' ('+($2?"used":"--FREE--")+') is corrupt (size markers in the beginning and at the end of the region do not match!)'), - r, r->size, region_ceiling_size(r) == r->size); - return 1; - } - uint8_t *lastRegionEnd = root->endPtr; - while((uint8_t*)r < lastRegionEnd) - { - if (!debug_region_is_consistent(r)) - { - MAIN_THREAD_ASYNC_EM_ASM(console.error('Used region 0x'+($0>>>0).toString(16)+', size: '+($1>>>0)+' ('+($2?"used":"--FREE--")+') is corrupt (size markers in the beginning and at the end of the region do not match!)'), - r, r->size, region_ceiling_size(r) == r->size); - return 1; - } - if (r->size == 0) - break; - r = next_region(r); - } - root = root->next; - } - for(int i = 0; i < NUM_FREE_BUCKETS; ++i) - { - Region *prev = &freeRegionBuckets[i]; - Region *fr = freeRegionBuckets[i].next; - while(fr != &freeRegionBuckets[i]) - { - if (!debug_region_is_consistent(fr) || !region_is_free(fr) || fr->prev != prev || fr->next == fr || fr->prev == fr) - { - MAIN_THREAD_ASYNC_EM_ASM(console.log('In bucket '+$0+', free region 0x'+($1>>>0).toString(16)+', size: ' + ($2>>>0) + ' (size at ceiling: '+($3>>>0)+'), prev: 0x' + ($4>>>0).toString(16) + ', next: 0x' + ($5>>>0).toString(16) + ' is corrupt!'), - i, fr, fr->size, size_of_region_from_ceiling(fr), fr->prev, fr->next); - return 1; - } - prev = fr; - fr = fr->next; - } - } - return 0; -} - -int emmalloc_validate_memory_regions() -{ - MALLOC_ACQUIRE(); - int memoryError = validate_memory_regions(); - MALLOC_RELEASE(); - return memoryError; -} -#endif - -static bool claim_more_memory(size_t numBytes) -{ -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('claim_more_memory(numBytes='+($0>>>0)+ ')'), numBytes); -#endif - -#ifdef EMMALLOC_MEMVALIDATE - validate_memory_regions(); -#endif - - uint8_t *startPtr; - uint8_t *endPtr; - do { - // If this is the first time we're called, see if we can use - // the initial heap memory set up by wasm-ld. - if (!listOfAllRegions) { - unsigned char *heap_base = &__heap_base; - unsigned char *heap_end = &__heap_end; - if (heap_end < heap_base) { - __builtin_trap(); - } - if (numBytes <= (size_t)(heap_end - heap_base)) { - startPtr = heap_base; - endPtr = heap_end; - break; - } - } - - // Round numBytes up to the nearest page size. - numBytes = (numBytes + (PAGE_SIZE-1)) & -PAGE_SIZE; - - // Claim memory via sbrk - startPtr = (uint8_t*)sbrk(numBytes); - if ((intptr_t)startPtr == -1) - { -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.error('claim_more_memory: sbrk failed!')); -#endif - return false; - } -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('claim_more_memory: claimed 0x' + ($0>>>0).toString(16) + ' - 0x' + ($1>>>0).toString(16) + ' (' + ($2>>>0) + ' bytes) via sbrk()'), startPtr, startPtr + numBytes, numBytes); -#endif - assert(HAS_ALIGNMENT(startPtr, alignof(size_t))); - endPtr = startPtr + numBytes; - } while (0); - - // Create a sentinel region at the end of the new heap block - Region *endSentinelRegion = (Region*)(endPtr - sizeof(Region)); - create_used_region(endSentinelRegion, sizeof(Region)); - - // If we are the sole user of sbrk(), it will feed us continuous/consecutive memory addresses - take advantage - // of that if so: instead of creating two disjoint memory regions blocks, expand the previous one to a larger size. - uint8_t *previousSbrkEndAddress = listOfAllRegions ? listOfAllRegions->endPtr : 0; - if (startPtr == previousSbrkEndAddress) - { - Region *prevEndSentinel = prev_region((Region*)startPtr); - assert(debug_region_is_consistent(prevEndSentinel)); - assert(region_is_in_use(prevEndSentinel)); - Region *prevRegion = prev_region(prevEndSentinel); - assert(debug_region_is_consistent(prevRegion)); - - listOfAllRegions->endPtr = endPtr; - - // Two scenarios, either the last region of the previous block was in use, in which case we need to create - // a new free region in the newly allocated space; or it was free, in which case we can extend that region - // to cover a larger size. - if (region_is_free(prevRegion)) - { - size_t newFreeRegionSize = (uint8_t*)endSentinelRegion - (uint8_t*)prevRegion; - unlink_from_free_list(prevRegion); - create_free_region(prevRegion, newFreeRegionSize); - link_to_free_list(prevRegion); - return true; - } - // else: last region of the previous block was in use. Since we are joining two consecutive sbrk() blocks, - // we can swallow the end sentinel of the previous block away. - startPtr -= sizeof(Region); - } - else - { - // Create a root region at the start of the heap block - create_used_region(startPtr, sizeof(Region)); - - // Dynamic heap start region: - RootRegion *newRegionBlock = (RootRegion*)startPtr; - newRegionBlock->next = listOfAllRegions; // Pointer to next region block head - newRegionBlock->endPtr = endPtr; // Pointer to the end address of this region block - listOfAllRegions = newRegionBlock; - startPtr += sizeof(Region); - } - - // Create a new memory region for the new claimed free space. - create_free_region(startPtr, (uint8_t*)endSentinelRegion - startPtr); - link_to_free_list((Region*)startPtr); - return true; -} - -#if 0 -// Initialize emmalloc during static initialization. -// See system/lib/README.md for static constructor ordering. -__attribute__((constructor(47))) -static void initialize_emmalloc_heap() -{ - // Initialize circular doubly linked lists representing free space - // Never useful to unroll this for loop, just takes up code size. -#pragma clang loop unroll(disable) - for(int i = 0; i < NUM_FREE_BUCKETS; ++i) - freeRegionBuckets[i].prev = freeRegionBuckets[i].next = &freeRegionBuckets[i]; - -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('initialize_emmalloc_heap()')); -#endif - - // Start with a tiny dynamic region. - claim_more_memory(3*sizeof(Region)); -} - -void emmalloc_blank_slate_from_orbit() -{ - MALLOC_ACQUIRE(); - listOfAllRegions = NULL; - freeRegionBucketsUsed = 0; - initialize_emmalloc_heap(); - MALLOC_RELEASE(); -} -#endif - -static void *attempt_allocate(Region *freeRegion, size_t alignment, size_t size) -{ - ASSERT_MALLOC_IS_ACQUIRED(); - assert(freeRegion); - // Look at the next potential free region to allocate into. - // First, we should check if the free region has enough of payload bytes contained - // in it to accommodate the new allocation. This check needs to take account the - // requested allocation alignment, so the payload memory area needs to be rounded - // upwards to the desired alignment. - uint8_t *payloadStartPtr = region_payload_start_ptr(freeRegion); - uint8_t *payloadStartPtrAligned = ALIGN_UP(payloadStartPtr, alignment); - uint8_t *payloadEndPtr = region_payload_end_ptr(freeRegion); - - // Do we have enough free space, taking into account alignment? - if (payloadStartPtrAligned + size > payloadEndPtr) - return NULL; - - // We have enough free space, so the memory allocation will be made into this region. Remove this free region - // from the list of free regions: whatever slop remains will be later added back to the free region pool. - unlink_from_free_list(freeRegion); - - // Before we proceed further, fix up the boundary of this region and the region that precedes this one, - // so that the boundary between the two regions happens at a right spot for the payload to be aligned. - if (payloadStartPtr != payloadStartPtrAligned) - { - Region *prevRegion = prev_region((Region*)freeRegion); - // We never have two free regions adjacent to each other, so the region before this free - // region should be in use. - assert(region_is_in_use(prevRegion)); - size_t regionBoundaryBumpAmount = payloadStartPtrAligned - payloadStartPtr; - size_t newThisRegionSize = freeRegion->size - regionBoundaryBumpAmount; - create_used_region(prevRegion, prevRegion->size + regionBoundaryBumpAmount); - freeRegion = (Region *)((uint8_t*)freeRegion + regionBoundaryBumpAmount); - freeRegion->size = newThisRegionSize; - } - // Next, we need to decide whether this region is so large that it should be split into two regions, - // one representing the newly used memory area, and at the high end a remaining leftover free area. - // This splitting to two is done always if there is enough space for the high end to fit a region. - // Carve 'size' bytes of payload off this region. So, - // [sz prev next sz] - // becomes - // [sz payload sz] [sz prev next sz] - if (sizeof(Region) + REGION_HEADER_SIZE + size <= freeRegion->size) - { - // There is enough space to keep a free region at the end of the carved out block - // -> construct the new block - Region *newFreeRegion = (Region *)((uint8_t*)freeRegion + REGION_HEADER_SIZE + size); - create_free_region(newFreeRegion, freeRegion->size - size - REGION_HEADER_SIZE); - link_to_free_list(newFreeRegion); - - // Recreate the resized Region under its new size. - create_used_region(freeRegion, size + REGION_HEADER_SIZE); - } - else - { - // There is not enough space to split the free memory region into used+free parts, so consume the whole - // region as used memory, not leaving a free memory region behind. - // Initialize the free region as used by resetting the ceiling size to the same value as the size at bottom. - ((size_t*)((uint8_t*)freeRegion + freeRegion->size))[-1] = freeRegion->size; - } - -#ifdef __EMSCRIPTEN_TRACING__ - emscripten_trace_record_allocation(freeRegion, freeRegion->size); -#endif - -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_allocate - succeeded allocating memory, region ptr=0x' + ($0>>>0).toString(16) + ', align=' + $1 + ', payload size=' + ($2>>>0) + ' bytes)'), freeRegion, alignment, size); -#endif - - return (uint8_t*)freeRegion + sizeof(size_t); -} - -static size_t validate_alloc_alignment(size_t alignment) -{ - // Cannot perform allocations that are less than 4 byte aligned, because the Region - // control structures need to be aligned. Also round up to minimum outputted alignment. - alignment = MAX(alignment, MALLOC_ALIGNMENT); - // Arbitrary upper limit on alignment - very likely a programming bug if alignment is higher than this. - assert(alignment <= 1024*1024); - return alignment; -} - -static size_t validate_alloc_size(size_t size) -{ - assert(size + REGION_HEADER_SIZE > size); - - // Allocation sizes must be a multiple of pointer sizes, and at least 2*sizeof(pointer). - size_t validatedSize = size > SMALLEST_ALLOCATION_SIZE ? (size_t)ALIGN_UP(size, sizeof(Region*)) : SMALLEST_ALLOCATION_SIZE; - assert(validatedSize >= size); // 32-bit wraparound should not occur, too large sizes should be stopped before - - return validatedSize; -} - -static void *allocate_memory(size_t alignment, size_t size) -{ - ASSERT_MALLOC_IS_ACQUIRED(); - -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('allocate_memory(align=' + $0 + ', size=' + ($1>>>0) + ' bytes)'), alignment, size); -#endif - -#ifdef EMMALLOC_MEMVALIDATE - validate_memory_regions(); -#endif - - if (!IS_POWER_OF_2(alignment)) - { -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: alignment not power of 2!')); -#endif - return 0; - } - - if (size > MAX_ALLOC_SIZE) - { -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size); -#endif - return 0; - } - - alignment = validate_alloc_alignment(alignment); - size = validate_alloc_size(size); - - // Attempt to allocate memory starting from smallest bucket that can contain the required amount of memory. - // Under normal alignment conditions this should always be the first or second bucket we look at, but if - // performing an allocation with complex alignment, we may need to look at multiple buckets. - int bucketIndex = compute_free_list_bucket(size); - BUCKET_BITMASK_T bucketMask = freeRegionBucketsUsed >> bucketIndex; - - // Loop through each bucket that has free regions in it, based on bits set in freeRegionBucketsUsed bitmap. - while(bucketMask) - { - BUCKET_BITMASK_T indexAdd = __builtin_ctzll(bucketMask); - bucketIndex += indexAdd; - bucketMask >>= indexAdd; - assert(bucketIndex >= 0); - assert(bucketIndex <= NUM_FREE_BUCKETS-1); - assert(freeRegionBucketsUsed & (((BUCKET_BITMASK_T)1) << bucketIndex)); - - Region *freeRegion = freeRegionBuckets[bucketIndex].next; - assert(freeRegion); - if (freeRegion != &freeRegionBuckets[bucketIndex]) - { - void *ptr = attempt_allocate(freeRegion, alignment, size); - if (ptr) - return ptr; - - // We were not able to allocate from the first region found in this bucket, so penalize - // the region by cycling it to the end of the doubly circular linked list. (constant time) - // This provides a randomized guarantee that when performing allocations of size k to a - // bucket of [k-something, k+something] range, we will not always attempt to satisfy the - // allocation from the same available region at the front of the list, but we try each - // region in turn. - unlink_from_free_list(freeRegion); - prepend_to_free_list(freeRegion, &freeRegionBuckets[bucketIndex]); - // But do not stick around to attempt to look at other regions in this bucket - move - // to search the next populated bucket index if this did not fit. This gives a practical - // "allocation in constant time" guarantee, since the next higher bucket will only have - // regions that are all of strictly larger size than the requested allocation. Only if - // there is a difficult alignment requirement we may fail to perform the allocation from - // a region in the next bucket, and if so, we keep trying higher buckets until one of them - // works. - ++bucketIndex; - bucketMask >>= 1; - } - else - { - // This bucket was not populated after all with any regions, - // but we just had a stale bit set to mark a populated bucket. - // Reset the bit to update latest status so that we do not - // redundantly look at this bucket again. - freeRegionBucketsUsed &= ~(((BUCKET_BITMASK_T)1) << bucketIndex); - bucketMask ^= 1; - } - // Instead of recomputing bucketMask from scratch at the end of each loop, it is updated as we go, - // to avoid undefined behavior with (x >> 32)/(x >> 64) when bucketIndex reaches 32/64, (the shift would comes out as a no-op instead of 0). - - assert((bucketIndex == NUM_FREE_BUCKETS && bucketMask == 0) || (bucketMask == freeRegionBucketsUsed >> bucketIndex)); - } - - // None of the buckets were able to accommodate an allocation. If this happens we are almost out of memory. - // The largest bucket might contain some suitable regions, but we only looked at one region in that bucket, so - // as a last resort, loop through more free regions in the bucket that represents the largest allocations available. - // But only if the bucket representing largest allocations available is not any of the first thirty buckets, - // these represent allocatable areas less than <1024 bytes - which could be a lot of scrap. - // In such case, prefer to sbrk() in more memory right away. - int largestBucketIndex = NUM_FREE_BUCKETS - 1 - __builtin_clzll(freeRegionBucketsUsed); - // freeRegion will be null if there is absolutely no memory left. (all buckets are 100% used) - Region *freeRegion = freeRegionBucketsUsed ? freeRegionBuckets[largestBucketIndex].next : 0; - if (freeRegionBucketsUsed >> 30) - { - // Look only at a constant number of regions in this bucket max, to avoid bad worst case behavior. - // If this many regions cannot find free space, we give up and prefer to sbrk() more instead. - const int maxRegionsToTryBeforeGivingUp = 99; - int numTriesLeft = maxRegionsToTryBeforeGivingUp; - while(freeRegion != &freeRegionBuckets[largestBucketIndex] && numTriesLeft-- > 0) - { - void *ptr = attempt_allocate(freeRegion, alignment, size); - if (ptr) - return ptr; - freeRegion = freeRegion->next; - } - } - - // We were unable to find a free memory region. Must sbrk() in more memory! - size_t numBytesToClaim = size+sizeof(Region)*3; - assert(numBytesToClaim > size); // 32-bit wraparound should not happen here, allocation size has been validated above! - bool success = claim_more_memory(numBytesToClaim); - if (success) - return allocate_memory(alignment, size); // Recurse back to itself to try again - - // also sbrk() failed, we are really really constrained :( As a last resort, go back to looking at the - // bucket we already looked at above, continuing where the above search left off - perhaps there are - // regions we overlooked the first time that might be able to satisfy the allocation. - if (freeRegion) - { - while(freeRegion != &freeRegionBuckets[largestBucketIndex]) - { - void *ptr = attempt_allocate(freeRegion, alignment, size); - if (ptr) - return ptr; - freeRegion = freeRegion->next; - } - } - -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('Could not find a free memory block!')); -#endif - - return 0; -} - -static -void *emmalloc_memalign(size_t alignment, size_t size) -{ - MALLOC_ACQUIRE(); - void *ptr = allocate_memory(alignment, size); - MALLOC_RELEASE(); - return ptr; -} - -#if 0 -void * EMMALLOC_EXPORT memalign(size_t alignment, size_t size) -{ - return emmalloc_memalign(alignment, size); -} -#endif - -void * EMMALLOC_EXPORT aligned_alloc(size_t alignment, size_t size) -{ - if ((alignment % sizeof(void *) != 0) || (size % alignment) != 0) - return 0; - return emmalloc_memalign(alignment, size); -} - -static -void *emmalloc_malloc(size_t size) -{ - return emmalloc_memalign(MALLOC_ALIGNMENT, size); -} - -void * EMMALLOC_EXPORT malloc(size_t size) -{ - return emmalloc_malloc(size); -} - -static -size_t emmalloc_usable_size(void *ptr) -{ - if (!ptr) - return 0; - - uint8_t *regionStartPtr = (uint8_t*)ptr - sizeof(size_t); - Region *region = (Region*)(regionStartPtr); - assert(HAS_ALIGNMENT(region, sizeof(size_t))); - - MALLOC_ACQUIRE(); - - size_t size = region->size; - assert(size >= sizeof(Region)); - assert(region_is_in_use(region)); - - MALLOC_RELEASE(); - - return size - REGION_HEADER_SIZE; -} - -size_t EMMALLOC_EXPORT malloc_usable_size(void *ptr) -{ - return emmalloc_usable_size(ptr); -} - -static -void emmalloc_free(void *ptr) -{ -#ifdef EMMALLOC_MEMVALIDATE - emmalloc_validate_memory_regions(); -#endif - - if (!ptr) - return; - -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('free(ptr=0x'+($0>>>0).toString(16)+')'), ptr); -#endif - - uint8_t *regionStartPtr = (uint8_t*)ptr - sizeof(size_t); - Region *region = (Region*)(regionStartPtr); - assert(HAS_ALIGNMENT(region, sizeof(size_t))); - - MALLOC_ACQUIRE(); - - size_t size = region->size; -#ifdef EMMALLOC_VERBOSE - if (size < sizeof(Region) || !region_is_in_use(region)) - { - if (debug_region_is_consistent(region)) - // LLVM wasm backend bug: cannot use MAIN_THREAD_ASYNC_EM_ASM() here, that generates internal compiler error - // Reproducible by running e.g. other.test_alloc_3GB - EM_ASM(console.error('Double free at region ptr 0x' + ($0>>>0).toString(16) + ', region->size: 0x' + ($1>>>0).toString(16) + ', region->sizeAtCeiling: 0x' + ($2>>>0).toString(16) + ')'), region, size, region_ceiling_size(region)); - else - MAIN_THREAD_ASYNC_EM_ASM(console.error('Corrupt region at region ptr 0x' + ($0>>>0).toString(16) + ' region->size: 0x' + ($1>>>0).toString(16) + ', region->sizeAtCeiling: 0x' + ($2>>>0).toString(16) + ')'), region, size, region_ceiling_size(region)); - } -#endif - assert(size >= sizeof(Region)); - assert(region_is_in_use(region)); - -#ifdef __EMSCRIPTEN_TRACING__ - emscripten_trace_record_free(region); -#endif - - // Check merging with left side - size_t prevRegionSizeField = ((size_t*)region)[-1]; - size_t prevRegionSize = prevRegionSizeField & ~FREE_REGION_FLAG; - if (prevRegionSizeField != prevRegionSize) // Previous region is free? - { - Region *prevRegion = (Region*)((uint8_t*)region - prevRegionSize); - assert(debug_region_is_consistent(prevRegion)); - unlink_from_free_list(prevRegion); - regionStartPtr = (uint8_t*)prevRegion; - size += prevRegionSize; - } - - // Check merging with right side - Region *nextRegion = next_region(region); - assert(debug_region_is_consistent(nextRegion)); - size_t sizeAtEnd = *(size_t*)region_payload_end_ptr(nextRegion); - if (nextRegion->size != sizeAtEnd) - { - unlink_from_free_list(nextRegion); - size += nextRegion->size; - } - - create_free_region(regionStartPtr, size); - link_to_free_list((Region*)regionStartPtr); - - MALLOC_RELEASE(); - -#ifdef EMMALLOC_MEMVALIDATE - emmalloc_validate_memory_regions(); -#endif -} - -void EMMALLOC_EXPORT free(void *ptr) -{ - emmalloc_free(ptr); -} - -// Can be called to attempt to increase or decrease the size of the given region -// to a new size (in-place). Returns 1 if resize succeeds, and 0 on failure. -static int attempt_region_resize(Region *region, size_t size) -{ - ASSERT_MALLOC_IS_ACQUIRED(); - assert(size > 0); - assert(HAS_ALIGNMENT(size, sizeof(size_t))); - -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_region_resize(region=0x' + ($0>>>0).toString(16) + ', size=' + ($1>>>0) + ' bytes)'), region, size); -#endif - - // First attempt to resize this region, if the next region that follows this one - // is a free region. - Region *nextRegion = next_region(region); - uint8_t *nextRegionEndPtr = (uint8_t*)nextRegion + nextRegion->size; - size_t sizeAtCeiling = ((size_t*)nextRegionEndPtr)[-1]; - if (nextRegion->size != sizeAtCeiling) // Next region is free? - { - assert(region_is_free(nextRegion)); - uint8_t *newNextRegionStartPtr = (uint8_t*)region + size; - assert(HAS_ALIGNMENT(newNextRegionStartPtr, sizeof(size_t))); - // Next region does not shrink to too small size? - if (newNextRegionStartPtr + sizeof(Region) <= nextRegionEndPtr) - { - unlink_from_free_list(nextRegion); - create_free_region(newNextRegionStartPtr, nextRegionEndPtr - newNextRegionStartPtr); - link_to_free_list((Region*)newNextRegionStartPtr); - create_used_region(region, newNextRegionStartPtr - (uint8_t*)region); - return 1; - } - // If we remove the next region altogether, allocation is satisfied? - if (newNextRegionStartPtr <= nextRegionEndPtr) - { - unlink_from_free_list(nextRegion); - create_used_region(region, region->size + nextRegion->size); - return 1; - } - } - else - { - // Next region is an used region - we cannot change its starting address. However if we are shrinking the - // size of this region, we can create a new free region between this and the next used region. - if (size + sizeof(Region) <= region->size) - { - size_t freeRegionSize = region->size - size; - create_used_region(region, size); - Region *freeRegion = (Region *)((uint8_t*)region + size); - create_free_region(freeRegion, freeRegionSize); - link_to_free_list(freeRegion); - return 1; - } - else if (size <= region->size) - { - // Caller was asking to shrink the size, but due to not being able to fit a full Region in the shrunk - // area, we cannot actually do anything. This occurs if the shrink amount is really small. In such case, - // just call it success without doing any work. - return 1; - } - } -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_region_resize failed.')); -#endif - return 0; -} - -static int acquire_and_attempt_region_resize(Region *region, size_t size) -{ - MALLOC_ACQUIRE(); - int success = attempt_region_resize(region, size); - MALLOC_RELEASE(); - return success; -} - -static -void *emmalloc_aligned_realloc(void *ptr, size_t alignment, size_t size) -{ -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('aligned_realloc(ptr=0x' + ($0>>>0).toString(16) + ', alignment=' + $1 + ', size=' + ($2>>>0)), ptr, alignment, size); -#endif - - if (!ptr) - return emmalloc_memalign(alignment, size); - - if (size == 0) - { - free(ptr); - return 0; - } - - if (size > MAX_ALLOC_SIZE) - { -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size); -#endif - return 0; - } - - assert(IS_POWER_OF_2(alignment)); - // aligned_realloc() cannot be used to ask to change the alignment of a pointer. - assert(HAS_ALIGNMENT(ptr, alignment)); - size = validate_alloc_size(size); - - // Calculate the region start address of the original allocation - Region *region = (Region*)((uint8_t*)ptr - sizeof(size_t)); - - // First attempt to resize the given region to avoid having to copy memory around - if (acquire_and_attempt_region_resize(region, size + REGION_HEADER_SIZE)) - { -#ifdef __EMSCRIPTEN_TRACING__ - emscripten_trace_record_reallocation(ptr, ptr, size); -#endif - return ptr; - } - - // If resize failed, we must allocate a new region, copy the data over, and then - // free the old region. - void *newptr = emmalloc_memalign(alignment, size); - if (newptr) - { - memcpy(newptr, ptr, MIN(size, region->size - REGION_HEADER_SIZE)); - free(ptr); - } - // N.B. If there is not enough memory, the old memory block should not be freed and - // null pointer is returned. - return newptr; -} - -#if 0 -void * EMMALLOC_EXPORT aligned_realloc(void *ptr, size_t alignment, size_t size) -{ - return emmalloc_aligned_realloc(ptr, alignment, size); -} -#endif - -#if 0 -// realloc_try() is like realloc(), but only attempts to try to resize the existing memory -// area. If resizing the existing memory area fails, then realloc_try() will return 0 -// (the original memory block is not freed or modified). If resizing succeeds, previous -// memory contents will be valid up to min(old length, new length) bytes. -void *emmalloc_realloc_try(void *ptr, size_t size) -{ - if (!ptr) - return 0; - - if (size == 0) - { - free(ptr); - return 0; - } - - if (size > MAX_ALLOC_SIZE) - { -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size); -#endif - return 0; - } - - size = validate_alloc_size(size); - - // Calculate the region start address of the original allocation - Region *region = (Region*)((uint8_t*)ptr - sizeof(size_t)); - - // Attempt to resize the given region to avoid having to copy memory around - int success = acquire_and_attempt_region_resize(region, size + REGION_HEADER_SIZE); -#ifdef __EMSCRIPTEN_TRACING__ - if (success) - emscripten_trace_record_reallocation(ptr, ptr, size); -#endif - return success ? ptr : 0; -} - -// emmalloc_aligned_realloc_uninitialized() is like aligned_realloc(), but old memory contents -// will be undefined after reallocation. (old memory is not preserved in any case) -void *emmalloc_aligned_realloc_uninitialized(void *ptr, size_t alignment, size_t size) -{ - if (!ptr) - return emmalloc_memalign(alignment, size); - - if (size == 0) - { - free(ptr); - return 0; - } - - if (size > MAX_ALLOC_SIZE) - { -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size); -#endif - return 0; - } - - size = validate_alloc_size(size); - - // Calculate the region start address of the original allocation - Region *region = (Region*)((uint8_t*)ptr - sizeof(size_t)); - - // First attempt to resize the given region to avoid having to copy memory around - if (acquire_and_attempt_region_resize(region, size + REGION_HEADER_SIZE)) - { -#ifdef __EMSCRIPTEN_TRACING__ - emscripten_trace_record_reallocation(ptr, ptr, size); -#endif - return ptr; - } - - // If resize failed, drop the old region and allocate a new region. Memory is not - // copied over - free(ptr); - return emmalloc_memalign(alignment, size); -} -#endif - -static -void *emmalloc_realloc(void *ptr, size_t size) -{ - return emmalloc_aligned_realloc(ptr, MALLOC_ALIGNMENT, size); -} - -void * EMMALLOC_EXPORT realloc(void *ptr, size_t size) -{ - return emmalloc_realloc(ptr, size); -} - -#if 0 -// realloc_uninitialized() is like realloc(), but old memory contents -// will be undefined after reallocation. (old memory is not preserved in any case) -void *emmalloc_realloc_uninitialized(void *ptr, size_t size) -{ - return emmalloc_aligned_realloc_uninitialized(ptr, MALLOC_ALIGNMENT, size); -} -#endif - -static -int emmalloc_posix_memalign(void **memptr, size_t alignment, size_t size) -{ - assert(memptr); - if (alignment % sizeof(void *) != 0) - return 22/* EINVAL*/; - *memptr = emmalloc_memalign(alignment, size); - return *memptr ? 0 : 12/*ENOMEM*/; -} - -int EMMALLOC_EXPORT posix_memalign(void **memptr, size_t alignment, size_t size) -{ - return emmalloc_posix_memalign(memptr, alignment, size); -} - -static -void *emmalloc_calloc(size_t num, size_t size) -{ - size_t bytes = num*size; - void *ptr = emmalloc_memalign(MALLOC_ALIGNMENT, bytes); - if (ptr) - memset(ptr, 0, bytes); - return ptr; -} - -void * EMMALLOC_EXPORT calloc(size_t num, size_t size) -{ - return emmalloc_calloc(num, size); -} - -#if 0 -static int count_linked_list_size(Region *list) -{ - int size = 1; - for(Region *i = list->next; i != list; list = list->next) - ++size; - return size; -} - -static size_t count_linked_list_space(Region *list) -{ - size_t space = 0; - for(Region *i = list->next; i != list; list = list->next) - space += region_payload_end_ptr(i) - region_payload_start_ptr(i); - return space; -} - -struct mallinfo emmalloc_mallinfo() -{ - MALLOC_ACQUIRE(); - - struct mallinfo info; - // Non-mmapped space allocated (bytes): For emmalloc, - // let's define this as the difference between heap size and dynamic top end. - info.arena = emscripten_get_heap_size() - (size_t)sbrk(0); - // Number of "ordinary" blocks. Let's define this as the number of highest - // size blocks. (subtract one from each, since there is a sentinel node in each list) - info.ordblks = count_linked_list_size(&freeRegionBuckets[NUM_FREE_BUCKETS-1])-1; - // Number of free "fastbin" blocks. For emmalloc, define this as the number - // of blocks that are not in the largest pristine block. - info.smblks = 0; - // The total number of bytes in free "fastbin" blocks. - info.fsmblks = 0; - for(int i = 0; i < NUM_FREE_BUCKETS-1; ++i) - { - info.smblks += count_linked_list_size(&freeRegionBuckets[i])-1; - info.fsmblks += count_linked_list_space(&freeRegionBuckets[i]); - } - - info.hblks = 0; // Number of mmapped regions: always 0. (no mmap support) - info.hblkhd = 0; // Amount of bytes in mmapped regions: always 0. (no mmap support) - - // Walk through all the heap blocks to report the following data: - // The "highwater mark" for allocated space—that is, the maximum amount of - // space that was ever allocated. Emmalloc does not want to pay code to - // track this, so this is only reported from current allocation data, and - // may not be accurate. - info.usmblks = 0; - info.uordblks = 0; // The total number of bytes used by in-use allocations. - info.fordblks = 0; // The total number of bytes in free blocks. - // The total amount of releasable free space at the top of the heap. - // This is the maximum number of bytes that could ideally be released by malloc_trim(3). - Region *lastActualRegion = prev_region((Region*)(listOfAllRegions->endPtr - sizeof(Region))); - info.keepcost = region_is_free(lastActualRegion) ? lastActualRegion->size : 0; - - RootRegion *root = listOfAllRegions; - while(root) - { - Region *r = (Region*)root; - assert(debug_region_is_consistent(r)); - uint8_t *lastRegionEnd = root->endPtr; - while((uint8_t*)r < lastRegionEnd) - { - assert(debug_region_is_consistent(r)); - - if (region_is_free(r)) - { - // Count only the payload of the free block towards free memory. - info.fordblks += region_payload_end_ptr(r) - region_payload_start_ptr(r); - // But the header data of the free block goes towards used memory. - info.uordblks += REGION_HEADER_SIZE; - } - else - { - info.uordblks += r->size; - } - // Update approximate watermark data - info.usmblks = MAX(info.usmblks, (intptr_t)r + r->size); - - if (r->size == 0) - break; - r = next_region(r); - } - root = root->next; - } - - MALLOC_RELEASE(); - return info; -} - -struct mallinfo EMMALLOC_EXPORT mallinfo() -{ - return emmalloc_mallinfo(); -} - -// Note! This function is not fully multithreadin safe: while this function is running, other threads should not be -// allowed to call sbrk()! -static int trim_dynamic_heap_reservation(size_t pad) -{ - ASSERT_MALLOC_IS_ACQUIRED(); - - if (!listOfAllRegions) - return 0; // emmalloc is not controlling any dynamic memory at all - cannot release memory. - uint8_t *previousSbrkEndAddress = listOfAllRegions->endPtr; - assert(sbrk(0) == previousSbrkEndAddress); - size_t lastMemoryRegionSize = ((size_t*)previousSbrkEndAddress)[-1]; - assert(lastMemoryRegionSize == 16); // // The last memory region should be a sentinel node of exactly 16 bytes in size. - Region *endSentinelRegion = (Region*)(previousSbrkEndAddress - sizeof(Region)); - Region *lastActualRegion = prev_region(endSentinelRegion); - - // Round padding up to multiple of 4 bytes to keep sbrk() and memory region alignment intact. - // Also have at least 8 bytes of payload so that we can form a full free region. - size_t newRegionSize = (size_t)ALIGN_UP(pad, 4); - if (pad > 0) - newRegionSize += sizeof(Region) - (newRegionSize - pad); - - if (!region_is_free(lastActualRegion) || lastActualRegion->size <= newRegionSize) - return 0; // Last actual region is in use, or caller desired to leave more free memory intact than there is. - - // This many bytes will be shrunk away. - size_t shrinkAmount = lastActualRegion->size - newRegionSize; - assert(HAS_ALIGNMENT(shrinkAmount, 4)); - - unlink_from_free_list(lastActualRegion); - // If pad == 0, we should delete the last free region altogether. If pad > 0, - // shrink the last free region to the desired size. - if (newRegionSize > 0) - { - create_free_region(lastActualRegion, newRegionSize); - link_to_free_list(lastActualRegion); - } - - // Recreate the sentinel region at the end of the last free region - endSentinelRegion = (Region*)((uint8_t*)lastActualRegion + newRegionSize); - create_used_region(endSentinelRegion, sizeof(Region)); - - // And update the size field of the whole region block. - listOfAllRegions->endPtr = (uint8_t*)endSentinelRegion + sizeof(Region); - - // Finally call sbrk() to shrink the memory area. - void *oldSbrk = sbrk(-(intptr_t)shrinkAmount); - assert((intptr_t)oldSbrk != -1); // Shrinking with sbrk() should never fail. - assert(oldSbrk == previousSbrkEndAddress); // Another thread should not have raced to increase sbrk() on us! - - // All successful, and we actually trimmed memory! - return 1; -} - -int emmalloc_trim(size_t pad) -{ - MALLOC_ACQUIRE(); - int success = trim_dynamic_heap_reservation(pad); - MALLOC_RELEASE(); - return success; -} - -int EMMALLOC_EXPORT malloc_trim(size_t pad) -{ - return emmalloc_trim(pad); -} - -size_t emmalloc_dynamic_heap_size() -{ - size_t dynamicHeapSize = 0; - - MALLOC_ACQUIRE(); - RootRegion *root = listOfAllRegions; - while(root) - { - dynamicHeapSize += root->endPtr - (uint8_t*)root; - root = root->next; - } - MALLOC_RELEASE(); - return dynamicHeapSize; -} - -size_t emmalloc_free_dynamic_memory() -{ - size_t freeDynamicMemory = 0; - - int bucketIndex = 0; - - MALLOC_ACQUIRE(); - BUCKET_BITMASK_T bucketMask = freeRegionBucketsUsed; - - // Loop through each bucket that has free regions in it, based on bits set in freeRegionBucketsUsed bitmap. - while(bucketMask) - { - BUCKET_BITMASK_T indexAdd = __builtin_ctzll(bucketMask); - bucketIndex += indexAdd; - bucketMask >>= indexAdd; - for(Region *freeRegion = freeRegionBuckets[bucketIndex].next; - freeRegion != &freeRegionBuckets[bucketIndex]; - freeRegion = freeRegion->next) - { - freeDynamicMemory += freeRegion->size - REGION_HEADER_SIZE; - } - ++bucketIndex; - bucketMask >>= 1; - } - MALLOC_RELEASE(); - return freeDynamicMemory; -} - -size_t emmalloc_compute_free_dynamic_memory_fragmentation_map(size_t freeMemorySizeMap[32]) -{ - memset((void*)freeMemorySizeMap, 0, sizeof(freeMemorySizeMap[0])*32); - - size_t numFreeMemoryRegions = 0; - int bucketIndex = 0; - MALLOC_ACQUIRE(); - BUCKET_BITMASK_T bucketMask = freeRegionBucketsUsed; - - // Loop through each bucket that has free regions in it, based on bits set in freeRegionBucketsUsed bitmap. - while(bucketMask) - { - BUCKET_BITMASK_T indexAdd = __builtin_ctzll(bucketMask); - bucketIndex += indexAdd; - bucketMask >>= indexAdd; - for(Region *freeRegion = freeRegionBuckets[bucketIndex].next; - freeRegion != &freeRegionBuckets[bucketIndex]; - freeRegion = freeRegion->next) - { - ++numFreeMemoryRegions; - size_t freeDynamicMemory = freeRegion->size - REGION_HEADER_SIZE; - if (freeDynamicMemory > 0) - ++freeMemorySizeMap[31-__builtin_clz(freeDynamicMemory)]; - else - ++freeMemorySizeMap[0]; - } - ++bucketIndex; - bucketMask >>= 1; - } - MALLOC_RELEASE(); - return numFreeMemoryRegions; -} - -size_t emmalloc_unclaimed_heap_memory(void) { - return emscripten_get_heap_max() - (size_t)sbrk(0); -} -#endif - -// Define these to satisfy musl references. -void *__libc_malloc(size_t) __attribute__((alias("malloc"))); -void __libc_free(void *) __attribute__((alias("free"))); -void *__libc_calloc(size_t nmemb, size_t size) __attribute__((alias("calloc"))); diff --git a/lib/std/heap/SmpAllocator.zig b/lib/std/heap/SmpAllocator.zig index 50380d29b5..5982026cbc 100644 --- a/lib/std/heap/SmpAllocator.zig +++ b/lib/std/heap/SmpAllocator.zig @@ -26,6 +26,7 @@ //! By limiting the thread-local metadata array to the same number as the CPU //! count, ensures that as threads are created and destroyed, they cycle //! through the full set of freelists. +const SmpAllocator = @This(); const builtin = @import("builtin"); @@ -34,7 +35,7 @@ const assert = std.debug.assert; const mem = std.mem; const math = std.math; const Allocator = std.mem.Allocator; -const SmpAllocator = @This(); +const Alignment = std.mem.Alignment; const PageAllocator = std.heap.PageAllocator; cpu_count: u32, @@ -114,7 +115,7 @@ comptime { assert(!builtin.single_threaded); // you're holding it wrong } -fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 { +fn alloc(context: *anyopaque, len: usize, alignment: Alignment, ra: usize) ?[*]u8 { _ = context; _ = ra; const class = sizeClassIndex(len, alignment); @@ -172,7 +173,7 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ? } } -fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) bool { +fn resize(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ra: usize) bool { _ = context; _ = ra; const class = sizeClassIndex(memory.len, alignment); @@ -184,7 +185,7 @@ fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: return new_class == class; } -fn remap(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) ?[*]u8 { +fn remap(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ra: usize) ?[*]u8 { _ = context; _ = ra; const class = sizeClassIndex(memory.len, alignment); @@ -196,7 +197,7 @@ fn remap(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: u return if (new_class == class) memory.ptr else null; } -fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize) void { +fn free(context: *anyopaque, memory: []u8, alignment: Alignment, ra: usize) void { _ = context; _ = ra; const class = sizeClassIndex(memory.len, alignment); @@ -214,7 +215,7 @@ fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize) t.frees[class] = @intFromPtr(node); } -fn sizeClassIndex(len: usize, alignment: mem.Alignment) usize { +fn sizeClassIndex(len: usize, alignment: Alignment) usize { return @max(@bitSizeOf(usize) - @clz(len - 1), @intFromEnum(alignment), min_class) - min_class; } diff --git a/src/libs/musl.zig b/src/libs/musl.zig index 8cdc4a966f..54fa1aed85 100644 --- a/src/libs/musl.zig +++ b/src/libs/musl.zig @@ -352,8 +352,7 @@ const Ext = enum { fn addSrcFile(arena: Allocator, source_table: *std.StringArrayHashMap(Ext), file_path: []const u8) !void { const ext: Ext = ext: { if (mem.endsWith(u8, file_path, ".c")) { - if (mem.startsWith(u8, file_path, "musl/src/malloc/") or - mem.startsWith(u8, file_path, "musl/src/string/") or + if (mem.startsWith(u8, file_path, "musl/src/string/") or mem.startsWith(u8, file_path, "musl/src/internal/")) { break :ext .o3; @@ -786,24 +785,6 @@ const src_files = [_][]const u8{ "musl/src/locale/uselocale.c", "musl/src/locale/wcscoll.c", "musl/src/locale/wcsxfrm.c", - "musl/src/malloc/calloc.c", - "musl/src/malloc/free.c", - "musl/src/malloc/libc_calloc.c", - "musl/src/malloc/lite_malloc.c", - "musl/src/malloc/mallocng/aligned_alloc.c", - "musl/src/malloc/mallocng/donate.c", - "musl/src/malloc/mallocng/free.c", - "musl/src/malloc/mallocng/malloc.c", - "musl/src/malloc/mallocng/malloc_usable_size.c", - "musl/src/malloc/mallocng/realloc.c", - "musl/src/malloc/memalign.c", - "musl/src/malloc/oldmalloc/aligned_alloc.c", - "musl/src/malloc/oldmalloc/malloc.c", - "musl/src/malloc/oldmalloc/malloc_usable_size.c", - "musl/src/malloc/posix_memalign.c", - "musl/src/malloc/reallocarray.c", - "musl/src/malloc/realloc.c", - "musl/src/malloc/replaced.c", "musl/src/math/aarch64/fma.c", "musl/src/math/aarch64/fmaf.c", "musl/src/math/aarch64/llrint.c", diff --git a/src/libs/wasi_libc.zig b/src/libs/wasi_libc.zig index ca6231bf53..2809f5c9f3 100644 --- a/src/libs/wasi_libc.zig +++ b/src/libs/wasi_libc.zig @@ -77,22 +77,6 @@ pub fn buildCrtFile(comp: *Compilation, crt_file: CrtFile, prog_node: std.Progre .libc_a => { var libc_sources = std.array_list.Managed(Compilation.CSourceFile).init(arena); - { - // Compile emmalloc. - var args = std.array_list.Managed([]const u8).init(arena); - try addCCArgs(comp, arena, &args, .{ .want_O3 = true, .no_strict_aliasing = true }); - - for (emmalloc_src_files) |file_path| { - try libc_sources.append(.{ - .src_path = try comp.dirs.zig_lib.join(arena, &.{ - "libc", try sanitize(arena, file_path), - }), - .extra_flags = args.items, - .owner = undefined, - }); - } - } - { // Compile libc-bottom-half. var args = std.array_list.Managed([]const u8).init(arena); @@ -472,10 +456,6 @@ fn addLibcTopHalfIncludes( }); } -const emmalloc_src_files = [_][]const u8{ - "wasi/emmalloc/emmalloc.c", -}; - const libc_bottom_half_src_files = [_][]const u8{ "wasi/libc-bottom-half/cloudlibc/src/libc/dirent/closedir.c", "wasi/libc-bottom-half/cloudlibc/src/libc/dirent/dirfd.c",