/usr/src/lib/libc/stdlib/malloc.c

Bug Summary

File:	src/lib/libc/stdlib/malloc.c
Warning:	line 1471, column 16 Access to field 'malloc_mt' results in a dereference of a null pointer (loaded from variable 'd')
Annotated Source Code

Press '?' to see keyboard shortcuts
Show analyzer invocation
clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name malloc.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 1 -pic-is-pie -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/lib/libc/obj -resource-dir /usr/local/llvm16/lib/clang/16 -include namespace.h -I /usr/src/lib/libc/include -I /usr/src/lib/libc/hidden -D __LIBC__ -D APIWARN -D YP -I /usr/src/lib/libc/yp -I /usr/src/lib/libc -I /usr/src/lib/libc/gdtoa -I /usr/src/lib/libc/arch/amd64/gdtoa -D INFNAN_CHECK -D MULTIPLE_THREADS -D NO_FENV_H -D USE_LOCALE -I /usr/src/lib/libc -I /usr/src/lib/libc/citrus -D RESOLVSORT -D FLOATING_POINT -D PRINTF_WIDE_CHAR -D SCANF_WIDE_CHAR -D FUTEX -internal-isystem /usr/local/llvm16/lib/clang/16/include -internal-externc-isystem /usr/include -O2 -fdebug-compilation-dir=/usr/src/lib/libc/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fno-jump-tables -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/ben/Projects/scan/2024-01-11-140451-98009-1 -x c /usr/src/lib/libc/stdlib/malloc.c
1/*	$OpenBSD: malloc.c,v 1.295 2023/12/19 06:59:28 otto Exp $	*/
2/*
* Copyright (c) 2008, 2010, 2011, 2016, 2023 Otto Moerbeek <otto@drijf.net>
* Copyright (c) 2012 Matthew Dempsky <matthew@openbsd.org>
* Copyright (c) 2008 Damien Miller <djm@openbsd.org>
* Copyright (c) 2000 Poul-Henning Kamp <phk@FreeBSD.org>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/

21/*
* If we meet some day, and you think this stuff is worth it, you
* can buy me a beer in return. Poul-Henning Kamp
*/

26#ifndef MALLOC_SMALL
27#define MALLOC_STATS
28#endif

30#include <sys/types.h>
31#include <sys/queue.h>
32#include <sys/mman.h>
33#include <sys/sysctl.h>
34#include <uvm/uvmexp.h>
35#include <errno(*__errno()).h>
36#include <stdarg.h>
37#include <stdint.h>
38#include <stdio.h>
39#include <stdlib.h>
40#include <string.h>
41#include <unistd.h>

43#ifdef MALLOC_STATS
44#include <sys/tree.h>
45#include <sys/ktrace.h>
46#include <dlfcn.h>
47#endif

49#include "thread_private.h"
50#include <tib.h>

52#define MALLOC_PAGESHIFT12	_MAX_PAGE_SHIFT12

54#define MALLOC_MINSHIFT4		4
55#define MALLOC_MAXSHIFT(12 - 1)		(MALLOC_PAGESHIFT12 - 1)
56#define MALLOC_PAGESIZE(1UL << 12)		(1UL << MALLOC_PAGESHIFT12)
57#define MALLOC_MINSIZE(1UL << 4)		(1UL << MALLOC_MINSHIFT4)
58#define MALLOC_PAGEMASK((1UL << 12) - 1)		(MALLOC_PAGESIZE(1UL << 12) - 1)
59#define MASK_POINTER(p)((void *)(((uintptr_t)(p)) & ~((1UL << 12) - 1)))		((void *)(((uintptr_t)(p)) & ~MALLOC_PAGEMASK((1UL << 12) - 1)))

61#define MALLOC_MAXCHUNK(1 << (12 - 1))		(1 << MALLOC_MAXSHIFT(12 - 1))
62#define MALLOC_MAXCACHE256		256
63#define MALLOC_DELAYED_CHUNK_MASK15	15
64#ifdef MALLOC_STATS
65#define MALLOC_INITIAL_REGIONS512	512
66#else
67#define MALLOC_INITIAL_REGIONS512	(MALLOC_PAGESIZE(1UL << 12) / sizeof(struct region_info))
68#endif
69#define MALLOC_DEFAULT_CACHE64	64
70#define MALLOC_CHUNK_LISTS4	4
71#define CHUNK_CHECK_LENGTH32	32

73#define B2SIZE(b)((b) * (1UL << 4))		((b) * MALLOC_MINSIZE(1UL << 4))
74#define B2ALLOC(b)((b) == 0 ? (1UL << 4) : (b) * (1UL << 4))		((b) == 0 ? MALLOC_MINSIZE(1UL << 4) : \
		    (b) * MALLOC_MINSIZE(1UL << 4))
76#define BUCKETS((1 << (12 - 1)) / (1UL << 4)) 		(MALLOC_MAXCHUNK(1 << (12 - 1)) / MALLOC_MINSIZE(1UL << 4))

78/*
* We move allocations between half a page and a whole page towards the end,
* subject to alignment constraints. This is the extra headroom we allow.
* Set to zero to be the most strict.
*/
83#define MALLOC_LEEWAY0		0
84#define MALLOC_MOVE_COND(sz)((sz) - malloc_readonly.mopts.malloc_guard < (1UL <<
 12) - 0)	((sz) - moptsmalloc_readonly.mopts.malloc_guard < 		\
		    MALLOC_PAGESIZE(1UL << 12) - MALLOC_LEEWAY0)
86#define MALLOC_MOVE(p, sz)(((char *)(p)) + (((1UL << 12) - 0 - ((sz) - malloc_readonly
.mopts.malloc_guard)) & ~((1UL << 4) - 1)))  	(((char *)(p)) +			\
		    ((MALLOC_PAGESIZE(1UL << 12) - MALLOC_LEEWAY0 -	\
	    	    ((sz) - moptsmalloc_readonly.mopts.malloc_guard)) & 	\
		    ~(MALLOC_MINSIZE(1UL << 4) - 1)))

91#define PAGEROUND(x)(((x) + (((1UL << 12) - 1))) & ~((1UL << 12) -
 1))  (((x) + (MALLOC_PAGEMASK((1UL << 12) - 1))) & ~MALLOC_PAGEMASK((1UL << 12) - 1))

93/*
* What to use for Junk.  This is the byte value we use to fill with
* when the 'J' option is enabled. Use SOME_JUNK right after alloc,
* and SOME_FREEJUNK right before free.
*/
98#define SOME_JUNK0xdb		0xdb	/* deadbeef */
99#define SOME_FREEJUNK0xdf		0xdf	/* dead, free */
100#define SOME_FREEJUNK_ULL0xdfdfdfdfdfdfdfdfULL	0xdfdfdfdfdfdfdfdfULL

102#define MMAP(sz,f)mmap(((void *)0), (sz), 0x01 | 0x02, 0x1000 | 0x0002 | (f), -
1, 0)	mmap(NULL((void *)0), (sz), PROT_READ0x01 | PROT_WRITE0x02, \
  MAP_ANON0x1000 | MAP_PRIVATE0x0002 | (f), -1, 0)

105#define MMAPNONE(sz,f)mmap(((void *)0), (sz), 0x00, 0x1000 | 0x0002 | (f), -1, 0)	mmap(NULL((void *)0), (sz), PROT_NONE0x00, \
  MAP_ANON0x1000 | MAP_PRIVATE0x0002 | (f), -1, 0)

108#define MMAPA(a,sz,f)mmap((a), (sz), 0x01 | 0x02, 0x1000 | 0x0002 | (f), -1, 0)	mmap((a), (sz), PROT_READ0x01 | PROT_WRITE0x02, \
  MAP_ANON0x1000 | MAP_PRIVATE0x0002 | (f), -1, 0)

111struct region_info {
void *p;		/* page; low bits used to mark chunks */
uintptr_t size;		/* size for pages, or chunk_info pointer */
114#ifdef MALLOC_STATS
void **f;		/* where allocated from */
116#endif
117};

119LIST_HEAD(chunk_head, chunk_info)struct chunk_head { struct chunk_info *lh_first; };

121/*
* Two caches, one for "small" regions, one for "big".
* Small cache is an array per size, big cache is one array with different
* sized regions
*/
126#define MAX_SMALLCACHEABLE_SIZE32	32
127#define MAX_BIGCACHEABLE_SIZE512	512
128/* If the total # of pages is larger than this, evict before inserting */
129#define BIGCACHE_FILL(sz)(512 * (sz) / 4)	(MAX_BIGCACHEABLE_SIZE512 * (sz) / 4)

131struct smallcache {
void **pages;
ushort length;
ushort max;
135};

137struct bigcache {
void *page;
size_t psize;
140};

142#ifdef MALLOC_STATS
143#define NUM_FRAMES4		4
144struct btnode {
RBT_ENTRY(btnode)struct rb_entry entry;
void *caller[NUM_FRAMES4];
147};
148RBT_HEAD(btshead, btnode)struct btshead { struct rb_tree rbh_root; };
149RBT_PROTOTYPE(btshead, btnode, entry, btcmp)extern const struct rb_type *const btshead_RBT_TYPE; __attribute__
((__unused__)) static inline void btshead_RBT_INIT(struct btshead
 *head) { _rb_init(&head->rbh_root); } __attribute__((
__unused__)) static inline struct btnode * btshead_RBT_INSERT
(struct btshead *head, struct btnode *elm) { return _rb_insert
(btshead_RBT_TYPE, &head->rbh_root, elm); } __attribute__
((__unused__)) static inline struct btnode * btshead_RBT_REMOVE
(struct btshead *head, struct btnode *elm) { return _rb_remove
(btshead_RBT_TYPE, &head->rbh_root, elm); } __attribute__
((__unused__)) static inline struct btnode * btshead_RBT_FIND
(struct btshead *head, const struct btnode *key) { return _rb_find
(btshead_RBT_TYPE, &head->rbh_root, key); } __attribute__
((__unused__)) static inline struct btnode * btshead_RBT_NFIND
(struct btshead *head, const struct btnode *key) { return _rb_nfind
(btshead_RBT_TYPE, &head->rbh_root, key); } __attribute__
((__unused__)) static inline struct btnode * btshead_RBT_ROOT
(struct btshead *head) { return _rb_root(btshead_RBT_TYPE, &
head->rbh_root); } __attribute__((__unused__)) static inline
 int btshead_RBT_EMPTY(struct btshead *head) { return _rb_empty
(&head->rbh_root); } __attribute__((__unused__)) static
 inline struct btnode * btshead_RBT_MIN(struct btshead *head)
 { return _rb_min(btshead_RBT_TYPE, &head->rbh_root); }
 __attribute__((__unused__)) static inline struct btnode * btshead_RBT_MAX
(struct btshead *head) { return _rb_max(btshead_RBT_TYPE, &
head->rbh_root); } __attribute__((__unused__)) static inline
 struct btnode * btshead_RBT_NEXT(struct btnode *elm) { return
 _rb_next(btshead_RBT_TYPE, elm); } __attribute__((__unused__
)) static inline struct btnode * btshead_RBT_PREV(struct btnode
 *elm) { return _rb_prev(btshead_RBT_TYPE, elm); } __attribute__
((__unused__)) static inline struct btnode * btshead_RBT_LEFT
(struct btnode *elm) { return _rb_left(btshead_RBT_TYPE, elm)
; } __attribute__((__unused__)) static inline struct btnode *
 btshead_RBT_RIGHT(struct btnode *elm) { return _rb_right(btshead_RBT_TYPE
, elm); } __attribute__((__unused__)) static inline struct btnode
 * btshead_RBT_PARENT(struct btnode *elm) { return _rb_parent
(btshead_RBT_TYPE, elm); } __attribute__((__unused__)) static
 inline void btshead_RBT_SET_LEFT(struct btnode *elm, struct btnode
 *left) { _rb_set_left(btshead_RBT_TYPE, elm, left); } __attribute__
((__unused__)) static inline void btshead_RBT_SET_RIGHT(struct
 btnode *elm, struct btnode *right) { _rb_set_right(btshead_RBT_TYPE
, elm, right); } __attribute__((__unused__)) static inline void
 btshead_RBT_SET_PARENT(struct btnode *elm, struct btnode *parent
) { _rb_set_parent(btshead_RBT_TYPE, elm, parent); } __attribute__
((__unused__)) static inline void btshead_RBT_POISON(struct btnode
 *elm, unsigned long poison) { _rb_poison(btshead_RBT_TYPE, elm
, poison); } __attribute__((__unused__)) static inline int btshead_RBT_CHECK
(struct btnode *elm, unsigned long poison) { return _rb_check
(btshead_RBT_TYPE, elm, poison); };
150#endif /* MALLOC_STATS */

152struct dir_info {
u_int32_t canary1;
int active;			/* status of malloc */
struct region_info *r;		/* region slots */
size_t regions_total;		/* number of region slots */
size_t regions_free;		/* number of free slots */
size_t rbytesused;		/* random bytes used */
const char *func;		/* current function */
int malloc_junk;		/* junk fill? */
int mmap_flag;			/* extra flag for mmap */
int mutex;
int malloc_mt;			/* multi-threaded mode? */
			/* lists of free chunk info structs */
struct chunk_head chunk_info_list[BUCKETS((1 << (12 - 1)) / (1UL << 4)) + 1];
			/* lists of chunks with free slots */
struct chunk_head chunk_dir[BUCKETS((1 << (12 - 1)) / (1UL << 4)) + 1][MALLOC_CHUNK_LISTS4];
			/* delayed free chunk slots */
void *delayed_chunks[MALLOC_DELAYED_CHUNK_MASK15 + 1];
u_char rbytes[32];		/* random bytes */
			/* free pages cache */
struct smallcache smallcache[MAX_SMALLCACHEABLE_SIZE32];
size_t bigcache_used;
size_t bigcache_size;
struct bigcache *bigcache;
void *chunk_pages;
size_t chunk_pages_used;
178#ifdef MALLOC_STATS
void *caller;
size_t inserts;
size_t insert_collisions;
size_t deletes;
size_t delete_moves;
size_t cheap_realloc_tries;
size_t cheap_reallocs;
size_t malloc_used;		/* bytes allocated */
size_t malloc_guarded;		/* bytes used for guards */
struct btshead btraces;		/* backtraces seen */
struct btnode *btnodes;		/* store of backtrace nodes */
size_t btnodesused;
191#define STATS_ADD(x,y)((x) += (y))	((x) += (y))
192#define STATS_SUB(x,y)((x) -= (y))	((x) -= (y))
193#define STATS_INC(x)((x)++)	((x)++)
194#define STATS_ZERO(x)((x) = 0)	((x) = 0)
195#define STATS_SETF(x,y)((x)->f = (y))	((x)->f = (y))
196#define STATS_SETFN(x,k,y)((x)->f[k] = (y))	((x)->f[k] = (y))
197#define SET_CALLER(x,y)if (malloc_readonly.mopts.malloc_stats) ((x)->caller = (y)
)	if (DO_STATSmalloc_readonly.mopts.malloc_stats) ((x)->caller = (y))
198#else
199#define STATS_ADD(x,y)((x) += (y))	/* nothing */
200#define STATS_SUB(x,y)((x) -= (y))	/* nothing */
201#define STATS_INC(x)((x)++)	/* nothing */
202#define STATS_ZERO(x)((x) = 0)	/* nothing */
203#define STATS_SETF(x,y)((x)->f = (y))	/* nothing */
204#define STATS_SETFN(x,k,y)((x)->f[k] = (y))	/* nothing */
205#define SET_CALLER(x,y)if (malloc_readonly.mopts.malloc_stats) ((x)->caller = (y)
)	/* nothing */
206#endif /* MALLOC_STATS */
u_int32_t canary2;
208};

210static void unmap(struct dir_info *d, void *p, size_t sz, size_t clear);

212/*
* This structure describes a page worth of chunks.
*
* How many bits per u_short in the bitmap
*/
217#define MALLOC_BITS(8 * sizeof(u_short))		(NBBY8 * sizeof(u_short))
218struct chunk_info {
LIST_ENTRY(chunk_info)struct { struct chunk_info *le_next; struct chunk_info **le_prev
; } entries;
void *page;			/* pointer to the page */
/* number of shorts should add up to 8, check alloc_chunk_info() */
u_short canary;
u_short bucket;
u_short free;			/* how many free chunks */
u_short total;			/* how many chunks */
u_short offset;			/* requested size table offset */
227#define CHUNK_INFO_TAIL3			3
u_short bits[CHUNK_INFO_TAIL3];	/* which chunks are free */
229};

231#define CHUNK_FREE(i, n)((i)->bits[(n) / (8 * sizeof(u_short))] & (1U <<
 ((n) % (8 * sizeof(u_short))))) ((i)->bits[(n) / MALLOC_BITS(8 * sizeof(u_short))] & \
  (1U << ((n) % MALLOC_BITS(8 * sizeof(u_short)))))

234struct malloc_readonly {
			/* Main bookkeeping information */
struct dir_info *malloc_pool[_MALLOC_MUTEXES32];
u_int	malloc_mutexes;		/* how much in actual use? */
int	malloc_freecheck;	/* Extensive double free check */
int	malloc_freeunmap;	/* mprotect free pages PROT_NONE? */
int	def_malloc_junk;	/* junk fill? */
int	malloc_realloc;		/* always realloc? */
int	malloc_xmalloc;		/* xmalloc behaviour? */
u_int	chunk_canaries;		/* use canaries after chunks? */
int	internal_funcs;		/* use better recallocarray/freezero? */
u_int	def_maxcache;		/* free pages we cache */
u_int	junk_loc;		/* variation in location of junk */
size_t	malloc_guard;		/* use guard pages after allocations? */
248#ifdef MALLOC_STATS
int	malloc_stats;		/* save callers, dump leak report */
int	malloc_verbose;		/* dump verbose statistics at end */
251#define	DO_STATSmalloc_readonly.mopts.malloc_stats	moptsmalloc_readonly.mopts.malloc_stats
252#else
253#define	DO_STATSmalloc_readonly.mopts.malloc_stats	0
254#endif
u_int32_t malloc_canary;	/* Matched against ones in pool */
256};


259/* This object is mapped PROT_READ after initialisation to prevent tampering */
260static union {
struct malloc_readonly moptsmalloc_readonly.mopts;
u_char _pad[MALLOC_PAGESIZE(1UL << 12)];
263} malloc_readonly __attribute__((aligned(MALLOC_PAGESIZE(1UL << 12))))
__attribute__((section(".openbsd.mutable")));
265#define moptsmalloc_readonly.mopts	malloc_readonly.moptsmalloc_readonly.mopts

267char		*malloc_options;	/* compile-time options */

269static __dead__attribute__((__noreturn__)) void wrterror(struct dir_info *d, char *msg, ...)
  __attribute__((__format__ (printf, 2, 3)));

272#ifdef MALLOC_STATS
273void malloc_dump(void);
274PROTO_NORMAL(malloc_dump)__attribute__((__visibility__("hidden"))) typeof(malloc_dump)
 malloc_dump asm("_libc_" "malloc_dump");
275static void malloc_exit(void);
276static void print_chunk_details(struct dir_info *, void *, size_t, size_t);
277static void* store_caller(struct dir_info *, struct btnode *);

279/* below are the arches for which deeper caller info has been tested */
280#if defined(__aarch64__) || \
defined(__amd64__1) || \
defined(__arm__) || \
defined(__i386__) || \
defined(__powerpc__)
285__attribute__((always_inline))
286static inline void*
287caller(struct dir_info *d)
288{
struct btnode p;
int level = DO_STATSmalloc_readonly.mopts.malloc_stats;

if (level == 0)
return NULL((void *)0);

memset(&p.caller, 0, sizeof(p.caller));
if (level >= 1)
p.caller[0] = __builtin_extract_return_addr(
    __builtin_return_address(0));
if (p.caller[0] != NULL((void *)0) && level >= 2)
p.caller[1] = __builtin_extract_return_addr(
    __builtin_return_address(1));
if (p.caller[1] != NULL((void *)0) && level >= 3)
p.caller[2] = __builtin_extract_return_addr(
    __builtin_return_address(2));
if (p.caller[2] != NULL((void *)0) && level >= 4)
p.caller[3] = __builtin_extract_return_addr(
    __builtin_return_address(3));
return store_caller(d, &p);
309}
310#else
311__attribute__((always_inline))
312static inline void* caller(struct dir_info *d)
313{
struct btnode p;

if (DO_STATSmalloc_readonly.mopts.malloc_stats == 0)
return NULL((void *)0);
memset(&p.caller, 0, sizeof(p.caller));
p.caller[0] = __builtin_extract_return_addr(__builtin_return_address(0));
return store_caller(d, &p);
321}
322#endif
323#endif /* MALLOC_STATS */

325/* low bits of r->p determine size: 0 means >= page size and r->size holding
* real size, otherwise low bits is the bucket + 1
*/
328#define REALSIZE(sz, r)(sz) = (uintptr_t)(r)->p & ((1UL << 12) - 1), (sz
) = ((sz) == 0 ? (r)->size : (((sz) - 1) * (1UL << 4
)))						\
(sz) = (uintptr_t)(r)->p & MALLOC_PAGEMASK((1UL << 12) - 1),		\
(sz) = ((sz) == 0 ? (r)->size : B2SIZE((sz) - 1)(((sz) - 1) * (1UL << 4)))

332static inline size_t
333hash(void *p)
334{
size_t sum;
uintptr_t u;

u = (uintptr_t)p >> MALLOC_PAGESHIFT12;
sum = u;
sum = (sum << 7) - sum + (u >> 16);
341#ifdef __LP64__1
sum = (sum << 7) - sum + (u >> 32);
sum = (sum << 7) - sum + (u >> 48);
344#endif
return sum;
346}

348static inline struct dir_info *
349getpool(void)
350{
if (moptsmalloc_readonly.mopts.malloc_pool[1] == NULL((void *)0) || !moptsmalloc_readonly.mopts.malloc_pool[1]->malloc_mt)
return moptsmalloc_readonly.mopts.malloc_pool[1];
else	/* first one reserved for special pool */
return moptsmalloc_readonly.mopts.malloc_pool[1 + TIB_GET()((struct tib *)((char *)(__amd64_read_tcb(0)) - (__builtin_offsetof
(struct tib, __tib_self) + 0)))->tib_tid %
    (moptsmalloc_readonly.mopts.malloc_mutexes - 1)];
356}

358static __dead__attribute__((__noreturn__)) void
359wrterror(struct dir_info *d, char *msg, ...)
360{
int		saved_errno = errno(*__errno());
va_list		ap;

dprintf(STDERR_FILENO2, "%s(%d) in %s(): ", __progname,
   getpid(), (d != NULL((void *)0) && d->func) ? d->func : "unknown");
va_start(ap, msg)__builtin_va_start((ap), msg);
vdprintf(STDERR_FILENO2, msg, ap);
va_end(ap)__builtin_va_end((ap));
dprintf(STDERR_FILENO2, "\n");

371#ifdef MALLOC_STATS
if (DO_STATSmalloc_readonly.mopts.malloc_stats && moptsmalloc_readonly.mopts.malloc_verbose)
malloc_dump();
374#endif

errno(*__errno()) = saved_errno;

abort();
379}

381static void
382rbytes_init(struct dir_info *d)
383{
arc4random_buf(d->rbytes, sizeof(d->rbytes));
/* add 1 to account for using d->rbytes[0] */
d->rbytesused = 1 + d->rbytes[0] % (sizeof(d->rbytes) / 2);
387}

389static inline u_char
390getrbyte(struct dir_info *d)
391{
u_char x;

if (d->rbytesused >= sizeof(d->rbytes))
rbytes_init(d);
x = d->rbytes[d->rbytesused++];
return x;
398}

400static void
401omalloc_parseopt(char opt)
402{
switch (opt) {
case '+':
moptsmalloc_readonly.mopts.malloc_mutexes <<= 1;
if (moptsmalloc_readonly.mopts.malloc_mutexes > _MALLOC_MUTEXES32)
	moptsmalloc_readonly.mopts.malloc_mutexes = _MALLOC_MUTEXES32;
break;
case '-':
moptsmalloc_readonly.mopts.malloc_mutexes >>= 1;
if (moptsmalloc_readonly.mopts.malloc_mutexes < 2)
	moptsmalloc_readonly.mopts.malloc_mutexes = 2;
break;
case '>':
moptsmalloc_readonly.mopts.def_maxcache <<= 1;
if (moptsmalloc_readonly.mopts.def_maxcache > MALLOC_MAXCACHE256)
	moptsmalloc_readonly.mopts.def_maxcache = MALLOC_MAXCACHE256;
break;
case '<':
moptsmalloc_readonly.mopts.def_maxcache >>= 1;
break;
case 'c':
moptsmalloc_readonly.mopts.chunk_canaries = 0;
break;
case 'C':
moptsmalloc_readonly.mopts.chunk_canaries = 1;
break;
428#ifdef MALLOC_STATS
case 'd':
moptsmalloc_readonly.mopts.malloc_stats = 0;
break;
case 'D':
case '1':
moptsmalloc_readonly.mopts.malloc_stats = 1;
break;
case '2':
moptsmalloc_readonly.mopts.malloc_stats = 2;
break;
case '3':
moptsmalloc_readonly.mopts.malloc_stats = 3;
break;
case '4':
moptsmalloc_readonly.mopts.malloc_stats = 4;
break;
445#endif /* MALLOC_STATS */
case 'f':
moptsmalloc_readonly.mopts.malloc_freecheck = 0;
moptsmalloc_readonly.mopts.malloc_freeunmap = 0;
break;
case 'F':
moptsmalloc_readonly.mopts.malloc_freecheck = 1;
moptsmalloc_readonly.mopts.malloc_freeunmap = 1;
break;
case 'g':
moptsmalloc_readonly.mopts.malloc_guard = 0;
break;
case 'G':
moptsmalloc_readonly.mopts.malloc_guard = MALLOC_PAGESIZE(1UL << 12);
break;
case 'j':
if (moptsmalloc_readonly.mopts.def_malloc_junk > 0)
	moptsmalloc_readonly.mopts.def_malloc_junk--;
break;
case 'J':
if (moptsmalloc_readonly.mopts.def_malloc_junk < 2)
	moptsmalloc_readonly.mopts.def_malloc_junk++;
break;
case 'r':
moptsmalloc_readonly.mopts.malloc_realloc = 0;
break;
case 'R':
moptsmalloc_readonly.mopts.malloc_realloc = 1;
break;
case 'u':
moptsmalloc_readonly.mopts.malloc_freeunmap = 0;
break;
case 'U':
moptsmalloc_readonly.mopts.malloc_freeunmap = 1;
break;
480#ifdef MALLOC_STATS
case 'v':
moptsmalloc_readonly.mopts.malloc_verbose = 0;
break;
case 'V':
moptsmalloc_readonly.mopts.malloc_verbose = 1;
break;
487#endif /* MALLOC_STATS */
case 'x':
moptsmalloc_readonly.mopts.malloc_xmalloc = 0;
break;
case 'X':
moptsmalloc_readonly.mopts.malloc_xmalloc = 1;
break;
default:
dprintf(STDERR_FILENO2, "malloc() warning: "
                  "unknown char in MALLOC_OPTIONS\n");
break;
}
499}

501static void
502omalloc_init(void)
503{
char *p, *q, b[16];
int i, j;
const int mib[2] = { CTL_VM2, VM_MALLOC_CONF12 };
size_t sb;

/*
* Default options
*/
moptsmalloc_readonly.mopts.malloc_mutexes = 8;
moptsmalloc_readonly.mopts.def_malloc_junk = 1;
moptsmalloc_readonly.mopts.def_maxcache = MALLOC_DEFAULT_CACHE64;

for (i = 0; i < 3; i++) {
switch (i) {
case 0:
	sb = sizeof(b);
	j = sysctl(mib, 2, b, &sb, NULL((void *)0), 0);
	if (j != 0)
		continue;
	p = b;
	break;
case 1:
	if (issetugid() == 0)
		p = getenv("MALLOC_OPTIONS");
	else
		continue;
	break;
case 2:
	p = malloc_options;
	break;
default:
	p = NULL((void *)0);
}

for (; p != NULL((void *)0) && *p != '\0'; p++) {
	switch (*p) {
	case 'S':
		for (q = "CFGJ"; *q != '\0'; q++)
			omalloc_parseopt(*q);
		moptsmalloc_readonly.mopts.def_maxcache = 0;
		break;
	case 's':
		for (q = "cfgj"; *q != '\0'; q++)
			omalloc_parseopt(*q);
		moptsmalloc_readonly.mopts.def_maxcache = MALLOC_DEFAULT_CACHE64;
		break;
	default:
		omalloc_parseopt(*p);
		break;
	}
}
}

557#ifdef MALLOC_STATS
if (DO_STATSmalloc_readonly.mopts.malloc_stats && (atexit(malloc_exit) == -1)) {
dprintf(STDERR_FILENO2, "malloc() warning: atexit(3) failed."
    " Will not be able to dump stats on exit\n");
}
562#endif

while ((moptsmalloc_readonly.mopts.malloc_canary = arc4random()) == 0)
;
moptsmalloc_readonly.mopts.junk_loc = arc4random();
if (moptsmalloc_readonly.mopts.chunk_canaries)
do {
	moptsmalloc_readonly.mopts.chunk_canaries = arc4random();
} while ((u_char)moptsmalloc_readonly.mopts.chunk_canaries == 0 ||
    (u_char)moptsmalloc_readonly.mopts.chunk_canaries == SOME_FREEJUNK0xdf);
572}

574static void
575omalloc_poolinit(struct dir_info *d, int mmap_flag)
576{
u_int i, j;

d->r = NULL((void *)0);
d->rbytesused = sizeof(d->rbytes);
d->regions_free = d->regions_total = 0;
for (i = 0; i <= BUCKETS((1 << (12 - 1)) / (1UL << 4)); i++) {
LIST_INIT(&d->chunk_info_list[i])do { ((&d->chunk_info_list[i])->lh_first) = ((void *
)0); } while (0);
for (j = 0; j < MALLOC_CHUNK_LISTS4; j++)
	LIST_INIT(&d->chunk_dir[i][j])do { ((&d->chunk_dir[i][j])->lh_first) = ((void *)0
); } while (0);
}
d->mmap_flag = mmap_flag;
d->malloc_junk = moptsmalloc_readonly.mopts.def_malloc_junk;
589#ifdef MALLOC_STATS
RBT_INIT(btshead, &d->btraces)btshead_RBT_INIT(&d->btraces);
591#endif
d->canary1 = moptsmalloc_readonly.mopts.malloc_canary ^ (u_int32_t)(uintptr_t)d;
d->canary2 = ~d->canary1;
594}

596static int
597omalloc_grow(struct dir_info *d)
598{
size_t newtotal;
size_t newsize;
size_t mask;
size_t i, oldpsz;
struct region_info *p;

if (d->regions_total > SIZE_MAX0xffffffffffffffffUL / sizeof(struct region_info) / 2)
return 1;

newtotal = d->regions_total == 0 ? MALLOC_INITIAL_REGIONS512 :
   d->regions_total * 2;
newsize = PAGEROUND(newtotal * sizeof(struct region_info))(((newtotal * sizeof(struct region_info)) + (((1UL << 12
) - 1))) & ~((1UL << 12) - 1));
mask = newtotal - 1;

/* Don't use cache here, we don't want user uaf touch this */
p = MMAP(newsize, d->mmap_flag)mmap(((void *)0), (newsize), 0x01 | 0x02, 0x1000 | 0x0002 | (
d->mmap_flag), -1, 0);
if (p == MAP_FAILED((void *)-1))
return 1;

STATS_ADD(d->malloc_used, newsize)((d->malloc_used) += (newsize));
STATS_ZERO(d->inserts)((d->inserts) = 0);
STATS_ZERO(d->insert_collisions)((d->insert_collisions) = 0);
for (i = 0; i < d->regions_total; i++) {
void *q = d->r[i].p;
if (q != NULL((void *)0)) {
	size_t index = hash(q) & mask;
	STATS_INC(d->inserts)((d->inserts)++);
	while (p[index].p != NULL((void *)0)) {
		index = (index - 1) & mask;
		STATS_INC(d->insert_collisions)((d->insert_collisions)++);
	}
	p[index] = d->r[i];
}
}

if (d->regions_total > 0) {
oldpsz = PAGEROUND(d->regions_total *(((d->regions_total * sizeof(struct region_info)) + (((1UL
 << 12) - 1))) & ~((1UL << 12) - 1))
    sizeof(struct region_info))(((d->regions_total * sizeof(struct region_info)) + (((1UL
 << 12) - 1))) & ~((1UL << 12) - 1));
/* clear to avoid meta info ending up in the cache */
unmap(d, d->r, oldpsz, oldpsz);
}
d->regions_free += newtotal - d->regions_total;
d->regions_total = newtotal;
d->r = p;
return 0;
644}

646/*
* The hashtable uses the assumption that p is never NULL. This holds since
* non-MAP_FIXED mappings with hint 0 start at BRKSIZ.
*/
650static int
651insert(struct dir_info *d, void *p, size_t sz, void *f)
652{
size_t index;
size_t mask;
void *q;

if (d->regions_free * 4 < d->regions_total || d->regions_total == 0) {
if (omalloc_grow(d))
	return 1;
}
mask = d->regions_total - 1;
index = hash(p) & mask;
q = d->r[index].p;
STATS_INC(d->inserts)((d->inserts)++);
while (q != NULL((void *)0)) {
index = (index - 1) & mask;
q = d->r[index].p;
STATS_INC(d->insert_collisions)((d->insert_collisions)++);
}
d->r[index].p = p;
d->r[index].size = sz;
STATS_SETF(&d->r[index], f)((&d->r[index])->f = (f));
d->regions_free--;
return 0;
675}

677static struct region_info *
678find(struct dir_info *d, void *p)
679{
size_t index;
size_t mask = d->regions_total - 1;
void *q, *r;

if (moptsmalloc_readonly.mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) ||
   d->canary1 != ~d->canary2)
wrterror(d, "internal struct corrupt");
if (d->r == NULL((void *)0))
return NULL((void *)0);
p = MASK_POINTER(p)((void *)(((uintptr_t)(p)) & ~((1UL << 12) - 1)));
index = hash(p) & mask;
r = d->r[index].p;
q = MASK_POINTER(r)((void *)(((uintptr_t)(r)) & ~((1UL << 12) - 1)));
while (q != p && r != NULL((void *)0)) {
index = (index - 1) & mask;
r = d->r[index].p;
q = MASK_POINTER(r)((void *)(((uintptr_t)(r)) & ~((1UL << 12) - 1)));
}
return (q == p && r != NULL((void *)0)) ? &d->r[index] : NULL((void *)0);
699}

701static void
702delete(struct dir_info *d, struct region_info *ri)
703{
/* algorithm R, Knuth Vol III section 6.4 */
size_t mask = d->regions_total - 1;
size_t i, j, r;

if (d->regions_total & (d->regions_total - 1))
wrterror(d, "regions_total not 2^x");
d->regions_free++;
STATS_INC(d->deletes)((d->deletes)++);

i = ri - d->r;
for (;;) {
d->r[i].p = NULL((void *)0);
d->r[i].size = 0;
j = i;
for (;;) {
	i = (i - 1) & mask;
	if (d->r[i].p == NULL((void *)0))
		return;
	r = hash(d->r[i].p) & mask;
	if ((i <= r && r < j) || (r < j && j < i) ||
	    (j < i && i <= r))
		continue;
	d->r[j] = d->r[i];
	STATS_INC(d->delete_moves)((d->delete_moves)++);
	break;
}

}
732}

734static inline void
735junk_free(int junk, void *p, size_t sz)
736{
size_t i, step = 1;
uint64_t *lp = p;

if (junk == 0 || sz == 0)
return;
sz /= sizeof(uint64_t);
if (junk == 1) {
if (sz > MALLOC_PAGESIZE(1UL << 12) / sizeof(uint64_t))
	sz = MALLOC_PAGESIZE(1UL << 12) / sizeof(uint64_t);
step = sz / 4;
if (step == 0)
	step = 1;
}
/* Do not always put the free junk bytes in the same spot.
  There is modulo bias here, but we ignore that. */
for (i = moptsmalloc_readonly.mopts.junk_loc % step; i < sz; i += step)
lp[i] = SOME_FREEJUNK_ULL0xdfdfdfdfdfdfdfdfULL;
754}

756static inline void
757validate_junk(struct dir_info *pool, void *p, size_t argsz)
758{
size_t i, sz, step = 1;
uint64_t *lp = p;

if (pool->malloc_junk == 0 || argsz == 0)
return;
sz = argsz / sizeof(uint64_t);
if (pool->malloc_junk == 1) {
if (sz > MALLOC_PAGESIZE(1UL << 12) / sizeof(uint64_t))
	sz = MALLOC_PAGESIZE(1UL << 12) / sizeof(uint64_t);
step = sz / 4;
if (step == 0)
	step = 1;
}
/* see junk_free */
for (i = moptsmalloc_readonly.mopts.junk_loc % step; i < sz; i += step) {
if (lp[i] != SOME_FREEJUNK_ULL0xdfdfdfdfdfdfdfdfULL) {
775#ifdef MALLOC_STATS
	if (DO_STATSmalloc_readonly.mopts.malloc_stats && argsz <= MALLOC_MAXCHUNK(1 << (12 - 1)))
		print_chunk_details(pool, lp, argsz, i);
	else
779#endif
		wrterror(pool,
		    "write to free mem %p[%zu..%zu]@%zu",
		    lp, i * sizeof(uint64_t),
		    (i + 1) * sizeof(uint64_t) - 1, argsz);
}
}
786}


789/*
* Cache maintenance.
* Opposed to the regular region data structure, the sizes in the
* cache are in MALLOC_PAGESIZE units.
*/
794static void
795unmap(struct dir_info *d, void *p, size_t sz, size_t clear)
796{
size_t psz = sz >> MALLOC_PAGESHIFT12;
void *r;
u_short i;
struct smallcache *cache;

if (sz != PAGEROUND(sz)(((sz) + (((1UL << 12) - 1))) & ~((1UL << 12)
 - 1)) || psz == 0)
wrterror(d, "munmap round");

if (d->bigcache_size > 0 && psz > MAX_SMALLCACHEABLE_SIZE32 &&
   psz <= MAX_BIGCACHEABLE_SIZE512) {
u_short base = getrbyte(d);
u_short j;

/* don't look through all slots */
for (j = 0; j < d->bigcache_size / 4; j++) {
	i = (base + j) & (d->bigcache_size - 1);
	if (d->bigcache_used <
	    BIGCACHE_FILL(d->bigcache_size)(512 * (d->bigcache_size) / 4))  {
		if (d->bigcache[i].psize == 0)
			break;
	} else {
		if (d->bigcache[i].psize != 0)
			break;
	}
}
/* if we didn't find a preferred slot, use random one */
if (d->bigcache[i].psize != 0) {
	size_t tmp;

	r = d->bigcache[i].page;
	d->bigcache_used -= d->bigcache[i].psize;
	tmp = d->bigcache[i].psize << MALLOC_PAGESHIFT12;
	if (!moptsmalloc_readonly.mopts.malloc_freeunmap)
		validate_junk(d, r, tmp);
	if (munmap(r, tmp))
		 wrterror(d, "munmap %p", r);
	STATS_SUB(d->malloc_used, tmp)((d->malloc_used) -= (tmp));
}

if (clear > 0)
	explicit_bzero(p, clear);
if (moptsmalloc_readonly.mopts.malloc_freeunmap) {
	if (mprotect(p, sz, PROT_NONE0x00))
		wrterror(d, "mprotect %p", r);
} else
	junk_free(d->malloc_junk, p, sz);
d->bigcache[i].page = p;
d->bigcache[i].psize = psz;
d->bigcache_used += psz;
return;
}
if (psz > MAX_SMALLCACHEABLE_SIZE32 || d->smallcache[psz - 1].max == 0) {
if (munmap(p, sz))
	wrterror(d, "munmap %p", p);
STATS_SUB(d->malloc_used, sz)((d->malloc_used) -= (sz));
return;
}
cache = &d->smallcache[psz - 1];
if (cache->length == cache->max) {
int fresh;
/* use a random slot */
i = getrbyte(d) & (cache->max - 1);
r = cache->pages[i];
fresh = (uintptr_t)r & 1;
*(uintptr_t*)&r &= ~1UL;
if (!fresh && !moptsmalloc_readonly.mopts.malloc_freeunmap)
	validate_junk(d, r, sz);
if (munmap(r, sz))
	wrterror(d, "munmap %p", r);
STATS_SUB(d->malloc_used, sz)((d->malloc_used) -= (sz));
cache->length--;
} else
i = cache->length;

/* fill slot */
if (clear > 0)
explicit_bzero(p, clear);
if (moptsmalloc_readonly.mopts.malloc_freeunmap)
mprotect(p, sz, PROT_NONE0x00);
else
junk_free(d->malloc_junk, p, sz);
cache->pages[i] = p;
cache->length++;
880}

882static void *
883map(struct dir_info *d, size_t sz, int zero_fill)
884{
size_t psz = sz >> MALLOC_PAGESHIFT12;
u_short i;
void *p;
struct smallcache *cache;

if (moptsmalloc_readonly.mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) ||
   d->canary1 != ~d->canary2)
wrterror(d, "internal struct corrupt");
if (sz != PAGEROUND(sz)(((sz) + (((1UL << 12) - 1))) & ~((1UL << 12)
 - 1)) || psz == 0)
wrterror(d, "map round");


if (d->bigcache_size > 0 && psz > MAX_SMALLCACHEABLE_SIZE32 &&
   psz <= MAX_BIGCACHEABLE_SIZE512) {
size_t base = getrbyte(d);
size_t cached = d->bigcache_used;
ushort j;

for (j = 0; j < d->bigcache_size && cached >= psz; j++) {
	i = (j + base) & (d->bigcache_size - 1);
	if (d->bigcache[i].psize == psz) {
		p = d->bigcache[i].page;
		d->bigcache_used -= psz;
		d->bigcache[i].page = NULL((void *)0);
		d->bigcache[i].psize = 0;

		if (!moptsmalloc_readonly.mopts.malloc_freeunmap)
			validate_junk(d, p, sz);
		if (moptsmalloc_readonly.mopts.malloc_freeunmap)
			mprotect(p, sz, PROT_READ0x01 | PROT_WRITE0x02);
		if (zero_fill)
			memset(p, 0, sz);
		else if (moptsmalloc_readonly.mopts.malloc_freeunmap)
			junk_free(d->malloc_junk, p, sz);
		return p;
	}
	cached -= d->bigcache[i].psize;
}
}
if (psz <= MAX_SMALLCACHEABLE_SIZE32 && d->smallcache[psz - 1].max > 0) {
cache = &d->smallcache[psz - 1];
if (cache->length > 0) {
	int fresh;
	if (cache->length == 1)
		p = cache->pages[--cache->length];
	else {
		i = getrbyte(d) % cache->length;
		p = cache->pages[i];
		cache->pages[i] = cache->pages[--cache->length];
	}
	/* check if page was not junked, i.e. "fresh
	   we use the lsb of the pointer for that */	
	fresh = (uintptr_t)p & 1UL;
	*(uintptr_t*)&p &= ~1UL;
	if (!fresh && !moptsmalloc_readonly.mopts.malloc_freeunmap)
		validate_junk(d, p, sz);
	if (moptsmalloc_readonly.mopts.malloc_freeunmap)
		mprotect(p, sz, PROT_READ0x01 | PROT_WRITE0x02);
	if (zero_fill)
		memset(p, 0, sz);
	else if (moptsmalloc_readonly.mopts.malloc_freeunmap)
		junk_free(d->malloc_junk, p, sz);
	return p;
}
if (psz <= 1) {
	p = MMAP(cache->max * sz, d->mmap_flag)mmap(((void *)0), (cache->max * sz), 0x01 | 0x02, 0x1000 |
 0x0002 | (d->mmap_flag), -1, 0);
	if (p != MAP_FAILED((void *)-1)) {
		STATS_ADD(d->malloc_used, cache->max * sz)((d->malloc_used) += (cache->max * sz));
		cache->length = cache->max - 1;
		for (i = 0; i < cache->max - 1; i++) {
			void *q = (char*)p + i * sz;
			cache->pages[i] = q;
			/* mark pointer in slot as not junked */
			*(uintptr_t*)&cache->pages[i] |= 1UL;
		}
		if (moptsmalloc_readonly.mopts.malloc_freeunmap)
			mprotect(p, (cache->max - 1) * sz,
			    PROT_NONE0x00);
		p = (char*)p + (cache->max - 1) * sz;
		/* zero fill not needed, freshly mmapped */
		return p;
	}
}

}
p = MMAP(sz, d->mmap_flag)mmap(((void *)0), (sz), 0x01 | 0x02, 0x1000 | 0x0002 | (d->
mmap_flag), -1, 0);
if (p != MAP_FAILED((void *)-1))
STATS_ADD(d->malloc_used, sz)((d->malloc_used) += (sz));
/* zero fill not needed */
return p;
975}

977static void
978init_chunk_info(struct dir_info *d, struct chunk_info *p, u_int bucket)
979{
u_int i;

p->bucket = bucket;
p->total = p->free = MALLOC_PAGESIZE(1UL << 12) / B2ALLOC(bucket)((bucket) == 0 ? (1UL << 4) : (bucket) * (1UL << 4
));
p->offset = howmany(p->total, MALLOC_BITS)(((p->total) + (((8 * sizeof(u_short))) - 1)) / ((8 * sizeof
(u_short))));
p->canary = (u_short)d->canary1;

/* set all valid bits in the bitmap */
i = p->total - 1;	
memset(p->bits, 0xff, sizeof(p->bits[0]) * (i / MALLOC_BITS(8 * sizeof(u_short))));
p->bits[i / MALLOC_BITS(8 * sizeof(u_short))] = (2U << (i % MALLOC_BITS(8 * sizeof(u_short)))) - 1;
991}

993static struct chunk_info *
994alloc_chunk_info(struct dir_info *d, u_int bucket)
995{
struct chunk_info *p;

if (LIST_EMPTY(&d->chunk_info_list[bucket])(((&d->chunk_info_list[bucket])->lh_first) == ((void
 *)0))) {
const size_t chunk_pages = 64;
size_t size, count, i;
char *q;

count = MALLOC_PAGESIZE(1UL << 12) / B2ALLOC(bucket)((bucket) == 0 ? (1UL << 4) : (bucket) * (1UL << 4
));

size = howmany(count, MALLOC_BITS)(((count) + (((8 * sizeof(u_short))) - 1)) / ((8 * sizeof(u_short
))));
/* see declaration of struct chunk_info */
if (size <= CHUNK_INFO_TAIL3)
	size = 0;
else
	size -= CHUNK_INFO_TAIL3;
size = sizeof(struct chunk_info) + size * sizeof(u_short);
if (moptsmalloc_readonly.mopts.chunk_canaries && bucket > 0)
	size += count * sizeof(u_short);
size = _ALIGN(size)(((unsigned long)(size) + (sizeof(long) - 1)) &~(sizeof(long
) - 1));
count = MALLOC_PAGESIZE(1UL << 12) / size;

/* Don't use cache here, we don't want user uaf touch this */
if (d->chunk_pages_used == chunk_pages ||
     d->chunk_pages == NULL((void *)0)) {
	q = MMAP(MALLOC_PAGESIZE * chunk_pages, d->mmap_flag)mmap(((void *)0), ((1UL << 12) * chunk_pages), 0x01 | 0x02
, 0x1000 | 0x0002 | (d->mmap_flag), -1, 0);
	if (q == MAP_FAILED((void *)-1))
		return NULL((void *)0);
	d->chunk_pages = q;
	d->chunk_pages_used = 0;
	STATS_ADD(d->malloc_used, MALLOC_PAGESIZE *((d->malloc_used) += ((1UL << 12) * chunk_pages))
	    chunk_pages)((d->malloc_used) += ((1UL << 12) * chunk_pages));
}
q = (char *)d->chunk_pages + d->chunk_pages_used *
    MALLOC_PAGESIZE(1UL << 12);
d->chunk_pages_used++;

for (i = 0; i < count; i++, q += size) {
	p = (struct chunk_info *)q;
	LIST_INSERT_HEAD(&d->chunk_info_list[bucket], p,do { if (((p)->entries.le_next = (&d->chunk_info_list
[bucket])->lh_first) != ((void *)0)) (&d->chunk_info_list
[bucket])->lh_first->entries.le_prev = &(p)->entries
.le_next; (&d->chunk_info_list[bucket])->lh_first =
 (p); (p)->entries.le_prev = &(&d->chunk_info_list
[bucket])->lh_first; } while (0)
	    entries)do { if (((p)->entries.le_next = (&d->chunk_info_list
[bucket])->lh_first) != ((void *)0)) (&d->chunk_info_list
[bucket])->lh_first->entries.le_prev = &(p)->entries
.le_next; (&d->chunk_info_list[bucket])->lh_first =
 (p); (p)->entries.le_prev = &(&d->chunk_info_list
[bucket])->lh_first; } while (0);
}
}
p = LIST_FIRST(&d->chunk_info_list[bucket])((&d->chunk_info_list[bucket])->lh_first);
LIST_REMOVE(p, entries)do { if ((p)->entries.le_next != ((void *)0)) (p)->entries
.le_next->entries.le_prev = (p)->entries.le_prev; *(p)->
entries.le_prev = (p)->entries.le_next; ; ; } while (0);
if (p->total == 0)
init_chunk_info(d, p, bucket);
return p;
1043}

1045/*
* Allocate a page of chunks
*/
1048static struct chunk_info *
1049omalloc_make_chunks(struct dir_info *d, u_int bucket, u_int listnum)
1050{
struct chunk_info *bp;
void *pp;
void *ff = NULL((void *)0);

/* Allocate a new bucket */
pp = map(d, MALLOC_PAGESIZE(1UL << 12), 0);
if (pp == MAP_FAILED((void *)-1))
return NULL((void *)0);
if (DO_STATSmalloc_readonly.mopts.malloc_stats) {
ff = map(d, MALLOC_PAGESIZE(1UL << 12), 0);
if (ff == MAP_FAILED((void *)-1))
	goto err;
memset(ff, 0, sizeof(void *) * MALLOC_PAGESIZE(1UL << 12) /
    B2ALLOC(bucket)((bucket) == 0 ? (1UL << 4) : (bucket) * (1UL << 4
)));
}

/* memory protect the page allocated in the malloc(0) case */
if (bucket == 0 && mprotect(pp, MALLOC_PAGESIZE(1UL << 12), PROT_NONE0x00) == -1)
goto err;

bp = alloc_chunk_info(d, bucket);
if (bp == NULL((void *)0))
goto err;
bp->page = pp;

if (insert(d, (void *)((uintptr_t)pp | (bucket + 1)), (uintptr_t)bp,
   ff))
goto err;
LIST_INSERT_HEAD(&d->chunk_dir[bucket][listnum], bp, entries)do { if (((bp)->entries.le_next = (&d->chunk_dir[bucket
][listnum])->lh_first) != ((void *)0)) (&d->chunk_dir
[bucket][listnum])->lh_first->entries.le_prev = &(bp
)->entries.le_next; (&d->chunk_dir[bucket][listnum]
)->lh_first = (bp); (bp)->entries.le_prev = &(&
d->chunk_dir[bucket][listnum])->lh_first; } while (0);

if (bucket > 0 && d->malloc_junk != 0)
memset(pp, SOME_FREEJUNK0xdf, MALLOC_PAGESIZE(1UL << 12));

return bp;

1086err:
unmap(d, pp, MALLOC_PAGESIZE(1UL << 12), 0);
if (ff != NULL((void *)0) && ff != MAP_FAILED((void *)-1))
unmap(d, ff, MALLOC_PAGESIZE(1UL << 12), 0);
return NULL((void *)0);
1091}

1093#if defined(__GNUC__4) && __GNUC__4 < 4
1094static inline unsigned int
1095lb(u_int x)
1096{
1097#if defined(__m88k__)
__asm__ __volatile__ ("ff1 %0, %0" : "=r" (x) : "0" (x));
return x;
1100#else
/* portable version */
unsigned int count = 0;
while ((x & (1U << (sizeof(int) * CHAR_BIT8 - 1))) == 0) {
count++;
x <<= 1;
}
return (sizeof(int) * CHAR_BIT8 - 1) - count;
1108#endif
1109}
1110#else
1111/* using built-in function version */
1112static inline unsigned int
1113lb(u_int x)
1114{
/* I need an extension just for integer-length (: */
return (sizeof(int) * CHAR_BIT8 - 1) - __builtin_clz(x);
1117}
1118#endif

1120/* https://pvk.ca/Blog/2015/06/27/linear-log-bucketing-fast-versatile-simple/
 via Tony Finch */
1122static inline unsigned int
1123bin_of(unsigned int size)
1124{
const unsigned int linear = 6;
const unsigned int subbin = 2;

unsigned int mask, rounded, rounded_size;
unsigned int n_bits, shift;

n_bits = lb(size | (1U << linear));
shift = n_bits - subbin;
mask = (1ULL << shift) - 1;
rounded = size + mask; /* XXX: overflow. */

rounded_size = rounded & ~mask;
return rounded_size;
1138}

1140static inline u_short
1141find_bucket(u_short size)
1142{
/* malloc(0) is special */
if (size == 0)
return 0;
if (size < MALLOC_MINSIZE(1UL << 4))
size = MALLOC_MINSIZE(1UL << 4);
if (moptsmalloc_readonly.mopts.def_maxcache != 0)
size = bin_of(size);
return howmany(size, MALLOC_MINSIZE)(((size) + (((1UL << 4)) - 1)) / ((1UL << 4)));
1151}

1153static void
1154fill_canary(char *ptr, size_t sz, size_t allocated)
1155{
size_t check_sz = allocated - sz;

if (check_sz > CHUNK_CHECK_LENGTH32)
check_sz = CHUNK_CHECK_LENGTH32;
memset(ptr + sz, moptsmalloc_readonly.mopts.chunk_canaries, check_sz);
1161}

1163/*
* Allocate a chunk
*/
1166static void *
1167malloc_bytes(struct dir_info *d, size_t size)
1168{
u_int i, j, k, r, bucket, listnum;
u_short	*lp;
struct chunk_info *bp;
void *p;

if (moptsmalloc_readonly.mopts.malloc_canary != (d->canary1 ^ (u_int32_t)(uintptr_t)d) ||
   d->canary1 != ~d->canary2)
wrterror(d, "internal struct corrupt");

bucket = find_bucket(size);

r = getrbyte(d);
listnum = r % MALLOC_CHUNK_LISTS4;

/* If it's empty, make a page more of that size chunks */
if ((bp = LIST_FIRST(&d->chunk_dir[bucket][listnum])((&d->chunk_dir[bucket][listnum])->lh_first)) == NULL((void *)0)) {
bp = omalloc_make_chunks(d, bucket, listnum);
if (bp == NULL((void *)0))
	return NULL((void *)0);
}

if (bp->canary != (u_short)d->canary1 || bucket != bp->bucket)
wrterror(d, "chunk info corrupted");

r /= MALLOC_CHUNK_LISTS4;
/* do we need more random bits? */
if (bp->total > 256 / MALLOC_CHUNK_LISTS4)
r = r << 8 | getrbyte(d);
/* bias, as bp->total is not a power of 2 */
i = r % bp->total;

j = i % MALLOC_BITS(8 * sizeof(u_short));
i /= MALLOC_BITS(8 * sizeof(u_short));
lp = &bp->bits[i];
/* potentially start somewhere in a short */
if (j > 0 && *lp >> j)
k = ffs(*lp >> j) + j;
else {
/* no bit halfway, go to next full short */
for (;;) {
	if (*lp) {
		k = ffs(*lp);
		break;
	}
	if (++i >= bp->offset)
		i = 0;
	lp = &bp->bits[i];
}
}
*lp ^= 1 << --k;

/* If there are no more free, remove from free-list */
if (--bp->free == 0)
LIST_REMOVE(bp, entries)do { if ((bp)->entries.le_next != ((void *)0)) (bp)->entries
.le_next->entries.le_prev = (bp)->entries.le_prev; *(bp
)->entries.le_prev = (bp)->entries.le_next; ; ; } while
 (0);

/* Adjust to the real offset of that chunk */
k += i * MALLOC_BITS(8 * sizeof(u_short));

if (moptsmalloc_readonly.mopts.chunk_canaries && size > 0)
bp->bits[bp->offset + k] = size;

if (DO_STATSmalloc_readonly.mopts.malloc_stats) {
struct region_info *r = find(d, bp->page);
STATS_SETFN(r, k, d->caller)((r)->f[k] = (d->caller));
}

p = (char *)bp->page + k * B2ALLOC(bucket)((bucket) == 0 ? (1UL << 4) : (bucket) * (1UL << 4
));
if (bucket > 0) {
validate_junk(d, p, B2SIZE(bucket)((bucket) * (1UL << 4)));
if (moptsmalloc_readonly.mopts.chunk_canaries)
	fill_canary(p, size, B2SIZE(bucket)((bucket) * (1UL << 4)));
}
return p;
1242}

1244static void
1245validate_canary(struct dir_info *d, u_char *ptr, size_t sz, size_t allocated)
1246{
size_t check_sz = allocated - sz;
u_char *p, *q;

if (check_sz > CHUNK_CHECK_LENGTH32)
check_sz = CHUNK_CHECK_LENGTH32;
p = ptr + sz;
q = p + check_sz;

while (p < q) {
if (*p != (u_char)moptsmalloc_readonly.mopts.chunk_canaries && *p != SOME_JUNK0xdb) {
	wrterror(d, "canary corrupted %p[%tu]@%zu/%zu%s",
	    ptr, p - ptr, sz, allocated,
	    *p == SOME_FREEJUNK0xdf ? " (double free?)" : "");
}
p++;
}
1263}

1265static inline uint32_t
1266find_chunknum(struct dir_info *d, struct chunk_info *info, void *ptr, int check)
1267{
uint32_t chunknum;

if (info->canary != (u_short)d->canary1)
wrterror(d, "chunk info corrupted");

/* Find the chunk number on the page */
chunknum = ((uintptr_t)ptr & MALLOC_PAGEMASK((1UL << 12) - 1)) / B2ALLOC(info->bucket)((info->bucket) == 0 ? (1UL << 4) : (info->bucket
) * (1UL << 4));

if ((uintptr_t)ptr & (MALLOC_MINSIZE(1UL << 4) - 1))
wrterror(d, "modified chunk-pointer %p", ptr);
if (CHUNK_FREE(info, chunknum)((info)->bits[(chunknum) / (8 * sizeof(u_short))] & (1U
 << ((chunknum) % (8 * sizeof(u_short))))))
wrterror(d, "double free %p", ptr);
if (check && info->bucket > 0) {
validate_canary(d, ptr, info->bits[info->offset + chunknum],
    B2SIZE(info->bucket)((info->bucket) * (1UL << 4)));
}
return chunknum;
1285}

1287/*
* Free a chunk, and possibly the page it's on, if the page becomes empty.
*/
1290static void
1291free_bytes(struct dir_info *d, struct region_info *r, void *ptr)
1292{
struct chunk_head *mp;
struct chunk_info *info;
uint32_t chunknum;
uint32_t listnum;

info = (struct chunk_info *)r->size;
chunknum = find_chunknum(d, info, ptr, 0);

info->bits[chunknum / MALLOC_BITS(8 * sizeof(u_short))] |= 1U << (chunknum % MALLOC_BITS(8 * sizeof(u_short)));
info->free++;

if (info->free == 1) {
/* Page became non-full */
listnum = getrbyte(d) % MALLOC_CHUNK_LISTS4;
mp = &d->chunk_dir[info->bucket][listnum];
LIST_INSERT_HEAD(mp, info, entries)do { if (((info)->entries.le_next = (mp)->lh_first) != (
(void *)0)) (mp)->lh_first->entries.le_prev = &(info
)->entries.le_next; (mp)->lh_first = (info); (info)->
entries.le_prev = &(mp)->lh_first; } while (0);
return;
}

if (info->free != info->total)
return;

LIST_REMOVE(info, entries)do { if ((info)->entries.le_next != ((void *)0)) (info)->
entries.le_next->entries.le_prev = (info)->entries.le_prev
; *(info)->entries.le_prev = (info)->entries.le_next; ;
 ; } while (0);

if (info->bucket == 0 && !moptsmalloc_readonly.mopts.malloc_freeunmap)
mprotect(info->page, MALLOC_PAGESIZE(1UL << 12), PROT_READ0x01 | PROT_WRITE0x02);
unmap(d, info->page, MALLOC_PAGESIZE(1UL << 12), 0);
1320#ifdef MALLOC_STATS
if (r->f != NULL((void *)0)) {
unmap(d, r->f, MALLOC_PAGESIZE(1UL << 12), MALLOC_PAGESIZE(1UL << 12));
r->f = NULL((void *)0);
}
1325#endif

delete(d, r);
mp = &d->chunk_info_list[info->bucket];
LIST_INSERT_HEAD(mp, info, entries)do { if (((info)->entries.le_next = (mp)->lh_first) != (
(void *)0)) (mp)->lh_first->entries.le_prev = &(info
)->entries.le_next; (mp)->lh_first = (info); (info)->
entries.le_prev = &(mp)->lh_first; } while (0);
1330}

1332static void *
1333omalloc(struct dir_info *pool, size_t sz, int zero_fill)
1334{
void *p, *caller = NULL((void *)0);
size_t psz;

if (sz > MALLOC_MAXCHUNK(1 << (12 - 1))) {
if (sz >= SIZE_MAX0xffffffffffffffffUL - moptsmalloc_readonly.mopts.malloc_guard - MALLOC_PAGESIZE(1UL << 12)) {
	errno(*__errno()) = ENOMEM12;
	return NULL((void *)0);
}
sz += moptsmalloc_readonly.mopts.malloc_guard;
psz = PAGEROUND(sz)(((sz) + (((1UL << 12) - 1))) & ~((1UL << 12)
 - 1));
p = map(pool, psz, zero_fill);
if (p == MAP_FAILED((void *)-1)) {
	errno(*__errno()) = ENOMEM12;
	return NULL((void *)0);
}
1350#ifdef MALLOC_STATS
if (DO_STATSmalloc_readonly.mopts.malloc_stats)
	caller = pool->caller;
1353#endif
if (insert(pool, p, sz, caller)) {
	unmap(pool, p, psz, 0);
	errno(*__errno()) = ENOMEM12;
	return NULL((void *)0);
}
if (moptsmalloc_readonly.mopts.malloc_guard) {
	if (mprotect((char *)p + psz - moptsmalloc_readonly.mopts.malloc_guard,
	    moptsmalloc_readonly.mopts.malloc_guard, PROT_NONE0x00))
		wrterror(pool, "mprotect");
	STATS_ADD(pool->malloc_guarded, mopts.malloc_guard)((pool->malloc_guarded) += (malloc_readonly.mopts.malloc_guard
));
}

if (MALLOC_MOVE_COND(sz)((sz) - malloc_readonly.mopts.malloc_guard < (1UL <<
 12) - 0)) {
	/* fill whole allocation */
	if (pool->malloc_junk == 2)
		memset(p, SOME_JUNK0xdb, psz - moptsmalloc_readonly.mopts.malloc_guard);
	/* shift towards the end */
	p = MALLOC_MOVE(p, sz)(((char *)(p)) + (((1UL << 12) - 0 - ((sz) - malloc_readonly
.mopts.malloc_guard)) & ~((1UL << 4) - 1)));
	/* fill zeros if needed and overwritten above */
	if (zero_fill && pool->malloc_junk == 2)
		memset(p, 0, sz - moptsmalloc_readonly.mopts.malloc_guard);
} else {
	if (pool->malloc_junk == 2) {
		if (zero_fill)
			memset((char *)p + sz -
			    moptsmalloc_readonly.mopts.malloc_guard, SOME_JUNK0xdb,
			    psz - sz);
		else
			memset(p, SOME_JUNK0xdb,
			    psz - moptsmalloc_readonly.mopts.malloc_guard);
	} else if (moptsmalloc_readonly.mopts.chunk_canaries)
		fill_canary(p, sz - moptsmalloc_readonly.mopts.malloc_guard,
		    psz - moptsmalloc_readonly.mopts.malloc_guard);
}

} else {
/* takes care of SOME_JUNK */
p = malloc_bytes(pool, sz);
if (zero_fill && p != NULL((void *)0) && sz > 0)
	memset(p, 0, sz);
}

return p;
1397}

1399/*
* Common function for handling recursion.  Only
* print the error message once, to avoid making the problem
* potentially worse.
*/
1404static void
1405malloc_recurse(struct dir_info *d)
1406{
static int noprint;

if (noprint == 0) {
noprint = 1;
wrterror(d, "recursive call");
}
d->active--;
_MALLOC_UNLOCK(d->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_unlock(d->mutex
); } while (0);
errno(*__errno()) = EDEADLK11;
1416}

1418void
1419_malloc_init(int from_rthreads)
1420{
u_int i, j, nmutexes;
struct dir_info *d;

_MALLOC_LOCK(1)do { if (__isthreaded) _thread_cb.tc_malloc_lock(1); } while (
0);
8
←
Assuming '__isthreaded' is not equal to 0→
9
←
Taking true branch→
10
←
Assigning value→
if (!from_rthreads10.1
'from_rthreads' is 0
 && moptsmalloc_readonly.mopts.malloc_pool[1]) {
11
←
Assuming pointer value is null→
12
←
Assuming the condition is false→
13
←
Taking false branch→
_MALLOC_UNLOCK(1)do { if (__isthreaded) _thread_cb.tc_malloc_unlock(1); } while
 (0);
return;
}
if (!moptsmalloc_readonly.mopts.malloc_canary) {
14
←
Assuming field 'malloc_canary' is not equal to 0→
char *p;
size_t sz, d_avail;

omalloc_init();
/*
 * Allocate dir_infos with a guard page on either side. Also
 * randomise offset inside the page at which the dir_infos
 * lay (subject to alignment by 1 << MALLOC_MINSHIFT)
 */
sz = moptsmalloc_readonly.mopts.malloc_mutexes * sizeof(*d) + 2 * MALLOC_PAGESIZE(1UL << 12);
if ((p = MMAPNONE(sz, 0)mmap(((void *)0), (sz), 0x00, 0x1000 | 0x0002 | (0), -1, 0)) == MAP_FAILED((void *)-1))
	wrterror(NULL((void *)0), "malloc_init mmap1 failed");
if (mprotect(p + MALLOC_PAGESIZE(1UL << 12), moptsmalloc_readonly.mopts.malloc_mutexes *
    sizeof(*d), PROT_READ0x01 | PROT_WRITE0x02))
	wrterror(NULL((void *)0), "malloc_init mprotect1 failed");
if (mimmutable(p, sz))
	wrterror(NULL((void *)0), "malloc_init mimmutable1 failed");
d_avail = (((moptsmalloc_readonly.mopts.malloc_mutexes * sizeof(*d) +
    MALLOC_PAGEMASK((1UL << 12) - 1)) & ~MALLOC_PAGEMASK((1UL << 12) - 1)) -
    (moptsmalloc_readonly.mopts.malloc_mutexes * sizeof(*d))) >> MALLOC_MINSHIFT4;
d = (struct dir_info *)(p + MALLOC_PAGESIZE(1UL << 12) +
    (arc4random_uniform(d_avail) << MALLOC_MINSHIFT4));
STATS_ADD(d[1].malloc_used, sz)((d[1].malloc_used) += (sz));
for (i = 0; i < moptsmalloc_readonly.mopts.malloc_mutexes; i++)
	moptsmalloc_readonly.mopts.malloc_pool[i] = &d[i];
moptsmalloc_readonly.mopts.internal_funcs = 1;
if (((uintptr_t)&malloc_readonly & MALLOC_PAGEMASK((1UL << 12) - 1)) == 0) {
	if (mprotect(&malloc_readonly, sizeof(malloc_readonly),
	    PROT_READ0x01))
		wrterror(NULL((void *)0),
		    "malloc_init mprotect r/o failed");
	if (mimmutable(&malloc_readonly,
	    sizeof(malloc_readonly)))
		wrterror(NULL((void *)0),
		    "malloc_init mimmutable r/o failed");
}
}

nmutexes = from_rthreads15.1
'from_rthreads' is 0
 ? moptsmalloc_readonly.mopts.malloc_mutexes : 2;
15
←
Taking false branch→
16
←
'?' condition is false→
for (i = 0; i < nmutexes; i++) {
17
←
Loop condition is true.  Entering loop body→
20
←
 Execution continues on line 1469→
21
←
The value 1 is assigned to 'i'→
22
←
Loop condition is true.  Entering loop body→
d = moptsmalloc_readonly.mopts.malloc_pool[i];
23
←
Null pointer value stored to 'd'→
d->malloc_mt = from_rthreads;
24
←
Access to field 'malloc_mt' results in a dereference of a null pointer (loaded from variable 'd')
if (d->canary1 == ~d->canary2)
18
←
Assuming the condition is true→
19
←
Taking true branch→
	continue;
if (i == 0) {
	omalloc_poolinit(d, MAP_CONCEAL0x8000);
	d->malloc_junk = 2;
	d->bigcache_size = 0;
	for (j = 0; j < MAX_SMALLCACHEABLE_SIZE32; j++)
		d->smallcache[j].max = 0;
} else {
	size_t sz = 0;

	omalloc_poolinit(d, 0);
	d->malloc_junk = moptsmalloc_readonly.mopts.def_malloc_junk;
	d->bigcache_size = moptsmalloc_readonly.mopts.def_maxcache;
	for (j = 0; j < MAX_SMALLCACHEABLE_SIZE32; j++) {
		d->smallcache[j].max =
		    moptsmalloc_readonly.mopts.def_maxcache >> (j / 8);
		sz += d->smallcache[j].max * sizeof(void *);
	}
	sz += d->bigcache_size * sizeof(struct bigcache);
	if (sz > 0) {
		void *p = MMAP(sz, 0)mmap(((void *)0), (sz), 0x01 | 0x02, 0x1000 | 0x0002 | (0), -
1, 0);
		if (p == MAP_FAILED((void *)-1))
			wrterror(NULL((void *)0),
			    "malloc_init mmap2 failed");
		if (mimmutable(p, sz))
			wrterror(NULL((void *)0),
			    "malloc_init mimmutable2 failed");
		for (j = 0; j < MAX_SMALLCACHEABLE_SIZE32; j++) {
			d->smallcache[j].pages = p;
			p = (char *)p + d->smallcache[j].max *
			    sizeof(void *);
		}
		d->bigcache = p;
	}
}
d->mutex = i;
}

_MALLOC_UNLOCK(1)do { if (__isthreaded) _thread_cb.tc_malloc_unlock(1); } while
 (0);
1512}
1513DEF_STRONG(_malloc_init)__asm__(".global " "_malloc_init" " ; " "_malloc_init" " = " "_libc__malloc_init"
);

1515#define PROLOGUE(p, fn)d = (p); if (d == ((void *)0)) { _malloc_init(0); d = (p); } do
 { if (__isthreaded) _thread_cb.tc_malloc_lock(d->mutex); }
 while (0); d->func = fn; if (d->active++) { malloc_recurse
(d); return ((void *)0); }			\
d = (p); 			\
if (d == NULL((void *)0)) { 		\
_malloc_init(0);	\
d = (p);		\
}				\
_MALLOC_LOCK(d->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_lock(d->mutex)
; } while (0);		\
d->func = fn;			\
if (d->active++) {		\
malloc_recurse(d);	\
return NULL((void *)0);		\
}				\

1528#define EPILOGUE()d->active--; do { if (__isthreaded) _thread_cb.tc_malloc_unlock
(d->mutex); } while (0); if (r == ((void *)0) && malloc_readonly
.mopts.malloc_xmalloc) wrterror(d, "out of memory"); if (r !=
 ((void *)0)) (*__errno()) = saved_errno;				\
d->active--;				\
_MALLOC_UNLOCK(d->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_unlock(d->mutex
); } while (0);		\
if (r == NULL((void *)0) && moptsmalloc_readonly.mopts.malloc_xmalloc)	\
wrterror(d, "out of memory");	\
if (r != NULL((void *)0))				\
errno(*__errno()) = saved_errno;		\

1536void *
1537malloc(size_t size)
1538{
void *r;
struct dir_info *d;
int saved_errno = errno(*__errno());

PROLOGUE(getpool(), "malloc")d = (getpool()); if (d == ((void *)0)) { _malloc_init(0); d =
 (getpool()); } do { if (__isthreaded) _thread_cb.tc_malloc_lock
(d->mutex); } while (0); d->func = "malloc"; if (d->
active++) { malloc_recurse(d); return ((void *)0); }
SET_CALLER(d, caller(d))if (malloc_readonly.mopts.malloc_stats) ((d)->caller = (caller
(d)));
r = omalloc(d, size, 0);
EPILOGUE()d->active--; do { if (__isthreaded) _thread_cb.tc_malloc_unlock
(d->mutex); } while (0); if (r == ((void *)0) && malloc_readonly
.mopts.malloc_xmalloc) wrterror(d, "out of memory"); if (r !=
 ((void *)0)) (*__errno()) = saved_errno;
return r;
1548}
1549DEF_STRONG(malloc)__asm__(".global " "malloc" " ; " "malloc" " = " "_libc_malloc"
);

1551void *
1552malloc_conceal(size_t size)
1553{
void *r;
struct dir_info *d;
int saved_errno = errno(*__errno());

PROLOGUE(mopts.malloc_pool[0], "malloc_conceal")d = (malloc_readonly.mopts.malloc_pool[0]); if (d == ((void *
)0)) { _malloc_init(0); d = (malloc_readonly.mopts.malloc_pool
[0]); } do { if (__isthreaded) _thread_cb.tc_malloc_lock(d->
mutex); } while (0); d->func = "malloc_conceal"; if (d->
active++) { malloc_recurse(d); return ((void *)0); }
SET_CALLER(d, caller(d))if (malloc_readonly.mopts.malloc_stats) ((d)->caller = (caller
(d)));
r = omalloc(d, size, 0);
EPILOGUE()d->active--; do { if (__isthreaded) _thread_cb.tc_malloc_unlock
(d->mutex); } while (0); if (r == ((void *)0) && malloc_readonly
.mopts.malloc_xmalloc) wrterror(d, "out of memory"); if (r !=
 ((void *)0)) (*__errno()) = saved_errno;
return r;
1563}
1564DEF_WEAK(malloc_conceal)__asm__(".weak " "malloc_conceal" " ; " "malloc_conceal" " = "
 "_libc_malloc_conceal");

1566static struct region_info *
1567findpool(void *p, struct dir_info *argpool, struct dir_info **foundpool,
  const char ** saved_function)
1569{
struct dir_info *pool = argpool;
struct region_info *r = find(pool, p);

if (r == NULL((void *)0)) {
u_int i, nmutexes;

nmutexes = moptsmalloc_readonly.mopts.malloc_pool[1]->malloc_mt ?
    moptsmalloc_readonly.mopts.malloc_mutexes : 2;
for (i = 1; i < nmutexes; i++) {
	u_int j = (argpool->mutex + i) & (nmutexes - 1);

	pool->active--;
	_MALLOC_UNLOCK(pool->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_unlock(pool->mutex
); } while (0);
	pool = moptsmalloc_readonly.mopts.malloc_pool[j];
	_MALLOC_LOCK(pool->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_lock(pool->mutex
); } while (0);
	pool->active++;
	r = find(pool, p);
	if (r != NULL((void *)0)) {
		*saved_function = pool->func;
		pool->func = argpool->func;
		break;
	}
}
if (r == NULL((void *)0))
	wrterror(argpool, "bogus pointer (double free?) %p", p);
}
*foundpool = pool;
return r;
1598}

1600static void
1601ofree(struct dir_info **argpool, void *p, int clear, int check, size_t argsz)
1602{
struct region_info *r;
struct dir_info *pool;
const char *saved_function;
size_t sz;

r = findpool(p, *argpool, &pool, &saved_function);

REALSIZE(sz, r)(sz) = (uintptr_t)(r)->p & ((1UL << 12) - 1), (sz
) = ((sz) == 0 ? (r)->size : (((sz) - 1) * (1UL << 4
)));
if (pool->mmap_flag) {
clear = 1;
if (!check) {
	argsz = sz;
	if (sz > MALLOC_MAXCHUNK(1 << (12 - 1)))
		argsz -= moptsmalloc_readonly.mopts.malloc_guard;
}
}
if (check) {
if (sz <= MALLOC_MAXCHUNK(1 << (12 - 1))) {
	if (moptsmalloc_readonly.mopts.chunk_canaries && sz > 0) {
		struct chunk_info *info =
		    (struct chunk_info *)r->size;
		uint32_t chunknum =
		    find_chunknum(pool, info, p, 0);

		if (info->bits[info->offset + chunknum] < argsz)
			wrterror(pool, "recorded size %hu"
			    " < %zu",
			    info->bits[info->offset + chunknum],
			    argsz);
	} else {
		if (sz < argsz)
			wrterror(pool, "chunk size %zu < %zu",
			    sz, argsz);
	}
} else if (sz - moptsmalloc_readonly.mopts.malloc_guard < argsz) {
	wrterror(pool, "recorded size %zu < %zu",
	    sz - moptsmalloc_readonly.mopts.malloc_guard, argsz);
}
}
if (sz > MALLOC_MAXCHUNK(1 << (12 - 1))) {
if (!MALLOC_MOVE_COND(sz)((sz) - malloc_readonly.mopts.malloc_guard < (1UL <<
 12) - 0)) {
	if (r->p != p)
		wrterror(pool, "bogus pointer %p", p);
	if (moptsmalloc_readonly.mopts.chunk_canaries)
		validate_canary(pool, p,
		    sz - moptsmalloc_readonly.mopts.malloc_guard,
		    PAGEROUND(sz - mopts.malloc_guard)(((sz - malloc_readonly.mopts.malloc_guard) + (((1UL <<
 12) - 1))) & ~((1UL << 12) - 1)));
} else {
	/* shifted towards the end */
	if (p != MALLOC_MOVE(r->p, sz)(((char *)(r->p)) + (((1UL << 12) - 0 - ((sz) - malloc_readonly
.mopts.malloc_guard)) & ~((1UL << 4) - 1))))
		wrterror(pool, "bogus moved pointer %p", p);
	p = r->p;
}
if (moptsmalloc_readonly.mopts.malloc_guard) {
	if (sz < moptsmalloc_readonly.mopts.malloc_guard)
		wrterror(pool, "guard size");
	if (!moptsmalloc_readonly.mopts.malloc_freeunmap) {
		if (mprotect((char *)p + PAGEROUND(sz)(((sz) + (((1UL << 12) - 1))) & ~((1UL << 12)
 - 1)) -
		    moptsmalloc_readonly.mopts.malloc_guard, moptsmalloc_readonly.mopts.malloc_guard,
		    PROT_READ0x01 | PROT_WRITE0x02))
			wrterror(pool, "mprotect");
	}
	STATS_SUB(pool->malloc_guarded, mopts.malloc_guard)((pool->malloc_guarded) -= (malloc_readonly.mopts.malloc_guard
));
}
unmap(pool, p, PAGEROUND(sz)(((sz) + (((1UL << 12) - 1))) & ~((1UL << 12)
 - 1)), clear ? argsz : 0);
delete(pool, r);
} else {
void *tmp;
u_int i;

/* Validate and optionally canary check */
struct chunk_info *info = (struct chunk_info *)r->size;
if (B2SIZE(info->bucket)((info->bucket) * (1UL << 4)) != sz)
	wrterror(pool, "internal struct corrupt");
find_chunknum(pool, info, p, moptsmalloc_readonly.mopts.chunk_canaries);

if (moptsmalloc_readonly.mopts.malloc_freecheck) {
	for (i = 0; i <= MALLOC_DELAYED_CHUNK_MASK15; i++) {
		tmp = pool->delayed_chunks[i];
		if (tmp == p)
			wrterror(pool,
			    "double free %p", p);
		if (tmp != NULL((void *)0)) {
			size_t tmpsz;

			r = find(pool, tmp);
			if (r == NULL((void *)0))
				wrterror(pool,
				    "bogus pointer ("
				    "double free?) %p", tmp);
			REALSIZE(tmpsz, r)(tmpsz) = (uintptr_t)(r)->p & ((1UL << 12) - 1),
 (tmpsz) = ((tmpsz) == 0 ? (r)->size : (((tmpsz) - 1) * (1UL
 << 4)));
			validate_junk(pool, tmp, tmpsz);
		}
	}
}

if (clear && argsz > 0)
	explicit_bzero(p, argsz);
junk_free(pool->malloc_junk, p, sz);

i = getrbyte(pool) & MALLOC_DELAYED_CHUNK_MASK15;
tmp = p;
p = pool->delayed_chunks[i];
if (tmp == p)
	wrterror(pool, "double free %p", p);
pool->delayed_chunks[i] = tmp;
if (p != NULL((void *)0)) {
	r = find(pool, p);
	if (r == NULL((void *)0))
		wrterror(pool,
		    "bogus pointer (double free?) %p", p);
	if (!moptsmalloc_readonly.mopts.malloc_freecheck) {
		REALSIZE(sz, r)(sz) = (uintptr_t)(r)->p & ((1UL << 12) - 1), (sz
) = ((sz) == 0 ? (r)->size : (((sz) - 1) * (1UL << 4
)));
		validate_junk(pool, p, sz);
	}
	free_bytes(pool, r, p);
}
}

if (*argpool != pool) {
pool->func = saved_function;
*argpool = pool;
}
1726}

1728void
1729free(void *ptr)
1730{
struct dir_info *d;
int saved_errno = errno(*__errno());

/* This is legal. */
if (ptr == NULL((void *)0))
return;

d = getpool();
if (d == NULL((void *)0))
wrterror(d, "free() called before allocation");
_MALLOC_LOCK(d->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_lock(d->mutex)
; } while (0);
d->func = "free";
if (d->active++) {
malloc_recurse(d);
return;
}
ofree(&d, ptr, 0, 0, 0);
d->active--;
_MALLOC_UNLOCK(d->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_unlock(d->mutex
); } while (0);
errno(*__errno()) = saved_errno;
1751}
1752DEF_STRONG(free)__asm__(".global " "free" " ; " "free" " = " "_libc_free");

1754static void
1755freezero_p(void *ptr, size_t sz)
1756{
explicit_bzero(ptr, sz);
free(ptr);
1759}

1761void
1762freezero(void *ptr, size_t sz)
1763{
struct dir_info *d;
int saved_errno = errno(*__errno());

/* This is legal. */
if (ptr == NULL((void *)0))
return;

if (!moptsmalloc_readonly.mopts.internal_funcs) {
freezero_p(ptr, sz);
return;
}

d = getpool();
if (d == NULL((void *)0))
wrterror(d, "freezero() called before allocation");
_MALLOC_LOCK(d->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_lock(d->mutex)
; } while (0);
d->func = "freezero";
if (d->active++) {
malloc_recurse(d);
return;
}
ofree(&d, ptr, 1, 1, sz);
d->active--;
_MALLOC_UNLOCK(d->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_unlock(d->mutex
); } while (0);
errno(*__errno()) = saved_errno;
1789}
1790DEF_WEAK(freezero)__asm__(".weak " "freezero" " ; " "freezero" " = " "_libc_freezero"
);

1792static void *
1793orealloc(struct dir_info **argpool, void *p, size_t newsz)
1794{
struct region_info *r;
struct dir_info *pool;
const char *saved_function;
struct chunk_info *info;
size_t oldsz, goldsz, gnewsz;
void *q, *ret;
uint32_t chunknum;
int forced;

if (p == NULL((void *)0))
return omalloc(*argpool, newsz, 0);

if (newsz >= SIZE_MAX0xffffffffffffffffUL - moptsmalloc_readonly.mopts.malloc_guard - MALLOC_PAGESIZE(1UL << 12)) {
errno(*__errno()) = ENOMEM12;
return  NULL((void *)0);
}

r = findpool(p, *argpool, &pool, &saved_function);

REALSIZE(oldsz, r)(oldsz) = (uintptr_t)(r)->p & ((1UL << 12) - 1),
 (oldsz) = ((oldsz) == 0 ? (r)->size : (((oldsz) - 1) * (1UL
 << 4)));
if (oldsz <= MALLOC_MAXCHUNK(1 << (12 - 1))) {
if (DO_STATSmalloc_readonly.mopts.malloc_stats || moptsmalloc_readonly.mopts.chunk_canaries) {
	info = (struct chunk_info *)r->size;
	chunknum = find_chunknum(pool, info, p, 0);
}
}

goldsz = oldsz;
if (oldsz > MALLOC_MAXCHUNK(1 << (12 - 1))) {
if (oldsz < moptsmalloc_readonly.mopts.malloc_guard)
	wrterror(pool, "guard size");
oldsz -= moptsmalloc_readonly.mopts.malloc_guard;
}

gnewsz = newsz;
if (gnewsz > MALLOC_MAXCHUNK(1 << (12 - 1)))
gnewsz += moptsmalloc_readonly.mopts.malloc_guard;

forced = moptsmalloc_readonly.mopts.malloc_realloc || pool->mmap_flag;
if (newsz > MALLOC_MAXCHUNK(1 << (12 - 1)) && oldsz > MALLOC_MAXCHUNK(1 << (12 - 1)) && !forced) {
/* First case: from n pages sized allocation to m pages sized
   allocation, m > n */
size_t roldsz = PAGEROUND(goldsz)(((goldsz) + (((1UL << 12) - 1))) & ~((1UL <<
 12) - 1));
size_t rnewsz = PAGEROUND(gnewsz)(((gnewsz) + (((1UL << 12) - 1))) & ~((1UL <<
 12) - 1));

if (rnewsz < roldsz && rnewsz > roldsz / 2 &&
    roldsz - rnewsz < moptsmalloc_readonly.mopts.def_maxcache * MALLOC_PAGESIZE(1UL << 12) &&
    !moptsmalloc_readonly.mopts.malloc_guard) {

	ret = p;
	goto done;
}

if (rnewsz > roldsz) {
	/* try to extend existing region */
	if (!moptsmalloc_readonly.mopts.malloc_guard) {
		void *hint = (char *)r->p + roldsz;
		size_t needed = rnewsz - roldsz;

		STATS_INC(pool->cheap_realloc_tries)((pool->cheap_realloc_tries)++);
		q = MMAPA(hint, needed, MAP_FIXED |mmap((hint), (needed), 0x01 | 0x02, 0x1000 | 0x0002 | (0x0010
 | 0x0800 | pool->mmap_flag), -1, 0)
		    __MAP_NOREPLACE | pool->mmap_flag)mmap((hint), (needed), 0x01 | 0x02, 0x1000 | 0x0002 | (0x0010
 | 0x0800 | pool->mmap_flag), -1, 0);
		if (q == hint) {
			STATS_ADD(pool->malloc_used, needed)((pool->malloc_used) += (needed));
			if (pool->malloc_junk == 2)
				memset(q, SOME_JUNK0xdb, needed);
			r->size = gnewsz;
			if (r->p != p) {
				/* old pointer is moved */
				memmove(r->p, p, oldsz);
				p = r->p;
			}
			if (moptsmalloc_readonly.mopts.chunk_canaries)
				fill_canary(p, newsz,
				    PAGEROUND(newsz)(((newsz) + (((1UL << 12) - 1))) & ~((1UL << 12
) - 1)));
			STATS_SETF(r, (*argpool)->caller)((r)->f = ((*argpool)->caller));
			STATS_INC(pool->cheap_reallocs)((pool->cheap_reallocs)++);
			ret = p;
			goto done;
		}
	}
} else if (rnewsz < roldsz) {
	/* shrink number of pages */
	if (moptsmalloc_readonly.mopts.malloc_guard) {
		if (mprotect((char *)r->p + rnewsz -
		    moptsmalloc_readonly.mopts.malloc_guard, moptsmalloc_readonly.mopts.malloc_guard,
		    PROT_NONE0x00))
			wrterror(pool, "mprotect");
	}
	if (munmap((char *)r->p + rnewsz, roldsz - rnewsz))
		wrterror(pool, "munmap %p", (char *)r->p +
		    rnewsz);
	STATS_SUB(pool->malloc_used, roldsz - rnewsz)((pool->malloc_used) -= (roldsz - rnewsz));
	r->size = gnewsz;
	if (MALLOC_MOVE_COND(gnewsz)((gnewsz) - malloc_readonly.mopts.malloc_guard < (1UL <<
 12) - 0)) {
		void *pp = MALLOC_MOVE(r->p, gnewsz)(((char *)(r->p)) + (((1UL << 12) - 0 - ((gnewsz) - malloc_readonly
.mopts.malloc_guard)) & ~((1UL << 4) - 1)));
		memmove(pp, p, newsz);
		p = pp;
	} else if (moptsmalloc_readonly.mopts.chunk_canaries)
		fill_canary(p, newsz, PAGEROUND(newsz)(((newsz) + (((1UL << 12) - 1))) & ~((1UL << 12
) - 1)));
	STATS_SETF(r, (*argpool)->caller)((r)->f = ((*argpool)->caller));
	ret = p;
	goto done;
} else {
	/* number of pages remains the same */
	void *pp = r->p;

	r->size = gnewsz;
	if (MALLOC_MOVE_COND(gnewsz)((gnewsz) - malloc_readonly.mopts.malloc_guard < (1UL <<
 12) - 0))
		pp = MALLOC_MOVE(r->p, gnewsz)(((char *)(r->p)) + (((1UL << 12) - 0 - ((gnewsz) - malloc_readonly
.mopts.malloc_guard)) & ~((1UL << 4) - 1)));
	if (p != pp) {
		memmove(pp, p, oldsz < newsz ? oldsz : newsz);
		p = pp;
	}
	if (p == r->p) {
		if (newsz > oldsz && pool->malloc_junk == 2)
			memset((char *)p + newsz, SOME_JUNK0xdb,
			    rnewsz - moptsmalloc_readonly.mopts.malloc_guard -
			    newsz);
		if (moptsmalloc_readonly.mopts.chunk_canaries)
			fill_canary(p, newsz, PAGEROUND(newsz)(((newsz) + (((1UL << 12) - 1))) & ~((1UL << 12
) - 1)));
	}
	STATS_SETF(r, (*argpool)->caller)((r)->f = ((*argpool)->caller));
	ret = p;
	goto done;
}
}
if (oldsz <= MALLOC_MAXCHUNK(1 << (12 - 1)) && oldsz > 0 &&
   newsz <= MALLOC_MAXCHUNK(1 << (12 - 1)) && newsz > 0 &&
   !forced && find_bucket(newsz) == find_bucket(oldsz)) {
/* do not reallocate if new size fits good in existing chunk */
if (pool->malloc_junk == 2)
	memset((char *)p + newsz, SOME_JUNK0xdb, oldsz - newsz);
if (moptsmalloc_readonly.mopts.chunk_canaries) {
	info->bits[info->offset + chunknum] = newsz;
	fill_canary(p, newsz, B2SIZE(info->bucket)((info->bucket) * (1UL << 4)));
}
if (DO_STATSmalloc_readonly.mopts.malloc_stats)
	STATS_SETFN(r, chunknum, (*argpool)->caller)((r)->f[chunknum] = ((*argpool)->caller));
ret = p;
} else if (newsz != oldsz || forced) {
/* create new allocation */
q = omalloc(pool, newsz, 0);
if (q == NULL((void *)0)) {
	ret = NULL((void *)0);
	goto done;
}
if (newsz != 0 && oldsz != 0)
	memcpy(q, p, oldsz < newsz ? oldsz : newsz);
ofree(&pool, p, 0, 0, 0);
ret = q;
} else {
/* oldsz == newsz */
if (newsz != 0)
	wrterror(pool, "realloc internal inconsistency");
if (DO_STATSmalloc_readonly.mopts.malloc_stats)
	STATS_SETFN(r, chunknum, (*argpool)->caller)((r)->f[chunknum] = ((*argpool)->caller));
ret = p;
}
1954done:
if (*argpool != pool) {
pool->func = saved_function;
*argpool = pool;
}
return ret;
1960}

1962void *
1963realloc(void *ptr, size_t size)
1964{
struct dir_info *d;
void *r;
int saved_errno = errno(*__errno());

PROLOGUE(getpool(), "realloc")d = (getpool()); if (d == ((void *)0)) { _malloc_init(0); d =
 (getpool()); } do { if (__isthreaded) _thread_cb.tc_malloc_lock
(d->mutex); } while (0); d->func = "realloc"; if (d->
active++) { malloc_recurse(d); return ((void *)0); }
SET_CALLER(d, caller(d))if (malloc_readonly.mopts.malloc_stats) ((d)->caller = (caller
(d)));
r = orealloc(&d, ptr, size);
EPILOGUE()d->active--; do { if (__isthreaded) _thread_cb.tc_malloc_unlock
(d->mutex); } while (0); if (r == ((void *)0) && malloc_readonly
.mopts.malloc_xmalloc) wrterror(d, "out of memory"); if (r !=
 ((void *)0)) (*__errno()) = saved_errno;
return r;
1974}
1975DEF_STRONG(realloc)__asm__(".global " "realloc" " ; " "realloc" " = " "_libc_realloc"
);

1977/*
* This is sqrt(SIZE_MAX+1), as s1*s2 <= SIZE_MAX
* if both s1 < MUL_NO_OVERFLOW and s2 < MUL_NO_OVERFLOW
*/
1981#define MUL_NO_OVERFLOW(1UL << (sizeof(size_t) * 4))	(1UL << (sizeof(size_t) * 4))

1983void *
1984calloc(size_t nmemb, size_t size)
1985{
struct dir_info *d;
void *r;
int saved_errno = errno(*__errno());

PROLOGUE(getpool(), "calloc")d = (getpool()); if (d == ((void *)0)) { _malloc_init(0); d =
 (getpool()); } do { if (__isthreaded) _thread_cb.tc_malloc_lock
(d->mutex); } while (0); d->func = "calloc"; if (d->
active++) { malloc_recurse(d); return ((void *)0); }
SET_CALLER(d, caller(d))if (malloc_readonly.mopts.malloc_stats) ((d)->caller = (caller
(d)));
if ((nmemb >= MUL_NO_OVERFLOW(1UL << (sizeof(size_t) * 4)) || size >= MUL_NO_OVERFLOW(1UL << (sizeof(size_t) * 4))) &&
   nmemb > 0 && SIZE_MAX0xffffffffffffffffUL / nmemb < size) {
d->active--;
_MALLOC_UNLOCK(d->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_unlock(d->mutex
); } while (0);
if (moptsmalloc_readonly.mopts.malloc_xmalloc)
	wrterror(d, "out of memory");
errno(*__errno()) = ENOMEM12;
return NULL((void *)0);
}

size *= nmemb;
r = omalloc(d, size, 1);
EPILOGUE()d->active--; do { if (__isthreaded) _thread_cb.tc_malloc_unlock
(d->mutex); } while (0); if (r == ((void *)0) && malloc_readonly
.mopts.malloc_xmalloc) wrterror(d, "out of memory"); if (r !=
 ((void *)0)) (*__errno()) = saved_errno;
return r;
2006}
2007DEF_STRONG(calloc)__asm__(".global " "calloc" " ; " "calloc" " = " "_libc_calloc"
);

2009void *
2010calloc_conceal(size_t nmemb, size_t size)
2011{
struct dir_info *d;
void *r;
int saved_errno = errno(*__errno());

PROLOGUE(mopts.malloc_pool[0], "calloc_conceal")d = (malloc_readonly.mopts.malloc_pool[0]); if (d == ((void *
)0)) { _malloc_init(0); d = (malloc_readonly.mopts.malloc_pool
[0]); } do { if (__isthreaded) _thread_cb.tc_malloc_lock(d->
mutex); } while (0); d->func = "calloc_conceal"; if (d->
active++) { malloc_recurse(d); return ((void *)0); }
SET_CALLER(d, caller(d))if (malloc_readonly.mopts.malloc_stats) ((d)->caller = (caller
(d)));
if ((nmemb >= MUL_NO_OVERFLOW(1UL << (sizeof(size_t) * 4)) || size >= MUL_NO_OVERFLOW(1UL << (sizeof(size_t) * 4))) &&
   nmemb > 0 && SIZE_MAX0xffffffffffffffffUL / nmemb < size) {
d->active--;
_MALLOC_UNLOCK(d->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_unlock(d->mutex
); } while (0);
if (moptsmalloc_readonly.mopts.malloc_xmalloc)
	wrterror(d, "out of memory");
errno(*__errno()) = ENOMEM12;
return NULL((void *)0);
}

size *= nmemb;
r = omalloc(d, size, 1);
EPILOGUE()d->active--; do { if (__isthreaded) _thread_cb.tc_malloc_unlock
(d->mutex); } while (0); if (r == ((void *)0) && malloc_readonly
.mopts.malloc_xmalloc) wrterror(d, "out of memory"); if (r !=
 ((void *)0)) (*__errno()) = saved_errno;
return r;
2032}
2033DEF_WEAK(calloc_conceal)__asm__(".weak " "calloc_conceal" " ; " "calloc_conceal" " = "
 "_libc_calloc_conceal");

2035static void *
2036orecallocarray(struct dir_info **argpool, void *p, size_t oldsize,
  size_t newsize)
2038{
struct region_info *r;
struct dir_info *pool;
const char *saved_function;
void *newptr;
size_t sz;

if (p == NULL((void *)0))
return omalloc(*argpool, newsize, 1);

if (oldsize == newsize)
return p;

r = findpool(p, *argpool, &pool, &saved_function);

REALSIZE(sz, r)(sz) = (uintptr_t)(r)->p & ((1UL << 12) - 1), (sz
) = ((sz) == 0 ? (r)->size : (((sz) - 1) * (1UL << 4
)));
if (sz <= MALLOC_MAXCHUNK(1 << (12 - 1))) {
if (moptsmalloc_readonly.mopts.chunk_canaries && sz > 0) {
	struct chunk_info *info = (struct chunk_info *)r->size;
	uint32_t chunknum = find_chunknum(pool, info, p, 0);

	if (info->bits[info->offset + chunknum] != oldsize)
		wrterror(pool, "recorded size %hu != %zu",
		    info->bits[info->offset + chunknum],
		    oldsize);
} else {
	if (sz < oldsize)
		wrterror(pool, "chunk size %zu < %zu",
		    sz, oldsize);
}
} else {
if (sz - moptsmalloc_readonly.mopts.malloc_guard < oldsize)
	wrterror(pool, "recorded size %zu < %zu",
	    sz - moptsmalloc_readonly.mopts.malloc_guard, oldsize);
if (oldsize < (sz - moptsmalloc_readonly.mopts.malloc_guard) / 2)
	wrterror(pool,
	    "recorded size %zu inconsistent with %zu",
	    sz - moptsmalloc_readonly.mopts.malloc_guard, oldsize);
}

newptr = omalloc(pool, newsize, 0);
if (newptr == NULL((void *)0))
goto done;

if (newsize > oldsize) {
memcpy(newptr, p, oldsize);
memset((char *)newptr + oldsize, 0, newsize - oldsize);
} else
memcpy(newptr, p, newsize);

ofree(&pool, p, 1, 0, oldsize);

2090done:
if (*argpool != pool) {
pool->func = saved_function;
*argpool = pool;
}

return newptr;
2097}

2099static void *
2100recallocarray_p(void *ptr, size_t oldnmemb, size_t newnmemb, size_t size)
2101{
size_t oldsize, newsize;
void *newptr;

if (ptr == NULL((void *)0))
return calloc(newnmemb, size);

if ((newnmemb >= MUL_NO_OVERFLOW(1UL << (sizeof(size_t) * 4)) || size >= MUL_NO_OVERFLOW(1UL << (sizeof(size_t) * 4))) &&
   newnmemb > 0 && SIZE_MAX0xffffffffffffffffUL / newnmemb < size) {
errno(*__errno()) = ENOMEM12;
return NULL((void *)0);
}
newsize = newnmemb * size;

if ((oldnmemb >= MUL_NO_OVERFLOW(1UL << (sizeof(size_t) * 4)) || size >= MUL_NO_OVERFLOW(1UL << (sizeof(size_t) * 4))) &&
   oldnmemb > 0 && SIZE_MAX0xffffffffffffffffUL / oldnmemb < size) {
errno(*__errno()) = EINVAL22;
return NULL((void *)0);
}
oldsize = oldnmemb * size;

/*
* Don't bother too much if we're shrinking just a bit,
* we do not shrink for series of small steps, oh well.
*/
if (newsize <= oldsize) {
size_t d = oldsize - newsize;

if (d < oldsize / 2 && d < MALLOC_PAGESIZE(1UL << 12)) {
	memset((char *)ptr + newsize, 0, d);
	return ptr;
}
}

newptr = malloc(newsize);
if (newptr == NULL((void *)0))
return NULL((void *)0);

if (newsize > oldsize) {
memcpy(newptr, ptr, oldsize);
memset((char *)newptr + oldsize, 0, newsize - oldsize);
} else
memcpy(newptr, ptr, newsize);

explicit_bzero(ptr, oldsize);
free(ptr);

return newptr;
2149}

2151void *
2152recallocarray(void *ptr, size_t oldnmemb, size_t newnmemb, size_t size)
2153{
struct dir_info *d;
size_t oldsize = 0, newsize;
void *r;
int saved_errno = errno(*__errno());

if (!moptsmalloc_readonly.mopts.internal_funcs)
return recallocarray_p(ptr, oldnmemb, newnmemb, size);

PROLOGUE(getpool(), "recallocarray")d = (getpool()); if (d == ((void *)0)) { _malloc_init(0); d =
 (getpool()); } do { if (__isthreaded) _thread_cb.tc_malloc_lock
(d->mutex); } while (0); d->func = "recallocarray"; if (
d->active++) { malloc_recurse(d); return ((void *)0); }
SET_CALLER(d, caller(d))if (malloc_readonly.mopts.malloc_stats) ((d)->caller = (caller
(d)));

if ((newnmemb >= MUL_NO_OVERFLOW(1UL << (sizeof(size_t) * 4)) || size >= MUL_NO_OVERFLOW(1UL << (sizeof(size_t) * 4))) &&
   newnmemb > 0 && SIZE_MAX0xffffffffffffffffUL / newnmemb < size) {
d->active--;
_MALLOC_UNLOCK(d->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_unlock(d->mutex
); } while (0);
if (moptsmalloc_readonly.mopts.malloc_xmalloc)
	wrterror(d, "out of memory");
errno(*__errno()) = ENOMEM12;
return NULL((void *)0);
}
newsize = newnmemb * size;

if (ptr != NULL((void *)0)) {
if ((oldnmemb >= MUL_NO_OVERFLOW(1UL << (sizeof(size_t) * 4)) || size >= MUL_NO_OVERFLOW(1UL << (sizeof(size_t) * 4))) &&
    oldnmemb > 0 && SIZE_MAX0xffffffffffffffffUL / oldnmemb < size) {
	d->active--;
	_MALLOC_UNLOCK(d->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_unlock(d->mutex
); } while (0);
	errno(*__errno()) = EINVAL22;
	return NULL((void *)0);
}
oldsize = oldnmemb * size;
}

r = orecallocarray(&d, ptr, oldsize, newsize);
EPILOGUE()d->active--; do { if (__isthreaded) _thread_cb.tc_malloc_unlock
(d->mutex); } while (0); if (r == ((void *)0) && malloc_readonly
.mopts.malloc_xmalloc) wrterror(d, "out of memory"); if (r !=
 ((void *)0)) (*__errno()) = saved_errno;
return r;
2190}
2191DEF_WEAK(recallocarray)__asm__(".weak " "recallocarray" " ; " "recallocarray" " = " "_libc_recallocarray"
);

2193static void *
2194mapalign(struct dir_info *d, size_t alignment, size_t sz, int zero_fill)
2195{
char *p, *q;

if (alignment < MALLOC_PAGESIZE(1UL << 12) || ((alignment - 1) & alignment) != 0)
wrterror(d, "mapalign bad alignment");
if (sz != PAGEROUND(sz)(((sz) + (((1UL << 12) - 1))) & ~((1UL << 12)
 - 1)))
wrterror(d, "mapalign round");

/* Allocate sz + alignment bytes of memory, which must include a
* subrange of size bytes that is properly aligned.  Unmap the
* other bytes, and then return that subrange.
*/

/* We need sz + alignment to fit into a size_t. */
if (alignment > SIZE_MAX0xffffffffffffffffUL - sz)
return MAP_FAILED((void *)-1);

p = map(d, sz + alignment, zero_fill);
if (p == MAP_FAILED((void *)-1))
return MAP_FAILED((void *)-1);
q = (char *)(((uintptr_t)p + alignment - 1) & ~(alignment - 1));
if (q != p) {
if (munmap(p, q - p))
	wrterror(d, "munmap %p", p);
}
if (munmap(q + sz, alignment - (q - p)))
wrterror(d, "munmap %p", q + sz);
STATS_SUB(d->malloc_used, alignment)((d->malloc_used) -= (alignment));

return q;
2225}

2227static void *
2228omemalign(struct dir_info *pool, size_t alignment, size_t sz, int zero_fill)
2229{
size_t psz;
void *p, *caller = NULL((void *)0);

/* If between half a page and a page, avoid MALLOC_MOVE. */
if (sz > MALLOC_MAXCHUNK(1 << (12 - 1)) && sz < MALLOC_PAGESIZE(1UL << 12))
sz = MALLOC_PAGESIZE(1UL << 12);
if (alignment <= MALLOC_PAGESIZE(1UL << 12)) {
size_t pof2;
/*
 * max(size, alignment) rounded up to power of 2 is enough
 * to assure the requested alignment. Large regions are
 * always page aligned.
 */
if (sz < alignment)
	sz = alignment;
if (sz < MALLOC_PAGESIZE(1UL << 12)) {
	pof2 = MALLOC_MINSIZE(1UL << 4);
	while (pof2 < sz)
		pof2 <<= 1;
} else
	pof2 = sz;
return omalloc(pool, pof2, zero_fill);
}

if (sz >= SIZE_MAX0xffffffffffffffffUL - moptsmalloc_readonly.mopts.malloc_guard - MALLOC_PAGESIZE(1UL << 12)) {
errno(*__errno()) = ENOMEM12;
return NULL((void *)0);
}

if (sz < MALLOC_PAGESIZE(1UL << 12))
sz = MALLOC_PAGESIZE(1UL << 12);
sz += moptsmalloc_readonly.mopts.malloc_guard;
psz = PAGEROUND(sz)(((sz) + (((1UL << 12) - 1))) & ~((1UL << 12)
 - 1));

p = mapalign(pool, alignment, psz, zero_fill);
if (p == MAP_FAILED((void *)-1)) {
errno(*__errno()) = ENOMEM12;
return NULL((void *)0);
}

2270#ifdef MALLOC_STATS
if (DO_STATSmalloc_readonly.mopts.malloc_stats)
caller = pool->caller;
2273#endif
if (insert(pool, p, sz, caller)) {
unmap(pool, p, psz, 0);
errno(*__errno()) = ENOMEM12;
return NULL((void *)0);
}

if (moptsmalloc_readonly.mopts.malloc_guard) {
if (mprotect((char *)p + psz - moptsmalloc_readonly.mopts.malloc_guard,
    moptsmalloc_readonly.mopts.malloc_guard, PROT_NONE0x00))
	wrterror(pool, "mprotect");
STATS_ADD(pool->malloc_guarded, mopts.malloc_guard)((pool->malloc_guarded) += (malloc_readonly.mopts.malloc_guard
));
}

if (pool->malloc_junk == 2) {
if (zero_fill)
	memset((char *)p + sz - moptsmalloc_readonly.mopts.malloc_guard,
	    SOME_JUNK0xdb, psz - sz);
else
	memset(p, SOME_JUNK0xdb, psz - moptsmalloc_readonly.mopts.malloc_guard);
} else if (moptsmalloc_readonly.mopts.chunk_canaries)
fill_canary(p, sz - moptsmalloc_readonly.mopts.malloc_guard,
    psz - moptsmalloc_readonly.mopts.malloc_guard);

return p;
2298}

2300int
2301posix_memalign(void **memptr, size_t alignment, size_t size)
2302{
struct dir_info *d;
int res, saved_errno = errno(*__errno());
void *r;

/* Make sure that alignment is a large enough power of 2. */
if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *))
return EINVAL22;

d = getpool();
if (d == NULL((void *)0)) {
_malloc_init(0);
d = getpool();
}
_MALLOC_LOCK(d->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_lock(d->mutex)
; } while (0);
d->func = "posix_memalign";
if (d->active++) {
malloc_recurse(d);
goto err;
}
SET_CALLER(d, caller(d))if (malloc_readonly.mopts.malloc_stats) ((d)->caller = (caller
(d)));
r = omemalign(d, alignment, size, 0);
d->active--;
_MALLOC_UNLOCK(d->mutex)do { if (__isthreaded) _thread_cb.tc_malloc_unlock(d->mutex
); } while (0);
if (r == NULL((void *)0)) {
if (moptsmalloc_readonly.mopts.malloc_xmalloc)
	wrterror(d, "out of memory");
goto err;
}
errno(*__errno()) = saved_errno;
*memptr = r;
return 0;

2335err:
res = errno(*__errno());
errno(*__errno()) = saved_errno;
return res;
2339}
2340DEF_STRONG(posix_memalign)__asm__(".global " "posix_memalign" " ; " "posix_memalign" " = "
 "_libc_posix_memalign");

2342void *
2343aligned_alloc(size_t alignment, size_t size)
2344{
struct dir_info *d;
int saved_errno = errno(*__errno());
void *r;

/* Make sure that alignment is a positive power of 2. */
if (((alignment - 1) & alignment) != 0 || alignment == 0) {
1
Assuming the condition is false→
2
←
Assuming 'alignment' is not equal to 0→
3
←
Taking false branch→
errno(*__errno()) = EINVAL22;
return NULL((void *)0);
};
/* Per spec, size should be a multiple of alignment */
if ((size & (alignment - 1)) != 0) {
4
←
Assuming the condition is false→
5
←
Taking false branch→
errno(*__errno()) = EINVAL22;
return NULL((void *)0);
}

PROLOGUE(getpool(), "aligned_alloc")d = (getpool()); if (d == ((void *)0)) { _malloc_init(0); d =
 (getpool()); } do { if (__isthreaded) _thread_cb.tc_malloc_lock
(d->mutex); } while (0); d->func = "aligned_alloc"; if (
d->active++) { malloc_recurse(d); return ((void *)0); }
6
←
Taking true branch→
7
←
Calling '_malloc_init'→
SET_CALLER(d, caller(d))if (malloc_readonly.mopts.malloc_stats) ((d)->caller = (caller
(d)));
r = omemalign(d, alignment, size, 0);
EPILOGUE()d->active--; do { if (__isthreaded) _thread_cb.tc_malloc_unlock
(d->mutex); } while (0); if (r == ((void *)0) && malloc_readonly
.mopts.malloc_xmalloc) wrterror(d, "out of memory"); if (r !=
 ((void *)0)) (*__errno()) = saved_errno;
return r;
2365}
2366DEF_STRONG(aligned_alloc)__asm__(".global " "aligned_alloc" " ; " "aligned_alloc" " = "
 "_libc_aligned_alloc");

2368#ifdef MALLOC_STATS

2370static int
2371btcmp(const struct btnode *e1, const struct btnode *e2)
2372{
return memcmp(e1->caller, e2->caller, sizeof(e1->caller));
2374}

2376RBT_GENERATE(btshead, btnode, entry, btcmp)static int btshead_RBT_COMPARE(const void *lptr, const void *
rptr) { const struct btnode *l = lptr, *r = rptr; return btcmp
(l, r); } static const struct rb_type btshead_RBT_INFO = { btshead_RBT_COMPARE
, ((void *)0), __builtin_offsetof(struct btnode, entry), }; const
 struct rb_type *const btshead_RBT_TYPE = &btshead_RBT_INFO;

2378static void*
2379store_caller(struct dir_info *d, struct btnode *f)
2380{
struct btnode *p;

if (DO_STATSmalloc_readonly.mopts.malloc_stats == 0 || d->btnodes == MAP_FAILED((void *)-1))
return NULL((void *)0);

p = RBT_FIND(btshead, &d->btraces, f)btshead_RBT_FIND(&d->btraces, f);
if (p != NULL((void *)0))
return p;
if (d->btnodes == NULL((void *)0) ||
   d->btnodesused >= MALLOC_PAGESIZE(1UL << 12) / sizeof(struct btnode)) {
d->btnodes = map(d, MALLOC_PAGESIZE(1UL << 12), 0);
if (d->btnodes == MAP_FAILED((void *)-1))
	return NULL((void *)0);
d->btnodesused = 0;
}
p = &d->btnodes[d->btnodesused++];
memcpy(p->caller, f->caller, sizeof(p->caller[0]) * DO_STATSmalloc_readonly.mopts.malloc_stats);
RBT_INSERT(btshead, &d->btraces, p)btshead_RBT_INSERT(&d->btraces, p);
return p;
2400}

2402static void fabstorel(const void *, char *, size_t);

2404static void
2405print_chunk_details(struct dir_info *pool, void *p, size_t sz, size_t index)
2406{
struct region_info *r;
struct chunk_info *chunkinfo;
struct btnode* btnode;
uint32_t chunknum;
int frame;
char buf1[128];
char buf2[128];
const char *msg = "";

r = find(pool, p);
chunkinfo = (struct chunk_info *)r->size;
chunknum = find_chunknum(pool, chunkinfo, p, 0);
btnode = (struct btnode *)r->f[chunknum];
frame = DO_STATSmalloc_readonly.mopts.malloc_stats - 1;
if (btnode != NULL((void *)0))
fabstorel(btnode->caller[frame], buf1, sizeof(buf1));
strlcpy(buf2, ". 0x0", sizeof(buf2));
if (chunknum > 0) {
chunknum--;
btnode = (struct btnode *)r->f[chunknum];
if (btnode != NULL((void *)0))
	fabstorel(btnode->caller[frame], buf2, sizeof(buf2));
if (CHUNK_FREE(chunkinfo, chunknum)((chunkinfo)->bits[(chunknum) / (8 * sizeof(u_short))] &
 (1U << ((chunknum) % (8 * sizeof(u_short))))))
	msg = " (now free)";
}

wrterror(pool,
   "write to free chunk %p[%zu..%zu]@%zu allocated at %s "
   "(preceding chunk %p allocated at %s%s)",
   p, index * sizeof(uint64_t), (index + 1) * sizeof(uint64_t) - 1,
   sz, buf1, p - sz, buf2, msg);
2438}

2440static void
2441ulog(const char *format, ...)
2442{
va_list ap;
static char* buf;
static size_t filled;
int len;

if (buf == NULL((void *)0))
buf = MMAP(KTR_USER_MAXLEN, 0)mmap(((void *)0), (2048), 0x01 | 0x02, 0x1000 | 0x0002 | (0),
 -1, 0);
if (buf == MAP_FAILED((void *)-1))
return;

va_start(ap, format)__builtin_va_start((ap), format);
len = vsnprintf(buf + filled, KTR_USER_MAXLEN2048 - filled, format, ap);
va_end(ap)__builtin_va_end((ap));
if (len < 0)
return;
if ((size_t)len > KTR_USER_MAXLEN2048 - filled)
len = KTR_USER_MAXLEN2048 - filled;
filled += len;
if (filled > 0) {
if (filled == KTR_USER_MAXLEN2048 || buf[filled - 1] == '\n') {
	utrace("malloc", buf, filled);
	filled = 0;
}
}
2467}

2469struct malloc_leak {
void *f;
size_t total_size;
int count;
2473};

2475struct leaknode {
RBT_ENTRY(leaknode)struct rb_entry entry;
struct malloc_leak d;
2478};

2480static inline int
2481leakcmp(const struct leaknode *e1, const struct leaknode *e2)
2482{
return e1->d.f < e2->d.f ? -1 : e1->d.f > e2->d.f;
2484}

2486RBT_HEAD(leaktree, leaknode)struct leaktree { struct rb_tree rbh_root; };
2487RBT_PROTOTYPE(leaktree, leaknode, entry, leakcmp)extern const struct rb_type *const leaktree_RBT_TYPE; __attribute__
((__unused__)) static inline void leaktree_RBT_INIT(struct leaktree
 *head) { _rb_init(&head->rbh_root); } __attribute__((
__unused__)) static inline struct leaknode * leaktree_RBT_INSERT
(struct leaktree *head, struct leaknode *elm) { return _rb_insert
(leaktree_RBT_TYPE, &head->rbh_root, elm); } __attribute__
((__unused__)) static inline struct leaknode * leaktree_RBT_REMOVE
(struct leaktree *head, struct leaknode *elm) { return _rb_remove
(leaktree_RBT_TYPE, &head->rbh_root, elm); } __attribute__
((__unused__)) static inline struct leaknode * leaktree_RBT_FIND
(struct leaktree *head, const struct leaknode *key) { return _rb_find
(leaktree_RBT_TYPE, &head->rbh_root, key); } __attribute__
((__unused__)) static inline struct leaknode * leaktree_RBT_NFIND
(struct leaktree *head, const struct leaknode *key) { return _rb_nfind
(leaktree_RBT_TYPE, &head->rbh_root, key); } __attribute__
((__unused__)) static inline struct leaknode * leaktree_RBT_ROOT
(struct leaktree *head) { return _rb_root(leaktree_RBT_TYPE, &
head->rbh_root); } __attribute__((__unused__)) static inline
 int leaktree_RBT_EMPTY(struct leaktree *head) { return _rb_empty
(&head->rbh_root); } __attribute__((__unused__)) static
 inline struct leaknode * leaktree_RBT_MIN(struct leaktree *head
) { return _rb_min(leaktree_RBT_TYPE, &head->rbh_root)
; } __attribute__((__unused__)) static inline struct leaknode
 * leaktree_RBT_MAX(struct leaktree *head) { return _rb_max(leaktree_RBT_TYPE
, &head->rbh_root); } __attribute__((__unused__)) static
 inline struct leaknode * leaktree_RBT_NEXT(struct leaknode *
elm) { return _rb_next(leaktree_RBT_TYPE, elm); } __attribute__
((__unused__)) static inline struct leaknode * leaktree_RBT_PREV
(struct leaknode *elm) { return _rb_prev(leaktree_RBT_TYPE, elm
); } __attribute__((__unused__)) static inline struct leaknode
 * leaktree_RBT_LEFT(struct leaknode *elm) { return _rb_left(
leaktree_RBT_TYPE, elm); } __attribute__((__unused__)) static
 inline struct leaknode * leaktree_RBT_RIGHT(struct leaknode *
elm) { return _rb_right(leaktree_RBT_TYPE, elm); } __attribute__
((__unused__)) static inline struct leaknode * leaktree_RBT_PARENT
(struct leaknode *elm) { return _rb_parent(leaktree_RBT_TYPE,
 elm); } __attribute__((__unused__)) static inline void leaktree_RBT_SET_LEFT
(struct leaknode *elm, struct leaknode *left) { _rb_set_left(
leaktree_RBT_TYPE, elm, left); } __attribute__((__unused__)) static
 inline void leaktree_RBT_SET_RIGHT(struct leaknode *elm, struct
 leaknode *right) { _rb_set_right(leaktree_RBT_TYPE, elm, right
); } __attribute__((__unused__)) static inline void leaktree_RBT_SET_PARENT
(struct leaknode *elm, struct leaknode *parent) { _rb_set_parent
(leaktree_RBT_TYPE, elm, parent); } __attribute__((__unused__
)) static inline void leaktree_RBT_POISON(struct leaknode *elm
, unsigned long poison) { _rb_poison(leaktree_RBT_TYPE, elm, poison
); } __attribute__((__unused__)) static inline int leaktree_RBT_CHECK
(struct leaknode *elm, unsigned long poison) { return _rb_check
(leaktree_RBT_TYPE, elm, poison); };
2488RBT_GENERATE(leaktree, leaknode, entry, leakcmp)static int leaktree_RBT_COMPARE(const void *lptr, const void *
rptr) { const struct leaknode *l = lptr, *r = rptr; return leakcmp
(l, r); } static const struct rb_type leaktree_RBT_INFO = { leaktree_RBT_COMPARE
, ((void *)0), __builtin_offsetof(struct leaknode, entry), };
 const struct rb_type *const leaktree_RBT_TYPE = &leaktree_RBT_INFO;

2490static void
2491wrtwarning(const char *func, char *msg, ...)
2492{
int		saved_errno = errno(*__errno());
va_list		ap;

dprintf(STDERR_FILENO2, "%s(%d) in %s(): ", __progname,
   getpid(), func != NULL((void *)0) ? func : "unknown");
va_start(ap, msg)__builtin_va_start((ap), msg);
vdprintf(STDERR_FILENO2, msg, ap);
va_end(ap)__builtin_va_end((ap));
dprintf(STDERR_FILENO2, "\n");

errno(*__errno()) = saved_errno;
2504}

2506static void
2507putleakinfo(struct leaktree *leaks, void *f, size_t sz, int cnt)
2508{
struct leaknode key, *p;
static struct leaknode *page;
static unsigned int used;

if (cnt == 0 || page == MAP_FAILED((void *)-1))
return;

key.d.f = f;
p = RBT_FIND(leaktree, leaks, &key)leaktree_RBT_FIND(leaks, &key);
if (p == NULL((void *)0)) {
if (page == NULL((void *)0) ||
    used >= MALLOC_PAGESIZE(1UL << 12) / sizeof(struct leaknode)) {
	page = MMAP(MALLOC_PAGESIZE, 0)mmap(((void *)0), ((1UL << 12)), 0x01 | 0x02, 0x1000 | 0x0002
 | (0), -1, 0);
	if (page == MAP_FAILED((void *)-1)) {
		wrtwarning(__func__, strerror(errno(*__errno())));
		return;
	}
	used = 0;
}
p = &page[used++];
p->d.f = f;
p->d.total_size = sz * cnt;
p->d.count = cnt;
RBT_INSERT(leaktree, leaks, p)leaktree_RBT_INSERT(leaks, p);
} else {
p->d.total_size += sz * cnt;
p->d.count += cnt;
}
2537}

2539static void
2540fabstorel(const void *f, char *buf, size_t size)
2541{
Dl_info info;
const char *object = ".";
const char *caller;

caller = f;
if (caller != NULL((void *)0) && dladdr(f, &info) != 0) {
caller -= (uintptr_t)info.dli_fbase;
object = info.dli_fname;
}
snprintf(buf, size, "%s %p", object, caller);
2552}

2554static void
2555dump_leak(struct leaknode *p)
2556{
int i;
char buf[128];

if (p->d.f == NULL((void *)0)) {
fabstorel(NULL((void *)0), buf, sizeof(buf));
ulog("%18p %7zu %6u %6zu addr2line -e %s\n",
    p->d.f, p->d.total_size, p->d.count,
    p->d.total_size / p->d.count, buf);
return;
}

for (i = 0; i < DO_STATSmalloc_readonly.mopts.malloc_stats; i++) {
const char *abscaller;

abscaller = ((struct btnode*)p->d.f)->caller[i];
if (abscaller == NULL((void *)0))
	break;
fabstorel(abscaller, buf, sizeof(buf));
if (i == 0)
	ulog("%18p %7zu %6u %6zu addr2line -e %s\n",
	    abscaller, p->d.total_size, p->d.count,
	    p->d.total_size / p->d.count, buf);
else
	ulog("%*p %*s %6s %6s addr2line -e %s\n",
	    i + 18, abscaller, 7 - i, "-", "-", "-", buf);
}
2583}

2585static void
2586dump_leaks(struct leaktree *leaks)
2587{
struct leaknode *p;

ulog("Leak report:\n");
ulog("                 f     sum      #    avg\n");

RBT_FOREACH(p, leaktree, leaks)for ((p) = leaktree_RBT_MIN((leaks)); (p) != ((void *)0); (p)
 = leaktree_RBT_NEXT((p))) 
dump_leak(p);
2595}

2597static void
2598dump_chunk(struct leaktree* leaks, struct chunk_info *p, void **f,
  int fromfreelist)
2600{
while (p != NULL((void *)0)) {
if (moptsmalloc_readonly.mopts.malloc_verbose)
	ulog("chunk %18p %18p %4zu %d/%d\n",
	    p->page, NULL((void *)0),
	    B2SIZE(p->bucket)((p->bucket) * (1UL << 4)), p->free, p->total);
if (!fromfreelist) {
	size_t i, sz =  B2SIZE(p->bucket)((p->bucket) * (1UL << 4));
	for (i = 0; i < p->total; i++) {
		if (!CHUNK_FREE(p, i)((p)->bits[(i) / (8 * sizeof(u_short))] & (1U <<
 ((i) % (8 * sizeof(u_short))))))
			putleakinfo(leaks, f[i], sz, 1);
	}
	break;
}
p = LIST_NEXT(p, entries)((p)->entries.le_next);
if (moptsmalloc_readonly.mopts.malloc_verbose && p != NULL((void *)0))
	ulog("       ->");
}
2618}

2620static void
2621dump_free_chunk_info(struct dir_info *d, struct leaktree *leaks)
2622{
u_int i, j, count;
struct chunk_info *p;

ulog("Free chunk structs:\n");
ulog("Bkt) #CI                     page"
   "                  f size free/n\n");
for (i = 0; i <= BUCKETS((1 << (12 - 1)) / (1UL << 4)); i++) {
count = 0;
LIST_FOREACH(p, &d->chunk_info_list[i], entries)for((p) = ((&d->chunk_info_list[i])->lh_first); (p)
!= ((void *)0); (p) = ((p)->entries.le_next))
	count++;
for (j = 0; j < MALLOC_CHUNK_LISTS4; j++) {
	p = LIST_FIRST(&d->chunk_dir[i][j])((&d->chunk_dir[i][j])->lh_first);
	if (p == NULL((void *)0) && count == 0)
		continue;
	if (j == 0)
		ulog("%3d) %3d ", i, count);
	else
		ulog("         ");
	if (p != NULL((void *)0))
		dump_chunk(leaks, p, NULL((void *)0), 1);
	else
		ulog(".\n");
}
}

2648}

2650static void
2651dump_free_page_info(struct dir_info *d)
2652{
struct smallcache *cache;
size_t i, total = 0;

ulog("Cached in small cache:\n");
for (i = 0; i < MAX_SMALLCACHEABLE_SIZE32; i++) {
cache = &d->smallcache[i];
if (cache->length != 0)
	ulog("%zu(%u): %u = %zu\n", i + 1, cache->max,
	    cache->length, cache->length * (i + 1));
total += cache->length * (i + 1);
}

ulog("Cached in big cache: %zu/%zu\n", d->bigcache_used,
   d->bigcache_size);
for (i = 0; i < d->bigcache_size; i++) {
if (d->bigcache[i].psize != 0)
	ulog("%zu: %zu\n", i, d->bigcache[i].psize);
total += d->bigcache[i].psize;
}
ulog("Free pages cached: %zu\n", total);
2673}

2675static void
2676malloc_dump1(int poolno, struct dir_info *d, struct leaktree *leaks)
2677{
size_t i, realsize;

if (moptsmalloc_readonly.mopts.malloc_verbose) {
ulog("Malloc dir of %s pool %d at %p\n", __progname, poolno, d);
ulog("MT=%d J=%d Fl=%#x\n", d->malloc_mt, d->malloc_junk,
    d->mmap_flag);
ulog("Region slots free %zu/%zu\n",
	d->regions_free, d->regions_total);
ulog("Inserts %zu/%zu\n", d->inserts, d->insert_collisions);
ulog("Deletes %zu/%zu\n", d->deletes, d->delete_moves);
ulog("Cheap reallocs %zu/%zu\n",
    d->cheap_reallocs, d->cheap_realloc_tries);
ulog("In use %zu\n", d->malloc_used);
ulog("Guarded %zu\n", d->malloc_guarded);
dump_free_chunk_info(d, leaks);
dump_free_page_info(d);
ulog("Hash table:\n");
ulog("slot)  hash d  type               page                  "
    "f size [free/n]\n");
}
for (i = 0; i < d->regions_total; i++) {
if (d->r[i].p != NULL((void *)0)) {
	size_t h = hash(d->r[i].p) &
	    (d->regions_total - 1);
	if (moptsmalloc_readonly.mopts.malloc_verbose)
		ulog("%4zx) #%4zx %zd ",
	        i, h, h - i);
	REALSIZE(realsize, &d->r[i])(realsize) = (uintptr_t)(&d->r[i])->p & ((1UL <<
 12) - 1), (realsize) = ((realsize) == 0 ? (&d->r[i])->
size : (((realsize) - 1) * (1UL << 4)));
	if (realsize > MALLOC_MAXCHUNK(1 << (12 - 1))) {
		putleakinfo(leaks, d->r[i].f, realsize, 1);
		if (moptsmalloc_readonly.mopts.malloc_verbose)
			ulog("pages %18p %18p %zu\n", d->r[i].p,
		        d->r[i].f, realsize);
	} else
		dump_chunk(leaks,
		    (struct chunk_info *)d->r[i].size,
		    d->r[i].f, 0);
}
}
if (moptsmalloc_readonly.mopts.malloc_verbose)
ulog("\n");
2719}

2721static void
2722malloc_dump0(int poolno, struct dir_info *pool, struct leaktree *leaks)
2723{
int i;
void *p;
struct region_info *r;

if (pool == NULL((void *)0) || pool->r == NULL((void *)0))
return;
for (i = 0; i < MALLOC_DELAYED_CHUNK_MASK15 + 1; i++) {
p = pool->delayed_chunks[i];
if (p == NULL((void *)0))
	continue;
r = find(pool, p);
if (r == NULL((void *)0))
	wrterror(pool, "bogus pointer in malloc_dump %p", p);
free_bytes(pool, r, p);
pool->delayed_chunks[i] = NULL((void *)0);
}
malloc_dump1(poolno, pool, leaks);
2741}

2743void
2744malloc_dump(void)
2745{
u_int i;
int saved_errno = errno(*__errno());

/* XXX leak when run multiple times */
struct leaktree leaks = RBT_INITIALIZER(&leaks){ { ((void *)0) } };

for (i = 0; i < moptsmalloc_readonly.mopts.malloc_mutexes; i++)
malloc_dump0(i, moptsmalloc_readonly.mopts.malloc_pool[i], &leaks);

dump_leaks(&leaks);
ulog("\n");
errno(*__errno()) = saved_errno;
2758}
2759DEF_WEAK(malloc_dump)__asm__(".weak " "malloc_dump" " ; " "malloc_dump" " = " "_libc_malloc_dump"
);

2761static void
2762malloc_exit(void)
2763{
int save_errno = errno(*__errno());

ulog("******** Start dump %s *******\n", __progname);
ulog("M=%u I=%d F=%d U=%d J=%d R=%d X=%d C=%#x cache=%u "
   "G=%zu\n",
   moptsmalloc_readonly.mopts.malloc_mutexes,
   moptsmalloc_readonly.mopts.internal_funcs, moptsmalloc_readonly.mopts.malloc_freecheck,
   moptsmalloc_readonly.mopts.malloc_freeunmap, moptsmalloc_readonly.mopts.def_malloc_junk,
   moptsmalloc_readonly.mopts.malloc_realloc, moptsmalloc_readonly.mopts.malloc_xmalloc,
   moptsmalloc_readonly.mopts.chunk_canaries, moptsmalloc_readonly.mopts.def_maxcache,
   moptsmalloc_readonly.mopts.malloc_guard);

malloc_dump();
ulog("******** End dump %s *******\n", __progname);
errno(*__errno()) = save_errno;
2779}

2781#endif /* MALLOC_STATS */