prof_data.c

#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"

#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/ckh.h"
#include "jemalloc/internal/hash.h"
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/prof_data.h"

/*
 * This file defines and manages the core profiling data structures.
 *
 * Conceptually, profiling data can be imagined as a table with three columns:
 * thread, stack trace, and current allocation size.  (When prof_accum is on,
 * there's one additional column which is the cumulative allocation size.)
 *
 * Implementation wise, each thread maintains a hash recording the stack trace
 * to allocation size correspondences, which are basically the individual rows
 * in the table.  In addition, two global "indices" are built to make data
 * aggregation efficient (for dumping): bt2gctx and tdatas, which are basically
 * the "grouped by stack trace" and "grouped by thread" views of the same table,
 * respectively.  Note that the allocation size is only aggregated to the two
 * indices at dumping time, so as to optimize for performance.
 */

/******************************************************************************/

malloc_mutex_t bt2gctx_mtx;
malloc_mutex_t tdatas_mtx;
malloc_mutex_t prof_dump_mtx;

/*
 * Table of mutexes that are shared among gctx's.  These are leaf locks, so
 * there is no problem with using them for more than one gctx at the same time.
 * The primary motivation for this sharing though is that gctx's are ephemeral,
 * and destroying mutexes causes complications for systems that allocate when
 * creating/destroying mutexes.
 */
malloc_mutex_t *gctx_locks;
static atomic_u_t cum_gctxs; /* Atomic counter. */

/*
 * Table of mutexes that are shared among tdata's.  No operations require
 * holding multiple tdata locks, so there is no problem with using them for more
 * than one tdata at the same time, even though a gctx lock may be acquired
 * while holding a tdata lock.
 */
malloc_mutex_t *tdata_locks;

/*
 * Global hash of (prof_bt_t *)-->(prof_gctx_t *).  This is the master data
 * structure that knows about all backtraces currently captured.
 */
static ckh_t bt2gctx;

/*
 * Tree of all extant prof_tdata_t structures, regardless of state,
 * {attached,detached,expired}.
 */
static prof_tdata_tree_t tdatas;

size_t prof_unbiased_sz[PROF_SC_NSIZES];
size_t prof_shifted_unbiased_cnt[PROF_SC_NSIZES];

/******************************************************************************/
/* Red-black trees. */

static int
prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) {
	uint64_t a_thr_uid = a->thr_uid;
	uint64_t b_thr_uid = b->thr_uid;
	int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid);
	if (ret == 0) {
		uint64_t a_thr_discrim = a->thr_discrim;
		uint64_t b_thr_discrim = b->thr_discrim;
		ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim <
		    b_thr_discrim);
		if (ret == 0) {
			uint64_t a_tctx_uid = a->tctx_uid;
			uint64_t b_tctx_uid = b->tctx_uid;
			ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid <
			    b_tctx_uid);
		}
	}
	return ret;
}

rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t,
    tctx_link, prof_tctx_comp)

static int
prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) {
	unsigned a_len = a->bt.len;
	unsigned b_len = b->bt.len;
	unsigned comp_len = (a_len < b_len) ? a_len : b_len;
	int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *));
	if (ret == 0) {
		ret = (a_len > b_len) - (a_len < b_len);
	}
	return ret;
}

rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link,
    prof_gctx_comp)

static int
prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) {
	int ret;
	uint64_t a_uid = a->thr_uid;
	uint64_t b_uid = b->thr_uid;

	ret = ((a_uid > b_uid) - (a_uid < b_uid));
	if (ret == 0) {
		uint64_t a_discrim = a->thr_discrim;
		uint64_t b_discrim = b->thr_discrim;

		ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim));
	}
	return ret;
}

rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
    prof_tdata_comp)

/******************************************************************************/

static malloc_mutex_t *
prof_gctx_mutex_choose(void) {
	unsigned ngctxs = atomic_fetch_add_u(&cum_gctxs, 1, ATOMIC_RELAXED);

	return &gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS];
}

static malloc_mutex_t *
prof_tdata_mutex_choose(uint64_t thr_uid) {
	return &tdata_locks[thr_uid % PROF_NTDATA_LOCKS];
}

bool
prof_data_init(tsd_t *tsd) {
	tdata_tree_new(&tdatas);
	return ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS,
	    prof_bt_hash, prof_bt_keycomp);
}

static void
prof_enter(tsd_t *tsd, prof_tdata_t *tdata) {
	cassert(config_prof);
	assert(tdata == prof_tdata_get(tsd, false));

	if (tdata != NULL) {
		assert(!tdata->enq);
		tdata->enq = true;
	}

	malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
}

static void
prof_leave(tsd_t *tsd, prof_tdata_t *tdata) {
	cassert(config_prof);
	assert(tdata == prof_tdata_get(tsd, false));

	malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);

	if (tdata != NULL) {
		bool idump, gdump;

		assert(tdata->enq);
		tdata->enq = false;
		idump = tdata->enq_idump;
		tdata->enq_idump = false;
		gdump = tdata->enq_gdump;
		tdata->enq_gdump = false;

		if (idump) {
			prof_idump(tsd_tsdn(tsd));
		}
		if (gdump) {
			prof_gdump(tsd_tsdn(tsd));
		}
	}
}

static prof_gctx_t *
prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) {
	/*
	 * Create a single allocation that has space for vec of length bt->len.
	 */
	size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *));
	prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size,
	    sz_size2index(size), false, NULL, true, arena_get(TSDN_NULL, 0, true),
	    true);
	if (gctx == NULL) {
		return NULL;
	}
	gctx->lock = prof_gctx_mutex_choose();
	/*
	 * Set nlimbo to 1, in order to avoid a race condition with
	 * prof_tctx_destroy()/prof_gctx_try_destroy().
	 */
	gctx->nlimbo = 1;
	tctx_tree_new(&gctx->tctxs);
	/* Duplicate bt. */
	memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *));
	gctx->bt.vec = gctx->vec;
	gctx->bt.len = bt->len;
	return gctx;
}

static void
prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self,
    prof_gctx_t *gctx) {
	cassert(config_prof);

	/*
	 * Check that gctx is still unused by any thread cache before destroying
	 * it.  prof_lookup() increments gctx->nlimbo in order to avoid a race
	 * condition with this function, as does prof_tctx_destroy() in order to
	 * avoid a race between the main body of prof_tctx_destroy() and entry
	 * into this function.
	 */
	prof_enter(tsd, tdata_self);
	malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
	assert(gctx->nlimbo != 0);
	if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
		/* Remove gctx from bt2gctx. */
		if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) {
			not_reached();
		}
		prof_leave(tsd, tdata_self);
		/* Destroy gctx. */
		malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
		idalloctm(tsd_tsdn(tsd), gctx, NULL, NULL, true, true);
	} else {
		/*
		 * Compensate for increment in prof_tctx_destroy() or
		 * prof_lookup().
		 */
		gctx->nlimbo--;
		malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
		prof_leave(tsd, tdata_self);
	}
}

static bool
prof_gctx_should_destroy(prof_gctx_t *gctx) {
	if (opt_prof_accum) {
		return false;
	}
	if (!tctx_tree_empty(&gctx->tctxs)) {
		return false;
	}
	if (gctx->nlimbo != 0) {
		return false;
	}
	return true;
}

static bool
prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
    void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) {
	union {
		prof_gctx_t	*p;
		void		*v;
	} gctx, tgctx;
	union {
		prof_bt_t	*p;
		void		*v;
	} btkey;
	bool new_gctx;

	prof_enter(tsd, tdata);
	if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
		/* bt has never been seen before.  Insert it. */
		prof_leave(tsd, tdata);
		tgctx.p = prof_gctx_create(tsd_tsdn(tsd), bt);
		if (tgctx.v == NULL) {
			return true;
		}
		prof_enter(tsd, tdata);
		if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
			gctx.p = tgctx.p;
			btkey.p = &gctx.p->bt;
			if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
				/* OOM. */
				prof_leave(tsd, tdata);
				idalloctm(tsd_tsdn(tsd), gctx.v, NULL, NULL,
				    true, true);
				return true;
			}
			new_gctx = true;
		} else {
			new_gctx = false;
		}
	} else {
		tgctx.v = NULL;
		new_gctx = false;
	}

	if (!new_gctx) {
		/*
		 * Increment nlimbo, in order to avoid a race condition with
		 * prof_tctx_destroy()/prof_gctx_try_destroy().
		 */
		malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock);
		gctx.p->nlimbo++;
		malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock);
		new_gctx = false;

		if (tgctx.v != NULL) {
			/* Lost race to insert. */
			idalloctm(tsd_tsdn(tsd), tgctx.v, NULL, NULL, true,
			    true);
		}
	}
	prof_leave(tsd, tdata);

	*p_btkey = btkey.v;
	*p_gctx = gctx.p;
	*p_new_gctx = new_gctx;
	return false;
}

prof_tctx_t *
prof_lookup(tsd_t *tsd, prof_bt_t *bt) {
	union {
		prof_tctx_t	*p;
		void		*v;
	} ret;
	prof_tdata_t *tdata;
	bool not_found;

	cassert(config_prof);

	tdata = prof_tdata_get(tsd, false);
	assert(tdata != NULL);

	malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
	not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);
	if (!not_found) { /* Note double negative! */
		ret.p->prepared = true;
	}
	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
	if (not_found) {
		void *btkey;
		prof_gctx_t *gctx;
		bool new_gctx, error;

		/*
		 * This thread's cache lacks bt.  Look for it in the global
		 * cache.
		 */
		if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
		    &new_gctx)) {
			return NULL;
		}

		/* Link a prof_tctx_t into gctx for this thread. */
		ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t),
		    sz_size2index(sizeof(prof_tctx_t)), false, NULL, true,
		    arena_ichoose(tsd, NULL), true);
		if (ret.p == NULL) {
			if (new_gctx) {
				prof_gctx_try_destroy(tsd, tdata, gctx);
			}
			return NULL;
		}
		ret.p->tdata = tdata;
		ret.p->thr_uid = tdata->thr_uid;
		ret.p->thr_discrim = tdata->thr_discrim;
		ret.p->recent_count = 0;
		memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
		ret.p->gctx = gctx;
		ret.p->tctx_uid = tdata->tctx_uid_next++;
		ret.p->prepared = true;
		ret.p->state = prof_tctx_state_initializing;
		malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
		error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
		malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
		if (error) {
			if (new_gctx) {
				prof_gctx_try_destroy(tsd, tdata, gctx);
			}
			idalloctm(tsd_tsdn(tsd), ret.v, NULL, NULL, true, true);
			return NULL;
		}
		malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
		ret.p->state = prof_tctx_state_nominal;
		tctx_tree_insert(&gctx->tctxs, ret.p);
		gctx->nlimbo--;
		malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
	}

	return ret.p;
}

/* Used in unit tests. */
static prof_tdata_t *
prof_tdata_count_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata,
    void *arg) {
	size_t *tdata_count = (size_t *)arg;

	(*tdata_count)++;

	return NULL;
}

/* Used in unit tests. */
size_t
prof_tdata_count(void) {
	size_t tdata_count = 0;
	tsdn_t *tsdn;

	tsdn = tsdn_fetch();
	malloc_mutex_lock(tsdn, &tdatas_mtx);
	tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter,
	    (void *)&tdata_count);
	malloc_mutex_unlock(tsdn, &tdatas_mtx);

	return tdata_count;
}

/* Used in unit tests. */
size_t
prof_bt_count(void) {
	size_t bt_count;
	tsd_t *tsd;
	prof_tdata_t *tdata;

	tsd = tsd_fetch();
	tdata = prof_tdata_get(tsd, false);
	if (tdata == NULL) {
		return 0;
	}

	malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
	bt_count = ckh_count(&bt2gctx);
	malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);

	return bt_count;
}

char *
prof_thread_name_alloc(tsd_t *tsd, const char *thread_name) {
	char *ret;
	size_t size;

	if (thread_name == NULL) {
		return NULL;
	}

	size = strlen(thread_name) + 1;
	if (size == 1) {
		return "";
	}

	ret = iallocztm(tsd_tsdn(tsd), size, sz_size2index(size), false, NULL,
	    true, arena_get(TSDN_NULL, 0, true), true);
	if (ret == NULL) {
		return NULL;
	}
	memcpy(ret, thread_name, size);
	return ret;
}

int
prof_thread_name_set_impl(tsd_t *tsd, const char *thread_name) {
	assert(tsd_reentrancy_level_get(tsd) == 0);

	prof_tdata_t *tdata;
	unsigned i;
	char *s;

	tdata = prof_tdata_get(tsd, true);
	if (tdata == NULL) {
		return EAGAIN;
	}

	/* Validate input. */
	if (thread_name == NULL) {
		return EFAULT;
	}
	for (i = 0; thread_name[i] != '\0'; i++) {
		char c = thread_name[i];
		if (!isgraph(c) && !isblank(c)) {
			return EFAULT;
		}
	}

	s = prof_thread_name_alloc(tsd, thread_name);
	if (s == NULL) {
		return EAGAIN;
	}

	if (tdata->thread_name != NULL) {
		idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,
		    true);
		tdata->thread_name = NULL;
	}
	if (strlen(s) > 0) {
		tdata->thread_name = s;
	}
	return 0;
}

JEMALLOC_FORMAT_PRINTF(3, 4)
static void
prof_dump_printf(write_cb_t *prof_dump_write, void *cbopaque,
    const char *format, ...) {
	va_list ap;
	char buf[PROF_PRINTF_BUFSIZE];

	va_start(ap, format);
	malloc_vsnprintf(buf, sizeof(buf), format, ap);
	va_end(ap);
	prof_dump_write(cbopaque, buf);
}

/*
 * Casting a double to a uint64_t may not necessarily be in range; this can be
 * UB.  I don't think this is practically possible with the cur counters, but
 * plausibly could be with the accum counters.
 */
#ifdef JEMALLOC_PROF
static uint64_t
prof_double_uint64_cast(double d) {
	/*
	 * Note: UINT64_MAX + 1 is exactly representable as a double on all
	 * reasonable platforms (certainly those we'll support).  Writing this
	 * as !(a < b) instead of (a >= b) means that we're NaN-safe.
	 */
	double rounded = round(d);
	if (!(rounded < (double)UINT64_MAX)) {
		return UINT64_MAX;
	}
	return (uint64_t)rounded;
}
#endif

void prof_unbias_map_init() {
	/* See the comment in prof_sample_new_event_wait */
#ifdef JEMALLOC_PROF
	for (szind_t i = 0; i < SC_NSIZES; i++) {
		double sz = (double)sz_index2size(i);
		double rate = (double)(ZU(1) << lg_prof_sample);
		double div_val = 1.0 - exp(-sz / rate);
		double unbiased_sz = sz / div_val;
		/*
		 * The "true" right value for the unbiased count is
		 * 1.0/(1 - exp(-sz/rate)).  The problem is, we keep the counts
		 * as integers (for a variety of reasons -- rounding errors
		 * could trigger asserts, and not all libcs can properly handle
		 * floating point arithmetic during malloc calls inside libc).
		 * Rounding to an integer, though, can lead to rounding errors
		 * of over 30% for sizes close to the sampling rate.  So
		 * instead, we multiply by a constant, dividing the maximum
		 * possible roundoff error by that constant.  To avoid overflow
		 * in summing up size_t values, the largest safe constant we can
		 * pick is the size of the smallest allocation.
		 */
		double cnt_shift = (double)(ZU(1) << SC_LG_TINY_MIN);
		double shifted_unbiased_cnt = cnt_shift / div_val;
		prof_unbiased_sz[i] = (size_t)round(unbiased_sz);
		prof_shifted_unbiased_cnt[i] = (size_t)round(
		    shifted_unbiased_cnt);
	}
#else
	unreachable();
#endif
}

/*
 * The unbiasing story is long.  The jeprof unbiasing logic was copied from
 * pprof.  Both shared an issue: they unbiased using the average size of the
 * allocations at a particular stack trace.  This can work out OK if allocations
 * are mostly of the same size given some stack, but not otherwise.  We now
 * internally track what the unbiased results ought to be.  We can't just report
 * them as they are though; they'll still go through the jeprof unbiasing
 * process.  Instead, we figure out what values we can feed *into* jeprof's
 * unbiasing mechanism that will lead to getting the right values out.
 *
 * It'll unbias count and aggregate size as:
 *
 *   c_out = c_in * 1/(1-exp(-s_in/c_in/R)
 *   s_out = s_in * 1/(1-exp(-s_in/c_in/R)
 *
 * We want to solve for the values of c_in and s_in that will
 * give the c_out and s_out that we've computed internally.
 *
 * Let's do a change of variables (both to make the math easier and to make it
 * easier to write):
 *   x = s_in / c_in
 *   y = s_in
 *   k = 1/R.
 *
 * Then
 *   c_out = y/x * 1/(1-exp(-k*x))
 *   s_out = y * 1/(1-exp(-k*x))
 *
 * The first equation gives:
 *   y = x * c_out * (1-exp(-k*x))
 * The second gives:
 *   y = s_out * (1-exp(-k*x))
 * So we have
 *   x = s_out / c_out.
 * And all the other values fall out from that.
 *
 * This is all a fair bit of work.  The thing we get out of it is that we don't
 * break backwards compatibility with jeprof (and the various tools that have
 * copied its unbiasing logic).  Eventually, we anticipate a v3 heap profile
 * dump format based on JSON, at which point I think much of this logic can get
 * cleaned up (since we'll be taking a compatibility break there anyways).
 */
static void
prof_do_unbias(uint64_t c_out_shifted_i, uint64_t s_out_i, uint64_t *r_c_in,
    uint64_t *r_s_in) {
#ifdef JEMALLOC_PROF
	if (c_out_shifted_i == 0 || s_out_i == 0) {
		*r_c_in = 0;
		*r_s_in = 0;
		return;
	}
	/*
	 * See the note in prof_unbias_map_init() to see why we take c_out in a
	 * shifted form.
	 */
	double c_out = (double)c_out_shifted_i
	    / (double)(ZU(1) << SC_LG_TINY_MIN);
	double s_out = (double)s_out_i;
	double R = (double)(ZU(1) << lg_prof_sample);

	double x = s_out / c_out;
	double y = s_out * (1.0 - exp(-x / R));

	double c_in = y / x;
	double s_in = y;

	*r_c_in = prof_double_uint64_cast(c_in);
	*r_s_in = prof_double_uint64_cast(s_in);
#else
	unreachable();
#endif
}

static void
prof_dump_print_cnts(write_cb_t *prof_dump_write, void *cbopaque,
    const prof_cnt_t *cnts) {
	uint64_t curobjs;
	uint64_t curbytes;
	uint64_t accumobjs;
	uint64_t accumbytes;
	if (opt_prof_unbias) {
		prof_do_unbias(cnts->curobjs_shifted_unbiased,
		    cnts->curbytes_unbiased, &curobjs, &curbytes);
		prof_do_unbias(cnts->accumobjs_shifted_unbiased,
		    cnts->accumbytes_unbiased, &accumobjs, &accumbytes);
	} else {
		curobjs = cnts->curobjs;
		curbytes = cnts->curbytes;
		accumobjs = cnts->accumobjs;
		accumbytes = cnts->accumbytes;
	}
	prof_dump_printf(prof_dump_write, cbopaque,
	    "%"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]",
	    curobjs, curbytes, accumobjs, accumbytes);
}

static void
prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) {
	malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);

	malloc_mutex_lock(tsdn, tctx->gctx->lock);

	switch (tctx->state) {
	case prof_tctx_state_initializing:
		malloc_mutex_unlock(tsdn, tctx->gctx->lock);
		return;
	case prof_tctx_state_nominal:
		tctx->state = prof_tctx_state_dumping;
		malloc_mutex_unlock(tsdn, tctx->gctx->lock);

		memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));

		tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
		tdata->cnt_summed.curobjs_shifted_unbiased
		    += tctx->dump_cnts.curobjs_shifted_unbiased;
		tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
		tdata->cnt_summed.curbytes_unbiased
		    += tctx->dump_cnts.curbytes_unbiased;
		if (opt_prof_accum) {
			tdata->cnt_summed.accumobjs +=
			    tctx->dump_cnts.accumobjs;
			tdata->cnt_summed.accumobjs_shifted_unbiased +=
			    tctx->dump_cnts.accumobjs_shifted_unbiased;
			tdata->cnt_summed.accumbytes +=
			    tctx->dump_cnts.accumbytes;
			tdata->cnt_summed.accumbytes_unbiased +=
			    tctx->dump_cnts.accumbytes_unbiased;
		}
		break;
	case prof_tctx_state_dumping:
	case prof_tctx_state_purgatory:
		not_reached();
	}
}

static void
prof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx) {
	malloc_mutex_assert_owner(tsdn, gctx->lock);

	gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
	gctx->cnt_summed.curobjs_shifted_unbiased
	    += tctx->dump_cnts.curobjs_shifted_unbiased;
	gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
	gctx->cnt_summed.curbytes_unbiased += tctx->dump_cnts.curbytes_unbiased;
	if (opt_prof_accum) {
		gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;
		gctx->cnt_summed.accumobjs_shifted_unbiased
		    += tctx->dump_cnts.accumobjs_shifted_unbiased;
		gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;
		gctx->cnt_summed.accumbytes_unbiased
		    += tctx->dump_cnts.accumbytes_unbiased;
	}
}

static prof_tctx_t *
prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
	tsdn_t *tsdn = (tsdn_t *)arg;

	malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);

	switch (tctx->state) {
	case prof_tctx_state_nominal:
		/* New since dumping started; ignore. */
		break;
	case prof_tctx_state_dumping:
	case prof_tctx_state_purgatory:
		prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx);
		break;
	default:
		not_reached();
	}

	return NULL;
}

typedef struct prof_dump_iter_arg_s prof_dump_iter_arg_t;
struct prof_dump_iter_arg_s {
	tsdn_t *tsdn;
	write_cb_t *prof_dump_write;
	void *cbopaque;
};

static prof_tctx_t *
prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) {
	prof_dump_iter_arg_t *arg = (prof_dump_iter_arg_t *)opaque;
	malloc_mutex_assert_owner(arg->tsdn, tctx->gctx->lock);

	switch (tctx->state) {
	case prof_tctx_state_initializing:
	case prof_tctx_state_nominal:
		/* Not captured by this dump. */
		break;
	case prof_tctx_state_dumping:
	case prof_tctx_state_purgatory:
		prof_dump_printf(arg->prof_dump_write, arg->cbopaque,
		    "  t%"FMTu64": ", tctx->thr_uid);
		prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque,
		    &tctx->dump_cnts);
		arg->prof_dump_write(arg->cbopaque, "\n");
		break;
	default:
		not_reached();
	}
	return NULL;
}

static prof_tctx_t *
prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
	tsdn_t *tsdn = (tsdn_t *)arg;
	prof_tctx_t *ret;

	malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);

	switch (tctx->state) {
	case prof_tctx_state_nominal:
		/* New since dumping started; ignore. */
		break;
	case prof_tctx_state_dumping:
		tctx->state = prof_tctx_state_nominal;
		break;
	case prof_tctx_state_purgatory:
		ret = tctx;
		goto label_return;
	default:
		not_reached();
	}

	ret = NULL;
label_return:
	return ret;
}

static void
prof_dump_gctx_prep(tsdn_t *tsdn, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) {
	cassert(config_prof);

	malloc_mutex_lock(tsdn, gctx->lock);

	/*
	 * Increment nlimbo so that gctx won't go away before dump.
	 * Additionally, link gctx into the dump list so that it is included in
	 * prof_dump()'s second pass.
	 */
	gctx->nlimbo++;
	gctx_tree_insert(gctxs, gctx);

	memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));

	malloc_mutex_unlock(tsdn, gctx->lock);
}

typedef struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg_t;
struct prof_gctx_merge_iter_arg_s {
	tsdn_t *tsdn;
	size_t *leak_ngctx;
};

static prof_gctx_t *
prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
	prof_gctx_merge_iter_arg_t *arg = (prof_gctx_merge_iter_arg_t *)opaque;

	malloc_mutex_lock(arg->tsdn, gctx->lock);
	tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter,
	    (void *)arg->tsdn);
	if (gctx->cnt_summed.curobjs != 0) {
		(*arg->leak_ngctx)++;
	}
	malloc_mutex_unlock(arg->tsdn, gctx->lock);

	return NULL;
}

static void
prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) {
	prof_tdata_t *tdata = prof_tdata_get(tsd, false);
	prof_gctx_t *gctx;

	/*
	 * Standard tree iteration won't work here, because as soon as we
	 * decrement gctx->nlimbo and unlock gctx, another thread can
	 * concurrently destroy it, which will corrupt the tree.  Therefore,
	 * tear down the tree one node at a time during iteration.
	 */
	while ((gctx = gctx_tree_first(gctxs)) != NULL) {
		gctx_tree_remove(gctxs, gctx);
		malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
		{
			prof_tctx_t *next;

			next = NULL;
			do {
				prof_tctx_t *to_destroy =
				    tctx_tree_iter(&gctx->tctxs, next,
				    prof_tctx_finish_iter,
				    (void *)tsd_tsdn(tsd));
				if (to_destroy != NULL) {
					next = tctx_tree_next(&gctx->tctxs,
					    to_destroy);
					tctx_tree_remove(&gctx->tctxs,
					    to_destroy);
					idalloctm(tsd_tsdn(tsd), to_destroy,
					    NULL, NULL, true, true);
				} else {
					next = NULL;
				}
			} while (next != NULL);
		}
		gctx->nlimbo--;
		if (prof_gctx_should_destroy(gctx)) {
			gctx->nlimbo++;
			malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
			prof_gctx_try_destroy(tsd, tdata, gctx);
		} else {
			malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
		}
	}
}

typedef struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg_t;
struct prof_tdata_merge_iter_arg_s {
	tsdn_t *tsdn;
	prof_cnt_t *cnt_all;
};

static prof_tdata_t *
prof_tdata_merge_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata,
    void *opaque) {
	prof_tdata_merge_iter_arg_t *arg =
	    (prof_tdata_merge_iter_arg_t *)opaque;

	malloc_mutex_lock(arg->tsdn, tdata->lock);
	if (!tdata->expired) {
		size_t tabind;
		union {
			prof_tctx_t	*p;
			void		*v;
		} tctx;

		tdata->dumping = true;
		memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));
		for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,
		    &tctx.v);) {
			prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata);
		}

		arg->cnt_all->curobjs += tdata->cnt_summed.curobjs;
		arg->cnt_all->curobjs_shifted_unbiased
		    += tdata->cnt_summed.curobjs_shifted_unbiased;
		arg->cnt_all->curbytes += tdata->cnt_summed.curbytes;
		arg->cnt_all->curbytes_unbiased
		    += tdata->cnt_summed.curbytes_unbiased;
		if (opt_prof_accum) {
			arg->cnt_all->accumobjs += tdata->cnt_summed.accumobjs;
			arg->cnt_all->accumobjs_shifted_unbiased
			    += tdata->cnt_summed.accumobjs_shifted_unbiased;
			arg->cnt_all->accumbytes +=
			    tdata->cnt_summed.accumbytes;
			arg->cnt_all->accumbytes_unbiased +=
			    tdata->cnt_summed.accumbytes_unbiased;
		}
	} else {
		tdata->dumping = false;
	}
	malloc_mutex_unlock(arg->tsdn, tdata->lock);

	return NULL;
}

static prof_tdata_t *
prof_tdata_dump_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata,
    void *opaque) {
	if (!tdata->dumping) {
		return NULL;
	}

	prof_dump_iter_arg_t *arg = (prof_dump_iter_arg_t *)opaque;
	prof_dump_printf(arg->prof_dump_write, arg->cbopaque, "  t%"FMTu64": ",
	    tdata->thr_uid);
	prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque,
	    &tdata->cnt_summed);
	if (tdata->thread_name != NULL) {
		arg->prof_dump_write(arg->cbopaque, " ");
		arg->prof_dump_write(arg->cbopaque, tdata->thread_name);
	}
	arg->prof_dump_write(arg->cbopaque, "\n");
	return NULL;
}

static void
prof_dump_header(prof_dump_iter_arg_t *arg, const prof_cnt_t *cnt_all) {
	prof_dump_printf(arg->prof_dump_write, arg->cbopaque,
	    "heap_v2/%"FMTu64"\n  t*: ", ((uint64_t)1U << lg_prof_sample));
	prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque, cnt_all);
	arg->prof_dump_write(arg->cbopaque, "\n");

	malloc_mutex_lock(arg->tsdn, &tdatas_mtx);
	tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter, arg);
	malloc_mutex_unlock(arg->tsdn, &tdatas_mtx);
}

static void
prof_dump_gctx(prof_dump_iter_arg_t *arg, prof_gctx_t *gctx,
    const prof_bt_t *bt, prof_gctx_tree_t *gctxs) {
	cassert(config_prof);
	malloc_mutex_assert_owner(arg->tsdn, gctx->lock);

	/* Avoid dumping such gctx's that have no useful data. */
	if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) ||
	    (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) {
		assert(gctx->cnt_summed.curobjs == 0);
		assert(gctx->cnt_summed.curbytes == 0);
		/*
		 * These asserts would not be correct -- see the comment on races
		 * in prof.c
		 * assert(gctx->cnt_summed.curobjs_unbiased == 0);
		 * assert(gctx->cnt_summed.curbytes_unbiased == 0);
		*/
		assert(gctx->cnt_summed.accumobjs == 0);
		assert(gctx->cnt_summed.accumobjs_shifted_unbiased == 0);
		assert(gctx->cnt_summed.accumbytes == 0);
		assert(gctx->cnt_summed.accumbytes_unbiased == 0);
		return;
	}

	arg->prof_dump_write(arg->cbopaque, "@");
	for (unsigned i = 0; i < bt->len; i++) {
		prof_dump_printf(arg->prof_dump_write, arg->cbopaque,
		    " %#"FMTxPTR, (uintptr_t)bt->vec[i]);
	}

	arg->prof_dump_write(arg->cbopaque, "\n  t*: ");
	prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque,
	    &gctx->cnt_summed);
	arg->prof_dump_write(arg->cbopaque, "\n");

	tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter, arg);
}

/*
 * See prof_sample_new_event_wait() comment for why the body of this function
 * is conditionally compiled.
 */
static void
prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx) {
#ifdef JEMALLOC_PROF
	/*
	 * Scaling is equivalent AdjustSamples() in jeprof, but the result may
	 * differ slightly from what jeprof reports, because here we scale the
	 * summary values, whereas jeprof scales each context individually and
	 * reports the sums of the scaled values.
	 */
	if (cnt_all->curbytes != 0) {
		double sample_period = (double)((uint64_t)1 << lg_prof_sample);
		double ratio = (((double)cnt_all->curbytes) /
		    (double)cnt_all->curobjs) / sample_period;
		double scale_factor = 1.0 / (1.0 - exp(-ratio));
		uint64_t curbytes = (uint64_t)round(((double)cnt_all->curbytes)
		    * scale_factor);
		uint64_t curobjs = (uint64_t)round(((double)cnt_all->curobjs) *
		    scale_factor);

		malloc_printf("<jemalloc>: Leak approximation summary: ~%"FMTu64
		    " byte%s, ~%"FMTu64" object%s, >= %zu context%s\n",
		    curbytes, (curbytes != 1) ? "s" : "", curobjs, (curobjs !=
		    1) ? "s" : "", leak_ngctx, (leak_ngctx != 1) ? "s" : "");
		malloc_printf(
		    "<jemalloc>: Run jeprof on dump output for leak detail\n");
		if (opt_prof_leak_error) {
			malloc_printf(
			    "<jemalloc>: Exiting with error code because memory"
			    " leaks were detected\n");
			/*
			 * Use _exit() with underscore to avoid calling atexit()
			 * and entering endless cycle.
			 */
			_exit(1);
		}
	}
#endif
}

static prof_gctx_t *
prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
	prof_dump_iter_arg_t *arg = (prof_dump_iter_arg_t *)opaque;
	malloc_mutex_lock(arg->tsdn, gctx->lock);
	prof_dump_gctx(arg, gctx, &gctx->bt, gctxs);
	malloc_mutex_unlock(arg->tsdn, gctx->lock);
	return NULL;
}

static void
prof_dump_prep(tsd_t *tsd, prof_tdata_t *tdata, prof_cnt_t *cnt_all,
    size_t *leak_ngctx, prof_gctx_tree_t *gctxs) {
	size_t tabind;
	union {
		prof_gctx_t	*p;
		void		*v;
	} gctx;

	prof_enter(tsd, tdata);

	/*
	 * Put gctx's in limbo and clear their counters in preparation for
	 * summing.
	 */
	gctx_tree_new(gctxs);
	for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) {
		prof_dump_gctx_prep(tsd_tsdn(tsd), gctx.p, gctxs);
	}

	/*
	 * Iterate over tdatas, and for the non-expired ones snapshot their tctx
	 * stats and merge them into the associated gctx's.
	 */
	memset(cnt_all, 0, sizeof(prof_cnt_t));
	prof_tdata_merge_iter_arg_t prof_tdata_merge_iter_arg = {tsd_tsdn(tsd),
	    cnt_all};
	malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
	tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter,
	    &prof_tdata_merge_iter_arg);
	malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);

	/* Merge tctx stats into gctx's. */
	*leak_ngctx = 0;
	prof_gctx_merge_iter_arg_t prof_gctx_merge_iter_arg = {tsd_tsdn(tsd),
	    leak_ngctx};
	gctx_tree_iter(gctxs, NULL, prof_gctx_merge_iter,
	    &prof_gctx_merge_iter_arg);

	prof_leave(tsd, tdata);
}

void
prof_dump_impl(tsd_t *tsd, write_cb_t *prof_dump_write, void *cbopaque,
    prof_tdata_t *tdata, bool leakcheck) {
	malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_dump_mtx);
	prof_cnt_t cnt_all;
	size_t leak_ngctx;
	prof_gctx_tree_t gctxs;
	prof_dump_prep(tsd, tdata, &cnt_all, &leak_ngctx, &gctxs);
	prof_dump_iter_arg_t prof_dump_iter_arg = {tsd_tsdn(tsd),
	    prof_dump_write, cbopaque};
	prof_dump_header(&prof_dump_iter_arg, &cnt_all);
	gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter, &prof_dump_iter_arg);
	prof_gctx_finish(tsd, &gctxs);
	if (leakcheck) {
		prof_leakcheck(&cnt_all, leak_ngctx);
	}
}

/* Used in unit tests. */
void
prof_cnt_all(prof_cnt_t *cnt_all) {
	tsd_t *tsd = tsd_fetch();
	prof_tdata_t *tdata = prof_tdata_get(tsd, false);
	if (tdata == NULL) {
		memset(cnt_all, 0, sizeof(prof_cnt_t));
	} else {
		size_t leak_ngctx;
		prof_gctx_tree_t gctxs;
		prof_dump_prep(tsd, tdata, cnt_all, &leak_ngctx, &gctxs);
		prof_gctx_finish(tsd, &gctxs);
	}
}

void
prof_bt_hash(const void *key, size_t r_hash[2]) {
	prof_bt_t *bt = (prof_bt_t *)key;

	cassert(config_prof);

	hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash);
}

bool
prof_bt_keycomp(const void *k1, const void *k2) {
	const prof_bt_t *bt1 = (prof_bt_t *)k1;
	const prof_bt_t *bt2 = (prof_bt_t *)k2;

	cassert(config_prof);

	if (bt1->len != bt2->len) {
		return false;
	}
	return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
}

prof_tdata_t *
prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,
    char *thread_name, bool active) {
	assert(tsd_reentrancy_level_get(tsd) == 0);

	prof_tdata_t *tdata;

	cassert(config_prof);

	/* Initialize an empty cache for this thread. */
	tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t),
	    sz_size2index(sizeof(prof_tdata_t)), false, NULL, true,
	    arena_get(TSDN_NULL, 0, true), true);
	if (tdata == NULL) {
		return NULL;
	}

	tdata->lock = prof_tdata_mutex_choose(thr_uid);
	tdata->thr_uid = thr_uid;
	tdata->thr_discrim = thr_discrim;
	tdata->thread_name = thread_name;
	tdata->attached = true;
	tdata->expired = false;
	tdata->tctx_uid_next = 0;

	if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash,
	    prof_bt_keycomp)) {
		idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
		return NULL;
	}

	tdata->enq = false;
	tdata->enq_idump = false;
	tdata->enq_gdump = false;

	tdata->dumping = false;
	tdata->active = active;

	malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
	tdata_tree_insert(&tdatas, tdata);
	malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);

	return tdata;
}

static bool
prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) {
	if (tdata->attached && !even_if_attached) {
		return false;
	}
	if (ckh_count(&tdata->bt2tctx) != 0) {
		return false;
	}
	return true;
}

static bool
prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
    bool even_if_attached) {
	malloc_mutex_assert_owner(tsdn, tdata->lock);

	return prof_tdata_should_destroy_unlocked(tdata, even_if_attached);
}

static void
prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,
    bool even_if_attached) {
	malloc_mutex_assert_owner(tsd_tsdn(tsd), &tdatas_mtx);
	malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tdata->lock);

	tdata_tree_remove(&tdatas, tdata);

	assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached));

	if (tdata->thread_name != NULL) {
		idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,
		    true);
	}
	ckh_delete(tsd, &tdata->bt2tctx);
	idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
}

static void
prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) {
	malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
	prof_tdata_destroy_locked(tsd, tdata, even_if_attached);
	malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
}

void
prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) {
	bool destroy_tdata;

	malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
	if (tdata->attached) {
		destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata,
		    true);
		/*
		 * Only detach if !destroy_tdata, because detaching would allow
		 * another thread to win the race to destroy tdata.
		 */
		if (!destroy_tdata) {
			tdata->attached = false;
		}
		tsd_prof_tdata_set(tsd, NULL);
	} else {
		destroy_tdata = false;
	}
	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
	if (destroy_tdata) {
		prof_tdata_destroy(tsd, tdata, true);
	}
}

static bool
prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) {
	bool destroy_tdata;

	malloc_mutex_lock(tsdn, tdata->lock);
	if (!tdata->expired) {
		tdata->expired = true;
		destroy_tdata = prof_tdata_should_destroy(tsdn, tdata, false);
	} else {
		destroy_tdata = false;
	}
	malloc_mutex_unlock(tsdn, tdata->lock);

	return destroy_tdata;
}

static prof_tdata_t *
prof_tdata_reset_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata,
    void *arg) {
	tsdn_t *tsdn = (tsdn_t *)arg;

	return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL);
}

void
prof_reset(tsd_t *tsd, size_t lg_sample) {
	prof_tdata_t *next;

	assert(lg_sample < (sizeof(uint64_t) << 3));

	malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
	malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);

	lg_prof_sample = lg_sample;
	prof_unbias_map_init();

	next = NULL;
	do {
		prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next,
		    prof_tdata_reset_iter, (void *)tsd);
		if (to_destroy != NULL) {
			next = tdata_tree_next(&tdatas, to_destroy);
			prof_tdata_destroy_locked(tsd, to_destroy, false);
		} else {
			next = NULL;
		}
	} while (next != NULL);

	malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
	malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
}

static bool
prof_tctx_should_destroy(tsd_t *tsd, prof_tctx_t *tctx) {
	malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);

	if (opt_prof_accum) {
		return false;
	}
	if (tctx->cnts.curobjs != 0) {
		return false;
	}
	if (tctx->prepared) {
		return false;
	}
	if (tctx->recent_count != 0) {
		return false;
	}
	return true;
}

static void
prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) {
	malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);

	assert(tctx->cnts.curobjs == 0);
	assert(tctx->cnts.curbytes == 0);
	/*
	 * These asserts are not correct -- see the comment about races in
	 * prof.c
	 *
	 * assert(tctx->cnts.curobjs_shifted_unbiased == 0);
	 * assert(tctx->cnts.curbytes_unbiased == 0);
	 */
	assert(!opt_prof_accum);
	assert(tctx->cnts.accumobjs == 0);
	assert(tctx->cnts.accumbytes == 0);
	/*
	 * These ones are, since accumbyte counts never go down.  Either
	 * prof_accum is off (in which case these should never have changed from
	 * their initial value of zero), or it's on (in which case we shouldn't
	 * be destroying this tctx).
	 */
	assert(tctx->cnts.accumobjs_shifted_unbiased == 0);
	assert(tctx->cnts.accumbytes_unbiased == 0);

	prof_gctx_t *gctx = tctx->gctx;

	{
		prof_tdata_t *tdata = tctx->tdata;
		tctx->tdata = NULL;
		ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
		bool destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd),
		    tdata, false);
		malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
		if (destroy_tdata) {
			prof_tdata_destroy(tsd, tdata, false);
		}
	}

	bool destroy_tctx, destroy_gctx;

	malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
	switch (tctx->state) {
	case prof_tctx_state_nominal:
		tctx_tree_remove(&gctx->tctxs, tctx);
		destroy_tctx = true;
		if (prof_gctx_should_destroy(gctx)) {
			/*
			 * Increment gctx->nlimbo in order to keep another
			 * thread from winning the race to destroy gctx while
			 * this one has gctx->lock dropped.  Without this, it
			 * would be possible for another thread to:
			 *
			 * 1) Sample an allocation associated with gctx.
			 * 2) Deallocate the sampled object.
			 * 3) Successfully prof_gctx_try_destroy(gctx).
			 *
			 * The result would be that gctx no longer exists by the
			 * time this thread accesses it in
			 * prof_gctx_try_destroy().
			 */
			gctx->nlimbo++;
			destroy_gctx = true;
		} else {
			destroy_gctx = false;
		}
		break;
	case prof_tctx_state_dumping:
		/*
		 * A dumping thread needs tctx to remain valid until dumping
		 * has finished.  Change state such that the dumping thread will
		 * complete destruction during a late dump iteration phase.
		 */
		tctx->state = prof_tctx_state_purgatory;
		destroy_tctx = false;
		destroy_gctx = false;
		break;
	default:
		not_reached();
		destroy_tctx = false;
		destroy_gctx = false;
	}
	malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
	if (destroy_gctx) {
		prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx);
	}
	if (destroy_tctx) {
		idalloctm(tsd_tsdn(tsd), tctx, NULL, NULL, true, true);
	}
}

void
prof_tctx_try_destroy(tsd_t *tsd, prof_tctx_t *tctx) {
	malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
	if (prof_tctx_should_destroy(tsd, tctx)) {
		/* tctx->tdata->lock will be released in prof_tctx_destroy(). */
		prof_tctx_destroy(tsd, tctx);
	} else {
		malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);
	}
}

/******************************************************************************/