arm_spe_dev.c

/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2024 Arm Ltd
 * Copyright (c) 2022 The FreeBSD Foundation
 *
 * Portions of this software were developed by Andrew Turner under sponsorship
 * from the FreeBSD Foundation.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/event.h>
#include <sys/hwt.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/rman.h>
#include <sys/smp.h>
#include <sys/systm.h>
#include <sys/taskqueue.h>

#include <machine/bus.h>

#include <arm64/spe/arm_spe.h>
#include <arm64/spe/arm_spe_dev.h>

MALLOC_DEFINE(M_ARM_SPE, "armspe", "Arm SPE tracing");

/*
 * taskqueue(9) used for sleepable routines called from interrupt handlers
 */
TASKQUEUE_FAST_DEFINE_THREAD(arm_spe);

void arm_spe_send_buffer(void *, int);
static void arm_spe_error(void *, int);
static int arm_spe_intr(void *);
device_attach_t arm_spe_attach;

static device_method_t arm_spe_methods[] = {
	/* Device interface */
	DEVMETHOD(device_attach,        arm_spe_attach),

	DEVMETHOD_END,
};

DEFINE_CLASS_0(spe, arm_spe_driver, arm_spe_methods,
    sizeof(struct arm_spe_softc));

#define ARM_SPE_KVA_MAX_ALIGN	UL(2048)

int
arm_spe_attach(device_t dev)
{
	struct arm_spe_softc *sc;
	int error, rid;

	sc = device_get_softc(dev);
	sc->dev = dev;

	sc->pmbidr = READ_SPECIALREG(PMBIDR_EL1_REG);
	sc->pmsidr = READ_SPECIALREG(PMSIDR_EL1_REG);
	device_printf(dev, "PMBIDR_EL1: %#lx\n", sc->pmbidr);
	device_printf(dev, "PMSIDR_EL1: %#lx\n", sc->pmsidr);
	if ((sc->pmbidr & PMBIDR_P) != 0) {
		device_printf(dev, "Profiling Buffer is owned by a higher Exception level\n");
		return (EPERM);
	}

	sc->kva_align = 1 << ((sc->pmbidr & PMBIDR_Align_MASK) >> PMBIDR_Align_SHIFT);
	if (sc->kva_align > ARM_SPE_KVA_MAX_ALIGN) {
		device_printf(dev, "Invalid PMBIDR.Align value of %d\n", sc->kva_align);
		return (EINVAL);
	}

	rid = 0;
	sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
	    RF_ACTIVE);
	if (sc->sc_irq_res == NULL) {
		device_printf(dev, "Unable to allocate interrupt\n");
		return (ENXIO);
	}
	error = bus_setup_intr(dev, sc->sc_irq_res,
	    INTR_TYPE_MISC | INTR_MPSAFE, arm_spe_intr, NULL, sc,
	    &sc->sc_irq_cookie);
	if (error != 0) {
		device_printf(dev, "Unable to set up interrupt\n");
		return (error);
	}

	mtx_init(&sc->sc_lock, "Arm SPE lock", NULL, MTX_SPIN);

	STAILQ_INIT(&sc->pending);
	sc->npending = 0;

	spe_register(dev);

	return (0);
}

/* Interrupt handler runs on the same core that triggered the exception */
static int
arm_spe_intr(void *arg)
{
	int cpu_id = PCPU_GET(cpuid);
	struct arm_spe_softc *sc = arg;
	uint64_t pmbsr;
	uint64_t base, limit;
	uint8_t ec;
	struct arm_spe_info *info = sc->spe_info[cpu_id];
	uint8_t i = info->buf_idx;
	struct arm_spe_buf_info *buf = &info->buf_info[i];
	struct arm_spe_buf_info *prev_buf = &info->buf_info[!i];
	device_t dev = sc->dev;

	/* Make sure the profiling data is visible to the CPU */
	psb_csync();
	dsb(nsh);

	/* Make sure any HW update of PMBPTR_EL1 is visible to the CPU */
	isb();

	pmbsr = READ_SPECIALREG(PMBSR_EL1_REG);

	if (!(pmbsr & PMBSR_S))
		return (FILTER_STRAY);

	/* Event Class */
	ec = PMBSR_EC_VAL(pmbsr);
	switch (ec)
	{
	case PMBSR_EC_OTHER_BUF_MGMT: /* Other buffer management event */
		break;
	case PMBSR_EC_GRAN_PROT_CHK: /* Granule Protection Check fault */
		device_printf(dev, "PMBSR_EC_GRAN_PROT_CHK\n");
		break;
	case PMBSR_EC_STAGE1_DA: /* Stage 1 Data Abort */
		device_printf(dev, "PMBSR_EC_STAGE1_DA\n");
		break;
	case PMBSR_EC_STAGE2_DA: /* Stage 2 Data Abort */
		device_printf(dev, "PMBSR_EC_STAGE2_DA\n");
		break;
	default:
		/* Unknown EC */
		device_printf(dev, "unknown PMBSR_EC: %#x\n", ec);
		arm_spe_disable(NULL);
		TASK_INIT(&sc->task, 0, (task_fn_t *)arm_spe_error, sc->ctx);
		taskqueue_enqueue(taskqueue_arm_spe, &sc->task);
		return (FILTER_HANDLED);
	}

	switch (ec) {
	case PMBSR_EC_OTHER_BUF_MGMT:
		/* Buffer Status Code = buffer filled */
		if ((pmbsr & PMBSR_MSS_BSC_MASK) == PMBSR_MSS_BSC_BUFFER_FILLED) {
			dprintf("%s SPE buffer full event (cpu:%d)\n",
			    __func__, cpu_id);
			break;
		}
	case PMBSR_EC_GRAN_PROT_CHK:
	case PMBSR_EC_STAGE1_DA:
	case PMBSR_EC_STAGE2_DA:
		/*
		 * If we have one of these, we've messed up the
		 * programming somehow (e.g. passed invalid memory to
		 * SPE) and can't recover
		 */
		arm_spe_disable(NULL);
		TASK_INIT(&sc->task, 0, (task_fn_t *)arm_spe_error, sc->ctx);
		taskqueue_enqueue(taskqueue_arm_spe, &sc->task);
		/* PMBPTR_EL1 is fault address if PMBSR_DL is 1 */
		device_printf(dev, "CPU:%d PMBSR_EL1:%#lx\n", cpu_id, pmbsr);
		device_printf(dev, "PMBPTR_EL1:%#lx PMBLIMITR_EL1:%#lx\n",
		    READ_SPECIALREG(PMBPTR_EL1_REG),
		    READ_SPECIALREG(PMBLIMITR_EL1_REG));
		return (FILTER_HANDLED);
	}

	mtx_lock_spin(&info->lock);

	/*
	 * Data Loss bit - pmbptr might not be pointing to the end of the last
	 * complete record
	 */
	if ((pmbsr & PMBSR_DL) == PMBSR_DL)
		buf->partial_rec = 1;
	buf->pmbptr = READ_SPECIALREG(PMBPTR_EL1_REG);
	buf->buf_svc = true;

	/* Setup regs ready to start writing to the other half of the buffer */
	info->buf_idx = !info->buf_idx;
	base = buf_start_addr(info->buf_idx, info);
	limit = base + (info->buf_size/2);
	limit &= PMBLIMITR_LIMIT_MASK;
	limit |= PMBLIMITR_E;
	WRITE_SPECIALREG(PMBPTR_EL1_REG, base);
	WRITE_SPECIALREG(PMBLIMITR_EL1_REG, limit);
	isb();

	/*
	 * Notify userspace via kqueue that buffer is full and needs copying
	 * out - since kqueue can sleep, don't do this in the interrupt handler,
	 * add to a taskqueue to be scheduled later instead
	 */
	TASK_INIT(&info->task[i], 0, (task_fn_t *)arm_spe_send_buffer, buf);
	taskqueue_enqueue(taskqueue_arm_spe, &info->task[i]);

	/*
	 * It's possible userspace hasn't yet notified us they've copied out the
	 * other half of the buffer
	 *
	 * This might be because:
	 *      a) Kernel hasn't scheduled the task via taskqueue to notify
	 *         userspace to copy out the data
	 *      b) Userspace is still copying the buffer or hasn't notified us
	 *         back via the HWT_IOC_SVC_BUF ioctl
	 *
	 * Either way we need to avoid overwriting uncopied data in the
	 * buffer, so disable profiling until we receive that SVC_BUF
	 * ioctl
	 *
	 * Using a larger buffer size should help to minimise these events and
	 * loss of profiling data while profiling is disabled
	 */
	if (prev_buf->buf_svc) {
		device_printf(sc->dev, "cpu%d: buffer full interrupt, but other"
		    " half of buffer has not been copied out - consider"
		    " increasing buffer size to minimise loss of profiling data\n",
		    cpu_id);
		WRITE_SPECIALREG(PMSCR_EL1_REG, 0x0);
		prev_buf->buf_wait = true;
	}

	mtx_unlock_spin(&info->lock);

	/* Clear Profiling Buffer Status Register */
	WRITE_SPECIALREG(PMBSR_EL1_REG, 0);

	isb();

	return (FILTER_HANDLED);
}

/* note: Scheduled and run via taskqueue, so can run on any CPU at any time */
void
arm_spe_send_buffer(void *arg, int pending __unused)
{
	struct arm_spe_buf_info *buf = (struct arm_spe_buf_info *)arg;
	struct arm_spe_info *info = buf->info;
	struct arm_spe_queue *queue;
	struct kevent kev;
	int ret;

	queue = malloc(sizeof(struct arm_spe_queue), M_ARM_SPE,
	    M_WAITOK | M_ZERO);

	mtx_lock_spin(&info->lock);

	/* Add to queue for userspace to pickup */
	queue->ident = info->ident;
	queue->offset = buf->pmbptr - buf_start_addr(buf->buf_idx, info);
	queue->buf_idx = buf->buf_idx;
	queue->final_buf = !info->enabled;
	queue->partial_rec = buf->partial_rec;
	mtx_unlock_spin(&info->lock);

	mtx_lock_spin(&info->sc->sc_lock);
	STAILQ_INSERT_TAIL(&info->sc->pending, queue, next);
	info->sc->npending++;
	EV_SET(&kev, ARM_SPE_KQ_BUF, EVFILT_USER, 0, NOTE_TRIGGER,
	    info->sc->npending, NULL);
	mtx_unlock_spin(&info->sc->sc_lock);

	/* Notify userspace */
	ret = kqfd_register(info->sc->kqueue_fd, &kev, info->sc->hwt_td,
	    M_WAITOK);
	if (ret) {
		dprintf("%s kqfd_register ret:%d\n", __func__, ret);
		arm_spe_error(info->sc->ctx, 0);
	}
}

static void
arm_spe_error(void *arg, int pending __unused)
{
	struct hwt_context *ctx = arg;
	struct kevent kev;
	int ret;

	if (!CPU_EMPTY(&ctx->cpu_map))
		smp_rendezvous_cpus(ctx->cpu_map, smp_no_rendezvous_barrier,
		    arm_spe_disable, smp_no_rendezvous_barrier, NULL);

	EV_SET(&kev, ARM_SPE_KQ_SHUTDOWN, EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL);
	ret = kqfd_register(ctx->kqueue_fd, &kev, ctx->hwt_td, M_WAITOK);
	if (ret)
		dprintf("%s kqfd_register ret:%d\n", __func__, ret);
}

MODULE_DEPEND(spe, hwt, 1, 1, 1);
MODULE_VERSION(spe, 1);