core.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Zhaoxin PMU; like Intel Architectural PerfMon-v2
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/stddef.h>
#include <linux/types.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/nmi.h>

#include <asm/cpufeature.h>
#include <asm/hardirq.h>
#include <asm/apic.h>
#include <asm/msr.h>

#include "../perf_event.h"

/*
 * Zhaoxin PerfMon, used on zxc and later.
 */
static u64 zx_pmon_event_map[PERF_COUNT_HW_MAX] __read_mostly = {

	[PERF_COUNT_HW_CPU_CYCLES]        = 0x0082,
	[PERF_COUNT_HW_INSTRUCTIONS]      = 0x00c0,
	[PERF_COUNT_HW_CACHE_REFERENCES]  = 0x0515,
	[PERF_COUNT_HW_CACHE_MISSES]      = 0x051a,
	[PERF_COUNT_HW_BUS_CYCLES]        = 0x0083,
};

static struct event_constraint zxc_event_constraints[] __read_mostly = {

	FIXED_EVENT_CONSTRAINT(0x0082, 1), /* unhalted core clock cycles */
	EVENT_CONSTRAINT_END
};

static struct event_constraint zxd_event_constraints[] __read_mostly = {

	FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* retired instructions */
	FIXED_EVENT_CONSTRAINT(0x0082, 1), /* unhalted core clock cycles */
	FIXED_EVENT_CONSTRAINT(0x0083, 2), /* unhalted bus clock cycles */
	EVENT_CONSTRAINT_END
};

static __initconst const u64 zxd_hw_cache_event_ids
				[PERF_COUNT_HW_CACHE_MAX]
				[PERF_COUNT_HW_CACHE_OP_MAX]
				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
[C(L1D)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = 0x0042,
		[C(RESULT_MISS)] = 0x0538,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = 0x0043,
		[C(RESULT_MISS)] = 0x0562,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
},
[C(L1I)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = 0x0300,
		[C(RESULT_MISS)] = 0x0301,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = 0x030a,
		[C(RESULT_MISS)] = 0x030b,
	},
},
[C(LL)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
},
[C(DTLB)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = 0x0042,
		[C(RESULT_MISS)] = 0x052c,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = 0x0043,
		[C(RESULT_MISS)] = 0x0530,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = 0x0564,
		[C(RESULT_MISS)] = 0x0565,
	},
},
[C(ITLB)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = 0x00c0,
		[C(RESULT_MISS)] = 0x0534,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
},
[C(BPU)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = 0x0700,
		[C(RESULT_MISS)] = 0x0709,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
},
[C(NODE)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
},
};

static __initconst const u64 zxe_hw_cache_event_ids
				[PERF_COUNT_HW_CACHE_MAX]
				[PERF_COUNT_HW_CACHE_OP_MAX]
				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
[C(L1D)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = 0x0568,
		[C(RESULT_MISS)] = 0x054b,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = 0x0669,
		[C(RESULT_MISS)] = 0x0562,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
},
[C(L1I)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = 0x0300,
		[C(RESULT_MISS)] = 0x0301,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = 0x030a,
		[C(RESULT_MISS)] = 0x030b,
	},
},
[C(LL)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = 0x0,
		[C(RESULT_MISS)] = 0x0,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = 0x0,
		[C(RESULT_MISS)] = 0x0,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = 0x0,
		[C(RESULT_MISS)] = 0x0,
	},
},
[C(DTLB)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = 0x0568,
		[C(RESULT_MISS)] = 0x052c,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = 0x0669,
		[C(RESULT_MISS)] = 0x0530,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = 0x0564,
		[C(RESULT_MISS)] = 0x0565,
	},
},
[C(ITLB)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = 0x00c0,
		[C(RESULT_MISS)] = 0x0534,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
},
[C(BPU)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = 0x0028,
		[C(RESULT_MISS)] = 0x0029,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
},
[C(NODE)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = -1,
		[C(RESULT_MISS)] = -1,
	},
},
};

static void zhaoxin_pmu_disable_all(void)
{
	wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, 0);
}

static void zhaoxin_pmu_enable_all(int added)
{
	wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
}

static inline u64 zhaoxin_pmu_get_status(void)
{
	u64 status;

	rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, status);

	return status;
}

static inline void zhaoxin_pmu_ack_status(u64 ack)
{
	wrmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
}

static inline void zxc_pmu_ack_status(u64 ack)
{
	/*
	 * ZXC needs global control enabled in order to clear status bits.
	 */
	zhaoxin_pmu_enable_all(0);
	zhaoxin_pmu_ack_status(ack);
	zhaoxin_pmu_disable_all();
}

static void zhaoxin_pmu_disable_fixed(struct hw_perf_event *hwc)
{
	int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
	u64 ctrl_val, mask;

	mask = 0xfULL << (idx * 4);

	rdmsrq(hwc->config_base, ctrl_val);
	ctrl_val &= ~mask;
	wrmsrq(hwc->config_base, ctrl_val);
}

static void zhaoxin_pmu_disable_event(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
		zhaoxin_pmu_disable_fixed(hwc);
		return;
	}

	x86_pmu_disable_event(event);
}

static void zhaoxin_pmu_enable_fixed(struct hw_perf_event *hwc)
{
	int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
	u64 ctrl_val, bits, mask;

	/*
	 * Enable IRQ generation (0x8),
	 * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
	 * if requested:
	 */
	bits = 0x8ULL;
	if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
		bits |= 0x2;
	if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
		bits |= 0x1;

	bits <<= (idx * 4);
	mask = 0xfULL << (idx * 4);

	rdmsrq(hwc->config_base, ctrl_val);
	ctrl_val &= ~mask;
	ctrl_val |= bits;
	wrmsrq(hwc->config_base, ctrl_val);
}

static void zhaoxin_pmu_enable_event(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
		zhaoxin_pmu_enable_fixed(hwc);
		return;
	}

	__x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
}

/*
 * This handler is triggered by the local APIC, so the APIC IRQ handling
 * rules apply:
 */
static int zhaoxin_pmu_handle_irq(struct pt_regs *regs)
{
	struct perf_sample_data data;
	struct cpu_hw_events *cpuc;
	int handled = 0;
	u64 status;
	int bit;

	cpuc = this_cpu_ptr(&cpu_hw_events);
	apic_write(APIC_LVTPC, APIC_DM_NMI);
	zhaoxin_pmu_disable_all();
	status = zhaoxin_pmu_get_status();
	if (!status)
		goto done;

again:
	if (x86_pmu.enabled_ack)
		zxc_pmu_ack_status(status);
	else
		zhaoxin_pmu_ack_status(status);

	inc_irq_stat(apic_perf_irqs);

	/*
	 * CondChgd bit 63 doesn't mean any overflow status. Ignore
	 * and clear the bit.
	 */
	if (__test_and_clear_bit(63, (unsigned long *)&status)) {
		if (!status)
			goto done;
	}

	for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
		struct perf_event *event = cpuc->events[bit];

		handled++;

		if (!test_bit(bit, cpuc->active_mask))
			continue;

		x86_perf_event_update(event);
		perf_sample_data_init(&data, 0, event->hw.last_period);

		if (!x86_perf_event_set_period(event))
			continue;

		perf_event_overflow(event, &data, regs);
	}

	/*
	 * Repeat if there is more work to be done:
	 */
	status = zhaoxin_pmu_get_status();
	if (status)
		goto again;

done:
	zhaoxin_pmu_enable_all(0);
	return handled;
}

static u64 zhaoxin_pmu_event_map(int hw_event)
{
	return zx_pmon_event_map[hw_event];
}

static struct event_constraint *
zhaoxin_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
			struct perf_event *event)
{
	struct event_constraint *c;

	if (x86_pmu.event_constraints) {
		for_each_event_constraint(c, x86_pmu.event_constraints) {
			if ((event->hw.config & c->cmask) == c->code)
				return c;
		}
	}

	return &unconstrained;
}

PMU_FORMAT_ATTR(event,	"config:0-7");
PMU_FORMAT_ATTR(umask,	"config:8-15");
PMU_FORMAT_ATTR(edge,	"config:18");
PMU_FORMAT_ATTR(inv,	"config:23");
PMU_FORMAT_ATTR(cmask,	"config:24-31");

static struct attribute *zx_arch_formats_attr[] = {
	&format_attr_event.attr,
	&format_attr_umask.attr,
	&format_attr_edge.attr,
	&format_attr_inv.attr,
	&format_attr_cmask.attr,
	NULL,
};

static ssize_t zhaoxin_event_sysfs_show(char *page, u64 config)
{
	u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT);

	return x86_event_sysfs_show(page, config, event);
}

static const struct x86_pmu zhaoxin_pmu __initconst = {
	.name			= "zhaoxin",
	.handle_irq		= zhaoxin_pmu_handle_irq,
	.disable_all		= zhaoxin_pmu_disable_all,
	.enable_all		= zhaoxin_pmu_enable_all,
	.enable			= zhaoxin_pmu_enable_event,
	.disable		= zhaoxin_pmu_disable_event,
	.hw_config		= x86_pmu_hw_config,
	.schedule_events	= x86_schedule_events,
	.eventsel		= MSR_ARCH_PERFMON_EVENTSEL0,
	.perfctr		= MSR_ARCH_PERFMON_PERFCTR0,
	.event_map		= zhaoxin_pmu_event_map,
	.max_events		= ARRAY_SIZE(zx_pmon_event_map),
	.apic			= 1,
	/*
	 * For zxd/zxe, read/write operation for PMCx MSR is 48 bits.
	 */
	.max_period		= (1ULL << 47) - 1,
	.get_event_constraints	= zhaoxin_get_event_constraints,

	.format_attrs		= zx_arch_formats_attr,
	.events_sysfs_show	= zhaoxin_event_sysfs_show,
};

static const struct { int id; char *name; } zx_arch_events_map[] __initconst = {
	{ PERF_COUNT_HW_CPU_CYCLES, "cpu cycles" },
	{ PERF_COUNT_HW_INSTRUCTIONS, "instructions" },
	{ PERF_COUNT_HW_BUS_CYCLES, "bus cycles" },
	{ PERF_COUNT_HW_CACHE_REFERENCES, "cache references" },
	{ PERF_COUNT_HW_CACHE_MISSES, "cache misses" },
	{ PERF_COUNT_HW_BRANCH_INSTRUCTIONS, "branch instructions" },
	{ PERF_COUNT_HW_BRANCH_MISSES, "branch misses" },
};

static __init void zhaoxin_arch_events_quirk(void)
{
	int bit;

	/* disable event that reported as not present by cpuid */
	for_each_set_bit(bit, x86_pmu.events_mask, ARRAY_SIZE(zx_arch_events_map)) {
		zx_pmon_event_map[zx_arch_events_map[bit].id] = 0;
		pr_warn("CPUID marked event: \'%s\' unavailable\n",
			zx_arch_events_map[bit].name);
	}
}

__init int zhaoxin_pmu_init(void)
{
	union cpuid10_edx edx;
	union cpuid10_eax eax;
	union cpuid10_ebx ebx;
	struct event_constraint *c;
	unsigned int unused;
	int version;

	pr_info("Welcome to zhaoxin pmu!\n");

	/*
	 * Check whether the Architectural PerfMon supports
	 * hw_event or not.
	 */
	cpuid(10, &eax.full, &ebx.full, &unused, &edx.full);

	if (eax.split.mask_length < ARCH_PERFMON_EVENTS_COUNT - 1)
		return -ENODEV;

	version = eax.split.version_id;
	if (version != 2)
		return -ENODEV;

	x86_pmu = zhaoxin_pmu;
	pr_info("Version check pass!\n");

	x86_pmu.version			= version;
	x86_pmu.cntr_mask64		= GENMASK_ULL(eax.split.num_counters - 1, 0);
	x86_pmu.cntval_bits		= eax.split.bit_width;
	x86_pmu.cntval_mask		= (1ULL << eax.split.bit_width) - 1;
	x86_pmu.events_maskl		= ebx.full;
	x86_pmu.events_mask_len		= eax.split.mask_length;

	x86_pmu.fixed_cntr_mask64	= GENMASK_ULL(edx.split.num_counters_fixed - 1, 0);
	x86_add_quirk(zhaoxin_arch_events_quirk);

	switch (boot_cpu_data.x86) {
	case 0x06:
		/*
		 * Support Zhaoxin CPU from ZXC series, exclude Nano series through FMS.
		 * Nano FMS: Family=6, Model=F, Stepping=[0-A][C-D]
		 * ZXC FMS: Family=6, Model=F, Stepping=E-F OR Family=6, Model=0x19, Stepping=0-3
		 */
		if ((boot_cpu_data.x86_model == 0x0f && boot_cpu_data.x86_stepping >= 0x0e) ||
			boot_cpu_data.x86_model == 0x19) {

			x86_pmu.max_period = x86_pmu.cntval_mask >> 1;

			/* Clearing status works only if the global control is enable on zxc. */
			x86_pmu.enabled_ack = 1;

			x86_pmu.event_constraints = zxc_event_constraints;
			zx_pmon_event_map[PERF_COUNT_HW_INSTRUCTIONS] = 0;
			zx_pmon_event_map[PERF_COUNT_HW_CACHE_REFERENCES] = 0;
			zx_pmon_event_map[PERF_COUNT_HW_CACHE_MISSES] = 0;
			zx_pmon_event_map[PERF_COUNT_HW_BUS_CYCLES] = 0;

			pr_cont("ZXC events, ");
			break;
		}
		return -ENODEV;

	case 0x07:
		zx_pmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
			X86_CONFIG(.event = 0x01, .umask = 0x01, .inv = 0x01, .cmask = 0x01);

		zx_pmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
			X86_CONFIG(.event = 0x0f, .umask = 0x04, .inv = 0, .cmask = 0);

		switch (boot_cpu_data.x86_model) {
		case 0x1b:
			memcpy(hw_cache_event_ids, zxd_hw_cache_event_ids,
			       sizeof(hw_cache_event_ids));

			x86_pmu.event_constraints = zxd_event_constraints;

			zx_pmon_event_map[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x0700;
			zx_pmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x0709;

			pr_cont("ZXD events, ");
			break;
		case 0x3b:
			memcpy(hw_cache_event_ids, zxe_hw_cache_event_ids,
			       sizeof(hw_cache_event_ids));

			x86_pmu.event_constraints = zxd_event_constraints;

			zx_pmon_event_map[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x0028;
			zx_pmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x0029;

			pr_cont("ZXE events, ");
			break;
		default:
			return -ENODEV;
		}
		break;

	default:
		return -ENODEV;
	}

	x86_pmu.intel_ctrl = x86_pmu.cntr_mask64;
	x86_pmu.intel_ctrl |= x86_pmu.fixed_cntr_mask64 << INTEL_PMC_IDX_FIXED;

	if (x86_pmu.event_constraints) {
		for_each_event_constraint(c, x86_pmu.event_constraints) {
			c->idxmsk64 |= x86_pmu.cntr_mask64;
			c->weight += x86_pmu_num_counters(NULL);
		}
	}

	return 0;
}