Path: blob/master/arch/powerpc/platforms/ps3/interrupt.c
26481 views
// SPDX-License-Identifier: GPL-2.0-only1/*2* PS3 interrupt routines.3*4* Copyright (C) 2006 Sony Computer Entertainment Inc.5* Copyright 2006 Sony Corp.6*/78#include <linux/kernel.h>9#include <linux/export.h>10#include <linux/irq.h>11#include <linux/irqdomain.h>1213#include <asm/machdep.h>14#include <asm/udbg.h>15#include <asm/lv1call.h>16#include <asm/smp.h>1718#include "platform.h"1920#if defined(DEBUG)21#define DBG udbg_printf22#define FAIL udbg_printf23#else24#define DBG pr_devel25#define FAIL pr_debug26#endif2728/**29* struct ps3_bmp - a per cpu irq status and mask bitmap structure30* @status: 256 bit status bitmap indexed by plug31* @unused_1: Alignment32* @mask: 256 bit mask bitmap indexed by plug33* @unused_2: Alignment34*35* The HV maintains per SMT thread mappings of HV outlet to HV plug on36* behalf of the guest. These mappings are implemented as 256 bit guest37* supplied bitmaps indexed by plug number. The addresses of the bitmaps38* are registered with the HV through lv1_configure_irq_state_bitmap().39* The HV requires that the 512 bits of status + mask not cross a page40* boundary. PS3_BMP_MINALIGN is used to define this minimal 64 byte41* alignment.42*43* The HV supports 256 plugs per thread, assigned as {0..255}, for a total44* of 512 plugs supported on a processor. To simplify the logic this45* implementation equates HV plug value to Linux virq value, constrains each46* interrupt to have a system wide unique plug number, and limits the range47* of the plug values to map into the first dword of the bitmaps. This48* gives a usable range of plug values of {NR_IRQS_LEGACY..63}. Note49* that there is no constraint on how many in this set an individual thread50* can acquire.51*52* The mask is declared as unsigned long so we can use set/clear_bit on it.53*/5455#define PS3_BMP_MINALIGN 645657struct ps3_bmp {58struct {59u64 status;60u64 unused_1[3];61unsigned long mask;62u64 unused_2[3];63};64};6566/**67* struct ps3_private - a per cpu data structure68* @bmp: ps3_bmp structure69* @bmp_lock: Synchronize access to bmp.70* @ipi_debug_brk_mask: Mask for debug break IPIs71* @ppe_id: HV logical_ppe_id72* @thread_id: HV thread_id73* @ipi_mask: Mask of IPI virqs74*/7576struct ps3_private {77struct ps3_bmp bmp __attribute__ ((aligned (PS3_BMP_MINALIGN)));78spinlock_t bmp_lock;79u64 ppe_id;80u64 thread_id;81unsigned long ipi_debug_brk_mask;82unsigned long ipi_mask;83};8485static DEFINE_PER_CPU(struct ps3_private, ps3_private);8687/**88* ps3_chip_mask - Set an interrupt mask bit in ps3_bmp.89* @virq: The assigned Linux virq.90*91* Sets ps3_bmp.mask and calls lv1_did_update_interrupt_mask().92*/9394static void ps3_chip_mask(struct irq_data *d)95{96struct ps3_private *pd = irq_data_get_irq_chip_data(d);97unsigned long flags;9899DBG("%s:%d: thread_id %llu, virq %d\n", __func__, __LINE__,100pd->thread_id, d->irq);101102local_irq_save(flags);103clear_bit(63 - d->irq, &pd->bmp.mask);104lv1_did_update_interrupt_mask(pd->ppe_id, pd->thread_id);105local_irq_restore(flags);106}107108/**109* ps3_chip_unmask - Clear an interrupt mask bit in ps3_bmp.110* @virq: The assigned Linux virq.111*112* Clears ps3_bmp.mask and calls lv1_did_update_interrupt_mask().113*/114115static void ps3_chip_unmask(struct irq_data *d)116{117struct ps3_private *pd = irq_data_get_irq_chip_data(d);118unsigned long flags;119120DBG("%s:%d: thread_id %llu, virq %d\n", __func__, __LINE__,121pd->thread_id, d->irq);122123local_irq_save(flags);124set_bit(63 - d->irq, &pd->bmp.mask);125lv1_did_update_interrupt_mask(pd->ppe_id, pd->thread_id);126local_irq_restore(flags);127}128129/**130* ps3_chip_eoi - HV end-of-interrupt.131* @virq: The assigned Linux virq.132*133* Calls lv1_end_of_interrupt_ext().134*/135136static void ps3_chip_eoi(struct irq_data *d)137{138const struct ps3_private *pd = irq_data_get_irq_chip_data(d);139140/* non-IPIs are EOIed here. */141142if (!test_bit(63 - d->irq, &pd->ipi_mask))143lv1_end_of_interrupt_ext(pd->ppe_id, pd->thread_id, d->irq);144}145146/**147* ps3_irq_chip - Represents the ps3_bmp as a Linux struct irq_chip.148*/149150static struct irq_chip ps3_irq_chip = {151.name = "ps3",152.irq_mask = ps3_chip_mask,153.irq_unmask = ps3_chip_unmask,154.irq_eoi = ps3_chip_eoi,155};156157/**158* ps3_virq_setup - virq related setup.159* @cpu: enum ps3_cpu_binding indicating the cpu the interrupt should be160* serviced on.161* @outlet: The HV outlet from the various create outlet routines.162* @virq: The assigned Linux virq.163*164* Calls irq_create_mapping() to get a virq and sets the chip data to165* ps3_private data.166*/167168static int ps3_virq_setup(enum ps3_cpu_binding cpu, unsigned long outlet,169unsigned int *virq)170{171int result;172struct ps3_private *pd;173174/* This defines the default interrupt distribution policy. */175176if (cpu == PS3_BINDING_CPU_ANY)177cpu = 0;178179pd = &per_cpu(ps3_private, cpu);180181*virq = irq_create_mapping(NULL, outlet);182183if (!*virq) {184FAIL("%s:%d: irq_create_mapping failed: outlet %lu\n",185__func__, __LINE__, outlet);186result = -ENOMEM;187goto fail_create;188}189190DBG("%s:%d: outlet %lu => cpu %u, virq %u\n", __func__, __LINE__,191outlet, cpu, *virq);192193result = irq_set_chip_data(*virq, pd);194195if (result) {196FAIL("%s:%d: irq_set_chip_data failed\n",197__func__, __LINE__);198goto fail_set;199}200201ps3_chip_mask(irq_get_irq_data(*virq));202203return result;204205fail_set:206irq_dispose_mapping(*virq);207fail_create:208return result;209}210211/**212* ps3_virq_destroy - virq related teardown.213* @virq: The assigned Linux virq.214*215* Clears chip data and calls irq_dispose_mapping() for the virq.216*/217218static int ps3_virq_destroy(unsigned int virq)219{220const struct ps3_private *pd = irq_get_chip_data(virq);221222DBG("%s:%d: ppe_id %llu, thread_id %llu, virq %u\n", __func__,223__LINE__, pd->ppe_id, pd->thread_id, virq);224225irq_set_chip_data(virq, NULL);226irq_dispose_mapping(virq);227228DBG("%s:%d <-\n", __func__, __LINE__);229return 0;230}231232/**233* ps3_irq_plug_setup - Generic outlet and virq related setup.234* @cpu: enum ps3_cpu_binding indicating the cpu the interrupt should be235* serviced on.236* @outlet: The HV outlet from the various create outlet routines.237* @virq: The assigned Linux virq.238*239* Sets up virq and connects the irq plug.240*/241242int ps3_irq_plug_setup(enum ps3_cpu_binding cpu, unsigned long outlet,243unsigned int *virq)244{245int result;246struct ps3_private *pd;247248result = ps3_virq_setup(cpu, outlet, virq);249250if (result) {251FAIL("%s:%d: ps3_virq_setup failed\n", __func__, __LINE__);252goto fail_setup;253}254255pd = irq_get_chip_data(*virq);256257/* Binds outlet to cpu + virq. */258259result = lv1_connect_irq_plug_ext(pd->ppe_id, pd->thread_id, *virq,260outlet, 0);261262if (result) {263FAIL("%s:%d: lv1_connect_irq_plug_ext failed: %s\n",264__func__, __LINE__, ps3_result(result));265result = -EPERM;266goto fail_connect;267}268269return result;270271fail_connect:272ps3_virq_destroy(*virq);273fail_setup:274return result;275}276EXPORT_SYMBOL_GPL(ps3_irq_plug_setup);277278/**279* ps3_irq_plug_destroy - Generic outlet and virq related teardown.280* @virq: The assigned Linux virq.281*282* Disconnects the irq plug and tears down virq.283* Do not call for system bus event interrupts setup with284* ps3_sb_event_receive_port_setup().285*/286287int ps3_irq_plug_destroy(unsigned int virq)288{289int result;290const struct ps3_private *pd = irq_get_chip_data(virq);291292DBG("%s:%d: ppe_id %llu, thread_id %llu, virq %u\n", __func__,293__LINE__, pd->ppe_id, pd->thread_id, virq);294295ps3_chip_mask(irq_get_irq_data(virq));296297result = lv1_disconnect_irq_plug_ext(pd->ppe_id, pd->thread_id, virq);298299if (result)300FAIL("%s:%d: lv1_disconnect_irq_plug_ext failed: %s\n",301__func__, __LINE__, ps3_result(result));302303ps3_virq_destroy(virq);304305return result;306}307EXPORT_SYMBOL_GPL(ps3_irq_plug_destroy);308309/**310* ps3_event_receive_port_setup - Setup an event receive port.311* @cpu: enum ps3_cpu_binding indicating the cpu the interrupt should be312* serviced on.313* @virq: The assigned Linux virq.314*315* The virq can be used with lv1_connect_interrupt_event_receive_port() to316* arrange to receive interrupts from system-bus devices, or with317* ps3_send_event_locally() to signal events.318*/319320int ps3_event_receive_port_setup(enum ps3_cpu_binding cpu, unsigned int *virq)321{322int result;323u64 outlet;324325result = lv1_construct_event_receive_port(&outlet);326327if (result) {328FAIL("%s:%d: lv1_construct_event_receive_port failed: %s\n",329__func__, __LINE__, ps3_result(result));330*virq = 0;331return result;332}333334result = ps3_irq_plug_setup(cpu, outlet, virq);335BUG_ON(result);336337return result;338}339EXPORT_SYMBOL_GPL(ps3_event_receive_port_setup);340341/**342* ps3_event_receive_port_destroy - Destroy an event receive port.343* @virq: The assigned Linux virq.344*345* Since ps3_event_receive_port_destroy destroys the receive port outlet,346* SB devices need to call disconnect_interrupt_event_receive_port() before347* this.348*/349350int ps3_event_receive_port_destroy(unsigned int virq)351{352int result;353354DBG(" -> %s:%d virq %u\n", __func__, __LINE__, virq);355356ps3_chip_mask(irq_get_irq_data(virq));357358result = lv1_destruct_event_receive_port(virq_to_hw(virq));359360if (result)361FAIL("%s:%d: lv1_destruct_event_receive_port failed: %s\n",362__func__, __LINE__, ps3_result(result));363364/*365* Don't call ps3_virq_destroy() here since ps3_smp_cleanup_cpu()366* calls from interrupt context (smp_call_function) when kexecing.367*/368369DBG(" <- %s:%d\n", __func__, __LINE__);370return result;371}372373int ps3_send_event_locally(unsigned int virq)374{375return lv1_send_event_locally(virq_to_hw(virq));376}377378/**379* ps3_sb_event_receive_port_setup - Setup a system bus event receive port.380* @dev: The system bus device instance.381* @cpu: enum ps3_cpu_binding indicating the cpu the interrupt should be382* serviced on.383* @virq: The assigned Linux virq.384*385* An event irq represents a virtual device interrupt. The interrupt_id386* coresponds to the software interrupt number.387*/388389int ps3_sb_event_receive_port_setup(struct ps3_system_bus_device *dev,390enum ps3_cpu_binding cpu, unsigned int *virq)391{392/* this should go in system-bus.c */393394int result;395396result = ps3_event_receive_port_setup(cpu, virq);397398if (result)399return result;400401result = lv1_connect_interrupt_event_receive_port(dev->bus_id,402dev->dev_id, virq_to_hw(*virq), dev->interrupt_id);403404if (result) {405FAIL("%s:%d: lv1_connect_interrupt_event_receive_port"406" failed: %s\n", __func__, __LINE__,407ps3_result(result));408ps3_event_receive_port_destroy(*virq);409*virq = 0;410return result;411}412413DBG("%s:%d: interrupt_id %u, virq %u\n", __func__, __LINE__,414dev->interrupt_id, *virq);415416return 0;417}418EXPORT_SYMBOL(ps3_sb_event_receive_port_setup);419420int ps3_sb_event_receive_port_destroy(struct ps3_system_bus_device *dev,421unsigned int virq)422{423/* this should go in system-bus.c */424425int result;426427DBG(" -> %s:%d: interrupt_id %u, virq %u\n", __func__, __LINE__,428dev->interrupt_id, virq);429430result = lv1_disconnect_interrupt_event_receive_port(dev->bus_id,431dev->dev_id, virq_to_hw(virq), dev->interrupt_id);432433if (result)434FAIL("%s:%d: lv1_disconnect_interrupt_event_receive_port"435" failed: %s\n", __func__, __LINE__,436ps3_result(result));437438result = ps3_event_receive_port_destroy(virq);439BUG_ON(result);440441/*442* ps3_event_receive_port_destroy() destroys the IRQ plug,443* so don't call ps3_irq_plug_destroy() here.444*/445446result = ps3_virq_destroy(virq);447BUG_ON(result);448449DBG(" <- %s:%d\n", __func__, __LINE__);450return result;451}452EXPORT_SYMBOL(ps3_sb_event_receive_port_destroy);453454/**455* ps3_io_irq_setup - Setup a system bus io irq.456* @cpu: enum ps3_cpu_binding indicating the cpu the interrupt should be457* serviced on.458* @interrupt_id: The device interrupt id read from the system repository.459* @virq: The assigned Linux virq.460*461* An io irq represents a non-virtualized device interrupt. interrupt_id462* coresponds to the interrupt number of the interrupt controller.463*/464465int ps3_io_irq_setup(enum ps3_cpu_binding cpu, unsigned int interrupt_id,466unsigned int *virq)467{468int result;469u64 outlet;470471result = lv1_construct_io_irq_outlet(interrupt_id, &outlet);472473if (result) {474FAIL("%s:%d: lv1_construct_io_irq_outlet failed: %s\n",475__func__, __LINE__, ps3_result(result));476return result;477}478479result = ps3_irq_plug_setup(cpu, outlet, virq);480BUG_ON(result);481482return result;483}484EXPORT_SYMBOL_GPL(ps3_io_irq_setup);485486int ps3_io_irq_destroy(unsigned int virq)487{488int result;489unsigned long outlet = virq_to_hw(virq);490491ps3_chip_mask(irq_get_irq_data(virq));492493/*494* lv1_destruct_io_irq_outlet() will destroy the IRQ plug,495* so call ps3_irq_plug_destroy() first.496*/497498result = ps3_irq_plug_destroy(virq);499BUG_ON(result);500501result = lv1_destruct_io_irq_outlet(outlet);502503if (result)504FAIL("%s:%d: lv1_destruct_io_irq_outlet failed: %s\n",505__func__, __LINE__, ps3_result(result));506507return result;508}509EXPORT_SYMBOL_GPL(ps3_io_irq_destroy);510511/**512* ps3_vuart_irq_setup - Setup the system virtual uart virq.513* @cpu: enum ps3_cpu_binding indicating the cpu the interrupt should be514* serviced on.515* @virt_addr_bmp: The caller supplied virtual uart interrupt bitmap.516* @virq: The assigned Linux virq.517*518* The system supports only a single virtual uart, so multiple calls without519* freeing the interrupt will return a wrong state error.520*/521522int ps3_vuart_irq_setup(enum ps3_cpu_binding cpu, void* virt_addr_bmp,523unsigned int *virq)524{525int result;526u64 outlet;527u64 lpar_addr;528529BUG_ON(!is_kernel_addr((u64)virt_addr_bmp));530531lpar_addr = ps3_mm_phys_to_lpar(__pa(virt_addr_bmp));532533result = lv1_configure_virtual_uart_irq(lpar_addr, &outlet);534535if (result) {536FAIL("%s:%d: lv1_configure_virtual_uart_irq failed: %s\n",537__func__, __LINE__, ps3_result(result));538return result;539}540541result = ps3_irq_plug_setup(cpu, outlet, virq);542BUG_ON(result);543544return result;545}546EXPORT_SYMBOL_GPL(ps3_vuart_irq_setup);547548int ps3_vuart_irq_destroy(unsigned int virq)549{550int result;551552ps3_chip_mask(irq_get_irq_data(virq));553result = lv1_deconfigure_virtual_uart_irq();554555if (result) {556FAIL("%s:%d: lv1_configure_virtual_uart_irq failed: %s\n",557__func__, __LINE__, ps3_result(result));558return result;559}560561result = ps3_irq_plug_destroy(virq);562BUG_ON(result);563564return result;565}566EXPORT_SYMBOL_GPL(ps3_vuart_irq_destroy);567568/**569* ps3_spe_irq_setup - Setup an spe virq.570* @cpu: enum ps3_cpu_binding indicating the cpu the interrupt should be571* serviced on.572* @spe_id: The spe_id returned from lv1_construct_logical_spe().573* @class: The spe interrupt class {0,1,2}.574* @virq: The assigned Linux virq.575*576*/577578int ps3_spe_irq_setup(enum ps3_cpu_binding cpu, unsigned long spe_id,579unsigned int class, unsigned int *virq)580{581int result;582u64 outlet;583584BUG_ON(class > 2);585586result = lv1_get_spe_irq_outlet(spe_id, class, &outlet);587588if (result) {589FAIL("%s:%d: lv1_get_spe_irq_outlet failed: %s\n",590__func__, __LINE__, ps3_result(result));591return result;592}593594result = ps3_irq_plug_setup(cpu, outlet, virq);595BUG_ON(result);596597return result;598}599600int ps3_spe_irq_destroy(unsigned int virq)601{602int result;603604ps3_chip_mask(irq_get_irq_data(virq));605606result = ps3_irq_plug_destroy(virq);607BUG_ON(result);608609return result;610}611612613#define PS3_INVALID_OUTLET ((irq_hw_number_t)-1)614#define PS3_PLUG_MAX 63615616#if defined(DEBUG)617static void _dump_64_bmp(const char *header, const u64 *p, unsigned cpu,618const char* func, int line)619{620pr_debug("%s:%d: %s %u {%04llx_%04llx_%04llx_%04llx}\n",621func, line, header, cpu,622*p >> 48, (*p >> 32) & 0xffff, (*p >> 16) & 0xffff,623*p & 0xffff);624}625626static void __maybe_unused _dump_256_bmp(const char *header,627const u64 *p, unsigned cpu, const char* func, int line)628{629pr_debug("%s:%d: %s %u {%016llx:%016llx:%016llx:%016llx}\n",630func, line, header, cpu, p[0], p[1], p[2], p[3]);631}632633#define dump_bmp(_x) _dump_bmp(_x, __func__, __LINE__)634static void _dump_bmp(struct ps3_private* pd, const char* func, int line)635{636unsigned long flags;637638spin_lock_irqsave(&pd->bmp_lock, flags);639_dump_64_bmp("stat", &pd->bmp.status, pd->thread_id, func, line);640_dump_64_bmp("mask", (u64*)&pd->bmp.mask, pd->thread_id, func, line);641spin_unlock_irqrestore(&pd->bmp_lock, flags);642}643644#define dump_mask(_x) _dump_mask(_x, __func__, __LINE__)645static void __maybe_unused _dump_mask(struct ps3_private *pd,646const char* func, int line)647{648unsigned long flags;649650spin_lock_irqsave(&pd->bmp_lock, flags);651_dump_64_bmp("mask", (u64*)&pd->bmp.mask, pd->thread_id, func, line);652spin_unlock_irqrestore(&pd->bmp_lock, flags);653}654#else655static void dump_bmp(struct ps3_private* pd) {};656#endif /* defined(DEBUG) */657658static int ps3_host_map(struct irq_domain *h, unsigned int virq,659irq_hw_number_t hwirq)660{661DBG("%s:%d: hwirq %lu, virq %u\n", __func__, __LINE__, hwirq,662virq);663664irq_set_chip_and_handler(virq, &ps3_irq_chip, handle_fasteoi_irq);665666return 0;667}668669static int ps3_host_match(struct irq_domain *h, struct device_node *np,670enum irq_domain_bus_token bus_token)671{672/* Match all */673return 1;674}675676static const struct irq_domain_ops ps3_host_ops = {677.map = ps3_host_map,678.match = ps3_host_match,679};680681void __init ps3_register_ipi_debug_brk(unsigned int cpu, unsigned int virq)682{683struct ps3_private *pd = &per_cpu(ps3_private, cpu);684685set_bit(63 - virq, &pd->ipi_debug_brk_mask);686687DBG("%s:%d: cpu %u, virq %u, mask %lxh\n", __func__, __LINE__,688cpu, virq, pd->ipi_debug_brk_mask);689}690691void __init ps3_register_ipi_irq(unsigned int cpu, unsigned int virq)692{693struct ps3_private *pd = &per_cpu(ps3_private, cpu);694695set_bit(63 - virq, &pd->ipi_mask);696697DBG("%s:%d: cpu %u, virq %u, ipi_mask %lxh\n", __func__, __LINE__,698cpu, virq, pd->ipi_mask);699}700701static unsigned int ps3_get_irq(void)702{703struct ps3_private *pd = this_cpu_ptr(&ps3_private);704u64 x = (pd->bmp.status & pd->bmp.mask);705unsigned int plug;706707/* check for ipi break first to stop this cpu ASAP */708709if (x & pd->ipi_debug_brk_mask)710x &= pd->ipi_debug_brk_mask;711712asm volatile("cntlzd %0,%1" : "=r" (plug) : "r" (x));713plug &= 0x3f;714715if (unlikely(!plug)) {716DBG("%s:%d: no plug found: thread_id %llu\n", __func__,717__LINE__, pd->thread_id);718dump_bmp(&per_cpu(ps3_private, 0));719dump_bmp(&per_cpu(ps3_private, 1));720return 0;721}722723#if defined(DEBUG)724if (unlikely(plug < NR_IRQS_LEGACY || plug > PS3_PLUG_MAX)) {725dump_bmp(&per_cpu(ps3_private, 0));726dump_bmp(&per_cpu(ps3_private, 1));727BUG();728}729#endif730731/* IPIs are EOIed here. */732733if (test_bit(63 - plug, &pd->ipi_mask))734lv1_end_of_interrupt_ext(pd->ppe_id, pd->thread_id, plug);735736return plug;737}738739void __init ps3_init_IRQ(void)740{741int result;742unsigned cpu;743struct irq_domain *host;744745host = irq_domain_create_nomap(NULL, PS3_PLUG_MAX + 1, &ps3_host_ops, NULL);746irq_set_default_domain(host);747748for_each_possible_cpu(cpu) {749struct ps3_private *pd = &per_cpu(ps3_private, cpu);750751lv1_get_logical_ppe_id(&pd->ppe_id);752pd->thread_id = get_hard_smp_processor_id(cpu);753spin_lock_init(&pd->bmp_lock);754755DBG("%s:%d: ppe_id %llu, thread_id %llu, bmp %lxh\n",756__func__, __LINE__, pd->ppe_id, pd->thread_id,757ps3_mm_phys_to_lpar(__pa(&pd->bmp)));758759result = lv1_configure_irq_state_bitmap(pd->ppe_id,760pd->thread_id, ps3_mm_phys_to_lpar(__pa(&pd->bmp)));761762if (result)763FAIL("%s:%d: lv1_configure_irq_state_bitmap failed:"764" %s\n", __func__, __LINE__,765ps3_result(result));766}767768ppc_md.get_irq = ps3_get_irq;769}770771void ps3_shutdown_IRQ(int cpu)772{773int result;774u64 ppe_id;775u64 thread_id = get_hard_smp_processor_id(cpu);776777lv1_get_logical_ppe_id(&ppe_id);778result = lv1_configure_irq_state_bitmap(ppe_id, thread_id, 0);779780DBG("%s:%d: lv1_configure_irq_state_bitmap (%llu:%llu/%d) %s\n", __func__,781__LINE__, ppe_id, thread_id, cpu, ps3_result(result));782}783784785