Path: blob/master/drivers/crypto/cavium/nitrox/nitrox_hal.c
26285 views
// SPDX-License-Identifier: GPL-2.01#include <linux/delay.h>23#include "nitrox_dev.h"4#include "nitrox_csr.h"5#include "nitrox_hal.h"67#define PLL_REF_CLK 508#define MAX_CSR_RETRIES 10910/**11* emu_enable_cores - Enable EMU cluster cores.12* @ndev: NITROX device13*/14static void emu_enable_cores(struct nitrox_device *ndev)15{16union emu_se_enable emu_se;17union emu_ae_enable emu_ae;18int i;1920/* AE cores 20 per cluster */21emu_ae.value = 0;22emu_ae.s.enable = 0xfffff;2324/* SE cores 16 per cluster */25emu_se.value = 0;26emu_se.s.enable = 0xffff;2728/* enable per cluster cores */29for (i = 0; i < NR_CLUSTERS; i++) {30nitrox_write_csr(ndev, EMU_AE_ENABLEX(i), emu_ae.value);31nitrox_write_csr(ndev, EMU_SE_ENABLEX(i), emu_se.value);32}33}3435/**36* nitrox_config_emu_unit - configure EMU unit.37* @ndev: NITROX device38*/39void nitrox_config_emu_unit(struct nitrox_device *ndev)40{41union emu_wd_int_ena_w1s emu_wd_int;42union emu_ge_int_ena_w1s emu_ge_int;43u64 offset;44int i;4546/* enable cores */47emu_enable_cores(ndev);4849/* enable general error and watch dog interrupts */50emu_ge_int.value = 0;51emu_ge_int.s.se_ge = 0xffff;52emu_ge_int.s.ae_ge = 0xfffff;53emu_wd_int.value = 0;54emu_wd_int.s.se_wd = 1;5556for (i = 0; i < NR_CLUSTERS; i++) {57offset = EMU_WD_INT_ENA_W1SX(i);58nitrox_write_csr(ndev, offset, emu_wd_int.value);59offset = EMU_GE_INT_ENA_W1SX(i);60nitrox_write_csr(ndev, offset, emu_ge_int.value);61}62}6364static void reset_pkt_input_ring(struct nitrox_device *ndev, int ring)65{66union nps_pkt_in_instr_ctl pkt_in_ctl;67union nps_pkt_in_done_cnts pkt_in_cnts;68int max_retries = MAX_CSR_RETRIES;69u64 offset;7071/* step 1: disable the ring, clear enable bit */72offset = NPS_PKT_IN_INSTR_CTLX(ring);73pkt_in_ctl.value = nitrox_read_csr(ndev, offset);74pkt_in_ctl.s.enb = 0;75nitrox_write_csr(ndev, offset, pkt_in_ctl.value);7677/* step 2: wait to clear [ENB] */78usleep_range(100, 150);79do {80pkt_in_ctl.value = nitrox_read_csr(ndev, offset);81if (!pkt_in_ctl.s.enb)82break;83udelay(50);84} while (max_retries--);8586/* step 3: clear done counts */87offset = NPS_PKT_IN_DONE_CNTSX(ring);88pkt_in_cnts.value = nitrox_read_csr(ndev, offset);89nitrox_write_csr(ndev, offset, pkt_in_cnts.value);90usleep_range(50, 100);91}9293void enable_pkt_input_ring(struct nitrox_device *ndev, int ring)94{95union nps_pkt_in_instr_ctl pkt_in_ctl;96int max_retries = MAX_CSR_RETRIES;97u64 offset;9899/* 64-byte instruction size */100offset = NPS_PKT_IN_INSTR_CTLX(ring);101pkt_in_ctl.value = nitrox_read_csr(ndev, offset);102pkt_in_ctl.s.is64b = 1;103pkt_in_ctl.s.enb = 1;104nitrox_write_csr(ndev, offset, pkt_in_ctl.value);105106/* wait for set [ENB] */107do {108pkt_in_ctl.value = nitrox_read_csr(ndev, offset);109if (pkt_in_ctl.s.enb)110break;111udelay(50);112} while (max_retries--);113}114115/**116* nitrox_config_pkt_input_rings - configure Packet Input Rings117* @ndev: NITROX device118*/119void nitrox_config_pkt_input_rings(struct nitrox_device *ndev)120{121int i;122123for (i = 0; i < ndev->nr_queues; i++) {124struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i];125union nps_pkt_in_instr_rsize pkt_in_rsize;126union nps_pkt_in_instr_baoff_dbell pkt_in_dbell;127u64 offset;128129reset_pkt_input_ring(ndev, i);130131/**132* step 4:133* configure ring base address 16-byte aligned,134* size and interrupt threshold.135*/136offset = NPS_PKT_IN_INSTR_BADDRX(i);137nitrox_write_csr(ndev, offset, cmdq->dma);138139/* configure ring size */140offset = NPS_PKT_IN_INSTR_RSIZEX(i);141pkt_in_rsize.value = 0;142pkt_in_rsize.s.rsize = ndev->qlen;143nitrox_write_csr(ndev, offset, pkt_in_rsize.value);144145/* set high threshold for pkt input ring interrupts */146offset = NPS_PKT_IN_INT_LEVELSX(i);147nitrox_write_csr(ndev, offset, 0xffffffff);148149/* step 5: clear off door bell counts */150offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i);151pkt_in_dbell.value = 0;152pkt_in_dbell.s.dbell = 0xffffffff;153nitrox_write_csr(ndev, offset, pkt_in_dbell.value);154155/* enable the ring */156enable_pkt_input_ring(ndev, i);157}158}159160static void reset_pkt_solicit_port(struct nitrox_device *ndev, int port)161{162union nps_pkt_slc_ctl pkt_slc_ctl;163union nps_pkt_slc_cnts pkt_slc_cnts;164int max_retries = MAX_CSR_RETRIES;165u64 offset;166167/* step 1: disable slc port */168offset = NPS_PKT_SLC_CTLX(port);169pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);170pkt_slc_ctl.s.enb = 0;171nitrox_write_csr(ndev, offset, pkt_slc_ctl.value);172173/* step 2 */174usleep_range(100, 150);175/* wait to clear [ENB] */176do {177pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);178if (!pkt_slc_ctl.s.enb)179break;180udelay(50);181} while (max_retries--);182183/* step 3: clear slc counters */184offset = NPS_PKT_SLC_CNTSX(port);185pkt_slc_cnts.value = nitrox_read_csr(ndev, offset);186nitrox_write_csr(ndev, offset, pkt_slc_cnts.value);187usleep_range(50, 100);188}189190void enable_pkt_solicit_port(struct nitrox_device *ndev, int port)191{192union nps_pkt_slc_ctl pkt_slc_ctl;193int max_retries = MAX_CSR_RETRIES;194u64 offset;195196offset = NPS_PKT_SLC_CTLX(port);197pkt_slc_ctl.value = 0;198pkt_slc_ctl.s.enb = 1;199/*200* 8 trailing 0x00 bytes will be added201* to the end of the outgoing packet.202*/203pkt_slc_ctl.s.z = 1;204/* enable response header */205pkt_slc_ctl.s.rh = 1;206nitrox_write_csr(ndev, offset, pkt_slc_ctl.value);207208/* wait to set [ENB] */209do {210pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);211if (pkt_slc_ctl.s.enb)212break;213udelay(50);214} while (max_retries--);215}216217static void config_pkt_solicit_port(struct nitrox_device *ndev, int port)218{219union nps_pkt_slc_int_levels pkt_slc_int;220u64 offset;221222reset_pkt_solicit_port(ndev, port);223224/* step 4: configure interrupt levels */225offset = NPS_PKT_SLC_INT_LEVELSX(port);226pkt_slc_int.value = 0;227/* time interrupt threshold */228pkt_slc_int.s.timet = 0x3fffff;229nitrox_write_csr(ndev, offset, pkt_slc_int.value);230231/* enable the solicit port */232enable_pkt_solicit_port(ndev, port);233}234235void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev)236{237int i;238239for (i = 0; i < ndev->nr_queues; i++)240config_pkt_solicit_port(ndev, i);241}242243/**244* enable_nps_core_interrupts - enable NPS core interrutps245* @ndev: NITROX device.246*247* This includes NPS core interrupts.248*/249static void enable_nps_core_interrupts(struct nitrox_device *ndev)250{251union nps_core_int_ena_w1s core_int;252253/* NPS core interrutps */254core_int.value = 0;255core_int.s.host_wr_err = 1;256core_int.s.host_wr_timeout = 1;257core_int.s.exec_wr_timeout = 1;258core_int.s.npco_dma_malform = 1;259core_int.s.host_nps_wr_err = 1;260nitrox_write_csr(ndev, NPS_CORE_INT_ENA_W1S, core_int.value);261}262263void nitrox_config_nps_core_unit(struct nitrox_device *ndev)264{265union nps_core_gbl_vfcfg core_gbl_vfcfg;266267/* endian control information */268nitrox_write_csr(ndev, NPS_CORE_CONTROL, 1ULL);269270/* disable ILK interface */271core_gbl_vfcfg.value = 0;272core_gbl_vfcfg.s.ilk_disable = 1;273core_gbl_vfcfg.s.cfg = __NDEV_MODE_PF;274nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value);275276/* enable nps core interrupts */277enable_nps_core_interrupts(ndev);278}279280/**281* enable_nps_pkt_interrupts - enable NPS packet interrutps282* @ndev: NITROX device.283*284* This includes NPS packet in and slc interrupts.285*/286static void enable_nps_pkt_interrupts(struct nitrox_device *ndev)287{288/* NPS packet in ring interrupts */289nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL));290nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL));291nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL));292/* NPS packet slc port interrupts */293nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL));294nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL));295nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL));296}297298void nitrox_config_nps_pkt_unit(struct nitrox_device *ndev)299{300/* config input and solicit ports */301nitrox_config_pkt_input_rings(ndev);302nitrox_config_pkt_solicit_ports(ndev);303304/* enable nps packet interrupts */305enable_nps_pkt_interrupts(ndev);306}307308static void reset_aqm_ring(struct nitrox_device *ndev, int ring)309{310union aqmq_en aqmq_en_reg;311union aqmq_activity_stat activity_stat;312union aqmq_cmp_cnt cmp_cnt;313int max_retries = MAX_CSR_RETRIES;314u64 offset;315316/* step 1: disable the queue */317offset = AQMQ_ENX(ring);318aqmq_en_reg.value = 0;319aqmq_en_reg.queue_enable = 0;320nitrox_write_csr(ndev, offset, aqmq_en_reg.value);321322/* step 2: wait for AQMQ_ACTIVITY_STATX[QUEUE_ACTIVE] to clear */323usleep_range(100, 150);324offset = AQMQ_ACTIVITY_STATX(ring);325do {326activity_stat.value = nitrox_read_csr(ndev, offset);327if (!activity_stat.queue_active)328break;329udelay(50);330} while (max_retries--);331332/* step 3: clear commands completed count */333offset = AQMQ_CMP_CNTX(ring);334cmp_cnt.value = nitrox_read_csr(ndev, offset);335nitrox_write_csr(ndev, offset, cmp_cnt.value);336usleep_range(50, 100);337}338339void enable_aqm_ring(struct nitrox_device *ndev, int ring)340{341union aqmq_en aqmq_en_reg;342u64 offset;343344offset = AQMQ_ENX(ring);345aqmq_en_reg.value = 0;346aqmq_en_reg.queue_enable = 1;347nitrox_write_csr(ndev, offset, aqmq_en_reg.value);348usleep_range(50, 100);349}350351void nitrox_config_aqm_rings(struct nitrox_device *ndev)352{353int ring;354355for (ring = 0; ring < ndev->nr_queues; ring++) {356struct nitrox_cmdq *cmdq = ndev->aqmq[ring];357union aqmq_drbl drbl;358union aqmq_qsz qsize;359union aqmq_cmp_thr cmp_thr;360u64 offset;361362/* steps 1 - 3 */363reset_aqm_ring(ndev, ring);364365/* step 4: clear doorbell count of ring */366offset = AQMQ_DRBLX(ring);367drbl.value = 0;368drbl.dbell_count = 0xFFFFFFFF;369nitrox_write_csr(ndev, offset, drbl.value);370371/* step 5: configure host ring details */372373/* set host address for next command of ring */374offset = AQMQ_NXT_CMDX(ring);375nitrox_write_csr(ndev, offset, 0ULL);376377/* set host address of ring base */378offset = AQMQ_BADRX(ring);379nitrox_write_csr(ndev, offset, cmdq->dma);380381/* set ring size */382offset = AQMQ_QSZX(ring);383qsize.value = 0;384qsize.host_queue_size = ndev->qlen;385nitrox_write_csr(ndev, offset, qsize.value);386387/* set command completion threshold */388offset = AQMQ_CMP_THRX(ring);389cmp_thr.value = 0;390cmp_thr.commands_completed_threshold = 1;391nitrox_write_csr(ndev, offset, cmp_thr.value);392393/* step 6: enable the queue */394enable_aqm_ring(ndev, ring);395}396}397398static void enable_aqm_interrupts(struct nitrox_device *ndev)399{400/* clear interrupt enable bits */401nitrox_write_csr(ndev, AQM_DBELL_OVF_LO_ENA_W1S, (~0ULL));402nitrox_write_csr(ndev, AQM_DBELL_OVF_HI_ENA_W1S, (~0ULL));403nitrox_write_csr(ndev, AQM_DMA_RD_ERR_LO_ENA_W1S, (~0ULL));404nitrox_write_csr(ndev, AQM_DMA_RD_ERR_HI_ENA_W1S, (~0ULL));405nitrox_write_csr(ndev, AQM_EXEC_NA_LO_ENA_W1S, (~0ULL));406nitrox_write_csr(ndev, AQM_EXEC_NA_HI_ENA_W1S, (~0ULL));407nitrox_write_csr(ndev, AQM_EXEC_ERR_LO_ENA_W1S, (~0ULL));408nitrox_write_csr(ndev, AQM_EXEC_ERR_HI_ENA_W1S, (~0ULL));409}410411void nitrox_config_aqm_unit(struct nitrox_device *ndev)412{413/* config aqm command queues */414nitrox_config_aqm_rings(ndev);415416/* enable aqm interrupts */417enable_aqm_interrupts(ndev);418}419420void nitrox_config_pom_unit(struct nitrox_device *ndev)421{422union pom_int_ena_w1s pom_int;423int i;424425/* enable pom interrupts */426pom_int.value = 0;427pom_int.s.illegal_dport = 1;428nitrox_write_csr(ndev, POM_INT_ENA_W1S, pom_int.value);429430/* enable perf counters */431for (i = 0; i < ndev->hw.se_cores; i++)432nitrox_write_csr(ndev, POM_PERF_CTL, BIT_ULL(i));433}434435/**436* nitrox_config_rand_unit - enable NITROX random number unit437* @ndev: NITROX device438*/439void nitrox_config_rand_unit(struct nitrox_device *ndev)440{441union efl_rnm_ctl_status efl_rnm_ctl;442u64 offset;443444offset = EFL_RNM_CTL_STATUS;445efl_rnm_ctl.value = nitrox_read_csr(ndev, offset);446efl_rnm_ctl.s.ent_en = 1;447efl_rnm_ctl.s.rng_en = 1;448nitrox_write_csr(ndev, offset, efl_rnm_ctl.value);449}450451void nitrox_config_efl_unit(struct nitrox_device *ndev)452{453int i;454455for (i = 0; i < NR_CLUSTERS; i++) {456union efl_core_int_ena_w1s efl_core_int;457u64 offset;458459/* EFL core interrupts */460offset = EFL_CORE_INT_ENA_W1SX(i);461efl_core_int.value = 0;462efl_core_int.s.len_ovr = 1;463efl_core_int.s.d_left = 1;464efl_core_int.s.epci_decode_err = 1;465nitrox_write_csr(ndev, offset, efl_core_int.value);466467offset = EFL_CORE_VF_ERR_INT0_ENA_W1SX(i);468nitrox_write_csr(ndev, offset, (~0ULL));469offset = EFL_CORE_VF_ERR_INT1_ENA_W1SX(i);470nitrox_write_csr(ndev, offset, (~0ULL));471}472}473474void nitrox_config_bmi_unit(struct nitrox_device *ndev)475{476union bmi_ctl bmi_ctl;477union bmi_int_ena_w1s bmi_int_ena;478u64 offset;479480/* no threshold limits for PCIe */481offset = BMI_CTL;482bmi_ctl.value = nitrox_read_csr(ndev, offset);483bmi_ctl.s.max_pkt_len = 0xff;484bmi_ctl.s.nps_free_thrsh = 0xff;485bmi_ctl.s.nps_hdrq_thrsh = 0x7a;486nitrox_write_csr(ndev, offset, bmi_ctl.value);487488/* enable interrupts */489offset = BMI_INT_ENA_W1S;490bmi_int_ena.value = 0;491bmi_int_ena.s.max_len_err_nps = 1;492bmi_int_ena.s.pkt_rcv_err_nps = 1;493bmi_int_ena.s.fpf_undrrn = 1;494nitrox_write_csr(ndev, offset, bmi_int_ena.value);495}496497void nitrox_config_bmo_unit(struct nitrox_device *ndev)498{499union bmo_ctl2 bmo_ctl2;500u64 offset;501502/* no threshold limits for PCIe */503offset = BMO_CTL2;504bmo_ctl2.value = nitrox_read_csr(ndev, offset);505bmo_ctl2.s.nps_slc_buf_thrsh = 0xff;506nitrox_write_csr(ndev, offset, bmo_ctl2.value);507}508509void invalidate_lbc(struct nitrox_device *ndev)510{511union lbc_inval_ctl lbc_ctl;512union lbc_inval_status lbc_stat;513int max_retries = MAX_CSR_RETRIES;514u64 offset;515516/* invalidate LBC */517offset = LBC_INVAL_CTL;518lbc_ctl.value = nitrox_read_csr(ndev, offset);519lbc_ctl.s.cam_inval_start = 1;520nitrox_write_csr(ndev, offset, lbc_ctl.value);521522offset = LBC_INVAL_STATUS;523do {524lbc_stat.value = nitrox_read_csr(ndev, offset);525if (lbc_stat.s.done)526break;527udelay(50);528} while (max_retries--);529}530531void nitrox_config_lbc_unit(struct nitrox_device *ndev)532{533union lbc_int_ena_w1s lbc_int_ena;534u64 offset;535536invalidate_lbc(ndev);537538/* enable interrupts */539offset = LBC_INT_ENA_W1S;540lbc_int_ena.value = 0;541lbc_int_ena.s.dma_rd_err = 1;542lbc_int_ena.s.over_fetch_err = 1;543lbc_int_ena.s.cam_inval_abort = 1;544lbc_int_ena.s.cam_hard_err = 1;545nitrox_write_csr(ndev, offset, lbc_int_ena.value);546547offset = LBC_PLM_VF1_64_INT_ENA_W1S;548nitrox_write_csr(ndev, offset, (~0ULL));549offset = LBC_PLM_VF65_128_INT_ENA_W1S;550nitrox_write_csr(ndev, offset, (~0ULL));551552offset = LBC_ELM_VF1_64_INT_ENA_W1S;553nitrox_write_csr(ndev, offset, (~0ULL));554offset = LBC_ELM_VF65_128_INT_ENA_W1S;555nitrox_write_csr(ndev, offset, (~0ULL));556}557558void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode)559{560union nps_core_gbl_vfcfg vfcfg;561562vfcfg.value = nitrox_read_csr(ndev, NPS_CORE_GBL_VFCFG);563vfcfg.s.cfg = mode & 0x7;564565nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, vfcfg.value);566}567568static const char *get_core_option(u8 se_cores, u8 ae_cores)569{570const char *option = "";571572if (ae_cores == AE_MAX_CORES) {573switch (se_cores) {574case SE_MAX_CORES:575option = "60";576break;577case 40:578option = "60s";579break;580}581} else if (ae_cores == (AE_MAX_CORES / 2)) {582option = "30";583} else {584option = "60i";585}586587return option;588}589590static const char *get_feature_option(u8 zip_cores, int core_freq)591{592if (zip_cores == 0)593return "";594else if (zip_cores < ZIP_MAX_CORES)595return "-C15";596597if (core_freq >= 850)598return "-C45";599else if (core_freq >= 750)600return "-C35";601else if (core_freq >= 550)602return "-C25";603604return "";605}606607void nitrox_get_hwinfo(struct nitrox_device *ndev)608{609union emu_fuse_map emu_fuse;610union rst_boot rst_boot;611union fus_dat1 fus_dat1;612unsigned char name[IFNAMSIZ * 2] = {};613int i, dead_cores;614u64 offset;615616/* get core frequency */617offset = RST_BOOT;618rst_boot.value = nitrox_read_csr(ndev, offset);619ndev->hw.freq = (rst_boot.pnr_mul + 3) * PLL_REF_CLK;620621for (i = 0; i < NR_CLUSTERS; i++) {622offset = EMU_FUSE_MAPX(i);623emu_fuse.value = nitrox_read_csr(ndev, offset);624if (emu_fuse.s.valid) {625dead_cores = hweight32(emu_fuse.s.ae_fuse);626ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;627dead_cores = hweight16(emu_fuse.s.se_fuse);628ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;629}630}631/* find zip hardware availability */632offset = FUS_DAT1;633fus_dat1.value = nitrox_read_csr(ndev, offset);634if (!fus_dat1.nozip) {635dead_cores = hweight8(fus_dat1.zip_info);636ndev->hw.zip_cores = ZIP_MAX_CORES - dead_cores;637}638639/* determine the partname640* CNN55<core option>-<freq><pincount>-<feature option>-<rev>641*/642snprintf(name, sizeof(name), "CNN55%s-%3dBG676%s-1.%u",643get_core_option(ndev->hw.se_cores, ndev->hw.ae_cores),644ndev->hw.freq,645get_feature_option(ndev->hw.zip_cores, ndev->hw.freq),646ndev->hw.revision_id);647648/* copy partname */649strscpy(ndev->hw.partname, name, sizeof(ndev->hw.partname));650}651652void enable_pf2vf_mbox_interrupts(struct nitrox_device *ndev)653{654u64 value = ~0ULL;655u64 reg_addr;656657/* Mailbox interrupt low enable set register */658reg_addr = NPS_PKT_MBOX_INT_LO_ENA_W1S;659nitrox_write_csr(ndev, reg_addr, value);660661/* Mailbox interrupt high enable set register */662reg_addr = NPS_PKT_MBOX_INT_HI_ENA_W1S;663nitrox_write_csr(ndev, reg_addr, value);664}665666void disable_pf2vf_mbox_interrupts(struct nitrox_device *ndev)667{668u64 value = ~0ULL;669u64 reg_addr;670671/* Mailbox interrupt low enable clear register */672reg_addr = NPS_PKT_MBOX_INT_LO_ENA_W1C;673nitrox_write_csr(ndev, reg_addr, value);674675/* Mailbox interrupt high enable clear register */676reg_addr = NPS_PKT_MBOX_INT_HI_ENA_W1C;677nitrox_write_csr(ndev, reg_addr, value);678}679680681