otx2_cptpf_main.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2020 Marvell. */

#include <linux/firmware.h>
#include <linux/sysfs.h>
#include "otx2_cpt_hw_types.h"
#include "otx2_cpt_common.h"
#include "otx2_cpt_devlink.h"
#include "otx2_cptpf_ucode.h"
#include "otx2_cptpf.h"
#include "cn10k_cpt.h"
#include "rvu_reg.h"

#define OTX2_CPT_DRV_NAME    "rvu_cptpf"
#define OTX2_CPT_DRV_STRING  "Marvell RVU CPT Physical Function Driver"

#define CPT_UC_RID_CN9K_B0   1
#define CPT_UC_RID_CN10K_A   4
#define CPT_UC_RID_CN10K_B   5

static void cptpf_enable_vfpf_mbox_intr(struct otx2_cptpf_dev *cptpf,
					int num_vfs)
{
	int ena_bits;

	/* Clear any pending interrupts */
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFPF_MBOX_INTX(0), ~0x0ULL);
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFPF_MBOX_INTX(1), ~0x0ULL);

	/* Enable VF interrupts for VFs from 0 to 63 */
	ena_bits = ((num_vfs - 1) % 64);
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0),
			 GENMASK_ULL(ena_bits, 0));

	if (num_vfs > 64) {
		/* Enable VF interrupts for VFs from 64 to 127 */
		ena_bits = num_vfs - 64 - 1;
		otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
				RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1),
				GENMASK_ULL(ena_bits, 0));
	}
}

static void cptpf_disable_vfpf_mbox_intr(struct otx2_cptpf_dev *cptpf,
					 int num_vfs)
{
	int vector;

	/* Disable VF-PF interrupts */
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), ~0ULL);
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1), ~0ULL);
	/* Clear any pending interrupts */
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFPF_MBOX_INTX(0), ~0ULL);

	vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0);
	free_irq(vector, cptpf);

	if (num_vfs > 64) {
		otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
				 RVU_PF_VFPF_MBOX_INTX(1), ~0ULL);
		vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFPF_MBOX1);
		free_irq(vector, cptpf);
	}
}

static void cptpf_enable_vf_flr_me_intrs(struct otx2_cptpf_dev *cptpf,
					 int num_vfs)
{
	/* Clear FLR interrupt if any */
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(0),
			 INTR_MASK(num_vfs));

	/* Enable VF FLR interrupts */
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(num_vfs));
	/* Clear ME interrupt if any */
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFME_INTX(0),
			 INTR_MASK(num_vfs));
	/* Enable VF ME interrupts */
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(num_vfs));

	if (num_vfs <= 64)
		return;

	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(1),
			 INTR_MASK(num_vfs - 64));
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFFLR_INT_ENA_W1SX(1), INTR_MASK(num_vfs - 64));

	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFME_INTX(1),
			 INTR_MASK(num_vfs - 64));
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFME_INT_ENA_W1SX(1), INTR_MASK(num_vfs - 64));
}

static void cptpf_disable_vf_flr_me_intrs(struct otx2_cptpf_dev *cptpf,
				       int num_vfs)
{
	int vector;

	/* Disable VF FLR interrupts */
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(num_vfs));
	vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR0);
	free_irq(vector, cptpf);

	/* Disable VF ME interrupts */
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(num_vfs));
	vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFME0);
	free_irq(vector, cptpf);

	if (num_vfs <= 64)
		return;

	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(num_vfs - 64));
	vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR1);
	free_irq(vector, cptpf);

	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
			 RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(num_vfs - 64));
	vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFME1);
	free_irq(vector, cptpf);
}

static void cptpf_flr_wq_handler(struct work_struct *work)
{
	struct cptpf_flr_work *flr_work;
	struct otx2_cptpf_dev *pf;
	struct mbox_msghdr *req;
	struct otx2_mbox *mbox;
	int vf, reg = 0;

	flr_work = container_of(work, struct cptpf_flr_work, work);
	pf = flr_work->pf;
	mbox = &pf->afpf_mbox;

	vf = flr_work - pf->flr_work;

	mutex_lock(&pf->lock);
	req = otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req),
				      sizeof(struct msg_rsp));
	if (!req) {
		mutex_unlock(&pf->lock);
		return;
	}

	req->sig = OTX2_MBOX_REQ_SIG;
	req->id = MBOX_MSG_VF_FLR;
	req->pcifunc &= RVU_PFVF_FUNC_MASK;
	req->pcifunc |= (vf + 1) & RVU_PFVF_FUNC_MASK;

	otx2_cpt_send_mbox_msg(mbox, pf->pdev);
	if (!otx2_cpt_sync_mbox_msg(&pf->afpf_mbox)) {

		if (vf >= 64) {
			reg = 1;
			vf = vf - 64;
		}
		/* Clear transaction pending register */
		otx2_cpt_write64(pf->reg_base, BLKADDR_RVUM, 0,
				 RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
		otx2_cpt_write64(pf->reg_base, BLKADDR_RVUM, 0,
				 RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf));
	}
	mutex_unlock(&pf->lock);
}

static irqreturn_t cptpf_vf_flr_intr(int __always_unused irq, void *arg)
{
	int reg, dev, vf, start_vf, num_reg = 1;
	struct otx2_cptpf_dev *cptpf = arg;
	u64 intr;

	if (cptpf->max_vfs > 64)
		num_reg = 2;

	for (reg = 0; reg < num_reg; reg++) {
		intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0,
				       RVU_PF_VFFLR_INTX(reg));
		if (!intr)
			continue;
		start_vf = 64 * reg;
		for (vf = 0; vf < 64; vf++) {
			if (!(intr & BIT_ULL(vf)))
				continue;
			dev = vf + start_vf;
			queue_work(cptpf->flr_wq, &cptpf->flr_work[dev].work);
			/* Clear interrupt */
			otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
					 RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf));
			/* Disable the interrupt */
			otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
					 RVU_PF_VFFLR_INT_ENA_W1CX(reg),
					 BIT_ULL(vf));
		}
	}
	return IRQ_HANDLED;
}

static irqreturn_t cptpf_vf_me_intr(int __always_unused irq, void *arg)
{
	struct otx2_cptpf_dev *cptpf = arg;
	int reg, vf, num_reg = 1;
	u64 intr;

	if (cptpf->max_vfs > 64)
		num_reg = 2;

	for (reg = 0; reg < num_reg; reg++) {
		intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0,
				       RVU_PF_VFME_INTX(reg));
		if (!intr)
			continue;
		for (vf = 0; vf < 64; vf++) {
			if (!(intr & BIT_ULL(vf)))
				continue;
			otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
					 RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
			/* Clear interrupt */
			otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
					 RVU_PF_VFME_INTX(reg), BIT_ULL(vf));
		}
	}
	return IRQ_HANDLED;
}

static void cptpf_unregister_vfpf_intr(struct otx2_cptpf_dev *cptpf,
				       int num_vfs)
{
	cptpf_disable_vfpf_mbox_intr(cptpf, num_vfs);
	cptpf_disable_vf_flr_me_intrs(cptpf, num_vfs);
}

static int cptpf_register_vfpf_intr(struct otx2_cptpf_dev *cptpf, int num_vfs)
{
	struct pci_dev *pdev = cptpf->pdev;
	struct device *dev = &pdev->dev;
	int ret, vector;

	vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX0);
	/* Register VF-PF mailbox interrupt handler */
	ret = request_irq(vector, otx2_cptpf_vfpf_mbox_intr, 0, "CPTVFPF Mbox0",
			  cptpf);
	if (ret) {
		dev_err(dev,
			"IRQ registration failed for PFVF mbox0 irq\n");
		return ret;
	}
	vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR0);
	/* Register VF FLR interrupt handler */
	ret = request_irq(vector, cptpf_vf_flr_intr, 0, "CPTPF FLR0", cptpf);
	if (ret) {
		dev_err(dev,
			"IRQ registration failed for VFFLR0 irq\n");
		goto free_mbox0_irq;
	}
	vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFME0);
	/* Register VF ME interrupt handler */
	ret = request_irq(vector, cptpf_vf_me_intr, 0, "CPTPF ME0", cptpf);
	if (ret) {
		dev_err(dev,
			"IRQ registration failed for PFVF mbox0 irq\n");
		goto free_flr0_irq;
	}

	if (num_vfs > 64) {
		vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX1);
		ret = request_irq(vector, otx2_cptpf_vfpf_mbox_intr, 0,
				  "CPTVFPF Mbox1", cptpf);
		if (ret) {
			dev_err(dev,
				"IRQ registration failed for PFVF mbox1 irq\n");
			goto free_me0_irq;
		}
		vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR1);
		/* Register VF FLR interrupt handler */
		ret = request_irq(vector, cptpf_vf_flr_intr, 0, "CPTPF FLR1",
				  cptpf);
		if (ret) {
			dev_err(dev,
				"IRQ registration failed for VFFLR1 irq\n");
			goto free_mbox1_irq;
		}
		vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFME1);
		/* Register VF FLR interrupt handler */
		ret = request_irq(vector, cptpf_vf_me_intr, 0, "CPTPF ME1",
				  cptpf);
		if (ret) {
			dev_err(dev,
				"IRQ registration failed for VFFLR1 irq\n");
			goto free_flr1_irq;
		}
	}
	cptpf_enable_vfpf_mbox_intr(cptpf, num_vfs);
	cptpf_enable_vf_flr_me_intrs(cptpf, num_vfs);

	return 0;

free_flr1_irq:
	vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR1);
	free_irq(vector, cptpf);
free_mbox1_irq:
	vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX1);
	free_irq(vector, cptpf);
free_me0_irq:
	vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFME0);
	free_irq(vector, cptpf);
free_flr0_irq:
	vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR0);
	free_irq(vector, cptpf);
free_mbox0_irq:
	vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX0);
	free_irq(vector, cptpf);
	return ret;
}

static void cptpf_flr_wq_destroy(struct otx2_cptpf_dev *pf)
{
	if (!pf->flr_wq)
		return;
	destroy_workqueue(pf->flr_wq);
	pf->flr_wq = NULL;
	kfree(pf->flr_work);
}

static int cptpf_flr_wq_init(struct otx2_cptpf_dev *cptpf, int num_vfs)
{
	int vf;

	cptpf->flr_wq = alloc_ordered_workqueue("cptpf_flr_wq", 0);
	if (!cptpf->flr_wq)
		return -ENOMEM;

	cptpf->flr_work = kcalloc(num_vfs, sizeof(struct cptpf_flr_work),
				  GFP_KERNEL);
	if (!cptpf->flr_work)
		goto destroy_wq;

	for (vf = 0; vf < num_vfs; vf++) {
		cptpf->flr_work[vf].pf = cptpf;
		INIT_WORK(&cptpf->flr_work[vf].work, cptpf_flr_wq_handler);
	}
	return 0;

destroy_wq:
	destroy_workqueue(cptpf->flr_wq);
	return -ENOMEM;
}

static int cptpf_vfpf_mbox_init(struct otx2_cptpf_dev *cptpf, int num_vfs)
{
	struct device *dev = &cptpf->pdev->dev;
	u64 vfpf_mbox_base;
	int err, i;

	cptpf->vfpf_mbox_wq =
		alloc_ordered_workqueue("cpt_vfpf_mailbox",
					WQ_HIGHPRI | WQ_MEM_RECLAIM);
	if (!cptpf->vfpf_mbox_wq)
		return -ENOMEM;

	/* Map VF-PF mailbox memory */
	if (test_bit(CN10K_MBOX, &cptpf->cap_flag))
		vfpf_mbox_base = readq(cptpf->reg_base + RVU_PF_VF_MBOX_ADDR);
	else
		vfpf_mbox_base = readq(cptpf->reg_base + RVU_PF_VF_BAR4_ADDR);

	if (!vfpf_mbox_base) {
		dev_err(dev, "VF-PF mailbox address not configured\n");
		err = -ENOMEM;
		goto free_wqe;
	}
	cptpf->vfpf_mbox_base = devm_ioremap_wc(dev, vfpf_mbox_base,
						MBOX_SIZE * cptpf->max_vfs);
	if (!cptpf->vfpf_mbox_base) {
		dev_err(dev, "Mapping of VF-PF mailbox address failed\n");
		err = -ENOMEM;
		goto free_wqe;
	}
	err = otx2_mbox_init(&cptpf->vfpf_mbox, cptpf->vfpf_mbox_base,
			     cptpf->pdev, cptpf->reg_base, MBOX_DIR_PFVF,
			     num_vfs);
	if (err)
		goto free_wqe;

	for (i = 0; i < num_vfs; i++) {
		cptpf->vf[i].vf_id = i;
		cptpf->vf[i].cptpf = cptpf;
		cptpf->vf[i].intr_idx = i % 64;
		INIT_WORK(&cptpf->vf[i].vfpf_mbox_work,
			  otx2_cptpf_vfpf_mbox_handler);
	}
	return 0;

free_wqe:
	destroy_workqueue(cptpf->vfpf_mbox_wq);
	return err;
}

static void cptpf_vfpf_mbox_destroy(struct otx2_cptpf_dev *cptpf)
{
	destroy_workqueue(cptpf->vfpf_mbox_wq);
	otx2_mbox_destroy(&cptpf->vfpf_mbox);
}

static void cptpf_disable_afpf_mbox_intr(struct otx2_cptpf_dev *cptpf)
{
	/* Disable AF-PF interrupt */
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT_ENA_W1C,
			 0x1ULL);
	/* Clear interrupt if any */
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT, 0x1ULL);
}

static int cptpf_register_afpf_mbox_intr(struct otx2_cptpf_dev *cptpf)
{
	struct pci_dev *pdev = cptpf->pdev;
	struct device *dev = &pdev->dev;
	int ret, irq;

	irq = pci_irq_vector(pdev, RVU_PF_INT_VEC_AFPF_MBOX);
	/* Register AF-PF mailbox interrupt handler */
	ret = devm_request_irq(dev, irq, otx2_cptpf_afpf_mbox_intr, 0,
			       "CPTAFPF Mbox", cptpf);
	if (ret) {
		dev_err(dev,
			"IRQ registration failed for PFAF mbox irq\n");
		return ret;
	}
	/* Clear interrupt if any, to avoid spurious interrupts */
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT, 0x1ULL);
	/* Enable AF-PF interrupt */
	otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT_ENA_W1S,
			 0x1ULL);

	ret = otx2_cpt_send_ready_msg(&cptpf->afpf_mbox, cptpf->pdev);
	if (ret) {
		dev_warn(dev,
			 "AF not responding to mailbox, deferring probe\n");
		cptpf_disable_afpf_mbox_intr(cptpf);
		return -EPROBE_DEFER;
	}
	return 0;
}

static int cptpf_afpf_mbox_init(struct otx2_cptpf_dev *cptpf)
{
	struct pci_dev *pdev = cptpf->pdev;
	resource_size_t offset;
	int err;

	cptpf->afpf_mbox_wq =
		alloc_ordered_workqueue("cpt_afpf_mailbox",
					WQ_HIGHPRI | WQ_MEM_RECLAIM);
	if (!cptpf->afpf_mbox_wq)
		return -ENOMEM;

	offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM);
	/* Map AF-PF mailbox memory */
	cptpf->afpf_mbox_base = devm_ioremap_wc(&pdev->dev, offset, MBOX_SIZE);
	if (!cptpf->afpf_mbox_base) {
		dev_err(&pdev->dev, "Unable to map BAR4\n");
		err = -ENOMEM;
		goto error;
	}

	err = otx2_mbox_init(&cptpf->afpf_mbox, cptpf->afpf_mbox_base,
			     pdev, cptpf->reg_base, MBOX_DIR_PFAF, 1);
	if (err)
		goto error;

	err = otx2_mbox_init(&cptpf->afpf_mbox_up, cptpf->afpf_mbox_base,
			     pdev, cptpf->reg_base, MBOX_DIR_PFAF_UP, 1);
	if (err)
		goto mbox_cleanup;

	INIT_WORK(&cptpf->afpf_mbox_work, otx2_cptpf_afpf_mbox_handler);
	INIT_WORK(&cptpf->afpf_mbox_up_work, otx2_cptpf_afpf_mbox_up_handler);
	mutex_init(&cptpf->lock);

	return 0;

mbox_cleanup:
	otx2_mbox_destroy(&cptpf->afpf_mbox);
error:
	destroy_workqueue(cptpf->afpf_mbox_wq);
	return err;
}

static void cptpf_afpf_mbox_destroy(struct otx2_cptpf_dev *cptpf)
{
	destroy_workqueue(cptpf->afpf_mbox_wq);
	otx2_mbox_destroy(&cptpf->afpf_mbox);
	otx2_mbox_destroy(&cptpf->afpf_mbox_up);
}

static ssize_t sso_pf_func_ovrd_show(struct device *dev,
				     struct device_attribute *attr, char *buf)
{
	struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev);

	return sysfs_emit(buf, "%d\n", cptpf->sso_pf_func_ovrd);
}

static ssize_t sso_pf_func_ovrd_store(struct device *dev,
				      struct device_attribute *attr,
				      const char *buf, size_t count)
{
	struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev);
	u8 sso_pf_func_ovrd;

	if (!(cptpf->pdev->revision == CPT_UC_RID_CN9K_B0))
		return count;

	if (kstrtou8(buf, 0, &sso_pf_func_ovrd))
		return -EINVAL;

	cptpf->sso_pf_func_ovrd = sso_pf_func_ovrd;

	return count;
}

static ssize_t kvf_limits_show(struct device *dev,
			       struct device_attribute *attr, char *buf)
{
	struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev);

	return sysfs_emit(buf, "%d\n", cptpf->kvf_limits);
}

static ssize_t kvf_limits_store(struct device *dev,
				struct device_attribute *attr,
				const char *buf, size_t count)
{
	struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev);
	int lfs_num;
	int ret;

	ret = kstrtoint(buf, 0, &lfs_num);
	if (ret)
		return ret;
	if (lfs_num < 1 || lfs_num > num_online_cpus()) {
		dev_err(dev, "lfs count %d must be in range [1 - %d]\n",
			lfs_num, num_online_cpus());
		return -EINVAL;
	}
	cptpf->kvf_limits = lfs_num;

	return count;
}

static DEVICE_ATTR_RW(kvf_limits);
static DEVICE_ATTR_RW(sso_pf_func_ovrd);

static struct attribute *cptpf_attrs[] = {
	&dev_attr_kvf_limits.attr,
	&dev_attr_sso_pf_func_ovrd.attr,
	NULL
};

static const struct attribute_group cptpf_sysfs_group = {
	.attrs = cptpf_attrs,
};

static int cpt_is_pf_usable(struct otx2_cptpf_dev *cptpf)
{
	u64 rev;

	rev = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0,
			      RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_RVUM));
	rev = (rev >> 12) & 0xFF;
	/*
	 * Check if AF has setup revision for RVUM block, otherwise
	 * driver probe should be deferred until AF driver comes up
	 */
	if (!rev) {
		dev_warn(&cptpf->pdev->dev,
			 "AF is not initialized, deferring probe\n");
		return -EPROBE_DEFER;
	}
	return 0;
}

static void cptpf_get_rid(struct pci_dev *pdev, struct otx2_cptpf_dev *cptpf)
{
	struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps;
	u64 reg_val = 0x0;

	if (is_dev_otx2(pdev)) {
		eng_grps->rid = pdev->revision;
		return;
	}
	otx2_cpt_read_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTL, &reg_val,
			     BLKADDR_CPT0);
	if ((cpt_feature_sgv2(pdev) && (reg_val & BIT_ULL(18))) ||
	    is_dev_cn10ka_ax(pdev))
		eng_grps->rid = CPT_UC_RID_CN10K_A;
	else if (cpt_feature_sgv2(pdev))
		eng_grps->rid = CPT_UC_RID_CN10K_B;
}

static void cptpf_check_block_implemented(struct otx2_cptpf_dev *cptpf)
{
	u64 cfg;

	cfg = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0,
			      RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_CPT1));
	if (cfg & BIT_ULL(11))
		cptpf->has_cpt1 = true;
}

static int cptpf_device_init(struct otx2_cptpf_dev *cptpf)
{
	union otx2_cptx_af_constants1 af_cnsts1 = {0};
	int ret = 0;

	/* check if 'implemented' bit is set for block BLKADDR_CPT1 */
	cptpf_check_block_implemented(cptpf);

	/* Get number of SE, IE and AE engines */
	ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev,
				   CPT_AF_CONSTANTS1, &af_cnsts1.u,
				   BLKADDR_CPT0);
	if (ret)
		return ret;

	cptpf->eng_grps.avail.max_se_cnt = af_cnsts1.s.se;
	cptpf->eng_grps.avail.max_ie_cnt = af_cnsts1.s.ie;
	cptpf->eng_grps.avail.max_ae_cnt = af_cnsts1.s.ae;

	/* Disable all cores */
	ret = otx2_cpt_disable_all_cores(cptpf);

	otx2_cptlf_set_dev_info(&cptpf->lfs, cptpf->pdev, cptpf->reg_base,
				&cptpf->afpf_mbox, BLKADDR_CPT0);
	if (cptpf->has_cpt1)
		otx2_cptlf_set_dev_info(&cptpf->cpt1_lfs, cptpf->pdev,
					cptpf->reg_base, &cptpf->afpf_mbox,
					BLKADDR_CPT1);
	return ret;
}

static int cptpf_sriov_disable(struct pci_dev *pdev)
{
	struct otx2_cptpf_dev *cptpf = pci_get_drvdata(pdev);
	int num_vfs = pci_num_vf(pdev);

	if (!num_vfs)
		return 0;

	pci_disable_sriov(pdev);
	cptpf_unregister_vfpf_intr(cptpf, num_vfs);
	cptpf_flr_wq_destroy(cptpf);
	cptpf_vfpf_mbox_destroy(cptpf);
	module_put(THIS_MODULE);
	cptpf->enabled_vfs = 0;

	return 0;
}

static int cptpf_sriov_enable(struct pci_dev *pdev, int num_vfs)
{
	struct otx2_cptpf_dev *cptpf = pci_get_drvdata(pdev);
	int ret;

	/* Initialize VF<=>PF mailbox */
	ret = cptpf_vfpf_mbox_init(cptpf, num_vfs);
	if (ret)
		return ret;

	ret = cptpf_flr_wq_init(cptpf, num_vfs);
	if (ret)
		goto destroy_mbox;
	/* Register VF<=>PF mailbox interrupt */
	ret = cptpf_register_vfpf_intr(cptpf, num_vfs);
	if (ret)
		goto destroy_flr;

	cptpf_get_rid(pdev, cptpf);
	/* Get CPT HW capabilities using LOAD_FVC operation. */
	ret = otx2_cpt_discover_eng_capabilities(cptpf);
	if (ret)
		goto disable_intr;

	ret = otx2_cpt_create_eng_grps(cptpf, &cptpf->eng_grps);
	if (ret)
		goto disable_intr;

	cptpf->enabled_vfs = num_vfs;
	ret = pci_enable_sriov(pdev, num_vfs);
	if (ret)
		goto disable_intr;

	dev_notice(&cptpf->pdev->dev, "VFs enabled: %d\n", num_vfs);

	try_module_get(THIS_MODULE);
	return num_vfs;

disable_intr:
	cptpf_unregister_vfpf_intr(cptpf, num_vfs);
	cptpf->enabled_vfs = 0;
destroy_flr:
	cptpf_flr_wq_destroy(cptpf);
destroy_mbox:
	cptpf_vfpf_mbox_destroy(cptpf);
	return ret;
}

static int otx2_cptpf_sriov_configure(struct pci_dev *pdev, int num_vfs)
{
	if (num_vfs > 0) {
		return cptpf_sriov_enable(pdev, num_vfs);
	} else {
		return cptpf_sriov_disable(pdev);
	}
}

static int otx2_cptpf_probe(struct pci_dev *pdev,
			    const struct pci_device_id *ent)
{
	struct device *dev = &pdev->dev;
	struct otx2_cptpf_dev *cptpf;
	int err, num_vec;

	cptpf = devm_kzalloc(dev, sizeof(*cptpf), GFP_KERNEL);
	if (!cptpf)
		return -ENOMEM;

	err = pcim_enable_device(pdev);
	if (err) {
		dev_err(dev, "Failed to enable PCI device\n");
		goto clear_drvdata;
	}

	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
	if (err) {
		dev_err(dev, "Unable to get usable DMA configuration\n");
		goto clear_drvdata;
	}
	err = pcim_request_all_regions(pdev, OTX2_CPT_DRV_NAME);
	if (err) {
		dev_err(dev, "Couldn't request PCI resources 0x%x\n", err);
		goto clear_drvdata;
	}
	pci_set_master(pdev);
	pci_set_drvdata(pdev, cptpf);
	cptpf->pdev = pdev;

	/* Map PF's configuration registers */
	cptpf->reg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0);
	if (!cptpf->reg_base) {
		err = -ENOMEM;
		dev_err(dev, "Couldn't ioremap PCI resource 0x%x\n", err);
		goto clear_drvdata;
	}

	/* Check if AF driver is up, otherwise defer probe */
	err = cpt_is_pf_usable(cptpf);
	if (err)
		goto clear_drvdata;

	num_vec = pci_msix_vec_count(cptpf->pdev);
	if (num_vec <= 0) {
		err = -EINVAL;
		goto clear_drvdata;
	}

	err = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSIX);
	if (err < 0) {
		dev_err(dev, "Request for %d msix vectors failed\n",
			RVU_PF_INT_VEC_CNT);
		goto clear_drvdata;
	}
	otx2_cpt_set_hw_caps(pdev, &cptpf->cap_flag);
	/* Initialize AF-PF mailbox */
	err = cptpf_afpf_mbox_init(cptpf);
	if (err)
		goto clear_drvdata;
	/* Register mailbox interrupt */
	err = cptpf_register_afpf_mbox_intr(cptpf);
	if (err)
		goto destroy_afpf_mbox;

	cptpf->max_vfs = pci_sriov_get_totalvfs(pdev);
	cptpf->kvf_limits = 1;

	/* Initialize CPT PF device */
	err = cptpf_device_init(cptpf);
	if (err)
		goto unregister_intr;

	err = cn10k_cptpf_lmtst_init(cptpf);
	if (err)
		goto unregister_intr;

	/* Initialize engine groups */
	err = otx2_cpt_init_eng_grps(pdev, &cptpf->eng_grps);
	if (err)
		goto free_lmtst;

	err = sysfs_create_group(&dev->kobj, &cptpf_sysfs_group);
	if (err)
		goto cleanup_eng_grps;

	err = otx2_cpt_register_dl(cptpf);
	if (err)
		goto sysfs_grp_del;

	return 0;

sysfs_grp_del:
	sysfs_remove_group(&dev->kobj, &cptpf_sysfs_group);
cleanup_eng_grps:
	otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps);
free_lmtst:
	cn10k_cpt_lmtst_free(pdev, &cptpf->lfs);
unregister_intr:
	cptpf_disable_afpf_mbox_intr(cptpf);
destroy_afpf_mbox:
	cptpf_afpf_mbox_destroy(cptpf);
clear_drvdata:
	pci_set_drvdata(pdev, NULL);
	return err;
}

static void otx2_cptpf_remove(struct pci_dev *pdev)
{
	struct otx2_cptpf_dev *cptpf = pci_get_drvdata(pdev);

	if (!cptpf)
		return;

	cptpf_sriov_disable(pdev);
	otx2_cpt_unregister_dl(cptpf);

	/* Cleanup Inline CPT LF's if attached */
	if (cptpf->lfs.lfs_num)
		otx2_inline_cptlf_cleanup(&cptpf->lfs);

	if (cptpf->cpt1_lfs.lfs_num)
		otx2_inline_cptlf_cleanup(&cptpf->cpt1_lfs);

	/* Delete sysfs entry created for kernel VF limits */
	sysfs_remove_group(&pdev->dev.kobj, &cptpf_sysfs_group);
	/* Cleanup engine groups */
	otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps);
	/* Disable AF-PF mailbox interrupt */
	cptpf_disable_afpf_mbox_intr(cptpf);
	/* Destroy AF-PF mbox */
	cptpf_afpf_mbox_destroy(cptpf);
	/* Free LMTST memory */
	cn10k_cpt_lmtst_free(pdev, &cptpf->lfs);
	pci_set_drvdata(pdev, NULL);
}

/* Supported devices */
static const struct pci_device_id otx2_cpt_id_table[] = {
	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OTX2_CPT_PCI_PF_DEVICE_ID) },
	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CN10K_CPT_PCI_PF_DEVICE_ID) },
	{ 0, }  /* end of table */
};

static struct pci_driver otx2_cpt_pci_driver = {
	.name = OTX2_CPT_DRV_NAME,
	.id_table = otx2_cpt_id_table,
	.probe = otx2_cptpf_probe,
	.remove = otx2_cptpf_remove,
	.sriov_configure = otx2_cptpf_sriov_configure
};

module_pci_driver(otx2_cpt_pci_driver);

MODULE_IMPORT_NS("CRYPTO_DEV_OCTEONTX2_CPT");

MODULE_AUTHOR("Marvell");
MODULE_DESCRIPTION(OTX2_CPT_DRV_STRING);
MODULE_LICENSE("GPL v2");
MODULE_DEVICE_TABLE(pci, otx2_cpt_id_table);