habanalabs_drv.c

// SPDX-License-Identifier: GPL-2.0

/*
 * Copyright 2016-2021 HabanaLabs, Ltd.
 * All Rights Reserved.
 *
 */

#define pr_fmt(fmt)		"habanalabs: " fmt

#include "habanalabs.h"
#include "../include/hw_ip/pci/pci_general.h"

#include <linux/pci.h>
#include <linux/module.h>
#include <linux/vmalloc.h>
#include <linux/version.h>

#include <drm/drm_accel.h>
#include <drm/drm_drv.h>
#include <drm/drm_ioctl.h>

#define CREATE_TRACE_POINTS
#include <trace/events/habanalabs.h>

#define HL_DRIVER_AUTHOR	"HabanaLabs Kernel Driver Team"

#define HL_DRIVER_DESC		"Driver for HabanaLabs's AI Accelerators"

MODULE_AUTHOR(HL_DRIVER_AUTHOR);
MODULE_DESCRIPTION(HL_DRIVER_DESC);
MODULE_LICENSE("GPL v2");

static int hl_major;
static DEFINE_IDR(hl_devs_idr);
static DEFINE_MUTEX(hl_devs_idr_lock);

#define HL_DEFAULT_TIMEOUT_LOCKED	30	/* 30 seconds */
#define GAUDI_DEFAULT_TIMEOUT_LOCKED	600	/* 10 minutes */

static int timeout_locked = HL_DEFAULT_TIMEOUT_LOCKED;
static int reset_on_lockup = 1;
static int memory_scrub;
static ulong boot_error_status_mask = ULONG_MAX;

module_param(timeout_locked, int, 0444);
MODULE_PARM_DESC(timeout_locked,
	"Device lockup timeout in seconds (0 = disabled, default 30s)");

module_param(reset_on_lockup, int, 0444);
MODULE_PARM_DESC(reset_on_lockup,
	"Do device reset on lockup (0 = no, 1 = yes, default yes)");

module_param(memory_scrub, int, 0444);
MODULE_PARM_DESC(memory_scrub,
	"Scrub device memory in various states (0 = no, 1 = yes, default no)");

module_param(boot_error_status_mask, ulong, 0444);
MODULE_PARM_DESC(boot_error_status_mask,
	"Mask of the error status during device CPU boot (If bitX is cleared then error X is masked. Default all 1's)");

#define PCI_IDS_GOYA			0x0001
#define PCI_IDS_GAUDI			0x1000
#define PCI_IDS_GAUDI_SEC		0x1010

#define PCI_IDS_GAUDI2			0x1020

static const struct pci_device_id ids[] = {
	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GOYA), },
	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI), },
	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI_SEC), },
	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI2), },
	{ 0, }
};
MODULE_DEVICE_TABLE(pci, ids);

static const struct drm_ioctl_desc hl_drm_ioctls[] = {
	DRM_IOCTL_DEF_DRV(HL_INFO, hl_info_ioctl, 0),
	DRM_IOCTL_DEF_DRV(HL_CB, hl_cb_ioctl, 0),
	DRM_IOCTL_DEF_DRV(HL_CS, hl_cs_ioctl, 0),
	DRM_IOCTL_DEF_DRV(HL_WAIT_CS, hl_wait_ioctl, 0),
	DRM_IOCTL_DEF_DRV(HL_MEMORY, hl_mem_ioctl, 0),
	DRM_IOCTL_DEF_DRV(HL_DEBUG, hl_debug_ioctl, 0),
};

static const struct file_operations hl_fops = {
	.owner = THIS_MODULE,
	.open = accel_open,
	.release = drm_release,
	.unlocked_ioctl = drm_ioctl,
	.compat_ioctl = drm_compat_ioctl,
	.llseek = noop_llseek,
	.mmap = hl_mmap
};

static const struct drm_driver hl_driver = {
	.driver_features = DRIVER_COMPUTE_ACCEL,

	.name = HL_NAME,
	.desc = HL_DRIVER_DESC,
	.major = LINUX_VERSION_MAJOR,
	.minor = LINUX_VERSION_PATCHLEVEL,
	.patchlevel = LINUX_VERSION_SUBLEVEL,

	.fops = &hl_fops,
	.open = hl_device_open,
	.postclose = hl_device_release,
	.ioctls = hl_drm_ioctls,
	.num_ioctls = ARRAY_SIZE(hl_drm_ioctls)
};

/*
 * get_asic_type - translate device id to asic type
 *
 * @hdev: pointer to habanalabs device structure.
 *
 * Translate device id and revision id to asic type.
 * In case of unidentified device, return -1
 */
static enum hl_asic_type get_asic_type(struct hl_device *hdev)
{
	struct pci_dev *pdev = hdev->pdev;
	enum hl_asic_type asic_type = ASIC_INVALID;

	switch (pdev->device) {
	case PCI_IDS_GOYA:
		asic_type = ASIC_GOYA;
		break;
	case PCI_IDS_GAUDI:
		asic_type = ASIC_GAUDI;
		break;
	case PCI_IDS_GAUDI_SEC:
		asic_type = ASIC_GAUDI_SEC;
		break;
	case PCI_IDS_GAUDI2:
		switch (pdev->revision) {
		case REV_ID_A:
			asic_type = ASIC_GAUDI2;
			break;
		case REV_ID_B:
			asic_type = ASIC_GAUDI2B;
			break;
		case REV_ID_C:
			asic_type = ASIC_GAUDI2C;
			break;
		case REV_ID_D:
			asic_type = ASIC_GAUDI2D;
			break;
		default:
			break;
		}
		break;
	default:
		break;
	}

	return asic_type;
}

static bool is_asic_secured(enum hl_asic_type asic_type)
{
	switch (asic_type) {
	case ASIC_GAUDI_SEC:
		return true;
	default:
		return false;
	}
}

/*
 * hl_device_open() - open function for habanalabs device.
 * @ddev: pointer to DRM device structure.
 * @file: pointer to DRM file private data structure.
 *
 * Called when process opens an habanalabs device.
 */
int hl_device_open(struct drm_device *ddev, struct drm_file *file_priv)
{
	struct hl_device *hdev = to_hl_device(ddev);
	enum hl_device_status status;
	struct hl_fpriv *hpriv;
	int rc;

	hpriv = kzalloc(sizeof(*hpriv), GFP_KERNEL);
	if (!hpriv)
		return -ENOMEM;

	hpriv->hdev = hdev;
	mutex_init(&hpriv->notifier_event.lock);
	mutex_init(&hpriv->restore_phase_mutex);
	mutex_init(&hpriv->ctx_lock);
	kref_init(&hpriv->refcount);

	hl_ctx_mgr_init(&hpriv->ctx_mgr);
	hl_mem_mgr_init(hpriv->hdev->dev, &hpriv->mem_mgr);

	hpriv->taskpid = get_task_pid(current, PIDTYPE_PID);

	mutex_lock(&hdev->fpriv_list_lock);

	if (!hl_device_operational(hdev, &status)) {
		dev_dbg_ratelimited(hdev->dev,
			"Can't open %s because it is %s\n",
			dev_name(hdev->dev), hdev->status[status]);

		if (status == HL_DEVICE_STATUS_IN_RESET ||
					status == HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE)
			rc = -EAGAIN;
		else
			rc = -EPERM;

		goto out_err;
	}

	if (hdev->is_in_dram_scrub) {
		dev_dbg_ratelimited(hdev->dev,
			"Can't open %s during dram scrub\n",
			dev_name(hdev->dev));
		rc = -EAGAIN;
		goto out_err;
	}

	if (hdev->compute_ctx_in_release) {
		dev_dbg_ratelimited(hdev->dev,
			"Can't open %s because another user is still releasing it\n",
			dev_name(hdev->dev));
		rc = -EAGAIN;
		goto out_err;
	}

	if (hdev->is_compute_ctx_active) {
		dev_dbg_ratelimited(hdev->dev,
			"Can't open %s because another user is working on it\n",
			dev_name(hdev->dev));
		rc = -EBUSY;
		goto out_err;
	}

	rc = hl_ctx_create(hdev, hpriv);
	if (rc) {
		dev_err(hdev->dev, "Failed to create context %d\n", rc);
		goto out_err;
	}

	list_add(&hpriv->dev_node, &hdev->fpriv_list);
	mutex_unlock(&hdev->fpriv_list_lock);

	hdev->asic_funcs->send_device_activity(hdev, true);

	hl_debugfs_add_file(hpriv);

	hl_enable_err_info_capture(&hdev->captured_err_info);

	hdev->open_counter++;
	hdev->last_successful_open_jif = jiffies;
	hdev->last_successful_open_ktime = ktime_get();

	file_priv->driver_priv = hpriv;
	hpriv->file_priv = file_priv;

	return 0;

out_err:
	mutex_unlock(&hdev->fpriv_list_lock);
	hl_mem_mgr_fini(&hpriv->mem_mgr, NULL);
	hl_mem_mgr_idr_destroy(&hpriv->mem_mgr);
	hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
	mutex_destroy(&hpriv->ctx_lock);
	mutex_destroy(&hpriv->restore_phase_mutex);
	mutex_destroy(&hpriv->notifier_event.lock);
	put_pid(hpriv->taskpid);

	kfree(hpriv);

	return rc;
}

int hl_device_open_ctrl(struct inode *inode, struct file *filp)
{
	struct hl_device *hdev;
	struct hl_fpriv *hpriv;
	int rc;

	mutex_lock(&hl_devs_idr_lock);
	hdev = idr_find(&hl_devs_idr, iminor(inode));
	mutex_unlock(&hl_devs_idr_lock);

	if (!hdev) {
		pr_err("Couldn't find device %d:%d\n",
			imajor(inode), iminor(inode));
		return -ENXIO;
	}

	hpriv = kzalloc(sizeof(*hpriv), GFP_KERNEL);
	if (!hpriv)
		return -ENOMEM;

	/* Prevent other routines from reading partial hpriv data by
	 * initializing hpriv fields before inserting it to the list
	 */
	hpriv->hdev = hdev;
	filp->private_data = hpriv;

	nonseekable_open(inode, filp);

	hpriv->taskpid = get_task_pid(current, PIDTYPE_PID);

	mutex_lock(&hdev->fpriv_ctrl_list_lock);

	if (!hl_ctrl_device_operational(hdev, NULL)) {
		dev_dbg_ratelimited(hdev->dev_ctrl,
			"Can't open %s because it is disabled\n",
			dev_name(hdev->dev_ctrl));
		rc = -EPERM;
		goto out_err;
	}

	list_add(&hpriv->dev_node, &hdev->fpriv_ctrl_list);
	mutex_unlock(&hdev->fpriv_ctrl_list_lock);

	return 0;

out_err:
	mutex_unlock(&hdev->fpriv_ctrl_list_lock);
	filp->private_data = NULL;
	put_pid(hpriv->taskpid);

	kfree(hpriv);

	return rc;
}

static void set_driver_behavior_per_device(struct hl_device *hdev)
{
	hdev->nic_ports_mask = 0;
	hdev->fw_components = FW_TYPE_ALL_TYPES;
	hdev->cpu_queues_enable = 1;
	hdev->pldm = 0;
	hdev->hard_reset_on_fw_events = 1;
	hdev->bmc_enable = 1;
	hdev->reset_on_preboot_fail = 1;
	hdev->heartbeat = 1;
}

static void copy_kernel_module_params_to_device(struct hl_device *hdev)
{
	hdev->asic_prop.fw_security_enabled = is_asic_secured(hdev->asic_type);

	hdev->major = hl_major;
	hdev->memory_scrub = memory_scrub;
	hdev->reset_on_lockup = reset_on_lockup;
	hdev->boot_error_status_mask = boot_error_status_mask;
}

static void fixup_device_params_per_asic(struct hl_device *hdev, int timeout)
{
	switch (hdev->asic_type) {
	case ASIC_GAUDI:
	case ASIC_GAUDI_SEC:
		/* If user didn't request a different timeout than the default one, we have
		 * a different default timeout for Gaudi
		 */
		if (timeout == HL_DEFAULT_TIMEOUT_LOCKED)
			hdev->timeout_jiffies = secs_to_jiffies(GAUDI_DEFAULT_TIMEOUT_LOCKED);

		hdev->reset_upon_device_release = 0;
		break;

	case ASIC_GOYA:
		hdev->reset_upon_device_release = 0;
		break;

	default:
		hdev->reset_upon_device_release = 1;
		break;
	}
}

static int fixup_device_params(struct hl_device *hdev)
{
	int tmp_timeout;

	tmp_timeout = timeout_locked;

	hdev->fw_poll_interval_usec = HL_FW_STATUS_POLL_INTERVAL_USEC;
	hdev->fw_comms_poll_interval_usec = HL_FW_STATUS_POLL_INTERVAL_USEC;

	if (tmp_timeout)
		hdev->timeout_jiffies = secs_to_jiffies(tmp_timeout);
	else
		hdev->timeout_jiffies = MAX_SCHEDULE_TIMEOUT;

	hdev->stop_on_err = true;
	hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
	hdev->reset_info.prev_reset_trigger = HL_RESET_TRIGGER_DEFAULT;

	/* Enable only after the initialization of the device */
	hdev->disabled = true;

	if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) &&
			(hdev->fw_components & ~FW_TYPE_PREBOOT_CPU)) {
		pr_err("Preboot must be set along with other components");
		return -EINVAL;
	}

	/* If CPU queues not enabled, no way to do heartbeat */
	if (!hdev->cpu_queues_enable)
		hdev->heartbeat = 0;
	fixup_device_params_per_asic(hdev, tmp_timeout);

	return 0;
}

static int allocate_device_id(struct hl_device *hdev)
{
	int id;

	mutex_lock(&hl_devs_idr_lock);
	id = idr_alloc(&hl_devs_idr, hdev, 0, HL_MAX_MINORS, GFP_KERNEL);
	mutex_unlock(&hl_devs_idr_lock);

	if (id < 0) {
		if (id == -ENOSPC)
			pr_err("too many devices in the system\n");
		return -EBUSY;
	}

	hdev->id = id;

	/*
	 * Firstly initialized with the internal device ID.
	 * Will be updated later after the DRM device registration to hold the minor ID.
	 */
	hdev->cdev_idx = hdev->id;

	return 0;
}

/**
 * create_hdev - create habanalabs device instance
 *
 * @dev: will hold the pointer to the new habanalabs device structure
 * @pdev: pointer to the pci device
 *
 * Allocate memory for habanalabs device and initialize basic fields
 * Identify the ASIC type
 * Allocate ID (minor) for the device (only for real devices)
 */
static int create_hdev(struct hl_device **dev, struct pci_dev *pdev)
{
	struct hl_device *hdev;
	int rc;

	*dev = NULL;

	hdev = devm_drm_dev_alloc(&pdev->dev, &hl_driver, struct hl_device, drm);
	if (IS_ERR(hdev))
		return PTR_ERR(hdev);

	hdev->dev = hdev->drm.dev;

	/* Will be NULL in case of simulator device */
	hdev->pdev = pdev;

	/* Assign status description string */
	strscpy(hdev->status[HL_DEVICE_STATUS_OPERATIONAL], "operational", HL_STR_MAX);
	strscpy(hdev->status[HL_DEVICE_STATUS_IN_RESET], "in reset", HL_STR_MAX);
	strscpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION], "disabled", HL_STR_MAX);
	strscpy(hdev->status[HL_DEVICE_STATUS_NEEDS_RESET], "needs reset", HL_STR_MAX);
	strscpy(hdev->status[HL_DEVICE_STATUS_IN_DEVICE_CREATION],
				"in device creation", HL_STR_MAX);
	strscpy(hdev->status[HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE],
				"in reset after device release", HL_STR_MAX);


	/* First, we must find out which ASIC are we handling. This is needed
	 * to configure the behavior of the driver (kernel parameters)
	 */
	hdev->asic_type = get_asic_type(hdev);
	if (hdev->asic_type == ASIC_INVALID) {
		dev_err(&pdev->dev, "Unsupported ASIC\n");
		rc = -ENODEV;
		goto out_err;
	}

	copy_kernel_module_params_to_device(hdev);

	set_driver_behavior_per_device(hdev);

	fixup_device_params(hdev);

	rc = allocate_device_id(hdev);
	if (rc)
		goto out_err;

	*dev = hdev;

	return 0;

out_err:
	return rc;
}

/*
 * destroy_hdev - destroy habanalabs device instance
 *
 * @dev: pointer to the habanalabs device structure
 *
 */
static void destroy_hdev(struct hl_device *hdev)
{
	/* Remove device from the device list */
	mutex_lock(&hl_devs_idr_lock);
	idr_remove(&hl_devs_idr, hdev->id);
	mutex_unlock(&hl_devs_idr_lock);

}

static int hl_pmops_suspend(struct device *dev)
{
	struct hl_device *hdev = dev_get_drvdata(dev);

	pr_debug("Going to suspend PCI device\n");

	if (!hdev) {
		pr_err("device pointer is NULL in suspend\n");
		return 0;
	}

	return hl_device_suspend(hdev);
}

static int hl_pmops_resume(struct device *dev)
{
	struct hl_device *hdev = dev_get_drvdata(dev);

	pr_debug("Going to resume PCI device\n");

	if (!hdev) {
		pr_err("device pointer is NULL in resume\n");
		return 0;
	}

	return hl_device_resume(hdev);
}

/**
 * hl_pci_probe - probe PCI habanalabs devices
 *
 * @pdev: pointer to pci device
 * @id: pointer to pci device id structure
 *
 * Standard PCI probe function for habanalabs device.
 * Create a new habanalabs device and initialize it according to the
 * device's type
 */
static int hl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
	struct hl_device *hdev;
	int rc;

	dev_info(&pdev->dev, HL_NAME
		 " device found [%04x:%04x] (rev %x)\n",
		 (int)pdev->vendor, (int)pdev->device, (int)pdev->revision);

	rc = create_hdev(&hdev, pdev);
	if (rc)
		return rc;

	pci_set_drvdata(pdev, hdev);

	rc = hl_device_init(hdev);
	if (rc) {
		dev_err(&pdev->dev, "Fatal error during habanalabs device init\n");
		rc = -ENODEV;
		goto disable_device;
	}

	return 0;

disable_device:
	pci_set_drvdata(pdev, NULL);
	destroy_hdev(hdev);

	return rc;
}

/*
 * hl_pci_remove - remove PCI habanalabs devices
 *
 * @pdev: pointer to pci device
 *
 * Standard PCI remove function for habanalabs device
 */
static void hl_pci_remove(struct pci_dev *pdev)
{
	struct hl_device *hdev;

	hdev = pci_get_drvdata(pdev);
	if (!hdev)
		return;

	hl_device_fini(hdev);
	pci_set_drvdata(pdev, NULL);
	destroy_hdev(hdev);
}

/**
 * hl_pci_err_detected - a PCI bus error detected on this device
 *
 * @pdev: pointer to pci device
 * @state: PCI error type
 *
 * Called by the PCI subsystem whenever a non-correctable
 * PCI bus error is detected
 */
static pci_ers_result_t
hl_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t state)
{
	struct hl_device *hdev = pci_get_drvdata(pdev);
	enum pci_ers_result result;

	switch (state) {
	case pci_channel_io_normal:
		dev_warn(hdev->dev, "PCI normal state error detected\n");
		return PCI_ERS_RESULT_CAN_RECOVER;

	case pci_channel_io_frozen:
		dev_warn(hdev->dev, "PCI frozen state error detected\n");
		result = PCI_ERS_RESULT_NEED_RESET;
		break;

	case pci_channel_io_perm_failure:
		dev_warn(hdev->dev, "PCI failure state error detected\n");
		result = PCI_ERS_RESULT_DISCONNECT;
		break;

	default:
		result = PCI_ERS_RESULT_NONE;
	}

	hdev->asic_funcs->halt_engines(hdev, true, false);

	return result;
}

/**
 * hl_pci_err_resume - resume after a PCI slot reset
 *
 * @pdev: pointer to pci device
 *
 */
static void hl_pci_err_resume(struct pci_dev *pdev)
{
	struct hl_device *hdev = pci_get_drvdata(pdev);

	dev_warn(hdev->dev, "Resuming device after PCI slot reset\n");
	hl_device_resume(hdev);
}

/**
 * hl_pci_err_slot_reset - a PCI slot reset has just happened
 *
 * @pdev: pointer to pci device
 *
 * Determine if the driver can recover from the PCI slot reset
 */
static pci_ers_result_t hl_pci_err_slot_reset(struct pci_dev *pdev)
{
	struct hl_device *hdev = pci_get_drvdata(pdev);

	dev_warn(hdev->dev, "PCI slot reset detected\n");

	return PCI_ERS_RESULT_RECOVERED;
}

static void hl_pci_reset_prepare(struct pci_dev *pdev)
{
	struct hl_device *hdev;

	hdev = pci_get_drvdata(pdev);
	if (!hdev)
		return;

	hdev->disabled = true;
}

static void hl_pci_reset_done(struct pci_dev *pdev)
{
	struct hl_device *hdev;
	u32 flags;

	hdev = pci_get_drvdata(pdev);
	if (!hdev)
		return;

	/*
	 * Schedule a thread to trigger hard reset.
	 * The reason for this handler, is for rare cases where the driver is up
	 * and FLR occurs. This is valid only when working with no VM, so FW handles FLR
	 * and resets the device. FW will go back preboot stage, so driver needs to perform
	 * hard reset in order to load FW fit again.
	 */
	flags = HL_DRV_RESET_HARD | HL_DRV_RESET_BYPASS_REQ_TO_FW;

	hl_device_reset(hdev, flags);
}

static const struct dev_pm_ops hl_pm_ops = {
	.suspend = hl_pmops_suspend,
	.resume = hl_pmops_resume,
};

static const struct pci_error_handlers hl_pci_err_handler = {
	.error_detected = hl_pci_err_detected,
	.slot_reset = hl_pci_err_slot_reset,
	.resume = hl_pci_err_resume,
	.reset_prepare = hl_pci_reset_prepare,
	.reset_done = hl_pci_reset_done,
};

static struct pci_driver hl_pci_driver = {
	.name = HL_NAME,
	.id_table = ids,
	.probe = hl_pci_probe,
	.remove = hl_pci_remove,
	.shutdown = hl_pci_remove,
	.driver = {
		.name = HL_NAME,
		.pm = &hl_pm_ops,
		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
	},
	.err_handler = &hl_pci_err_handler,
};

/*
 * hl_init - Initialize the habanalabs kernel driver
 */
static int __init hl_init(void)
{
	int rc;
	dev_t dev;

	pr_info("loading driver\n");

	rc = alloc_chrdev_region(&dev, 0, HL_MAX_MINORS, HL_NAME);
	if (rc < 0) {
		pr_err("unable to get major\n");
		return rc;
	}

	hl_major = MAJOR(dev);

	rc = pci_register_driver(&hl_pci_driver);
	if (rc) {
		pr_err("failed to register pci device\n");
		goto remove_major;
	}

	pr_debug("driver loaded\n");

	return 0;

remove_major:
	unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);
	return rc;
}

/*
 * hl_exit - Release all resources of the habanalabs kernel driver
 */
static void __exit hl_exit(void)
{
	pci_unregister_driver(&hl_pci_driver);

	unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);

	idr_destroy(&hl_devs_idr);

	pr_debug("driver removed\n");
}

module_init(hl_init);
module_exit(hl_exit);