vtd.c

/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2011 NetApp, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/malloc.h>

#include <vm/vm.h>
#include <vm/pmap.h>

#include <dev/pci/pcireg.h>

#include <machine/vmparam.h>
#include <contrib/dev/acpica/include/acpi.h>

#include "io/iommu.h"

/*
 * Documented in the "Intel Virtualization Technology for Directed I/O",
 * Architecture Spec, September 2008.
 */

#define VTD_DRHD_INCLUDE_PCI_ALL(Flags)  (((Flags) >> 0) & 0x1)

/* Section 10.4 "Register Descriptions" */
struct vtdmap {
	volatile uint32_t	version;
	volatile uint32_t	res0;
	volatile uint64_t	cap;
	volatile uint64_t	ext_cap;
	volatile uint32_t	gcr;
	volatile uint32_t	gsr;
	volatile uint64_t	rta;
	volatile uint64_t	ccr;
};

#define	VTD_CAP_SAGAW(cap)	(((cap) >> 8) & 0x1F)
#define	VTD_CAP_ND(cap)		((cap) & 0x7)
#define	VTD_CAP_CM(cap)		(((cap) >> 7) & 0x1)
#define	VTD_CAP_SPS(cap)	(((cap) >> 34) & 0xF)
#define	VTD_CAP_RWBF(cap)	(((cap) >> 4) & 0x1)

#define	VTD_ECAP_DI(ecap)	(((ecap) >> 2) & 0x1)
#define	VTD_ECAP_COHERENCY(ecap) ((ecap) & 0x1)
#define	VTD_ECAP_IRO(ecap)	(((ecap) >> 8) & 0x3FF)

#define	VTD_GCR_WBF		(1 << 27)
#define	VTD_GCR_SRTP		(1 << 30)
#define	VTD_GCR_TE		(1U << 31)

#define	VTD_GSR_WBFS		(1 << 27)
#define	VTD_GSR_RTPS		(1 << 30)
#define	VTD_GSR_TES		(1U << 31)

#define	VTD_CCR_ICC		(1UL << 63)	/* invalidate context cache */
#define	VTD_CCR_CIRG_GLOBAL	(1UL << 61)	/* global invalidation */

#define	VTD_IIR_IVT		(1UL << 63)	/* invalidation IOTLB */
#define	VTD_IIR_IIRG_GLOBAL	(1ULL << 60)	/* global IOTLB invalidation */
#define	VTD_IIR_IIRG_DOMAIN	(2ULL << 60)	/* domain IOTLB invalidation */
#define	VTD_IIR_IIRG_PAGE	(3ULL << 60)	/* page IOTLB invalidation */
#define	VTD_IIR_DRAIN_READS	(1ULL << 49)	/* drain pending DMA reads */
#define	VTD_IIR_DRAIN_WRITES	(1ULL << 48)	/* drain pending DMA writes */
#define	VTD_IIR_DOMAIN_P	32

#define	VTD_ROOT_PRESENT	0x1
#define	VTD_CTX_PRESENT		0x1
#define	VTD_CTX_TT_ALL		(1UL << 2)

#define	VTD_PTE_RD		(1UL << 0)
#define	VTD_PTE_WR		(1UL << 1)
#define	VTD_PTE_SUPERPAGE	(1UL << 7)
#define	VTD_PTE_ADDR_M		(0x000FFFFFFFFFF000UL)

#define VTD_RID2IDX(rid)	(((rid) & 0xff) * 2)

struct domain {
	uint64_t	*ptp;		/* first level page table page */
	int		pt_levels;	/* number of page table levels */
	int		addrwidth;	/* 'AW' field in context entry */
	int		spsmask;	/* supported super page sizes */
	u_int		id;		/* domain id */
	vm_paddr_t	maxaddr;	/* highest address to be mapped */
	SLIST_ENTRY(domain) next;
};

static SLIST_HEAD(, domain) domhead;

#define	DRHD_MAX_UNITS	16
static ACPI_DMAR_HARDWARE_UNIT	*drhds[DRHD_MAX_UNITS];
static int			drhd_num;
static struct vtdmap		*vtdmaps[DRHD_MAX_UNITS];
static int			max_domains;
typedef int			(*drhd_ident_func_t)(void);

static uint64_t root_table[PAGE_SIZE / sizeof(uint64_t)] __aligned(4096);
static uint64_t ctx_tables[256][PAGE_SIZE / sizeof(uint64_t)] __aligned(4096);

static MALLOC_DEFINE(M_VTD, "vtd", "vtd");

static int
vtd_max_domains(struct vtdmap *vtdmap)
{
	int nd;

	nd = VTD_CAP_ND(vtdmap->cap);

	switch (nd) {
	case 0:
		return (16);
	case 1:
		return (64);
	case 2:
		return (256);
	case 3:
		return (1024);
	case 4:
		return (4 * 1024);
	case 5:
		return (16 * 1024);
	case 6:
		return (64 * 1024);
	default:
		panic("vtd_max_domains: invalid value of nd (0x%0x)", nd);
	}
}

static u_int
domain_id(void)
{
	u_int id;
	struct domain *dom;

	/* Skip domain id 0 - it is reserved when Caching Mode field is set */
	for (id = 1; id < max_domains; id++) {
		SLIST_FOREACH(dom, &domhead, next) {
			if (dom->id == id)
				break;
		}
		if (dom == NULL)
			break;		/* found it */
	}

	if (id >= max_domains)
		panic("domain ids exhausted");

	return (id);
}

static struct vtdmap *
vtd_device_scope(uint16_t rid)
{
	int i, remaining, pathremaining;
	char *end, *pathend;
	struct vtdmap *vtdmap;
	ACPI_DMAR_HARDWARE_UNIT *drhd;
	ACPI_DMAR_DEVICE_SCOPE *device_scope;
	ACPI_DMAR_PCI_PATH *path;

	for (i = 0; i < drhd_num; i++) {
		drhd = drhds[i];

		if (VTD_DRHD_INCLUDE_PCI_ALL(drhd->Flags)) {
			/*
			 * From Intel VT-d arch spec, version 3.0:
			 * If a DRHD structure with INCLUDE_PCI_ALL flag Set is reported
			 * for a Segment, it must be enumerated by BIOS after all other
			 * DRHD structures for the same Segment.
			 */
			vtdmap = vtdmaps[i];
			return(vtdmap);
		}

		end = (char *)drhd + drhd->Header.Length;
		remaining = drhd->Header.Length - sizeof(ACPI_DMAR_HARDWARE_UNIT);
		while (remaining > sizeof(ACPI_DMAR_DEVICE_SCOPE)) {
			device_scope = (ACPI_DMAR_DEVICE_SCOPE *)(end - remaining);
			remaining -= device_scope->Length;

			switch (device_scope->EntryType){
				/* 0x01 and 0x02 are PCI device entries */
				case 0x01:
				case 0x02:
					break;
				default:
					continue;
			}

			if (PCI_RID2BUS(rid) != device_scope->Bus)
				continue;

			pathend = (char *)device_scope + device_scope->Length;
			pathremaining = device_scope->Length - sizeof(ACPI_DMAR_DEVICE_SCOPE);
			while (pathremaining >= sizeof(ACPI_DMAR_PCI_PATH)) {
				path = (ACPI_DMAR_PCI_PATH *)(pathend - pathremaining);
				pathremaining -= sizeof(ACPI_DMAR_PCI_PATH);

				if (PCI_RID2SLOT(rid) != path->Device)
					continue;
				if (PCI_RID2FUNC(rid) != path->Function)
					continue;

				vtdmap = vtdmaps[i];
				return (vtdmap);
			}
		}
	}

	/* No matching scope */
	return (NULL);
}

static void
vtd_wbflush(struct vtdmap *vtdmap)
{

	if (VTD_ECAP_COHERENCY(vtdmap->ext_cap) == 0)
		pmap_invalidate_cache();

	if (VTD_CAP_RWBF(vtdmap->cap)) {
		vtdmap->gcr = VTD_GCR_WBF;
		while ((vtdmap->gsr & VTD_GSR_WBFS) != 0)
			;
	}
}

static void
vtd_ctx_global_invalidate(struct vtdmap *vtdmap)
{

	vtdmap->ccr = VTD_CCR_ICC | VTD_CCR_CIRG_GLOBAL;
	while ((vtdmap->ccr & VTD_CCR_ICC) != 0)
		;
}

static void
vtd_iotlb_global_invalidate(struct vtdmap *vtdmap)
{
	int offset;
	volatile uint64_t *iotlb_reg, val;

	vtd_wbflush(vtdmap);

	offset = VTD_ECAP_IRO(vtdmap->ext_cap) * 16;
	iotlb_reg = (volatile uint64_t *)((caddr_t)vtdmap + offset + 8);

	*iotlb_reg =  VTD_IIR_IVT | VTD_IIR_IIRG_GLOBAL |
		      VTD_IIR_DRAIN_READS | VTD_IIR_DRAIN_WRITES;

	while (1) {
		val = *iotlb_reg;
		if ((val & VTD_IIR_IVT) == 0)
			break;
	}
}

static void
vtd_translation_enable(struct vtdmap *vtdmap)
{

	vtdmap->gcr = VTD_GCR_TE;
	while ((vtdmap->gsr & VTD_GSR_TES) == 0)
		;
}

static void
vtd_translation_disable(struct vtdmap *vtdmap)
{

	vtdmap->gcr = 0;
	while ((vtdmap->gsr & VTD_GSR_TES) != 0)
		;
}

static int
vtd_init(void)
{
	int i, units, remaining, tmp;
	struct vtdmap *vtdmap;
	vm_paddr_t ctx_paddr;
	char *end, envname[32];
	unsigned long mapaddr;
	ACPI_STATUS status;
	ACPI_TABLE_DMAR *dmar;
	ACPI_DMAR_HEADER *hdr;
	ACPI_DMAR_HARDWARE_UNIT *drhd;

	/*
	 * Allow the user to override the ACPI DMAR table by specifying the
	 * physical address of each remapping unit.
	 *
	 * The following example specifies two remapping units at
	 * physical addresses 0xfed90000 and 0xfeda0000 respectively.
	 * set vtd.regmap.0.addr=0xfed90000
	 * set vtd.regmap.1.addr=0xfeda0000
	 */
	for (units = 0; units < DRHD_MAX_UNITS; units++) {
		snprintf(envname, sizeof(envname), "vtd.regmap.%d.addr", units);
		if (getenv_ulong(envname, &mapaddr) == 0)
			break;
		vtdmaps[units] = (struct vtdmap *)PHYS_TO_DMAP(mapaddr);
	}

	if (units > 0)
		goto skip_dmar;

	/* Search for DMAR table. */
	status = AcpiGetTable(ACPI_SIG_DMAR, 0, (ACPI_TABLE_HEADER **)&dmar);
	if (ACPI_FAILURE(status))
		return (ENXIO);

	end = (char *)dmar + dmar->Header.Length;
	remaining = dmar->Header.Length - sizeof(ACPI_TABLE_DMAR);
	while (remaining > sizeof(ACPI_DMAR_HEADER)) {
		hdr = (ACPI_DMAR_HEADER *)(end - remaining);
		if (hdr->Length > remaining)
			break;
		/*
		 * From Intel VT-d arch spec, version 1.3:
		 * BIOS implementations must report mapping structures
		 * in numerical order, i.e. All remapping structures of
		 * type 0 (DRHD) enumerated before remapping structures of
		 * type 1 (RMRR) and so forth.
		 */
		if (hdr->Type != ACPI_DMAR_TYPE_HARDWARE_UNIT)
			break;

		drhd = (ACPI_DMAR_HARDWARE_UNIT *)hdr;
		drhds[units] = drhd;
		vtdmaps[units] = (struct vtdmap *)PHYS_TO_DMAP(drhd->Address);
		if (++units >= DRHD_MAX_UNITS)
			break;
		remaining -= hdr->Length;
	}

	if (units <= 0)
		return (ENXIO);

skip_dmar:
	drhd_num = units;

	max_domains = 64 * 1024; /* maximum valid value */
	for (i = 0; i < drhd_num; i++){
		vtdmap = vtdmaps[i];

		if (VTD_CAP_CM(vtdmap->cap) != 0)
			panic("vtd_init: invalid caching mode");

		/* take most compatible (minimum) value */
		if ((tmp = vtd_max_domains(vtdmap)) < max_domains)
			max_domains = tmp;
	}

	/*
	 * Set up the root-table to point to the context-entry tables
	 */
	for (i = 0; i < 256; i++) {
		ctx_paddr = vtophys(ctx_tables[i]);
		if (ctx_paddr & PAGE_MASK)
			panic("ctx table (0x%0lx) not page aligned", ctx_paddr);

		root_table[i * 2] = ctx_paddr | VTD_ROOT_PRESENT;
	}

	return (0);
}

static void
vtd_cleanup(void)
{
}

static void
vtd_enable(void)
{
	int i;
	struct vtdmap *vtdmap;

	for (i = 0; i < drhd_num; i++) {
		vtdmap = vtdmaps[i];
		vtd_wbflush(vtdmap);

		/* Update the root table address */
		vtdmap->rta = vtophys(root_table);
		vtdmap->gcr = VTD_GCR_SRTP;
		while ((vtdmap->gsr & VTD_GSR_RTPS) == 0)
			;

		vtd_ctx_global_invalidate(vtdmap);
		vtd_iotlb_global_invalidate(vtdmap);

		vtd_translation_enable(vtdmap);
	}
}

static void
vtd_disable(void)
{
	int i;
	struct vtdmap *vtdmap;

	for (i = 0; i < drhd_num; i++) {
		vtdmap = vtdmaps[i];
		vtd_translation_disable(vtdmap);
	}
}

static int
vtd_add_device(void *arg, device_t dev __unused, uint16_t rid)
{
	int idx;
	uint64_t *ctxp;
	struct domain *dom = arg;
	vm_paddr_t pt_paddr;
	struct vtdmap *vtdmap;
	uint8_t bus;

	KASSERT(dom != NULL, ("domain is NULL"));

	bus = PCI_RID2BUS(rid);
	ctxp = ctx_tables[bus];
	pt_paddr = vtophys(dom->ptp);
	idx = VTD_RID2IDX(rid);

	if (ctxp[idx] & VTD_CTX_PRESENT) {
		panic("vtd_add_device: device %x is already owned by "
		      "domain %d", rid,
		      (uint16_t)(ctxp[idx + 1] >> 8));
	}

	if ((vtdmap = vtd_device_scope(rid)) == NULL)
		panic("vtd_add_device: device %x is not in scope for "
		      "any DMA remapping unit", rid);

	/*
	 * Order is important. The 'present' bit is set only after all fields
	 * of the context pointer are initialized.
	 */
	ctxp[idx + 1] = dom->addrwidth | (dom->id << 8);

	if (VTD_ECAP_DI(vtdmap->ext_cap))
		ctxp[idx] = VTD_CTX_TT_ALL;
	else
		ctxp[idx] = 0;

	ctxp[idx] |= pt_paddr | VTD_CTX_PRESENT;

	/*
	 * 'Not Present' entries are not cached in either the Context Cache
	 * or in the IOTLB, so there is no need to invalidate either of them.
	 */
	return (0);
}

static int
vtd_remove_device(void *arg, device_t dev __unused, uint16_t rid)
{
	int i, idx;
	uint64_t *ctxp;
	struct vtdmap *vtdmap;
	uint8_t bus;

	bus = PCI_RID2BUS(rid);
	ctxp = ctx_tables[bus];
	idx = VTD_RID2IDX(rid);

	/*
	 * Order is important. The 'present' bit is must be cleared first.
	 */
	ctxp[idx] = 0;
	ctxp[idx + 1] = 0;

	/*
	 * Invalidate the Context Cache and the IOTLB.
	 *
	 * XXX use device-selective invalidation for Context Cache
	 * XXX use domain-selective invalidation for IOTLB
	 */
	for (i = 0; i < drhd_num; i++) {
		vtdmap = vtdmaps[i];
		vtd_ctx_global_invalidate(vtdmap);
		vtd_iotlb_global_invalidate(vtdmap);
	}
	return (0);
}

#define	CREATE_MAPPING	0
#define	REMOVE_MAPPING	1

static uint64_t
vtd_update_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len,
		   int remove)
{
	struct domain *dom;
	int i, spshift, ptpshift, ptpindex, nlevels;
	uint64_t spsize, *ptp;

	dom = arg;
	ptpindex = 0;
	ptpshift = 0;

	KASSERT(gpa + len > gpa, ("%s: invalid gpa range %#lx/%#lx", __func__,
	    gpa, len));
	KASSERT(gpa + len <= dom->maxaddr, ("%s: gpa range %#lx/%#lx beyond "
	    "domain maxaddr %#lx", __func__, gpa, len, dom->maxaddr));

	if (gpa & PAGE_MASK)
		panic("vtd_create_mapping: unaligned gpa 0x%0lx", gpa);

	if (hpa & PAGE_MASK)
		panic("vtd_create_mapping: unaligned hpa 0x%0lx", hpa);

	if (len & PAGE_MASK)
		panic("vtd_create_mapping: unaligned len 0x%0lx", len);

	/*
	 * Compute the size of the mapping that we can accommodate.
	 *
	 * This is based on three factors:
	 * - supported super page size
	 * - alignment of the region starting at 'gpa' and 'hpa'
	 * - length of the region 'len'
	 */
	spshift = 48;
	for (i = 3; i >= 0; i--) {
		spsize = 1UL << spshift;
		if ((dom->spsmask & (1 << i)) != 0 &&
		    (gpa & (spsize - 1)) == 0 &&
		    (hpa & (spsize - 1)) == 0 &&
		    (len >= spsize)) {
			break;
		}
		spshift -= 9;
	}

	ptp = dom->ptp;
	nlevels = dom->pt_levels;
	while (--nlevels >= 0) {
		ptpshift = 12 + nlevels * 9;
		ptpindex = (gpa >> ptpshift) & 0x1FF;

		/* We have reached the leaf mapping */
		if (spshift >= ptpshift) {
			break;
		}

		/*
		 * We are working on a non-leaf page table page.
		 *
		 * Create a downstream page table page if necessary and point
		 * to it from the current page table.
		 */
		if (ptp[ptpindex] == 0) {
			void *nlp = malloc(PAGE_SIZE, M_VTD, M_WAITOK | M_ZERO);
			ptp[ptpindex] = vtophys(nlp)| VTD_PTE_RD | VTD_PTE_WR;
		}

		ptp = (uint64_t *)PHYS_TO_DMAP(ptp[ptpindex] & VTD_PTE_ADDR_M);
	}

	if ((gpa & ((1UL << ptpshift) - 1)) != 0)
		panic("gpa 0x%lx and ptpshift %d mismatch", gpa, ptpshift);

	/*
	 * Update the 'gpa' -> 'hpa' mapping
	 */
	if (remove) {
		ptp[ptpindex] = 0;
	} else {
		ptp[ptpindex] = hpa | VTD_PTE_RD | VTD_PTE_WR;

		if (nlevels > 0)
			ptp[ptpindex] |= VTD_PTE_SUPERPAGE;
	}

	return (1UL << ptpshift);
}

static int
vtd_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len,
    uint64_t *res_len)
{

	*res_len = vtd_update_mapping(arg, gpa, hpa, len, CREATE_MAPPING);
	return (0);
}

static int
vtd_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len, uint64_t *res_len)
{

	*res_len = vtd_update_mapping(arg, gpa, 0, len, REMOVE_MAPPING);
	return (0);
}

static int
vtd_invalidate_tlb(void *dom)
{
	int i;
	struct vtdmap *vtdmap;

	/*
	 * Invalidate the IOTLB.
	 * XXX use domain-selective invalidation for IOTLB
	 */
	for (i = 0; i < drhd_num; i++) {
		vtdmap = vtdmaps[i];
		vtd_iotlb_global_invalidate(vtdmap);
	}
	return (0);
}

static void *
vtd_create_domain(vm_paddr_t maxaddr)
{
	struct domain *dom;
	vm_paddr_t addr;
	int tmp, i, gaw, agaw, sagaw, res, pt_levels, addrwidth;
	struct vtdmap *vtdmap;

	if (drhd_num <= 0)
		panic("vtd_create_domain: no dma remapping hardware available");

	/*
	 * Calculate AGAW.
	 * Section 3.4.2 "Adjusted Guest Address Width", Architecture Spec.
	 */
	addr = 0;
	for (gaw = 0; addr < maxaddr; gaw++)
		addr = 1ULL << gaw;

	res = (gaw - 12) % 9;
	if (res == 0)
		agaw = gaw;
	else
		agaw = gaw + 9 - res;

	if (agaw > 64)
		agaw = 64;

	/*
	 * Select the smallest Supported AGAW and the corresponding number
	 * of page table levels.
	 */
	pt_levels = 2;
	sagaw = 30;
	addrwidth = 0;

	tmp = ~0;
	for (i = 0; i < drhd_num; i++) {
		vtdmap = vtdmaps[i];
		/* take most compatible value */
		tmp &= VTD_CAP_SAGAW(vtdmap->cap);
	}

	for (i = 0; i < 5; i++) {
		if ((tmp & (1 << i)) != 0 && sagaw >= agaw)
			break;
		pt_levels++;
		addrwidth++;
		sagaw += 9;
		if (sagaw > 64)
			sagaw = 64;
	}

	if (i >= 5) {
		panic("vtd_create_domain: SAGAW 0x%x does not support AGAW %d",
		      tmp, agaw);
	}

	dom = malloc(sizeof(struct domain), M_VTD, M_ZERO | M_WAITOK);
	dom->pt_levels = pt_levels;
	dom->addrwidth = addrwidth;
	dom->id = domain_id();
	dom->maxaddr = maxaddr;
	dom->ptp = malloc(PAGE_SIZE, M_VTD, M_ZERO | M_WAITOK);
	if ((uintptr_t)dom->ptp & PAGE_MASK)
		panic("vtd_create_domain: ptp (%p) not page aligned", dom->ptp);

#ifdef notyet
	/*
	 * XXX superpage mappings for the iommu do not work correctly.
	 *
	 * By default all physical memory is mapped into the host_domain.
	 * When a VM is allocated wired memory the pages belonging to it
	 * are removed from the host_domain and added to the vm's domain.
	 *
	 * If the page being removed was mapped using a superpage mapping
	 * in the host_domain then we need to demote the mapping before
	 * removing the page.
	 *
	 * There is not any code to deal with the demotion at the moment
	 * so we disable superpage mappings altogether.
	 */
	dom->spsmask = ~0;
	for (i = 0; i < drhd_num; i++) {
		vtdmap = vtdmaps[i];
		/* take most compatible value */
		dom->spsmask &= VTD_CAP_SPS(vtdmap->cap);
	}
#endif

	SLIST_INSERT_HEAD(&domhead, dom, next);

	return (dom);
}

static void
vtd_free_ptp(uint64_t *ptp, int level)
{
	int i;
	uint64_t *nlp;

	if (level > 1) {
		for (i = 0; i < 512; i++) {
			if ((ptp[i] & (VTD_PTE_RD | VTD_PTE_WR)) == 0)
				continue;
			if ((ptp[i] & VTD_PTE_SUPERPAGE) != 0)
				continue;
			nlp = (uint64_t *)PHYS_TO_DMAP(ptp[i] & VTD_PTE_ADDR_M);
			vtd_free_ptp(nlp, level - 1);
		}
	}

	bzero(ptp, PAGE_SIZE);
	free(ptp, M_VTD);
}

static void
vtd_destroy_domain(void *arg)
{
	struct domain *dom;

	dom = arg;

	SLIST_REMOVE(&domhead, dom, domain, next);
	vtd_free_ptp(dom->ptp, dom->pt_levels);
	free(dom, M_VTD);
}

const struct iommu_ops iommu_ops_intel = {
	.init = vtd_init,
	.cleanup = vtd_cleanup,
	.enable = vtd_enable,
	.disable = vtd_disable,
	.create_domain = vtd_create_domain,
	.destroy_domain = vtd_destroy_domain,
	.create_mapping = vtd_create_mapping,
	.remove_mapping = vtd_remove_mapping,
	.add_device = vtd_add_device,
	.remove_device = vtd_remove_device,
	.invalidate_tlb = vtd_invalidate_tlb,
};