linux_netdev.c

/*-
 * Copyright (c) 2021 The FreeBSD Foundation
 * Copyright (c) 2022 Bjoern A. Zeeb
 *
 * This software was developed by Björn Zeeb under sponsorship from
 * the FreeBSD Foundation.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/types.h>
#include <sys/kernel.h>
#include <sys/sysctl.h>

#include <linux/bitops.h>
#include <linux/list.h>
#include <linux/netdevice.h>

MALLOC_DEFINE(M_NETDEV, "lkpindev", "Linux KPI netdevice compat");

#define	NAPI_LOCK_INIT(_ndev)		\
    mtx_init(&(_ndev)->napi_mtx, "napi_mtx", NULL, MTX_DEF)
#define	NAPI_LOCK_DESTROY(_ndev)	mtx_destroy(&(_ndev)->napi_mtx)
#define	NAPI_LOCK_ASSERT(_ndev)		mtx_assert(&(_ndev)->napi_mtx, MA_OWNED)
#define	NAPI_LOCK(_ndev)		mtx_lock(&(_ndev)->napi_mtx)
#define	NAPI_UNLOCK(_ndev)		mtx_unlock(&(_ndev)->napi_mtx)

/* -------------------------------------------------------------------------- */

#define LKPI_NAPI_FLAGS \
        "\20\1DISABLE_PENDING\2IS_SCHEDULED\3LOST_RACE_TRY_AGAIN"

/* #define	NAPI_DEBUG */
#ifdef NAPI_DEBUG
static int debug_napi;
SYSCTL_INT(_compat_linuxkpi, OID_AUTO, debug_napi, CTLFLAG_RWTUN,
    &debug_napi, 0, "NAPI debug level");

#define	DNAPI_TODO		0x01
#define	DNAPI_IMPROVE		0x02
#define	DNAPI_TRACE		0x10
#define	DNAPI_TRACE_TASK	0x20
#define	DNAPI_DIRECT_DISPATCH	0x1000

#define	NAPI_TRACE(_n)		if (debug_napi & DNAPI_TRACE)		\
    printf("NAPI_TRACE %s:%d %lu %p (%#jx %b)\n", __func__, __LINE__,	\
	jiffies, _n, (uintmax_t)(_n)->state,				\
	(int)(_n)->state, LKPI_NAPI_FLAGS)
#define	NAPI_TRACE2D(_n, _d)	if (debug_napi & DNAPI_TRACE)		\
    printf("NAPI_TRACE %s:%d %lu %p (%#jx %b) %d\n", __func__, __LINE__, \
	jiffies, _n, (uintmax_t)(_n)->state,				\
	(int)(_n)->state, LKPI_NAPI_FLAGS, _d)
#define	NAPI_TRACE_TASK(_n, _p, _c) if (debug_napi & DNAPI_TRACE_TASK)	\
    printf("NAPI_TRACE %s:%d %lu %p (%#jx %b) pending %d count %d "	\
	"rx_count %d\n", __func__, __LINE__,				\
	jiffies, _n, (uintmax_t)(_n)->state,				\
	(int)(_n)->state, LKPI_NAPI_FLAGS, _p, _c, (_n)->rx_count)
#define	NAPI_TODO()		if (debug_napi & DNAPI_TODO)		\
    printf("NAPI_TODO %s:%d %lu\n", __func__, __LINE__, jiffies)
#define	NAPI_IMPROVE()		if (debug_napi & DNAPI_IMPROVE)		\
    printf("NAPI_IMPROVE %s:%d %lu\n", __func__, __LINE__, jiffies)

#define	NAPI_DIRECT_DISPATCH()	((debug_napi & DNAPI_DIRECT_DISPATCH) != 0)
#else
#define	NAPI_TRACE(_n)			do { } while(0)
#define	NAPI_TRACE2D(_n, _d)		do { } while(0)
#define	NAPI_TRACE_TASK(_n, _p, _c)	do { } while(0)
#define	NAPI_TODO()			do { } while(0)
#define	NAPI_IMPROVE()			do { } while(0)

#define	NAPI_DIRECT_DISPATCH()		(0)
#endif

/* -------------------------------------------------------------------------- */

/*
 * Check if a poll is running or can run and and if the latter
 * make us as running.  That way we ensure that only one poll
 * can only ever run at the same time.  Returns true if no poll
 * was scheduled yet.
 */
bool
linuxkpi_napi_schedule_prep(struct napi_struct *napi)
{
	unsigned long old, new;

	NAPI_TRACE(napi);

	/* Can can only update/return if all flags agree. */
	do {
		old = READ_ONCE(napi->state);

		/* If we are stopping, cannot run again. */
		if ((old & BIT(LKPI_NAPI_FLAG_DISABLE_PENDING)) != 0) {
			NAPI_TRACE(napi);
			return (false);
		}

		new = old;
		/* We were already scheduled. Need to try again? */
		if ((old & BIT(LKPI_NAPI_FLAG_IS_SCHEDULED)) != 0)
			new |= BIT(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN);
		new |= BIT(LKPI_NAPI_FLAG_IS_SCHEDULED);

	} while (atomic_cmpset_acq_long(&napi->state, old, new) == 0);

	NAPI_TRACE(napi);
        return ((old & BIT(LKPI_NAPI_FLAG_IS_SCHEDULED)) == 0);
}

static void
lkpi___napi_schedule_dd(struct napi_struct *napi)
{
	unsigned long old, new;
	int rc;

	rc = 0;
again:
	NAPI_TRACE2D(napi, rc);
	if (napi->poll != NULL)
		rc = napi->poll(napi, napi->budget);
	napi->rx_count += rc;

	/* Check if interrupts are still disabled, more work to do. */
	/* Bandaid for now. */
	if (rc >= napi->budget)
		goto again;

	/* Bandaid for now. */
	if (test_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &napi->state))
		goto again;

	do {
		new = old = READ_ONCE(napi->state);
		clear_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &new);
		clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &new);
	} while (atomic_cmpset_acq_long(&napi->state, old, new) == 0);

	NAPI_TRACE2D(napi, rc);
}

void
linuxkpi___napi_schedule(struct napi_struct *napi)
{
	int rc;

	NAPI_TRACE(napi);
	if (test_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->state)) {
		clear_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &napi->state);
		clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state);
		NAPI_TRACE(napi);
		return;
	}

	if (NAPI_DIRECT_DISPATCH()) {
		lkpi___napi_schedule_dd(napi);
	} else {
		rc = taskqueue_enqueue(napi->dev->napi_tq, &napi->napi_task);
		NAPI_TRACE2D(napi, rc);
		if (rc != 0) {
			/* Should we assert EPIPE? */
			return;
		}
	}
}

bool
linuxkpi_napi_schedule(struct napi_struct *napi)
{

	NAPI_TRACE(napi);

	/*
	 * iwlwifi calls this sequence instead of napi_schedule()
	 * to be able to test the prep result.
	 */
	if (napi_schedule_prep(napi)) {
		__napi_schedule(napi);
		return (true);
	}

	return (false);
}

void
linuxkpi_napi_reschedule(struct napi_struct *napi)
{

	NAPI_TRACE(napi);

	/* Not sure what is different to napi_schedule yet. */
	if (napi_schedule_prep(napi))
		__napi_schedule(napi);
}

bool
linuxkpi_napi_complete_done(struct napi_struct *napi, int ret)
{
	unsigned long old, new;

	NAPI_TRACE(napi);
	if (NAPI_DIRECT_DISPATCH())
		return (true);

	do {
		new = old = READ_ONCE(napi->state);

		/*
		 * If we lost a race before, we need to re-schedule.
		 * Leave IS_SCHEDULED set essentially doing "_prep".
		 */
		if (!test_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &old))
			clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &new);
		clear_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &new);
	} while (atomic_cmpset_acq_long(&napi->state, old, new) == 0);

	NAPI_TRACE(napi);

	/* Someone tried to schedule while poll was running. Re-sched. */
	if (test_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &old)) {
		__napi_schedule(napi);
		return (false);
	}

	return (true);
}

bool
linuxkpi_napi_complete(struct napi_struct *napi)
{

	NAPI_TRACE(napi);
	return (napi_complete_done(napi, 0));
}

void
linuxkpi_napi_disable(struct napi_struct *napi)
{
	NAPI_TRACE(napi);
	set_bit(LKPI_NAPI_FLAG_DISABLE_PENDING, &napi->state);
	while (test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state))
		pause_sbt("napidslp", SBT_1MS, 0, C_HARDCLOCK);
	clear_bit(LKPI_NAPI_FLAG_DISABLE_PENDING, &napi->state);
}

void
linuxkpi_napi_enable(struct napi_struct *napi)
{

	NAPI_TRACE(napi);
	KASSERT(!test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state),
	    ("%s: enabling napi %p already scheduled\n", __func__, napi));
	mb();
	/* Let us be scheduled. */
	clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state);
}

void
linuxkpi_napi_synchronize(struct napi_struct *napi)
{
	NAPI_TRACE(napi);
#if defined(SMP)
	/* Check & sleep while a napi is scheduled. */
	while (test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state))
		pause_sbt("napisslp", SBT_1MS, 0, C_HARDCLOCK);
#else
	mb();
#endif
}

/* -------------------------------------------------------------------------- */

static void
lkpi_napi_task(void *ctx, int pending)
{
	struct napi_struct *napi;
	int count;

	KASSERT(ctx != NULL, ("%s: napi %p, pending %d\n",
	    __func__, ctx, pending));
	napi = ctx;
	KASSERT(napi->poll != NULL, ("%s: napi %p poll is NULL\n",
	    __func__, napi));

	NAPI_TRACE_TASK(napi, pending, napi->budget);
	count = napi->poll(napi, napi->budget);
	napi->rx_count += count;
	NAPI_TRACE_TASK(napi, pending, count);

	/*
	 * We must not check against count < pending here.  There are situations
	 * when a driver may "poll" and we may not have any work to do and that
	 * would make us re-schedule ourseless for ever.
	 */
	if (count >= napi->budget) {
		/*
		 * Have to re-schedule ourselves.  napi_complete() was not run
		 * in this case which means we are still SCHEDULED.
		 * In order to queue another task we have to directly call
		 * __napi_schedule() without _prep() in the way.
		 */
		__napi_schedule(napi);
	}
}

/* -------------------------------------------------------------------------- */

void
linuxkpi_netif_napi_add(struct net_device *ndev, struct napi_struct *napi,
    int(*napi_poll)(struct napi_struct *, int))
{

	napi->dev = ndev;
	napi->poll = napi_poll;
	napi->budget = NAPI_POLL_WEIGHT;

	INIT_LIST_HEAD(&napi->rx_list);
	napi->rx_count = 0;

	TASK_INIT(&napi->napi_task, 0, lkpi_napi_task, napi);

	NAPI_LOCK(ndev);
	TAILQ_INSERT_TAIL(&ndev->napi_head, napi, entry);
	NAPI_UNLOCK(ndev);

	/* Anything else to do on the ndev? */
	clear_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->state);
}

static void
lkpi_netif_napi_del_locked(struct napi_struct *napi)
{
	struct net_device *ndev;

	ndev = napi->dev;
	NAPI_LOCK_ASSERT(ndev);

	set_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->state);
	TAILQ_REMOVE(&ndev->napi_head, napi, entry);
	while (taskqueue_cancel(ndev->napi_tq, &napi->napi_task, NULL) != 0)
		taskqueue_drain(ndev->napi_tq, &napi->napi_task);
}

void
linuxkpi_netif_napi_del(struct napi_struct *napi)
{
	struct net_device *ndev;

	ndev = napi->dev;
	NAPI_LOCK(ndev);
	lkpi_netif_napi_del_locked(napi);
	NAPI_UNLOCK(ndev);
}

/* -------------------------------------------------------------------------- */

void
linuxkpi_init_dummy_netdev(struct net_device *ndev)
{

	memset(ndev, 0, sizeof(*ndev));

	ndev->reg_state = NETREG_DUMMY;
	NAPI_LOCK_INIT(ndev);
	TAILQ_INIT(&ndev->napi_head);
	/* Anything else? */

	ndev->napi_tq = taskqueue_create("tq_ndev_napi", M_WAITOK,
	    taskqueue_thread_enqueue, &ndev->napi_tq);
	/* One thread for now. */
	(void) taskqueue_start_threads(&ndev->napi_tq, 1, PWAIT,
	    "ndev napi taskq");
}

struct net_device *
linuxkpi_alloc_netdev(size_t len, const char *name, uint32_t flags,
    void(*setup_func)(struct net_device *))
{
	struct net_device *ndev;

	ndev = malloc(sizeof(*ndev) + len, M_NETDEV, M_NOWAIT);
	if (ndev == NULL)
		return (ndev);

	/* Always first as it zeros! */
	linuxkpi_init_dummy_netdev(ndev);

	strlcpy(ndev->name, name, sizeof(*ndev->name));

	/* This needs extending as we support more. */

	if (setup_func != NULL)
		setup_func(ndev);

	return (ndev);
}

void
linuxkpi_free_netdev(struct net_device *ndev)
{
	struct napi_struct *napi, *temp;

	NAPI_LOCK(ndev);
	TAILQ_FOREACH_SAFE(napi, &ndev->napi_head, entry, temp) {
		lkpi_netif_napi_del_locked(napi);
	}
	NAPI_UNLOCK(ndev);

	taskqueue_free(ndev->napi_tq);
	ndev->napi_tq = NULL;
	NAPI_LOCK_DESTROY(ndev);

	/* This needs extending as we support more. */

	free(ndev, M_NETDEV);
}