opnsense-src/sys/dev/firmware/arm/scmi.c

/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2022 Ruslan Bukin <br@bsdpad.com>
 * Copyright (c) 2023 Arm Ltd
 *
 * This work was supported by Innovate UK project 105694, "Digital Security
 * by Design (DSbD) Technology Platform Prototype".
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/_bitset.h>
#include <sys/bitset.h>
#include <sys/bus.h>
#include <sys/cpu.h>
#include <sys/endian.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/queue.h>
#include <sys/refcount.h>
#include <sys/sdt.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>

#include <dev/clk/clk.h>
#include <dev/fdt/simplebus.h>
#include <dev/fdt/fdt_common.h>
#include <dev/ofw/ofw_bus_subr.h>

#include "scmi.h"
#include "scmi_protocols.h"

SDT_PROVIDER_DEFINE(scmi);
SDT_PROBE_DEFINE3(scmi, func, scmi_req_alloc, req_alloc,
    "int", "int", "int");
SDT_PROBE_DEFINE3(scmi, func, scmi_req_free_unlocked, req_alloc,
    "int", "int", "int");
SDT_PROBE_DEFINE3(scmi, func, scmi_req_get, req_alloc,
    "int", "int", "int");
SDT_PROBE_DEFINE3(scmi, func, scmi_req_put, req_alloc,
    "int", "int", "int");
SDT_PROBE_DEFINE5(scmi, func, scmi_request_tx, xfer_track,
    "int", "int", "int", "int", "int");
SDT_PROBE_DEFINE5(scmi, entry, scmi_wait_for_response, xfer_track,
    "int", "int", "int", "int", "int");
SDT_PROBE_DEFINE5(scmi, exit, scmi_wait_for_response, xfer_track,
    "int", "int", "int", "int", "int");
SDT_PROBE_DEFINE2(scmi, func, scmi_rx_irq_callback, hdr_dump,
    "int", "int");
SDT_PROBE_DEFINE5(scmi, func, scmi_process_response, xfer_track,
    "int", "int", "int", "int", "int");

#define SCMI_MAX_TOKEN		1024

#define	SCMI_HDR_TOKEN_S		18
#define SCMI_HDR_TOKEN_BF		(0x3ff)
#define	SCMI_HDR_TOKEN_M		(SCMI_HDR_TOKEN_BF << SCMI_HDR_TOKEN_S)

#define	SCMI_HDR_PROTOCOL_ID_S		10
#define	SCMI_HDR_PROTOCOL_ID_BF		(0xff)
#define	SCMI_HDR_PROTOCOL_ID_M		\
    (SCMI_HDR_PROTOCOL_ID_BF << SCMI_HDR_PROTOCOL_ID_S)

#define	SCMI_HDR_MESSAGE_TYPE_S		8
#define	SCMI_HDR_MESSAGE_TYPE_BF	(0x3)
#define	SCMI_HDR_MESSAGE_TYPE_M		\
    (SCMI_HDR_MESSAGE_TYPE_BF << SCMI_HDR_MESSAGE_TYPE_S)

#define	SCMI_HDR_MESSAGE_ID_S		0
#define	SCMI_HDR_MESSAGE_ID_BF		(0xff)
#define	SCMI_HDR_MESSAGE_ID_M		\
    (SCMI_HDR_MESSAGE_ID_BF << SCMI_HDR_MESSAGE_ID_S)

#define SCMI_MSG_TYPE_CMD	0
#define SCMI_MSG_TYPE_DRESP	2
#define SCMI_MSG_TYPE_NOTIF	3

#define SCMI_MSG_TYPE_CHECK(_h, _t)					\
    ((((_h) & SCMI_HDR_MESSAGE_TYPE_M) >> SCMI_HDR_MESSAGE_TYPE_S) == (_t))

#define SCMI_IS_MSG_TYPE_NOTIF(h)					\
    SCMI_MSG_TYPE_CHECK((h), SCMI_MSG_TYPE_NOTIF)
#define SCMI_IS_MSG_TYPE_DRESP(h)					\
    SCMI_MSG_TYPE_CHECK((h), SCMI_MSG_TYPE_DRESP)

#define SCMI_MSG_TOKEN(_hdr)		\
    (((_hdr) & SCMI_HDR_TOKEN_M) >> SCMI_HDR_TOKEN_S)
#define SCMI_MSG_PROTOCOL_ID(_hdr)		\
    (((_hdr) & SCMI_HDR_PROTOCOL_ID_M) >> SCMI_HDR_PROTOCOL_ID_S)
#define SCMI_MSG_MESSAGE_ID(_hdr)		\
    (((_hdr) & SCMI_HDR_MESSAGE_ID_M) >> SCMI_HDR_MESSAGE_ID_S)
#define SCMI_MSG_TYPE(_hdr)		\
    (((_hdr) & SCMI_HDR_TYPE_ID_M) >> SCMI_HDR_TYPE_ID_S)

struct scmi_req {
	int		cnt;
	bool		timed_out;
	bool		use_polling;
	bool		done;
	bool		is_raw;
	device_t	dev;
	struct task	tsk;
	struct mtx	mtx;
	LIST_ENTRY(scmi_req)	next;
	int		protocol_id;
	int		message_id;
	int		token;
	uint32_t	header;
	struct scmi_msg msg;
};

#define tsk_to_req(t)	__containerof((t), struct scmi_req, tsk)
#define buf_to_msg(b)	__containerof((b), struct scmi_msg, payld)
#define msg_to_req(m)	__containerof((m), struct scmi_req, msg)
#define buf_to_req(b)	msg_to_req(buf_to_msg(b))

LIST_HEAD(reqs_head, scmi_req);

struct scmi_reqs_pool {
	struct mtx		mtx;
	struct reqs_head	head;
};

BITSET_DEFINE(_scmi_tokens, SCMI_MAX_TOKEN);
LIST_HEAD(inflight_head, scmi_req);
#define	REQHASH(_sc, _tk)		\
    (&((_sc)->trs->inflight_ht[(_tk) & (_sc)->trs->inflight_mask]))

struct scmi_transport {
	unsigned long		next_id;
	struct _scmi_tokens	avail_tokens;
	struct inflight_head	*inflight_ht;
	unsigned long		inflight_mask;
	struct scmi_reqs_pool	*chans[SCMI_CHAN_MAX];
	struct mtx		mtx;
};

static void		scmi_transport_configure(struct scmi_transport_desc *, phandle_t);
static int		scmi_transport_init(struct scmi_softc *, phandle_t);
static void		scmi_transport_cleanup(struct scmi_softc *);
static void		scmi_req_async_waiter(void *, int);
static struct scmi_reqs_pool *scmi_reqs_pool_allocate(device_t, const int,
			    const int);
static void		scmi_reqs_pool_free(struct scmi_reqs_pool *);
static struct scmi_req	*scmi_req_alloc(struct scmi_softc *, enum scmi_chan);
static struct scmi_req	*scmi_req_initialized_alloc(device_t, int, int);
static void		scmi_req_free_unlocked(struct scmi_softc *,
			    enum scmi_chan, struct scmi_req *);
static void		scmi_req_get(struct scmi_softc *, struct scmi_req *);
static void		scmi_req_put(struct scmi_softc *, struct scmi_req *);
static int		scmi_token_pick(struct scmi_softc *);
static int		scmi_token_reserve(struct scmi_softc *, uint16_t);
static void		scmi_token_release_unlocked(struct scmi_softc *, int);
static int		scmi_req_track_inflight(struct scmi_softc *,
			    struct scmi_req *);
static int		scmi_req_drop_inflight(struct scmi_softc *,
			    struct scmi_req *);
static struct scmi_req *scmi_req_lookup_inflight(struct scmi_softc *, uint32_t);

static int		scmi_wait_for_response(struct scmi_softc *,
			    struct scmi_req *, void **);
static void		scmi_process_response(struct scmi_softc *, uint32_t,
			    unsigned int);

int
scmi_attach(device_t dev)
{
	struct sysctl_oid *sysctl_trans;
	struct scmi_softc *sc;
	phandle_t node;
	int error;

	sc = device_get_softc(dev);
	sc->dev = dev;

	node = ofw_bus_get_node(dev);
	if (node == -1)
		return (ENXIO);

	simplebus_init(dev, node);

	error = scmi_transport_init(sc, node);
	if (error != 0)
		return (error);

	device_printf(dev, "Transport - max_msg:%d  max_payld_sz:%lu  reply_timo_ms:%d\n",
	    SCMI_MAX_MSG(sc), SCMI_MAX_MSG_PAYLD_SIZE(sc), SCMI_MAX_MSG_TIMEOUT_MS(sc));

	sc->sysctl_root = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_hw),
	    OID_AUTO, "scmi", CTLFLAG_RD, 0, "SCMI root");
	sysctl_trans = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(sc->sysctl_root),
	    OID_AUTO, "transport", CTLFLAG_RD, 0, "SCMI Transport properties");
	SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(sysctl_trans), OID_AUTO, "max_msg",
	    CTLFLAG_RD, &sc->trs_desc.max_msg, 0, "SCMI Max number of inflight messages");
	SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(sysctl_trans), OID_AUTO, "max_msg_size",
	    CTLFLAG_RD, &sc->trs_desc.max_payld_sz, 0, "SCMI Max message payload size");
	SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(sysctl_trans), OID_AUTO, "max_rx_timeout_ms",
	    CTLFLAG_RD, &sc->trs_desc.reply_timo_ms, 0, "SCMI Max message RX timeout ms");

	/*
	 * Allow devices to identify.
	 */
	bus_identify_children(dev);

	/*
	 * Now walk the OFW tree and attach top-level devices.
	 */
	for (node = OF_child(node); node > 0; node = OF_peer(node))
		simplebus_add_device(dev, node, 0, NULL, -1, NULL);

	bus_attach_children(dev);

	return (0);
}

static int
scmi_detach(device_t dev)
{
	struct scmi_softc *sc;

	sc = device_get_softc(dev);
	scmi_transport_cleanup(sc);

	return (0);
}

static device_method_t scmi_methods[] = {
	DEVMETHOD(device_attach,	scmi_attach),
	DEVMETHOD(device_detach,	scmi_detach),

	DEVMETHOD_END
};

DEFINE_CLASS_1(scmi, scmi_driver, scmi_methods, sizeof(struct scmi_softc),
    simplebus_driver);

DRIVER_MODULE(scmi, simplebus, scmi_driver, 0, 0);
MODULE_VERSION(scmi, 1);

static struct scmi_reqs_pool *
scmi_reqs_pool_allocate(device_t dev, const int max_msg, const int max_payld_sz)
{
	struct scmi_reqs_pool *rp;
	struct scmi_req *req;

	rp = malloc(sizeof(*rp), M_DEVBUF, M_ZERO | M_WAITOK);

	LIST_INIT(&rp->head);
	for (int i = 0; i < max_msg; i++) {
		req = malloc(sizeof(*req) + max_payld_sz,
		    M_DEVBUF, M_ZERO | M_WAITOK);

		req->dev = dev;
		req->tsk.ta_context = &req->tsk;
		req->tsk.ta_func = scmi_req_async_waiter;

		mtx_init(&req->mtx, "req", "SCMI", MTX_SPIN);
		LIST_INSERT_HEAD(&rp->head, req, next);
	}

	mtx_init(&rp->mtx, "reqs_pool", "SCMI", MTX_SPIN);

	return (rp);
}

static void
scmi_reqs_pool_free(struct scmi_reqs_pool *rp)
{
	struct scmi_req *req;

	LIST_FOREACH(req, &rp->head, next) {
		mtx_destroy(&req->mtx);
		free(req, M_DEVBUF);
	}

	mtx_destroy(&rp->mtx);
	free(rp, M_DEVBUF);
}

static void
scmi_transport_configure(struct scmi_transport_desc *td, phandle_t node)
{
	if (OF_getencprop(node, "arm,max-msg", &td->max_msg, sizeof(td->max_msg)) == -1)
		td->max_msg = SCMI_DEF_MAX_MSG;

	if (OF_getencprop(node, "arm,max-msg-size", &td->max_payld_sz,
	    sizeof(td->max_payld_sz)) == -1)
		td->max_payld_sz = SCMI_DEF_MAX_MSG_PAYLD_SIZE;
}

static int
scmi_transport_init(struct scmi_softc *sc, phandle_t node)
{
	struct scmi_transport_desc *td = &sc->trs_desc;
	struct scmi_transport *trs;
	int ret;

	trs = malloc(sizeof(*trs), M_DEVBUF, M_ZERO | M_WAITOK);

	scmi_transport_configure(td, node);

	BIT_FILL(SCMI_MAX_TOKEN, &trs->avail_tokens);
	mtx_init(&trs->mtx, "tokens", "SCMI", MTX_SPIN);

	trs->inflight_ht = hashinit(td->max_msg, M_DEVBUF, &trs->inflight_mask);

	trs->chans[SCMI_CHAN_A2P] =
	    scmi_reqs_pool_allocate(sc->dev, td->max_msg, td->max_payld_sz);
	if (trs->chans[SCMI_CHAN_A2P] == NULL) {
		free(trs, M_DEVBUF);
		return (ENOMEM);
	}

	trs->chans[SCMI_CHAN_P2A] =
	    scmi_reqs_pool_allocate(sc->dev, td->max_msg, td->max_payld_sz);
	if (trs->chans[SCMI_CHAN_P2A] == NULL) {
		scmi_reqs_pool_free(trs->chans[SCMI_CHAN_A2P]);
		free(trs, M_DEVBUF);
		return (ENOMEM);
	}

	sc->trs = trs;
	ret = SCMI_TRANSPORT_INIT(sc->dev);
	if (ret != 0) {
		scmi_reqs_pool_free(trs->chans[SCMI_CHAN_A2P]);
		scmi_reqs_pool_free(trs->chans[SCMI_CHAN_P2A]);
		free(trs, M_DEVBUF);
		return (ret);
	}

	/* Use default transport timeout if not overridden by OF */
	OF_getencprop(node, "arm,max-rx-timeout-ms", &td->reply_timo_ms,
	    sizeof(td->reply_timo_ms));

	return (0);
}

static void
scmi_transport_cleanup(struct scmi_softc *sc)
{

	SCMI_TRANSPORT_CLEANUP(sc->dev);
	mtx_destroy(&sc->trs->mtx);
	hashdestroy(sc->trs->inflight_ht, M_DEVBUF, sc->trs->inflight_mask);
	scmi_reqs_pool_free(sc->trs->chans[SCMI_CHAN_A2P]);
	scmi_reqs_pool_free(sc->trs->chans[SCMI_CHAN_P2A]);
	free(sc->trs, M_DEVBUF);
}

static struct scmi_req *
scmi_req_initialized_alloc(device_t dev, int tx_payld_sz, int rx_payld_sz)
{
	struct scmi_softc *sc;
	struct scmi_req *req;

	sc = device_get_softc(dev);

	if (tx_payld_sz > SCMI_MAX_MSG_PAYLD_SIZE(sc) ||
	    rx_payld_sz > SCMI_MAX_MSG_REPLY_SIZE(sc)) {
		device_printf(dev, "Unsupported payload size. Drop.\n");
		return (NULL);
	}

	/* Pick one from free list */
	req = scmi_req_alloc(sc, SCMI_CHAN_A2P);
	if (req == NULL)
		return (NULL);

	req->msg.tx_len = sizeof(req->msg.hdr) + tx_payld_sz;
	req->msg.rx_len = rx_payld_sz ?
	    rx_payld_sz + 2 * sizeof(uint32_t) : SCMI_MAX_MSG_SIZE(sc);

	return (req);
}

static struct scmi_req *
scmi_req_alloc(struct scmi_softc *sc, enum scmi_chan ch_idx)
{
	struct scmi_reqs_pool *rp;
	struct scmi_req *req = NULL;

	rp = sc->trs->chans[ch_idx];
	mtx_lock_spin(&rp->mtx);
	if (!LIST_EMPTY(&rp->head)) {
		req = LIST_FIRST(&rp->head);
		LIST_REMOVE_HEAD(&rp->head, next);
	}
	mtx_unlock_spin(&rp->mtx);

	if (req != NULL) {
		refcount_init(&req->cnt, 1);
		SDT_PROBE3(scmi, func, scmi_req_alloc, req_alloc,
		    req, refcount_load(&req->cnt), -1);
	}

	return (req);
}

static void
scmi_req_free_unlocked(struct scmi_softc *sc, enum scmi_chan ch_idx,
    struct scmi_req *req)
{
	struct scmi_reqs_pool *rp;

	rp = sc->trs->chans[ch_idx];
	mtx_lock_spin(&rp->mtx);
	req->timed_out = false;
	req->done = false;
	req->is_raw = false;
	refcount_init(&req->cnt, 0);
	LIST_INSERT_HEAD(&rp->head, req, next);
	mtx_unlock_spin(&rp->mtx);

	SDT_PROBE3(scmi, func, scmi_req_free_unlocked, req_alloc,
	    req, refcount_load(&req->cnt), -1);
}

static void
scmi_req_get(struct scmi_softc *sc, struct scmi_req *req)
{
	bool ok;

	mtx_lock_spin(&req->mtx);
	ok = refcount_acquire_if_not_zero(&req->cnt);
	mtx_unlock_spin(&req->mtx);

	if (!ok)
		device_printf(sc->dev, "%s() -- BAD REFCOUNT\n", __func__);

	SDT_PROBE3(scmi, func, scmi_req_get, req_alloc,
	    req, refcount_load(&req->cnt), SCMI_MSG_TOKEN(req->msg.hdr));

	return;
}

static void
scmi_req_put(struct scmi_softc *sc, struct scmi_req *req)
{
	mtx_lock_spin(&req->mtx);
	if (!refcount_release_if_not_last(&req->cnt)) {
		req->protocol_id = 0;
		req->message_id = 0;
		req->token = 0;
		req->header = 0;
		bzero(&req->msg, sizeof(req->msg) + SCMI_MAX_MSG_PAYLD_SIZE(sc));
		scmi_req_free_unlocked(sc, SCMI_CHAN_A2P, req);
	} else {
		SDT_PROBE3(scmi, func, scmi_req_put, req_alloc,
		    req, refcount_load(&req->cnt), SCMI_MSG_TOKEN(req->msg.hdr));
	}
	mtx_unlock_spin(&req->mtx);
}

static int
scmi_token_pick(struct scmi_softc *sc)
{
	unsigned long next_msg_id, token;

	mtx_lock_spin(&sc->trs->mtx);
	/*
	 * next_id is a monotonically increasing unsigned long that can be used
	 * for tracing purposes; next_msg_id is a 10-bit sequence number derived
	 * from it.
	 */
	next_msg_id = sc->trs->next_id++ & SCMI_HDR_TOKEN_BF;
	token = BIT_FFS_AT(SCMI_MAX_TOKEN, &sc->trs->avail_tokens, next_msg_id);
	if (token != 0)
		BIT_CLR(SCMI_MAX_TOKEN, token - 1, &sc->trs->avail_tokens);
	mtx_unlock_spin(&sc->trs->mtx);

	/*
	 * BIT_FFS_AT returns 1-indexed values, so 0 means failure to find a
	 * free slot: all possible SCMI messages are in-flight using all of the
	 * SCMI_MAX_TOKEN sequence numbers.
	 */
	if (!token)
		return (-EBUSY);

	return ((int)(token - 1));
}

static int
scmi_token_reserve(struct scmi_softc *sc, uint16_t candidate)
{
	int token = -EBUSY, retries = 3;

	do {
		mtx_lock_spin(&sc->trs->mtx);
		if (BIT_ISSET(SCMI_MAX_TOKEN, candidate, &sc->trs->avail_tokens)) {
			BIT_CLR(SCMI_MAX_TOKEN, candidate, &sc->trs->avail_tokens);
			token = candidate;
			sc->trs->next_id++;
		}
		mtx_unlock_spin(&sc->trs->mtx);
		if (token == candidate || retries-- == 0)
			break;

		pause("scmi_tk_reserve", hz);
	} while (1);

	return (token);
}

static void
scmi_token_release_unlocked(struct scmi_softc *sc, int token)
{

	BIT_SET(SCMI_MAX_TOKEN, token, &sc->trs->avail_tokens);
}

static int
scmi_finalize_req(struct scmi_softc *sc, struct scmi_req *req)
{
	if (!req->is_raw)
		req->token = scmi_token_pick(sc);
	else
		req->token = scmi_token_reserve(sc, SCMI_MSG_TOKEN(req->msg.hdr));

	if (req->token < 0)
		return (EBUSY);

	if (!req->is_raw) {
		req->msg.hdr = req->message_id;
		req->msg.hdr |= SCMI_MSG_TYPE_CMD << SCMI_HDR_MESSAGE_TYPE_S;
		req->msg.hdr |= req->protocol_id << SCMI_HDR_PROTOCOL_ID_S;
		req->msg.hdr |= req->token << SCMI_HDR_TOKEN_S;
	}

	/* Save requested header */
	req->header = req->msg.hdr;

	return (0);
}

static int
scmi_req_track_inflight(struct scmi_softc *sc, struct scmi_req *req)
{
	int error;

	/* build hdr, pick token */
	error = scmi_finalize_req(sc, req);
	if (error != 0)
		return (error);

	/* Bump refcount to get hold of this in-flight transaction */
	scmi_req_get(sc, req);
	/* Register in the inflight hashtable */
	mtx_lock_spin(&sc->trs->mtx);
	LIST_INSERT_HEAD(REQHASH(sc, req->token), req, next);
	mtx_unlock_spin(&sc->trs->mtx);

	return (0);
}

static int
scmi_req_drop_inflight(struct scmi_softc *sc, struct scmi_req *req)
{

	/* Remove from inflight hashtable at first ... */
	mtx_lock_spin(&sc->trs->mtx);
	LIST_REMOVE(req, next);
	scmi_token_release_unlocked(sc, req->token);
	mtx_unlock_spin(&sc->trs->mtx);
	/* ...and drop refcount..potentially releasing *req */
	scmi_req_put(sc, req);

	return (0);
}

static struct scmi_req *
scmi_req_lookup_inflight(struct scmi_softc *sc, uint32_t hdr)
{
	struct scmi_req *req = NULL;
	unsigned int token;

	token = SCMI_MSG_TOKEN(hdr);
	mtx_lock_spin(&sc->trs->mtx);
	LIST_FOREACH(req, REQHASH(sc, token), next) {
		if (req->token == token)
			break;
	}
	mtx_unlock_spin(&sc->trs->mtx);

	return (req);
}

static void
scmi_process_response(struct scmi_softc *sc, uint32_t hdr, uint32_t rx_len)
{
	bool timed_out = false;
	struct scmi_req *req;

	req = scmi_req_lookup_inflight(sc, hdr);
	if (req == NULL) {
		device_printf(sc->dev,
		    "Unexpected reply with header |%X| - token: 0x%X Drop.\n",
		    hdr, SCMI_MSG_TOKEN(hdr));
		return;
	}

	SDT_PROBE5(scmi, func, scmi_process_response, xfer_track, req,
	    SCMI_MSG_PROTOCOL_ID(req->msg.hdr), SCMI_MSG_MESSAGE_ID(req->msg.hdr),
	    SCMI_MSG_TOKEN(req->msg.hdr), req->timed_out);

	mtx_lock_spin(&req->mtx);
	req->done = true;
	req->msg.rx_len = rx_len;
	if (!req->timed_out) {
		/*
		 * Consider the case in which a polled message is picked
		 * by chance on the IRQ path on another CPU: setting poll_done
		 * will terminate the other poll loop.
		 */
		if (!req->msg.polling)
			wakeup(req);
		else
			atomic_store_rel_int(&req->msg.poll_done, 1);
	} else {
		timed_out = true;
	}
	mtx_unlock_spin(&req->mtx);

	if (timed_out)
		device_printf(sc->dev,
		    "Late reply for timed-out request - token: 0x%X. Ignore.\n",
		    req->token);

	/*
	 * In case of a late reply to a timed-out transaction this will
	 * finally free the pending scmi_req
	 */
	scmi_req_drop_inflight(sc, req);
}

void
scmi_rx_irq_callback(device_t dev, void *chan, uint32_t hdr, uint32_t rx_len)
{
	struct scmi_softc *sc;

	sc = device_get_softc(dev);

	SDT_PROBE2(scmi, func, scmi_rx_irq_callback, hdr_dump, hdr, rx_len);

	if (SCMI_IS_MSG_TYPE_NOTIF(hdr) || SCMI_IS_MSG_TYPE_DRESP(hdr)) {
		device_printf(dev, "DRESP/NOTIF unsupported. Drop.\n");
		SCMI_CLEAR_CHANNEL(dev, chan);
		return;
	}

	scmi_process_response(sc, hdr, rx_len);
}

static int
scmi_wait_for_response(struct scmi_softc *sc, struct scmi_req *req, void **out)
{
	unsigned int reply_timo_ms = SCMI_MAX_MSG_TIMEOUT_MS(sc);
	int ret;

	SDT_PROBE5(scmi, entry, scmi_wait_for_response, xfer_track, req,
	    SCMI_MSG_PROTOCOL_ID(req->msg.hdr), SCMI_MSG_MESSAGE_ID(req->msg.hdr),
	    SCMI_MSG_TOKEN(req->msg.hdr), reply_timo_ms);

	if (req->msg.polling) {
		bool needs_drop;

		ret = SCMI_POLL_MSG(sc->dev, &req->msg, reply_timo_ms);
		/*
		 * Drop reference to successfully polled req unless it had
		 * already also been processed on the IRQ path.
		 * Addresses a possible race-condition between polling and
		 * interrupt reception paths.
		 */
		mtx_lock_spin(&req->mtx);
		needs_drop = (ret == 0) && !req->done;
		req->timed_out = ret != 0;
		mtx_unlock_spin(&req->mtx);
		if (needs_drop)
			scmi_req_drop_inflight(sc, req);
		if (ret == 0 && req->msg.hdr != req->header) {
			device_printf(sc->dev,
			    "Malformed reply with header |%08X|. Expected: |%08X|Drop.\n",
			    le32toh(req->msg.hdr), le32toh(req->header));
		}
	} else {
		ret = tsleep(req, 0, "scmi_wait4", (reply_timo_ms * hz) / 1000);
		/* Check for lost wakeups since there is no associated lock */
		mtx_lock_spin(&req->mtx);
		if (ret != 0 && req->done)
			ret = 0;
		req->timed_out = ret != 0;
		mtx_unlock_spin(&req->mtx);
	}

	if (ret == 0) {
		SCMI_COLLECT_REPLY(sc->dev, &req->msg);
		if (req->msg.payld[0] != 0)
			ret = req->msg.payld[0];
		if (out != NULL)
			*out = &req->msg.payld[SCMI_MSG_HDR_SIZE];
	} else {
		device_printf(sc->dev,
		    "Request for token 0x%X timed-out.\n", req->token);
	}

	SCMI_TX_COMPLETE(sc->dev, NULL);

	SDT_PROBE5(scmi, exit, scmi_wait_for_response, xfer_track, req,
	    SCMI_MSG_PROTOCOL_ID(req->msg.hdr), SCMI_MSG_MESSAGE_ID(req->msg.hdr),
	    SCMI_MSG_TOKEN(req->msg.hdr), req->timed_out);

	return (ret);
}

void *
scmi_buf_get(device_t dev, uint8_t protocol_id, uint8_t message_id,
    int tx_payld_sz, int rx_payld_sz)
{
	struct scmi_req *req;

	/* Pick a pre-built req */
	req = scmi_req_initialized_alloc(dev, tx_payld_sz, rx_payld_sz);
	if (req == NULL)
		return (NULL);

	req->protocol_id = protocol_id & SCMI_HDR_PROTOCOL_ID_BF;
	req->message_id = message_id & SCMI_HDR_MESSAGE_ID_BF;

	return (&req->msg.payld[0]);
}

void
scmi_buf_put(device_t dev, void *buf)
{
	struct scmi_softc *sc;
	struct scmi_req *req;

	sc = device_get_softc(dev);

	req = buf_to_req(buf);
	scmi_req_put(sc, req);
}

struct scmi_msg *
scmi_msg_get(device_t dev, int tx_payld_sz, int rx_payld_sz)
{
	struct scmi_req *req;

	/* Pick a pre-built req */
	req = scmi_req_initialized_alloc(dev, tx_payld_sz, rx_payld_sz);
	if (req == NULL)
		return (NULL);

	req->is_raw = true;

	return (&req->msg);
}

static void
scmi_req_async_waiter(void *context, int pending)
{
	struct task *ta = context;
	struct scmi_softc *sc;
	struct scmi_req *req;

	req = tsk_to_req(ta);
	sc = device_get_softc(req->dev);
	scmi_wait_for_response(sc, req, NULL);

	scmi_msg_put(req->dev, &req->msg);
}

void
scmi_msg_put(device_t dev, struct scmi_msg *msg)
{
	struct scmi_softc *sc;
	struct scmi_req *req;

	sc = device_get_softc(dev);

	req = msg_to_req(msg);

	scmi_req_put(sc, req);
}

int
scmi_request_tx(device_t dev, void *in)
{
	struct scmi_softc *sc;
	struct scmi_req *req;
	int error;

	sc = device_get_softc(dev);

	req = buf_to_req(in);

	req->msg.polling =
	    (cold || sc->trs_desc.no_completion_irq || req->use_polling);

	/* Set inflight and send using transport specific method - refc-2 */
	error = scmi_req_track_inflight(sc, req);
	if (error != 0) {
		device_printf(dev, "Failed to build req with HDR |%0X|\n",
		    req->msg.hdr);
		return (error);
	}

	error = SCMI_XFER_MSG(sc->dev, &req->msg);
	if (error != 0) {
		scmi_req_drop_inflight(sc, req);
		return (error);
	}

	SDT_PROBE5(scmi, func, scmi_request_tx, xfer_track, req,
	    SCMI_MSG_PROTOCOL_ID(req->msg.hdr), SCMI_MSG_MESSAGE_ID(req->msg.hdr),
	    SCMI_MSG_TOKEN(req->msg.hdr), req->msg.polling);

	return (0);
}

int
scmi_request(device_t dev, void *in, void **out)
{
	struct scmi_softc *sc;
	struct scmi_req *req;
	int error;

	error = scmi_request_tx(dev, in);
	if (error != 0)
		return (error);

	sc = device_get_softc(dev);
	req = buf_to_req(in);

	return (scmi_wait_for_response(sc, req, out));
}

int
scmi_msg_async_enqueue(struct scmi_msg *msg)
{
	struct scmi_req *req;

	req = msg_to_req(msg);

	return taskqueue_enqueue_flags(taskqueue_thread, &req->tsk,
	    TASKQUEUE_FAIL_IF_PENDING | TASKQUEUE_FAIL_IF_CANCELING);
}