Merge branch 'lukeshu/dma'

6 files changed, 427 insertions, 48 deletions

diff --git a/libhw/CMakeLists.txt b/libhw/CMakeLists.txt
index bd92e04..242a3fa 100644
--- a/libhw/CMakeLists.txt
+++ b/libhw/CMakeLists.txt
@@ -14,6 +14,7 @@ if (PICO_PLATFORM STREQUAL "rp2040")
 		libcr_ipc
 	)
 	target_sources(libhw INTERFACE
+		rp2040_dma.c
 		rp2040_gpioirq.c
 		rp2040_hwspi.c
 		rp2040_hwtimer.c
diff --git a/libhw/rp2040_dma.c b/libhw/rp2040_dma.c
new file mode 100644
index 0000000..dfbf136
--- /dev/null
+++ b/libhw/rp2040_dma.c
@@ -0,0 +1,56 @@
+/* libhw/rp2040_dma.c - Utilities for sharing the DMA IRQs
+ *
+ * Copyright (C) 2025  Luke T. Shumaker <lukeshu@lukeshu.com>
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+#include <stdbool.h>
+
+#include <hardware/irq.h> /* for irq_set_exclusive_handler() */
+
+#include "rp2040_dma.h"
+
+struct dmairq_handler_entry {
+	dmairq_handler_t         fn;
+	void                    *arg;
+};
+struct dmairq_handler_entry dmairq_handlers[NUM_DMA_CHANNELS] = {0};
+
+bool dmairq_initialized[NUM_DMA_IRQS] = {0};
+
+static void dmairq_handler(void) {
+	enum dmairq irq = __get_current_exception() - VTABLE_FIRST_IRQ;
+	size_t irq_idx = irq - DMAIRQ_0;
+	assert(irq_idx < NUM_DMA_IRQS);
+
+	uint32_t regval = dma_hw->irq_ctrl[irq_idx].ints;
+	for (uint channel = 0; channel < NUM_DMA_CHANNELS; channel++) {
+		if (regval & 1u<<channel) {
+			struct dmairq_handler_entry *handler = &dmairq_handlers[channel];
+			if (handler->fn)
+				handler->fn(handler->arg, irq, channel);
+		}
+	}
+	/* acknowledge irq */
+	dma_hw->intr = regval;
+}
+
+void dmairq_set_and_enable_exclusive_handler(enum dmairq irq, uint channel, dmairq_handler_t fn, void *arg) {
+	assert(irq == DMAIRQ_0 || irq == DMAIRQ_1);
+	assert(channel < NUM_DMA_CHANNELS);
+	assert(fn);
+
+	assert(dmairq_handlers[channel].fn == NULL);
+
+	dmairq_handlers[channel].fn = fn;
+	dmairq_handlers[channel].arg = arg;
+
+	size_t irq_idx = irq - DMAIRQ_0;
+	hw_set_bits(&dma_hw->irq_ctrl[irq_idx].inte, 1u<<channel);
+
+	if (!dmairq_initialized[irq_idx]) {
+		irq_set_exclusive_handler(irq, dmairq_handler);
+		irq_set_enabled(irq, true);
+		dmairq_initialized[irq_idx] = true;
+	}
+}
diff --git a/libhw/rp2040_dma.h b/libhw/rp2040_dma.h
new file mode 100644
index 0000000..c8d69b1
--- /dev/null
+++ b/libhw/rp2040_dma.h
@@ -0,0 +1,128 @@
+/* libhw/rp2040_dma.h - Utilities for using DMA on the RP2040 (replaces <hardware/dma.h>)
+ *
+ * Copyright (c) 2020 Raspberry Pi (Trading) Ltd.
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (C) 2025  Luke T. Shumaker <lukeshu@lukeshu.com>
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+#ifndef _LIBHW_RP2040_DMA_H_
+#define _LIBHW_RP2040_DMA_H_
+
+#include <assert.h>
+#include <stdint.h> /* for uint32_t */
+
+#include <hardware/regs/dreq.h>   /* for DREQ_* for use with DMA_CTRL_TREQ_SEL() */
+#include <hardware/structs/dma.h> /* for dma_hw, dma_channel_hw_t, DMA_NUM_CHANNELS */
+
+#include <libmisc/macro.h> /* for LM_FLOORLOG2() */
+
+/* Borrowed from <hardware/dma.h> *********************************************/
+
+static inline dma_channel_hw_t *dma_channel_hw_addr(uint channel) {
+	assert(channel < NUM_DMA_CHANNELS);
+	return &dma_hw->ch[channel];
+}
+
+enum dma_channel_transfer_size {
+    DMA_SIZE_8 = 0,    ///< Byte transfer (8 bits)
+    DMA_SIZE_16 = 1,   ///< Half word transfer (16 bits)
+    DMA_SIZE_32 = 2    ///< Word transfer (32 bits)
+};
+
+/* Our own code ***************************************************************/
+
+enum dmairq {
+	DMAIRQ_0 = DMA_IRQ_0,
+	DMAIRQ_1 = DMA_IRQ_1,
+};
+
+typedef void (*dmairq_handler_t)(void *arg, enum dmairq irq, uint channel);
+
+/**
+ * Register `fn(arg, ...)` to be called when `channel` completes or
+ * has a NULL trigger (depending on the channel's configuration).
+ *
+ * Your handler does not need to acknowledge the IRQ; that will be
+ * done for you after your handler is called.
+ *
+ * It is illegal to enable the same channel on more than one IRQ.
+ */
+void dmairq_set_and_enable_exclusive_handler(enum dmairq irq, uint channel, dmairq_handler_t fn, void *arg);
+
+#define DMA_CTRL_ENABLE          (1<<0)
+#define DMA_CTRL_HI_PRIO         (1<<1)
+#define DMA_CTRL_DATA_SIZE(sz)   ((sz)<<2)
+#define DMA_CTRL_INCR_READ       (1<<4)
+#define DMA_CTRL_INCR_WRITE      (1<<5)
+#define _DMA_CTRL_RING_BITS(b)   ((b)<<6)
+#define _DMA_CTRL_RING_RD        (0)
+#define _DMA_CTRL_RING_WR        (1<<10)
+#define DMA_CTRL_RING(rdwr, bits) (_DMA_CTRL_RING_##rdwr | _DMA_CTRL_RING_BITS(bits))
+#define DMA_CTRL_CHAIN_TO(ch)    ((ch)<<11)
+#define DMA_CTRL_TREQ_SEL(dreq)  ((dreq)<<15)
+#define DMA_CTRL_IRQ_QUIET       (1<<21)
+#define DMA_CTRL_BSWAP           (1<<22)
+#define DMA_CTRL_SNIFF_EN        (1<<23)
+
+/*                    |          elem         |     val    |  name */
+#define READ_ADDR   /*|*/volatile const void/*|*/    *   /*|*/read_addr
+#define WRITE_ADDR  /*|*/volatile       void/*|*/    *   /*|*/write_addr
+#define TRANS_COUNT /*|*/                   /*|*/uint32_t/*|*/trans_count
+#define CTRL        /*|*/                   /*|*/uint32_t/*|*/ctrl
+
+/*                { +0x0      ; +0x4        ; +0x8        ; +0xC (Trigger) */
+struct dma_alias0 { READ_ADDR ; WRITE_ADDR  ; TRANS_COUNT ; CTRL        ; };
+struct dma_alias1 { CTRL      ; READ_ADDR   ; WRITE_ADDR  ; TRANS_COUNT ; };
+struct dma_alias2 { CTRL      ; TRANS_COUNT ; READ_ADDR   ; WRITE_ADDR  ; };
+struct dma_alias3 { CTRL      ; WRITE_ADDR  ; TRANS_COUNT ; READ_ADDR   ; };
+struct dma_alias0_short2                    { TRANS_COUNT ; CTRL        ; };
+struct dma_alias1_short2                    { WRITE_ADDR  ; TRANS_COUNT ; };
+struct dma_alias2_short2                    { READ_ADDR   ; WRITE_ADDR  ; };
+struct dma_alias3_short2                    { TRANS_COUNT ; READ_ADDR   ; };
+struct dma_alias0_short3                                  { CTRL        ; };
+struct dma_alias1_short3                                  { TRANS_COUNT ; };
+struct dma_alias2_short3                                  { WRITE_ADDR  ; };
+struct dma_alias3_short3                                  { READ_ADDR   ; };
+
+#undef CTRL
+#undef TRANS_COUNT
+#undef WRITE_ADDR
+#undef READ_ADDR
+
+#define DMA_CHAN_ADDR(CH, TYP) ((TYP *volatile)_Generic((TYP){},                     \
+	struct dma_alias0:        &dma_channel_hw_addr(CH)->read_addr,               \
+	struct dma_alias1:        &dma_channel_hw_addr(CH)->al1_ctrl,                \
+	struct dma_alias2:        &dma_channel_hw_addr(CH)->al2_ctrl,                \
+	struct dma_alias3:        &dma_channel_hw_addr(CH)->al3_ctrl,                \
+	struct dma_alias0_short2: &dma_channel_hw_addr(CH)->transfer_count,          \
+	struct dma_alias1_short2: &dma_channel_hw_addr(CH)->al1_write_addr,          \
+	struct dma_alias2_short2: &dma_channel_hw_addr(CH)->al2_read_addr,           \
+	struct dma_alias3_short2: &dma_channel_hw_addr(CH)->al3_transfer_count,      \
+	struct dma_alias0_short3: &dma_channel_hw_addr(CH)->ctrl_trig,               \
+	struct dma_alias1_short3: &dma_channel_hw_addr(CH)->al1_transfer_count_trig, \
+	struct dma_alias2_short3: &dma_channel_hw_addr(CH)->al2_write_addr_trig,     \
+	struct dma_alias3_short3: &dma_channel_hw_addr(CH)->al3_read_addr_trig))
+
+#define DMA_CHAN_WR_TRANS_COUNT(TYP) \
+	(sizeof(TYP)/4)
+
+#define DMA_CHAN_WR_CTRL(TYP) ( DMA_CTRL_DATA_SIZE(DMA_SIZE_32) \
+                              | DMA_CTRL_INCR_WRITE \
+                              | DMA_CTRL_RING(WR, LM_FLOORLOG2(sizeof(TYP))) \
+                              )
+
+#define DMA_NONTRIGGER(CH, FIELD) (DMA_CHAN_ADDR(CH, _DMA_NONTRIGGER_##FIELD)->FIELD)
+#define _DMA_NONTRIGGER_read_addr   struct dma_alias0
+#define _DMA_NONTRIGGER_write_addr  struct dma_alias0
+#define _DMA_NONTRIGGER_trans_count struct dma_alias0
+#define _DMA_NONTRIGGER_ctrl        struct dma_alias1
+
+#define DMA_TRIGGER(CH, FIELD)    (DMA_CHAN_ADDR(CH, _DMA_TRIGGER_##FIELD)->FIELD)
+#define _DMA_TRIGGER_read_addr      struct dma_alias3
+#define _DMA_TRIGGER_write_addr     struct dma_alias2
+#define _DMA_TRIGGER_trans_count    struct dma_alias1
+#define _DMA_TRIGGER_ctrl           struct dma_alias0
+
+#endif /* _LIBHW_RP2040_DMA_H_ */
diff --git a/libhw/rp2040_hwspi.c b/libhw/rp2040_hwspi.c
index 8c87666..f747b1e 100644
--- a/libhw/rp2040_hwspi.c
+++ b/libhw/rp2040_hwspi.c
@@ -4,37 +4,76 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
-#include <hardware/spi.h>  /* pico-sdk:hardware_spi */
-#include <hardware/gpio.h> /* pico-sdk:hardware_gpio */
+#include <alloca.h>
+#include <inttypes.h> /* for PRIu{n} */
 
+#include <hardware/clocks.h> /* for clock_get_hz() and clk_peri */
+#include <hardware/gpio.h>
+#include <hardware/spi.h>
+
+#include <libcr/coroutine.h>
 #include <libmisc/assert.h>
 
+#define LOG_NAME RP2040_SPI
+#include <libmisc/log.h>
+
 #define IMPLEMENTATION_FOR_LIBHW_RP2040_HWSPI_H YES
 #include <libhw/rp2040_hwspi.h>
 
+#include <libhw/generic/alarmclock.h>
+
+#include "rp2040_dma.h"
+
+#include "config.h"
+
+#ifndef CONFIG_RP2040_SPI_DEBUG
+	#error config.h must define CONFIG_RP2040_SPI_DEBUG (bool)
+#endif
+
 LO_IMPLEMENTATION_C(io_duplex_readwriter, struct rp2040_hwspi, rp2040_hwspi, static)
 LO_IMPLEMENTATION_C(spi, struct rp2040_hwspi, rp2040_hwspi, static)
 
+static void rp2040_hwspi_intrhandler(void *_self, enum dmairq LM_UNUSED(irq), uint LM_UNUSED(channel)) {
+	struct rp2040_hwspi *self = _self;
+	gpio_put(self->pin_cs, 1);
+	cr_sema_signal_from_intrhandler(&self->sema);
+}
+
+#define assert_4distinct(a, b, c, d) \
+	assert(a != b); \
+	assert(a != c); \
+	assert(a != d); \
+	assert(b != c); \
+	assert(b != d); \
+	assert(c != d);
+
 void _rp2040_hwspi_init(struct rp2040_hwspi *self,
                         enum rp2040_hwspi_instance inst_num,
                         enum spi_mode mode,
                         uint baudrate_hz,
+                        uint64_t min_delay_ns,
+                        uint8_t bogus_data,
                         uint pin_miso,
                         uint pin_mosi,
                         uint pin_clk,
-                        uint pin_cs) {
+                        uint pin_cs,
+                        uint dma1,
+                        uint dma2,
+                        uint dma3,
+                        uint dma4)
+{
 	/* Be not weary: This is but 12 lines of actual code; and many
 	 * lines of comments and assert()s.  */
 	spi_inst_t *inst;
+	uint actual_baudrate_hz;
 
 	assert(self);
 	assert(baudrate_hz);
-	assert(pin_miso != pin_mosi);
-	assert(pin_miso != pin_clk);
-	assert(pin_miso != pin_cs);
-	assert(pin_mosi != pin_clk);
-	assert(pin_mosi != pin_cs);
-	assert(pin_clk != pin_cs);
+	uint32_t clk_peri_hz = clock_get_hz(clk_peri);
+	debugf("clk_peri = %"PRIu32"Hz", clk_peri_hz);
+	assert(baudrate_hz*2 <= clk_peri_hz);
+	assert_4distinct(pin_miso, pin_mosi, pin_clk, pin_cs);
+	assert_4distinct(dma1, dma2, dma3, dma4);
 
 	/* Regarding the constraints on pin assignments: see the
 	 * RP2040 datasheet, table 2, in §1.4.3 "GPIO Functions".  */
@@ -57,7 +96,9 @@ void _rp2040_hwspi_init(struct rp2040_hwspi *self,
 		assert_notreached("invalid hwspi instance number");
 	}
 
-	spi_init(inst, baudrate_hz);
+	actual_baudrate_hz = spi_init(inst, baudrate_hz);
+	debugf("baudrate = %uHz", actual_baudrate_hz);
+	assert(actual_baudrate_hz == baudrate_hz);
 	spi_set_format(inst, 8,
 	               (mode & 0b10) ? SPI_CPOL_1 : SPI_CPOL_0,
 	               (mode & 0b01) ? SPI_CPHA_1 : SPI_CPHA_0,
@@ -79,30 +120,125 @@ void _rp2040_hwspi_init(struct rp2040_hwspi *self,
 	gpio_set_dir(pin_cs, GPIO_OUT);
 	gpio_put(pin_cs, 1);
 
-	/* Return.  */
+	/* Initialize self.  */
 	self->inst = inst;
+	self->min_delay_ns = min_delay_ns;
+	self->bogus_data = bogus_data;
 	self->pin_cs = pin_cs;
+	self->dma_tx_ctrl = dma1;
+	self->dma_rx_ctrl = dma2;
+	self->dma_tx_data = dma3;
+	self->dma_rx_data = dma4;
+	self->dead_until_ns = 0;
+	self->sema = (cr_sema_t){0};
+
+	/* Initialize the interrupt handler.  */
+	dmairq_set_and_enable_exclusive_handler(DMAIRQ_0, self->dma_rx_data, rp2040_hwspi_intrhandler, self);
 }
 
 static void rp2040_hwspi_readwritev(struct rp2040_hwspi *self, const struct duplex_iovec *iov, int iovcnt) {
 	assert(self);
-	spi_inst_t *inst = self->inst;
-
-	assert(inst);
+	assert(self->inst);
 	assert(iov);
-	assert(iovcnt);
+	assert(iovcnt > 0);
 
-	gpio_put(self->pin_cs, 0);
-	/* TODO: Replace blocking reads+writes with DMA.  */
-	for (int i = 0; i < iovcnt; i++) {
-		if (iov[i].iov_write_src && iov[i].iov_read_dst)
-			spi_write_read_blocking(inst, iov[i].iov_write_src, iov[i].iov_read_dst, iov[i].iov_len);
-		else if (iov[i].iov_write_src)
-			spi_write_blocking(inst, iov[i].iov_write_src, iov[i].iov_len);
-		else if (iov[i].iov_read_dst)
-			spi_read_blocking(inst, 0, iov[i].iov_read_dst, iov[i].iov_len);
-		else
-			assert_notreached("duplex_iovec is neither read nor write");
+	/* At this time, I have no intention to run SPI faster than
+	 * 80MHz (= 80Mb/s = 10MB/s).  If we ran the CPU at just
+	 * 100MHz (we'll be running it faster than that, maybe even
+	 * 200MHz), that means we'd have 10 clock cycles to send each
+	 * byte.
+	 *
+	 * This affords us substantial simplifications, like being
+	 * able to afford 4-cycle changeovers between DMA blocks, and
+	 * not having to worry about alignment because we can just use
+	 * DMA_SIZE_8.
+	 */
+
+	uint8_t bogus_rx_dst;
+
+	int pruned_iovcnt = 0;
+	for (int i = 0; i < iovcnt; i++)
+		if (iov[i].iov_len)
+			pruned_iovcnt++;
+	if (!pruned_iovcnt)
+		return;
+
+	/* For tx_data_blocks, it doesn't really matter which aliases
+	 * we choose:
+	 *  - None of our fields can be NULL (so no
+	 *    false-termination).
+	 *  - Moving const fields first so they don't have to be
+	 *    re-programmed each time isn't possible for us there need
+	 *    to be at least 2 const fields, and we only have 1
+	 *    (read_addr for rx_data_blocks, and write_addr for
+	 *    tx_data_blocks).
+	 *
+	 * But for rx_data_blocks, we need ctrl to be the trigger
+	 * register so that the DMA_CTRL_IRQ_QUIET flag isn't cleared
+	 * before we get to the trigger; and while for tx_data_blocks
+	 * it doesn't really matter, the inverse would be nice.
+	 */
+	struct dma_alias1 *tx_data_blocks = alloca(sizeof(struct dma_alias1)*(pruned_iovcnt+1));
+	struct dma_alias0 *rx_data_blocks = alloca(sizeof(struct dma_alias0)*(pruned_iovcnt+1));
+
+	for (int i = 0, j = 0; i < iovcnt; i++) {
+		if (!iov[i].iov_len)
+			continue;
+		tx_data_blocks[j]    = (typeof(tx_data_blocks[0])){
+			.read_addr   = iov[i].iov_write_src ?: &self->bogus_data,
+			.write_addr  = &spi_get_hw(self->inst)->dr,
+			.trans_count = iov[i].iov_len,
+			.ctrl        = (DMA_CTRL_ENABLE
+			                | DMA_CTRL_DATA_SIZE(DMA_SIZE_8)
+			                | (iov[i].iov_write_src ? DMA_CTRL_INCR_READ : 0)
+			                | DMA_CTRL_CHAIN_TO(self->dma_tx_ctrl)
+			                | DMA_CTRL_TREQ_SEL(SPI_DREQ_NUM(self->inst, true))
+			                | DMA_CTRL_IRQ_QUIET),
+		};
+		rx_data_blocks[j]    = (typeof(rx_data_blocks[0])){
+			.read_addr   = &spi_get_hw(self->inst)->dr,
+			.write_addr  = iov[i].iov_read_dst ?: &bogus_rx_dst,
+			.trans_count = iov[i].iov_len,
+			.ctrl        = (DMA_CTRL_ENABLE
+			                | DMA_CTRL_DATA_SIZE(DMA_SIZE_8)
+			                | (iov[i].iov_read_dst ? DMA_CTRL_INCR_WRITE : 0)
+			                | DMA_CTRL_CHAIN_TO(self->dma_rx_ctrl)
+			                | DMA_CTRL_TREQ_SEL(SPI_DREQ_NUM(self->inst, false))
+			                | DMA_CTRL_IRQ_QUIET),
+		};
+		j++;
 	}
-	gpio_put(self->pin_cs, 1);
+	tx_data_blocks[pruned_iovcnt] = (typeof(tx_data_blocks[0])){0};
+	rx_data_blocks[pruned_iovcnt] = (typeof(rx_data_blocks[0])){0};
+
+	/* Set up ctrl.  */
+	DMA_NONTRIGGER(self->dma_tx_ctrl, read_addr) = tx_data_blocks;
+	DMA_NONTRIGGER(self->dma_tx_ctrl, write_addr) = DMA_CHAN_ADDR(self->dma_tx_data, typeof(tx_data_blocks[0]));
+	DMA_NONTRIGGER(self->dma_tx_ctrl, trans_count) = DMA_CHAN_WR_TRANS_COUNT(typeof(tx_data_blocks[0]));
+	DMA_NONTRIGGER(self->dma_tx_ctrl, ctrl) = (DMA_CTRL_ENABLE
+	                                           | DMA_CHAN_WR_CTRL(typeof(tx_data_blocks[0]))
+	                                           | DMA_CTRL_INCR_READ
+	                                           | DMA_CTRL_CHAIN_TO(self->dma_tx_data)
+	                                           | DMA_CTRL_TREQ_SEL(DREQ_FORCE)
+	                                           | DMA_CTRL_IRQ_QUIET);
+	DMA_NONTRIGGER(self->dma_rx_ctrl, read_addr) = rx_data_blocks;
+	DMA_NONTRIGGER(self->dma_rx_ctrl, write_addr) = DMA_CHAN_ADDR(self->dma_rx_data, typeof(rx_data_blocks[0]));
+	DMA_NONTRIGGER(self->dma_rx_ctrl, trans_count) = DMA_CHAN_WR_TRANS_COUNT(typeof(rx_data_blocks[0]));
+	DMA_NONTRIGGER(self->dma_rx_ctrl, ctrl) = (DMA_CTRL_ENABLE
+	                                           | DMA_CHAN_WR_CTRL(typeof(rx_data_blocks[0]))
+	                                           | DMA_CTRL_INCR_READ
+	                                           | DMA_CTRL_CHAIN_TO(self->dma_rx_data)
+	                                           | DMA_CTRL_TREQ_SEL(DREQ_FORCE)
+	                                           | DMA_CTRL_IRQ_QUIET);
+
+	/* Run.  */
+	uint64_t now = LO_CALL(bootclock, get_time_ns);
+	if (now < self->dead_until_ns)
+		sleep_until_ns(self->dead_until_ns);
+	bool saved = cr_save_and_disable_interrupts();
+	gpio_put(self->pin_cs, 0);
+	dma_hw->multi_channel_trigger = (1u<<self->dma_tx_ctrl) | (1u<<self->dma_rx_ctrl);
+	cr_restore_interrupts(saved);
+	cr_sema_wait(&self->sema);
+	self->dead_until_ns = LO_CALL(bootclock, get_time_ns) + self->min_delay_ns;
 }
diff --git a/libhw/rp2040_include/libhw/rp2040_hwspi.h b/libhw/rp2040_include/libhw/rp2040_hwspi.h
index 76e0709..eb54cdc 100644
--- a/libhw/rp2040_include/libhw/rp2040_hwspi.h
+++ b/libhw/rp2040_include/libhw/rp2040_hwspi.h
@@ -9,6 +9,7 @@
 
 #include <pico/binary_info.h> /* for bi_* */
 
+#include <libcr_ipc/sema.h>
 #include <libmisc/private.h>
 
 #include <libhw/generic/spi.h>
@@ -20,9 +21,19 @@ enum rp2040_hwspi_instance {
 
 struct rp2040_hwspi {
 	BEGIN_PRIVATE(LIBHW_RP2040_HWSPI_H)
+	/* const */
 	LM_IF(IS_IMPLEMENTATION_FOR(LIBHW_RP2040_HWSPI_H))(spi_inst_t)(void)    *inst;
+	uint64_t        min_delay_ns;
+	uint8_t         bogus_data;
+	uint            pin_cs;
+	uint            dma_tx_data;
+	uint            dma_tx_ctrl;
+	uint            dma_rx_data;
+	uint            dma_rx_ctrl;
 
-	uint    pin_cs;
+	/* mutable */
+	uint64_t        dead_until_ns;
+	cr_sema_t       sema;
 	END_PRIVATE(LIBHW_RP2040_HWSPI_H)
 };
 LO_IMPLEMENTATION_H(io_duplex_readwriter, struct rp2040_hwspi, rp2040_hwspi)
@@ -31,14 +42,18 @@ LO_IMPLEMENTATION_H(spi, struct rp2040_hwspi, rp2040_hwspi)
 /**
  * Initialize an instance of `struct rp2040_hwspi`.
  *
- * @param self     : struct rp2040_hwspi        : the structure to initialize
- * @param name     : char *                     : a name for the SPI port; to include in the bininfo
- * @param inst_num : enum rp2040_hwspi_instance : the PL220 instance number; RP2040_HWSPI_{0,1}
- * @param mode     : enum spi_mode              : the SPI mode; SPI_MODE_{0..3}
- * @param pin_miso : uint                       : pin number; 0, 4, 16, or 20 for _HWSPI_0; 8, 12, 24, or 28 for _HWSPI_1
- * @param pin_mosi : uint                       : pin number; 3, 7, 19, or 23 for _HWSPI_0; 11, 15, or 27 for _HWSPI_1
- * @param pin_clk  : uint                       : pin number; 2, 6, 18, or 22 for _HWSPI_0; 10, 14, or 26 for _HWSPI_1
- * @param pin_cs   : uint                       : pin number; any unused GPIO pin
+ * @param self        : struct rp2040_hwspi        : the structure to initialize
+ * @param name        : char *                     : a name for the SPI port; to include in the bininfo
+ * @param inst_num    : enum rp2040_hwspi_instance : the PL220 instance number; RP2040_HWSPI_{0,1}
+ * @param mode        : enum spi_mode              : the SPI mode; SPI_MODE_{0..3}
+ * @param baudrate_hz : uint                       : baudrate in Hz
+ * @param min_delay_ns: uint64_t                   : minimum time for pin_cs to be high between messages
+ * @param bogus_data  : uint8_t                    : bogus data to write when .iov_write_src is NULL
+ * @param pin_miso    : uint                       : pin number; 0, 4, 16, or 20 for _HWSPI_0; 8, 12, 24, or 28 for _HWSPI_1
+ * @param pin_mosi    : uint                       : pin number; 3, 7, 19, or 23 for _HWSPI_0; 11, 15, or 27 for _HWSPI_1
+ * @param pin_clk     : uint                       : pin number; 2, 6, 18, or 22 for _HWSPI_0; 10, 14, or 26 for _HWSPI_1
+ * @param pin_cs      : uint                       : pin number; any unused GPIO pin
+ * @param dma{1-4}    : uint                       : DMA channel; any unused channel
  *
  * There is no bit-order argument; the RP2040's hardware SPI always
  * uses MSB-first bit order.
@@ -48,16 +63,32 @@ LO_IMPLEMENTATION_H(spi, struct rp2040_hwspi, rp2040_hwspi)
  * the CPU.  This is because the PL022 has a maximum of 16-bit frames,
  * but we need to be able to do *at least* 32-bit frames (and ideally,
  * much larger).  By managing it ourselves, we can just keep CS pulled
- * low extra-long, making the frame extra-long.  However, this means
- * that we can't SPI so fast that the CPU can't do things in time;
- * experimentally much faster than 60MHz seems to be when I start
- * getting mangled messages.  We wouldn't have this speed limit with a
- * PIO-based SPI driver, because it could toggle CLK and CS in
- * lock-step with receiving data from the FIFO.
+ * low extra-long, making the frame extra-long.
+ *
+ * Restrictions on baudrate:
+ *
+ * - The PL022 requires that the baudrate is an even-number fraction
+ *   of clk_peri.
+ *    + This implies that the maximum baudrate is clk_peri/2.
+ *    + Pico-SDK' default clk_peri is 125MHz, max is 200MHz.
+ * - The CS-from-GPIO hack above means that that we can't go so fast
+ *   that the CPU can't do things in time.
+ *    + Experimentally:
+ *      | clk_sys=125MHz | baud=31.25MHz | works OK                         |
+ *      | clk_sys=160MHz | baud=40   MHz | works OK                         |
+ *      | clk_sys=170MHz | baud=42.5 MHz | works OK                         |
+ *      | clk_sys=180MHz | baud=45   MHz | mangled in funny ways?           |
+ *      | clk_sys=200MHz | baud=50   MHz | messages get shifted right a bit |
+ *      | clk_sys=125MHz | baud=62.5 MHz | messages get shifted right a bit |
+ *
+ * Both of these restrictions aught to be avoidable by using a
+ * PIO-based SPI driver instead of this PLL02-based driver.
  */
 #define rp2040_hwspi_init(self, name,                                    \
                           inst_num, mode, baudrate_hz,                   \
-                          pin_miso, pin_mosi, pin_clk, pin_cs)           \
+                          min_delay_ns, bogus_data,                      \
+                          pin_miso, pin_mosi, pin_clk, pin_cs,           \
+                          dma1, dma2, dma3, dma4)                        \
 	do {                                                             \
 		bi_decl(bi_4pins_with_names(pin_miso, name" SPI MISO",   \
 		                            pin_mosi, name" SPI MOSI",   \
@@ -65,15 +96,23 @@ LO_IMPLEMENTATION_H(spi, struct rp2040_hwspi, rp2040_hwspi)
 		                            pin_mosi, name" SPI CS"));   \
 		_rp2040_hwspi_init(self,                                 \
 		                   inst_num, mode, baudrate_hz,          \
-		                   pin_miso, pin_mosi, pin_clk, pin_cs); \
+		                   min_delay_ns, bogus_data,             \
+		                   pin_miso, pin_mosi, pin_clk, pin_cs,  \
+		                   dma1, dma2, dma3, dma4);              \
 	} while(0)
 void _rp2040_hwspi_init(struct rp2040_hwspi *self,
                         enum rp2040_hwspi_instance inst_num,
                         enum spi_mode mode,
                         uint baudrate_hz,
+                        uint64_t min_delay_ns,
+                        uint8_t bogus_data,
                         uint pin_miso,
                         uint pin_mosi,
                         uint pin_clk,
-                        uint pin_cs);
+                        uint pin_cs,
+                        uint dma1,
+                        uint dma2,
+                        uint dma3,
+                        uint dma4);
 
 #endif /* _LIBHW_RP2040_HWSPI_H_ */
diff --git a/libhw/w5500.c b/libhw/w5500.c
index b4ad86b..c4d36f3 100644
--- a/libhw/w5500.c
+++ b/libhw/w5500.c
@@ -96,6 +96,9 @@
 #ifndef CONFIG_W5500_LOCAL_PORT_MAX
 	#error config.h must define CONFIG_W5500_LOCAL_PORT_MAX
 #endif
+#ifndef CONFIG_W5500_VALIDATE_SPI
+	#error config.h must define CONFIG_W5500_VALIDATE_SPI
+#endif
 #ifndef CONFIG_W5500_DEBUG
 	#error config.h must define CONFIG_W5500_DEBUG
 #endif
@@ -312,6 +315,22 @@ void _w5500_init(struct w5500 *chip,
 		};
 	}
 
+#if CONFIG_W5500_VALIDATE_SPI
+	/* Validate that SPI works correctly.  */
+	bool spi_ok = true;
+	for (uint16_t a = 0; a < 0x100; a++) {
+		w5500ll_write_sock_reg(chip->spidev, 0, mode, a);
+		uint8_t b = w5500ll_read_sock_reg(chip->spidev, 0, mode);
+		if (b != a) {
+			errorf("SPI to W5500 does not appear to be functional: wrote:0x%02"PRIx16" != read:0x%02"PRIx8, a, b);
+			spi_ok = false;
+		}
+	}
+	if (!spi_ok)
+		__lm_abort();
+	w5500ll_write_sock_reg(chip->spidev, 0, mode, 0);
+#endif
+
 	/* Initialize the hardware.  */
 	gpioirq_set_and_enable_exclusive_handler(pin_intr, GPIO_IRQ_EDGE_FALL, w5500_intrhandler, chip);
 	gpio_set_dir(chip->pin_reset, GPIO_OUT);
@@ -329,9 +348,9 @@ static void w5500_post_reset(struct w5500 *chip) {
 	w5500ll_write_common_reg(chip->spidev, eth_addr, chip->hwaddr);
 
 	/* The RP2040 needs a 1/sys_clk hysteresis between interrupts
-	 * for us to notice them.  At the maximum-rated clock-rate of
-	 * 133MHz, that means 7.5ns (but the sbc-harness overclocks
-	 * the RP2040, so we could get away with even shorter).
+	 * for us to notice them.  At the default clock-rate of
+	 * 125MHz, that means 8ns; and at the maximum-rated clock-rate
+	 * of 200MHz, that means 5ns.
 	 *
 	 * If intlevel is non-zero, then the hysteresis is
 	 * (intlevel+1)*4/(150MHz), or (intlevel+1)*26.7ns; so even