Merge branch 'lukeshu/spi-dma-cleanup'HEADmain

5 files changed, 157 insertions, 132 deletions

diff --git a/flashimg/cpu_main/main.c b/flashimg/cpu_main/main.c
index a471d2a..812dbc1 100644
--- a/flashimg/cpu_main/main.c
+++ b/flashimg/cpu_main/main.c
@@ -205,7 +205,7 @@ COROUTINE init_cr(void *) {
 	                  19, /* PIN_MOSI */
 	                  18, /* PIN_CLK */
 	                  17, /* PIN_CS */
-	                  0, 1, 2, 3); /* DMA channels */
+	                  1, 2, 3, 4); /* DMA channels */
 	w5500_init(&globals.dev_w5500, "W5500",
 	           LO_BOX(spi, &globals.dev_spi),
 	           21, /* PIN_INTR */
diff --git a/gdb-helpers/rp2040.py b/gdb-helpers/rp2040.py
index f019299..03a5673 100644
--- a/gdb-helpers/rp2040.py
+++ b/gdb-helpers/rp2040.py
@@ -152,7 +152,7 @@ ICSR     : {fmt32(icsr)                     } Control and State
                              ╓endiannes (0=little, 1=big)
              vect_key┐       ║            ╓sys_reset_req
              ┌───────┴──────┐║            ║╓vect_clr_active
-AIRCR    : {fmt32(read_mmreg(base+0xed0c))  } Application {{Interrupt+Reset}} Control
+AIRCR    : {fmt32(read_mmreg(base+0xed0c))  } App {{Interrupt+Reset}} Control
 
                                           ╓sleep_deep
                             s_ev_on_pend╖ ║╓sleep_on_exit
diff --git a/libhw_cr/rp2040_dma.c b/libhw_cr/rp2040_dma.c
index 0248f23..9c2c9ce 100644
--- a/libhw_cr/rp2040_dma.c
+++ b/libhw_cr/rp2040_dma.c
@@ -11,6 +11,10 @@
 
 #include "rp2040_dma.h"
 
+/* static_assert("rp2040_dma.h" == <hardware/irq.h>); */
+static_assert((uint)DMAIRQ_0 == (uint)DMA_IRQ_0);
+static_assert((uint)DMAIRQ_1 == (uint)DMA_IRQ_1);
+
 /* Borrowed from <hardware/dma.h> *********************************************/
 
 dma_channel_hw_t *dma_channel_hw_addr(uint channel) {
@@ -18,12 +22,6 @@ dma_channel_hw_t *dma_channel_hw_addr(uint channel) {
 	return &dma_hw->ch[channel];
 }
 
-enum dma_channel_transfer_size {
-	DMA_SIZE_8  = 0, ///< Byte transfer (8 bits)
-	DMA_SIZE_16 = 1, ///< Half word transfer (16 bits)
-	DMA_SIZE_32 = 2  ///< Word transfer (32 bits)
-};
-
 /* Our own code ***************************************************************/
 
 struct dmairq_handler_entry {
diff --git a/libhw_cr/rp2040_dma.h b/libhw_cr/rp2040_dma.h
index 13862d0..b382952 100644
--- a/libhw_cr/rp2040_dma.h
+++ b/libhw_cr/rp2040_dma.h
@@ -10,30 +10,20 @@
 #ifndef _LIBHW_CR_RP2040_DMA_H_
 #define _LIBHW_CR_RP2040_DMA_H_
 
-#include <assert.h>
 #include <stddef.h> /* for offsetof() */
 #include <stdint.h> /* for uint32_t */
 
 #include <hardware/regs/dreq.h>   /* for DREQ_* for use with DMA_CTRL_TREQ_SEL() */
-#include <hardware/structs/dma.h> /* for dma_hw, dma_channel_hw_t, DMA_NUM_CHANNELS */
+#include <hardware/structs/dma.h> /* for dma_hw, DMA_NUM_CHANNELS */
 
+#include <libmisc/assert.h>
 #include <libmisc/macro.h> /* for LM_FLOORLOG2() */
 
-/* Borrowed from <hardware/dma.h> *********************************************/
-
-dma_channel_hw_t *dma_channel_hw_addr(uint channel);
-
-enum dma_channel_transfer_size {
-	DMA_SIZE_8  = 0, ///< Byte transfer (8 bits)
-	DMA_SIZE_16 = 1, ///< Half word transfer (16 bits)
-	DMA_SIZE_32 = 2  ///< Word transfer (32 bits)
-};
-
-/* Our own code ***************************************************************/
+/* IRQ ************************************************************************/
 
 enum dmairq {
-	DMAIRQ_0 = DMA_IRQ_0,
-	DMAIRQ_1 = DMA_IRQ_1,
+	DMAIRQ_0 = 11,
+	DMAIRQ_1 = 12,
 };
 
 typedef void (*dmairq_handler_t)(void *arg, enum dmairq irq, uint channel);
@@ -49,55 +39,51 @@ typedef void (*dmairq_handler_t)(void *arg, enum dmairq irq, uint channel);
  */
 void dmairq_set_and_enable_exclusive_handler(enum dmairq irq, uint channel, dmairq_handler_t fn, void *arg);
 
-#define DMA_CTRL_ENABLE           (1<<0)
-#define DMA_CTRL_HI_PRIO          (1<<1)
-#define DMA_CTRL_DATA_SIZE(sz)    ((sz)<<2)
-#define DMA_CTRL_INCR_READ        (1<<4)
-#define DMA_CTRL_INCR_WRITE       (1<<5)
-#define _DMA_CTRL_RING_BITS(b)    ((b)<<6)
-#define _DMA_CTRL_RING_RD         (0)
-#define _DMA_CTRL_RING_WR         (1<<10)
-#define DMA_CTRL_RING(rdwr, bits) (_DMA_CTRL_RING_##rdwr | _DMA_CTRL_RING_BITS(bits))
-#define DMA_CTRL_CHAIN_TO(ch)     ((ch)<<11)
-#define DMA_CTRL_TREQ_SEL(dreq)   ((dreq)<<15)
-#define DMA_CTRL_IRQ_QUIET        (1<<21)
-#define DMA_CTRL_BSWAP            (1<<22)
-#define DMA_CTRL_SNIFF_EN         (1<<23)
-
-/*                    |          elem         |     val    |  name */
-#define READ_ADDR   /*|*/volatile const void/*|*/    *   /*|*/read_addr
-#define WRITE_ADDR  /*|*/volatile       void/*|*/    *   /*|*/write_addr
-#define TRANS_COUNT /*|*/                   /*|*/uint32_t/*|*/trans_count
-#define CTRL        /*|*/                   /*|*/uint32_t/*|*/ctrl
-
-/*                { +0x0      ; +0x4        ; +0x8        ; +0xC (Trigger) */
-struct dma_alias0 { READ_ADDR ; WRITE_ADDR  ; TRANS_COUNT ; CTRL        ; };
-struct dma_alias1 { CTRL      ; READ_ADDR   ; WRITE_ADDR  ; TRANS_COUNT ; };
-struct dma_alias2 { CTRL      ; TRANS_COUNT ; READ_ADDR   ; WRITE_ADDR  ; };
-struct dma_alias3 { CTRL      ; WRITE_ADDR  ; TRANS_COUNT ; READ_ADDR   ; };
-struct dma_alias0_short2                    { TRANS_COUNT ; CTRL        ; };
-struct dma_alias1_short2                    { WRITE_ADDR  ; TRANS_COUNT ; };
-struct dma_alias2_short2                    { READ_ADDR   ; WRITE_ADDR  ; };
-struct dma_alias3_short2                    { TRANS_COUNT ; READ_ADDR   ; };
-struct dma_alias0_short3                                  { CTRL        ; };
-struct dma_alias1_short3                                  { TRANS_COUNT ; };
-struct dma_alias2_short3                                  { WRITE_ADDR  ; };
-struct dma_alias3_short3                                  { READ_ADDR   ; };
+/* Control blocks *************************************************************/
+
+/* types =================================================*/
+
+/*                   |          elem         |     val    |  name */
+#define READ_ADDR  /*|*/volatile const void/*|*/    *   /*|*/read_addr
+#define WRITE_ADDR /*|*/volatile       void/*|*/    *   /*|*/write_addr
+#define XFER_COUNT /*|*/                   /*|*/uint32_t/*|*/xfer_count
+#define CTRL       /*|*/                   /*|*/uint32_t/*|*/ctrl
+
+/*                { +0x0      ; +0x4       ; +0x8       ; +0xC (Trigger) */
+struct dma_alias0 { READ_ADDR ; WRITE_ADDR ; XFER_COUNT ; CTRL       ; };
+struct dma_alias1 { CTRL      ; READ_ADDR  ; WRITE_ADDR ; XFER_COUNT ; };
+struct dma_alias2 { CTRL      ; XFER_COUNT ; READ_ADDR  ; WRITE_ADDR ; };
+struct dma_alias3 { CTRL      ; WRITE_ADDR ; XFER_COUNT ; READ_ADDR  ; };
+struct dma_alias0_short2                   { XFER_COUNT ; CTRL       ; };
+struct dma_alias1_short2                   { WRITE_ADDR ; XFER_COUNT ; };
+struct dma_alias2_short2                   { READ_ADDR  ; WRITE_ADDR ; };
+struct dma_alias3_short2                   { XFER_COUNT ; READ_ADDR  ; };
+struct dma_alias0_short3                                { CTRL       ; };
+struct dma_alias1_short3                                { XFER_COUNT ; };
+struct dma_alias2_short3                                { WRITE_ADDR ; };
+struct dma_alias3_short3                                { READ_ADDR  ; };
 
 #undef CTRL
-#undef TRANS_COUNT
+#undef XFER_COUNT
 #undef WRITE_ADDR
 #undef READ_ADDR
 
+/* locations =============================================*/
+
+dma_channel_hw_t *dma_channel_hw_addr(uint channel);
+
 #define DMA_CHAN_ADDR(CH, TYP) ((TYP *volatile)_Generic((TYP){},                     \
+	                                                                             \
 	struct dma_alias0:        &dma_channel_hw_addr(CH)->read_addr,               \
 	struct dma_alias1:        &dma_channel_hw_addr(CH)->al1_ctrl,                \
 	struct dma_alias2:        &dma_channel_hw_addr(CH)->al2_ctrl,                \
 	struct dma_alias3:        &dma_channel_hw_addr(CH)->al3_ctrl,                \
+	                                                                             \
 	struct dma_alias0_short2: &dma_channel_hw_addr(CH)->transfer_count,          \
 	struct dma_alias1_short2: &dma_channel_hw_addr(CH)->al1_write_addr,          \
 	struct dma_alias2_short2: &dma_channel_hw_addr(CH)->al2_read_addr,           \
 	struct dma_alias3_short2: &dma_channel_hw_addr(CH)->al3_transfer_count,      \
+	                                                                             \
 	struct dma_alias0_short3: &dma_channel_hw_addr(CH)->ctrl_trig,               \
 	struct dma_alias1_short3: &dma_channel_hw_addr(CH)->al1_transfer_count_trig, \
 	struct dma_alias2_short3: &dma_channel_hw_addr(CH)->al2_write_addr_trig,     \
@@ -105,24 +91,48 @@ struct dma_alias3_short3                                  { READ_ADDR   ; };
 
 #define DMA_IS_TRIGGER(TYP, FIELD) (offsetof(TYP, FIELD) == 0xC)
 
-#define DMA_CHAN_WR_TRANS_COUNT(TYP) \
-	(sizeof(TYP)/4)
-
-#define DMA_CHAN_WR_CTRL(TYP) ( DMA_CTRL_DATA_SIZE(DMA_SIZE_32) \
-                              | DMA_CTRL_INCR_WRITE \
-                              | DMA_CTRL_RING(WR, LM_FLOORLOG2(sizeof(TYP))) \
-                              )
-
 #define DMA_NONTRIGGER(CH, FIELD) (DMA_CHAN_ADDR(CH, _DMA_NONTRIGGER_##FIELD)->FIELD)
 #define _DMA_NONTRIGGER_read_addr   struct dma_alias0
 #define _DMA_NONTRIGGER_write_addr  struct dma_alias0
-#define _DMA_NONTRIGGER_trans_count struct dma_alias0
+#define _DMA_NONTRIGGER_xfer_count struct dma_alias0
 #define _DMA_NONTRIGGER_ctrl        struct dma_alias1
 
 #define DMA_TRIGGER(CH, FIELD)    (DMA_CHAN_ADDR(CH, _DMA_TRIGGER_##FIELD)->FIELD)
 #define _DMA_TRIGGER_read_addr      struct dma_alias3
 #define _DMA_TRIGGER_write_addr     struct dma_alias2
-#define _DMA_TRIGGER_trans_count    struct dma_alias1
+#define _DMA_TRIGGER_xfer_count    struct dma_alias1
 #define _DMA_TRIGGER_ctrl           struct dma_alias0
 
+/* block->ctrl register *******************************************************/
+
+enum dma_transfer_size {
+	DMA_SIZE_8  = 0, /** Byte transfer (8 bits) */
+	DMA_SIZE_16 = 1, /** Half word transfer (16 bits) */
+	DMA_SIZE_32 = 2, /** Word transfer (32 bits) */
+};
+
+#define DMA_CTRL_ENABLE           (1<<0)
+#define DMA_CTRL_HI_PRIO          (1<<1)
+#define DMA_CTRL_DATA_SIZE(sz)    ((sz)<<2)
+#define DMA_CTRL_INCR_READ        (1<<4)
+#define DMA_CTRL_INCR_WRITE       (1<<5)
+#define _DMA_CTRL_RING_BITS(b)    ((b)<<6)
+#define _DMA_CTRL_RING_RD         (0)
+#define _DMA_CTRL_RING_WR         (1<<10)
+#define DMA_CTRL_RING(rdwr, bits) (_DMA_CTRL_RING_##rdwr | _DMA_CTRL_RING_BITS(bits))
+#define DMA_CTRL_CHAIN_TO(ch)     ((ch)<<11)
+#define DMA_CTRL_TREQ_SEL(dreq)   ((dreq)<<15)
+#define DMA_CTRL_IRQ_QUIET        (1<<21)
+#define DMA_CTRL_BSWAP            (1<<22)
+#define DMA_CTRL_SNIFF_EN         (1<<23)
+
+/* Utilities to build control blocks that write to other control blocks *******/
+
+#define DMA_CHAN_WR_XFER_COUNT(TYP) \
+	(sizeof(TYP)/4)
+#define DMA_CHAN_WR_CTRL(TYP) ( DMA_CTRL_DATA_SIZE(DMA_SIZE_32) \
+                              | DMA_CTRL_INCR_WRITE \
+                              | DMA_CTRL_RING(WR, LM_FLOORLOG2(sizeof(TYP))) \
+                              )
+
 #endif /* _LIBHW_CR_RP2040_DMA_H_ */
diff --git a/libhw_cr/rp2040_hwspi.c b/libhw_cr/rp2040_hwspi.c
index 6b0d1f5..14b2762 100644
--- a/libhw_cr/rp2040_hwspi.c
+++ b/libhw_cr/rp2040_hwspi.c
@@ -37,13 +37,26 @@ static void rp2040_hwspi_intrhandler(void *_self, enum dmairq LM_UNUSED(irq), ui
 	cr_sema_signal_from_intrhandler(&self->sema);
 }
 
+#define assert_2distinct(a, b) \
+	assert(a != b)
+
+#define assert_3distinct(a, b, c) \
+	assert_2distinct(a, b); \
+	assert(c != a); \
+	assert(c != b)
+
 #define assert_4distinct(a, b, c, d) \
-	assert(a != b); \
-	assert(a != c); \
-	assert(a != d); \
-	assert(b != c); \
-	assert(b != d); \
-	assert(c != d);
+	assert_3distinct(a, b, c); \
+	assert(d != a); \
+	assert(d != b); \
+	assert(d != c)
+
+#define assert_5distinct(a, b, c, d, e) \
+	assert_4distinct(a, b, c, d); \
+	assert(e != a); \
+	assert(e != b); \
+	assert(e != c); \
+	assert(e != d)
 
 void _rp2040_hwspi_init(struct rp2040_hwspi *self,
                         enum rp2040_hwspi_instance inst_num,
@@ -71,7 +84,9 @@ void _rp2040_hwspi_init(struct rp2040_hwspi *self,
 	log_debugln("clk_peri = ", clk_peri_hz, "Hz");
 	assert(baudrate_hz*2 <= clk_peri_hz);
 	assert_4distinct(pin_miso, pin_mosi, pin_clk, pin_cs);
-	assert_4distinct(dma1, dma2, dma3, dma4);
+	/* I don't trust DMA channel 0
+	 * https://github.com/raspberrypi/pico-feedback/issues/464 */
+	assert_5distinct(0, dma1, dma2, dma3, dma4);
 
 	/* Regarding the constraints on pin assignments: see the
 	 * RP2040 datasheet, table 2, in §1.4.3 "GPIO Functions".  */
@@ -166,22 +181,22 @@ size_t_and_error rp2040_hwspi_readwritev(struct rp2040_hwspi *self, const struct
 	}
 	assert(count);
 
-	/* It doesn't *really* matter which aliases we choose:
+	/* The code following this initial declaration is generic to
+	 * the alias, so changing which alias is used is easy.  But
+	 * which aliases should we choose?
 	 *
-	 *  - None of our fields can be NULL (so no
-	 *    false-termination).
+	 * Hard requirements:
 	 *
-	 *  - Moving const fields first so they don't have to be
-	 *    re-programmed each time isn't possible for us; there
-	 *    need to be at least 2 const fields, and we only have 1
-	 *    (read_addr for rx_data_blocks, and write_addr for
-	 *    tx_data_blocks).
+	 *  - The RP2040 can read from NULL (that's where the ROM is),
+	 *    so we need the tx channel's read_addr to not be the
+	 *    trigger, to avoid accidental null-triggers.
+	 *    false-termination).
 	 *
-	 * The code following this initial declaration is generic to
-	 * the alias, so changing which alias is used is easy.
+	 * Soft requirements:
 	 *
-	 * Since we have no hard requirements, here are some mild
-	 * preferences:
+	 *  - We can't write to NULL (it's ROM), but let's give the
+	 *    same consideration to the rx channel's write_addr
+	 *    anyway.
 	 *
 	 *  - I like the aliases being different for each channel,
 	 *    because it helps prevent alias-specific code from
@@ -193,71 +208,73 @@ size_t_and_error rp2040_hwspi_readwritev(struct rp2040_hwspi *self, const struct
 	 *    cleared before the trigger, and at the end the control
 	 *    block is clean and zeroed-out.
 	 *
-	 *  - Conversely, I like the tx channel (the non-interrupt
-	 *    channel) having ctrl *not* be the trigger, so that
-	 *    DMA_CTRL_IRQ_QUIET is cleared by the time the trigger
-	 *    happens, so the IRQ machinery doesn't need to be engaged
-	 *    at all.
+	 * Non-requirements:
+	 *
+	 *  - Moving const fields first so they don't have to be
+	 *    re-programmed each time isn't possible for us; there
+	 *    need to be at least 2 const fields, and we only have 1
+	 *    (read_addr for rx_data_blocks, and write_addr for
+	 *    tx_data_blocks).
 	 */
-	[[gnu::cleanup(heap_cleanup)]] struct dma_alias1 *tx_data_blocks = heap_alloc(pruned_iovcnt+1, struct dma_alias1);
-	[[gnu::cleanup(heap_cleanup)]] struct dma_alias0 *rx_data_blocks = heap_alloc(pruned_iovcnt+1, struct dma_alias0);
-	static_assert(!DMA_IS_TRIGGER(typeof(tx_data_blocks[0]), ctrl));
-	static_assert(DMA_IS_TRIGGER(typeof(rx_data_blocks[0]), ctrl));
+	[[gnu::cleanup(heap_cleanup)]] struct dma_alias1 *tx_data_blocks = heap_alloc(pruned_iovcnt, struct dma_alias1);
+	[[gnu::cleanup(heap_cleanup)]] struct dma_alias0 *rx_data_blocks = heap_alloc(pruned_iovcnt+1, struct dma_alias0); /* extra +1 block for null trigger */
+	/* hard requirements */
+	static_assert(!DMA_IS_TRIGGER(typeof(tx_data_blocks[0]), read_addr)); /* avoid accidental null-trigger */
+	/* soft requirements */
+	static_assert(!DMA_IS_TRIGGER(typeof(rx_data_blocks[0]), write_addr)); /* avoid accidental null-trigger */
+	static_assert(!__builtin_types_compatible_p(typeof(tx_data_blocks[0]), typeof(rx_data_blocks[0]))); /* help detect code errors */
+	static_assert(DMA_IS_TRIGGER(typeof(rx_data_blocks[0]), ctrl)); /* avoid needing to set IRQ_QUIET in the null-trigger block */
 
+	/* Core data blocks.  */
 	for (int i = 0, j = 0; i < iovcnt; i++) {
 		if (!iov[i].iov_len)
 			continue;
-		tx_data_blocks[j]    = (typeof(tx_data_blocks[0])){
-			.read_addr   = (iov[i].iov_write_from != IOVEC_DISCARD) ? iov[i].iov_write_from : &self->bogus_data,
-			.write_addr  = &spi_get_hw(self->inst)->dr,
-			.trans_count = iov[i].iov_len,
-			.ctrl        = (DMA_CTRL_ENABLE
-			                | DMA_CTRL_DATA_SIZE(DMA_SIZE_8)
-			                | ((iov[i].iov_write_from != IOVEC_DISCARD) ? DMA_CTRL_INCR_READ : 0)
-			                | DMA_CTRL_CHAIN_TO(self->dma_tx_ctrl)
-			                | DMA_CTRL_TREQ_SEL(SPI_DREQ_NUM(self->inst, true))
-			                | DMA_CTRL_IRQ_QUIET),
+		tx_data_blocks[j]   = (typeof(tx_data_blocks[0])){
+			.read_addr  = (iov[i].iov_write_from != IOVEC_DISCARD) ? iov[i].iov_write_from : &self->bogus_data,
+			.write_addr = &spi_get_hw(self->inst)->dr,
+			.xfer_count = iov[i].iov_len,
+			.ctrl       = (DMA_CTRL_ENABLE
+			               | DMA_CTRL_DATA_SIZE(DMA_SIZE_8)
+			               | ((iov[i].iov_write_from != IOVEC_DISCARD) ? DMA_CTRL_INCR_READ : 0)
+			               | ((j+1 < pruned_iovcnt) ? DMA_CTRL_CHAIN_TO(self->dma_tx_ctrl) : 0)
+			               | DMA_CTRL_TREQ_SEL(SPI_DREQ_NUM(self->inst, true))),
 		};
-		rx_data_blocks[j]    = (typeof(rx_data_blocks[0])){
-			.read_addr   = &spi_get_hw(self->inst)->dr,
-			.write_addr  = (iov[i].iov_read_to != IOVEC_DISCARD) ? iov[i].iov_read_to : &bogus_rx_dst,
-			.trans_count = iov[i].iov_len,
-			.ctrl        = (DMA_CTRL_ENABLE
-			                | DMA_CTRL_DATA_SIZE(DMA_SIZE_8)
-			                | ((iov[i].iov_read_to != IOVEC_DISCARD) ? DMA_CTRL_INCR_WRITE : 0)
-			                | DMA_CTRL_CHAIN_TO(self->dma_rx_ctrl)
-			                | DMA_CTRL_TREQ_SEL(SPI_DREQ_NUM(self->inst, false))
-			                | DMA_CTRL_IRQ_QUIET),
+		rx_data_blocks[j]   = (typeof(rx_data_blocks[0])){
+			.read_addr  = &spi_get_hw(self->inst)->dr,
+			.write_addr = (iov[i].iov_read_to != IOVEC_DISCARD) ? iov[i].iov_read_to : &bogus_rx_dst,
+			.xfer_count = iov[i].iov_len,
+			.ctrl       = (DMA_CTRL_ENABLE
+			               | DMA_CTRL_DATA_SIZE(DMA_SIZE_8)
+			               | ((iov[i].iov_read_to != IOVEC_DISCARD) ? DMA_CTRL_INCR_WRITE : 0)
+			               | DMA_CTRL_CHAIN_TO(self->dma_rx_ctrl)
+			               | DMA_CTRL_TREQ_SEL(SPI_DREQ_NUM(self->inst, false))
+			               | DMA_CTRL_IRQ_QUIET),
 		};
 		j++;
 	}
-	tx_data_blocks[pruned_iovcnt] = (typeof(tx_data_blocks[0])){};
-	rx_data_blocks[pruned_iovcnt] = (typeof(rx_data_blocks[0])){};
-	/* If ctrl isn't the trigger then we need to make sure that
-	 * DMA_CTRL_IRQ_QUIET isn't cleared before the trigger
-	 * happens.  */
-	if (!DMA_IS_TRIGGER(typeof(rx_data_blocks[0]), ctrl))
-		rx_data_blocks[pruned_iovcnt].ctrl = DMA_CTRL_IRQ_QUIET;
+	/* Null-trigger (generate IRQ).  */
+	rx_data_blocks[pruned_iovcnt] = (typeof(rx_data_blocks[0])){
+		/* If ctrl isn't the trigger then we need to make sure
+		 * that DMA_CTRL_IRQ_QUIET isn't cleared before the
+		 * trigger happens.  */
+		.ctrl = DMA_IS_TRIGGER(typeof(rx_data_blocks[0]), ctrl) ? 0 : DMA_CTRL_IRQ_QUIET,
+	};
 
 	/* Set up ctrl.  */
 	DMA_NONTRIGGER(self->dma_tx_ctrl, read_addr) = tx_data_blocks;
 	DMA_NONTRIGGER(self->dma_tx_ctrl, write_addr) = DMA_CHAN_ADDR(self->dma_tx_data, typeof(tx_data_blocks[0]));
-	DMA_NONTRIGGER(self->dma_tx_ctrl, trans_count) = DMA_CHAN_WR_TRANS_COUNT(typeof(tx_data_blocks[0]));
+	DMA_NONTRIGGER(self->dma_tx_ctrl, xfer_count) = DMA_CHAN_WR_XFER_COUNT(typeof(tx_data_blocks[0]));
 	DMA_NONTRIGGER(self->dma_tx_ctrl, ctrl) = (DMA_CTRL_ENABLE
 	                                           | DMA_CHAN_WR_CTRL(typeof(tx_data_blocks[0]))
 	                                           | DMA_CTRL_INCR_READ
-	                                           | DMA_CTRL_CHAIN_TO(self->dma_tx_data)
-	                                           | DMA_CTRL_TREQ_SEL(DREQ_FORCE)
-	                                           | DMA_CTRL_IRQ_QUIET);
+	                                           | DMA_CTRL_TREQ_SEL(DREQ_FORCE));
 	DMA_NONTRIGGER(self->dma_rx_ctrl, read_addr) = rx_data_blocks;
 	DMA_NONTRIGGER(self->dma_rx_ctrl, write_addr) = DMA_CHAN_ADDR(self->dma_rx_data, typeof(rx_data_blocks[0]));
-	DMA_NONTRIGGER(self->dma_rx_ctrl, trans_count) = DMA_CHAN_WR_TRANS_COUNT(typeof(rx_data_blocks[0]));
+	DMA_NONTRIGGER(self->dma_rx_ctrl, xfer_count) = DMA_CHAN_WR_XFER_COUNT(typeof(rx_data_blocks[0]));
 	DMA_NONTRIGGER(self->dma_rx_ctrl, ctrl) = (DMA_CTRL_ENABLE
 	                                           | DMA_CHAN_WR_CTRL(typeof(rx_data_blocks[0]))
 	                                           | DMA_CTRL_INCR_READ
-	                                           | DMA_CTRL_CHAIN_TO(self->dma_rx_data)
-	                                           | DMA_CTRL_TREQ_SEL(DREQ_FORCE)
-	                                           | DMA_CTRL_IRQ_QUIET);
+	                                           | DMA_CTRL_TREQ_SEL(DREQ_FORCE));
 
 	/* Run.  */
 	uint64_t now = LO_CALL(bootclock, get_time_ns);
@@ -265,7 +282,7 @@ size_t_and_error rp2040_hwspi_readwritev(struct rp2040_hwspi *self, const struct
 		sleep_until_ns(self->dead_until_ns);
 	bool saved = cr_save_and_disable_interrupts();
 	gpio_put(self->pin_cs, 0);
-	dma_hw->multi_channel_trigger = (1u<<self->dma_tx_ctrl) | (1u<<self->dma_rx_ctrl);
+	dma_hw->multi_channel_trigger = (1<<self->dma_tx_ctrl) | (1<<self->dma_rx_ctrl);
 	cr_restore_interrupts(saved);
 	cr_sema_wait(&self->sema);
 	self->dead_until_ns = LO_CALL(bootclock, get_time_ns) + self->min_delay_ns;