diff options
Diffstat (limited to 'libhw_cr/rp2040_hwspi.c')
| -rw-r--r-- | libhw_cr/rp2040_hwspi.c | 218 |
1 files changed, 136 insertions, 82 deletions
diff --git a/libhw_cr/rp2040_hwspi.c b/libhw_cr/rp2040_hwspi.c index 646d8ba..f667332 100644 --- a/libhw_cr/rp2040_hwspi.c +++ b/libhw_cr/rp2040_hwspi.c @@ -4,14 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -#include <alloca.h> -#include <inttypes.h> /* for PRIu{n} */ - #include <hardware/clocks.h> /* for clock_get_hz() and clk_peri */ #include <hardware/gpio.h> #include <hardware/spi.h> #include <libcr/coroutine.h> +#include <libmisc/alloc.h> #include <libmisc/assert.h> #define LOG_NAME RP2040_SPI @@ -29,23 +27,40 @@ #ifndef CONFIG_RP2040_SPI_DEBUG #error config.h must define CONFIG_RP2040_SPI_DEBUG (bool) #endif +#ifndef CONFIG_RP2040_SPI_MAX_DMABUF + #error config.h must define CONFIG_RP2040_SPI_DEBUG (non-negative integer) +#endif -LO_IMPLEMENTATION_C(io_duplex_readwriter, struct rp2040_hwspi, rp2040_hwspi, static); -LO_IMPLEMENTATION_C(spi, struct rp2040_hwspi, rp2040_hwspi, static); +LO_IMPLEMENTATION_C(io_duplex_readwriter, struct rp2040_hwspi, rp2040_hwspi); +LO_IMPLEMENTATION_C(spi, struct rp2040_hwspi, rp2040_hwspi); static void rp2040_hwspi_intrhandler(void *_self, enum dmairq LM_UNUSED(irq), uint LM_UNUSED(channel)) { struct rp2040_hwspi *self = _self; gpio_put(self->pin_cs, 1); - cr_sema_signal_from_intrhandler(&self->sema); + assert(((spi_hw_t *)self->inst)->sr == 0b11); + cr_unpause_from_intrhandler(self->waiter); } +#define assert_2distinct(a, b) \ + assert(a != b) + +#define assert_3distinct(a, b, c) \ + assert_2distinct(a, b); \ + assert(c != a); \ + assert(c != b) + #define assert_4distinct(a, b, c, d) \ - assert(a != b); \ - assert(a != c); \ - assert(a != d); \ - assert(b != c); \ - assert(b != d); \ - assert(c != d); + assert_3distinct(a, b, c); \ + assert(d != a); \ + assert(d != b); \ + assert(d != c) + +#define assert_5distinct(a, b, c, d, e) \ + assert_4distinct(a, b, c, d); \ + assert(e != a); \ + assert(e != b); \ + assert(e != c); \ + assert(e != d) void _rp2040_hwspi_init(struct rp2040_hwspi *self, enum rp2040_hwspi_instance inst_num, @@ -70,10 +85,12 @@ void _rp2040_hwspi_init(struct rp2040_hwspi *self, assert(self); assert(baudrate_hz); uint32_t clk_peri_hz = clock_get_hz(clk_peri); - debugf("clk_peri = %"PRIu32"Hz", clk_peri_hz); + log_debugln("clk_peri = ", clk_peri_hz, "Hz"); assert(baudrate_hz*2 <= clk_peri_hz); assert_4distinct(pin_miso, pin_mosi, pin_clk, pin_cs); - assert_4distinct(dma1, dma2, dma3, dma4); + /* I don't trust DMA channel 0 + * https://github.com/raspberrypi/pico-feedback/issues/464 */ + assert_5distinct(0, dma1, dma2, dma3, dma4); /* Regarding the constraints on pin assignments: see the * RP2040 datasheet, table 2, in ยง1.4.3 "GPIO Functions". */ @@ -96,8 +113,9 @@ void _rp2040_hwspi_init(struct rp2040_hwspi *self, assert_notreached("invalid hwspi instance number"); } + /* Initialize the PL022. */ actual_baudrate_hz = spi_init(inst, baudrate_hz); - debugf("baudrate = %uHz", actual_baudrate_hz); + log_debugln("baudrate = ", actual_baudrate_hz, "Hz"); assert(actual_baudrate_hz == baudrate_hz); spi_set_format(inst, 8, (mode & 0b10) ? SPI_CPOL_1 : SPI_CPOL_0, @@ -130,7 +148,6 @@ void _rp2040_hwspi_init(struct rp2040_hwspi *self, self->dma_tx_data = dma3; self->dma_rx_data = dma4; self->dead_until_ns = 0; - self->sema = (cr_sema_t){0}; /* Initialize the interrupt handler. */ /* We do this on (just) the rx channel, because the way the @@ -138,7 +155,7 @@ void _rp2040_hwspi_init(struct rp2040_hwspi *self, dmairq_set_and_enable_exclusive_handler(DMAIRQ_0, self->dma_rx_data, rp2040_hwspi_intrhandler, self); } -static void rp2040_hwspi_readwritev(struct rp2040_hwspi *self, const struct duplex_iovec *iov, int iovcnt) { +size_t_and_error rp2040_hwspi_readwritev(struct rp2040_hwspi *self, const struct duplex_iovec *iov, int iovcnt) { assert(self); assert(self->inst); assert(iov); @@ -158,29 +175,38 @@ static void rp2040_hwspi_readwritev(struct rp2040_hwspi *self, const struct dupl uint8_t bogus_rx_dst; + size_t count = 0; + size_t unsafe_count = 0; int pruned_iovcnt = 0; - for (int i = 0; i < iovcnt; i++) - if (iov[i].iov_len) - pruned_iovcnt++; - if (!pruned_iovcnt) - return; + for (int i = 0; i < iovcnt; i++) { + if (!iov[i].iov_len) + continue; + pruned_iovcnt++; + count += iov[i].iov_len; + if (dma_is_unsafe(iov[i].iov_write_from)) + unsafe_count += iov[i].iov_len; + } + assert(count); + assert(unsafe_count <= CONFIG_RP2040_SPI_MAX_DMABUF); - /* It doesn't *really* matter which aliases we choose: + assert(((spi_hw_t *)self->inst)->sr == 0b11); + + /* The code following this initial declaration is generic to + * the alias, so changing which alias is used is easy. But + * which aliases should we choose? * - * - None of our fields can be NULL (so no - * false-termination). + * Hard requirements: * - * - Moving const fields first so they don't have to be - * re-programmed each time isn't possible for us; there - * need to be at least 2 const fields, and we only have 1 - * (read_addr for rx_data_blocks, and write_addr for - * tx_data_blocks). + * - The RP2040 can read from NULL (that's where the ROM is), + * so we need the tx channel's read_addr to not be the + * trigger, to avoid accidental null-triggers. + * false-termination). * - * The code following this initial declaration is generic to - * the alias, so changing which alias is used is easy. + * Soft requirements: * - * Since we have no hard requirements, here are some mild - * preferences: + * - We can't write to NULL (it's ROM), but let's give the + * same consideration to the rx channel's write_addr + * anyway. * * - I like the aliases being different for each channel, * because it helps prevent alias-specific code from @@ -192,80 +218,108 @@ static void rp2040_hwspi_readwritev(struct rp2040_hwspi *self, const struct dupl * cleared before the trigger, and at the end the control * block is clean and zeroed-out. * - * - Conversely, I like the tx channel (the non-interrupt - * channel) having ctrl *not* be the trigger, so that - * DMA_CTRL_IRQ_QUIET is cleared by the time the trigger - * happens, so the IRQ machinery doesn't need to be engaged - * at all. + * Non-requirements: + * + * - Moving const fields first so they don't have to be + * re-programmed each time isn't possible for us; there + * need to be at least 2 const fields, and we only have 1 + * (read_addr for rx_data_blocks, and write_addr for + * tx_data_blocks). */ - struct dma_alias1 *tx_data_blocks = alloca(sizeof(struct dma_alias1)*(pruned_iovcnt+1)); - struct dma_alias0 *rx_data_blocks = alloca(sizeof(struct dma_alias0)*(pruned_iovcnt+1)); - static_assert(!DMA_IS_TRIGGER(typeof(tx_data_blocks[0]), ctrl)); - static_assert(DMA_IS_TRIGGER(typeof(rx_data_blocks[0]), ctrl)); + [[gnu::cleanup(heap_cleanup)]] struct dma_alias1 *tx_data_blocks = heap_alloc(pruned_iovcnt, struct dma_alias1); + [[gnu::cleanup(heap_cleanup)]] struct dma_alias0 *rx_data_blocks = heap_alloc(pruned_iovcnt+1, struct dma_alias0); /* extra +1 block for null trigger */ + /* hard requirements */ + static_assert(!DMA_IS_TRIGGER(typeof(tx_data_blocks[0]), read_addr)); /* avoid accidental null-trigger */ + /* soft requirements */ + static_assert(!DMA_IS_TRIGGER(typeof(rx_data_blocks[0]), write_addr)); /* avoid accidental null-trigger */ + static_assert(!__builtin_types_compatible_p(typeof(tx_data_blocks[0]), typeof(rx_data_blocks[0]))); /* help detect code errors */ + static_assert(DMA_IS_TRIGGER(typeof(rx_data_blocks[0]), ctrl)); /* avoid needing to set IRQ_QUIET in the null-trigger block */ + /* Core data blocks. */ + [[gnu::cleanup(heap_cleanup)]] void *dmabuf = NULL; + if (unsafe_count) + dmabuf = heap_alloc(unsafe_count, char); + size_t dmabuf_pos = 0; for (int i = 0, j = 0; i < iovcnt; i++) { if (!iov[i].iov_len) continue; - tx_data_blocks[j] = (typeof(tx_data_blocks[0])){ - .read_addr = (iov[i].iov_write_from != IOVEC_DISCARD) ? iov[i].iov_write_from : &self->bogus_data, - .write_addr = &spi_get_hw(self->inst)->dr, - .trans_count = iov[i].iov_len, - .ctrl = (DMA_CTRL_ENABLE - | DMA_CTRL_DATA_SIZE(DMA_SIZE_8) - | ((iov[i].iov_write_from != IOVEC_DISCARD) ? DMA_CTRL_INCR_READ : 0) - | DMA_CTRL_CHAIN_TO(self->dma_tx_ctrl) - | DMA_CTRL_TREQ_SEL(SPI_DREQ_NUM(self->inst, true)) - | DMA_CTRL_IRQ_QUIET), + + const void *write_from = iov[i].iov_write_from; + if (write_from == IOVEC_DISCARD) + write_from = &self->bogus_data; + else if (dma_is_unsafe(write_from)) { + memcpy(dmabuf+dmabuf_pos, write_from, iov[i].iov_len); + write_from = dmabuf+dmabuf_pos; + dmabuf_pos += iov[i].iov_len; + } + tx_data_blocks[j] = (typeof(tx_data_blocks[0])){ + .read_addr = write_from, + .write_addr = &spi_get_hw(self->inst)->dr, + .xfer_count = iov[i].iov_len, + .ctrl = (DMA_CTRL_ENABLE + | DMA_CTRL_DATA_SIZE(DMA_SIZE_8) + | ((iov[i].iov_write_from != IOVEC_DISCARD) ? DMA_CTRL_INCR_READ : 0) + | ((j+1 < pruned_iovcnt) ? DMA_CTRL_CHAIN_TO(self->dma_tx_ctrl) : 0) + | DMA_CTRL_TREQ_SEL(SPI_DREQ_NUM(self->inst, true))), }; - rx_data_blocks[j] = (typeof(rx_data_blocks[0])){ - .read_addr = &spi_get_hw(self->inst)->dr, - .write_addr = (iov[i].iov_read_to != IOVEC_DISCARD) ? iov[i].iov_read_to : &bogus_rx_dst, - .trans_count = iov[i].iov_len, - .ctrl = (DMA_CTRL_ENABLE - | DMA_CTRL_DATA_SIZE(DMA_SIZE_8) - | ((iov[i].iov_read_to != IOVEC_DISCARD) ? DMA_CTRL_INCR_WRITE : 0) - | DMA_CTRL_CHAIN_TO(self->dma_rx_ctrl) - | DMA_CTRL_TREQ_SEL(SPI_DREQ_NUM(self->inst, false)) - | DMA_CTRL_IRQ_QUIET), + dma_assert_addrs(tx_data_blocks[j].write_addr, tx_data_blocks[j].read_addr); + + void *read_to = iov[i].iov_read_to; + if (read_to == IOVEC_DISCARD) + read_to = &bogus_rx_dst; + rx_data_blocks[j] = (typeof(rx_data_blocks[0])){ + .read_addr = &spi_get_hw(self->inst)->dr, + .write_addr = read_to, + .xfer_count = iov[i].iov_len, + .ctrl = (DMA_CTRL_ENABLE + | DMA_CTRL_DATA_SIZE(DMA_SIZE_8) + | ((iov[i].iov_read_to != IOVEC_DISCARD) ? DMA_CTRL_INCR_WRITE : 0) + | DMA_CTRL_CHAIN_TO(self->dma_rx_ctrl) + | DMA_CTRL_TREQ_SEL(SPI_DREQ_NUM(self->inst, false)) + | DMA_CTRL_IRQ_QUIET), }; + dma_assert_addrs(rx_data_blocks[j].write_addr, rx_data_blocks[j].read_addr); + j++; } - tx_data_blocks[pruned_iovcnt] = (typeof(tx_data_blocks[0])){0}; - rx_data_blocks[pruned_iovcnt] = (typeof(rx_data_blocks[0])){0}; - /* If ctrl isn't the trigger then we need to make sure that - * DMA_CTRL_IRQ_QUIET isn't cleared before the trigger - * happens. */ - if (!DMA_IS_TRIGGER(typeof(rx_data_blocks[0]), ctrl)) - rx_data_blocks[pruned_iovcnt].ctrl = DMA_CTRL_IRQ_QUIET; + /* Null-trigger (generate IRQ). */ + rx_data_blocks[pruned_iovcnt] = (typeof(rx_data_blocks[0])){ + /* If ctrl isn't the trigger then we need to make sure + * that DMA_CTRL_IRQ_QUIET isn't cleared before the + * trigger happens. */ + .ctrl = DMA_IS_TRIGGER(typeof(rx_data_blocks[0]), ctrl) ? 0 : DMA_CTRL_IRQ_QUIET, + }; /* Set up ctrl. */ DMA_NONTRIGGER(self->dma_tx_ctrl, read_addr) = tx_data_blocks; DMA_NONTRIGGER(self->dma_tx_ctrl, write_addr) = DMA_CHAN_ADDR(self->dma_tx_data, typeof(tx_data_blocks[0])); - DMA_NONTRIGGER(self->dma_tx_ctrl, trans_count) = DMA_CHAN_WR_TRANS_COUNT(typeof(tx_data_blocks[0])); + DMA_NONTRIGGER(self->dma_tx_ctrl, xfer_count) = DMA_CHAN_WR_XFER_COUNT(typeof(tx_data_blocks[0])); DMA_NONTRIGGER(self->dma_tx_ctrl, ctrl) = (DMA_CTRL_ENABLE | DMA_CHAN_WR_CTRL(typeof(tx_data_blocks[0])) | DMA_CTRL_INCR_READ - | DMA_CTRL_CHAIN_TO(self->dma_tx_data) - | DMA_CTRL_TREQ_SEL(DREQ_FORCE) - | DMA_CTRL_IRQ_QUIET); + | DMA_CTRL_TREQ_SEL(DREQ_FORCE)); DMA_NONTRIGGER(self->dma_rx_ctrl, read_addr) = rx_data_blocks; DMA_NONTRIGGER(self->dma_rx_ctrl, write_addr) = DMA_CHAN_ADDR(self->dma_rx_data, typeof(rx_data_blocks[0])); - DMA_NONTRIGGER(self->dma_rx_ctrl, trans_count) = DMA_CHAN_WR_TRANS_COUNT(typeof(rx_data_blocks[0])); + DMA_NONTRIGGER(self->dma_rx_ctrl, xfer_count) = DMA_CHAN_WR_XFER_COUNT(typeof(rx_data_blocks[0])); DMA_NONTRIGGER(self->dma_rx_ctrl, ctrl) = (DMA_CTRL_ENABLE | DMA_CHAN_WR_CTRL(typeof(rx_data_blocks[0])) | DMA_CTRL_INCR_READ - | DMA_CTRL_CHAIN_TO(self->dma_rx_data) - | DMA_CTRL_TREQ_SEL(DREQ_FORCE) - | DMA_CTRL_IRQ_QUIET); + | DMA_CTRL_TREQ_SEL(DREQ_FORCE)); /* Run. */ - uint64_t now = LO_CALL(bootclock, get_time_ns); - if (now < self->dead_until_ns) + + self->waiter = cr_getcid(); + + if (LO_CALL(bootclock, get_time_ns) < self->dead_until_ns) sleep_until_ns(self->dead_until_ns); + bool saved = cr_save_and_disable_interrupts(); gpio_put(self->pin_cs, 0); - dma_hw->multi_channel_trigger = (1u<<self->dma_tx_ctrl) | (1u<<self->dma_rx_ctrl); + dma_hw->multi_channel_trigger = (1<<self->dma_tx_ctrl) | (1<<self->dma_rx_ctrl); + cr_pause_and_yield(); + assert(((spi_hw_t *)self->inst)->sr == 0b11); cr_restore_interrupts(saved); - cr_sema_wait(&self->sema); + self->dead_until_ns = LO_CALL(bootclock, get_time_ns) + self->min_delay_ns; + return ERROR_AND(size_t, count, ERROR_NULL); } |