Implement QPI support

With the quad-i/o support in place, this is actually straight-
forward:
* we check for compatibility of the flash chip and programmer,
* select an appropriate fast-read function, and
* always set the respective io-mode when passing a SPI command
  to the programmer.

Tested with FT4222H + W25Q128FV and linux_gpio_spi + MX25L25645G.

Change-Id: I2287034f6818f24f892d66d1a505cb719838f75d
Signed-off-by: Nico Huber <nico.h@gmx.de>
Reviewed-on: https://review.sourcearcade.org/c/flashprog/+/165
Reviewed-by: Arthur Heymans <arthur@aheymans.xyz>
diff --git a/dediprog.c b/dediprog.c
index 43d8b84..6b72938 100644
--- a/dediprog.c
+++ b/dediprog.c
@@ -646,7 +646,7 @@
 	msg_pdbg("Slow read for partial block from 0x%x, length 0x%x\n", start, len);
 
 	/* Override fast-read function for a moment: */
-	const struct spi_read_op *const backup = flash->spi_fast_read;
+	struct spi_read_op *const backup = flash->spi_fast_read;
 	flash->spi_fast_read = NULL;
 
 	const int ret = default_spi_read(flash, buf, start, len);
diff --git a/include/flash.h b/include/flash.h
index f839995..fb64304 100644
--- a/include/flash.h
+++ b/include/flash.h
@@ -460,8 +460,9 @@
            of the extended address register. */
 	int address_high_byte;
 	bool in_4ba_mode;
+	bool in_qpi_mode;
 	/* For SPI flash chips, we dynamically select the fast-read operation. */
-	const struct spi_read_op *spi_fast_read;
+	struct spi_read_op *spi_fast_read;
 
 	int chip_restore_fn_count;
 	struct chip_restore_func_data {
diff --git a/include/programmer.h b/include/programmer.h
index 54cbdc0..2633450 100644
--- a/include/programmer.h
+++ b/include/programmer.h
@@ -506,6 +506,10 @@
 {
 	return flash->mst.spi->features & SPI_MASTER_QUAD;
 }
+static inline bool spi_master_qpi(const struct flashctx *const flash)
+{
+	return flash->mst.spi->features & SPI_MASTER_QPI;
+}
 
 /* usbdev.c */
 struct libusb_device_handle;
diff --git a/include/spi_command.h b/include/spi_command.h
index 3af0cd0..6560f22 100644
--- a/include/spi_command.h
+++ b/include/spi_command.h
@@ -68,6 +68,8 @@
 	QPI_4_4_4,
 };
 
+enum io_mode spi_current_io_mode(const struct flashctx *);
+
 /* describes properties of a read operation */
 struct spi_read_op {
 	enum io_mode io_mode;
diff --git a/spi.c b/spi.c
index de7965f..748ef99 100644
--- a/spi.c
+++ b/spi.c
@@ -31,6 +31,9 @@
 		     unsigned int readcnt, const unsigned char *writearr,
 		     unsigned char *readarr)
 {
+	if (spi_current_io_mode(flash) != SINGLE_IO_1_1_1)
+		return default_spi_send_command(flash, writecnt, readcnt, writearr, readarr);
+
 	return flash->mst.spi->command(flash, writecnt, readcnt, writearr,
 				       readarr);
 }
@@ -47,7 +50,7 @@
 {
 	struct spi_command cmd[] = {
 	{
-		.io_mode = SINGLE_IO_1_1_1,
+		.io_mode = spi_current_io_mode(flash),
 		.opcode_len = 1,
 		.address_len = writecnt - 1,
 		.read_len = readcnt,
diff --git a/spi25.c b/spi25.c
index 34f30df..0f49a13 100644
--- a/spi25.c
+++ b/spi25.c
@@ -312,10 +312,12 @@
 {
 	struct spi_command cmds[] = {
 	{
+		.io_mode = spi_current_io_mode(flash),
 		.readarr = 0,
 		.opcode_len = 1,
 		.writearr = (const unsigned char[]){ JEDEC_WREN },
 	}, {
+		.io_mode = spi_current_io_mode(flash),
 		.readarr = 0,
 		.opcode_len = 1,
 		.writearr = (const unsigned char[]){ op },
@@ -346,10 +348,12 @@
 
 	struct spi_command cmds[] = {
 	{
+		.io_mode = spi_current_io_mode(flash),
 		.readarr = 0,
 		.opcode_len = 1,
 		.writearr = (const unsigned char[]){ JEDEC_WREN },
 	}, {
+		.io_mode = spi_current_io_mode(flash),
 		.readarr = 0,
 		.opcode_len = 1,
 		.write_len = 1,
@@ -424,10 +428,12 @@
 	uint8_t cmd[1 + JEDEC_MAX_ADDR_LEN + 256];
 	struct spi_command cmds[] = {
 	{
+		.io_mode = spi_current_io_mode(flash),
 		.readarr = 0,
 		.opcode_len = 1,
 		.writearr = (const unsigned char[]){ JEDEC_WREN },
 	}, {
+		.io_mode = spi_current_io_mode(flash),
 		.readarr = 0,
 		.writearr = cmd,
 	},
diff --git a/spi25_prepare.c b/spi25_prepare.c
index 279f2e4..6ad1f7f 100644
--- a/spi25_prepare.c
+++ b/spi25_prepare.c
@@ -77,9 +77,36 @@
 	return 0;
 }
 
+static int spi_enter_qpi(struct flashctx *const flash)
+{
+	const unsigned char cmd = flash->chip->feature_bits & FEATURE_QPI_35_F5 ? 0x35 : 0x38;
+	const int ret = spi_send_command(flash, sizeof(cmd), 0, &cmd, NULL);
+	if (!ret) {
+		msg_cdbg("Entered QPI mode.\n");
+		flash->in_qpi_mode = true;
+	}
+	return ret;
+}
+
+static int spi_exit_qpi(struct flashctx *const flash)
+{
+	const unsigned char cmd = flash->chip->feature_bits & FEATURE_QPI_35_F5 ? 0xf5 : 0xff;
+	const int ret = spi_send_command(flash, sizeof(cmd), 0, &cmd, NULL);
+	if (!ret) {
+		msg_cdbg("Left QPI mode.\n");
+		flash->in_qpi_mode = false;
+	}
+	return ret;
+}
+
+enum io_mode spi_current_io_mode(const struct flashctx *const flash)
+{
+	return flash->in_qpi_mode ? QPI_4_4_4 : SINGLE_IO_1_1_1;
+}
+
 static int spi_prepare_quad_io(struct flashctx *const flash)
 {
-	if (!spi_master_quad(flash))
+	if (!spi_master_quad(flash) && !spi_master_qpi(flash))
 		return 0;
 
 	/* Check QE bit if present */
@@ -99,10 +126,58 @@
 		}
 	}
 
+	flash->in_qpi_mode = false;
+
+	if (!(flash->chip->feature_bits & (FEATURE_QPI_35_F5 | FEATURE_QPI_38_FF)) || !spi_master_qpi(flash))
+		return 0;
+
+	if (spi_enter_qpi(flash))
+		msg_cwarn("Failed to switch to QPI mode!\n");
+
 	return 0;
 }
 
-static const struct spi_read_op *select_spi_fast_read(const struct flashctx *flash)
+static bool qpi_use_fast_read_qio(const struct flashctx *flash)
+{
+	return flash->chip->feature_bits & FEATURE_SET_READ_PARAMS ||
+		flash->chip->reg_bits.dc[0].reg != INVALID_REG ||
+		(flash->chip->dummy_cycles.qpi_fast_read_qio != 0 &&
+		 (flash->chip->dummy_cycles.qpi_fast_read == 0 ||
+		  flash->chip->dummy_cycles.qpi_fast_read_qio <=
+			flash->chip->dummy_cycles.qpi_fast_read));
+}
+
+static int qpi_dummy_cycles(const struct flashctx *flash)
+{
+	if (flash->chip->feature_bits & FEATURE_SET_READ_PARAMS ||
+	    flash->chip->reg_bits.dc[0].reg != INVALID_REG)
+		/* TODO: Index 00 is assumed to be the default.
+		         Could switch to potentially faster params. */
+		return flash->chip->dummy_cycles.qpi_read_params.clks00;
+	else if (qpi_use_fast_read_qio(flash))
+		return flash->chip->dummy_cycles.qpi_fast_read_qio;
+	else
+		return flash->chip->dummy_cycles.qpi_fast_read;
+}
+
+static const struct spi_read_op *select_qpi_fast_read(const struct flashctx *flash)
+{
+	static const struct spi_read_op fast_read = { QPI_4_4_4, false, JEDEC_FAST_READ, 0x00, 0 };
+	static const struct spi_read_op fast_read_qio = { QPI_4_4_4, false, JEDEC_FAST_READ_QIO, 0xff, 0 };
+	static const struct spi_read_op fast_read_qio_4ba = { QPI_4_4_4, true, JEDEC_FAST_READ_QIO_4BA, 0xff, 0 };
+
+	if (qpi_use_fast_read_qio(flash)) {
+		if (flash->chip->feature_bits & FEATURE_FAST_READ_QPI4B &&
+		    spi_master_4ba(flash) && flash->mst.spi->probe_opcode(flash, fast_read_qio_4ba.opcode))
+			return &fast_read_qio_4ba;
+		else
+			return &fast_read_qio;
+	} else {
+		return &fast_read;
+	}
+}
+
+static const struct spi_read_op *select_multi_io_fast_read(const struct flashctx *flash)
 {
 	static const struct {
 		unsigned int feature_check;
@@ -138,6 +213,26 @@
 	return NULL;
 }
 
+static struct spi_read_op *select_spi_fast_read(const struct flashctx *flash)
+{
+	const struct spi_read_op *const fast_read =
+		flash->in_qpi_mode
+		? select_qpi_fast_read(flash)
+		: select_multi_io_fast_read(flash);
+	if (!fast_read)
+		return NULL;
+
+	struct spi_read_op *const fast_read_copy = malloc(sizeof(*flash->spi_fast_read));
+	if (!fast_read_copy)
+		return NULL;
+
+	*fast_read_copy = *fast_read;
+	if (flash->in_qpi_mode)
+		fast_read_copy->dummy_len = qpi_dummy_cycles(flash) / 2;
+
+	return fast_read_copy;
+}
+
 int spi_prepare_io(struct flashctx *const flash, const enum preparation_steps prep)
 {
 	if (prep != PREPARE_FULL)
@@ -153,9 +248,24 @@
 
 	flash->spi_fast_read = select_spi_fast_read(flash);
 
+	if (!flash->spi_fast_read && flash->in_qpi_mode) {
+		msg_cwarn("No compatible fast-read operation! Leaving QPI mode.\n");
+		if (spi_exit_qpi(flash)) {
+			msg_cerr("Failed to exit QPI mode!\n");
+			return 1;
+		}
+		/* Try again w/o QPI */
+		flash->spi_fast_read = select_spi_fast_read(flash);
+	}
+
 	return 0;
 }
 
 void spi_finish_io(struct flashctx *const flash)
 {
+	if (flash->in_qpi_mode) {
+		if (spi_exit_qpi(flash))
+			msg_cwarn("Failed to exit QPI mode!\n");
+	}
+	free(flash->spi_fast_read);
 }
diff --git a/spi25_statusreg.c b/spi25_statusreg.c
index b363b5f..24bef6c 100644
--- a/spi25_statusreg.c
+++ b/spi25_statusreg.c
@@ -139,9 +139,11 @@
 
 	struct spi_command cmds[] = {
 	{
+		.io_mode	= spi_current_io_mode(flash),
 		.opcode_len	= JEDEC_WREN_OUTSIZE,
 		.writearr	= &enable_cmd,
 	}, {
+		.io_mode	= spi_current_io_mode(flash),
 		.opcode_len	= 1,
 		.write_len	= write_cmd_len - 1,
 		.writearr	= write_cmd,