drivers/spi/spi-qcom-qspi.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 // Copyright (c) 2017-2018, The Linux foundation. All rights reserved.
0003
0004 #include <linux/clk.h>
0005 #include <linux/interconnect.h>
0006 #include <linux/interrupt.h>
0007 #include <linux/io.h>
0008 #include <linux/module.h>
0009 #include <linux/of.h>
0010 #include <linux/of_platform.h>
0011 #include <linux/pm_runtime.h>
0012 #include <linux/pm_opp.h>
0013 #include <linux/spi/spi.h>
0014 #include <linux/spi/spi-mem.h>
0015
0016
0017 #define QSPI_NUM_CS     2
0018 #define QSPI_BYTES_PER_WORD 4
0019
0020 #define MSTR_CONFIG     0x0000
0021 #define FULL_CYCLE_MODE     BIT(3)
0022 #define FB_CLK_EN       BIT(4)
0023 #define PIN_HOLDN       BIT(6)
0024 #define PIN_WPN         BIT(7)
0025 #define DMA_ENABLE      BIT(8)
0026 #define BIG_ENDIAN_MODE     BIT(9)
0027 #define SPI_MODE_MSK        0xc00
0028 #define SPI_MODE_SHFT       10
0029 #define CHIP_SELECT_NUM     BIT(12)
0030 #define SBL_EN          BIT(13)
0031 #define LPA_BASE_MSK        0x3c000
0032 #define LPA_BASE_SHFT       14
0033 #define TX_DATA_DELAY_MSK   0xc0000
0034 #define TX_DATA_DELAY_SHFT  18
0035 #define TX_CLK_DELAY_MSK    0x300000
0036 #define TX_CLK_DELAY_SHFT   20
0037 #define TX_CS_N_DELAY_MSK   0xc00000
0038 #define TX_CS_N_DELAY_SHFT  22
0039 #define TX_DATA_OE_DELAY_MSK    0x3000000
0040 #define TX_DATA_OE_DELAY_SHFT   24
0041
0042 #define AHB_MASTER_CFG              0x0004
0043 #define HMEM_TYPE_START_MID_TRANS_MSK       0x7
0044 #define HMEM_TYPE_START_MID_TRANS_SHFT      0
0045 #define HMEM_TYPE_LAST_TRANS_MSK        0x38
0046 #define HMEM_TYPE_LAST_TRANS_SHFT       3
0047 #define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_MSK  0xc0
0048 #define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_SHFT 6
0049 #define HMEMTYPE_READ_TRANS_MSK         0x700
0050 #define HMEMTYPE_READ_TRANS_SHFT        8
0051 #define HSHARED                 BIT(11)
0052 #define HINNERSHARED                BIT(12)
0053
0054 #define MSTR_INT_EN     0x000C
0055 #define MSTR_INT_STATUS     0x0010
0056 #define RESP_FIFO_UNDERRUN  BIT(0)
0057 #define RESP_FIFO_NOT_EMPTY BIT(1)
0058 #define RESP_FIFO_RDY       BIT(2)
0059 #define HRESP_FROM_NOC_ERR  BIT(3)
0060 #define WR_FIFO_EMPTY       BIT(9)
0061 #define WR_FIFO_FULL        BIT(10)
0062 #define WR_FIFO_OVERRUN     BIT(11)
0063 #define TRANSACTION_DONE    BIT(16)
0064 #define QSPI_ERR_IRQS       (RESP_FIFO_UNDERRUN | HRESP_FROM_NOC_ERR | \
0065                  WR_FIFO_OVERRUN)
0066 #define QSPI_ALL_IRQS       (QSPI_ERR_IRQS | RESP_FIFO_RDY | \
0067                  WR_FIFO_EMPTY | WR_FIFO_FULL | \
0068                  TRANSACTION_DONE)
0069
0070 #define PIO_XFER_CTRL       0x0014
0071 #define REQUEST_COUNT_MSK   0xffff
0072
0073 #define PIO_XFER_CFG        0x0018
0074 #define TRANSFER_DIRECTION  BIT(0)
0075 #define MULTI_IO_MODE_MSK   0xe
0076 #define MULTI_IO_MODE_SHFT  1
0077 #define TRANSFER_FRAGMENT   BIT(8)
0078 #define SDR_1BIT        1
0079 #define SDR_2BIT        2
0080 #define SDR_4BIT        3
0081 #define DDR_1BIT        5
0082 #define DDR_2BIT        6
0083 #define DDR_4BIT        7
0084 #define DMA_DESC_SINGLE_SPI 1
0085 #define DMA_DESC_DUAL_SPI   2
0086 #define DMA_DESC_QUAD_SPI   3
0087
0088 #define PIO_XFER_STATUS     0x001c
0089 #define WR_FIFO_BYTES_MSK   0xffff0000
0090 #define WR_FIFO_BYTES_SHFT  16
0091
0092 #define PIO_DATAOUT_1B      0x0020
0093 #define PIO_DATAOUT_4B      0x0024
0094
0095 #define RD_FIFO_CFG     0x0028
0096 #define CONTINUOUS_MODE     BIT(0)
0097
0098 #define RD_FIFO_STATUS  0x002c
0099 #define FIFO_EMPTY  BIT(11)
0100 #define WR_CNTS_MSK 0x7f0
0101 #define WR_CNTS_SHFT    4
0102 #define RDY_64BYTE  BIT(3)
0103 #define RDY_32BYTE  BIT(2)
0104 #define RDY_16BYTE  BIT(1)
0105 #define FIFO_RDY    BIT(0)
0106
0107 #define RD_FIFO_RESET       0x0030
0108 #define RESET_FIFO      BIT(0)
0109
0110 #define CUR_MEM_ADDR        0x0048
0111 #define HW_VERSION      0x004c
0112 #define RD_FIFO         0x0050
0113 #define SAMPLING_CLK_CFG    0x0090
0114 #define SAMPLING_CLK_STATUS 0x0094
0115
0116
0117 enum qspi_dir {
0118     QSPI_READ,
0119     QSPI_WRITE,
0120 };
0121
0122 struct qspi_xfer {
0123     union {
0124         const void *tx_buf;
0125         void *rx_buf;
0126     };
0127     unsigned int rem_bytes;
0128     unsigned int buswidth;
0129     enum qspi_dir dir;
0130     bool is_last;
0131 };
0132
0133 enum qspi_clocks {
0134     QSPI_CLK_CORE,
0135     QSPI_CLK_IFACE,
0136     QSPI_NUM_CLKS
0137 };
0138
0139 struct qcom_qspi {
0140     void __iomem *base;
0141     struct device *dev;
0142     struct clk_bulk_data *clks;
0143     struct qspi_xfer xfer;
0144     struct icc_path *icc_path_cpu_to_qspi;
0145     unsigned long last_speed;
0146     /* Lock to protect data accessed by IRQs */
0147     spinlock_t lock;
0148 };
0149
0150 static u32 qspi_buswidth_to_iomode(struct qcom_qspi *ctrl,
0151                    unsigned int buswidth)
0152 {
0153     switch (buswidth) {
0154     case 1:
0155         return SDR_1BIT << MULTI_IO_MODE_SHFT;
0156     case 2:
0157         return SDR_2BIT << MULTI_IO_MODE_SHFT;
0158     case 4:
0159         return SDR_4BIT << MULTI_IO_MODE_SHFT;
0160     default:
0161         dev_warn_once(ctrl->dev,
0162                 "Unexpected bus width: %u\n", buswidth);
0163         return SDR_1BIT << MULTI_IO_MODE_SHFT;
0164     }
0165 }
0166
0167 static void qcom_qspi_pio_xfer_cfg(struct qcom_qspi *ctrl)
0168 {
0169     u32 pio_xfer_cfg;
0170     const struct qspi_xfer *xfer;
0171
0172     xfer = &ctrl->xfer;
0173     pio_xfer_cfg = readl(ctrl->base + PIO_XFER_CFG);
0174     pio_xfer_cfg &= ~TRANSFER_DIRECTION;
0175     pio_xfer_cfg |= xfer->dir;
0176     if (xfer->is_last)
0177         pio_xfer_cfg &= ~TRANSFER_FRAGMENT;
0178     else
0179         pio_xfer_cfg |= TRANSFER_FRAGMENT;
0180     pio_xfer_cfg &= ~MULTI_IO_MODE_MSK;
0181     pio_xfer_cfg |= qspi_buswidth_to_iomode(ctrl, xfer->buswidth);
0182
0183     writel(pio_xfer_cfg, ctrl->base + PIO_XFER_CFG);
0184 }
0185
0186 static void qcom_qspi_pio_xfer_ctrl(struct qcom_qspi *ctrl)
0187 {
0188     u32 pio_xfer_ctrl;
0189
0190     pio_xfer_ctrl = readl(ctrl->base + PIO_XFER_CTRL);
0191     pio_xfer_ctrl &= ~REQUEST_COUNT_MSK;
0192     pio_xfer_ctrl |= ctrl->xfer.rem_bytes;
0193     writel(pio_xfer_ctrl, ctrl->base + PIO_XFER_CTRL);
0194 }
0195
0196 static void qcom_qspi_pio_xfer(struct qcom_qspi *ctrl)
0197 {
0198     u32 ints;
0199
0200     qcom_qspi_pio_xfer_cfg(ctrl);
0201
0202     /* Ack any previous interrupts that might be hanging around */
0203     writel(QSPI_ALL_IRQS, ctrl->base + MSTR_INT_STATUS);
0204
0205     /* Setup new interrupts */
0206     if (ctrl->xfer.dir == QSPI_WRITE)
0207         ints = QSPI_ERR_IRQS | WR_FIFO_EMPTY;
0208     else
0209         ints = QSPI_ERR_IRQS | RESP_FIFO_RDY;
0210     writel(ints, ctrl->base + MSTR_INT_EN);
0211
0212     /* Kick off the transfer */
0213     qcom_qspi_pio_xfer_ctrl(ctrl);
0214 }
0215
0216 static void qcom_qspi_handle_err(struct spi_master *master,
0217                  struct spi_message *msg)
0218 {
0219     struct qcom_qspi *ctrl = spi_master_get_devdata(master);
0220     unsigned long flags;
0221
0222     spin_lock_irqsave(&ctrl->lock, flags);
0223     writel(0, ctrl->base + MSTR_INT_EN);
0224     ctrl->xfer.rem_bytes = 0;
0225     spin_unlock_irqrestore(&ctrl->lock, flags);
0226 }
0227
0228 static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz)
0229 {
0230     int ret;
0231     unsigned int avg_bw_cpu;
0232
0233     if (speed_hz == ctrl->last_speed)
0234         return 0;
0235
0236     /* In regular operation (SBL_EN=1) core must be 4x transfer clock */
0237     ret = dev_pm_opp_set_rate(ctrl->dev, speed_hz * 4);
0238     if (ret) {
0239         dev_err(ctrl->dev, "Failed to set core clk %d\n", ret);
0240         return ret;
0241     }
0242
0243     /*
0244      * Set BW quota for CPU as driver supports FIFO mode only.
0245      * We don't have explicit peak requirement so keep it equal to avg_bw.
0246      */
0247     avg_bw_cpu = Bps_to_icc(speed_hz);
0248     ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, avg_bw_cpu, avg_bw_cpu);
0249     if (ret) {
0250         dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
0251             __func__, ret);
0252         return ret;
0253     }
0254
0255     ctrl->last_speed = speed_hz;
0256
0257     return 0;
0258 }
0259
0260 static int qcom_qspi_transfer_one(struct spi_master *master,
0261                   struct spi_device *slv,
0262                   struct spi_transfer *xfer)
0263 {
0264     struct qcom_qspi *ctrl = spi_master_get_devdata(master);
0265     int ret;
0266     unsigned long speed_hz;
0267     unsigned long flags;
0268
0269     speed_hz = slv->max_speed_hz;
0270     if (xfer->speed_hz)
0271         speed_hz = xfer->speed_hz;
0272
0273     ret = qcom_qspi_set_speed(ctrl, speed_hz);
0274     if (ret)
0275         return ret;
0276
0277     spin_lock_irqsave(&ctrl->lock, flags);
0278
0279     /* We are half duplex, so either rx or tx will be set */
0280     if (xfer->rx_buf) {
0281         ctrl->xfer.dir = QSPI_READ;
0282         ctrl->xfer.buswidth = xfer->rx_nbits;
0283         ctrl->xfer.rx_buf = xfer->rx_buf;
0284     } else {
0285         ctrl->xfer.dir = QSPI_WRITE;
0286         ctrl->xfer.buswidth = xfer->tx_nbits;
0287         ctrl->xfer.tx_buf = xfer->tx_buf;
0288     }
0289     ctrl->xfer.is_last = list_is_last(&xfer->transfer_list,
0290                       &master->cur_msg->transfers);
0291     ctrl->xfer.rem_bytes = xfer->len;
0292     qcom_qspi_pio_xfer(ctrl);
0293
0294     spin_unlock_irqrestore(&ctrl->lock, flags);
0295
0296     /* We'll call spi_finalize_current_transfer() when done */
0297     return 1;
0298 }
0299
0300 static int qcom_qspi_prepare_message(struct spi_master *master,
0301                      struct spi_message *message)
0302 {
0303     u32 mstr_cfg;
0304     struct qcom_qspi *ctrl;
0305     int tx_data_oe_delay = 1;
0306     int tx_data_delay = 1;
0307     unsigned long flags;
0308
0309     ctrl = spi_master_get_devdata(master);
0310     spin_lock_irqsave(&ctrl->lock, flags);
0311
0312     mstr_cfg = readl(ctrl->base + MSTR_CONFIG);
0313     mstr_cfg &= ~CHIP_SELECT_NUM;
0314     if (message->spi->chip_select)
0315         mstr_cfg |= CHIP_SELECT_NUM;
0316
0317     mstr_cfg |= FB_CLK_EN | PIN_WPN | PIN_HOLDN | SBL_EN | FULL_CYCLE_MODE;
0318     mstr_cfg &= ~(SPI_MODE_MSK | TX_DATA_OE_DELAY_MSK | TX_DATA_DELAY_MSK);
0319     mstr_cfg |= message->spi->mode << SPI_MODE_SHFT;
0320     mstr_cfg |= tx_data_oe_delay << TX_DATA_OE_DELAY_SHFT;
0321     mstr_cfg |= tx_data_delay << TX_DATA_DELAY_SHFT;
0322     mstr_cfg &= ~DMA_ENABLE;
0323
0324     writel(mstr_cfg, ctrl->base + MSTR_CONFIG);
0325     spin_unlock_irqrestore(&ctrl->lock, flags);
0326
0327     return 0;
0328 }
0329
0330 static irqreturn_t pio_read(struct qcom_qspi *ctrl)
0331 {
0332     u32 rd_fifo_status;
0333     u32 rd_fifo;
0334     unsigned int wr_cnts;
0335     unsigned int bytes_to_read;
0336     unsigned int words_to_read;
0337     u32 *word_buf;
0338     u8 *byte_buf;
0339     int i;
0340
0341     rd_fifo_status = readl(ctrl->base + RD_FIFO_STATUS);
0342
0343     if (!(rd_fifo_status & FIFO_RDY)) {
0344         dev_dbg(ctrl->dev, "Spurious IRQ %#x\n", rd_fifo_status);
0345         return IRQ_NONE;
0346     }
0347
0348     wr_cnts = (rd_fifo_status & WR_CNTS_MSK) >> WR_CNTS_SHFT;
0349     wr_cnts = min(wr_cnts, ctrl->xfer.rem_bytes);
0350
0351     words_to_read = wr_cnts / QSPI_BYTES_PER_WORD;
0352     bytes_to_read = wr_cnts % QSPI_BYTES_PER_WORD;
0353
0354     if (words_to_read) {
0355         word_buf = ctrl->xfer.rx_buf;
0356         ctrl->xfer.rem_bytes -= words_to_read * QSPI_BYTES_PER_WORD;
0357         ioread32_rep(ctrl->base + RD_FIFO, word_buf, words_to_read);
0358         ctrl->xfer.rx_buf = word_buf + words_to_read;
0359     }
0360
0361     if (bytes_to_read) {
0362         byte_buf = ctrl->xfer.rx_buf;
0363         rd_fifo = readl(ctrl->base + RD_FIFO);
0364         ctrl->xfer.rem_bytes -= bytes_to_read;
0365         for (i = 0; i < bytes_to_read; i++)
0366             *byte_buf++ = rd_fifo >> (i * BITS_PER_BYTE);
0367         ctrl->xfer.rx_buf = byte_buf;
0368     }
0369
0370     return IRQ_HANDLED;
0371 }
0372
0373 static irqreturn_t pio_write(struct qcom_qspi *ctrl)
0374 {
0375     const void *xfer_buf = ctrl->xfer.tx_buf;
0376     const int *word_buf;
0377     const char *byte_buf;
0378     unsigned int wr_fifo_bytes;
0379     unsigned int wr_fifo_words;
0380     unsigned int wr_size;
0381     unsigned int rem_words;
0382
0383     wr_fifo_bytes = readl(ctrl->base + PIO_XFER_STATUS);
0384     wr_fifo_bytes >>= WR_FIFO_BYTES_SHFT;
0385
0386     if (ctrl->xfer.rem_bytes < QSPI_BYTES_PER_WORD) {
0387         /* Process the last 1-3 bytes */
0388         wr_size = min(wr_fifo_bytes, ctrl->xfer.rem_bytes);
0389         ctrl->xfer.rem_bytes -= wr_size;
0390
0391         byte_buf = xfer_buf;
0392         while (wr_size--)
0393             writel(*byte_buf++,
0394                    ctrl->base + PIO_DATAOUT_1B);
0395         ctrl->xfer.tx_buf = byte_buf;
0396     } else {
0397         /*
0398          * Process all the whole words; to keep things simple we'll
0399          * just wait for the next interrupt to handle the last 1-3
0400          * bytes if we don't have an even number of words.
0401          */
0402         rem_words = ctrl->xfer.rem_bytes / QSPI_BYTES_PER_WORD;
0403         wr_fifo_words = wr_fifo_bytes / QSPI_BYTES_PER_WORD;
0404
0405         wr_size = min(rem_words, wr_fifo_words);
0406         ctrl->xfer.rem_bytes -= wr_size * QSPI_BYTES_PER_WORD;
0407
0408         word_buf = xfer_buf;
0409         iowrite32_rep(ctrl->base + PIO_DATAOUT_4B, word_buf, wr_size);
0410         ctrl->xfer.tx_buf = word_buf + wr_size;
0411
0412     }
0413
0414     return IRQ_HANDLED;
0415 }
0416
0417 static irqreturn_t qcom_qspi_irq(int irq, void *dev_id)
0418 {
0419     u32 int_status;
0420     struct qcom_qspi *ctrl = dev_id;
0421     irqreturn_t ret = IRQ_NONE;
0422
0423     spin_lock(&ctrl->lock);
0424
0425     int_status = readl(ctrl->base + MSTR_INT_STATUS);
0426     writel(int_status, ctrl->base + MSTR_INT_STATUS);
0427
0428     if (ctrl->xfer.dir == QSPI_WRITE) {
0429         if (int_status & WR_FIFO_EMPTY)
0430             ret = pio_write(ctrl);
0431     } else {
0432         if (int_status & RESP_FIFO_RDY)
0433             ret = pio_read(ctrl);
0434     }
0435
0436     if (int_status & QSPI_ERR_IRQS) {
0437         if (int_status & RESP_FIFO_UNDERRUN)
0438             dev_err(ctrl->dev, "IRQ error: FIFO underrun\n");
0439         if (int_status & WR_FIFO_OVERRUN)
0440             dev_err(ctrl->dev, "IRQ error: FIFO overrun\n");
0441         if (int_status & HRESP_FROM_NOC_ERR)
0442             dev_err(ctrl->dev, "IRQ error: NOC response error\n");
0443         ret = IRQ_HANDLED;
0444     }
0445
0446     if (!ctrl->xfer.rem_bytes) {
0447         writel(0, ctrl->base + MSTR_INT_EN);
0448         spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev));
0449     }
0450
0451     spin_unlock(&ctrl->lock);
0452     return ret;
0453 }
0454
0455 static int qcom_qspi_probe(struct platform_device *pdev)
0456 {
0457     int ret;
0458     struct device *dev;
0459     struct spi_master *master;
0460     struct qcom_qspi *ctrl;
0461
0462     dev = &pdev->dev;
0463
0464     master = devm_spi_alloc_master(dev, sizeof(*ctrl));
0465     if (!master)
0466         return -ENOMEM;
0467
0468     platform_set_drvdata(pdev, master);
0469
0470     ctrl = spi_master_get_devdata(master);
0471
0472     spin_lock_init(&ctrl->lock);
0473     ctrl->dev = dev;
0474     ctrl->base = devm_platform_ioremap_resource(pdev, 0);
0475     if (IS_ERR(ctrl->base))
0476         return PTR_ERR(ctrl->base);
0477
0478     ctrl->clks = devm_kcalloc(dev, QSPI_NUM_CLKS,
0479                   sizeof(*ctrl->clks), GFP_KERNEL);
0480     if (!ctrl->clks)
0481         return -ENOMEM;
0482
0483     ctrl->clks[QSPI_CLK_CORE].id = "core";
0484     ctrl->clks[QSPI_CLK_IFACE].id = "iface";
0485     ret = devm_clk_bulk_get(dev, QSPI_NUM_CLKS, ctrl->clks);
0486     if (ret)
0487         return ret;
0488
0489     ctrl->icc_path_cpu_to_qspi = devm_of_icc_get(dev, "qspi-config");
0490     if (IS_ERR(ctrl->icc_path_cpu_to_qspi))
0491         return dev_err_probe(dev, PTR_ERR(ctrl->icc_path_cpu_to_qspi),
0492                      "Failed to get cpu path\n");
0493
0494     /* Set BW vote for register access */
0495     ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, Bps_to_icc(1000),
0496                 Bps_to_icc(1000));
0497     if (ret) {
0498         dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
0499                 __func__, ret);
0500         return ret;
0501     }
0502
0503     ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
0504     if (ret) {
0505         dev_err(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
0506                 __func__, ret);
0507         return ret;
0508     }
0509
0510     ret = platform_get_irq(pdev, 0);
0511     if (ret < 0)
0512         return ret;
0513     ret = devm_request_irq(dev, ret, qcom_qspi_irq, 0, dev_name(dev), ctrl);
0514     if (ret) {
0515         dev_err(dev, "Failed to request irq %d\n", ret);
0516         return ret;
0517     }
0518
0519     master->max_speed_hz = 300000000;
0520     master->num_chipselect = QSPI_NUM_CS;
0521     master->bus_num = -1;
0522     master->dev.of_node = pdev->dev.of_node;
0523     master->mode_bits = SPI_MODE_0 |
0524                 SPI_TX_DUAL | SPI_RX_DUAL |
0525                 SPI_TX_QUAD | SPI_RX_QUAD;
0526     master->flags = SPI_MASTER_HALF_DUPLEX;
0527     master->prepare_message = qcom_qspi_prepare_message;
0528     master->transfer_one = qcom_qspi_transfer_one;
0529     master->handle_err = qcom_qspi_handle_err;
0530     master->auto_runtime_pm = true;
0531
0532     ret = devm_pm_opp_set_clkname(&pdev->dev, "core");
0533     if (ret)
0534         return ret;
0535     /* OPP table is optional */
0536     ret = devm_pm_opp_of_add_table(&pdev->dev);
0537     if (ret && ret != -ENODEV) {
0538         dev_err(&pdev->dev, "invalid OPP table in device tree\n");
0539         return ret;
0540     }
0541
0542     pm_runtime_use_autosuspend(dev);
0543     pm_runtime_set_autosuspend_delay(dev, 250);
0544     pm_runtime_enable(dev);
0545
0546     ret = spi_register_master(master);
0547     if (!ret)
0548         return 0;
0549
0550     pm_runtime_disable(dev);
0551
0552     return ret;
0553 }
0554
0555 static int qcom_qspi_remove(struct platform_device *pdev)
0556 {
0557     struct spi_master *master = platform_get_drvdata(pdev);
0558
0559     /* Unregister _before_ disabling pm_runtime() so we stop transfers */
0560     spi_unregister_master(master);
0561
0562     pm_runtime_disable(&pdev->dev);
0563
0564     return 0;
0565 }
0566
0567 static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev)
0568 {
0569     struct spi_master *master = dev_get_drvdata(dev);
0570     struct qcom_qspi *ctrl = spi_master_get_devdata(master);
0571     int ret;
0572
0573     /* Drop the performance state vote */
0574     dev_pm_opp_set_rate(dev, 0);
0575     clk_bulk_disable_unprepare(QSPI_NUM_CLKS, ctrl->clks);
0576
0577     ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
0578     if (ret) {
0579         dev_err_ratelimited(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
0580             __func__, ret);
0581         return ret;
0582     }
0583
0584     return 0;
0585 }
0586
0587 static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev)
0588 {
0589     struct spi_master *master = dev_get_drvdata(dev);
0590     struct qcom_qspi *ctrl = spi_master_get_devdata(master);
0591     int ret;
0592
0593     ret = icc_enable(ctrl->icc_path_cpu_to_qspi);
0594     if (ret) {
0595         dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n",
0596             __func__, ret);
0597         return ret;
0598     }
0599
0600     ret = clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks);
0601     if (ret)
0602         return ret;
0603
0604     return dev_pm_opp_set_rate(dev, ctrl->last_speed * 4);
0605 }
0606
0607 static int __maybe_unused qcom_qspi_suspend(struct device *dev)
0608 {
0609     struct spi_master *master = dev_get_drvdata(dev);
0610     int ret;
0611
0612     ret = spi_master_suspend(master);
0613     if (ret)
0614         return ret;
0615
0616     ret = pm_runtime_force_suspend(dev);
0617     if (ret)
0618         spi_master_resume(master);
0619
0620     return ret;
0621 }
0622
0623 static int __maybe_unused qcom_qspi_resume(struct device *dev)
0624 {
0625     struct spi_master *master = dev_get_drvdata(dev);
0626     int ret;
0627
0628     ret = pm_runtime_force_resume(dev);
0629     if (ret)
0630         return ret;
0631
0632     ret = spi_master_resume(master);
0633     if (ret)
0634         pm_runtime_force_suspend(dev);
0635
0636     return ret;
0637 }
0638
0639 static const struct dev_pm_ops qcom_qspi_dev_pm_ops = {
0640     SET_RUNTIME_PM_OPS(qcom_qspi_runtime_suspend,
0641                qcom_qspi_runtime_resume, NULL)
0642     SET_SYSTEM_SLEEP_PM_OPS(qcom_qspi_suspend, qcom_qspi_resume)
0643 };
0644
0645 static const struct of_device_id qcom_qspi_dt_match[] = {
0646     { .compatible = "qcom,qspi-v1", },
0647     { }
0648 };
0649 MODULE_DEVICE_TABLE(of, qcom_qspi_dt_match);
0650
0651 static struct platform_driver qcom_qspi_driver = {
0652     .driver = {
0653         .name       = "qcom_qspi",
0654         .pm     = &qcom_qspi_dev_pm_ops,
0655         .of_match_table = qcom_qspi_dt_match,
0656     },
0657     .probe = qcom_qspi_probe,
0658     .remove = qcom_qspi_remove,
0659 };
0660 module_platform_driver(qcom_qspi_driver);
0661
0662 MODULE_DESCRIPTION("SPI driver for QSPI cores");
0663 MODULE_LICENSE("GPL v2");