Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * MHI Endpoint bus stack
0004  *
0005  * Copyright (C) 2022 Linaro Ltd.
0006  * Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
0007  */
0008 
0009 #include <linux/bitfield.h>
0010 #include <linux/delay.h>
0011 #include <linux/dma-direction.h>
0012 #include <linux/interrupt.h>
0013 #include <linux/io.h>
0014 #include <linux/irq.h>
0015 #include <linux/mhi_ep.h>
0016 #include <linux/mod_devicetable.h>
0017 #include <linux/module.h>
0018 #include "internal.h"
0019 
0020 #define M0_WAIT_DELAY_MS    100
0021 #define M0_WAIT_COUNT       100
0022 
0023 static DEFINE_IDA(mhi_ep_cntrl_ida);
0024 
0025 static int mhi_ep_create_device(struct mhi_ep_cntrl *mhi_cntrl, u32 ch_id);
0026 static int mhi_ep_destroy_device(struct device *dev, void *data);
0027 
0028 static int mhi_ep_send_event(struct mhi_ep_cntrl *mhi_cntrl, u32 ring_idx,
0029                  struct mhi_ring_element *el, bool bei)
0030 {
0031     struct device *dev = &mhi_cntrl->mhi_dev->dev;
0032     union mhi_ep_ring_ctx *ctx;
0033     struct mhi_ep_ring *ring;
0034     int ret;
0035 
0036     mutex_lock(&mhi_cntrl->event_lock);
0037     ring = &mhi_cntrl->mhi_event[ring_idx].ring;
0038     ctx = (union mhi_ep_ring_ctx *)&mhi_cntrl->ev_ctx_cache[ring_idx];
0039     if (!ring->started) {
0040         ret = mhi_ep_ring_start(mhi_cntrl, ring, ctx);
0041         if (ret) {
0042             dev_err(dev, "Error starting event ring (%u)\n", ring_idx);
0043             goto err_unlock;
0044         }
0045     }
0046 
0047     /* Add element to the event ring */
0048     ret = mhi_ep_ring_add_element(ring, el);
0049     if (ret) {
0050         dev_err(dev, "Error adding element to event ring (%u)\n", ring_idx);
0051         goto err_unlock;
0052     }
0053 
0054     mutex_unlock(&mhi_cntrl->event_lock);
0055 
0056     /*
0057      * Raise IRQ to host only if the BEI flag is not set in TRE. Host might
0058      * set this flag for interrupt moderation as per MHI protocol.
0059      */
0060     if (!bei)
0061         mhi_cntrl->raise_irq(mhi_cntrl, ring->irq_vector);
0062 
0063     return 0;
0064 
0065 err_unlock:
0066     mutex_unlock(&mhi_cntrl->event_lock);
0067 
0068     return ret;
0069 }
0070 
0071 static int mhi_ep_send_completion_event(struct mhi_ep_cntrl *mhi_cntrl, struct mhi_ep_ring *ring,
0072                     struct mhi_ring_element *tre, u32 len, enum mhi_ev_ccs code)
0073 {
0074     struct mhi_ring_element event = {};
0075 
0076     event.ptr = cpu_to_le64(ring->rbase + ring->rd_offset * sizeof(*tre));
0077     event.dword[0] = MHI_TRE_EV_DWORD0(code, len);
0078     event.dword[1] = MHI_TRE_EV_DWORD1(ring->ch_id, MHI_PKT_TYPE_TX_EVENT);
0079 
0080     return mhi_ep_send_event(mhi_cntrl, ring->er_index, &event, MHI_TRE_DATA_GET_BEI(tre));
0081 }
0082 
0083 int mhi_ep_send_state_change_event(struct mhi_ep_cntrl *mhi_cntrl, enum mhi_state state)
0084 {
0085     struct mhi_ring_element event = {};
0086 
0087     event.dword[0] = MHI_SC_EV_DWORD0(state);
0088     event.dword[1] = MHI_SC_EV_DWORD1(MHI_PKT_TYPE_STATE_CHANGE_EVENT);
0089 
0090     return mhi_ep_send_event(mhi_cntrl, 0, &event, 0);
0091 }
0092 
0093 int mhi_ep_send_ee_event(struct mhi_ep_cntrl *mhi_cntrl, enum mhi_ee_type exec_env)
0094 {
0095     struct mhi_ring_element event = {};
0096 
0097     event.dword[0] = MHI_EE_EV_DWORD0(exec_env);
0098     event.dword[1] = MHI_SC_EV_DWORD1(MHI_PKT_TYPE_EE_EVENT);
0099 
0100     return mhi_ep_send_event(mhi_cntrl, 0, &event, 0);
0101 }
0102 
0103 static int mhi_ep_send_cmd_comp_event(struct mhi_ep_cntrl *mhi_cntrl, enum mhi_ev_ccs code)
0104 {
0105     struct mhi_ep_ring *ring = &mhi_cntrl->mhi_cmd->ring;
0106     struct mhi_ring_element event = {};
0107 
0108     event.ptr = cpu_to_le64(ring->rbase + ring->rd_offset * sizeof(struct mhi_ring_element));
0109     event.dword[0] = MHI_CC_EV_DWORD0(code);
0110     event.dword[1] = MHI_CC_EV_DWORD1(MHI_PKT_TYPE_CMD_COMPLETION_EVENT);
0111 
0112     return mhi_ep_send_event(mhi_cntrl, 0, &event, 0);
0113 }
0114 
0115 static int mhi_ep_process_cmd_ring(struct mhi_ep_ring *ring, struct mhi_ring_element *el)
0116 {
0117     struct mhi_ep_cntrl *mhi_cntrl = ring->mhi_cntrl;
0118     struct device *dev = &mhi_cntrl->mhi_dev->dev;
0119     struct mhi_result result = {};
0120     struct mhi_ep_chan *mhi_chan;
0121     struct mhi_ep_ring *ch_ring;
0122     u32 tmp, ch_id;
0123     int ret;
0124 
0125     ch_id = MHI_TRE_GET_CMD_CHID(el);
0126     mhi_chan = &mhi_cntrl->mhi_chan[ch_id];
0127     ch_ring = &mhi_cntrl->mhi_chan[ch_id].ring;
0128 
0129     switch (MHI_TRE_GET_CMD_TYPE(el)) {
0130     case MHI_PKT_TYPE_START_CHAN_CMD:
0131         dev_dbg(dev, "Received START command for channel (%u)\n", ch_id);
0132 
0133         mutex_lock(&mhi_chan->lock);
0134         /* Initialize and configure the corresponding channel ring */
0135         if (!ch_ring->started) {
0136             ret = mhi_ep_ring_start(mhi_cntrl, ch_ring,
0137                 (union mhi_ep_ring_ctx *)&mhi_cntrl->ch_ctx_cache[ch_id]);
0138             if (ret) {
0139                 dev_err(dev, "Failed to start ring for channel (%u)\n", ch_id);
0140                 ret = mhi_ep_send_cmd_comp_event(mhi_cntrl,
0141                             MHI_EV_CC_UNDEFINED_ERR);
0142                 if (ret)
0143                     dev_err(dev, "Error sending completion event: %d\n", ret);
0144 
0145                 goto err_unlock;
0146             }
0147         }
0148 
0149         /* Set channel state to RUNNING */
0150         mhi_chan->state = MHI_CH_STATE_RUNNING;
0151         tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[ch_id].chcfg);
0152         tmp &= ~CHAN_CTX_CHSTATE_MASK;
0153         tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_RUNNING);
0154         mhi_cntrl->ch_ctx_cache[ch_id].chcfg = cpu_to_le32(tmp);
0155 
0156         ret = mhi_ep_send_cmd_comp_event(mhi_cntrl, MHI_EV_CC_SUCCESS);
0157         if (ret) {
0158             dev_err(dev, "Error sending command completion event (%u)\n",
0159                 MHI_EV_CC_SUCCESS);
0160             goto err_unlock;
0161         }
0162 
0163         mutex_unlock(&mhi_chan->lock);
0164 
0165         /*
0166          * Create MHI device only during UL channel start. Since the MHI
0167          * channels operate in a pair, we'll associate both UL and DL
0168          * channels to the same device.
0169          *
0170          * We also need to check for mhi_dev != NULL because, the host
0171          * will issue START_CHAN command during resume and we don't
0172          * destroy the device during suspend.
0173          */
0174         if (!(ch_id % 2) && !mhi_chan->mhi_dev) {
0175             ret = mhi_ep_create_device(mhi_cntrl, ch_id);
0176             if (ret) {
0177                 dev_err(dev, "Error creating device for channel (%u)\n", ch_id);
0178                 mhi_ep_handle_syserr(mhi_cntrl);
0179                 return ret;
0180             }
0181         }
0182 
0183         /* Finally, enable DB for the channel */
0184         mhi_ep_mmio_enable_chdb(mhi_cntrl, ch_id);
0185 
0186         break;
0187     case MHI_PKT_TYPE_STOP_CHAN_CMD:
0188         dev_dbg(dev, "Received STOP command for channel (%u)\n", ch_id);
0189         if (!ch_ring->started) {
0190             dev_err(dev, "Channel (%u) not opened\n", ch_id);
0191             return -ENODEV;
0192         }
0193 
0194         mutex_lock(&mhi_chan->lock);
0195         /* Disable DB for the channel */
0196         mhi_ep_mmio_disable_chdb(mhi_cntrl, ch_id);
0197 
0198         /* Send channel disconnect status to client drivers */
0199         result.transaction_status = -ENOTCONN;
0200         result.bytes_xferd = 0;
0201         mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
0202 
0203         /* Set channel state to STOP */
0204         mhi_chan->state = MHI_CH_STATE_STOP;
0205         tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[ch_id].chcfg);
0206         tmp &= ~CHAN_CTX_CHSTATE_MASK;
0207         tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_STOP);
0208         mhi_cntrl->ch_ctx_cache[ch_id].chcfg = cpu_to_le32(tmp);
0209 
0210         ret = mhi_ep_send_cmd_comp_event(mhi_cntrl, MHI_EV_CC_SUCCESS);
0211         if (ret) {
0212             dev_err(dev, "Error sending command completion event (%u)\n",
0213                 MHI_EV_CC_SUCCESS);
0214             goto err_unlock;
0215         }
0216 
0217         mutex_unlock(&mhi_chan->lock);
0218         break;
0219     case MHI_PKT_TYPE_RESET_CHAN_CMD:
0220         dev_dbg(dev, "Received STOP command for channel (%u)\n", ch_id);
0221         if (!ch_ring->started) {
0222             dev_err(dev, "Channel (%u) not opened\n", ch_id);
0223             return -ENODEV;
0224         }
0225 
0226         mutex_lock(&mhi_chan->lock);
0227         /* Stop and reset the transfer ring */
0228         mhi_ep_ring_reset(mhi_cntrl, ch_ring);
0229 
0230         /* Send channel disconnect status to client driver */
0231         result.transaction_status = -ENOTCONN;
0232         result.bytes_xferd = 0;
0233         mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
0234 
0235         /* Set channel state to DISABLED */
0236         mhi_chan->state = MHI_CH_STATE_DISABLED;
0237         tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[ch_id].chcfg);
0238         tmp &= ~CHAN_CTX_CHSTATE_MASK;
0239         tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_DISABLED);
0240         mhi_cntrl->ch_ctx_cache[ch_id].chcfg = cpu_to_le32(tmp);
0241 
0242         ret = mhi_ep_send_cmd_comp_event(mhi_cntrl, MHI_EV_CC_SUCCESS);
0243         if (ret) {
0244             dev_err(dev, "Error sending command completion event (%u)\n",
0245                 MHI_EV_CC_SUCCESS);
0246             goto err_unlock;
0247         }
0248 
0249         mutex_unlock(&mhi_chan->lock);
0250         break;
0251     default:
0252         dev_err(dev, "Invalid command received: %lu for channel (%u)\n",
0253             MHI_TRE_GET_CMD_TYPE(el), ch_id);
0254         return -EINVAL;
0255     }
0256 
0257     return 0;
0258 
0259 err_unlock:
0260     mutex_unlock(&mhi_chan->lock);
0261 
0262     return ret;
0263 }
0264 
0265 bool mhi_ep_queue_is_empty(struct mhi_ep_device *mhi_dev, enum dma_data_direction dir)
0266 {
0267     struct mhi_ep_chan *mhi_chan = (dir == DMA_FROM_DEVICE) ? mhi_dev->dl_chan :
0268                                 mhi_dev->ul_chan;
0269     struct mhi_ep_cntrl *mhi_cntrl = mhi_dev->mhi_cntrl;
0270     struct mhi_ep_ring *ring = &mhi_cntrl->mhi_chan[mhi_chan->chan].ring;
0271 
0272     return !!(ring->rd_offset == ring->wr_offset);
0273 }
0274 EXPORT_SYMBOL_GPL(mhi_ep_queue_is_empty);
0275 
0276 static int mhi_ep_read_channel(struct mhi_ep_cntrl *mhi_cntrl,
0277                 struct mhi_ep_ring *ring,
0278                 struct mhi_result *result,
0279                 u32 len)
0280 {
0281     struct mhi_ep_chan *mhi_chan = &mhi_cntrl->mhi_chan[ring->ch_id];
0282     struct device *dev = &mhi_cntrl->mhi_dev->dev;
0283     size_t tr_len, read_offset, write_offset;
0284     struct mhi_ring_element *el;
0285     bool tr_done = false;
0286     void *write_addr;
0287     u64 read_addr;
0288     u32 buf_left;
0289     int ret;
0290 
0291     buf_left = len;
0292 
0293     do {
0294         /* Don't process the transfer ring if the channel is not in RUNNING state */
0295         if (mhi_chan->state != MHI_CH_STATE_RUNNING) {
0296             dev_err(dev, "Channel not available\n");
0297             return -ENODEV;
0298         }
0299 
0300         el = &ring->ring_cache[ring->rd_offset];
0301 
0302         /* Check if there is data pending to be read from previous read operation */
0303         if (mhi_chan->tre_bytes_left) {
0304             dev_dbg(dev, "TRE bytes remaining: %u\n", mhi_chan->tre_bytes_left);
0305             tr_len = min(buf_left, mhi_chan->tre_bytes_left);
0306         } else {
0307             mhi_chan->tre_loc = MHI_TRE_DATA_GET_PTR(el);
0308             mhi_chan->tre_size = MHI_TRE_DATA_GET_LEN(el);
0309             mhi_chan->tre_bytes_left = mhi_chan->tre_size;
0310 
0311             tr_len = min(buf_left, mhi_chan->tre_size);
0312         }
0313 
0314         read_offset = mhi_chan->tre_size - mhi_chan->tre_bytes_left;
0315         write_offset = len - buf_left;
0316         read_addr = mhi_chan->tre_loc + read_offset;
0317         write_addr = result->buf_addr + write_offset;
0318 
0319         dev_dbg(dev, "Reading %zd bytes from channel (%u)\n", tr_len, ring->ch_id);
0320         ret = mhi_cntrl->read_from_host(mhi_cntrl, read_addr, write_addr, tr_len);
0321         if (ret < 0) {
0322             dev_err(&mhi_chan->mhi_dev->dev, "Error reading from channel\n");
0323             return ret;
0324         }
0325 
0326         buf_left -= tr_len;
0327         mhi_chan->tre_bytes_left -= tr_len;
0328 
0329         /*
0330          * Once the TRE (Transfer Ring Element) of a TD (Transfer Descriptor) has been
0331          * read completely:
0332          *
0333          * 1. Send completion event to the host based on the flags set in TRE.
0334          * 2. Increment the local read offset of the transfer ring.
0335          */
0336         if (!mhi_chan->tre_bytes_left) {
0337             /*
0338              * The host will split the data packet into multiple TREs if it can't fit
0339              * the packet in a single TRE. In that case, CHAIN flag will be set by the
0340              * host for all TREs except the last one.
0341              */
0342             if (MHI_TRE_DATA_GET_CHAIN(el)) {
0343                 /*
0344                  * IEOB (Interrupt on End of Block) flag will be set by the host if
0345                  * it expects the completion event for all TREs of a TD.
0346                  */
0347                 if (MHI_TRE_DATA_GET_IEOB(el)) {
0348                     ret = mhi_ep_send_completion_event(mhi_cntrl, ring, el,
0349                                      MHI_TRE_DATA_GET_LEN(el),
0350                                      MHI_EV_CC_EOB);
0351                     if (ret < 0) {
0352                         dev_err(&mhi_chan->mhi_dev->dev,
0353                             "Error sending transfer compl. event\n");
0354                         return ret;
0355                     }
0356                 }
0357             } else {
0358                 /*
0359                  * IEOT (Interrupt on End of Transfer) flag will be set by the host
0360                  * for the last TRE of the TD and expects the completion event for
0361                  * the same.
0362                  */
0363                 if (MHI_TRE_DATA_GET_IEOT(el)) {
0364                     ret = mhi_ep_send_completion_event(mhi_cntrl, ring, el,
0365                                      MHI_TRE_DATA_GET_LEN(el),
0366                                      MHI_EV_CC_EOT);
0367                     if (ret < 0) {
0368                         dev_err(&mhi_chan->mhi_dev->dev,
0369                             "Error sending transfer compl. event\n");
0370                         return ret;
0371                     }
0372                 }
0373 
0374                 tr_done = true;
0375             }
0376 
0377             mhi_ep_ring_inc_index(ring);
0378         }
0379 
0380         result->bytes_xferd += tr_len;
0381     } while (buf_left && !tr_done);
0382 
0383     return 0;
0384 }
0385 
0386 static int mhi_ep_process_ch_ring(struct mhi_ep_ring *ring, struct mhi_ring_element *el)
0387 {
0388     struct mhi_ep_cntrl *mhi_cntrl = ring->mhi_cntrl;
0389     struct mhi_result result = {};
0390     u32 len = MHI_EP_DEFAULT_MTU;
0391     struct mhi_ep_chan *mhi_chan;
0392     int ret;
0393 
0394     mhi_chan = &mhi_cntrl->mhi_chan[ring->ch_id];
0395 
0396     /*
0397      * Bail out if transfer callback is not registered for the channel.
0398      * This is most likely due to the client driver not loaded at this point.
0399      */
0400     if (!mhi_chan->xfer_cb) {
0401         dev_err(&mhi_chan->mhi_dev->dev, "Client driver not available\n");
0402         return -ENODEV;
0403     }
0404 
0405     if (ring->ch_id % 2) {
0406         /* DL channel */
0407         result.dir = mhi_chan->dir;
0408         mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
0409     } else {
0410         /* UL channel */
0411         result.buf_addr = kzalloc(len, GFP_KERNEL);
0412         if (!result.buf_addr)
0413             return -ENOMEM;
0414 
0415         do {
0416             ret = mhi_ep_read_channel(mhi_cntrl, ring, &result, len);
0417             if (ret < 0) {
0418                 dev_err(&mhi_chan->mhi_dev->dev, "Failed to read channel\n");
0419                 kfree(result.buf_addr);
0420                 return ret;
0421             }
0422 
0423             result.dir = mhi_chan->dir;
0424             mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
0425             result.bytes_xferd = 0;
0426             memset(result.buf_addr, 0, len);
0427 
0428             /* Read until the ring becomes empty */
0429         } while (!mhi_ep_queue_is_empty(mhi_chan->mhi_dev, DMA_TO_DEVICE));
0430 
0431         kfree(result.buf_addr);
0432     }
0433 
0434     return 0;
0435 }
0436 
0437 /* TODO: Handle partially formed TDs */
0438 int mhi_ep_queue_skb(struct mhi_ep_device *mhi_dev, struct sk_buff *skb)
0439 {
0440     struct mhi_ep_cntrl *mhi_cntrl = mhi_dev->mhi_cntrl;
0441     struct mhi_ep_chan *mhi_chan = mhi_dev->dl_chan;
0442     struct device *dev = &mhi_chan->mhi_dev->dev;
0443     struct mhi_ring_element *el;
0444     u32 buf_left, read_offset;
0445     struct mhi_ep_ring *ring;
0446     enum mhi_ev_ccs code;
0447     void *read_addr;
0448     u64 write_addr;
0449     size_t tr_len;
0450     u32 tre_len;
0451     int ret;
0452 
0453     buf_left = skb->len;
0454     ring = &mhi_cntrl->mhi_chan[mhi_chan->chan].ring;
0455 
0456     mutex_lock(&mhi_chan->lock);
0457 
0458     do {
0459         /* Don't process the transfer ring if the channel is not in RUNNING state */
0460         if (mhi_chan->state != MHI_CH_STATE_RUNNING) {
0461             dev_err(dev, "Channel not available\n");
0462             ret = -ENODEV;
0463             goto err_exit;
0464         }
0465 
0466         if (mhi_ep_queue_is_empty(mhi_dev, DMA_FROM_DEVICE)) {
0467             dev_err(dev, "TRE not available!\n");
0468             ret = -ENOSPC;
0469             goto err_exit;
0470         }
0471 
0472         el = &ring->ring_cache[ring->rd_offset];
0473         tre_len = MHI_TRE_DATA_GET_LEN(el);
0474 
0475         tr_len = min(buf_left, tre_len);
0476         read_offset = skb->len - buf_left;
0477         read_addr = skb->data + read_offset;
0478         write_addr = MHI_TRE_DATA_GET_PTR(el);
0479 
0480         dev_dbg(dev, "Writing %zd bytes to channel (%u)\n", tr_len, ring->ch_id);
0481         ret = mhi_cntrl->write_to_host(mhi_cntrl, read_addr, write_addr, tr_len);
0482         if (ret < 0) {
0483             dev_err(dev, "Error writing to the channel\n");
0484             goto err_exit;
0485         }
0486 
0487         buf_left -= tr_len;
0488         /*
0489          * For all TREs queued by the host for DL channel, only the EOT flag will be set.
0490          * If the packet doesn't fit into a single TRE, send the OVERFLOW event to
0491          * the host so that the host can adjust the packet boundary to next TREs. Else send
0492          * the EOT event to the host indicating the packet boundary.
0493          */
0494         if (buf_left)
0495             code = MHI_EV_CC_OVERFLOW;
0496         else
0497             code = MHI_EV_CC_EOT;
0498 
0499         ret = mhi_ep_send_completion_event(mhi_cntrl, ring, el, tr_len, code);
0500         if (ret) {
0501             dev_err(dev, "Error sending transfer completion event\n");
0502             goto err_exit;
0503         }
0504 
0505         mhi_ep_ring_inc_index(ring);
0506     } while (buf_left);
0507 
0508     mutex_unlock(&mhi_chan->lock);
0509 
0510     return 0;
0511 
0512 err_exit:
0513     mutex_unlock(&mhi_chan->lock);
0514 
0515     return ret;
0516 }
0517 EXPORT_SYMBOL_GPL(mhi_ep_queue_skb);
0518 
0519 static int mhi_ep_cache_host_cfg(struct mhi_ep_cntrl *mhi_cntrl)
0520 {
0521     size_t cmd_ctx_host_size, ch_ctx_host_size, ev_ctx_host_size;
0522     struct device *dev = &mhi_cntrl->mhi_dev->dev;
0523     int ret;
0524 
0525     /* Update the number of event rings (NER) programmed by the host */
0526     mhi_ep_mmio_update_ner(mhi_cntrl);
0527 
0528     dev_dbg(dev, "Number of Event rings: %u, HW Event rings: %u\n",
0529          mhi_cntrl->event_rings, mhi_cntrl->hw_event_rings);
0530 
0531     ch_ctx_host_size = sizeof(struct mhi_chan_ctxt) * mhi_cntrl->max_chan;
0532     ev_ctx_host_size = sizeof(struct mhi_event_ctxt) * mhi_cntrl->event_rings;
0533     cmd_ctx_host_size = sizeof(struct mhi_cmd_ctxt) * NR_OF_CMD_RINGS;
0534 
0535     /* Get the channel context base pointer from host */
0536     mhi_ep_mmio_get_chc_base(mhi_cntrl);
0537 
0538     /* Allocate and map memory for caching host channel context */
0539     ret = mhi_cntrl->alloc_map(mhi_cntrl, mhi_cntrl->ch_ctx_host_pa,
0540                    &mhi_cntrl->ch_ctx_cache_phys,
0541                    (void __iomem **) &mhi_cntrl->ch_ctx_cache,
0542                    ch_ctx_host_size);
0543     if (ret) {
0544         dev_err(dev, "Failed to allocate and map ch_ctx_cache\n");
0545         return ret;
0546     }
0547 
0548     /* Get the event context base pointer from host */
0549     mhi_ep_mmio_get_erc_base(mhi_cntrl);
0550 
0551     /* Allocate and map memory for caching host event context */
0552     ret = mhi_cntrl->alloc_map(mhi_cntrl, mhi_cntrl->ev_ctx_host_pa,
0553                    &mhi_cntrl->ev_ctx_cache_phys,
0554                    (void __iomem **) &mhi_cntrl->ev_ctx_cache,
0555                    ev_ctx_host_size);
0556     if (ret) {
0557         dev_err(dev, "Failed to allocate and map ev_ctx_cache\n");
0558         goto err_ch_ctx;
0559     }
0560 
0561     /* Get the command context base pointer from host */
0562     mhi_ep_mmio_get_crc_base(mhi_cntrl);
0563 
0564     /* Allocate and map memory for caching host command context */
0565     ret = mhi_cntrl->alloc_map(mhi_cntrl, mhi_cntrl->cmd_ctx_host_pa,
0566                    &mhi_cntrl->cmd_ctx_cache_phys,
0567                    (void __iomem **) &mhi_cntrl->cmd_ctx_cache,
0568                    cmd_ctx_host_size);
0569     if (ret) {
0570         dev_err(dev, "Failed to allocate and map cmd_ctx_cache\n");
0571         goto err_ev_ctx;
0572     }
0573 
0574     /* Initialize command ring */
0575     ret = mhi_ep_ring_start(mhi_cntrl, &mhi_cntrl->mhi_cmd->ring,
0576                 (union mhi_ep_ring_ctx *)mhi_cntrl->cmd_ctx_cache);
0577     if (ret) {
0578         dev_err(dev, "Failed to start the command ring\n");
0579         goto err_cmd_ctx;
0580     }
0581 
0582     return ret;
0583 
0584 err_cmd_ctx:
0585     mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->cmd_ctx_host_pa, mhi_cntrl->cmd_ctx_cache_phys,
0586                   (void __iomem *) mhi_cntrl->cmd_ctx_cache, cmd_ctx_host_size);
0587 
0588 err_ev_ctx:
0589     mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->ev_ctx_host_pa, mhi_cntrl->ev_ctx_cache_phys,
0590                   (void __iomem *) mhi_cntrl->ev_ctx_cache, ev_ctx_host_size);
0591 
0592 err_ch_ctx:
0593     mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->ch_ctx_host_pa, mhi_cntrl->ch_ctx_cache_phys,
0594                   (void __iomem *) mhi_cntrl->ch_ctx_cache, ch_ctx_host_size);
0595 
0596     return ret;
0597 }
0598 
0599 static void mhi_ep_free_host_cfg(struct mhi_ep_cntrl *mhi_cntrl)
0600 {
0601     size_t cmd_ctx_host_size, ch_ctx_host_size, ev_ctx_host_size;
0602 
0603     ch_ctx_host_size = sizeof(struct mhi_chan_ctxt) * mhi_cntrl->max_chan;
0604     ev_ctx_host_size = sizeof(struct mhi_event_ctxt) * mhi_cntrl->event_rings;
0605     cmd_ctx_host_size = sizeof(struct mhi_cmd_ctxt) * NR_OF_CMD_RINGS;
0606 
0607     mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->cmd_ctx_host_pa, mhi_cntrl->cmd_ctx_cache_phys,
0608                   (void __iomem *) mhi_cntrl->cmd_ctx_cache, cmd_ctx_host_size);
0609 
0610     mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->ev_ctx_host_pa, mhi_cntrl->ev_ctx_cache_phys,
0611                   (void __iomem *) mhi_cntrl->ev_ctx_cache, ev_ctx_host_size);
0612 
0613     mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->ch_ctx_host_pa, mhi_cntrl->ch_ctx_cache_phys,
0614                   (void __iomem *) mhi_cntrl->ch_ctx_cache, ch_ctx_host_size);
0615 }
0616 
0617 static void mhi_ep_enable_int(struct mhi_ep_cntrl *mhi_cntrl)
0618 {
0619     /*
0620      * Doorbell interrupts are enabled when the corresponding channel gets started.
0621      * Enabling all interrupts here triggers spurious irqs as some of the interrupts
0622      * associated with hw channels always get triggered.
0623      */
0624     mhi_ep_mmio_enable_ctrl_interrupt(mhi_cntrl);
0625     mhi_ep_mmio_enable_cmdb_interrupt(mhi_cntrl);
0626 }
0627 
0628 static int mhi_ep_enable(struct mhi_ep_cntrl *mhi_cntrl)
0629 {
0630     struct device *dev = &mhi_cntrl->mhi_dev->dev;
0631     enum mhi_state state;
0632     bool mhi_reset;
0633     u32 count = 0;
0634     int ret;
0635 
0636     /* Wait for Host to set the M0 state */
0637     do {
0638         msleep(M0_WAIT_DELAY_MS);
0639         mhi_ep_mmio_get_mhi_state(mhi_cntrl, &state, &mhi_reset);
0640         if (mhi_reset) {
0641             /* Clear the MHI reset if host is in reset state */
0642             mhi_ep_mmio_clear_reset(mhi_cntrl);
0643             dev_info(dev, "Detected Host reset while waiting for M0\n");
0644         }
0645         count++;
0646     } while (state != MHI_STATE_M0 && count < M0_WAIT_COUNT);
0647 
0648     if (state != MHI_STATE_M0) {
0649         dev_err(dev, "Host failed to enter M0\n");
0650         return -ETIMEDOUT;
0651     }
0652 
0653     ret = mhi_ep_cache_host_cfg(mhi_cntrl);
0654     if (ret) {
0655         dev_err(dev, "Failed to cache host config\n");
0656         return ret;
0657     }
0658 
0659     mhi_ep_mmio_set_env(mhi_cntrl, MHI_EE_AMSS);
0660 
0661     /* Enable all interrupts now */
0662     mhi_ep_enable_int(mhi_cntrl);
0663 
0664     return 0;
0665 }
0666 
0667 static void mhi_ep_cmd_ring_worker(struct work_struct *work)
0668 {
0669     struct mhi_ep_cntrl *mhi_cntrl = container_of(work, struct mhi_ep_cntrl, cmd_ring_work);
0670     struct mhi_ep_ring *ring = &mhi_cntrl->mhi_cmd->ring;
0671     struct device *dev = &mhi_cntrl->mhi_dev->dev;
0672     struct mhi_ring_element *el;
0673     int ret;
0674 
0675     /* Update the write offset for the ring */
0676     ret = mhi_ep_update_wr_offset(ring);
0677     if (ret) {
0678         dev_err(dev, "Error updating write offset for ring\n");
0679         return;
0680     }
0681 
0682     /* Sanity check to make sure there are elements in the ring */
0683     if (ring->rd_offset == ring->wr_offset)
0684         return;
0685 
0686     /*
0687      * Process command ring element till write offset. In case of an error, just try to
0688      * process next element.
0689      */
0690     while (ring->rd_offset != ring->wr_offset) {
0691         el = &ring->ring_cache[ring->rd_offset];
0692 
0693         ret = mhi_ep_process_cmd_ring(ring, el);
0694         if (ret)
0695             dev_err(dev, "Error processing cmd ring element: %zu\n", ring->rd_offset);
0696 
0697         mhi_ep_ring_inc_index(ring);
0698     }
0699 }
0700 
0701 static void mhi_ep_ch_ring_worker(struct work_struct *work)
0702 {
0703     struct mhi_ep_cntrl *mhi_cntrl = container_of(work, struct mhi_ep_cntrl, ch_ring_work);
0704     struct device *dev = &mhi_cntrl->mhi_dev->dev;
0705     struct mhi_ep_ring_item *itr, *tmp;
0706     struct mhi_ring_element *el;
0707     struct mhi_ep_ring *ring;
0708     struct mhi_ep_chan *chan;
0709     unsigned long flags;
0710     LIST_HEAD(head);
0711     int ret;
0712 
0713     spin_lock_irqsave(&mhi_cntrl->list_lock, flags);
0714     list_splice_tail_init(&mhi_cntrl->ch_db_list, &head);
0715     spin_unlock_irqrestore(&mhi_cntrl->list_lock, flags);
0716 
0717     /* Process each queued channel ring. In case of an error, just process next element. */
0718     list_for_each_entry_safe(itr, tmp, &head, node) {
0719         list_del(&itr->node);
0720         ring = itr->ring;
0721 
0722         /* Update the write offset for the ring */
0723         ret = mhi_ep_update_wr_offset(ring);
0724         if (ret) {
0725             dev_err(dev, "Error updating write offset for ring\n");
0726             kfree(itr);
0727             continue;
0728         }
0729 
0730         /* Sanity check to make sure there are elements in the ring */
0731         if (ring->rd_offset == ring->wr_offset) {
0732             kfree(itr);
0733             continue;
0734         }
0735 
0736         el = &ring->ring_cache[ring->rd_offset];
0737         chan = &mhi_cntrl->mhi_chan[ring->ch_id];
0738 
0739         mutex_lock(&chan->lock);
0740         dev_dbg(dev, "Processing the ring for channel (%u)\n", ring->ch_id);
0741         ret = mhi_ep_process_ch_ring(ring, el);
0742         if (ret) {
0743             dev_err(dev, "Error processing ring for channel (%u): %d\n",
0744                 ring->ch_id, ret);
0745             mutex_unlock(&chan->lock);
0746             kfree(itr);
0747             continue;
0748         }
0749 
0750         mutex_unlock(&chan->lock);
0751         kfree(itr);
0752     }
0753 }
0754 
0755 static void mhi_ep_state_worker(struct work_struct *work)
0756 {
0757     struct mhi_ep_cntrl *mhi_cntrl = container_of(work, struct mhi_ep_cntrl, state_work);
0758     struct device *dev = &mhi_cntrl->mhi_dev->dev;
0759     struct mhi_ep_state_transition *itr, *tmp;
0760     unsigned long flags;
0761     LIST_HEAD(head);
0762     int ret;
0763 
0764     spin_lock_irqsave(&mhi_cntrl->list_lock, flags);
0765     list_splice_tail_init(&mhi_cntrl->st_transition_list, &head);
0766     spin_unlock_irqrestore(&mhi_cntrl->list_lock, flags);
0767 
0768     list_for_each_entry_safe(itr, tmp, &head, node) {
0769         list_del(&itr->node);
0770         dev_dbg(dev, "Handling MHI state transition to %s\n",
0771              mhi_state_str(itr->state));
0772 
0773         switch (itr->state) {
0774         case MHI_STATE_M0:
0775             ret = mhi_ep_set_m0_state(mhi_cntrl);
0776             if (ret)
0777                 dev_err(dev, "Failed to transition to M0 state\n");
0778             break;
0779         case MHI_STATE_M3:
0780             ret = mhi_ep_set_m3_state(mhi_cntrl);
0781             if (ret)
0782                 dev_err(dev, "Failed to transition to M3 state\n");
0783             break;
0784         default:
0785             dev_err(dev, "Invalid MHI state transition: %d\n", itr->state);
0786             break;
0787         }
0788         kfree(itr);
0789     }
0790 }
0791 
0792 static void mhi_ep_queue_channel_db(struct mhi_ep_cntrl *mhi_cntrl, unsigned long ch_int,
0793                     u32 ch_idx)
0794 {
0795     struct mhi_ep_ring_item *item;
0796     struct mhi_ep_ring *ring;
0797     bool work = !!ch_int;
0798     LIST_HEAD(head);
0799     u32 i;
0800 
0801     /* First add the ring items to a local list */
0802     for_each_set_bit(i, &ch_int, 32) {
0803         /* Channel index varies for each register: 0, 32, 64, 96 */
0804         u32 ch_id = ch_idx + i;
0805 
0806         ring = &mhi_cntrl->mhi_chan[ch_id].ring;
0807         item = kzalloc(sizeof(*item), GFP_ATOMIC);
0808         if (!item)
0809             return;
0810 
0811         item->ring = ring;
0812         list_add_tail(&item->node, &head);
0813     }
0814 
0815     /* Now, splice the local list into ch_db_list and queue the work item */
0816     if (work) {
0817         spin_lock(&mhi_cntrl->list_lock);
0818         list_splice_tail_init(&head, &mhi_cntrl->ch_db_list);
0819         spin_unlock(&mhi_cntrl->list_lock);
0820 
0821         queue_work(mhi_cntrl->wq, &mhi_cntrl->ch_ring_work);
0822     }
0823 }
0824 
0825 /*
0826  * Channel interrupt statuses are contained in 4 registers each of 32bit length.
0827  * For checking all interrupts, we need to loop through each registers and then
0828  * check for bits set.
0829  */
0830 static void mhi_ep_check_channel_interrupt(struct mhi_ep_cntrl *mhi_cntrl)
0831 {
0832     u32 ch_int, ch_idx, i;
0833 
0834     /* Bail out if there is no channel doorbell interrupt */
0835     if (!mhi_ep_mmio_read_chdb_status_interrupts(mhi_cntrl))
0836         return;
0837 
0838     for (i = 0; i < MHI_MASK_ROWS_CH_DB; i++) {
0839         ch_idx = i * MHI_MASK_CH_LEN;
0840 
0841         /* Only process channel interrupt if the mask is enabled */
0842         ch_int = mhi_cntrl->chdb[i].status & mhi_cntrl->chdb[i].mask;
0843         if (ch_int) {
0844             mhi_ep_queue_channel_db(mhi_cntrl, ch_int, ch_idx);
0845             mhi_ep_mmio_write(mhi_cntrl, MHI_CHDB_INT_CLEAR_n(i),
0846                             mhi_cntrl->chdb[i].status);
0847         }
0848     }
0849 }
0850 
0851 static void mhi_ep_process_ctrl_interrupt(struct mhi_ep_cntrl *mhi_cntrl,
0852                      enum mhi_state state)
0853 {
0854     struct mhi_ep_state_transition *item;
0855 
0856     item = kzalloc(sizeof(*item), GFP_ATOMIC);
0857     if (!item)
0858         return;
0859 
0860     item->state = state;
0861     spin_lock(&mhi_cntrl->list_lock);
0862     list_add_tail(&item->node, &mhi_cntrl->st_transition_list);
0863     spin_unlock(&mhi_cntrl->list_lock);
0864 
0865     queue_work(mhi_cntrl->wq, &mhi_cntrl->state_work);
0866 }
0867 
0868 /*
0869  * Interrupt handler that services interrupts raised by the host writing to
0870  * MHICTRL and Command ring doorbell (CRDB) registers for state change and
0871  * channel interrupts.
0872  */
0873 static irqreturn_t mhi_ep_irq(int irq, void *data)
0874 {
0875     struct mhi_ep_cntrl *mhi_cntrl = data;
0876     struct device *dev = &mhi_cntrl->mhi_dev->dev;
0877     enum mhi_state state;
0878     u32 int_value;
0879     bool mhi_reset;
0880 
0881     /* Acknowledge the ctrl interrupt */
0882     int_value = mhi_ep_mmio_read(mhi_cntrl, MHI_CTRL_INT_STATUS);
0883     mhi_ep_mmio_write(mhi_cntrl, MHI_CTRL_INT_CLEAR, int_value);
0884 
0885     /* Check for ctrl interrupt */
0886     if (FIELD_GET(MHI_CTRL_INT_STATUS_MSK, int_value)) {
0887         dev_dbg(dev, "Processing ctrl interrupt\n");
0888         mhi_ep_mmio_get_mhi_state(mhi_cntrl, &state, &mhi_reset);
0889         if (mhi_reset) {
0890             dev_info(dev, "Host triggered MHI reset!\n");
0891             disable_irq_nosync(mhi_cntrl->irq);
0892             schedule_work(&mhi_cntrl->reset_work);
0893             return IRQ_HANDLED;
0894         }
0895 
0896         mhi_ep_process_ctrl_interrupt(mhi_cntrl, state);
0897     }
0898 
0899     /* Check for command doorbell interrupt */
0900     if (FIELD_GET(MHI_CTRL_INT_STATUS_CRDB_MSK, int_value)) {
0901         dev_dbg(dev, "Processing command doorbell interrupt\n");
0902         queue_work(mhi_cntrl->wq, &mhi_cntrl->cmd_ring_work);
0903     }
0904 
0905     /* Check for channel interrupts */
0906     mhi_ep_check_channel_interrupt(mhi_cntrl);
0907 
0908     return IRQ_HANDLED;
0909 }
0910 
0911 static void mhi_ep_abort_transfer(struct mhi_ep_cntrl *mhi_cntrl)
0912 {
0913     struct mhi_ep_ring *ch_ring, *ev_ring;
0914     struct mhi_result result = {};
0915     struct mhi_ep_chan *mhi_chan;
0916     int i;
0917 
0918     /* Stop all the channels */
0919     for (i = 0; i < mhi_cntrl->max_chan; i++) {
0920         mhi_chan = &mhi_cntrl->mhi_chan[i];
0921         if (!mhi_chan->ring.started)
0922             continue;
0923 
0924         mutex_lock(&mhi_chan->lock);
0925         /* Send channel disconnect status to client drivers */
0926         if (mhi_chan->xfer_cb) {
0927             result.transaction_status = -ENOTCONN;
0928             result.bytes_xferd = 0;
0929             mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
0930         }
0931 
0932         mhi_chan->state = MHI_CH_STATE_DISABLED;
0933         mutex_unlock(&mhi_chan->lock);
0934     }
0935 
0936     flush_workqueue(mhi_cntrl->wq);
0937 
0938     /* Destroy devices associated with all channels */
0939     device_for_each_child(&mhi_cntrl->mhi_dev->dev, NULL, mhi_ep_destroy_device);
0940 
0941     /* Stop and reset the transfer rings */
0942     for (i = 0; i < mhi_cntrl->max_chan; i++) {
0943         mhi_chan = &mhi_cntrl->mhi_chan[i];
0944         if (!mhi_chan->ring.started)
0945             continue;
0946 
0947         ch_ring = &mhi_cntrl->mhi_chan[i].ring;
0948         mutex_lock(&mhi_chan->lock);
0949         mhi_ep_ring_reset(mhi_cntrl, ch_ring);
0950         mutex_unlock(&mhi_chan->lock);
0951     }
0952 
0953     /* Stop and reset the event rings */
0954     for (i = 0; i < mhi_cntrl->event_rings; i++) {
0955         ev_ring = &mhi_cntrl->mhi_event[i].ring;
0956         if (!ev_ring->started)
0957             continue;
0958 
0959         mutex_lock(&mhi_cntrl->event_lock);
0960         mhi_ep_ring_reset(mhi_cntrl, ev_ring);
0961         mutex_unlock(&mhi_cntrl->event_lock);
0962     }
0963 
0964     /* Stop and reset the command ring */
0965     mhi_ep_ring_reset(mhi_cntrl, &mhi_cntrl->mhi_cmd->ring);
0966 
0967     mhi_ep_free_host_cfg(mhi_cntrl);
0968     mhi_ep_mmio_mask_interrupts(mhi_cntrl);
0969 
0970     mhi_cntrl->enabled = false;
0971 }
0972 
0973 static void mhi_ep_reset_worker(struct work_struct *work)
0974 {
0975     struct mhi_ep_cntrl *mhi_cntrl = container_of(work, struct mhi_ep_cntrl, reset_work);
0976     struct device *dev = &mhi_cntrl->mhi_dev->dev;
0977     enum mhi_state cur_state;
0978     int ret;
0979 
0980     mhi_ep_abort_transfer(mhi_cntrl);
0981 
0982     spin_lock_bh(&mhi_cntrl->state_lock);
0983     /* Reset MMIO to signal host that the MHI_RESET is completed in endpoint */
0984     mhi_ep_mmio_reset(mhi_cntrl);
0985     cur_state = mhi_cntrl->mhi_state;
0986     spin_unlock_bh(&mhi_cntrl->state_lock);
0987 
0988     /*
0989      * Only proceed further if the reset is due to SYS_ERR. The host will
0990      * issue reset during shutdown also and we don't need to do re-init in
0991      * that case.
0992      */
0993     if (cur_state == MHI_STATE_SYS_ERR) {
0994         mhi_ep_mmio_init(mhi_cntrl);
0995 
0996         /* Set AMSS EE before signaling ready state */
0997         mhi_ep_mmio_set_env(mhi_cntrl, MHI_EE_AMSS);
0998 
0999         /* All set, notify the host that we are ready */
1000         ret = mhi_ep_set_ready_state(mhi_cntrl);
1001         if (ret)
1002             return;
1003 
1004         dev_dbg(dev, "READY state notification sent to the host\n");
1005 
1006         ret = mhi_ep_enable(mhi_cntrl);
1007         if (ret) {
1008             dev_err(dev, "Failed to enable MHI endpoint: %d\n", ret);
1009             return;
1010         }
1011 
1012         enable_irq(mhi_cntrl->irq);
1013     }
1014 }
1015 
1016 /*
1017  * We don't need to do anything special other than setting the MHI SYS_ERR
1018  * state. The host will reset all contexts and issue MHI RESET so that we
1019  * could also recover from error state.
1020  */
1021 void mhi_ep_handle_syserr(struct mhi_ep_cntrl *mhi_cntrl)
1022 {
1023     struct device *dev = &mhi_cntrl->mhi_dev->dev;
1024     int ret;
1025 
1026     ret = mhi_ep_set_mhi_state(mhi_cntrl, MHI_STATE_SYS_ERR);
1027     if (ret)
1028         return;
1029 
1030     /* Signal host that the device went to SYS_ERR state */
1031     ret = mhi_ep_send_state_change_event(mhi_cntrl, MHI_STATE_SYS_ERR);
1032     if (ret)
1033         dev_err(dev, "Failed sending SYS_ERR state change event: %d\n", ret);
1034 }
1035 
1036 int mhi_ep_power_up(struct mhi_ep_cntrl *mhi_cntrl)
1037 {
1038     struct device *dev = &mhi_cntrl->mhi_dev->dev;
1039     int ret, i;
1040 
1041     /*
1042      * Mask all interrupts until the state machine is ready. Interrupts will
1043      * be enabled later with mhi_ep_enable().
1044      */
1045     mhi_ep_mmio_mask_interrupts(mhi_cntrl);
1046     mhi_ep_mmio_init(mhi_cntrl);
1047 
1048     mhi_cntrl->mhi_event = kzalloc(mhi_cntrl->event_rings * (sizeof(*mhi_cntrl->mhi_event)),
1049                     GFP_KERNEL);
1050     if (!mhi_cntrl->mhi_event)
1051         return -ENOMEM;
1052 
1053     /* Initialize command, channel and event rings */
1054     mhi_ep_ring_init(&mhi_cntrl->mhi_cmd->ring, RING_TYPE_CMD, 0);
1055     for (i = 0; i < mhi_cntrl->max_chan; i++)
1056         mhi_ep_ring_init(&mhi_cntrl->mhi_chan[i].ring, RING_TYPE_CH, i);
1057     for (i = 0; i < mhi_cntrl->event_rings; i++)
1058         mhi_ep_ring_init(&mhi_cntrl->mhi_event[i].ring, RING_TYPE_ER, i);
1059 
1060     mhi_cntrl->mhi_state = MHI_STATE_RESET;
1061 
1062     /* Set AMSS EE before signaling ready state */
1063     mhi_ep_mmio_set_env(mhi_cntrl, MHI_EE_AMSS);
1064 
1065     /* All set, notify the host that we are ready */
1066     ret = mhi_ep_set_ready_state(mhi_cntrl);
1067     if (ret)
1068         goto err_free_event;
1069 
1070     dev_dbg(dev, "READY state notification sent to the host\n");
1071 
1072     ret = mhi_ep_enable(mhi_cntrl);
1073     if (ret) {
1074         dev_err(dev, "Failed to enable MHI endpoint\n");
1075         goto err_free_event;
1076     }
1077 
1078     enable_irq(mhi_cntrl->irq);
1079     mhi_cntrl->enabled = true;
1080 
1081     return 0;
1082 
1083 err_free_event:
1084     kfree(mhi_cntrl->mhi_event);
1085 
1086     return ret;
1087 }
1088 EXPORT_SYMBOL_GPL(mhi_ep_power_up);
1089 
1090 void mhi_ep_power_down(struct mhi_ep_cntrl *mhi_cntrl)
1091 {
1092     if (mhi_cntrl->enabled)
1093         mhi_ep_abort_transfer(mhi_cntrl);
1094 
1095     kfree(mhi_cntrl->mhi_event);
1096     disable_irq(mhi_cntrl->irq);
1097 }
1098 EXPORT_SYMBOL_GPL(mhi_ep_power_down);
1099 
1100 void mhi_ep_suspend_channels(struct mhi_ep_cntrl *mhi_cntrl)
1101 {
1102     struct mhi_ep_chan *mhi_chan;
1103     u32 tmp;
1104     int i;
1105 
1106     for (i = 0; i < mhi_cntrl->max_chan; i++) {
1107         mhi_chan = &mhi_cntrl->mhi_chan[i];
1108 
1109         if (!mhi_chan->mhi_dev)
1110             continue;
1111 
1112         mutex_lock(&mhi_chan->lock);
1113         /* Skip if the channel is not currently running */
1114         tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[i].chcfg);
1115         if (FIELD_GET(CHAN_CTX_CHSTATE_MASK, tmp) != MHI_CH_STATE_RUNNING) {
1116             mutex_unlock(&mhi_chan->lock);
1117             continue;
1118         }
1119 
1120         dev_dbg(&mhi_chan->mhi_dev->dev, "Suspending channel\n");
1121         /* Set channel state to SUSPENDED */
1122         tmp &= ~CHAN_CTX_CHSTATE_MASK;
1123         tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_SUSPENDED);
1124         mhi_cntrl->ch_ctx_cache[i].chcfg = cpu_to_le32(tmp);
1125         mutex_unlock(&mhi_chan->lock);
1126     }
1127 }
1128 
1129 void mhi_ep_resume_channels(struct mhi_ep_cntrl *mhi_cntrl)
1130 {
1131     struct mhi_ep_chan *mhi_chan;
1132     u32 tmp;
1133     int i;
1134 
1135     for (i = 0; i < mhi_cntrl->max_chan; i++) {
1136         mhi_chan = &mhi_cntrl->mhi_chan[i];
1137 
1138         if (!mhi_chan->mhi_dev)
1139             continue;
1140 
1141         mutex_lock(&mhi_chan->lock);
1142         /* Skip if the channel is not currently suspended */
1143         tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[i].chcfg);
1144         if (FIELD_GET(CHAN_CTX_CHSTATE_MASK, tmp) != MHI_CH_STATE_SUSPENDED) {
1145             mutex_unlock(&mhi_chan->lock);
1146             continue;
1147         }
1148 
1149         dev_dbg(&mhi_chan->mhi_dev->dev, "Resuming channel\n");
1150         /* Set channel state to RUNNING */
1151         tmp &= ~CHAN_CTX_CHSTATE_MASK;
1152         tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_RUNNING);
1153         mhi_cntrl->ch_ctx_cache[i].chcfg = cpu_to_le32(tmp);
1154         mutex_unlock(&mhi_chan->lock);
1155     }
1156 }
1157 
1158 static void mhi_ep_release_device(struct device *dev)
1159 {
1160     struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1161 
1162     if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1163         mhi_dev->mhi_cntrl->mhi_dev = NULL;
1164 
1165     /*
1166      * We need to set the mhi_chan->mhi_dev to NULL here since the MHI
1167      * devices for the channels will only get created in mhi_ep_create_device()
1168      * if the mhi_dev associated with it is NULL.
1169      */
1170     if (mhi_dev->ul_chan)
1171         mhi_dev->ul_chan->mhi_dev = NULL;
1172 
1173     if (mhi_dev->dl_chan)
1174         mhi_dev->dl_chan->mhi_dev = NULL;
1175 
1176     kfree(mhi_dev);
1177 }
1178 
1179 static struct mhi_ep_device *mhi_ep_alloc_device(struct mhi_ep_cntrl *mhi_cntrl,
1180                          enum mhi_device_type dev_type)
1181 {
1182     struct mhi_ep_device *mhi_dev;
1183     struct device *dev;
1184 
1185     mhi_dev = kzalloc(sizeof(*mhi_dev), GFP_KERNEL);
1186     if (!mhi_dev)
1187         return ERR_PTR(-ENOMEM);
1188 
1189     dev = &mhi_dev->dev;
1190     device_initialize(dev);
1191     dev->bus = &mhi_ep_bus_type;
1192     dev->release = mhi_ep_release_device;
1193 
1194     /* Controller device is always allocated first */
1195     if (dev_type == MHI_DEVICE_CONTROLLER)
1196         /* for MHI controller device, parent is the bus device (e.g. PCI EPF) */
1197         dev->parent = mhi_cntrl->cntrl_dev;
1198     else
1199         /* for MHI client devices, parent is the MHI controller device */
1200         dev->parent = &mhi_cntrl->mhi_dev->dev;
1201 
1202     mhi_dev->mhi_cntrl = mhi_cntrl;
1203     mhi_dev->dev_type = dev_type;
1204 
1205     return mhi_dev;
1206 }
1207 
1208 /*
1209  * MHI channels are always defined in pairs with UL as the even numbered
1210  * channel and DL as odd numbered one. This function gets UL channel (primary)
1211  * as the ch_id and always looks after the next entry in channel list for
1212  * the corresponding DL channel (secondary).
1213  */
1214 static int mhi_ep_create_device(struct mhi_ep_cntrl *mhi_cntrl, u32 ch_id)
1215 {
1216     struct mhi_ep_chan *mhi_chan = &mhi_cntrl->mhi_chan[ch_id];
1217     struct device *dev = mhi_cntrl->cntrl_dev;
1218     struct mhi_ep_device *mhi_dev;
1219     int ret;
1220 
1221     /* Check if the channel name is same for both UL and DL */
1222     if (strcmp(mhi_chan->name, mhi_chan[1].name)) {
1223         dev_err(dev, "UL and DL channel names are not same: (%s) != (%s)\n",
1224             mhi_chan->name, mhi_chan[1].name);
1225         return -EINVAL;
1226     }
1227 
1228     mhi_dev = mhi_ep_alloc_device(mhi_cntrl, MHI_DEVICE_XFER);
1229     if (IS_ERR(mhi_dev))
1230         return PTR_ERR(mhi_dev);
1231 
1232     /* Configure primary channel */
1233     mhi_dev->ul_chan = mhi_chan;
1234     get_device(&mhi_dev->dev);
1235     mhi_chan->mhi_dev = mhi_dev;
1236 
1237     /* Configure secondary channel as well */
1238     mhi_chan++;
1239     mhi_dev->dl_chan = mhi_chan;
1240     get_device(&mhi_dev->dev);
1241     mhi_chan->mhi_dev = mhi_dev;
1242 
1243     /* Channel name is same for both UL and DL */
1244     mhi_dev->name = mhi_chan->name;
1245     ret = dev_set_name(&mhi_dev->dev, "%s_%s",
1246              dev_name(&mhi_cntrl->mhi_dev->dev),
1247              mhi_dev->name);
1248     if (ret) {
1249         put_device(&mhi_dev->dev);
1250         return ret;
1251     }
1252 
1253     ret = device_add(&mhi_dev->dev);
1254     if (ret)
1255         put_device(&mhi_dev->dev);
1256 
1257     return ret;
1258 }
1259 
1260 static int mhi_ep_destroy_device(struct device *dev, void *data)
1261 {
1262     struct mhi_ep_device *mhi_dev;
1263     struct mhi_ep_cntrl *mhi_cntrl;
1264     struct mhi_ep_chan *ul_chan, *dl_chan;
1265 
1266     if (dev->bus != &mhi_ep_bus_type)
1267         return 0;
1268 
1269     mhi_dev = to_mhi_ep_device(dev);
1270     mhi_cntrl = mhi_dev->mhi_cntrl;
1271 
1272     /* Only destroy devices created for channels */
1273     if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1274         return 0;
1275 
1276     ul_chan = mhi_dev->ul_chan;
1277     dl_chan = mhi_dev->dl_chan;
1278 
1279     if (ul_chan)
1280         put_device(&ul_chan->mhi_dev->dev);
1281 
1282     if (dl_chan)
1283         put_device(&dl_chan->mhi_dev->dev);
1284 
1285     dev_dbg(&mhi_cntrl->mhi_dev->dev, "Destroying device for chan:%s\n",
1286          mhi_dev->name);
1287 
1288     /* Notify the client and remove the device from MHI bus */
1289     device_del(dev);
1290     put_device(dev);
1291 
1292     return 0;
1293 }
1294 
1295 static int mhi_ep_chan_init(struct mhi_ep_cntrl *mhi_cntrl,
1296                 const struct mhi_ep_cntrl_config *config)
1297 {
1298     const struct mhi_ep_channel_config *ch_cfg;
1299     struct device *dev = mhi_cntrl->cntrl_dev;
1300     u32 chan, i;
1301     int ret = -EINVAL;
1302 
1303     mhi_cntrl->max_chan = config->max_channels;
1304 
1305     /*
1306      * Allocate max_channels supported by the MHI endpoint and populate
1307      * only the defined channels
1308      */
1309     mhi_cntrl->mhi_chan = kcalloc(mhi_cntrl->max_chan, sizeof(*mhi_cntrl->mhi_chan),
1310                       GFP_KERNEL);
1311     if (!mhi_cntrl->mhi_chan)
1312         return -ENOMEM;
1313 
1314     for (i = 0; i < config->num_channels; i++) {
1315         struct mhi_ep_chan *mhi_chan;
1316 
1317         ch_cfg = &config->ch_cfg[i];
1318 
1319         chan = ch_cfg->num;
1320         if (chan >= mhi_cntrl->max_chan) {
1321             dev_err(dev, "Channel (%u) exceeds maximum available channels (%u)\n",
1322                 chan, mhi_cntrl->max_chan);
1323             goto error_chan_cfg;
1324         }
1325 
1326         /* Bi-directional and direction less channels are not supported */
1327         if (ch_cfg->dir == DMA_BIDIRECTIONAL || ch_cfg->dir == DMA_NONE) {
1328             dev_err(dev, "Invalid direction (%u) for channel (%u)\n",
1329                 ch_cfg->dir, chan);
1330             goto error_chan_cfg;
1331         }
1332 
1333         mhi_chan = &mhi_cntrl->mhi_chan[chan];
1334         mhi_chan->name = ch_cfg->name;
1335         mhi_chan->chan = chan;
1336         mhi_chan->dir = ch_cfg->dir;
1337         mutex_init(&mhi_chan->lock);
1338     }
1339 
1340     return 0;
1341 
1342 error_chan_cfg:
1343     kfree(mhi_cntrl->mhi_chan);
1344 
1345     return ret;
1346 }
1347 
1348 /*
1349  * Allocate channel and command rings here. Event rings will be allocated
1350  * in mhi_ep_power_up() as the config comes from the host.
1351  */
1352 int mhi_ep_register_controller(struct mhi_ep_cntrl *mhi_cntrl,
1353                 const struct mhi_ep_cntrl_config *config)
1354 {
1355     struct mhi_ep_device *mhi_dev;
1356     int ret;
1357 
1358     if (!mhi_cntrl || !mhi_cntrl->cntrl_dev || !mhi_cntrl->mmio || !mhi_cntrl->irq)
1359         return -EINVAL;
1360 
1361     ret = mhi_ep_chan_init(mhi_cntrl, config);
1362     if (ret)
1363         return ret;
1364 
1365     mhi_cntrl->mhi_cmd = kcalloc(NR_OF_CMD_RINGS, sizeof(*mhi_cntrl->mhi_cmd), GFP_KERNEL);
1366     if (!mhi_cntrl->mhi_cmd) {
1367         ret = -ENOMEM;
1368         goto err_free_ch;
1369     }
1370 
1371     INIT_WORK(&mhi_cntrl->state_work, mhi_ep_state_worker);
1372     INIT_WORK(&mhi_cntrl->reset_work, mhi_ep_reset_worker);
1373     INIT_WORK(&mhi_cntrl->cmd_ring_work, mhi_ep_cmd_ring_worker);
1374     INIT_WORK(&mhi_cntrl->ch_ring_work, mhi_ep_ch_ring_worker);
1375 
1376     mhi_cntrl->wq = alloc_workqueue("mhi_ep_wq", 0, 0);
1377     if (!mhi_cntrl->wq) {
1378         ret = -ENOMEM;
1379         goto err_free_cmd;
1380     }
1381 
1382     INIT_LIST_HEAD(&mhi_cntrl->st_transition_list);
1383     INIT_LIST_HEAD(&mhi_cntrl->ch_db_list);
1384     spin_lock_init(&mhi_cntrl->state_lock);
1385     spin_lock_init(&mhi_cntrl->list_lock);
1386     mutex_init(&mhi_cntrl->event_lock);
1387 
1388     /* Set MHI version and AMSS EE before enumeration */
1389     mhi_ep_mmio_write(mhi_cntrl, EP_MHIVER, config->mhi_version);
1390     mhi_ep_mmio_set_env(mhi_cntrl, MHI_EE_AMSS);
1391 
1392     /* Set controller index */
1393     ret = ida_alloc(&mhi_ep_cntrl_ida, GFP_KERNEL);
1394     if (ret < 0)
1395         goto err_destroy_wq;
1396 
1397     mhi_cntrl->index = ret;
1398 
1399     irq_set_status_flags(mhi_cntrl->irq, IRQ_NOAUTOEN);
1400     ret = request_irq(mhi_cntrl->irq, mhi_ep_irq, IRQF_TRIGGER_HIGH,
1401               "doorbell_irq", mhi_cntrl);
1402     if (ret) {
1403         dev_err(mhi_cntrl->cntrl_dev, "Failed to request Doorbell IRQ\n");
1404         goto err_ida_free;
1405     }
1406 
1407     /* Allocate the controller device */
1408     mhi_dev = mhi_ep_alloc_device(mhi_cntrl, MHI_DEVICE_CONTROLLER);
1409     if (IS_ERR(mhi_dev)) {
1410         dev_err(mhi_cntrl->cntrl_dev, "Failed to allocate controller device\n");
1411         ret = PTR_ERR(mhi_dev);
1412         goto err_free_irq;
1413     }
1414 
1415     ret = dev_set_name(&mhi_dev->dev, "mhi_ep%u", mhi_cntrl->index);
1416     if (ret)
1417         goto err_put_dev;
1418 
1419     mhi_dev->name = dev_name(&mhi_dev->dev);
1420     mhi_cntrl->mhi_dev = mhi_dev;
1421 
1422     ret = device_add(&mhi_dev->dev);
1423     if (ret)
1424         goto err_put_dev;
1425 
1426     dev_dbg(&mhi_dev->dev, "MHI EP Controller registered\n");
1427 
1428     return 0;
1429 
1430 err_put_dev:
1431     put_device(&mhi_dev->dev);
1432 err_free_irq:
1433     free_irq(mhi_cntrl->irq, mhi_cntrl);
1434 err_ida_free:
1435     ida_free(&mhi_ep_cntrl_ida, mhi_cntrl->index);
1436 err_destroy_wq:
1437     destroy_workqueue(mhi_cntrl->wq);
1438 err_free_cmd:
1439     kfree(mhi_cntrl->mhi_cmd);
1440 err_free_ch:
1441     kfree(mhi_cntrl->mhi_chan);
1442 
1443     return ret;
1444 }
1445 EXPORT_SYMBOL_GPL(mhi_ep_register_controller);
1446 
1447 /*
1448  * It is expected that the controller drivers will power down the MHI EP stack
1449  * using "mhi_ep_power_down()" before calling this function to unregister themselves.
1450  */
1451 void mhi_ep_unregister_controller(struct mhi_ep_cntrl *mhi_cntrl)
1452 {
1453     struct mhi_ep_device *mhi_dev = mhi_cntrl->mhi_dev;
1454 
1455     destroy_workqueue(mhi_cntrl->wq);
1456 
1457     free_irq(mhi_cntrl->irq, mhi_cntrl);
1458 
1459     kfree(mhi_cntrl->mhi_cmd);
1460     kfree(mhi_cntrl->mhi_chan);
1461 
1462     device_del(&mhi_dev->dev);
1463     put_device(&mhi_dev->dev);
1464 
1465     ida_free(&mhi_ep_cntrl_ida, mhi_cntrl->index);
1466 }
1467 EXPORT_SYMBOL_GPL(mhi_ep_unregister_controller);
1468 
1469 static int mhi_ep_driver_probe(struct device *dev)
1470 {
1471     struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1472     struct mhi_ep_driver *mhi_drv = to_mhi_ep_driver(dev->driver);
1473     struct mhi_ep_chan *ul_chan = mhi_dev->ul_chan;
1474     struct mhi_ep_chan *dl_chan = mhi_dev->dl_chan;
1475 
1476     ul_chan->xfer_cb = mhi_drv->ul_xfer_cb;
1477     dl_chan->xfer_cb = mhi_drv->dl_xfer_cb;
1478 
1479     return mhi_drv->probe(mhi_dev, mhi_dev->id);
1480 }
1481 
1482 static int mhi_ep_driver_remove(struct device *dev)
1483 {
1484     struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1485     struct mhi_ep_driver *mhi_drv = to_mhi_ep_driver(dev->driver);
1486     struct mhi_result result = {};
1487     struct mhi_ep_chan *mhi_chan;
1488     int dir;
1489 
1490     /* Skip if it is a controller device */
1491     if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1492         return 0;
1493 
1494     /* Disconnect the channels associated with the driver */
1495     for (dir = 0; dir < 2; dir++) {
1496         mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan;
1497 
1498         if (!mhi_chan)
1499             continue;
1500 
1501         mutex_lock(&mhi_chan->lock);
1502         /* Send channel disconnect status to the client driver */
1503         if (mhi_chan->xfer_cb) {
1504             result.transaction_status = -ENOTCONN;
1505             result.bytes_xferd = 0;
1506             mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
1507         }
1508 
1509         mhi_chan->state = MHI_CH_STATE_DISABLED;
1510         mhi_chan->xfer_cb = NULL;
1511         mutex_unlock(&mhi_chan->lock);
1512     }
1513 
1514     /* Remove the client driver now */
1515     mhi_drv->remove(mhi_dev);
1516 
1517     return 0;
1518 }
1519 
1520 int __mhi_ep_driver_register(struct mhi_ep_driver *mhi_drv, struct module *owner)
1521 {
1522     struct device_driver *driver = &mhi_drv->driver;
1523 
1524     if (!mhi_drv->probe || !mhi_drv->remove)
1525         return -EINVAL;
1526 
1527     /* Client drivers should have callbacks defined for both channels */
1528     if (!mhi_drv->ul_xfer_cb || !mhi_drv->dl_xfer_cb)
1529         return -EINVAL;
1530 
1531     driver->bus = &mhi_ep_bus_type;
1532     driver->owner = owner;
1533     driver->probe = mhi_ep_driver_probe;
1534     driver->remove = mhi_ep_driver_remove;
1535 
1536     return driver_register(driver);
1537 }
1538 EXPORT_SYMBOL_GPL(__mhi_ep_driver_register);
1539 
1540 void mhi_ep_driver_unregister(struct mhi_ep_driver *mhi_drv)
1541 {
1542     driver_unregister(&mhi_drv->driver);
1543 }
1544 EXPORT_SYMBOL_GPL(mhi_ep_driver_unregister);
1545 
1546 static int mhi_ep_uevent(struct device *dev, struct kobj_uevent_env *env)
1547 {
1548     struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1549 
1550     return add_uevent_var(env, "MODALIAS=" MHI_EP_DEVICE_MODALIAS_FMT,
1551                     mhi_dev->name);
1552 }
1553 
1554 static int mhi_ep_match(struct device *dev, struct device_driver *drv)
1555 {
1556     struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1557     struct mhi_ep_driver *mhi_drv = to_mhi_ep_driver(drv);
1558     const struct mhi_device_id *id;
1559 
1560     /*
1561      * If the device is a controller type then there is no client driver
1562      * associated with it
1563      */
1564     if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1565         return 0;
1566 
1567     for (id = mhi_drv->id_table; id->chan[0]; id++)
1568         if (!strcmp(mhi_dev->name, id->chan)) {
1569             mhi_dev->id = id;
1570             return 1;
1571         }
1572 
1573     return 0;
1574 };
1575 
1576 struct bus_type mhi_ep_bus_type = {
1577     .name = "mhi_ep",
1578     .dev_name = "mhi_ep",
1579     .match = mhi_ep_match,
1580     .uevent = mhi_ep_uevent,
1581 };
1582 
1583 static int __init mhi_ep_init(void)
1584 {
1585     return bus_register(&mhi_ep_bus_type);
1586 }
1587 
1588 static void __exit mhi_ep_exit(void)
1589 {
1590     bus_unregister(&mhi_ep_bus_type);
1591 }
1592 
1593 postcore_initcall(mhi_ep_init);
1594 module_exit(mhi_ep_exit);
1595 
1596 MODULE_LICENSE("GPL v2");
1597 MODULE_DESCRIPTION("MHI Bus Endpoint stack");
1598 MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!