wwan/iosm/iosm_ipc_protocol_ops.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2020-21 Intel Corporation.
0004  */
0005
0006 #include "iosm_ipc_protocol.h"
0007 #include "iosm_ipc_protocol_ops.h"
0008
0009 /* Get the next free message element.*/
0010 static union ipc_mem_msg_entry *
0011 ipc_protocol_free_msg_get(struct iosm_protocol *ipc_protocol, int *index)
0012 {
0013     u32 head = le32_to_cpu(ipc_protocol->p_ap_shm->msg_head);
0014     u32 new_head = (head + 1) % IPC_MEM_MSG_ENTRIES;
0015     union ipc_mem_msg_entry *msg;
0016
0017     if (new_head == le32_to_cpu(ipc_protocol->p_ap_shm->msg_tail)) {
0018         dev_err(ipc_protocol->dev, "message ring is full");
0019         return NULL;
0020     }
0021
0022     /* Get the pointer to the next free message element,
0023      * reset the fields and mark is as invalid.
0024      */
0025     msg = &ipc_protocol->p_ap_shm->msg_ring[head];
0026     memset(msg, 0, sizeof(*msg));
0027
0028     /* return index in message ring */
0029     *index = head;
0030
0031     return msg;
0032 }
0033
0034 /* Updates the message ring Head pointer */
0035 void ipc_protocol_msg_hp_update(struct iosm_imem *ipc_imem)
0036 {
0037     struct iosm_protocol *ipc_protocol = ipc_imem->ipc_protocol;
0038     u32 head = le32_to_cpu(ipc_protocol->p_ap_shm->msg_head);
0039     u32 new_head = (head + 1) % IPC_MEM_MSG_ENTRIES;
0040
0041     /* Update head pointer and fire doorbell. */
0042     ipc_protocol->p_ap_shm->msg_head = cpu_to_le32(new_head);
0043     ipc_protocol->old_msg_tail =
0044         le32_to_cpu(ipc_protocol->p_ap_shm->msg_tail);
0045
0046     ipc_pm_signal_hpda_doorbell(&ipc_protocol->pm, IPC_HP_MR, false);
0047 }
0048
0049 /* Allocate and prepare a OPEN_PIPE message.
0050  * This also allocates the memory for the new TDR structure and
0051  * updates the pipe structure referenced in the preparation arguments.
0052  */
0053 static int ipc_protocol_msg_prepipe_open(struct iosm_protocol *ipc_protocol,
0054                      union ipc_msg_prep_args *args)
0055 {
0056     int index;
0057     union ipc_mem_msg_entry *msg =
0058         ipc_protocol_free_msg_get(ipc_protocol, &index);
0059     struct ipc_pipe *pipe = args->pipe_open.pipe;
0060     struct ipc_protocol_td *tdr;
0061     struct sk_buff **skbr;
0062
0063     if (!msg) {
0064         dev_err(ipc_protocol->dev, "failed to get free message");
0065         return -EIO;
0066     }
0067
0068     /* Allocate the skbuf elements for the skbuf which are on the way.
0069      * SKB ring is internal memory allocation for driver. No need to
0070      * re-calculate the start and end addresses.
0071      */
0072     skbr = kcalloc(pipe->nr_of_entries, sizeof(*skbr), GFP_ATOMIC);
0073     if (!skbr)
0074         return -ENOMEM;
0075
0076     /* Allocate the transfer descriptors for the pipe. */
0077     tdr = dma_alloc_coherent(&ipc_protocol->pcie->pci->dev,
0078                  pipe->nr_of_entries * sizeof(*tdr),
0079                  &pipe->phy_tdr_start, GFP_ATOMIC);
0080     if (!tdr) {
0081         kfree(skbr);
0082         dev_err(ipc_protocol->dev, "tdr alloc error");
0083         return -ENOMEM;
0084     }
0085
0086     pipe->max_nr_of_queued_entries = pipe->nr_of_entries - 1;
0087     pipe->nr_of_queued_entries = 0;
0088     pipe->tdr_start = tdr;
0089     pipe->skbr_start = skbr;
0090     pipe->old_tail = 0;
0091
0092     ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr] = 0;
0093
0094     msg->open_pipe.type_of_message = IPC_MEM_MSG_OPEN_PIPE;
0095     msg->open_pipe.pipe_nr = pipe->pipe_nr;
0096     msg->open_pipe.tdr_addr = cpu_to_le64(pipe->phy_tdr_start);
0097     msg->open_pipe.tdr_entries = cpu_to_le16(pipe->nr_of_entries);
0098     msg->open_pipe.accumulation_backoff =
0099                 cpu_to_le32(pipe->accumulation_backoff);
0100     msg->open_pipe.irq_vector = cpu_to_le32(pipe->irq);
0101
0102     return index;
0103 }
0104
0105 static int ipc_protocol_msg_prepipe_close(struct iosm_protocol *ipc_protocol,
0106                       union ipc_msg_prep_args *args)
0107 {
0108     int index = -1;
0109     union ipc_mem_msg_entry *msg =
0110         ipc_protocol_free_msg_get(ipc_protocol, &index);
0111     struct ipc_pipe *pipe = args->pipe_close.pipe;
0112
0113     if (!msg)
0114         return -EIO;
0115
0116     msg->close_pipe.type_of_message = IPC_MEM_MSG_CLOSE_PIPE;
0117     msg->close_pipe.pipe_nr = pipe->pipe_nr;
0118
0119     dev_dbg(ipc_protocol->dev, "IPC_MEM_MSG_CLOSE_PIPE(pipe_nr=%d)",
0120         msg->close_pipe.pipe_nr);
0121
0122     return index;
0123 }
0124
0125 static int ipc_protocol_msg_prep_sleep(struct iosm_protocol *ipc_protocol,
0126                        union ipc_msg_prep_args *args)
0127 {
0128     int index = -1;
0129     union ipc_mem_msg_entry *msg =
0130         ipc_protocol_free_msg_get(ipc_protocol, &index);
0131
0132     if (!msg) {
0133         dev_err(ipc_protocol->dev, "failed to get free message");
0134         return -EIO;
0135     }
0136
0137     /* Prepare and send the host sleep message to CP to enter or exit D3. */
0138     msg->host_sleep.type_of_message = IPC_MEM_MSG_SLEEP;
0139     msg->host_sleep.target = args->sleep.target; /* 0=host, 1=device */
0140
0141     /* state; 0=enter, 1=exit 2=enter w/o protocol */
0142     msg->host_sleep.state = args->sleep.state;
0143
0144     dev_dbg(ipc_protocol->dev, "IPC_MEM_MSG_SLEEP(target=%d; state=%d)",
0145         msg->host_sleep.target, msg->host_sleep.state);
0146
0147     return index;
0148 }
0149
0150 static int ipc_protocol_msg_prep_feature_set(struct iosm_protocol *ipc_protocol,
0151                          union ipc_msg_prep_args *args)
0152 {
0153     int index = -1;
0154     union ipc_mem_msg_entry *msg =
0155         ipc_protocol_free_msg_get(ipc_protocol, &index);
0156
0157     if (!msg) {
0158         dev_err(ipc_protocol->dev, "failed to get free message");
0159         return -EIO;
0160     }
0161
0162     msg->feature_set.type_of_message = IPC_MEM_MSG_FEATURE_SET;
0163     msg->feature_set.reset_enable = args->feature_set.reset_enable <<
0164                     RESET_BIT;
0165
0166     dev_dbg(ipc_protocol->dev, "IPC_MEM_MSG_FEATURE_SET(reset_enable=%d)",
0167         msg->feature_set.reset_enable >> RESET_BIT);
0168
0169     return index;
0170 }
0171
0172 /* Processes the message consumed by CP. */
0173 bool ipc_protocol_msg_process(struct iosm_imem *ipc_imem, int irq)
0174 {
0175     struct iosm_protocol *ipc_protocol = ipc_imem->ipc_protocol;
0176     struct ipc_rsp **rsp_ring = ipc_protocol->rsp_ring;
0177     bool msg_processed = false;
0178     u32 i;
0179
0180     if (le32_to_cpu(ipc_protocol->p_ap_shm->msg_tail) >=
0181             IPC_MEM_MSG_ENTRIES) {
0182         dev_err(ipc_protocol->dev, "msg_tail out of range: %d",
0183             le32_to_cpu(ipc_protocol->p_ap_shm->msg_tail));
0184         return msg_processed;
0185     }
0186
0187     if (irq != IMEM_IRQ_DONT_CARE &&
0188         irq != ipc_protocol->p_ap_shm->ci.msg_irq_vector)
0189         return msg_processed;
0190
0191     for (i = ipc_protocol->old_msg_tail;
0192          i != le32_to_cpu(ipc_protocol->p_ap_shm->msg_tail);
0193          i = (i + 1) % IPC_MEM_MSG_ENTRIES) {
0194         union ipc_mem_msg_entry *msg =
0195             &ipc_protocol->p_ap_shm->msg_ring[i];
0196
0197         dev_dbg(ipc_protocol->dev, "msg[%d]: type=%u status=%d", i,
0198             msg->common.type_of_message,
0199             msg->common.completion_status);
0200
0201         /* Update response with status and wake up waiting requestor */
0202         if (rsp_ring[i]) {
0203             rsp_ring[i]->status =
0204                 le32_to_cpu(msg->common.completion_status);
0205             complete(&rsp_ring[i]->completion);
0206             rsp_ring[i] = NULL;
0207         }
0208         msg_processed = true;
0209     }
0210
0211     ipc_protocol->old_msg_tail = i;
0212     return msg_processed;
0213 }
0214
0215 /* Sends data from UL list to CP for the provided pipe by updating the Head
0216  * pointer of given pipe.
0217  */
0218 bool ipc_protocol_ul_td_send(struct iosm_protocol *ipc_protocol,
0219                  struct ipc_pipe *pipe,
0220                  struct sk_buff_head *p_ul_list)
0221 {
0222     struct ipc_protocol_td *td;
0223     bool hpda_pending = false;
0224     struct sk_buff *skb;
0225     s32 free_elements;
0226     u32 head;
0227     u32 tail;
0228
0229     if (!ipc_protocol->p_ap_shm) {
0230         dev_err(ipc_protocol->dev, "driver is not initialized");
0231         return false;
0232     }
0233
0234     /* Get head and tail of the td list and calculate
0235      * the number of free elements.
0236      */
0237     head = le32_to_cpu(ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr]);
0238     tail = pipe->old_tail;
0239
0240     while (!skb_queue_empty(p_ul_list)) {
0241         if (head < tail)
0242             free_elements = tail - head - 1;
0243         else
0244             free_elements =
0245                 pipe->nr_of_entries - head + ((s32)tail - 1);
0246
0247         if (free_elements <= 0) {
0248             dev_dbg(ipc_protocol->dev,
0249                 "no free td elements for UL pipe %d",
0250                 pipe->pipe_nr);
0251             break;
0252         }
0253
0254         /* Get the td address. */
0255         td = &pipe->tdr_start[head];
0256
0257         /* Take the first element of the uplink list and add it
0258          * to the td list.
0259          */
0260         skb = skb_dequeue(p_ul_list);
0261         if (WARN_ON(!skb))
0262             break;
0263
0264         /* Save the reference to the uplink skbuf. */
0265         pipe->skbr_start[head] = skb;
0266
0267         td->buffer.address = IPC_CB(skb)->mapping;
0268         td->scs = cpu_to_le32(skb->len) & cpu_to_le32(SIZE_MASK);
0269         td->next = 0;
0270
0271         pipe->nr_of_queued_entries++;
0272
0273         /* Calculate the new head and save it. */
0274         head++;
0275         if (head >= pipe->nr_of_entries)
0276             head = 0;
0277
0278         ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr] =
0279             cpu_to_le32(head);
0280     }
0281
0282     if (pipe->old_head != head) {
0283         dev_dbg(ipc_protocol->dev, "New UL TDs Pipe:%d", pipe->pipe_nr);
0284
0285         pipe->old_head = head;
0286         /* Trigger doorbell because of pending UL packets. */
0287         hpda_pending = true;
0288     }
0289
0290     return hpda_pending;
0291 }
0292
0293 /* Checks for Tail pointer update from CP and returns the data as SKB. */
0294 struct sk_buff *ipc_protocol_ul_td_process(struct iosm_protocol *ipc_protocol,
0295                        struct ipc_pipe *pipe)
0296 {
0297     struct ipc_protocol_td *p_td = &pipe->tdr_start[pipe->old_tail];
0298     struct sk_buff *skb = pipe->skbr_start[pipe->old_tail];
0299
0300     pipe->nr_of_queued_entries--;
0301     pipe->old_tail++;
0302     if (pipe->old_tail >= pipe->nr_of_entries)
0303         pipe->old_tail = 0;
0304
0305     if (!p_td->buffer.address) {
0306         dev_err(ipc_protocol->dev, "Td buffer address is NULL");
0307         return NULL;
0308     }
0309
0310     if (p_td->buffer.address != IPC_CB(skb)->mapping) {
0311         dev_err(ipc_protocol->dev,
0312             "pipe %d: invalid buf_addr or skb_data",
0313             pipe->pipe_nr);
0314         return NULL;
0315     }
0316
0317     return skb;
0318 }
0319
0320 /* Allocates an SKB for CP to send data and updates the Head Pointer
0321  * of the given Pipe#.
0322  */
0323 bool ipc_protocol_dl_td_prepare(struct iosm_protocol *ipc_protocol,
0324                 struct ipc_pipe *pipe)
0325 {
0326     struct ipc_protocol_td *td;
0327     dma_addr_t mapping = 0;
0328     u32 head, new_head;
0329     struct sk_buff *skb;
0330     u32 tail;
0331
0332     /* Get head and tail of the td list and calculate
0333      * the number of free elements.
0334      */
0335     head = le32_to_cpu(ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr]);
0336     tail = le32_to_cpu(ipc_protocol->p_ap_shm->tail_array[pipe->pipe_nr]);
0337
0338     new_head = head + 1;
0339     if (new_head >= pipe->nr_of_entries)
0340         new_head = 0;
0341
0342     if (new_head == tail)
0343         return false;
0344
0345     /* Get the td address. */
0346     td = &pipe->tdr_start[head];
0347
0348     /* Allocate the skbuf for the descriptor. */
0349     skb = ipc_pcie_alloc_skb(ipc_protocol->pcie, pipe->buf_size, GFP_ATOMIC,
0350                  &mapping, DMA_FROM_DEVICE,
0351                  IPC_MEM_DL_ETH_OFFSET);
0352     if (!skb)
0353         return false;
0354
0355     td->buffer.address = mapping;
0356     td->scs = cpu_to_le32(pipe->buf_size) & cpu_to_le32(SIZE_MASK);
0357     td->next = 0;
0358
0359     /* store the new head value. */
0360     ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr] =
0361         cpu_to_le32(new_head);
0362
0363     /* Save the reference to the skbuf. */
0364     pipe->skbr_start[head] = skb;
0365
0366     pipe->nr_of_queued_entries++;
0367
0368     return true;
0369 }
0370
0371 /* Processes DL TD's */
0372 struct sk_buff *ipc_protocol_dl_td_process(struct iosm_protocol *ipc_protocol,
0373                        struct ipc_pipe *pipe)
0374 {
0375     struct ipc_protocol_td *p_td;
0376     struct sk_buff *skb;
0377
0378     if (!pipe->tdr_start)
0379         return NULL;
0380
0381     /* Copy the reference to the downlink buffer. */
0382     p_td = &pipe->tdr_start[pipe->old_tail];
0383     skb = pipe->skbr_start[pipe->old_tail];
0384
0385     /* Reset the ring elements. */
0386     pipe->skbr_start[pipe->old_tail] = NULL;
0387
0388     pipe->nr_of_queued_entries--;
0389
0390     pipe->old_tail++;
0391     if (pipe->old_tail >= pipe->nr_of_entries)
0392         pipe->old_tail = 0;
0393
0394     if (!skb) {
0395         dev_err(ipc_protocol->dev, "skb is null");
0396         goto ret;
0397     } else if (!p_td->buffer.address) {
0398         dev_err(ipc_protocol->dev, "td/buffer address is null");
0399         ipc_pcie_kfree_skb(ipc_protocol->pcie, skb);
0400         skb = NULL;
0401         goto ret;
0402     }
0403
0404     if (p_td->buffer.address != IPC_CB(skb)->mapping) {
0405         dev_err(ipc_protocol->dev, "invalid buf=%llx or skb=%p",
0406             (unsigned long long)p_td->buffer.address, skb->data);
0407         ipc_pcie_kfree_skb(ipc_protocol->pcie, skb);
0408         skb = NULL;
0409         goto ret;
0410     } else if ((le32_to_cpu(p_td->scs) & SIZE_MASK) > pipe->buf_size) {
0411         dev_err(ipc_protocol->dev, "invalid buffer size %d > %d",
0412             le32_to_cpu(p_td->scs) & SIZE_MASK,
0413             pipe->buf_size);
0414         ipc_pcie_kfree_skb(ipc_protocol->pcie, skb);
0415         skb = NULL;
0416         goto ret;
0417     } else if (le32_to_cpu(p_td->scs) >> COMPLETION_STATUS ==
0418           IPC_MEM_TD_CS_ABORT) {
0419         /* Discard aborted buffers. */
0420         dev_dbg(ipc_protocol->dev, "discard 'aborted' buffers");
0421         ipc_pcie_kfree_skb(ipc_protocol->pcie, skb);
0422         skb = NULL;
0423         goto ret;
0424     }
0425
0426     /* Set the length field in skbuf. */
0427     skb_put(skb, le32_to_cpu(p_td->scs) & SIZE_MASK);
0428
0429 ret:
0430     return skb;
0431 }
0432
0433 void ipc_protocol_get_head_tail_index(struct iosm_protocol *ipc_protocol,
0434                       struct ipc_pipe *pipe, u32 *head,
0435                       u32 *tail)
0436 {
0437     struct ipc_protocol_ap_shm *ipc_ap_shm = ipc_protocol->p_ap_shm;
0438
0439     if (head)
0440         *head = le32_to_cpu(ipc_ap_shm->head_array[pipe->pipe_nr]);
0441
0442     if (tail)
0443         *tail = le32_to_cpu(ipc_ap_shm->tail_array[pipe->pipe_nr]);
0444 }
0445
0446 /* Frees the TDs given to CP.  */
0447 void ipc_protocol_pipe_cleanup(struct iosm_protocol *ipc_protocol,
0448                    struct ipc_pipe *pipe)
0449 {
0450     struct sk_buff *skb;
0451     u32 head;
0452     u32 tail;
0453
0454     /* Get the start and the end of the buffer list. */
0455     head = le32_to_cpu(ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr]);
0456     tail = pipe->old_tail;
0457
0458     /* Reset tail and head to 0. */
0459     ipc_protocol->p_ap_shm->tail_array[pipe->pipe_nr] = 0;
0460     ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr] = 0;
0461
0462     /* Free pending uplink and downlink buffers. */
0463     if (pipe->skbr_start) {
0464         while (head != tail) {
0465             /* Get the reference to the skbuf,
0466              * which is on the way and free it.
0467              */
0468             skb = pipe->skbr_start[tail];
0469             if (skb)
0470                 ipc_pcie_kfree_skb(ipc_protocol->pcie, skb);
0471
0472             tail++;
0473             if (tail >= pipe->nr_of_entries)
0474                 tail = 0;
0475         }
0476
0477         kfree(pipe->skbr_start);
0478         pipe->skbr_start = NULL;
0479     }
0480
0481     pipe->old_tail = 0;
0482
0483     /* Free and reset the td and skbuf circular buffers. kfree is save! */
0484     if (pipe->tdr_start) {
0485         dma_free_coherent(&ipc_protocol->pcie->pci->dev,
0486                   sizeof(*pipe->tdr_start) * pipe->nr_of_entries,
0487                   pipe->tdr_start, pipe->phy_tdr_start);
0488
0489         pipe->tdr_start = NULL;
0490     }
0491 }
0492
0493 enum ipc_mem_device_ipc_state ipc_protocol_get_ipc_status(struct iosm_protocol
0494                               *ipc_protocol)
0495 {
0496     return (enum ipc_mem_device_ipc_state)
0497         le32_to_cpu(ipc_protocol->p_ap_shm->device_info.ipc_status);
0498 }
0499
0500 enum ipc_mem_exec_stage
0501 ipc_protocol_get_ap_exec_stage(struct iosm_protocol *ipc_protocol)
0502 {
0503     return le32_to_cpu(ipc_protocol->p_ap_shm->device_info.execution_stage);
0504 }
0505
0506 int ipc_protocol_msg_prep(struct iosm_imem *ipc_imem,
0507               enum ipc_msg_prep_type msg_type,
0508               union ipc_msg_prep_args *args)
0509 {
0510     struct iosm_protocol *ipc_protocol = ipc_imem->ipc_protocol;
0511
0512     switch (msg_type) {
0513     case IPC_MSG_PREP_SLEEP:
0514         return ipc_protocol_msg_prep_sleep(ipc_protocol, args);
0515
0516     case IPC_MSG_PREP_PIPE_OPEN:
0517         return ipc_protocol_msg_prepipe_open(ipc_protocol, args);
0518
0519     case IPC_MSG_PREP_PIPE_CLOSE:
0520         return ipc_protocol_msg_prepipe_close(ipc_protocol, args);
0521
0522     case IPC_MSG_PREP_FEATURE_SET:
0523         return ipc_protocol_msg_prep_feature_set(ipc_protocol, args);
0524
0525         /* Unsupported messages in protocol */
0526     case IPC_MSG_PREP_MAP:
0527     case IPC_MSG_PREP_UNMAP:
0528     default:
0529         dev_err(ipc_protocol->dev,
0530             "unsupported message type: %d in protocol", msg_type);
0531         return -EINVAL;
0532     }
0533 }
0534
0535 u32
0536 ipc_protocol_pm_dev_get_sleep_notification(struct iosm_protocol *ipc_protocol)
0537 {
0538     struct ipc_protocol_ap_shm *ipc_ap_shm = ipc_protocol->p_ap_shm;
0539
0540     return le32_to_cpu(ipc_ap_shm->device_info.device_sleep_notification);
0541 }