Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * rio_cm - RapidIO Channelized Messaging Driver
0004  *
0005  * Copyright 2013-2016 Integrated Device Technology, Inc.
0006  * Copyright (c) 2015, Prodrive Technologies
0007  * Copyright (c) 2015, RapidIO Trade Association
0008  */
0009 
0010 #include <linux/module.h>
0011 #include <linux/kernel.h>
0012 #include <linux/dma-mapping.h>
0013 #include <linux/delay.h>
0014 #include <linux/sched.h>
0015 #include <linux/rio.h>
0016 #include <linux/rio_drv.h>
0017 #include <linux/slab.h>
0018 #include <linux/idr.h>
0019 #include <linux/interrupt.h>
0020 #include <linux/cdev.h>
0021 #include <linux/fs.h>
0022 #include <linux/poll.h>
0023 #include <linux/reboot.h>
0024 #include <linux/bitops.h>
0025 #include <linux/printk.h>
0026 #include <linux/rio_cm_cdev.h>
0027 
0028 #define DRV_NAME        "rio_cm"
0029 #define DRV_VERSION     "1.0.0"
0030 #define DRV_AUTHOR      "Alexandre Bounine <alexandre.bounine@idt.com>"
0031 #define DRV_DESC        "RapidIO Channelized Messaging Driver"
0032 #define DEV_NAME    "rio_cm"
0033 
0034 /* Debug output filtering masks */
0035 enum {
0036     DBG_NONE    = 0,
0037     DBG_INIT    = BIT(0), /* driver init */
0038     DBG_EXIT    = BIT(1), /* driver exit */
0039     DBG_MPORT   = BIT(2), /* mport add/remove */
0040     DBG_RDEV    = BIT(3), /* RapidIO device add/remove */
0041     DBG_CHOP    = BIT(4), /* channel operations */
0042     DBG_WAIT    = BIT(5), /* waiting for events */
0043     DBG_TX      = BIT(6), /* message TX */
0044     DBG_TX_EVENT    = BIT(7), /* message TX event */
0045     DBG_RX_DATA = BIT(8), /* inbound data messages */
0046     DBG_RX_CMD  = BIT(9), /* inbound REQ/ACK/NACK messages */
0047     DBG_ALL     = ~0,
0048 };
0049 
0050 #ifdef DEBUG
0051 #define riocm_debug(level, fmt, arg...) \
0052     do { \
0053         if (DBG_##level & dbg_level) \
0054             pr_debug(DRV_NAME ": %s " fmt "\n", \
0055                 __func__, ##arg); \
0056     } while (0)
0057 #else
0058 #define riocm_debug(level, fmt, arg...) \
0059         no_printk(KERN_DEBUG pr_fmt(DRV_NAME fmt "\n"), ##arg)
0060 #endif
0061 
0062 #define riocm_warn(fmt, arg...) \
0063     pr_warn(DRV_NAME ": %s WARNING " fmt "\n", __func__, ##arg)
0064 
0065 #define riocm_error(fmt, arg...) \
0066     pr_err(DRV_NAME ": %s ERROR " fmt "\n", __func__, ##arg)
0067 
0068 
0069 static int cmbox = 1;
0070 module_param(cmbox, int, S_IRUGO);
0071 MODULE_PARM_DESC(cmbox, "RapidIO Mailbox number (default 1)");
0072 
0073 static int chstart = 256;
0074 module_param(chstart, int, S_IRUGO);
0075 MODULE_PARM_DESC(chstart,
0076          "Start channel number for dynamic allocation (default 256)");
0077 
0078 #ifdef DEBUG
0079 static u32 dbg_level = DBG_NONE;
0080 module_param(dbg_level, uint, S_IWUSR | S_IRUGO);
0081 MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)");
0082 #endif
0083 
0084 MODULE_AUTHOR(DRV_AUTHOR);
0085 MODULE_DESCRIPTION(DRV_DESC);
0086 MODULE_LICENSE("GPL");
0087 MODULE_VERSION(DRV_VERSION);
0088 
0089 #define RIOCM_TX_RING_SIZE  128
0090 #define RIOCM_RX_RING_SIZE  128
0091 #define RIOCM_CONNECT_TO    3 /* connect response TO (in sec) */
0092 
0093 #define RIOCM_MAX_CHNUM     0xffff /* Use full range of u16 field */
0094 #define RIOCM_CHNUM_AUTO    0
0095 #define RIOCM_MAX_EP_COUNT  0x10000 /* Max number of endpoints */
0096 
0097 enum rio_cm_state {
0098     RIO_CM_IDLE,
0099     RIO_CM_CONNECT,
0100     RIO_CM_CONNECTED,
0101     RIO_CM_DISCONNECT,
0102     RIO_CM_CHAN_BOUND,
0103     RIO_CM_LISTEN,
0104     RIO_CM_DESTROYING,
0105 };
0106 
0107 enum rio_cm_pkt_type {
0108     RIO_CM_SYS  = 0xaa,
0109     RIO_CM_CHAN = 0x55,
0110 };
0111 
0112 enum rio_cm_chop {
0113     CM_CONN_REQ,
0114     CM_CONN_ACK,
0115     CM_CONN_CLOSE,
0116     CM_DATA_MSG,
0117 };
0118 
0119 struct rio_ch_base_bhdr {
0120     u32 src_id;
0121     u32 dst_id;
0122 #define RIO_HDR_LETTER_MASK 0xffff0000
0123 #define RIO_HDR_MBOX_MASK   0x0000ffff
0124     u8  src_mbox;
0125     u8  dst_mbox;
0126     u8  type;
0127 } __attribute__((__packed__));
0128 
0129 struct rio_ch_chan_hdr {
0130     struct rio_ch_base_bhdr bhdr;
0131     u8 ch_op;
0132     u16 dst_ch;
0133     u16 src_ch;
0134     u16 msg_len;
0135     u16 rsrvd;
0136 } __attribute__((__packed__));
0137 
0138 struct tx_req {
0139     struct list_head node;
0140     struct rio_dev   *rdev;
0141     void         *buffer;
0142     size_t       len;
0143 };
0144 
0145 struct cm_dev {
0146     struct list_head    list;
0147     struct rio_mport    *mport;
0148     void            *rx_buf[RIOCM_RX_RING_SIZE];
0149     int         rx_slots;
0150     struct mutex        rx_lock;
0151 
0152     void            *tx_buf[RIOCM_TX_RING_SIZE];
0153     int         tx_slot;
0154     int         tx_cnt;
0155     int         tx_ack_slot;
0156     struct list_head    tx_reqs;
0157     spinlock_t      tx_lock;
0158 
0159     struct list_head    peers;
0160     u32         npeers;
0161     struct workqueue_struct *rx_wq;
0162     struct work_struct  rx_work;
0163 };
0164 
0165 struct chan_rx_ring {
0166     void    *buf[RIOCM_RX_RING_SIZE];
0167     int head;
0168     int tail;
0169     int count;
0170 
0171     /* Tracking RX buffers reported to upper level */
0172     void    *inuse[RIOCM_RX_RING_SIZE];
0173     int inuse_cnt;
0174 };
0175 
0176 struct rio_channel {
0177     u16         id; /* local channel ID */
0178     struct kref     ref;    /* channel refcount */
0179     struct file     *filp;
0180     struct cm_dev       *cmdev; /* associated CM device object */
0181     struct rio_dev      *rdev;  /* remote RapidIO device */
0182     enum rio_cm_state   state;
0183     int         error;
0184     spinlock_t      lock;
0185     void            *context;
0186     u32         loc_destid; /* local destID */
0187     u32         rem_destid; /* remote destID */
0188     u16         rem_channel;    /* remote channel ID */
0189     struct list_head    accept_queue;
0190     struct list_head    ch_node;
0191     struct completion   comp;
0192     struct completion   comp_close;
0193     struct chan_rx_ring rx_ring;
0194 };
0195 
0196 struct cm_peer {
0197     struct list_head node;
0198     struct rio_dev *rdev;
0199 };
0200 
0201 struct rio_cm_work {
0202     struct work_struct work;
0203     struct cm_dev *cm;
0204     void *data;
0205 };
0206 
0207 struct conn_req {
0208     struct list_head node;
0209     u32 destid; /* requester destID */
0210     u16 chan;   /* requester channel ID */
0211     struct cm_dev *cmdev;
0212 };
0213 
0214 /*
0215  * A channel_dev structure represents a CM_CDEV
0216  * @cdev    Character device
0217  * @dev     Associated device object
0218  */
0219 struct channel_dev {
0220     struct cdev cdev;
0221     struct device   *dev;
0222 };
0223 
0224 static struct rio_channel *riocm_ch_alloc(u16 ch_num);
0225 static void riocm_ch_free(struct kref *ref);
0226 static int riocm_post_send(struct cm_dev *cm, struct rio_dev *rdev,
0227                void *buffer, size_t len);
0228 static int riocm_ch_close(struct rio_channel *ch);
0229 
0230 static DEFINE_SPINLOCK(idr_lock);
0231 static DEFINE_IDR(ch_idr);
0232 
0233 static LIST_HEAD(cm_dev_list);
0234 static DECLARE_RWSEM(rdev_sem);
0235 
0236 static struct class *dev_class;
0237 static unsigned int dev_major;
0238 static unsigned int dev_minor_base;
0239 static dev_t dev_number;
0240 static struct channel_dev riocm_cdev;
0241 
0242 #define is_msg_capable(src_ops, dst_ops)            \
0243             ((src_ops & RIO_SRC_OPS_DATA_MSG) &&    \
0244              (dst_ops & RIO_DST_OPS_DATA_MSG))
0245 #define dev_cm_capable(dev) \
0246     is_msg_capable(dev->src_ops, dev->dst_ops)
0247 
0248 static int riocm_cmp(struct rio_channel *ch, enum rio_cm_state cmp)
0249 {
0250     int ret;
0251 
0252     spin_lock_bh(&ch->lock);
0253     ret = (ch->state == cmp);
0254     spin_unlock_bh(&ch->lock);
0255     return ret;
0256 }
0257 
0258 static int riocm_cmp_exch(struct rio_channel *ch,
0259                enum rio_cm_state cmp, enum rio_cm_state exch)
0260 {
0261     int ret;
0262 
0263     spin_lock_bh(&ch->lock);
0264     ret = (ch->state == cmp);
0265     if (ret)
0266         ch->state = exch;
0267     spin_unlock_bh(&ch->lock);
0268     return ret;
0269 }
0270 
0271 static enum rio_cm_state riocm_exch(struct rio_channel *ch,
0272                     enum rio_cm_state exch)
0273 {
0274     enum rio_cm_state old;
0275 
0276     spin_lock_bh(&ch->lock);
0277     old = ch->state;
0278     ch->state = exch;
0279     spin_unlock_bh(&ch->lock);
0280     return old;
0281 }
0282 
0283 static struct rio_channel *riocm_get_channel(u16 nr)
0284 {
0285     struct rio_channel *ch;
0286 
0287     spin_lock_bh(&idr_lock);
0288     ch = idr_find(&ch_idr, nr);
0289     if (ch)
0290         kref_get(&ch->ref);
0291     spin_unlock_bh(&idr_lock);
0292     return ch;
0293 }
0294 
0295 static void riocm_put_channel(struct rio_channel *ch)
0296 {
0297     kref_put(&ch->ref, riocm_ch_free);
0298 }
0299 
0300 static void *riocm_rx_get_msg(struct cm_dev *cm)
0301 {
0302     void *msg;
0303     int i;
0304 
0305     msg = rio_get_inb_message(cm->mport, cmbox);
0306     if (msg) {
0307         for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
0308             if (cm->rx_buf[i] == msg) {
0309                 cm->rx_buf[i] = NULL;
0310                 cm->rx_slots++;
0311                 break;
0312             }
0313         }
0314 
0315         if (i == RIOCM_RX_RING_SIZE)
0316             riocm_warn("no record for buffer 0x%p", msg);
0317     }
0318 
0319     return msg;
0320 }
0321 
0322 /*
0323  * riocm_rx_fill - fills a ring of receive buffers for given cm device
0324  * @cm: cm_dev object
0325  * @nent: max number of entries to fill
0326  *
0327  * Returns: none
0328  */
0329 static void riocm_rx_fill(struct cm_dev *cm, int nent)
0330 {
0331     int i;
0332 
0333     if (cm->rx_slots == 0)
0334         return;
0335 
0336     for (i = 0; i < RIOCM_RX_RING_SIZE && cm->rx_slots && nent; i++) {
0337         if (cm->rx_buf[i] == NULL) {
0338             cm->rx_buf[i] = kmalloc(RIO_MAX_MSG_SIZE, GFP_KERNEL);
0339             if (cm->rx_buf[i] == NULL)
0340                 break;
0341             rio_add_inb_buffer(cm->mport, cmbox, cm->rx_buf[i]);
0342             cm->rx_slots--;
0343             nent--;
0344         }
0345     }
0346 }
0347 
0348 /*
0349  * riocm_rx_free - frees all receive buffers associated with given cm device
0350  * @cm: cm_dev object
0351  *
0352  * Returns: none
0353  */
0354 static void riocm_rx_free(struct cm_dev *cm)
0355 {
0356     int i;
0357 
0358     for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
0359         if (cm->rx_buf[i] != NULL) {
0360             kfree(cm->rx_buf[i]);
0361             cm->rx_buf[i] = NULL;
0362         }
0363     }
0364 }
0365 
0366 /*
0367  * riocm_req_handler - connection request handler
0368  * @cm: cm_dev object
0369  * @req_data: pointer to the request packet
0370  *
0371  * Returns: 0 if success, or
0372  *          -EINVAL if channel is not in correct state,
0373  *          -ENODEV if cannot find a channel with specified ID,
0374  *          -ENOMEM if unable to allocate memory to store the request
0375  */
0376 static int riocm_req_handler(struct cm_dev *cm, void *req_data)
0377 {
0378     struct rio_channel *ch;
0379     struct conn_req *req;
0380     struct rio_ch_chan_hdr *hh = req_data;
0381     u16 chnum;
0382 
0383     chnum = ntohs(hh->dst_ch);
0384 
0385     ch = riocm_get_channel(chnum);
0386 
0387     if (!ch)
0388         return -ENODEV;
0389 
0390     if (ch->state != RIO_CM_LISTEN) {
0391         riocm_debug(RX_CMD, "channel %d is not in listen state", chnum);
0392         riocm_put_channel(ch);
0393         return -EINVAL;
0394     }
0395 
0396     req = kzalloc(sizeof(*req), GFP_KERNEL);
0397     if (!req) {
0398         riocm_put_channel(ch);
0399         return -ENOMEM;
0400     }
0401 
0402     req->destid = ntohl(hh->bhdr.src_id);
0403     req->chan = ntohs(hh->src_ch);
0404     req->cmdev = cm;
0405 
0406     spin_lock_bh(&ch->lock);
0407     list_add_tail(&req->node, &ch->accept_queue);
0408     spin_unlock_bh(&ch->lock);
0409     complete(&ch->comp);
0410     riocm_put_channel(ch);
0411 
0412     return 0;
0413 }
0414 
0415 /*
0416  * riocm_resp_handler - response to connection request handler
0417  * @resp_data: pointer to the response packet
0418  *
0419  * Returns: 0 if success, or
0420  *          -EINVAL if channel is not in correct state,
0421  *          -ENODEV if cannot find a channel with specified ID,
0422  */
0423 static int riocm_resp_handler(void *resp_data)
0424 {
0425     struct rio_channel *ch;
0426     struct rio_ch_chan_hdr *hh = resp_data;
0427     u16 chnum;
0428 
0429     chnum = ntohs(hh->dst_ch);
0430     ch = riocm_get_channel(chnum);
0431     if (!ch)
0432         return -ENODEV;
0433 
0434     if (ch->state != RIO_CM_CONNECT) {
0435         riocm_put_channel(ch);
0436         return -EINVAL;
0437     }
0438 
0439     riocm_exch(ch, RIO_CM_CONNECTED);
0440     ch->rem_channel = ntohs(hh->src_ch);
0441     complete(&ch->comp);
0442     riocm_put_channel(ch);
0443 
0444     return 0;
0445 }
0446 
0447 /*
0448  * riocm_close_handler - channel close request handler
0449  * @req_data: pointer to the request packet
0450  *
0451  * Returns: 0 if success, or
0452  *          -ENODEV if cannot find a channel with specified ID,
0453  *            + error codes returned by riocm_ch_close.
0454  */
0455 static int riocm_close_handler(void *data)
0456 {
0457     struct rio_channel *ch;
0458     struct rio_ch_chan_hdr *hh = data;
0459     int ret;
0460 
0461     riocm_debug(RX_CMD, "for ch=%d", ntohs(hh->dst_ch));
0462 
0463     spin_lock_bh(&idr_lock);
0464     ch = idr_find(&ch_idr, ntohs(hh->dst_ch));
0465     if (!ch) {
0466         spin_unlock_bh(&idr_lock);
0467         return -ENODEV;
0468     }
0469     idr_remove(&ch_idr, ch->id);
0470     spin_unlock_bh(&idr_lock);
0471 
0472     riocm_exch(ch, RIO_CM_DISCONNECT);
0473 
0474     ret = riocm_ch_close(ch);
0475     if (ret)
0476         riocm_debug(RX_CMD, "riocm_ch_close() returned %d", ret);
0477 
0478     return 0;
0479 }
0480 
0481 /*
0482  * rio_cm_handler - function that services request (non-data) packets
0483  * @cm: cm_dev object
0484  * @data: pointer to the packet
0485  */
0486 static void rio_cm_handler(struct cm_dev *cm, void *data)
0487 {
0488     struct rio_ch_chan_hdr *hdr;
0489 
0490     if (!rio_mport_is_running(cm->mport))
0491         goto out;
0492 
0493     hdr = data;
0494 
0495     riocm_debug(RX_CMD, "OP=%x for ch=%d from %d",
0496             hdr->ch_op, ntohs(hdr->dst_ch), ntohs(hdr->src_ch));
0497 
0498     switch (hdr->ch_op) {
0499     case CM_CONN_REQ:
0500         riocm_req_handler(cm, data);
0501         break;
0502     case CM_CONN_ACK:
0503         riocm_resp_handler(data);
0504         break;
0505     case CM_CONN_CLOSE:
0506         riocm_close_handler(data);
0507         break;
0508     default:
0509         riocm_error("Invalid packet header");
0510         break;
0511     }
0512 out:
0513     kfree(data);
0514 }
0515 
0516 /*
0517  * rio_rx_data_handler - received data packet handler
0518  * @cm: cm_dev object
0519  * @buf: data packet
0520  *
0521  * Returns: 0 if success, or
0522  *          -ENODEV if cannot find a channel with specified ID,
0523  *          -EIO if channel is not in CONNECTED state,
0524  *          -ENOMEM if channel RX queue is full (packet discarded)
0525  */
0526 static int rio_rx_data_handler(struct cm_dev *cm, void *buf)
0527 {
0528     struct rio_ch_chan_hdr *hdr;
0529     struct rio_channel *ch;
0530 
0531     hdr = buf;
0532 
0533     riocm_debug(RX_DATA, "for ch=%d", ntohs(hdr->dst_ch));
0534 
0535     ch = riocm_get_channel(ntohs(hdr->dst_ch));
0536     if (!ch) {
0537         /* Discard data message for non-existing channel */
0538         kfree(buf);
0539         return -ENODEV;
0540     }
0541 
0542     /* Place pointer to the buffer into channel's RX queue */
0543     spin_lock(&ch->lock);
0544 
0545     if (ch->state != RIO_CM_CONNECTED) {
0546         /* Channel is not ready to receive data, discard a packet */
0547         riocm_debug(RX_DATA, "ch=%d is in wrong state=%d",
0548                 ch->id, ch->state);
0549         spin_unlock(&ch->lock);
0550         kfree(buf);
0551         riocm_put_channel(ch);
0552         return -EIO;
0553     }
0554 
0555     if (ch->rx_ring.count == RIOCM_RX_RING_SIZE) {
0556         /* If RX ring is full, discard a packet */
0557         riocm_debug(RX_DATA, "ch=%d is full", ch->id);
0558         spin_unlock(&ch->lock);
0559         kfree(buf);
0560         riocm_put_channel(ch);
0561         return -ENOMEM;
0562     }
0563 
0564     ch->rx_ring.buf[ch->rx_ring.head] = buf;
0565     ch->rx_ring.head++;
0566     ch->rx_ring.count++;
0567     ch->rx_ring.head %= RIOCM_RX_RING_SIZE;
0568 
0569     complete(&ch->comp);
0570 
0571     spin_unlock(&ch->lock);
0572     riocm_put_channel(ch);
0573 
0574     return 0;
0575 }
0576 
0577 /*
0578  * rio_ibmsg_handler - inbound message packet handler
0579  */
0580 static void rio_ibmsg_handler(struct work_struct *work)
0581 {
0582     struct cm_dev *cm = container_of(work, struct cm_dev, rx_work);
0583     void *data;
0584     struct rio_ch_chan_hdr *hdr;
0585 
0586     if (!rio_mport_is_running(cm->mport))
0587         return;
0588 
0589     while (1) {
0590         mutex_lock(&cm->rx_lock);
0591         data = riocm_rx_get_msg(cm);
0592         if (data)
0593             riocm_rx_fill(cm, 1);
0594         mutex_unlock(&cm->rx_lock);
0595 
0596         if (data == NULL)
0597             break;
0598 
0599         hdr = data;
0600 
0601         if (hdr->bhdr.type != RIO_CM_CHAN) {
0602             /* For now simply discard packets other than channel */
0603             riocm_error("Unsupported TYPE code (0x%x). Msg dropped",
0604                     hdr->bhdr.type);
0605             kfree(data);
0606             continue;
0607         }
0608 
0609         /* Process a channel message */
0610         if (hdr->ch_op == CM_DATA_MSG)
0611             rio_rx_data_handler(cm, data);
0612         else
0613             rio_cm_handler(cm, data);
0614     }
0615 }
0616 
0617 static void riocm_inb_msg_event(struct rio_mport *mport, void *dev_id,
0618                 int mbox, int slot)
0619 {
0620     struct cm_dev *cm = dev_id;
0621 
0622     if (rio_mport_is_running(cm->mport) && !work_pending(&cm->rx_work))
0623         queue_work(cm->rx_wq, &cm->rx_work);
0624 }
0625 
0626 /*
0627  * rio_txcq_handler - TX completion handler
0628  * @cm: cm_dev object
0629  * @slot: TX queue slot
0630  *
0631  * TX completion handler also ensures that pending request packets are placed
0632  * into transmit queue as soon as a free slot becomes available. This is done
0633  * to give higher priority to request packets during high intensity data flow.
0634  */
0635 static void rio_txcq_handler(struct cm_dev *cm, int slot)
0636 {
0637     int ack_slot;
0638 
0639     /* ATTN: Add TX completion notification if/when direct buffer
0640      * transfer is implemented. At this moment only correct tracking
0641      * of tx_count is important.
0642      */
0643     riocm_debug(TX_EVENT, "for mport_%d slot %d tx_cnt %d",
0644             cm->mport->id, slot, cm->tx_cnt);
0645 
0646     spin_lock(&cm->tx_lock);
0647     ack_slot = cm->tx_ack_slot;
0648 
0649     if (ack_slot == slot)
0650         riocm_debug(TX_EVENT, "slot == ack_slot");
0651 
0652     while (cm->tx_cnt && ((ack_slot != slot) ||
0653            (cm->tx_cnt == RIOCM_TX_RING_SIZE))) {
0654 
0655         cm->tx_buf[ack_slot] = NULL;
0656         ++ack_slot;
0657         ack_slot &= (RIOCM_TX_RING_SIZE - 1);
0658         cm->tx_cnt--;
0659     }
0660 
0661     if (cm->tx_cnt < 0 || cm->tx_cnt > RIOCM_TX_RING_SIZE)
0662         riocm_error("tx_cnt %d out of sync", cm->tx_cnt);
0663 
0664     WARN_ON((cm->tx_cnt < 0) || (cm->tx_cnt > RIOCM_TX_RING_SIZE));
0665 
0666     cm->tx_ack_slot = ack_slot;
0667 
0668     /*
0669      * If there are pending requests, insert them into transmit queue
0670      */
0671     if (!list_empty(&cm->tx_reqs) && (cm->tx_cnt < RIOCM_TX_RING_SIZE)) {
0672         struct tx_req *req, *_req;
0673         int rc;
0674 
0675         list_for_each_entry_safe(req, _req, &cm->tx_reqs, node) {
0676             list_del(&req->node);
0677             cm->tx_buf[cm->tx_slot] = req->buffer;
0678             rc = rio_add_outb_message(cm->mport, req->rdev, cmbox,
0679                           req->buffer, req->len);
0680             kfree(req->buffer);
0681             kfree(req);
0682 
0683             ++cm->tx_cnt;
0684             ++cm->tx_slot;
0685             cm->tx_slot &= (RIOCM_TX_RING_SIZE - 1);
0686             if (cm->tx_cnt == RIOCM_TX_RING_SIZE)
0687                 break;
0688         }
0689     }
0690 
0691     spin_unlock(&cm->tx_lock);
0692 }
0693 
0694 static void riocm_outb_msg_event(struct rio_mport *mport, void *dev_id,
0695                  int mbox, int slot)
0696 {
0697     struct cm_dev *cm = dev_id;
0698 
0699     if (cm && rio_mport_is_running(cm->mport))
0700         rio_txcq_handler(cm, slot);
0701 }
0702 
0703 static int riocm_queue_req(struct cm_dev *cm, struct rio_dev *rdev,
0704                void *buffer, size_t len)
0705 {
0706     unsigned long flags;
0707     struct tx_req *treq;
0708 
0709     treq = kzalloc(sizeof(*treq), GFP_KERNEL);
0710     if (treq == NULL)
0711         return -ENOMEM;
0712 
0713     treq->rdev = rdev;
0714     treq->buffer = buffer;
0715     treq->len = len;
0716 
0717     spin_lock_irqsave(&cm->tx_lock, flags);
0718     list_add_tail(&treq->node, &cm->tx_reqs);
0719     spin_unlock_irqrestore(&cm->tx_lock, flags);
0720     return 0;
0721 }
0722 
0723 /*
0724  * riocm_post_send - helper function that places packet into msg TX queue
0725  * @cm: cm_dev object
0726  * @rdev: target RapidIO device object (required by outbound msg interface)
0727  * @buffer: pointer to a packet buffer to send
0728  * @len: length of data to transfer
0729  * @req: request priority flag
0730  *
0731  * Returns: 0 if success, or error code otherwise.
0732  */
0733 static int riocm_post_send(struct cm_dev *cm, struct rio_dev *rdev,
0734                void *buffer, size_t len)
0735 {
0736     int rc;
0737     unsigned long flags;
0738 
0739     spin_lock_irqsave(&cm->tx_lock, flags);
0740 
0741     if (cm->mport == NULL) {
0742         rc = -ENODEV;
0743         goto err_out;
0744     }
0745 
0746     if (cm->tx_cnt == RIOCM_TX_RING_SIZE) {
0747         riocm_debug(TX, "Tx Queue is full");
0748         rc = -EBUSY;
0749         goto err_out;
0750     }
0751 
0752     cm->tx_buf[cm->tx_slot] = buffer;
0753     rc = rio_add_outb_message(cm->mport, rdev, cmbox, buffer, len);
0754 
0755     riocm_debug(TX, "Add buf@%p destid=%x tx_slot=%d tx_cnt=%d",
0756          buffer, rdev->destid, cm->tx_slot, cm->tx_cnt);
0757 
0758     ++cm->tx_cnt;
0759     ++cm->tx_slot;
0760     cm->tx_slot &= (RIOCM_TX_RING_SIZE - 1);
0761 
0762 err_out:
0763     spin_unlock_irqrestore(&cm->tx_lock, flags);
0764     return rc;
0765 }
0766 
0767 /*
0768  * riocm_ch_send - sends a data packet to a remote device
0769  * @ch_id: local channel ID
0770  * @buf: pointer to a data buffer to send (including CM header)
0771  * @len: length of data to transfer (including CM header)
0772  *
0773  * ATTN: ASSUMES THAT THE HEADER SPACE IS RESERVED PART OF THE DATA PACKET
0774  *
0775  * Returns: 0 if success, or
0776  *          -EINVAL if one or more input parameters is/are not valid,
0777  *          -ENODEV if cannot find a channel with specified ID,
0778  *          -EAGAIN if a channel is not in CONNECTED state,
0779  *      + error codes returned by HW send routine.
0780  */
0781 static int riocm_ch_send(u16 ch_id, void *buf, int len)
0782 {
0783     struct rio_channel *ch;
0784     struct rio_ch_chan_hdr *hdr;
0785     int ret;
0786 
0787     if (buf == NULL || ch_id == 0 || len == 0 || len > RIO_MAX_MSG_SIZE)
0788         return -EINVAL;
0789 
0790     ch = riocm_get_channel(ch_id);
0791     if (!ch) {
0792         riocm_error("%s(%d) ch_%d not found", current->comm,
0793                 task_pid_nr(current), ch_id);
0794         return -ENODEV;
0795     }
0796 
0797     if (!riocm_cmp(ch, RIO_CM_CONNECTED)) {
0798         ret = -EAGAIN;
0799         goto err_out;
0800     }
0801 
0802     /*
0803      * Fill buffer header section with corresponding channel data
0804      */
0805     hdr = buf;
0806 
0807     hdr->bhdr.src_id = htonl(ch->loc_destid);
0808     hdr->bhdr.dst_id = htonl(ch->rem_destid);
0809     hdr->bhdr.src_mbox = cmbox;
0810     hdr->bhdr.dst_mbox = cmbox;
0811     hdr->bhdr.type = RIO_CM_CHAN;
0812     hdr->ch_op = CM_DATA_MSG;
0813     hdr->dst_ch = htons(ch->rem_channel);
0814     hdr->src_ch = htons(ch->id);
0815     hdr->msg_len = htons((u16)len);
0816 
0817     /* ATTN: the function call below relies on the fact that underlying
0818      * HW-specific add_outb_message() routine copies TX data into its own
0819      * internal transfer buffer (true for all RIONET compatible mport
0820      * drivers). Must be reviewed if mport driver uses the buffer directly.
0821      */
0822 
0823     ret = riocm_post_send(ch->cmdev, ch->rdev, buf, len);
0824     if (ret)
0825         riocm_debug(TX, "ch %d send_err=%d", ch->id, ret);
0826 err_out:
0827     riocm_put_channel(ch);
0828     return ret;
0829 }
0830 
0831 static int riocm_ch_free_rxbuf(struct rio_channel *ch, void *buf)
0832 {
0833     int i, ret = -EINVAL;
0834 
0835     spin_lock_bh(&ch->lock);
0836 
0837     for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
0838         if (ch->rx_ring.inuse[i] == buf) {
0839             ch->rx_ring.inuse[i] = NULL;
0840             ch->rx_ring.inuse_cnt--;
0841             ret = 0;
0842             break;
0843         }
0844     }
0845 
0846     spin_unlock_bh(&ch->lock);
0847 
0848     if (!ret)
0849         kfree(buf);
0850 
0851     return ret;
0852 }
0853 
0854 /*
0855  * riocm_ch_receive - fetch a data packet received for the specified channel
0856  * @ch: local channel ID
0857  * @buf: pointer to a packet buffer
0858  * @timeout: timeout to wait for incoming packet (in jiffies)
0859  *
0860  * Returns: 0 and valid buffer pointer if success, or NULL pointer and one of:
0861  *          -EAGAIN if a channel is not in CONNECTED state,
0862  *          -ENOMEM if in-use tracking queue is full,
0863  *          -ETIME if wait timeout expired,
0864  *      -EINTR if wait was interrupted.
0865  */
0866 static int riocm_ch_receive(struct rio_channel *ch, void **buf, long timeout)
0867 {
0868     void *rxmsg = NULL;
0869     int i, ret = 0;
0870     long wret;
0871 
0872     if (!riocm_cmp(ch, RIO_CM_CONNECTED)) {
0873         ret = -EAGAIN;
0874         goto out;
0875     }
0876 
0877     if (ch->rx_ring.inuse_cnt == RIOCM_RX_RING_SIZE) {
0878         /* If we do not have entries to track buffers given to upper
0879          * layer, reject request.
0880          */
0881         ret = -ENOMEM;
0882         goto out;
0883     }
0884 
0885     wret = wait_for_completion_interruptible_timeout(&ch->comp, timeout);
0886 
0887     riocm_debug(WAIT, "wait on %d returned %ld", ch->id, wret);
0888 
0889     if (!wret)
0890         ret = -ETIME;
0891     else if (wret == -ERESTARTSYS)
0892         ret = -EINTR;
0893     else
0894         ret = riocm_cmp(ch, RIO_CM_CONNECTED) ? 0 : -ECONNRESET;
0895 
0896     if (ret)
0897         goto out;
0898 
0899     spin_lock_bh(&ch->lock);
0900 
0901     rxmsg = ch->rx_ring.buf[ch->rx_ring.tail];
0902     ch->rx_ring.buf[ch->rx_ring.tail] = NULL;
0903     ch->rx_ring.count--;
0904     ch->rx_ring.tail++;
0905     ch->rx_ring.tail %= RIOCM_RX_RING_SIZE;
0906     ret = -ENOMEM;
0907 
0908     for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
0909         if (ch->rx_ring.inuse[i] == NULL) {
0910             ch->rx_ring.inuse[i] = rxmsg;
0911             ch->rx_ring.inuse_cnt++;
0912             ret = 0;
0913             break;
0914         }
0915     }
0916 
0917     if (ret) {
0918         /* We have no entry to store pending message: drop it */
0919         kfree(rxmsg);
0920         rxmsg = NULL;
0921     }
0922 
0923     spin_unlock_bh(&ch->lock);
0924 out:
0925     *buf = rxmsg;
0926     return ret;
0927 }
0928 
0929 /*
0930  * riocm_ch_connect - sends a connect request to a remote device
0931  * @loc_ch: local channel ID
0932  * @cm: CM device to send connect request
0933  * @peer: target RapidIO device
0934  * @rem_ch: remote channel ID
0935  *
0936  * Returns: 0 if success, or
0937  *          -EINVAL if the channel is not in IDLE state,
0938  *          -EAGAIN if no connection request available immediately,
0939  *          -ETIME if ACK response timeout expired,
0940  *          -EINTR if wait for response was interrupted.
0941  */
0942 static int riocm_ch_connect(u16 loc_ch, struct cm_dev *cm,
0943                 struct cm_peer *peer, u16 rem_ch)
0944 {
0945     struct rio_channel *ch = NULL;
0946     struct rio_ch_chan_hdr *hdr;
0947     int ret;
0948     long wret;
0949 
0950     ch = riocm_get_channel(loc_ch);
0951     if (!ch)
0952         return -ENODEV;
0953 
0954     if (!riocm_cmp_exch(ch, RIO_CM_IDLE, RIO_CM_CONNECT)) {
0955         ret = -EINVAL;
0956         goto conn_done;
0957     }
0958 
0959     ch->cmdev = cm;
0960     ch->rdev = peer->rdev;
0961     ch->context = NULL;
0962     ch->loc_destid = cm->mport->host_deviceid;
0963     ch->rem_channel = rem_ch;
0964 
0965     /*
0966      * Send connect request to the remote RapidIO device
0967      */
0968 
0969     hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
0970     if (hdr == NULL) {
0971         ret = -ENOMEM;
0972         goto conn_done;
0973     }
0974 
0975     hdr->bhdr.src_id = htonl(ch->loc_destid);
0976     hdr->bhdr.dst_id = htonl(peer->rdev->destid);
0977     hdr->bhdr.src_mbox = cmbox;
0978     hdr->bhdr.dst_mbox = cmbox;
0979     hdr->bhdr.type = RIO_CM_CHAN;
0980     hdr->ch_op = CM_CONN_REQ;
0981     hdr->dst_ch = htons(rem_ch);
0982     hdr->src_ch = htons(loc_ch);
0983 
0984     /* ATTN: the function call below relies on the fact that underlying
0985      * HW-specific add_outb_message() routine copies TX data into its
0986      * internal transfer buffer. Must be reviewed if mport driver uses
0987      * this buffer directly.
0988      */
0989     ret = riocm_post_send(cm, peer->rdev, hdr, sizeof(*hdr));
0990 
0991     if (ret != -EBUSY) {
0992         kfree(hdr);
0993     } else {
0994         ret = riocm_queue_req(cm, peer->rdev, hdr, sizeof(*hdr));
0995         if (ret)
0996             kfree(hdr);
0997     }
0998 
0999     if (ret) {
1000         riocm_cmp_exch(ch, RIO_CM_CONNECT, RIO_CM_IDLE);
1001         goto conn_done;
1002     }
1003 
1004     /* Wait for connect response from the remote device */
1005     wret = wait_for_completion_interruptible_timeout(&ch->comp,
1006                              RIOCM_CONNECT_TO * HZ);
1007     riocm_debug(WAIT, "wait on %d returns %ld", ch->id, wret);
1008 
1009     if (!wret)
1010         ret = -ETIME;
1011     else if (wret == -ERESTARTSYS)
1012         ret = -EINTR;
1013     else
1014         ret = riocm_cmp(ch, RIO_CM_CONNECTED) ? 0 : -1;
1015 
1016 conn_done:
1017     riocm_put_channel(ch);
1018     return ret;
1019 }
1020 
1021 static int riocm_send_ack(struct rio_channel *ch)
1022 {
1023     struct rio_ch_chan_hdr *hdr;
1024     int ret;
1025 
1026     hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
1027     if (hdr == NULL)
1028         return -ENOMEM;
1029 
1030     hdr->bhdr.src_id = htonl(ch->loc_destid);
1031     hdr->bhdr.dst_id = htonl(ch->rem_destid);
1032     hdr->dst_ch = htons(ch->rem_channel);
1033     hdr->src_ch = htons(ch->id);
1034     hdr->bhdr.src_mbox = cmbox;
1035     hdr->bhdr.dst_mbox = cmbox;
1036     hdr->bhdr.type = RIO_CM_CHAN;
1037     hdr->ch_op = CM_CONN_ACK;
1038 
1039     /* ATTN: the function call below relies on the fact that underlying
1040      * add_outb_message() routine copies TX data into its internal transfer
1041      * buffer. Review if switching to direct buffer version.
1042      */
1043     ret = riocm_post_send(ch->cmdev, ch->rdev, hdr, sizeof(*hdr));
1044 
1045     if (ret == -EBUSY && !riocm_queue_req(ch->cmdev,
1046                           ch->rdev, hdr, sizeof(*hdr)))
1047         return 0;
1048     kfree(hdr);
1049 
1050     if (ret)
1051         riocm_error("send ACK to ch_%d on %s failed (ret=%d)",
1052                 ch->id, rio_name(ch->rdev), ret);
1053     return ret;
1054 }
1055 
1056 /*
1057  * riocm_ch_accept - accept incoming connection request
1058  * @ch_id: channel ID
1059  * @new_ch_id: local mport device
1060  * @timeout: wait timeout (if 0 non-blocking call, do not wait if connection
1061  *           request is not available).
1062  *
1063  * Returns: pointer to new channel struct if success, or error-valued pointer:
1064  *          -ENODEV - cannot find specified channel or mport,
1065  *          -EINVAL - the channel is not in IDLE state,
1066  *          -EAGAIN - no connection request available immediately (timeout=0),
1067  *          -ENOMEM - unable to allocate new channel,
1068  *          -ETIME - wait timeout expired,
1069  *          -EINTR - wait was interrupted.
1070  */
1071 static struct rio_channel *riocm_ch_accept(u16 ch_id, u16 *new_ch_id,
1072                        long timeout)
1073 {
1074     struct rio_channel *ch;
1075     struct rio_channel *new_ch;
1076     struct conn_req *req;
1077     struct cm_peer *peer;
1078     int found = 0;
1079     int err = 0;
1080     long wret;
1081 
1082     ch = riocm_get_channel(ch_id);
1083     if (!ch)
1084         return ERR_PTR(-EINVAL);
1085 
1086     if (!riocm_cmp(ch, RIO_CM_LISTEN)) {
1087         err = -EINVAL;
1088         goto err_put;
1089     }
1090 
1091     /* Don't sleep if this is a non blocking call */
1092     if (!timeout) {
1093         if (!try_wait_for_completion(&ch->comp)) {
1094             err = -EAGAIN;
1095             goto err_put;
1096         }
1097     } else {
1098         riocm_debug(WAIT, "on %d", ch->id);
1099 
1100         wret = wait_for_completion_interruptible_timeout(&ch->comp,
1101                                  timeout);
1102         if (!wret) {
1103             err = -ETIME;
1104             goto err_put;
1105         } else if (wret == -ERESTARTSYS) {
1106             err = -EINTR;
1107             goto err_put;
1108         }
1109     }
1110 
1111     spin_lock_bh(&ch->lock);
1112 
1113     if (ch->state != RIO_CM_LISTEN) {
1114         err = -ECANCELED;
1115     } else if (list_empty(&ch->accept_queue)) {
1116         riocm_debug(WAIT, "on %d accept_queue is empty on completion",
1117                 ch->id);
1118         err = -EIO;
1119     }
1120 
1121     spin_unlock_bh(&ch->lock);
1122 
1123     if (err) {
1124         riocm_debug(WAIT, "on %d returns %d", ch->id, err);
1125         goto err_put;
1126     }
1127 
1128     /* Create new channel for this connection */
1129     new_ch = riocm_ch_alloc(RIOCM_CHNUM_AUTO);
1130 
1131     if (IS_ERR(new_ch)) {
1132         riocm_error("failed to get channel for new req (%ld)",
1133             PTR_ERR(new_ch));
1134         err = -ENOMEM;
1135         goto err_put;
1136     }
1137 
1138     spin_lock_bh(&ch->lock);
1139 
1140     req = list_first_entry(&ch->accept_queue, struct conn_req, node);
1141     list_del(&req->node);
1142     new_ch->cmdev = ch->cmdev;
1143     new_ch->loc_destid = ch->loc_destid;
1144     new_ch->rem_destid = req->destid;
1145     new_ch->rem_channel = req->chan;
1146 
1147     spin_unlock_bh(&ch->lock);
1148     riocm_put_channel(ch);
1149     ch = NULL;
1150     kfree(req);
1151 
1152     down_read(&rdev_sem);
1153     /* Find requester's device object */
1154     list_for_each_entry(peer, &new_ch->cmdev->peers, node) {
1155         if (peer->rdev->destid == new_ch->rem_destid) {
1156             riocm_debug(RX_CMD, "found matching device(%s)",
1157                     rio_name(peer->rdev));
1158             found = 1;
1159             break;
1160         }
1161     }
1162     up_read(&rdev_sem);
1163 
1164     if (!found) {
1165         /* If peer device object not found, simply ignore the request */
1166         err = -ENODEV;
1167         goto err_put_new_ch;
1168     }
1169 
1170     new_ch->rdev = peer->rdev;
1171     new_ch->state = RIO_CM_CONNECTED;
1172     spin_lock_init(&new_ch->lock);
1173 
1174     /* Acknowledge the connection request. */
1175     riocm_send_ack(new_ch);
1176 
1177     *new_ch_id = new_ch->id;
1178     return new_ch;
1179 
1180 err_put_new_ch:
1181     spin_lock_bh(&idr_lock);
1182     idr_remove(&ch_idr, new_ch->id);
1183     spin_unlock_bh(&idr_lock);
1184     riocm_put_channel(new_ch);
1185 
1186 err_put:
1187     if (ch)
1188         riocm_put_channel(ch);
1189     *new_ch_id = 0;
1190     return ERR_PTR(err);
1191 }
1192 
1193 /*
1194  * riocm_ch_listen - puts a channel into LISTEN state
1195  * @ch_id: channel ID
1196  *
1197  * Returns: 0 if success, or
1198  *          -EINVAL if the specified channel does not exists or
1199  *                  is not in CHAN_BOUND state.
1200  */
1201 static int riocm_ch_listen(u16 ch_id)
1202 {
1203     struct rio_channel *ch = NULL;
1204     int ret = 0;
1205 
1206     riocm_debug(CHOP, "(ch_%d)", ch_id);
1207 
1208     ch = riocm_get_channel(ch_id);
1209     if (!ch)
1210         return -EINVAL;
1211     if (!riocm_cmp_exch(ch, RIO_CM_CHAN_BOUND, RIO_CM_LISTEN))
1212         ret = -EINVAL;
1213     riocm_put_channel(ch);
1214     return ret;
1215 }
1216 
1217 /*
1218  * riocm_ch_bind - associate a channel object and an mport device
1219  * @ch_id: channel ID
1220  * @mport_id: local mport device ID
1221  * @context: pointer to the additional caller's context
1222  *
1223  * Returns: 0 if success, or
1224  *          -ENODEV if cannot find specified mport,
1225  *          -EINVAL if the specified channel does not exist or
1226  *                  is not in IDLE state.
1227  */
1228 static int riocm_ch_bind(u16 ch_id, u8 mport_id, void *context)
1229 {
1230     struct rio_channel *ch = NULL;
1231     struct cm_dev *cm;
1232     int rc = -ENODEV;
1233 
1234     riocm_debug(CHOP, "ch_%d to mport_%d", ch_id, mport_id);
1235 
1236     /* Find matching cm_dev object */
1237     down_read(&rdev_sem);
1238     list_for_each_entry(cm, &cm_dev_list, list) {
1239         if ((cm->mport->id == mport_id) &&
1240              rio_mport_is_running(cm->mport)) {
1241             rc = 0;
1242             break;
1243         }
1244     }
1245 
1246     if (rc)
1247         goto exit;
1248 
1249     ch = riocm_get_channel(ch_id);
1250     if (!ch) {
1251         rc = -EINVAL;
1252         goto exit;
1253     }
1254 
1255     spin_lock_bh(&ch->lock);
1256     if (ch->state != RIO_CM_IDLE) {
1257         spin_unlock_bh(&ch->lock);
1258         rc = -EINVAL;
1259         goto err_put;
1260     }
1261 
1262     ch->cmdev = cm;
1263     ch->loc_destid = cm->mport->host_deviceid;
1264     ch->context = context;
1265     ch->state = RIO_CM_CHAN_BOUND;
1266     spin_unlock_bh(&ch->lock);
1267 err_put:
1268     riocm_put_channel(ch);
1269 exit:
1270     up_read(&rdev_sem);
1271     return rc;
1272 }
1273 
1274 /*
1275  * riocm_ch_alloc - channel object allocation helper routine
1276  * @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1277  *
1278  * Return value: pointer to newly created channel object,
1279  *               or error-valued pointer
1280  */
1281 static struct rio_channel *riocm_ch_alloc(u16 ch_num)
1282 {
1283     int id;
1284     int start, end;
1285     struct rio_channel *ch;
1286 
1287     ch = kzalloc(sizeof(*ch), GFP_KERNEL);
1288     if (!ch)
1289         return ERR_PTR(-ENOMEM);
1290 
1291     if (ch_num) {
1292         /* If requested, try to obtain the specified channel ID */
1293         start = ch_num;
1294         end = ch_num + 1;
1295     } else {
1296         /* Obtain channel ID from the dynamic allocation range */
1297         start = chstart;
1298         end = RIOCM_MAX_CHNUM + 1;
1299     }
1300 
1301     idr_preload(GFP_KERNEL);
1302     spin_lock_bh(&idr_lock);
1303     id = idr_alloc_cyclic(&ch_idr, ch, start, end, GFP_NOWAIT);
1304     spin_unlock_bh(&idr_lock);
1305     idr_preload_end();
1306 
1307     if (id < 0) {
1308         kfree(ch);
1309         return ERR_PTR(id == -ENOSPC ? -EBUSY : id);
1310     }
1311 
1312     ch->id = (u16)id;
1313     ch->state = RIO_CM_IDLE;
1314     spin_lock_init(&ch->lock);
1315     INIT_LIST_HEAD(&ch->accept_queue);
1316     INIT_LIST_HEAD(&ch->ch_node);
1317     init_completion(&ch->comp);
1318     init_completion(&ch->comp_close);
1319     kref_init(&ch->ref);
1320     ch->rx_ring.head = 0;
1321     ch->rx_ring.tail = 0;
1322     ch->rx_ring.count = 0;
1323     ch->rx_ring.inuse_cnt = 0;
1324 
1325     return ch;
1326 }
1327 
1328 /*
1329  * riocm_ch_create - creates a new channel object and allocates ID for it
1330  * @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1331  *
1332  * Allocates and initializes a new channel object. If the parameter ch_num > 0
1333  * and is within the valid range, riocm_ch_create tries to allocate the
1334  * specified ID for the new channel. If ch_num = 0, channel ID will be assigned
1335  * automatically from the range (chstart ... RIOCM_MAX_CHNUM).
1336  * Module parameter 'chstart' defines start of an ID range available for dynamic
1337  * allocation. Range below 'chstart' is reserved for pre-defined ID numbers.
1338  * Available channel numbers are limited by 16-bit size of channel numbers used
1339  * in the packet header.
1340  *
1341  * Return value: PTR to rio_channel structure if successful (with channel number
1342  *               updated via pointer) or error-valued pointer if error.
1343  */
1344 static struct rio_channel *riocm_ch_create(u16 *ch_num)
1345 {
1346     struct rio_channel *ch = NULL;
1347 
1348     ch = riocm_ch_alloc(*ch_num);
1349 
1350     if (IS_ERR(ch))
1351         riocm_debug(CHOP, "Failed to allocate channel %d (err=%ld)",
1352                 *ch_num, PTR_ERR(ch));
1353     else
1354         *ch_num = ch->id;
1355 
1356     return ch;
1357 }
1358 
1359 /*
1360  * riocm_ch_free - channel object release routine
1361  * @ref: pointer to a channel's kref structure
1362  */
1363 static void riocm_ch_free(struct kref *ref)
1364 {
1365     struct rio_channel *ch = container_of(ref, struct rio_channel, ref);
1366     int i;
1367 
1368     riocm_debug(CHOP, "(ch_%d)", ch->id);
1369 
1370     if (ch->rx_ring.inuse_cnt) {
1371         for (i = 0;
1372              i < RIOCM_RX_RING_SIZE && ch->rx_ring.inuse_cnt; i++) {
1373             if (ch->rx_ring.inuse[i] != NULL) {
1374                 kfree(ch->rx_ring.inuse[i]);
1375                 ch->rx_ring.inuse_cnt--;
1376             }
1377         }
1378     }
1379 
1380     if (ch->rx_ring.count)
1381         for (i = 0; i < RIOCM_RX_RING_SIZE && ch->rx_ring.count; i++) {
1382             if (ch->rx_ring.buf[i] != NULL) {
1383                 kfree(ch->rx_ring.buf[i]);
1384                 ch->rx_ring.count--;
1385             }
1386         }
1387 
1388     complete(&ch->comp_close);
1389 }
1390 
1391 static int riocm_send_close(struct rio_channel *ch)
1392 {
1393     struct rio_ch_chan_hdr *hdr;
1394     int ret;
1395 
1396     /*
1397      * Send CH_CLOSE notification to the remote RapidIO device
1398      */
1399 
1400     hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
1401     if (hdr == NULL)
1402         return -ENOMEM;
1403 
1404     hdr->bhdr.src_id = htonl(ch->loc_destid);
1405     hdr->bhdr.dst_id = htonl(ch->rem_destid);
1406     hdr->bhdr.src_mbox = cmbox;
1407     hdr->bhdr.dst_mbox = cmbox;
1408     hdr->bhdr.type = RIO_CM_CHAN;
1409     hdr->ch_op = CM_CONN_CLOSE;
1410     hdr->dst_ch = htons(ch->rem_channel);
1411     hdr->src_ch = htons(ch->id);
1412 
1413     /* ATTN: the function call below relies on the fact that underlying
1414      * add_outb_message() routine copies TX data into its internal transfer
1415      * buffer. Needs to be reviewed if switched to direct buffer mode.
1416      */
1417     ret = riocm_post_send(ch->cmdev, ch->rdev, hdr, sizeof(*hdr));
1418 
1419     if (ret == -EBUSY && !riocm_queue_req(ch->cmdev, ch->rdev,
1420                           hdr, sizeof(*hdr)))
1421         return 0;
1422     kfree(hdr);
1423 
1424     if (ret)
1425         riocm_error("ch(%d) send CLOSE failed (ret=%d)", ch->id, ret);
1426 
1427     return ret;
1428 }
1429 
1430 /*
1431  * riocm_ch_close - closes a channel object with specified ID (by local request)
1432  * @ch: channel to be closed
1433  */
1434 static int riocm_ch_close(struct rio_channel *ch)
1435 {
1436     unsigned long tmo = msecs_to_jiffies(3000);
1437     enum rio_cm_state state;
1438     long wret;
1439     int ret = 0;
1440 
1441     riocm_debug(CHOP, "ch_%d by %s(%d)",
1442             ch->id, current->comm, task_pid_nr(current));
1443 
1444     state = riocm_exch(ch, RIO_CM_DESTROYING);
1445     if (state == RIO_CM_CONNECTED)
1446         riocm_send_close(ch);
1447 
1448     complete_all(&ch->comp);
1449 
1450     riocm_put_channel(ch);
1451     wret = wait_for_completion_interruptible_timeout(&ch->comp_close, tmo);
1452 
1453     riocm_debug(WAIT, "wait on %d returns %ld", ch->id, wret);
1454 
1455     if (wret == 0) {
1456         /* Timeout on wait occurred */
1457         riocm_debug(CHOP, "%s(%d) timed out waiting for ch %d",
1458                current->comm, task_pid_nr(current), ch->id);
1459         ret = -ETIMEDOUT;
1460     } else if (wret == -ERESTARTSYS) {
1461         /* Wait_for_completion was interrupted by a signal */
1462         riocm_debug(CHOP, "%s(%d) wait for ch %d was interrupted",
1463             current->comm, task_pid_nr(current), ch->id);
1464         ret = -EINTR;
1465     }
1466 
1467     if (!ret) {
1468         riocm_debug(CHOP, "ch_%d resources released", ch->id);
1469         kfree(ch);
1470     } else {
1471         riocm_debug(CHOP, "failed to release ch_%d resources", ch->id);
1472     }
1473 
1474     return ret;
1475 }
1476 
1477 /*
1478  * riocm_cdev_open() - Open character device
1479  */
1480 static int riocm_cdev_open(struct inode *inode, struct file *filp)
1481 {
1482     riocm_debug(INIT, "by %s(%d) filp=%p ",
1483             current->comm, task_pid_nr(current), filp);
1484 
1485     if (list_empty(&cm_dev_list))
1486         return -ENODEV;
1487 
1488     return 0;
1489 }
1490 
1491 /*
1492  * riocm_cdev_release() - Release character device
1493  */
1494 static int riocm_cdev_release(struct inode *inode, struct file *filp)
1495 {
1496     struct rio_channel *ch, *_c;
1497     unsigned int i;
1498     LIST_HEAD(list);
1499 
1500     riocm_debug(EXIT, "by %s(%d) filp=%p",
1501             current->comm, task_pid_nr(current), filp);
1502 
1503     /* Check if there are channels associated with this file descriptor */
1504     spin_lock_bh(&idr_lock);
1505     idr_for_each_entry(&ch_idr, ch, i) {
1506         if (ch && ch->filp == filp) {
1507             riocm_debug(EXIT, "ch_%d not released by %s(%d)",
1508                     ch->id, current->comm,
1509                     task_pid_nr(current));
1510             idr_remove(&ch_idr, ch->id);
1511             list_add(&ch->ch_node, &list);
1512         }
1513     }
1514     spin_unlock_bh(&idr_lock);
1515 
1516     if (!list_empty(&list)) {
1517         list_for_each_entry_safe(ch, _c, &list, ch_node) {
1518             list_del(&ch->ch_node);
1519             riocm_ch_close(ch);
1520         }
1521     }
1522 
1523     return 0;
1524 }
1525 
1526 /*
1527  * cm_ep_get_list_size() - Reports number of endpoints in the network
1528  */
1529 static int cm_ep_get_list_size(void __user *arg)
1530 {
1531     u32 __user *p = arg;
1532     u32 mport_id;
1533     u32 count = 0;
1534     struct cm_dev *cm;
1535 
1536     if (get_user(mport_id, p))
1537         return -EFAULT;
1538     if (mport_id >= RIO_MAX_MPORTS)
1539         return -EINVAL;
1540 
1541     /* Find a matching cm_dev object */
1542     down_read(&rdev_sem);
1543     list_for_each_entry(cm, &cm_dev_list, list) {
1544         if (cm->mport->id == mport_id) {
1545             count = cm->npeers;
1546             up_read(&rdev_sem);
1547             if (copy_to_user(arg, &count, sizeof(u32)))
1548                 return -EFAULT;
1549             return 0;
1550         }
1551     }
1552     up_read(&rdev_sem);
1553 
1554     return -ENODEV;
1555 }
1556 
1557 /*
1558  * cm_ep_get_list() - Returns list of attached endpoints
1559  */
1560 static int cm_ep_get_list(void __user *arg)
1561 {
1562     struct cm_dev *cm;
1563     struct cm_peer *peer;
1564     u32 info[2];
1565     void *buf;
1566     u32 nent;
1567     u32 *entry_ptr;
1568     u32 i = 0;
1569     int ret = 0;
1570 
1571     if (copy_from_user(&info, arg, sizeof(info)))
1572         return -EFAULT;
1573 
1574     if (info[1] >= RIO_MAX_MPORTS || info[0] > RIOCM_MAX_EP_COUNT)
1575         return -EINVAL;
1576 
1577     /* Find a matching cm_dev object */
1578     down_read(&rdev_sem);
1579     list_for_each_entry(cm, &cm_dev_list, list)
1580         if (cm->mport->id == (u8)info[1])
1581             goto found;
1582 
1583     up_read(&rdev_sem);
1584     return -ENODEV;
1585 
1586 found:
1587     nent = min(info[0], cm->npeers);
1588     buf = kcalloc(nent + 2, sizeof(u32), GFP_KERNEL);
1589     if (!buf) {
1590         up_read(&rdev_sem);
1591         return -ENOMEM;
1592     }
1593 
1594     entry_ptr = (u32 *)((uintptr_t)buf + 2*sizeof(u32));
1595 
1596     list_for_each_entry(peer, &cm->peers, node) {
1597         *entry_ptr = (u32)peer->rdev->destid;
1598         entry_ptr++;
1599         if (++i == nent)
1600             break;
1601     }
1602     up_read(&rdev_sem);
1603 
1604     ((u32 *)buf)[0] = i; /* report an updated number of entries */
1605     ((u32 *)buf)[1] = info[1]; /* put back an mport ID */
1606     if (copy_to_user(arg, buf, sizeof(u32) * (info[0] + 2)))
1607         ret = -EFAULT;
1608 
1609     kfree(buf);
1610     return ret;
1611 }
1612 
1613 /*
1614  * cm_mport_get_list() - Returns list of available local mport devices
1615  */
1616 static int cm_mport_get_list(void __user *arg)
1617 {
1618     int ret = 0;
1619     u32 entries;
1620     void *buf;
1621     struct cm_dev *cm;
1622     u32 *entry_ptr;
1623     int count = 0;
1624 
1625     if (copy_from_user(&entries, arg, sizeof(entries)))
1626         return -EFAULT;
1627     if (entries == 0 || entries > RIO_MAX_MPORTS)
1628         return -EINVAL;
1629     buf = kcalloc(entries + 1, sizeof(u32), GFP_KERNEL);
1630     if (!buf)
1631         return -ENOMEM;
1632 
1633     /* Scan all registered cm_dev objects */
1634     entry_ptr = (u32 *)((uintptr_t)buf + sizeof(u32));
1635     down_read(&rdev_sem);
1636     list_for_each_entry(cm, &cm_dev_list, list) {
1637         if (count++ < entries) {
1638             *entry_ptr = (cm->mport->id << 16) |
1639                       cm->mport->host_deviceid;
1640             entry_ptr++;
1641         }
1642     }
1643     up_read(&rdev_sem);
1644 
1645     *((u32 *)buf) = count; /* report a real number of entries */
1646     if (copy_to_user(arg, buf, sizeof(u32) * (count + 1)))
1647         ret = -EFAULT;
1648 
1649     kfree(buf);
1650     return ret;
1651 }
1652 
1653 /*
1654  * cm_chan_create() - Create a message exchange channel
1655  */
1656 static int cm_chan_create(struct file *filp, void __user *arg)
1657 {
1658     u16 __user *p = arg;
1659     u16 ch_num;
1660     struct rio_channel *ch;
1661 
1662     if (get_user(ch_num, p))
1663         return -EFAULT;
1664 
1665     riocm_debug(CHOP, "ch_%d requested by %s(%d)",
1666             ch_num, current->comm, task_pid_nr(current));
1667     ch = riocm_ch_create(&ch_num);
1668     if (IS_ERR(ch))
1669         return PTR_ERR(ch);
1670 
1671     ch->filp = filp;
1672     riocm_debug(CHOP, "ch_%d created by %s(%d)",
1673             ch_num, current->comm, task_pid_nr(current));
1674     return put_user(ch_num, p);
1675 }
1676 
1677 /*
1678  * cm_chan_close() - Close channel
1679  * @filp:   Pointer to file object
1680  * @arg:    Channel to close
1681  */
1682 static int cm_chan_close(struct file *filp, void __user *arg)
1683 {
1684     u16 __user *p = arg;
1685     u16 ch_num;
1686     struct rio_channel *ch;
1687 
1688     if (get_user(ch_num, p))
1689         return -EFAULT;
1690 
1691     riocm_debug(CHOP, "ch_%d by %s(%d)",
1692             ch_num, current->comm, task_pid_nr(current));
1693 
1694     spin_lock_bh(&idr_lock);
1695     ch = idr_find(&ch_idr, ch_num);
1696     if (!ch) {
1697         spin_unlock_bh(&idr_lock);
1698         return 0;
1699     }
1700     if (ch->filp != filp) {
1701         spin_unlock_bh(&idr_lock);
1702         return -EINVAL;
1703     }
1704     idr_remove(&ch_idr, ch->id);
1705     spin_unlock_bh(&idr_lock);
1706 
1707     return riocm_ch_close(ch);
1708 }
1709 
1710 /*
1711  * cm_chan_bind() - Bind channel
1712  * @arg:    Channel number
1713  */
1714 static int cm_chan_bind(void __user *arg)
1715 {
1716     struct rio_cm_channel chan;
1717 
1718     if (copy_from_user(&chan, arg, sizeof(chan)))
1719         return -EFAULT;
1720     if (chan.mport_id >= RIO_MAX_MPORTS)
1721         return -EINVAL;
1722 
1723     return riocm_ch_bind(chan.id, chan.mport_id, NULL);
1724 }
1725 
1726 /*
1727  * cm_chan_listen() - Listen on channel
1728  * @arg:    Channel number
1729  */
1730 static int cm_chan_listen(void __user *arg)
1731 {
1732     u16 __user *p = arg;
1733     u16 ch_num;
1734 
1735     if (get_user(ch_num, p))
1736         return -EFAULT;
1737 
1738     return riocm_ch_listen(ch_num);
1739 }
1740 
1741 /*
1742  * cm_chan_accept() - Accept incoming connection
1743  * @filp:   Pointer to file object
1744  * @arg:    Channel number
1745  */
1746 static int cm_chan_accept(struct file *filp, void __user *arg)
1747 {
1748     struct rio_cm_accept param;
1749     long accept_to;
1750     struct rio_channel *ch;
1751 
1752     if (copy_from_user(&param, arg, sizeof(param)))
1753         return -EFAULT;
1754 
1755     riocm_debug(CHOP, "on ch_%d by %s(%d)",
1756             param.ch_num, current->comm, task_pid_nr(current));
1757 
1758     accept_to = param.wait_to ?
1759             msecs_to_jiffies(param.wait_to) : 0;
1760 
1761     ch = riocm_ch_accept(param.ch_num, &param.ch_num, accept_to);
1762     if (IS_ERR(ch))
1763         return PTR_ERR(ch);
1764     ch->filp = filp;
1765 
1766     riocm_debug(CHOP, "new ch_%d for %s(%d)",
1767             ch->id, current->comm, task_pid_nr(current));
1768 
1769     if (copy_to_user(arg, &param, sizeof(param)))
1770         return -EFAULT;
1771     return 0;
1772 }
1773 
1774 /*
1775  * cm_chan_connect() - Connect on channel
1776  * @arg:    Channel information
1777  */
1778 static int cm_chan_connect(void __user *arg)
1779 {
1780     struct rio_cm_channel chan;
1781     struct cm_dev *cm;
1782     struct cm_peer *peer;
1783     int ret = -ENODEV;
1784 
1785     if (copy_from_user(&chan, arg, sizeof(chan)))
1786         return -EFAULT;
1787     if (chan.mport_id >= RIO_MAX_MPORTS)
1788         return -EINVAL;
1789 
1790     down_read(&rdev_sem);
1791 
1792     /* Find matching cm_dev object */
1793     list_for_each_entry(cm, &cm_dev_list, list) {
1794         if (cm->mport->id == chan.mport_id) {
1795             ret = 0;
1796             break;
1797         }
1798     }
1799 
1800     if (ret)
1801         goto err_out;
1802 
1803     if (chan.remote_destid >= RIO_ANY_DESTID(cm->mport->sys_size)) {
1804         ret = -EINVAL;
1805         goto err_out;
1806     }
1807 
1808     /* Find corresponding RapidIO endpoint device object */
1809     ret = -ENODEV;
1810 
1811     list_for_each_entry(peer, &cm->peers, node) {
1812         if (peer->rdev->destid == chan.remote_destid) {
1813             ret = 0;
1814             break;
1815         }
1816     }
1817 
1818     if (ret)
1819         goto err_out;
1820 
1821     up_read(&rdev_sem);
1822 
1823     return riocm_ch_connect(chan.id, cm, peer, chan.remote_channel);
1824 err_out:
1825     up_read(&rdev_sem);
1826     return ret;
1827 }
1828 
1829 /*
1830  * cm_chan_msg_send() - Send a message through channel
1831  * @arg:    Outbound message information
1832  */
1833 static int cm_chan_msg_send(void __user *arg)
1834 {
1835     struct rio_cm_msg msg;
1836     void *buf;
1837     int ret;
1838 
1839     if (copy_from_user(&msg, arg, sizeof(msg)))
1840         return -EFAULT;
1841     if (msg.size > RIO_MAX_MSG_SIZE)
1842         return -EINVAL;
1843 
1844     buf = memdup_user((void __user *)(uintptr_t)msg.msg, msg.size);
1845     if (IS_ERR(buf))
1846         return PTR_ERR(buf);
1847 
1848     ret = riocm_ch_send(msg.ch_num, buf, msg.size);
1849 
1850     kfree(buf);
1851     return ret;
1852 }
1853 
1854 /*
1855  * cm_chan_msg_rcv() - Receive a message through channel
1856  * @arg:    Inbound message information
1857  */
1858 static int cm_chan_msg_rcv(void __user *arg)
1859 {
1860     struct rio_cm_msg msg;
1861     struct rio_channel *ch;
1862     void *buf;
1863     long rxto;
1864     int ret = 0, msg_size;
1865 
1866     if (copy_from_user(&msg, arg, sizeof(msg)))
1867         return -EFAULT;
1868 
1869     if (msg.ch_num == 0 || msg.size == 0)
1870         return -EINVAL;
1871 
1872     ch = riocm_get_channel(msg.ch_num);
1873     if (!ch)
1874         return -ENODEV;
1875 
1876     rxto = msg.rxto ? msecs_to_jiffies(msg.rxto) : MAX_SCHEDULE_TIMEOUT;
1877 
1878     ret = riocm_ch_receive(ch, &buf, rxto);
1879     if (ret)
1880         goto out;
1881 
1882     msg_size = min(msg.size, (u16)(RIO_MAX_MSG_SIZE));
1883 
1884     if (copy_to_user((void __user *)(uintptr_t)msg.msg, buf, msg_size))
1885         ret = -EFAULT;
1886 
1887     riocm_ch_free_rxbuf(ch, buf);
1888 out:
1889     riocm_put_channel(ch);
1890     return ret;
1891 }
1892 
1893 /*
1894  * riocm_cdev_ioctl() - IOCTL requests handler
1895  */
1896 static long
1897 riocm_cdev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
1898 {
1899     switch (cmd) {
1900     case RIO_CM_EP_GET_LIST_SIZE:
1901         return cm_ep_get_list_size((void __user *)arg);
1902     case RIO_CM_EP_GET_LIST:
1903         return cm_ep_get_list((void __user *)arg);
1904     case RIO_CM_CHAN_CREATE:
1905         return cm_chan_create(filp, (void __user *)arg);
1906     case RIO_CM_CHAN_CLOSE:
1907         return cm_chan_close(filp, (void __user *)arg);
1908     case RIO_CM_CHAN_BIND:
1909         return cm_chan_bind((void __user *)arg);
1910     case RIO_CM_CHAN_LISTEN:
1911         return cm_chan_listen((void __user *)arg);
1912     case RIO_CM_CHAN_ACCEPT:
1913         return cm_chan_accept(filp, (void __user *)arg);
1914     case RIO_CM_CHAN_CONNECT:
1915         return cm_chan_connect((void __user *)arg);
1916     case RIO_CM_CHAN_SEND:
1917         return cm_chan_msg_send((void __user *)arg);
1918     case RIO_CM_CHAN_RECEIVE:
1919         return cm_chan_msg_rcv((void __user *)arg);
1920     case RIO_CM_MPORT_GET_LIST:
1921         return cm_mport_get_list((void __user *)arg);
1922     default:
1923         break;
1924     }
1925 
1926     return -EINVAL;
1927 }
1928 
1929 static const struct file_operations riocm_cdev_fops = {
1930     .owner      = THIS_MODULE,
1931     .open       = riocm_cdev_open,
1932     .release    = riocm_cdev_release,
1933     .unlocked_ioctl = riocm_cdev_ioctl,
1934 };
1935 
1936 /*
1937  * riocm_add_dev - add new remote RapidIO device into channel management core
1938  * @dev: device object associated with RapidIO device
1939  * @sif: subsystem interface
1940  *
1941  * Adds the specified RapidIO device (if applicable) into peers list of
1942  * the corresponding channel management device (cm_dev).
1943  */
1944 static int riocm_add_dev(struct device *dev, struct subsys_interface *sif)
1945 {
1946     struct cm_peer *peer;
1947     struct rio_dev *rdev = to_rio_dev(dev);
1948     struct cm_dev *cm;
1949 
1950     /* Check if the remote device has capabilities required to support CM */
1951     if (!dev_cm_capable(rdev))
1952         return 0;
1953 
1954     riocm_debug(RDEV, "(%s)", rio_name(rdev));
1955 
1956     peer = kmalloc(sizeof(*peer), GFP_KERNEL);
1957     if (!peer)
1958         return -ENOMEM;
1959 
1960     /* Find a corresponding cm_dev object */
1961     down_write(&rdev_sem);
1962     list_for_each_entry(cm, &cm_dev_list, list) {
1963         if (cm->mport == rdev->net->hport)
1964             goto found;
1965     }
1966 
1967     up_write(&rdev_sem);
1968     kfree(peer);
1969     return -ENODEV;
1970 
1971 found:
1972     peer->rdev = rdev;
1973     list_add_tail(&peer->node, &cm->peers);
1974     cm->npeers++;
1975 
1976     up_write(&rdev_sem);
1977     return 0;
1978 }
1979 
1980 /*
1981  * riocm_remove_dev - remove remote RapidIO device from channel management core
1982  * @dev: device object associated with RapidIO device
1983  * @sif: subsystem interface
1984  *
1985  * Removes the specified RapidIO device (if applicable) from peers list of
1986  * the corresponding channel management device (cm_dev).
1987  */
1988 static void riocm_remove_dev(struct device *dev, struct subsys_interface *sif)
1989 {
1990     struct rio_dev *rdev = to_rio_dev(dev);
1991     struct cm_dev *cm;
1992     struct cm_peer *peer;
1993     struct rio_channel *ch, *_c;
1994     unsigned int i;
1995     bool found = false;
1996     LIST_HEAD(list);
1997 
1998     /* Check if the remote device has capabilities required to support CM */
1999     if (!dev_cm_capable(rdev))
2000         return;
2001 
2002     riocm_debug(RDEV, "(%s)", rio_name(rdev));
2003 
2004     /* Find matching cm_dev object */
2005     down_write(&rdev_sem);
2006     list_for_each_entry(cm, &cm_dev_list, list) {
2007         if (cm->mport == rdev->net->hport) {
2008             found = true;
2009             break;
2010         }
2011     }
2012 
2013     if (!found) {
2014         up_write(&rdev_sem);
2015         return;
2016     }
2017 
2018     /* Remove remote device from the list of peers */
2019     found = false;
2020     list_for_each_entry(peer, &cm->peers, node) {
2021         if (peer->rdev == rdev) {
2022             riocm_debug(RDEV, "removing peer %s", rio_name(rdev));
2023             found = true;
2024             list_del(&peer->node);
2025             cm->npeers--;
2026             kfree(peer);
2027             break;
2028         }
2029     }
2030 
2031     up_write(&rdev_sem);
2032 
2033     if (!found)
2034         return;
2035 
2036     /*
2037      * Release channels associated with this peer
2038      */
2039 
2040     spin_lock_bh(&idr_lock);
2041     idr_for_each_entry(&ch_idr, ch, i) {
2042         if (ch && ch->rdev == rdev) {
2043             if (atomic_read(&rdev->state) != RIO_DEVICE_SHUTDOWN)
2044                 riocm_exch(ch, RIO_CM_DISCONNECT);
2045             idr_remove(&ch_idr, ch->id);
2046             list_add(&ch->ch_node, &list);
2047         }
2048     }
2049     spin_unlock_bh(&idr_lock);
2050 
2051     if (!list_empty(&list)) {
2052         list_for_each_entry_safe(ch, _c, &list, ch_node) {
2053             list_del(&ch->ch_node);
2054             riocm_ch_close(ch);
2055         }
2056     }
2057 }
2058 
2059 /*
2060  * riocm_cdev_add() - Create rio_cm char device
2061  * @devno: device number assigned to device (MAJ + MIN)
2062  */
2063 static int riocm_cdev_add(dev_t devno)
2064 {
2065     int ret;
2066 
2067     cdev_init(&riocm_cdev.cdev, &riocm_cdev_fops);
2068     riocm_cdev.cdev.owner = THIS_MODULE;
2069     ret = cdev_add(&riocm_cdev.cdev, devno, 1);
2070     if (ret < 0) {
2071         riocm_error("Cannot register a device with error %d", ret);
2072         return ret;
2073     }
2074 
2075     riocm_cdev.dev = device_create(dev_class, NULL, devno, NULL, DEV_NAME);
2076     if (IS_ERR(riocm_cdev.dev)) {
2077         cdev_del(&riocm_cdev.cdev);
2078         return PTR_ERR(riocm_cdev.dev);
2079     }
2080 
2081     riocm_debug(MPORT, "Added %s cdev(%d:%d)",
2082             DEV_NAME, MAJOR(devno), MINOR(devno));
2083 
2084     return 0;
2085 }
2086 
2087 /*
2088  * riocm_add_mport - add new local mport device into channel management core
2089  * @dev: device object associated with mport
2090  * @class_intf: class interface
2091  *
2092  * When a new mport device is added, CM immediately reserves inbound and
2093  * outbound RapidIO mailboxes that will be used.
2094  */
2095 static int riocm_add_mport(struct device *dev,
2096                struct class_interface *class_intf)
2097 {
2098     int rc;
2099     int i;
2100     struct cm_dev *cm;
2101     struct rio_mport *mport = to_rio_mport(dev);
2102 
2103     riocm_debug(MPORT, "add mport %s", mport->name);
2104 
2105     cm = kzalloc(sizeof(*cm), GFP_KERNEL);
2106     if (!cm)
2107         return -ENOMEM;
2108 
2109     cm->mport = mport;
2110 
2111     rc = rio_request_outb_mbox(mport, cm, cmbox,
2112                    RIOCM_TX_RING_SIZE, riocm_outb_msg_event);
2113     if (rc) {
2114         riocm_error("failed to allocate OBMBOX_%d on %s",
2115                 cmbox, mport->name);
2116         kfree(cm);
2117         return -ENODEV;
2118     }
2119 
2120     rc = rio_request_inb_mbox(mport, cm, cmbox,
2121                   RIOCM_RX_RING_SIZE, riocm_inb_msg_event);
2122     if (rc) {
2123         riocm_error("failed to allocate IBMBOX_%d on %s",
2124                 cmbox, mport->name);
2125         rio_release_outb_mbox(mport, cmbox);
2126         kfree(cm);
2127         return -ENODEV;
2128     }
2129 
2130     cm->rx_wq = create_workqueue(DRV_NAME "/rxq");
2131     if (!cm->rx_wq) {
2132         rio_release_inb_mbox(mport, cmbox);
2133         rio_release_outb_mbox(mport, cmbox);
2134         kfree(cm);
2135         return -ENOMEM;
2136     }
2137 
2138     /*
2139      * Allocate and register inbound messaging buffers to be ready
2140      * to receive channel and system management requests
2141      */
2142     for (i = 0; i < RIOCM_RX_RING_SIZE; i++)
2143         cm->rx_buf[i] = NULL;
2144 
2145     cm->rx_slots = RIOCM_RX_RING_SIZE;
2146     mutex_init(&cm->rx_lock);
2147     riocm_rx_fill(cm, RIOCM_RX_RING_SIZE);
2148     INIT_WORK(&cm->rx_work, rio_ibmsg_handler);
2149 
2150     cm->tx_slot = 0;
2151     cm->tx_cnt = 0;
2152     cm->tx_ack_slot = 0;
2153     spin_lock_init(&cm->tx_lock);
2154 
2155     INIT_LIST_HEAD(&cm->peers);
2156     cm->npeers = 0;
2157     INIT_LIST_HEAD(&cm->tx_reqs);
2158 
2159     down_write(&rdev_sem);
2160     list_add_tail(&cm->list, &cm_dev_list);
2161     up_write(&rdev_sem);
2162 
2163     return 0;
2164 }
2165 
2166 /*
2167  * riocm_remove_mport - remove local mport device from channel management core
2168  * @dev: device object associated with mport
2169  * @class_intf: class interface
2170  *
2171  * Removes a local mport device from the list of registered devices that provide
2172  * channel management services. Returns an error if the specified mport is not
2173  * registered with the CM core.
2174  */
2175 static void riocm_remove_mport(struct device *dev,
2176                    struct class_interface *class_intf)
2177 {
2178     struct rio_mport *mport = to_rio_mport(dev);
2179     struct cm_dev *cm;
2180     struct cm_peer *peer, *temp;
2181     struct rio_channel *ch, *_c;
2182     unsigned int i;
2183     bool found = false;
2184     LIST_HEAD(list);
2185 
2186     riocm_debug(MPORT, "%s", mport->name);
2187 
2188     /* Find a matching cm_dev object */
2189     down_write(&rdev_sem);
2190     list_for_each_entry(cm, &cm_dev_list, list) {
2191         if (cm->mport == mport) {
2192             list_del(&cm->list);
2193             found = true;
2194             break;
2195         }
2196     }
2197     up_write(&rdev_sem);
2198     if (!found)
2199         return;
2200 
2201     flush_workqueue(cm->rx_wq);
2202     destroy_workqueue(cm->rx_wq);
2203 
2204     /* Release channels bound to this mport */
2205     spin_lock_bh(&idr_lock);
2206     idr_for_each_entry(&ch_idr, ch, i) {
2207         if (ch->cmdev == cm) {
2208             riocm_debug(RDEV, "%s drop ch_%d",
2209                     mport->name, ch->id);
2210             idr_remove(&ch_idr, ch->id);
2211             list_add(&ch->ch_node, &list);
2212         }
2213     }
2214     spin_unlock_bh(&idr_lock);
2215 
2216     if (!list_empty(&list)) {
2217         list_for_each_entry_safe(ch, _c, &list, ch_node) {
2218             list_del(&ch->ch_node);
2219             riocm_ch_close(ch);
2220         }
2221     }
2222 
2223     rio_release_inb_mbox(mport, cmbox);
2224     rio_release_outb_mbox(mport, cmbox);
2225 
2226     /* Remove and free peer entries */
2227     if (!list_empty(&cm->peers))
2228         riocm_debug(RDEV, "ATTN: peer list not empty");
2229     list_for_each_entry_safe(peer, temp, &cm->peers, node) {
2230         riocm_debug(RDEV, "removing peer %s", rio_name(peer->rdev));
2231         list_del(&peer->node);
2232         kfree(peer);
2233     }
2234 
2235     riocm_rx_free(cm);
2236     kfree(cm);
2237     riocm_debug(MPORT, "%s done", mport->name);
2238 }
2239 
2240 static int rio_cm_shutdown(struct notifier_block *nb, unsigned long code,
2241     void *unused)
2242 {
2243     struct rio_channel *ch;
2244     unsigned int i;
2245     LIST_HEAD(list);
2246 
2247     riocm_debug(EXIT, ".");
2248 
2249     /*
2250      * If there are any channels left in connected state send
2251      * close notification to the connection partner.
2252      * First build a list of channels that require a closing
2253      * notification because function riocm_send_close() should
2254      * be called outside of spinlock protected code.
2255      */
2256     spin_lock_bh(&idr_lock);
2257     idr_for_each_entry(&ch_idr, ch, i) {
2258         if (ch->state == RIO_CM_CONNECTED) {
2259             riocm_debug(EXIT, "close ch %d", ch->id);
2260             idr_remove(&ch_idr, ch->id);
2261             list_add(&ch->ch_node, &list);
2262         }
2263     }
2264     spin_unlock_bh(&idr_lock);
2265 
2266     list_for_each_entry(ch, &list, ch_node)
2267         riocm_send_close(ch);
2268 
2269     return NOTIFY_DONE;
2270 }
2271 
2272 /*
2273  * riocm_interface handles addition/removal of remote RapidIO devices
2274  */
2275 static struct subsys_interface riocm_interface = {
2276     .name       = "rio_cm",
2277     .subsys     = &rio_bus_type,
2278     .add_dev    = riocm_add_dev,
2279     .remove_dev = riocm_remove_dev,
2280 };
2281 
2282 /*
2283  * rio_mport_interface handles addition/removal local mport devices
2284  */
2285 static struct class_interface rio_mport_interface __refdata = {
2286     .class = &rio_mport_class,
2287     .add_dev = riocm_add_mport,
2288     .remove_dev = riocm_remove_mport,
2289 };
2290 
2291 static struct notifier_block rio_cm_notifier = {
2292     .notifier_call = rio_cm_shutdown,
2293 };
2294 
2295 static int __init riocm_init(void)
2296 {
2297     int ret;
2298 
2299     /* Create device class needed by udev */
2300     dev_class = class_create(THIS_MODULE, DRV_NAME);
2301     if (IS_ERR(dev_class)) {
2302         riocm_error("Cannot create " DRV_NAME " class");
2303         return PTR_ERR(dev_class);
2304     }
2305 
2306     ret = alloc_chrdev_region(&dev_number, 0, 1, DRV_NAME);
2307     if (ret) {
2308         class_destroy(dev_class);
2309         return ret;
2310     }
2311 
2312     dev_major = MAJOR(dev_number);
2313     dev_minor_base = MINOR(dev_number);
2314     riocm_debug(INIT, "Registered class with %d major", dev_major);
2315 
2316     /*
2317      * Register as rapidio_port class interface to get notifications about
2318      * mport additions and removals.
2319      */
2320     ret = class_interface_register(&rio_mport_interface);
2321     if (ret) {
2322         riocm_error("class_interface_register error: %d", ret);
2323         goto err_reg;
2324     }
2325 
2326     /*
2327      * Register as RapidIO bus interface to get notifications about
2328      * addition/removal of remote RapidIO devices.
2329      */
2330     ret = subsys_interface_register(&riocm_interface);
2331     if (ret) {
2332         riocm_error("subsys_interface_register error: %d", ret);
2333         goto err_cl;
2334     }
2335 
2336     ret = register_reboot_notifier(&rio_cm_notifier);
2337     if (ret) {
2338         riocm_error("failed to register reboot notifier (err=%d)", ret);
2339         goto err_sif;
2340     }
2341 
2342     ret = riocm_cdev_add(dev_number);
2343     if (ret) {
2344         unregister_reboot_notifier(&rio_cm_notifier);
2345         ret = -ENODEV;
2346         goto err_sif;
2347     }
2348 
2349     return 0;
2350 err_sif:
2351     subsys_interface_unregister(&riocm_interface);
2352 err_cl:
2353     class_interface_unregister(&rio_mport_interface);
2354 err_reg:
2355     unregister_chrdev_region(dev_number, 1);
2356     class_destroy(dev_class);
2357     return ret;
2358 }
2359 
2360 static void __exit riocm_exit(void)
2361 {
2362     riocm_debug(EXIT, "enter");
2363     unregister_reboot_notifier(&rio_cm_notifier);
2364     subsys_interface_unregister(&riocm_interface);
2365     class_interface_unregister(&rio_mport_interface);
2366     idr_destroy(&ch_idr);
2367 
2368     device_unregister(riocm_cdev.dev);
2369     cdev_del(&(riocm_cdev.cdev));
2370 
2371     class_destroy(dev_class);
2372     unregister_chrdev_region(dev_number, 1);
2373 }
2374 
2375 late_initcall(riocm_init);
2376 module_exit(riocm_exit);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!