Back to home page

OSCL-LXR
 
 

     

0001 /* Copyright (c) 2013 Coraid, Inc.  See COPYING for GPL terms. */
0002 /*
0003  * aoecmd.c
0004  * Filesystem request handling methods
0005  */
0006 
0007 #include <linux/ata.h>
0008 #include <linux/slab.h>
0009 #include <linux/hdreg.h>
0010 #include <linux/blk-mq.h>
0011 #include <linux/skbuff.h>
0012 #include <linux/netdevice.h>
0013 #include <linux/moduleparam.h>
0014 #include <linux/workqueue.h>
0015 #include <linux/kthread.h>
0016 #include <net/net_namespace.h>
0017 #include <asm/unaligned.h>
0018 #include <linux/uio.h>
0019 #include "aoe.h"
0020 
0021 #define MAXIOC (8192)   /* default meant to avoid most soft lockups */
0022 
0023 static void ktcomplete(struct frame *, struct sk_buff *);
0024 static int count_targets(struct aoedev *d, int *untainted);
0025 
0026 static struct buf *nextbuf(struct aoedev *);
0027 
0028 static int aoe_deadsecs = 60 * 3;
0029 module_param(aoe_deadsecs, int, 0644);
0030 MODULE_PARM_DESC(aoe_deadsecs, "After aoe_deadsecs seconds, give up and fail dev.");
0031 
0032 static int aoe_maxout = 64;
0033 module_param(aoe_maxout, int, 0644);
0034 MODULE_PARM_DESC(aoe_maxout,
0035     "Only aoe_maxout outstanding packets for every MAC on eX.Y.");
0036 
0037 /* The number of online cpus during module initialization gives us a
0038  * convenient heuristic cap on the parallelism used for ktio threads
0039  * doing I/O completion.  It is not important that the cap equal the
0040  * actual number of running CPUs at any given time, but because of CPU
0041  * hotplug, we take care to use ncpus instead of using
0042  * num_online_cpus() after module initialization.
0043  */
0044 static int ncpus;
0045 
0046 /* mutex lock used for synchronization while thread spawning */
0047 static DEFINE_MUTEX(ktio_spawn_lock);
0048 
0049 static wait_queue_head_t *ktiowq;
0050 static struct ktstate *kts;
0051 
0052 /* io completion queue */
0053 struct iocq_ktio {
0054     struct list_head head;
0055     spinlock_t lock;
0056 };
0057 static struct iocq_ktio *iocq;
0058 
0059 static struct page *empty_page;
0060 
0061 static struct sk_buff *
0062 new_skb(ulong len)
0063 {
0064     struct sk_buff *skb;
0065 
0066     skb = alloc_skb(len + MAX_HEADER, GFP_ATOMIC);
0067     if (skb) {
0068         skb_reserve(skb, MAX_HEADER);
0069         skb_reset_mac_header(skb);
0070         skb_reset_network_header(skb);
0071         skb->protocol = __constant_htons(ETH_P_AOE);
0072         skb_checksum_none_assert(skb);
0073     }
0074     return skb;
0075 }
0076 
0077 static struct frame *
0078 getframe_deferred(struct aoedev *d, u32 tag)
0079 {
0080     struct list_head *head, *pos, *nx;
0081     struct frame *f;
0082 
0083     head = &d->rexmitq;
0084     list_for_each_safe(pos, nx, head) {
0085         f = list_entry(pos, struct frame, head);
0086         if (f->tag == tag) {
0087             list_del(pos);
0088             return f;
0089         }
0090     }
0091     return NULL;
0092 }
0093 
0094 static struct frame *
0095 getframe(struct aoedev *d, u32 tag)
0096 {
0097     struct frame *f;
0098     struct list_head *head, *pos, *nx;
0099     u32 n;
0100 
0101     n = tag % NFACTIVE;
0102     head = &d->factive[n];
0103     list_for_each_safe(pos, nx, head) {
0104         f = list_entry(pos, struct frame, head);
0105         if (f->tag == tag) {
0106             list_del(pos);
0107             return f;
0108         }
0109     }
0110     return NULL;
0111 }
0112 
0113 /*
0114  * Leave the top bit clear so we have tagspace for userland.
0115  * The bottom 16 bits are the xmit tick for rexmit/rttavg processing.
0116  * This driver reserves tag -1 to mean "unused frame."
0117  */
0118 static int
0119 newtag(struct aoedev *d)
0120 {
0121     register ulong n;
0122 
0123     n = jiffies & 0xffff;
0124     return n | (++d->lasttag & 0x7fff) << 16;
0125 }
0126 
0127 static u32
0128 aoehdr_atainit(struct aoedev *d, struct aoetgt *t, struct aoe_hdr *h)
0129 {
0130     u32 host_tag = newtag(d);
0131 
0132     memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
0133     memcpy(h->dst, t->addr, sizeof h->dst);
0134     h->type = __constant_cpu_to_be16(ETH_P_AOE);
0135     h->verfl = AOE_HVER;
0136     h->major = cpu_to_be16(d->aoemajor);
0137     h->minor = d->aoeminor;
0138     h->cmd = AOECMD_ATA;
0139     h->tag = cpu_to_be32(host_tag);
0140 
0141     return host_tag;
0142 }
0143 
0144 static inline void
0145 put_lba(struct aoe_atahdr *ah, sector_t lba)
0146 {
0147     ah->lba0 = lba;
0148     ah->lba1 = lba >>= 8;
0149     ah->lba2 = lba >>= 8;
0150     ah->lba3 = lba >>= 8;
0151     ah->lba4 = lba >>= 8;
0152     ah->lba5 = lba >>= 8;
0153 }
0154 
0155 static struct aoeif *
0156 ifrotate(struct aoetgt *t)
0157 {
0158     struct aoeif *ifp;
0159 
0160     ifp = t->ifp;
0161     ifp++;
0162     if (ifp >= &t->ifs[NAOEIFS] || ifp->nd == NULL)
0163         ifp = t->ifs;
0164     if (ifp->nd == NULL)
0165         return NULL;
0166     return t->ifp = ifp;
0167 }
0168 
0169 static void
0170 skb_pool_put(struct aoedev *d, struct sk_buff *skb)
0171 {
0172     __skb_queue_tail(&d->skbpool, skb);
0173 }
0174 
0175 static struct sk_buff *
0176 skb_pool_get(struct aoedev *d)
0177 {
0178     struct sk_buff *skb = skb_peek(&d->skbpool);
0179 
0180     if (skb && atomic_read(&skb_shinfo(skb)->dataref) == 1) {
0181         __skb_unlink(skb, &d->skbpool);
0182         return skb;
0183     }
0184     if (skb_queue_len(&d->skbpool) < NSKBPOOLMAX &&
0185         (skb = new_skb(ETH_ZLEN)))
0186         return skb;
0187 
0188     return NULL;
0189 }
0190 
0191 void
0192 aoe_freetframe(struct frame *f)
0193 {
0194     struct aoetgt *t;
0195 
0196     t = f->t;
0197     f->buf = NULL;
0198     memset(&f->iter, 0, sizeof(f->iter));
0199     f->r_skb = NULL;
0200     f->flags = 0;
0201     list_add(&f->head, &t->ffree);
0202 }
0203 
0204 static struct frame *
0205 newtframe(struct aoedev *d, struct aoetgt *t)
0206 {
0207     struct frame *f;
0208     struct sk_buff *skb;
0209     struct list_head *pos;
0210 
0211     if (list_empty(&t->ffree)) {
0212         if (t->falloc >= NSKBPOOLMAX*2)
0213             return NULL;
0214         f = kcalloc(1, sizeof(*f), GFP_ATOMIC);
0215         if (f == NULL)
0216             return NULL;
0217         t->falloc++;
0218         f->t = t;
0219     } else {
0220         pos = t->ffree.next;
0221         list_del(pos);
0222         f = list_entry(pos, struct frame, head);
0223     }
0224 
0225     skb = f->skb;
0226     if (skb == NULL) {
0227         f->skb = skb = new_skb(ETH_ZLEN);
0228         if (!skb) {
0229 bail:           aoe_freetframe(f);
0230             return NULL;
0231         }
0232     }
0233 
0234     if (atomic_read(&skb_shinfo(skb)->dataref) != 1) {
0235         skb = skb_pool_get(d);
0236         if (skb == NULL)
0237             goto bail;
0238         skb_pool_put(d, f->skb);
0239         f->skb = skb;
0240     }
0241 
0242     skb->truesize -= skb->data_len;
0243     skb_shinfo(skb)->nr_frags = skb->data_len = 0;
0244     skb_trim(skb, 0);
0245     return f;
0246 }
0247 
0248 static struct frame *
0249 newframe(struct aoedev *d)
0250 {
0251     struct frame *f;
0252     struct aoetgt *t, **tt;
0253     int totout = 0;
0254     int use_tainted;
0255     int has_untainted;
0256 
0257     if (!d->targets || !d->targets[0]) {
0258         printk(KERN_ERR "aoe: NULL TARGETS!\n");
0259         return NULL;
0260     }
0261     tt = d->tgt;    /* last used target */
0262     for (use_tainted = 0, has_untainted = 0;;) {
0263         tt++;
0264         if (tt >= &d->targets[d->ntargets] || !*tt)
0265             tt = d->targets;
0266         t = *tt;
0267         if (!t->taint) {
0268             has_untainted = 1;
0269             totout += t->nout;
0270         }
0271         if (t->nout < t->maxout
0272         && (use_tainted || !t->taint)
0273         && t->ifp->nd) {
0274             f = newtframe(d, t);
0275             if (f) {
0276                 ifrotate(t);
0277                 d->tgt = tt;
0278                 return f;
0279             }
0280         }
0281         if (tt == d->tgt) { /* we've looped and found nada */
0282             if (!use_tainted && !has_untainted)
0283                 use_tainted = 1;
0284             else
0285                 break;
0286         }
0287     }
0288     if (totout == 0) {
0289         d->kicked++;
0290         d->flags |= DEVFL_KICKME;
0291     }
0292     return NULL;
0293 }
0294 
0295 static void
0296 skb_fillup(struct sk_buff *skb, struct bio *bio, struct bvec_iter iter)
0297 {
0298     int frag = 0;
0299     struct bio_vec bv;
0300 
0301     __bio_for_each_segment(bv, bio, iter, iter)
0302         skb_fill_page_desc(skb, frag++, bv.bv_page,
0303                    bv.bv_offset, bv.bv_len);
0304 }
0305 
0306 static void
0307 fhash(struct frame *f)
0308 {
0309     struct aoedev *d = f->t->d;
0310     u32 n;
0311 
0312     n = f->tag % NFACTIVE;
0313     list_add_tail(&f->head, &d->factive[n]);
0314 }
0315 
0316 static void
0317 ata_rw_frameinit(struct frame *f)
0318 {
0319     struct aoetgt *t;
0320     struct aoe_hdr *h;
0321     struct aoe_atahdr *ah;
0322     struct sk_buff *skb;
0323     char writebit, extbit;
0324 
0325     skb = f->skb;
0326     h = (struct aoe_hdr *) skb_mac_header(skb);
0327     ah = (struct aoe_atahdr *) (h + 1);
0328     skb_put(skb, sizeof(*h) + sizeof(*ah));
0329     memset(h, 0, skb->len);
0330 
0331     writebit = 0x10;
0332     extbit = 0x4;
0333 
0334     t = f->t;
0335     f->tag = aoehdr_atainit(t->d, t, h);
0336     fhash(f);
0337     t->nout++;
0338     f->waited = 0;
0339     f->waited_total = 0;
0340 
0341     /* set up ata header */
0342     ah->scnt = f->iter.bi_size >> 9;
0343     put_lba(ah, f->iter.bi_sector);
0344     if (t->d->flags & DEVFL_EXT) {
0345         ah->aflags |= AOEAFL_EXT;
0346     } else {
0347         extbit = 0;
0348         ah->lba3 &= 0x0f;
0349         ah->lba3 |= 0xe0;   /* LBA bit + obsolete 0xa0 */
0350     }
0351     if (f->buf && bio_data_dir(f->buf->bio) == WRITE) {
0352         skb_fillup(skb, f->buf->bio, f->iter);
0353         ah->aflags |= AOEAFL_WRITE;
0354         skb->len += f->iter.bi_size;
0355         skb->data_len = f->iter.bi_size;
0356         skb->truesize += f->iter.bi_size;
0357         t->wpkts++;
0358     } else {
0359         t->rpkts++;
0360         writebit = 0;
0361     }
0362 
0363     ah->cmdstat = ATA_CMD_PIO_READ | writebit | extbit;
0364     skb->dev = t->ifp->nd;
0365 }
0366 
0367 static int
0368 aoecmd_ata_rw(struct aoedev *d)
0369 {
0370     struct frame *f;
0371     struct buf *buf;
0372     struct sk_buff *skb;
0373     struct sk_buff_head queue;
0374 
0375     buf = nextbuf(d);
0376     if (buf == NULL)
0377         return 0;
0378     f = newframe(d);
0379     if (f == NULL)
0380         return 0;
0381 
0382     /* initialize the headers & frame */
0383     f->buf = buf;
0384     f->iter = buf->iter;
0385     f->iter.bi_size = min_t(unsigned long,
0386                 d->maxbcnt ?: DEFAULTBCNT,
0387                 f->iter.bi_size);
0388     bio_advance_iter(buf->bio, &buf->iter, f->iter.bi_size);
0389 
0390     if (!buf->iter.bi_size)
0391         d->ip.buf = NULL;
0392 
0393     /* mark all tracking fields and load out */
0394     buf->nframesout += 1;
0395 
0396     ata_rw_frameinit(f);
0397 
0398     skb = skb_clone(f->skb, GFP_ATOMIC);
0399     if (skb) {
0400         f->sent = ktime_get();
0401         __skb_queue_head_init(&queue);
0402         __skb_queue_tail(&queue, skb);
0403         aoenet_xmit(&queue);
0404     }
0405     return 1;
0406 }
0407 
0408 /* some callers cannot sleep, and they can call this function,
0409  * transmitting the packets later, when interrupts are on
0410  */
0411 static void
0412 aoecmd_cfg_pkts(ushort aoemajor, unsigned char aoeminor, struct sk_buff_head *queue)
0413 {
0414     struct aoe_hdr *h;
0415     struct aoe_cfghdr *ch;
0416     struct sk_buff *skb;
0417     struct net_device *ifp;
0418 
0419     rcu_read_lock();
0420     for_each_netdev_rcu(&init_net, ifp) {
0421         dev_hold(ifp);
0422         if (!is_aoe_netif(ifp))
0423             goto cont;
0424 
0425         skb = new_skb(sizeof *h + sizeof *ch);
0426         if (skb == NULL) {
0427             printk(KERN_INFO "aoe: skb alloc failure\n");
0428             goto cont;
0429         }
0430         skb_put(skb, sizeof *h + sizeof *ch);
0431         skb->dev = ifp;
0432         __skb_queue_tail(queue, skb);
0433         h = (struct aoe_hdr *) skb_mac_header(skb);
0434         memset(h, 0, sizeof *h + sizeof *ch);
0435 
0436         memset(h->dst, 0xff, sizeof h->dst);
0437         memcpy(h->src, ifp->dev_addr, sizeof h->src);
0438         h->type = __constant_cpu_to_be16(ETH_P_AOE);
0439         h->verfl = AOE_HVER;
0440         h->major = cpu_to_be16(aoemajor);
0441         h->minor = aoeminor;
0442         h->cmd = AOECMD_CFG;
0443 
0444 cont:
0445         dev_put(ifp);
0446     }
0447     rcu_read_unlock();
0448 }
0449 
0450 static void
0451 resend(struct aoedev *d, struct frame *f)
0452 {
0453     struct sk_buff *skb;
0454     struct sk_buff_head queue;
0455     struct aoe_hdr *h;
0456     struct aoetgt *t;
0457     char buf[128];
0458     u32 n;
0459 
0460     t = f->t;
0461     n = newtag(d);
0462     skb = f->skb;
0463     if (ifrotate(t) == NULL) {
0464         /* probably can't happen, but set it up to fail anyway */
0465         pr_info("aoe: resend: no interfaces to rotate to.\n");
0466         ktcomplete(f, NULL);
0467         return;
0468     }
0469     h = (struct aoe_hdr *) skb_mac_header(skb);
0470 
0471     if (!(f->flags & FFL_PROBE)) {
0472         snprintf(buf, sizeof(buf),
0473             "%15s e%ld.%d oldtag=%08x@%08lx newtag=%08x s=%pm d=%pm nout=%d\n",
0474             "retransmit", d->aoemajor, d->aoeminor,
0475             f->tag, jiffies, n,
0476             h->src, h->dst, t->nout);
0477         aoechr_error(buf);
0478     }
0479 
0480     f->tag = n;
0481     fhash(f);
0482     h->tag = cpu_to_be32(n);
0483     memcpy(h->dst, t->addr, sizeof h->dst);
0484     memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
0485 
0486     skb->dev = t->ifp->nd;
0487     skb = skb_clone(skb, GFP_ATOMIC);
0488     if (skb == NULL)
0489         return;
0490     f->sent = ktime_get();
0491     __skb_queue_head_init(&queue);
0492     __skb_queue_tail(&queue, skb);
0493     aoenet_xmit(&queue);
0494 }
0495 
0496 static int
0497 tsince_hr(struct frame *f)
0498 {
0499     u64 delta = ktime_to_ns(ktime_sub(ktime_get(), f->sent));
0500 
0501     /* delta is normally under 4.2 seconds, avoid 64-bit division */
0502     if (likely(delta <= UINT_MAX))
0503         return (u32)delta / NSEC_PER_USEC;
0504 
0505     /* avoid overflow after 71 minutes */
0506     if (delta > ((u64)INT_MAX * NSEC_PER_USEC))
0507         return INT_MAX;
0508 
0509     return div_u64(delta, NSEC_PER_USEC);
0510 }
0511 
0512 static int
0513 tsince(u32 tag)
0514 {
0515     int n;
0516 
0517     n = jiffies & 0xffff;
0518     n -= tag & 0xffff;
0519     if (n < 0)
0520         n += 1<<16;
0521     return jiffies_to_usecs(n + 1);
0522 }
0523 
0524 static struct aoeif *
0525 getif(struct aoetgt *t, struct net_device *nd)
0526 {
0527     struct aoeif *p, *e;
0528 
0529     p = t->ifs;
0530     e = p + NAOEIFS;
0531     for (; p < e; p++)
0532         if (p->nd == nd)
0533             return p;
0534     return NULL;
0535 }
0536 
0537 static void
0538 ejectif(struct aoetgt *t, struct aoeif *ifp)
0539 {
0540     struct aoeif *e;
0541     struct net_device *nd;
0542     ulong n;
0543 
0544     nd = ifp->nd;
0545     e = t->ifs + NAOEIFS - 1;
0546     n = (e - ifp) * sizeof *ifp;
0547     memmove(ifp, ifp+1, n);
0548     e->nd = NULL;
0549     dev_put(nd);
0550 }
0551 
0552 static struct frame *
0553 reassign_frame(struct frame *f)
0554 {
0555     struct frame *nf;
0556     struct sk_buff *skb;
0557 
0558     nf = newframe(f->t->d);
0559     if (!nf)
0560         return NULL;
0561     if (nf->t == f->t) {
0562         aoe_freetframe(nf);
0563         return NULL;
0564     }
0565 
0566     skb = nf->skb;
0567     nf->skb = f->skb;
0568     nf->buf = f->buf;
0569     nf->iter = f->iter;
0570     nf->waited = 0;
0571     nf->waited_total = f->waited_total;
0572     nf->sent = f->sent;
0573     f->skb = skb;
0574 
0575     return nf;
0576 }
0577 
0578 static void
0579 probe(struct aoetgt *t)
0580 {
0581     struct aoedev *d;
0582     struct frame *f;
0583     struct sk_buff *skb;
0584     struct sk_buff_head queue;
0585     size_t n, m;
0586     int frag;
0587 
0588     d = t->d;
0589     f = newtframe(d, t);
0590     if (!f) {
0591         pr_err("%s %pm for e%ld.%d: %s\n",
0592             "aoe: cannot probe remote address",
0593             t->addr,
0594             (long) d->aoemajor, d->aoeminor,
0595             "no frame available");
0596         return;
0597     }
0598     f->flags |= FFL_PROBE;
0599     ifrotate(t);
0600     f->iter.bi_size = t->d->maxbcnt ? t->d->maxbcnt : DEFAULTBCNT;
0601     ata_rw_frameinit(f);
0602     skb = f->skb;
0603     for (frag = 0, n = f->iter.bi_size; n > 0; ++frag, n -= m) {
0604         if (n < PAGE_SIZE)
0605             m = n;
0606         else
0607             m = PAGE_SIZE;
0608         skb_fill_page_desc(skb, frag, empty_page, 0, m);
0609     }
0610     skb->len += f->iter.bi_size;
0611     skb->data_len = f->iter.bi_size;
0612     skb->truesize += f->iter.bi_size;
0613 
0614     skb = skb_clone(f->skb, GFP_ATOMIC);
0615     if (skb) {
0616         f->sent = ktime_get();
0617         __skb_queue_head_init(&queue);
0618         __skb_queue_tail(&queue, skb);
0619         aoenet_xmit(&queue);
0620     }
0621 }
0622 
0623 static long
0624 rto(struct aoedev *d)
0625 {
0626     long t;
0627 
0628     t = 2 * d->rttavg >> RTTSCALE;
0629     t += 8 * d->rttdev >> RTTDSCALE;
0630     if (t == 0)
0631         t = 1;
0632 
0633     return t;
0634 }
0635 
0636 static void
0637 rexmit_deferred(struct aoedev *d)
0638 {
0639     struct aoetgt *t;
0640     struct frame *f;
0641     struct frame *nf;
0642     struct list_head *pos, *nx, *head;
0643     int since;
0644     int untainted;
0645 
0646     count_targets(d, &untainted);
0647 
0648     head = &d->rexmitq;
0649     list_for_each_safe(pos, nx, head) {
0650         f = list_entry(pos, struct frame, head);
0651         t = f->t;
0652         if (t->taint) {
0653             if (!(f->flags & FFL_PROBE)) {
0654                 nf = reassign_frame(f);
0655                 if (nf) {
0656                     if (t->nout_probes == 0
0657                     && untainted > 0) {
0658                         probe(t);
0659                         t->nout_probes++;
0660                     }
0661                     list_replace(&f->head, &nf->head);
0662                     pos = &nf->head;
0663                     aoe_freetframe(f);
0664                     f = nf;
0665                     t = f->t;
0666                 }
0667             } else if (untainted < 1) {
0668                 /* don't probe w/o other untainted aoetgts */
0669                 goto stop_probe;
0670             } else if (tsince_hr(f) < t->taint * rto(d)) {
0671                 /* reprobe slowly when taint is high */
0672                 continue;
0673             }
0674         } else if (f->flags & FFL_PROBE) {
0675 stop_probe:     /* don't probe untainted aoetgts */
0676             list_del(pos);
0677             aoe_freetframe(f);
0678             /* leaving d->kicked, because this is routine */
0679             f->t->d->flags |= DEVFL_KICKME;
0680             continue;
0681         }
0682         if (t->nout >= t->maxout)
0683             continue;
0684         list_del(pos);
0685         t->nout++;
0686         if (f->flags & FFL_PROBE)
0687             t->nout_probes++;
0688         since = tsince_hr(f);
0689         f->waited += since;
0690         f->waited_total += since;
0691         resend(d, f);
0692     }
0693 }
0694 
0695 /* An aoetgt accumulates demerits quickly, and successful
0696  * probing redeems the aoetgt slowly.
0697  */
0698 static void
0699 scorn(struct aoetgt *t)
0700 {
0701     int n;
0702 
0703     n = t->taint++;
0704     t->taint += t->taint * 2;
0705     if (n > t->taint)
0706         t->taint = n;
0707     if (t->taint > MAX_TAINT)
0708         t->taint = MAX_TAINT;
0709 }
0710 
0711 static int
0712 count_targets(struct aoedev *d, int *untainted)
0713 {
0714     int i, good;
0715 
0716     for (i = good = 0; i < d->ntargets && d->targets[i]; ++i)
0717         if (d->targets[i]->taint == 0)
0718             good++;
0719 
0720     if (untainted)
0721         *untainted = good;
0722     return i;
0723 }
0724 
0725 static void
0726 rexmit_timer(struct timer_list *timer)
0727 {
0728     struct aoedev *d;
0729     struct aoetgt *t;
0730     struct aoeif *ifp;
0731     struct frame *f;
0732     struct list_head *head, *pos, *nx;
0733     LIST_HEAD(flist);
0734     register long timeout;
0735     ulong flags, n;
0736     int i;
0737     int utgts;  /* number of aoetgt descriptors (not slots) */
0738     int since;
0739 
0740     d = from_timer(d, timer, timer);
0741 
0742     spin_lock_irqsave(&d->lock, flags);
0743 
0744     /* timeout based on observed timings and variations */
0745     timeout = rto(d);
0746 
0747     utgts = count_targets(d, NULL);
0748 
0749     if (d->flags & DEVFL_TKILL) {
0750         spin_unlock_irqrestore(&d->lock, flags);
0751         return;
0752     }
0753 
0754     /* collect all frames to rexmit into flist */
0755     for (i = 0; i < NFACTIVE; i++) {
0756         head = &d->factive[i];
0757         list_for_each_safe(pos, nx, head) {
0758             f = list_entry(pos, struct frame, head);
0759             if (tsince_hr(f) < timeout)
0760                 break;  /* end of expired frames */
0761             /* move to flist for later processing */
0762             list_move_tail(pos, &flist);
0763         }
0764     }
0765 
0766     /* process expired frames */
0767     while (!list_empty(&flist)) {
0768         pos = flist.next;
0769         f = list_entry(pos, struct frame, head);
0770         since = tsince_hr(f);
0771         n = f->waited_total + since;
0772         n /= USEC_PER_SEC;
0773         if (aoe_deadsecs
0774         && n > aoe_deadsecs
0775         && !(f->flags & FFL_PROBE)) {
0776             /* Waited too long.  Device failure.
0777              * Hang all frames on first hash bucket for downdev
0778              * to clean up.
0779              */
0780             list_splice(&flist, &d->factive[0]);
0781             aoedev_downdev(d);
0782             goto out;
0783         }
0784 
0785         t = f->t;
0786         n = f->waited + since;
0787         n /= USEC_PER_SEC;
0788         if (aoe_deadsecs && utgts > 0
0789         && (n > aoe_deadsecs / utgts || n > HARD_SCORN_SECS))
0790             scorn(t); /* avoid this target */
0791 
0792         if (t->maxout != 1) {
0793             t->ssthresh = t->maxout / 2;
0794             t->maxout = 1;
0795         }
0796 
0797         if (f->flags & FFL_PROBE) {
0798             t->nout_probes--;
0799         } else {
0800             ifp = getif(t, f->skb->dev);
0801             if (ifp && ++ifp->lost > (t->nframes << 1)
0802             && (ifp != t->ifs || t->ifs[1].nd)) {
0803                 ejectif(t, ifp);
0804                 ifp = NULL;
0805             }
0806         }
0807         list_move_tail(pos, &d->rexmitq);
0808         t->nout--;
0809     }
0810     rexmit_deferred(d);
0811 
0812 out:
0813     if ((d->flags & DEVFL_KICKME) && d->blkq) {
0814         d->flags &= ~DEVFL_KICKME;
0815         blk_mq_run_hw_queues(d->blkq, true);
0816     }
0817 
0818     d->timer.expires = jiffies + TIMERTICK;
0819     add_timer(&d->timer);
0820 
0821     spin_unlock_irqrestore(&d->lock, flags);
0822 }
0823 
0824 static void
0825 bufinit(struct buf *buf, struct request *rq, struct bio *bio)
0826 {
0827     memset(buf, 0, sizeof(*buf));
0828     buf->rq = rq;
0829     buf->bio = bio;
0830     buf->iter = bio->bi_iter;
0831 }
0832 
0833 static struct buf *
0834 nextbuf(struct aoedev *d)
0835 {
0836     struct request *rq;
0837     struct request_queue *q;
0838     struct aoe_req *req;
0839     struct buf *buf;
0840     struct bio *bio;
0841 
0842     q = d->blkq;
0843     if (q == NULL)
0844         return NULL;    /* initializing */
0845     if (d->ip.buf)
0846         return d->ip.buf;
0847     rq = d->ip.rq;
0848     if (rq == NULL) {
0849         rq = list_first_entry_or_null(&d->rq_list, struct request,
0850                         queuelist);
0851         if (rq == NULL)
0852             return NULL;
0853         list_del_init(&rq->queuelist);
0854         blk_mq_start_request(rq);
0855         d->ip.rq = rq;
0856         d->ip.nxbio = rq->bio;
0857 
0858         req = blk_mq_rq_to_pdu(rq);
0859         req->nr_bios = 0;
0860         __rq_for_each_bio(bio, rq)
0861             req->nr_bios++;
0862     }
0863     buf = mempool_alloc(d->bufpool, GFP_ATOMIC);
0864     if (buf == NULL) {
0865         pr_err("aoe: nextbuf: unable to mempool_alloc!\n");
0866         return NULL;
0867     }
0868     bio = d->ip.nxbio;
0869     bufinit(buf, rq, bio);
0870     bio = bio->bi_next;
0871     d->ip.nxbio = bio;
0872     if (bio == NULL)
0873         d->ip.rq = NULL;
0874     return d->ip.buf = buf;
0875 }
0876 
0877 /* enters with d->lock held */
0878 void
0879 aoecmd_work(struct aoedev *d)
0880 {
0881     rexmit_deferred(d);
0882     while (aoecmd_ata_rw(d))
0883         ;
0884 }
0885 
0886 /* this function performs work that has been deferred until sleeping is OK
0887  */
0888 void
0889 aoecmd_sleepwork(struct work_struct *work)
0890 {
0891     struct aoedev *d = container_of(work, struct aoedev, work);
0892 
0893     if (d->flags & DEVFL_GDALLOC)
0894         aoeblk_gdalloc(d);
0895 
0896     if (d->flags & DEVFL_NEWSIZE) {
0897         set_capacity_and_notify(d->gd, d->ssize);
0898 
0899         spin_lock_irq(&d->lock);
0900         d->flags |= DEVFL_UP;
0901         d->flags &= ~DEVFL_NEWSIZE;
0902         spin_unlock_irq(&d->lock);
0903     }
0904 }
0905 
0906 static void
0907 ata_ident_fixstring(u16 *id, int ns)
0908 {
0909     u16 s;
0910 
0911     while (ns-- > 0) {
0912         s = *id;
0913         *id++ = s >> 8 | s << 8;
0914     }
0915 }
0916 
0917 static void
0918 ataid_complete(struct aoedev *d, struct aoetgt *t, unsigned char *id)
0919 {
0920     u64 ssize;
0921     u16 n;
0922 
0923     /* word 83: command set supported */
0924     n = get_unaligned_le16(&id[83 << 1]);
0925 
0926     /* word 86: command set/feature enabled */
0927     n |= get_unaligned_le16(&id[86 << 1]);
0928 
0929     if (n & (1<<10)) {  /* bit 10: LBA 48 */
0930         d->flags |= DEVFL_EXT;
0931 
0932         /* word 100: number lba48 sectors */
0933         ssize = get_unaligned_le64(&id[100 << 1]);
0934 
0935         /* set as in ide-disk.c:init_idedisk_capacity */
0936         d->geo.cylinders = ssize;
0937         d->geo.cylinders /= (255 * 63);
0938         d->geo.heads = 255;
0939         d->geo.sectors = 63;
0940     } else {
0941         d->flags &= ~DEVFL_EXT;
0942 
0943         /* number lba28 sectors */
0944         ssize = get_unaligned_le32(&id[60 << 1]);
0945 
0946         /* NOTE: obsolete in ATA 6 */
0947         d->geo.cylinders = get_unaligned_le16(&id[54 << 1]);
0948         d->geo.heads = get_unaligned_le16(&id[55 << 1]);
0949         d->geo.sectors = get_unaligned_le16(&id[56 << 1]);
0950     }
0951 
0952     ata_ident_fixstring((u16 *) &id[10<<1], 10);    /* serial */
0953     ata_ident_fixstring((u16 *) &id[23<<1], 4); /* firmware */
0954     ata_ident_fixstring((u16 *) &id[27<<1], 20);    /* model */
0955     memcpy(d->ident, id, sizeof(d->ident));
0956 
0957     if (d->ssize != ssize)
0958         printk(KERN_INFO
0959             "aoe: %pm e%ld.%d v%04x has %llu sectors\n",
0960             t->addr,
0961             d->aoemajor, d->aoeminor,
0962             d->fw_ver, (long long)ssize);
0963     d->ssize = ssize;
0964     d->geo.start = 0;
0965     if (d->flags & (DEVFL_GDALLOC|DEVFL_NEWSIZE))
0966         return;
0967     if (d->gd != NULL)
0968         d->flags |= DEVFL_NEWSIZE;
0969     else
0970         d->flags |= DEVFL_GDALLOC;
0971     queue_work(aoe_wq, &d->work);
0972 }
0973 
0974 static void
0975 calc_rttavg(struct aoedev *d, struct aoetgt *t, int rtt)
0976 {
0977     register long n;
0978 
0979     n = rtt;
0980 
0981     /* cf. Congestion Avoidance and Control, Jacobson & Karels, 1988 */
0982     n -= d->rttavg >> RTTSCALE;
0983     d->rttavg += n;
0984     if (n < 0)
0985         n = -n;
0986     n -= d->rttdev >> RTTDSCALE;
0987     d->rttdev += n;
0988 
0989     if (!t || t->maxout >= t->nframes)
0990         return;
0991     if (t->maxout < t->ssthresh)
0992         t->maxout += 1;
0993     else if (t->nout == t->maxout && t->next_cwnd-- == 0) {
0994         t->maxout += 1;
0995         t->next_cwnd = t->maxout;
0996     }
0997 }
0998 
0999 static struct aoetgt *
1000 gettgt(struct aoedev *d, char *addr)
1001 {
1002     struct aoetgt **t, **e;
1003 
1004     t = d->targets;
1005     e = t + d->ntargets;
1006     for (; t < e && *t; t++)
1007         if (memcmp((*t)->addr, addr, sizeof((*t)->addr)) == 0)
1008             return *t;
1009     return NULL;
1010 }
1011 
1012 static void
1013 bvcpy(struct sk_buff *skb, struct bio *bio, struct bvec_iter iter, long cnt)
1014 {
1015     int soff = 0;
1016     struct bio_vec bv;
1017 
1018     iter.bi_size = cnt;
1019 
1020     __bio_for_each_segment(bv, bio, iter, iter) {
1021         char *p = bvec_kmap_local(&bv);
1022         skb_copy_bits(skb, soff, p, bv.bv_len);
1023         kunmap_local(p);
1024         soff += bv.bv_len;
1025     }
1026 }
1027 
1028 void
1029 aoe_end_request(struct aoedev *d, struct request *rq, int fastfail)
1030 {
1031     struct bio *bio;
1032     int bok;
1033     struct request_queue *q;
1034     blk_status_t err = BLK_STS_OK;
1035 
1036     q = d->blkq;
1037     if (rq == d->ip.rq)
1038         d->ip.rq = NULL;
1039     do {
1040         bio = rq->bio;
1041         bok = !fastfail && !bio->bi_status;
1042         if (!bok)
1043             err = BLK_STS_IOERR;
1044     } while (blk_update_request(rq, bok ? BLK_STS_OK : BLK_STS_IOERR, bio->bi_iter.bi_size));
1045 
1046     __blk_mq_end_request(rq, err);
1047 
1048     /* cf. https://lore.kernel.org/lkml/20061031071040.GS14055@kernel.dk/ */
1049     if (!fastfail)
1050         blk_mq_run_hw_queues(q, true);
1051 }
1052 
1053 static void
1054 aoe_end_buf(struct aoedev *d, struct buf *buf)
1055 {
1056     struct request *rq = buf->rq;
1057     struct aoe_req *req = blk_mq_rq_to_pdu(rq);
1058 
1059     if (buf == d->ip.buf)
1060         d->ip.buf = NULL;
1061     mempool_free(buf, d->bufpool);
1062     if (--req->nr_bios == 0)
1063         aoe_end_request(d, rq, 0);
1064 }
1065 
1066 static void
1067 ktiocomplete(struct frame *f)
1068 {
1069     struct aoe_hdr *hin, *hout;
1070     struct aoe_atahdr *ahin, *ahout;
1071     struct buf *buf;
1072     struct sk_buff *skb;
1073     struct aoetgt *t;
1074     struct aoeif *ifp;
1075     struct aoedev *d;
1076     long n;
1077     int untainted;
1078 
1079     if (f == NULL)
1080         return;
1081 
1082     t = f->t;
1083     d = t->d;
1084     skb = f->r_skb;
1085     buf = f->buf;
1086     if (f->flags & FFL_PROBE)
1087         goto out;
1088     if (!skb)       /* just fail the buf. */
1089         goto noskb;
1090 
1091     hout = (struct aoe_hdr *) skb_mac_header(f->skb);
1092     ahout = (struct aoe_atahdr *) (hout+1);
1093 
1094     hin = (struct aoe_hdr *) skb->data;
1095     skb_pull(skb, sizeof(*hin));
1096     ahin = (struct aoe_atahdr *) skb->data;
1097     skb_pull(skb, sizeof(*ahin));
1098     if (ahin->cmdstat & 0xa9) { /* these bits cleared on success */
1099         pr_err("aoe: ata error cmd=%2.2Xh stat=%2.2Xh from e%ld.%d\n",
1100             ahout->cmdstat, ahin->cmdstat,
1101             d->aoemajor, d->aoeminor);
1102 noskb:      if (buf)
1103             buf->bio->bi_status = BLK_STS_IOERR;
1104         goto out;
1105     }
1106 
1107     n = ahout->scnt << 9;
1108     switch (ahout->cmdstat) {
1109     case ATA_CMD_PIO_READ:
1110     case ATA_CMD_PIO_READ_EXT:
1111         if (skb->len < n) {
1112             pr_err("%s e%ld.%d.  skb->len=%d need=%ld\n",
1113                 "aoe: runt data size in read from",
1114                 (long) d->aoemajor, d->aoeminor,
1115                    skb->len, n);
1116             buf->bio->bi_status = BLK_STS_IOERR;
1117             break;
1118         }
1119         if (n > f->iter.bi_size) {
1120             pr_err_ratelimited("%s e%ld.%d.  bytes=%ld need=%u\n",
1121                 "aoe: too-large data size in read from",
1122                 (long) d->aoemajor, d->aoeminor,
1123                 n, f->iter.bi_size);
1124             buf->bio->bi_status = BLK_STS_IOERR;
1125             break;
1126         }
1127         bvcpy(skb, f->buf->bio, f->iter, n);
1128         fallthrough;
1129     case ATA_CMD_PIO_WRITE:
1130     case ATA_CMD_PIO_WRITE_EXT:
1131         spin_lock_irq(&d->lock);
1132         ifp = getif(t, skb->dev);
1133         if (ifp)
1134             ifp->lost = 0;
1135         spin_unlock_irq(&d->lock);
1136         break;
1137     case ATA_CMD_ID_ATA:
1138         if (skb->len < 512) {
1139             pr_info("%s e%ld.%d.  skb->len=%d need=512\n",
1140                 "aoe: runt data size in ataid from",
1141                 (long) d->aoemajor, d->aoeminor,
1142                 skb->len);
1143             break;
1144         }
1145         if (skb_linearize(skb))
1146             break;
1147         spin_lock_irq(&d->lock);
1148         ataid_complete(d, t, skb->data);
1149         spin_unlock_irq(&d->lock);
1150         break;
1151     default:
1152         pr_info("aoe: unrecognized ata command %2.2Xh for %d.%d\n",
1153             ahout->cmdstat,
1154             be16_to_cpu(get_unaligned(&hin->major)),
1155             hin->minor);
1156     }
1157 out:
1158     spin_lock_irq(&d->lock);
1159     if (t->taint > 0
1160     && --t->taint > 0
1161     && t->nout_probes == 0) {
1162         count_targets(d, &untainted);
1163         if (untainted > 0) {
1164             probe(t);
1165             t->nout_probes++;
1166         }
1167     }
1168 
1169     aoe_freetframe(f);
1170 
1171     if (buf && --buf->nframesout == 0 && buf->iter.bi_size == 0)
1172         aoe_end_buf(d, buf);
1173 
1174     spin_unlock_irq(&d->lock);
1175     aoedev_put(d);
1176     dev_kfree_skb(skb);
1177 }
1178 
1179 /* Enters with iocq.lock held.
1180  * Returns true iff responses needing processing remain.
1181  */
1182 static int
1183 ktio(int id)
1184 {
1185     struct frame *f;
1186     struct list_head *pos;
1187     int i;
1188     int actual_id;
1189 
1190     for (i = 0; ; ++i) {
1191         if (i == MAXIOC)
1192             return 1;
1193         if (list_empty(&iocq[id].head))
1194             return 0;
1195         pos = iocq[id].head.next;
1196         list_del(pos);
1197         f = list_entry(pos, struct frame, head);
1198         spin_unlock_irq(&iocq[id].lock);
1199         ktiocomplete(f);
1200 
1201         /* Figure out if extra threads are required. */
1202         actual_id = f->t->d->aoeminor % ncpus;
1203 
1204         if (!kts[actual_id].active) {
1205             BUG_ON(id != 0);
1206             mutex_lock(&ktio_spawn_lock);
1207             if (!kts[actual_id].active
1208                 && aoe_ktstart(&kts[actual_id]) == 0)
1209                 kts[actual_id].active = 1;
1210             mutex_unlock(&ktio_spawn_lock);
1211         }
1212         spin_lock_irq(&iocq[id].lock);
1213     }
1214 }
1215 
1216 static int
1217 kthread(void *vp)
1218 {
1219     struct ktstate *k;
1220     DECLARE_WAITQUEUE(wait, current);
1221     int more;
1222 
1223     k = vp;
1224     current->flags |= PF_NOFREEZE;
1225     set_user_nice(current, -10);
1226     complete(&k->rendez);   /* tell spawner we're running */
1227     do {
1228         spin_lock_irq(k->lock);
1229         more = k->fn(k->id);
1230         if (!more) {
1231             add_wait_queue(k->waitq, &wait);
1232             __set_current_state(TASK_INTERRUPTIBLE);
1233         }
1234         spin_unlock_irq(k->lock);
1235         if (!more) {
1236             schedule();
1237             remove_wait_queue(k->waitq, &wait);
1238         } else
1239             cond_resched();
1240     } while (!kthread_should_stop());
1241     complete(&k->rendez);   /* tell spawner we're stopping */
1242     return 0;
1243 }
1244 
1245 void
1246 aoe_ktstop(struct ktstate *k)
1247 {
1248     kthread_stop(k->task);
1249     wait_for_completion(&k->rendez);
1250 }
1251 
1252 int
1253 aoe_ktstart(struct ktstate *k)
1254 {
1255     struct task_struct *task;
1256 
1257     init_completion(&k->rendez);
1258     task = kthread_run(kthread, k, "%s", k->name);
1259     if (task == NULL || IS_ERR(task))
1260         return -ENOMEM;
1261     k->task = task;
1262     wait_for_completion(&k->rendez); /* allow kthread to start */
1263     init_completion(&k->rendez);    /* for waiting for exit later */
1264     return 0;
1265 }
1266 
1267 /* pass it off to kthreads for processing */
1268 static void
1269 ktcomplete(struct frame *f, struct sk_buff *skb)
1270 {
1271     int id;
1272     ulong flags;
1273 
1274     f->r_skb = skb;
1275     id = f->t->d->aoeminor % ncpus;
1276     spin_lock_irqsave(&iocq[id].lock, flags);
1277     if (!kts[id].active) {
1278         spin_unlock_irqrestore(&iocq[id].lock, flags);
1279         /* The thread with id has not been spawned yet,
1280          * so delegate the work to the main thread and
1281          * try spawning a new thread.
1282          */
1283         id = 0;
1284         spin_lock_irqsave(&iocq[id].lock, flags);
1285     }
1286     list_add_tail(&f->head, &iocq[id].head);
1287     spin_unlock_irqrestore(&iocq[id].lock, flags);
1288     wake_up(&ktiowq[id]);
1289 }
1290 
1291 struct sk_buff *
1292 aoecmd_ata_rsp(struct sk_buff *skb)
1293 {
1294     struct aoedev *d;
1295     struct aoe_hdr *h;
1296     struct frame *f;
1297     u32 n;
1298     ulong flags;
1299     char ebuf[128];
1300     u16 aoemajor;
1301 
1302     h = (struct aoe_hdr *) skb->data;
1303     aoemajor = be16_to_cpu(get_unaligned(&h->major));
1304     d = aoedev_by_aoeaddr(aoemajor, h->minor, 0);
1305     if (d == NULL) {
1306         snprintf(ebuf, sizeof ebuf, "aoecmd_ata_rsp: ata response "
1307             "for unknown device %d.%d\n",
1308             aoemajor, h->minor);
1309         aoechr_error(ebuf);
1310         return skb;
1311     }
1312 
1313     spin_lock_irqsave(&d->lock, flags);
1314 
1315     n = be32_to_cpu(get_unaligned(&h->tag));
1316     f = getframe(d, n);
1317     if (f) {
1318         calc_rttavg(d, f->t, tsince_hr(f));
1319         f->t->nout--;
1320         if (f->flags & FFL_PROBE)
1321             f->t->nout_probes--;
1322     } else {
1323         f = getframe_deferred(d, n);
1324         if (f) {
1325             calc_rttavg(d, NULL, tsince_hr(f));
1326         } else {
1327             calc_rttavg(d, NULL, tsince(n));
1328             spin_unlock_irqrestore(&d->lock, flags);
1329             aoedev_put(d);
1330             snprintf(ebuf, sizeof(ebuf),
1331                  "%15s e%d.%d    tag=%08x@%08lx s=%pm d=%pm\n",
1332                  "unexpected rsp",
1333                  get_unaligned_be16(&h->major),
1334                  h->minor,
1335                  get_unaligned_be32(&h->tag),
1336                  jiffies,
1337                  h->src,
1338                  h->dst);
1339             aoechr_error(ebuf);
1340             return skb;
1341         }
1342     }
1343     aoecmd_work(d);
1344 
1345     spin_unlock_irqrestore(&d->lock, flags);
1346 
1347     ktcomplete(f, skb);
1348 
1349     /*
1350      * Note here that we do not perform an aoedev_put, as we are
1351      * leaving this reference for the ktio to release.
1352      */
1353     return NULL;
1354 }
1355 
1356 void
1357 aoecmd_cfg(ushort aoemajor, unsigned char aoeminor)
1358 {
1359     struct sk_buff_head queue;
1360 
1361     __skb_queue_head_init(&queue);
1362     aoecmd_cfg_pkts(aoemajor, aoeminor, &queue);
1363     aoenet_xmit(&queue);
1364 }
1365 
1366 struct sk_buff *
1367 aoecmd_ata_id(struct aoedev *d)
1368 {
1369     struct aoe_hdr *h;
1370     struct aoe_atahdr *ah;
1371     struct frame *f;
1372     struct sk_buff *skb;
1373     struct aoetgt *t;
1374 
1375     f = newframe(d);
1376     if (f == NULL)
1377         return NULL;
1378 
1379     t = *d->tgt;
1380 
1381     /* initialize the headers & frame */
1382     skb = f->skb;
1383     h = (struct aoe_hdr *) skb_mac_header(skb);
1384     ah = (struct aoe_atahdr *) (h+1);
1385     skb_put(skb, sizeof *h + sizeof *ah);
1386     memset(h, 0, skb->len);
1387     f->tag = aoehdr_atainit(d, t, h);
1388     fhash(f);
1389     t->nout++;
1390     f->waited = 0;
1391     f->waited_total = 0;
1392 
1393     /* set up ata header */
1394     ah->scnt = 1;
1395     ah->cmdstat = ATA_CMD_ID_ATA;
1396     ah->lba3 = 0xa0;
1397 
1398     skb->dev = t->ifp->nd;
1399 
1400     d->rttavg = RTTAVG_INIT;
1401     d->rttdev = RTTDEV_INIT;
1402     d->timer.function = rexmit_timer;
1403 
1404     skb = skb_clone(skb, GFP_ATOMIC);
1405     if (skb)
1406         f->sent = ktime_get();
1407 
1408     return skb;
1409 }
1410 
1411 static struct aoetgt **
1412 grow_targets(struct aoedev *d)
1413 {
1414     ulong oldn, newn;
1415     struct aoetgt **tt;
1416 
1417     oldn = d->ntargets;
1418     newn = oldn * 2;
1419     tt = kcalloc(newn, sizeof(*d->targets), GFP_ATOMIC);
1420     if (!tt)
1421         return NULL;
1422     memmove(tt, d->targets, sizeof(*d->targets) * oldn);
1423     d->tgt = tt + (d->tgt - d->targets);
1424     kfree(d->targets);
1425     d->targets = tt;
1426     d->ntargets = newn;
1427 
1428     return &d->targets[oldn];
1429 }
1430 
1431 static struct aoetgt *
1432 addtgt(struct aoedev *d, char *addr, ulong nframes)
1433 {
1434     struct aoetgt *t, **tt, **te;
1435 
1436     tt = d->targets;
1437     te = tt + d->ntargets;
1438     for (; tt < te && *tt; tt++)
1439         ;
1440 
1441     if (tt == te) {
1442         tt = grow_targets(d);
1443         if (!tt)
1444             goto nomem;
1445     }
1446     t = kzalloc(sizeof(*t), GFP_ATOMIC);
1447     if (!t)
1448         goto nomem;
1449     t->nframes = nframes;
1450     t->d = d;
1451     memcpy(t->addr, addr, sizeof t->addr);
1452     t->ifp = t->ifs;
1453     aoecmd_wreset(t);
1454     t->maxout = t->nframes / 2;
1455     INIT_LIST_HEAD(&t->ffree);
1456     return *tt = t;
1457 
1458  nomem:
1459     pr_info("aoe: cannot allocate memory to add target\n");
1460     return NULL;
1461 }
1462 
1463 static void
1464 setdbcnt(struct aoedev *d)
1465 {
1466     struct aoetgt **t, **e;
1467     int bcnt = 0;
1468 
1469     t = d->targets;
1470     e = t + d->ntargets;
1471     for (; t < e && *t; t++)
1472         if (bcnt == 0 || bcnt > (*t)->minbcnt)
1473             bcnt = (*t)->minbcnt;
1474     if (bcnt != d->maxbcnt) {
1475         d->maxbcnt = bcnt;
1476         pr_info("aoe: e%ld.%d: setting %d byte data frames\n",
1477             d->aoemajor, d->aoeminor, bcnt);
1478     }
1479 }
1480 
1481 static void
1482 setifbcnt(struct aoetgt *t, struct net_device *nd, int bcnt)
1483 {
1484     struct aoedev *d;
1485     struct aoeif *p, *e;
1486     int minbcnt;
1487 
1488     d = t->d;
1489     minbcnt = bcnt;
1490     p = t->ifs;
1491     e = p + NAOEIFS;
1492     for (; p < e; p++) {
1493         if (p->nd == NULL)
1494             break;      /* end of the valid interfaces */
1495         if (p->nd == nd) {
1496             p->bcnt = bcnt; /* we're updating */
1497             nd = NULL;
1498         } else if (minbcnt > p->bcnt)
1499             minbcnt = p->bcnt; /* find the min interface */
1500     }
1501     if (nd) {
1502         if (p == e) {
1503             pr_err("aoe: device setifbcnt failure; too many interfaces.\n");
1504             return;
1505         }
1506         dev_hold(nd);
1507         p->nd = nd;
1508         p->bcnt = bcnt;
1509     }
1510     t->minbcnt = minbcnt;
1511     setdbcnt(d);
1512 }
1513 
1514 void
1515 aoecmd_cfg_rsp(struct sk_buff *skb)
1516 {
1517     struct aoedev *d;
1518     struct aoe_hdr *h;
1519     struct aoe_cfghdr *ch;
1520     struct aoetgt *t;
1521     ulong flags, aoemajor;
1522     struct sk_buff *sl;
1523     struct sk_buff_head queue;
1524     u16 n;
1525 
1526     sl = NULL;
1527     h = (struct aoe_hdr *) skb_mac_header(skb);
1528     ch = (struct aoe_cfghdr *) (h+1);
1529 
1530     /*
1531      * Enough people have their dip switches set backwards to
1532      * warrant a loud message for this special case.
1533      */
1534     aoemajor = get_unaligned_be16(&h->major);
1535     if (aoemajor == 0xfff) {
1536         printk(KERN_ERR "aoe: Warning: shelf address is all ones.  "
1537             "Check shelf dip switches.\n");
1538         return;
1539     }
1540     if (aoemajor == 0xffff) {
1541         pr_info("aoe: e%ld.%d: broadcast shelf number invalid\n",
1542             aoemajor, (int) h->minor);
1543         return;
1544     }
1545     if (h->minor == 0xff) {
1546         pr_info("aoe: e%ld.%d: broadcast slot number invalid\n",
1547             aoemajor, (int) h->minor);
1548         return;
1549     }
1550 
1551     n = be16_to_cpu(ch->bufcnt);
1552     if (n > aoe_maxout) /* keep it reasonable */
1553         n = aoe_maxout;
1554 
1555     d = aoedev_by_aoeaddr(aoemajor, h->minor, 1);
1556     if (d == NULL) {
1557         pr_info("aoe: device allocation failure\n");
1558         return;
1559     }
1560 
1561     spin_lock_irqsave(&d->lock, flags);
1562 
1563     t = gettgt(d, h->src);
1564     if (t) {
1565         t->nframes = n;
1566         if (n < t->maxout)
1567             aoecmd_wreset(t);
1568     } else {
1569         t = addtgt(d, h->src, n);
1570         if (!t)
1571             goto bail;
1572     }
1573     n = skb->dev->mtu;
1574     n -= sizeof(struct aoe_hdr) + sizeof(struct aoe_atahdr);
1575     n /= 512;
1576     if (n > ch->scnt)
1577         n = ch->scnt;
1578     n = n ? n * 512 : DEFAULTBCNT;
1579     setifbcnt(t, skb->dev, n);
1580 
1581     /* don't change users' perspective */
1582     if (d->nopen == 0) {
1583         d->fw_ver = be16_to_cpu(ch->fwver);
1584         sl = aoecmd_ata_id(d);
1585     }
1586 bail:
1587     spin_unlock_irqrestore(&d->lock, flags);
1588     aoedev_put(d);
1589     if (sl) {
1590         __skb_queue_head_init(&queue);
1591         __skb_queue_tail(&queue, sl);
1592         aoenet_xmit(&queue);
1593     }
1594 }
1595 
1596 void
1597 aoecmd_wreset(struct aoetgt *t)
1598 {
1599     t->maxout = 1;
1600     t->ssthresh = t->nframes / 2;
1601     t->next_cwnd = t->nframes;
1602 }
1603 
1604 void
1605 aoecmd_cleanslate(struct aoedev *d)
1606 {
1607     struct aoetgt **t, **te;
1608 
1609     d->rttavg = RTTAVG_INIT;
1610     d->rttdev = RTTDEV_INIT;
1611     d->maxbcnt = 0;
1612 
1613     t = d->targets;
1614     te = t + d->ntargets;
1615     for (; t < te && *t; t++)
1616         aoecmd_wreset(*t);
1617 }
1618 
1619 void
1620 aoe_failbuf(struct aoedev *d, struct buf *buf)
1621 {
1622     if (buf == NULL)
1623         return;
1624     buf->iter.bi_size = 0;
1625     buf->bio->bi_status = BLK_STS_IOERR;
1626     if (buf->nframesout == 0)
1627         aoe_end_buf(d, buf);
1628 }
1629 
1630 void
1631 aoe_flush_iocq(void)
1632 {
1633     int i;
1634 
1635     for (i = 0; i < ncpus; i++) {
1636         if (kts[i].active)
1637             aoe_flush_iocq_by_index(i);
1638     }
1639 }
1640 
1641 void
1642 aoe_flush_iocq_by_index(int id)
1643 {
1644     struct frame *f;
1645     struct aoedev *d;
1646     LIST_HEAD(flist);
1647     struct list_head *pos;
1648     struct sk_buff *skb;
1649     ulong flags;
1650 
1651     spin_lock_irqsave(&iocq[id].lock, flags);
1652     list_splice_init(&iocq[id].head, &flist);
1653     spin_unlock_irqrestore(&iocq[id].lock, flags);
1654     while (!list_empty(&flist)) {
1655         pos = flist.next;
1656         list_del(pos);
1657         f = list_entry(pos, struct frame, head);
1658         d = f->t->d;
1659         skb = f->r_skb;
1660         spin_lock_irqsave(&d->lock, flags);
1661         if (f->buf) {
1662             f->buf->nframesout--;
1663             aoe_failbuf(d, f->buf);
1664         }
1665         aoe_freetframe(f);
1666         spin_unlock_irqrestore(&d->lock, flags);
1667         dev_kfree_skb(skb);
1668         aoedev_put(d);
1669     }
1670 }
1671 
1672 int __init
1673 aoecmd_init(void)
1674 {
1675     void *p;
1676     int i;
1677     int ret;
1678 
1679     /* get_zeroed_page returns page with ref count 1 */
1680     p = (void *) get_zeroed_page(GFP_KERNEL);
1681     if (!p)
1682         return -ENOMEM;
1683     empty_page = virt_to_page(p);
1684 
1685     ncpus = num_online_cpus();
1686 
1687     iocq = kcalloc(ncpus, sizeof(struct iocq_ktio), GFP_KERNEL);
1688     if (!iocq)
1689         return -ENOMEM;
1690 
1691     kts = kcalloc(ncpus, sizeof(struct ktstate), GFP_KERNEL);
1692     if (!kts) {
1693         ret = -ENOMEM;
1694         goto kts_fail;
1695     }
1696 
1697     ktiowq = kcalloc(ncpus, sizeof(wait_queue_head_t), GFP_KERNEL);
1698     if (!ktiowq) {
1699         ret = -ENOMEM;
1700         goto ktiowq_fail;
1701     }
1702 
1703     for (i = 0; i < ncpus; i++) {
1704         INIT_LIST_HEAD(&iocq[i].head);
1705         spin_lock_init(&iocq[i].lock);
1706         init_waitqueue_head(&ktiowq[i]);
1707         snprintf(kts[i].name, sizeof(kts[i].name), "aoe_ktio%d", i);
1708         kts[i].fn = ktio;
1709         kts[i].waitq = &ktiowq[i];
1710         kts[i].lock = &iocq[i].lock;
1711         kts[i].id = i;
1712         kts[i].active = 0;
1713     }
1714     kts[0].active = 1;
1715     if (aoe_ktstart(&kts[0])) {
1716         ret = -ENOMEM;
1717         goto ktstart_fail;
1718     }
1719     return 0;
1720 
1721 ktstart_fail:
1722     kfree(ktiowq);
1723 ktiowq_fail:
1724     kfree(kts);
1725 kts_fail:
1726     kfree(iocq);
1727 
1728     return ret;
1729 }
1730 
1731 void
1732 aoecmd_exit(void)
1733 {
1734     int i;
1735 
1736     for (i = 0; i < ncpus; i++)
1737         if (kts[i].active)
1738             aoe_ktstop(&kts[i]);
1739 
1740     aoe_flush_iocq();
1741 
1742     /* Free up the iocq and thread speicific configuration
1743     * allocated during startup.
1744     */
1745     kfree(iocq);
1746     kfree(kts);
1747     kfree(ktiowq);
1748 
1749     free_page((unsigned long) page_address(empty_page));
1750     empty_page = NULL;
1751 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!