hw/hfi1/pio_copy.c

0001 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
0002 /*
0003  * Copyright(c) 2015, 2016 Intel Corporation.
0004  */
0005
0006 #include "hfi.h"
0007
0008 /* additive distance between non-SOP and SOP space */
0009 #define SOP_DISTANCE (TXE_PIO_SIZE / 2)
0010 #define PIO_BLOCK_MASK (PIO_BLOCK_SIZE - 1)
0011 /* number of QUADWORDs in a block */
0012 #define PIO_BLOCK_QWS (PIO_BLOCK_SIZE / sizeof(u64))
0013
0014 /**
0015  * pio_copy - copy data block to MMIO space
0016  * @dd: hfi1 dev data
0017  * @pbuf: a number of blocks allocated within a PIO send context
0018  * @pbc: PBC to send
0019  * @from: source, must be 8 byte aligned
0020  * @count: number of DWORD (32-bit) quantities to copy from source
0021  *
0022  * Copy data from source to PIO Send Buffer memory, 8 bytes at a time.
0023  * Must always write full BLOCK_SIZE bytes blocks.  The first block must
0024  * be written to the corresponding SOP=1 address.
0025  *
0026  * Known:
0027  * o pbuf->start always starts on a block boundary
0028  * o pbuf can wrap only at a block boundary
0029  */
0030 void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc,
0031           const void *from, size_t count)
0032 {
0033     void __iomem *dest = pbuf->start + SOP_DISTANCE;
0034     void __iomem *send = dest + PIO_BLOCK_SIZE;
0035     void __iomem *dend;         /* 8-byte data end */
0036
0037     /* write the PBC */
0038     writeq(pbc, dest);
0039     dest += sizeof(u64);
0040
0041     /* calculate where the QWORD data ends - in SOP=1 space */
0042     dend = dest + ((count >> 1) * sizeof(u64));
0043
0044     if (dend < send) {
0045         /*
0046          * all QWORD data is within the SOP block, does *not*
0047          * reach the end of the SOP block
0048          */
0049
0050         while (dest < dend) {
0051             writeq(*(u64 *)from, dest);
0052             from += sizeof(u64);
0053             dest += sizeof(u64);
0054         }
0055         /*
0056          * No boundary checks are needed here:
0057          * 0. We're not on the SOP block boundary
0058          * 1. The possible DWORD dangle will still be within
0059          *    the SOP block
0060          * 2. We cannot wrap except on a block boundary.
0061          */
0062     } else {
0063         /* QWORD data extends _to_ or beyond the SOP block */
0064
0065         /* write 8-byte SOP chunk data */
0066         while (dest < send) {
0067             writeq(*(u64 *)from, dest);
0068             from += sizeof(u64);
0069             dest += sizeof(u64);
0070         }
0071         /* drop out of the SOP range */
0072         dest -= SOP_DISTANCE;
0073         dend -= SOP_DISTANCE;
0074
0075         /*
0076          * If the wrap comes before or matches the data end,
0077          * copy until until the wrap, then wrap.
0078          *
0079          * If the data ends at the end of the SOP above and
0080          * the buffer wraps, then pbuf->end == dend == dest
0081          * and nothing will get written, but we will wrap in
0082          * case there is a dangling DWORD.
0083          */
0084         if (pbuf->end <= dend) {
0085             while (dest < pbuf->end) {
0086                 writeq(*(u64 *)from, dest);
0087                 from += sizeof(u64);
0088                 dest += sizeof(u64);
0089             }
0090
0091             dest -= pbuf->sc->size;
0092             dend -= pbuf->sc->size;
0093         }
0094
0095         /* write 8-byte non-SOP, non-wrap chunk data */
0096         while (dest < dend) {
0097             writeq(*(u64 *)from, dest);
0098             from += sizeof(u64);
0099             dest += sizeof(u64);
0100         }
0101     }
0102     /* at this point we have wrapped if we are going to wrap */
0103
0104     /* write dangling u32, if any */
0105     if (count & 1) {
0106         union mix val;
0107
0108         val.val64 = 0;
0109         val.val32[0] = *(u32 *)from;
0110         writeq(val.val64, dest);
0111         dest += sizeof(u64);
0112     }
0113     /*
0114      * fill in rest of block, no need to check pbuf->end
0115      * as we only wrap on a block boundary
0116      */
0117     while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) {
0118         writeq(0, dest);
0119         dest += sizeof(u64);
0120     }
0121
0122     /* finished with this buffer */
0123     this_cpu_dec(*pbuf->sc->buffers_allocated);
0124     preempt_enable();
0125 }
0126
0127 /*
0128  * Handle carry bytes using shifts and masks.
0129  *
0130  * NOTE: the value the unused portion of carry is expected to always be zero.
0131  */
0132
0133 /*
0134  * "zero" shift - bit shift used to zero out upper bytes.  Input is
0135  * the count of LSB bytes to preserve.
0136  */
0137 #define zshift(x) (8 * (8 - (x)))
0138
0139 /*
0140  * "merge" shift - bit shift used to merge with carry bytes.  Input is
0141  * the LSB byte count to move beyond.
0142  */
0143 #define mshift(x) (8 * (x))
0144
0145 /*
0146  * Jump copy - no-loop copy for < 8 bytes.
0147  */
0148 static inline void jcopy(u8 *dest, const u8 *src, u32 n)
0149 {
0150     switch (n) {
0151     case 7:
0152         *dest++ = *src++;
0153         fallthrough;
0154     case 6:
0155         *dest++ = *src++;
0156         fallthrough;
0157     case 5:
0158         *dest++ = *src++;
0159         fallthrough;
0160     case 4:
0161         *dest++ = *src++;
0162         fallthrough;
0163     case 3:
0164         *dest++ = *src++;
0165         fallthrough;
0166     case 2:
0167         *dest++ = *src++;
0168         fallthrough;
0169     case 1:
0170         *dest++ = *src++;
0171     }
0172 }
0173
0174 /*
0175  * Read nbytes from "from" and place them in the low bytes
0176  * of pbuf->carry.  Other bytes are left as-is.  Any previous
0177  * value in pbuf->carry is lost.
0178  *
0179  * NOTES:
0180  * o do not read from from if nbytes is zero
0181  * o from may _not_ be u64 aligned.
0182  */
0183 static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,
0184                   unsigned int nbytes)
0185 {
0186     pbuf->carry.val64 = 0;
0187     jcopy(&pbuf->carry.val8[0], from, nbytes);
0188     pbuf->carry_bytes = nbytes;
0189 }
0190
0191 /*
0192  * Read nbytes bytes from "from" and put them at the end of pbuf->carry.
0193  * It is expected that the extra read does not overfill carry.
0194  *
0195  * NOTES:
0196  * o from may _not_ be u64 aligned
0197  * o nbytes may span a QW boundary
0198  */
0199 static inline void read_extra_bytes(struct pio_buf *pbuf,
0200                     const void *from, unsigned int nbytes)
0201 {
0202     jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);
0203     pbuf->carry_bytes += nbytes;
0204 }
0205
0206 /*
0207  * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
0208  * Put the unused part of the next 8 bytes of src into the LSB bytes of
0209  * pbuf->carry with the upper bytes zeroed..
0210  *
0211  * NOTES:
0212  * o result must keep unused bytes zeroed
0213  * o src must be u64 aligned
0214  */
0215 static inline void merge_write8(
0216     struct pio_buf *pbuf,
0217     void __iomem *dest,
0218     const void *src)
0219 {
0220     u64 new, temp;
0221
0222     new = *(u64 *)src;
0223     temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));
0224     writeq(temp, dest);
0225     pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);
0226 }
0227
0228 /*
0229  * Write a quad word using all bytes of carry.
0230  */
0231 static inline void carry8_write8(union mix carry, void __iomem *dest)
0232 {
0233     writeq(carry.val64, dest);
0234 }
0235
0236 /*
0237  * Write a quad word using all the valid bytes of carry.  If carry
0238  * has zero valid bytes, nothing is written.
0239  * Returns 0 on nothing written, non-zero on quad word written.
0240  */
0241 static inline int carry_write8(struct pio_buf *pbuf, void __iomem *dest)
0242 {
0243     if (pbuf->carry_bytes) {
0244         /* unused bytes are always kept zeroed, so just write */
0245         writeq(pbuf->carry.val64, dest);
0246         return 1;
0247     }
0248
0249     return 0;
0250 }
0251
0252 /*
0253  * Segmented PIO Copy - start
0254  *
0255  * Start a PIO copy.
0256  *
0257  * @pbuf: destination buffer
0258  * @pbc: the PBC for the PIO buffer
0259  * @from: data source, QWORD aligned
0260  * @nbytes: bytes to copy
0261  */
0262 void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc,
0263             const void *from, size_t nbytes)
0264 {
0265     void __iomem *dest = pbuf->start + SOP_DISTANCE;
0266     void __iomem *send = dest + PIO_BLOCK_SIZE;
0267     void __iomem *dend;         /* 8-byte data end */
0268
0269     writeq(pbc, dest);
0270     dest += sizeof(u64);
0271
0272     /* calculate where the QWORD data ends - in SOP=1 space */
0273     dend = dest + ((nbytes >> 3) * sizeof(u64));
0274
0275     if (dend < send) {
0276         /*
0277          * all QWORD data is within the SOP block, does *not*
0278          * reach the end of the SOP block
0279          */
0280
0281         while (dest < dend) {
0282             writeq(*(u64 *)from, dest);
0283             from += sizeof(u64);
0284             dest += sizeof(u64);
0285         }
0286         /*
0287          * No boundary checks are needed here:
0288          * 0. We're not on the SOP block boundary
0289          * 1. The possible DWORD dangle will still be within
0290          *    the SOP block
0291          * 2. We cannot wrap except on a block boundary.
0292          */
0293     } else {
0294         /* QWORD data extends _to_ or beyond the SOP block */
0295
0296         /* write 8-byte SOP chunk data */
0297         while (dest < send) {
0298             writeq(*(u64 *)from, dest);
0299             from += sizeof(u64);
0300             dest += sizeof(u64);
0301         }
0302         /* drop out of the SOP range */
0303         dest -= SOP_DISTANCE;
0304         dend -= SOP_DISTANCE;
0305
0306         /*
0307          * If the wrap comes before or matches the data end,
0308          * copy until until the wrap, then wrap.
0309          *
0310          * If the data ends at the end of the SOP above and
0311          * the buffer wraps, then pbuf->end == dend == dest
0312          * and nothing will get written, but we will wrap in
0313          * case there is a dangling DWORD.
0314          */
0315         if (pbuf->end <= dend) {
0316             while (dest < pbuf->end) {
0317                 writeq(*(u64 *)from, dest);
0318                 from += sizeof(u64);
0319                 dest += sizeof(u64);
0320             }
0321
0322             dest -= pbuf->sc->size;
0323             dend -= pbuf->sc->size;
0324         }
0325
0326         /* write 8-byte non-SOP, non-wrap chunk data */
0327         while (dest < dend) {
0328             writeq(*(u64 *)from, dest);
0329             from += sizeof(u64);
0330             dest += sizeof(u64);
0331         }
0332     }
0333     /* at this point we have wrapped if we are going to wrap */
0334
0335     /* ...but it doesn't matter as we're done writing */
0336
0337     /* save dangling bytes, if any */
0338     read_low_bytes(pbuf, from, nbytes & 0x7);
0339
0340     pbuf->qw_written = 1 /*PBC*/ + (nbytes >> 3);
0341 }
0342
0343 /*
0344  * Mid copy helper, "mixed case" - source is 64-bit aligned but carry
0345  * bytes are non-zero.
0346  *
0347  * Whole u64s must be written to the chip, so bytes must be manually merged.
0348  *
0349  * @pbuf: destination buffer
0350  * @from: data source, is QWORD aligned.
0351  * @nbytes: bytes to copy
0352  *
0353  * Must handle nbytes < 8.
0354  */
0355 static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)
0356 {
0357     void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
0358     void __iomem *dend;         /* 8-byte data end */
0359     unsigned long qw_to_write = nbytes >> 3;
0360     unsigned long bytes_left = nbytes & 0x7;
0361
0362     /* calculate 8-byte data end */
0363     dend = dest + (qw_to_write * sizeof(u64));
0364
0365     if (pbuf->qw_written < PIO_BLOCK_QWS) {
0366         /*
0367          * Still within SOP block.  We don't need to check for
0368          * wrap because we are still in the first block and
0369          * can only wrap on block boundaries.
0370          */
0371         void __iomem *send;     /* SOP end */
0372         void __iomem *xend;
0373
0374         /*
0375          * calculate the end of data or end of block, whichever
0376          * comes first
0377          */
0378         send = pbuf->start + PIO_BLOCK_SIZE;
0379         xend = min(send, dend);
0380
0381         /* shift up to SOP=1 space */
0382         dest += SOP_DISTANCE;
0383         xend += SOP_DISTANCE;
0384
0385         /* write 8-byte chunk data */
0386         while (dest < xend) {
0387             merge_write8(pbuf, dest, from);
0388             from += sizeof(u64);
0389             dest += sizeof(u64);
0390         }
0391
0392         /* shift down to SOP=0 space */
0393         dest -= SOP_DISTANCE;
0394     }
0395     /*
0396      * At this point dest could be (either, both, or neither):
0397      * - at dend
0398      * - at the wrap
0399      */
0400
0401     /*
0402      * If the wrap comes before or matches the data end,
0403      * copy until until the wrap, then wrap.
0404      *
0405      * If dest is at the wrap, we will fall into the if,
0406      * not do the loop, when wrap.
0407      *
0408      * If the data ends at the end of the SOP above and
0409      * the buffer wraps, then pbuf->end == dend == dest
0410      * and nothing will get written.
0411      */
0412     if (pbuf->end <= dend) {
0413         while (dest < pbuf->end) {
0414             merge_write8(pbuf, dest, from);
0415             from += sizeof(u64);
0416             dest += sizeof(u64);
0417         }
0418
0419         dest -= pbuf->sc->size;
0420         dend -= pbuf->sc->size;
0421     }
0422
0423     /* write 8-byte non-SOP, non-wrap chunk data */
0424     while (dest < dend) {
0425         merge_write8(pbuf, dest, from);
0426         from += sizeof(u64);
0427         dest += sizeof(u64);
0428     }
0429
0430     pbuf->qw_written += qw_to_write;
0431
0432     /* handle carry and left-over bytes */
0433     if (pbuf->carry_bytes + bytes_left >= 8) {
0434         unsigned long nread;
0435
0436         /* there is enough to fill another qw - fill carry */
0437         nread = 8 - pbuf->carry_bytes;
0438         read_extra_bytes(pbuf, from, nread);
0439
0440         /*
0441          * One more write - but need to make sure dest is correct.
0442          * Check for wrap and the possibility the write
0443          * should be in SOP space.
0444          *
0445          * The two checks immediately below cannot both be true, hence
0446          * the else. If we have wrapped, we cannot still be within the
0447          * first block. Conversely, if we are still in the first block,
0448          * we cannot have wrapped. We do the wrap check first as that
0449          * is more likely.
0450          */
0451         /* adjust if we have wrapped */
0452         if (dest >= pbuf->end)
0453             dest -= pbuf->sc->size;
0454         /* jump to the SOP range if within the first block */
0455         else if (pbuf->qw_written < PIO_BLOCK_QWS)
0456             dest += SOP_DISTANCE;
0457
0458         /* flush out full carry */
0459         carry8_write8(pbuf->carry, dest);
0460         pbuf->qw_written++;
0461
0462         /* now adjust and read the rest of the bytes into carry */
0463         bytes_left -= nread;
0464         from += nread; /* from is now not aligned */
0465         read_low_bytes(pbuf, from, bytes_left);
0466     } else {
0467         /* not enough to fill another qw, append the rest to carry */
0468         read_extra_bytes(pbuf, from, bytes_left);
0469     }
0470 }
0471
0472 /*
0473  * Mid copy helper, "straight case" - source pointer is 64-bit aligned
0474  * with no carry bytes.
0475  *
0476  * @pbuf: destination buffer
0477  * @from: data source, is QWORD aligned
0478  * @nbytes: bytes to copy
0479  *
0480  * Must handle nbytes < 8.
0481  */
0482 static void mid_copy_straight(struct pio_buf *pbuf,
0483                   const void *from, size_t nbytes)
0484 {
0485     void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
0486     void __iomem *dend;         /* 8-byte data end */
0487
0488     /* calculate 8-byte data end */
0489     dend = dest + ((nbytes >> 3) * sizeof(u64));
0490
0491     if (pbuf->qw_written < PIO_BLOCK_QWS) {
0492         /*
0493          * Still within SOP block.  We don't need to check for
0494          * wrap because we are still in the first block and
0495          * can only wrap on block boundaries.
0496          */
0497         void __iomem *send;     /* SOP end */
0498         void __iomem *xend;
0499
0500         /*
0501          * calculate the end of data or end of block, whichever
0502          * comes first
0503          */
0504         send = pbuf->start + PIO_BLOCK_SIZE;
0505         xend = min(send, dend);
0506
0507         /* shift up to SOP=1 space */
0508         dest += SOP_DISTANCE;
0509         xend += SOP_DISTANCE;
0510
0511         /* write 8-byte chunk data */
0512         while (dest < xend) {
0513             writeq(*(u64 *)from, dest);
0514             from += sizeof(u64);
0515             dest += sizeof(u64);
0516         }
0517
0518         /* shift down to SOP=0 space */
0519         dest -= SOP_DISTANCE;
0520     }
0521     /*
0522      * At this point dest could be (either, both, or neither):
0523      * - at dend
0524      * - at the wrap
0525      */
0526
0527     /*
0528      * If the wrap comes before or matches the data end,
0529      * copy until until the wrap, then wrap.
0530      *
0531      * If dest is at the wrap, we will fall into the if,
0532      * not do the loop, when wrap.
0533      *
0534      * If the data ends at the end of the SOP above and
0535      * the buffer wraps, then pbuf->end == dend == dest
0536      * and nothing will get written.
0537      */
0538     if (pbuf->end <= dend) {
0539         while (dest < pbuf->end) {
0540             writeq(*(u64 *)from, dest);
0541             from += sizeof(u64);
0542             dest += sizeof(u64);
0543         }
0544
0545         dest -= pbuf->sc->size;
0546         dend -= pbuf->sc->size;
0547     }
0548
0549     /* write 8-byte non-SOP, non-wrap chunk data */
0550     while (dest < dend) {
0551         writeq(*(u64 *)from, dest);
0552         from += sizeof(u64);
0553         dest += sizeof(u64);
0554     }
0555
0556     /* we know carry_bytes was zero on entry to this routine */
0557     read_low_bytes(pbuf, from, nbytes & 0x7);
0558
0559     pbuf->qw_written += nbytes >> 3;
0560 }
0561
0562 /*
0563  * Segmented PIO Copy - middle
0564  *
0565  * Must handle any aligned tail and any aligned source with any byte count.
0566  *
0567  * @pbuf: a number of blocks allocated within a PIO send context
0568  * @from: data source
0569  * @nbytes: number of bytes to copy
0570  */
0571 void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)
0572 {
0573     unsigned long from_align = (unsigned long)from & 0x7;
0574
0575     if (pbuf->carry_bytes + nbytes < 8) {
0576         /* not enough bytes to fill a QW */
0577         read_extra_bytes(pbuf, from, nbytes);
0578         return;
0579     }
0580
0581     if (from_align) {
0582         /* misaligned source pointer - align it */
0583         unsigned long to_align;
0584
0585         /* bytes to read to align "from" */
0586         to_align = 8 - from_align;
0587
0588         /*
0589          * In the advance-to-alignment logic below, we do not need
0590          * to check if we are using more than nbytes.  This is because
0591          * if we are here, we already know that carry+nbytes will
0592          * fill at least one QW.
0593          */
0594         if (pbuf->carry_bytes + to_align < 8) {
0595             /* not enough align bytes to fill a QW */
0596             read_extra_bytes(pbuf, from, to_align);
0597             from += to_align;
0598             nbytes -= to_align;
0599         } else {
0600             /* bytes to fill carry */
0601             unsigned long to_fill = 8 - pbuf->carry_bytes;
0602             /* bytes left over to be read */
0603             unsigned long extra = to_align - to_fill;
0604             void __iomem *dest;
0605
0606             /* fill carry... */
0607             read_extra_bytes(pbuf, from, to_fill);
0608             from += to_fill;
0609             nbytes -= to_fill;
0610             /* may not be enough valid bytes left to align */
0611             if (extra > nbytes)
0612                 extra = nbytes;
0613
0614             /* ...now write carry */
0615             dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
0616
0617             /*
0618              * The two checks immediately below cannot both be
0619              * true, hence the else.  If we have wrapped, we
0620              * cannot still be within the first block.
0621              * Conversely, if we are still in the first block, we
0622              * cannot have wrapped.  We do the wrap check first
0623              * as that is more likely.
0624              */
0625             /* adjust if we've wrapped */
0626             if (dest >= pbuf->end)
0627                 dest -= pbuf->sc->size;
0628             /* jump to SOP range if within the first block */
0629             else if (pbuf->qw_written < PIO_BLOCK_QWS)
0630                 dest += SOP_DISTANCE;
0631
0632             carry8_write8(pbuf->carry, dest);
0633             pbuf->qw_written++;
0634
0635             /* read any extra bytes to do final alignment */
0636             /* this will overwrite anything in pbuf->carry */
0637             read_low_bytes(pbuf, from, extra);
0638             from += extra;
0639             nbytes -= extra;
0640             /*
0641              * If no bytes are left, return early - we are done.
0642              * NOTE: This short-circuit is *required* because
0643              * "extra" may have been reduced in size and "from"
0644              * is not aligned, as required when leaving this
0645              * if block.
0646              */
0647             if (nbytes == 0)
0648                 return;
0649         }
0650
0651         /* at this point, from is QW aligned */
0652     }
0653
0654     if (pbuf->carry_bytes)
0655         mid_copy_mix(pbuf, from, nbytes);
0656     else
0657         mid_copy_straight(pbuf, from, nbytes);
0658 }
0659
0660 /*
0661  * Segmented PIO Copy - end
0662  *
0663  * Write any remainder (in pbuf->carry) and finish writing the whole block.
0664  *
0665  * @pbuf: a number of blocks allocated within a PIO send context
0666  */
0667 void seg_pio_copy_end(struct pio_buf *pbuf)
0668 {
0669     void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
0670
0671     /*
0672      * The two checks immediately below cannot both be true, hence the
0673      * else.  If we have wrapped, we cannot still be within the first
0674      * block.  Conversely, if we are still in the first block, we
0675      * cannot have wrapped.  We do the wrap check first as that is
0676      * more likely.
0677      */
0678     /* adjust if we have wrapped */
0679     if (dest >= pbuf->end)
0680         dest -= pbuf->sc->size;
0681     /* jump to the SOP range if within the first block */
0682     else if (pbuf->qw_written < PIO_BLOCK_QWS)
0683         dest += SOP_DISTANCE;
0684
0685     /* write final bytes, if any */
0686     if (carry_write8(pbuf, dest)) {
0687         dest += sizeof(u64);
0688         /*
0689          * NOTE: We do not need to recalculate whether dest needs
0690          * SOP_DISTANCE or not.
0691          *
0692          * If we are in the first block and the dangle write
0693          * keeps us in the same block, dest will need
0694          * to retain SOP_DISTANCE in the loop below.
0695          *
0696          * If we are in the first block and the dangle write pushes
0697          * us to the next block, then loop below will not run
0698          * and dest is not used.  Hence we do not need to update
0699          * it.
0700          *
0701          * If we are past the first block, then SOP_DISTANCE
0702          * was never added, so there is nothing to do.
0703          */
0704     }
0705
0706     /* fill in rest of block */
0707     while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) {
0708         writeq(0, dest);
0709         dest += sizeof(u64);
0710     }
0711
0712     /* finished with this buffer */
0713     this_cpu_dec(*pbuf->sc->buffers_allocated);
0714     preempt_enable();
0715 }