include/linux/kfifo.h

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  * A generic kernel FIFO implementation
0004  *
0005  * Copyright (C) 2013 Stefani Seibold <stefani@seibold.net>
0006  */
0007
0008 #ifndef _LINUX_KFIFO_H
0009 #define _LINUX_KFIFO_H
0010
0011 /*
0012  * How to porting drivers to the new generic FIFO API:
0013  *
0014  * - Modify the declaration of the "struct kfifo *" object into a
0015  *   in-place "struct kfifo" object
0016  * - Init the in-place object with kfifo_alloc() or kfifo_init()
0017  *   Note: The address of the in-place "struct kfifo" object must be
0018  *   passed as the first argument to this functions
0019  * - Replace the use of __kfifo_put into kfifo_in and __kfifo_get
0020  *   into kfifo_out
0021  * - Replace the use of kfifo_put into kfifo_in_spinlocked and kfifo_get
0022  *   into kfifo_out_spinlocked
0023  *   Note: the spinlock pointer formerly passed to kfifo_init/kfifo_alloc
0024  *   must be passed now to the kfifo_in_spinlocked and kfifo_out_spinlocked
0025  *   as the last parameter
0026  * - The formerly __kfifo_* functions are renamed into kfifo_*
0027  */
0028
0029 /*
0030  * Note about locking: There is no locking required until only one reader
0031  * and one writer is using the fifo and no kfifo_reset() will be called.
0032  * kfifo_reset_out() can be safely used, until it will be only called
0033  * in the reader thread.
0034  * For multiple writer and one reader there is only a need to lock the writer.
0035  * And vice versa for only one writer and multiple reader there is only a need
0036  * to lock the reader.
0037  */
0038
0039 #include <linux/kernel.h>
0040 #include <linux/spinlock.h>
0041 #include <linux/stddef.h>
0042 #include <linux/scatterlist.h>
0043
0044 struct __kfifo {
0045     unsigned int    in;
0046     unsigned int    out;
0047     unsigned int    mask;
0048     unsigned int    esize;
0049     void        *data;
0050 };
0051
0052 #define __STRUCT_KFIFO_COMMON(datatype, recsize, ptrtype) \
0053     union { \
0054         struct __kfifo  kfifo; \
0055         datatype    *type; \
0056         const datatype  *const_type; \
0057         char        (*rectype)[recsize]; \
0058         ptrtype     *ptr; \
0059         ptrtype const   *ptr_const; \
0060     }
0061
0062 #define __STRUCT_KFIFO(type, size, recsize, ptrtype) \
0063 { \
0064     __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
0065     type        buf[((size < 2) || (size & (size - 1))) ? -1 : size]; \
0066 }
0067
0068 #define STRUCT_KFIFO(type, size) \
0069     struct __STRUCT_KFIFO(type, size, 0, type)
0070
0071 #define __STRUCT_KFIFO_PTR(type, recsize, ptrtype) \
0072 { \
0073     __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
0074     type        buf[0]; \
0075 }
0076
0077 #define STRUCT_KFIFO_PTR(type) \
0078     struct __STRUCT_KFIFO_PTR(type, 0, type)
0079
0080 /*
0081  * define compatibility "struct kfifo" for dynamic allocated fifos
0082  */
0083 struct kfifo __STRUCT_KFIFO_PTR(unsigned char, 0, void);
0084
0085 #define STRUCT_KFIFO_REC_1(size) \
0086     struct __STRUCT_KFIFO(unsigned char, size, 1, void)
0087
0088 #define STRUCT_KFIFO_REC_2(size) \
0089     struct __STRUCT_KFIFO(unsigned char, size, 2, void)
0090
0091 /*
0092  * define kfifo_rec types
0093  */
0094 struct kfifo_rec_ptr_1 __STRUCT_KFIFO_PTR(unsigned char, 1, void);
0095 struct kfifo_rec_ptr_2 __STRUCT_KFIFO_PTR(unsigned char, 2, void);
0096
0097 /*
0098  * helper macro to distinguish between real in place fifo where the fifo
0099  * array is a part of the structure and the fifo type where the array is
0100  * outside of the fifo structure.
0101  */
0102 #define __is_kfifo_ptr(fifo) \
0103     (sizeof(*fifo) == sizeof(STRUCT_KFIFO_PTR(typeof(*(fifo)->type))))
0104
0105 /**
0106  * DECLARE_KFIFO_PTR - macro to declare a fifo pointer object
0107  * @fifo: name of the declared fifo
0108  * @type: type of the fifo elements
0109  */
0110 #define DECLARE_KFIFO_PTR(fifo, type)   STRUCT_KFIFO_PTR(type) fifo
0111
0112 /**
0113  * DECLARE_KFIFO - macro to declare a fifo object
0114  * @fifo: name of the declared fifo
0115  * @type: type of the fifo elements
0116  * @size: the number of elements in the fifo, this must be a power of 2
0117  */
0118 #define DECLARE_KFIFO(fifo, type, size) STRUCT_KFIFO(type, size) fifo
0119
0120 /**
0121  * INIT_KFIFO - Initialize a fifo declared by DECLARE_KFIFO
0122  * @fifo: name of the declared fifo datatype
0123  */
0124 #define INIT_KFIFO(fifo) \
0125 (void)({ \
0126     typeof(&(fifo)) __tmp = &(fifo); \
0127     struct __kfifo *__kfifo = &__tmp->kfifo; \
0128     __kfifo->in = 0; \
0129     __kfifo->out = 0; \
0130     __kfifo->mask = __is_kfifo_ptr(__tmp) ? 0 : ARRAY_SIZE(__tmp->buf) - 1;\
0131     __kfifo->esize = sizeof(*__tmp->buf); \
0132     __kfifo->data = __is_kfifo_ptr(__tmp) ?  NULL : __tmp->buf; \
0133 })
0134
0135 /**
0136  * DEFINE_KFIFO - macro to define and initialize a fifo
0137  * @fifo: name of the declared fifo datatype
0138  * @type: type of the fifo elements
0139  * @size: the number of elements in the fifo, this must be a power of 2
0140  *
0141  * Note: the macro can be used for global and local fifo data type variables.
0142  */
0143 #define DEFINE_KFIFO(fifo, type, size) \
0144     DECLARE_KFIFO(fifo, type, size) = \
0145     (typeof(fifo)) { \
0146         { \
0147             { \
0148             .in = 0, \
0149             .out    = 0, \
0150             .mask   = __is_kfifo_ptr(&(fifo)) ? \
0151                   0 : \
0152                   ARRAY_SIZE((fifo).buf) - 1, \
0153             .esize  = sizeof(*(fifo).buf), \
0154             .data   = __is_kfifo_ptr(&(fifo)) ? \
0155                 NULL : \
0156                 (fifo).buf, \
0157             } \
0158         } \
0159     }
0160
0161
0162 static inline unsigned int __must_check
0163 __kfifo_uint_must_check_helper(unsigned int val)
0164 {
0165     return val;
0166 }
0167
0168 static inline int __must_check
0169 __kfifo_int_must_check_helper(int val)
0170 {
0171     return val;
0172 }
0173
0174 /**
0175  * kfifo_initialized - Check if the fifo is initialized
0176  * @fifo: address of the fifo to check
0177  *
0178  * Return %true if fifo is initialized, otherwise %false.
0179  * Assumes the fifo was 0 before.
0180  */
0181 #define kfifo_initialized(fifo) ((fifo)->kfifo.mask)
0182
0183 /**
0184  * kfifo_esize - returns the size of the element managed by the fifo
0185  * @fifo: address of the fifo to be used
0186  */
0187 #define kfifo_esize(fifo)   ((fifo)->kfifo.esize)
0188
0189 /**
0190  * kfifo_recsize - returns the size of the record length field
0191  * @fifo: address of the fifo to be used
0192  */
0193 #define kfifo_recsize(fifo) (sizeof(*(fifo)->rectype))
0194
0195 /**
0196  * kfifo_size - returns the size of the fifo in elements
0197  * @fifo: address of the fifo to be used
0198  */
0199 #define kfifo_size(fifo)    ((fifo)->kfifo.mask + 1)
0200
0201 /**
0202  * kfifo_reset - removes the entire fifo content
0203  * @fifo: address of the fifo to be used
0204  *
0205  * Note: usage of kfifo_reset() is dangerous. It should be only called when the
0206  * fifo is exclusived locked or when it is secured that no other thread is
0207  * accessing the fifo.
0208  */
0209 #define kfifo_reset(fifo) \
0210 (void)({ \
0211     typeof((fifo) + 1) __tmp = (fifo); \
0212     __tmp->kfifo.in = __tmp->kfifo.out = 0; \
0213 })
0214
0215 /**
0216  * kfifo_reset_out - skip fifo content
0217  * @fifo: address of the fifo to be used
0218  *
0219  * Note: The usage of kfifo_reset_out() is safe until it will be only called
0220  * from the reader thread and there is only one concurrent reader. Otherwise
0221  * it is dangerous and must be handled in the same way as kfifo_reset().
0222  */
0223 #define kfifo_reset_out(fifo)   \
0224 (void)({ \
0225     typeof((fifo) + 1) __tmp = (fifo); \
0226     __tmp->kfifo.out = __tmp->kfifo.in; \
0227 })
0228
0229 /**
0230  * kfifo_len - returns the number of used elements in the fifo
0231  * @fifo: address of the fifo to be used
0232  */
0233 #define kfifo_len(fifo) \
0234 ({ \
0235     typeof((fifo) + 1) __tmpl = (fifo); \
0236     __tmpl->kfifo.in - __tmpl->kfifo.out; \
0237 })
0238
0239 /**
0240  * kfifo_is_empty - returns true if the fifo is empty
0241  * @fifo: address of the fifo to be used
0242  */
0243 #define kfifo_is_empty(fifo) \
0244 ({ \
0245     typeof((fifo) + 1) __tmpq = (fifo); \
0246     __tmpq->kfifo.in == __tmpq->kfifo.out; \
0247 })
0248
0249 /**
0250  * kfifo_is_empty_spinlocked - returns true if the fifo is empty using
0251  * a spinlock for locking
0252  * @fifo: address of the fifo to be used
0253  * @lock: spinlock to be used for locking
0254  */
0255 #define kfifo_is_empty_spinlocked(fifo, lock) \
0256 ({ \
0257     unsigned long __flags; \
0258     bool __ret; \
0259     spin_lock_irqsave(lock, __flags); \
0260     __ret = kfifo_is_empty(fifo); \
0261     spin_unlock_irqrestore(lock, __flags); \
0262     __ret; \
0263 })
0264
0265 /**
0266  * kfifo_is_empty_spinlocked_noirqsave  - returns true if the fifo is empty
0267  * using a spinlock for locking, doesn't disable interrupts
0268  * @fifo: address of the fifo to be used
0269  * @lock: spinlock to be used for locking
0270  */
0271 #define kfifo_is_empty_spinlocked_noirqsave(fifo, lock) \
0272 ({ \
0273     bool __ret; \
0274     spin_lock(lock); \
0275     __ret = kfifo_is_empty(fifo); \
0276     spin_unlock(lock); \
0277     __ret; \
0278 })
0279
0280 /**
0281  * kfifo_is_full - returns true if the fifo is full
0282  * @fifo: address of the fifo to be used
0283  */
0284 #define kfifo_is_full(fifo) \
0285 ({ \
0286     typeof((fifo) + 1) __tmpq = (fifo); \
0287     kfifo_len(__tmpq) > __tmpq->kfifo.mask; \
0288 })
0289
0290 /**
0291  * kfifo_avail - returns the number of unused elements in the fifo
0292  * @fifo: address of the fifo to be used
0293  */
0294 #define kfifo_avail(fifo) \
0295 __kfifo_uint_must_check_helper( \
0296 ({ \
0297     typeof((fifo) + 1) __tmpq = (fifo); \
0298     const size_t __recsize = sizeof(*__tmpq->rectype); \
0299     unsigned int __avail = kfifo_size(__tmpq) - kfifo_len(__tmpq); \
0300     (__recsize) ? ((__avail <= __recsize) ? 0 : \
0301     __kfifo_max_r(__avail - __recsize, __recsize)) : \
0302     __avail; \
0303 }) \
0304 )
0305
0306 /**
0307  * kfifo_skip - skip output data
0308  * @fifo: address of the fifo to be used
0309  */
0310 #define kfifo_skip(fifo) \
0311 (void)({ \
0312     typeof((fifo) + 1) __tmp = (fifo); \
0313     const size_t __recsize = sizeof(*__tmp->rectype); \
0314     struct __kfifo *__kfifo = &__tmp->kfifo; \
0315     if (__recsize) \
0316         __kfifo_skip_r(__kfifo, __recsize); \
0317     else \
0318         __kfifo->out++; \
0319 })
0320
0321 /**
0322  * kfifo_peek_len - gets the size of the next fifo record
0323  * @fifo: address of the fifo to be used
0324  *
0325  * This function returns the size of the next fifo record in number of bytes.
0326  */
0327 #define kfifo_peek_len(fifo) \
0328 __kfifo_uint_must_check_helper( \
0329 ({ \
0330     typeof((fifo) + 1) __tmp = (fifo); \
0331     const size_t __recsize = sizeof(*__tmp->rectype); \
0332     struct __kfifo *__kfifo = &__tmp->kfifo; \
0333     (!__recsize) ? kfifo_len(__tmp) * sizeof(*__tmp->type) : \
0334     __kfifo_len_r(__kfifo, __recsize); \
0335 }) \
0336 )
0337
0338 /**
0339  * kfifo_alloc - dynamically allocates a new fifo buffer
0340  * @fifo: pointer to the fifo
0341  * @size: the number of elements in the fifo, this must be a power of 2
0342  * @gfp_mask: get_free_pages mask, passed to kmalloc()
0343  *
0344  * This macro dynamically allocates a new fifo buffer.
0345  *
0346  * The number of elements will be rounded-up to a power of 2.
0347  * The fifo will be release with kfifo_free().
0348  * Return 0 if no error, otherwise an error code.
0349  */
0350 #define kfifo_alloc(fifo, size, gfp_mask) \
0351 __kfifo_int_must_check_helper( \
0352 ({ \
0353     typeof((fifo) + 1) __tmp = (fifo); \
0354     struct __kfifo *__kfifo = &__tmp->kfifo; \
0355     __is_kfifo_ptr(__tmp) ? \
0356     __kfifo_alloc(__kfifo, size, sizeof(*__tmp->type), gfp_mask) : \
0357     -EINVAL; \
0358 }) \
0359 )
0360
0361 /**
0362  * kfifo_free - frees the fifo
0363  * @fifo: the fifo to be freed
0364  */
0365 #define kfifo_free(fifo) \
0366 ({ \
0367     typeof((fifo) + 1) __tmp = (fifo); \
0368     struct __kfifo *__kfifo = &__tmp->kfifo; \
0369     if (__is_kfifo_ptr(__tmp)) \
0370         __kfifo_free(__kfifo); \
0371 })
0372
0373 /**
0374  * kfifo_init - initialize a fifo using a preallocated buffer
0375  * @fifo: the fifo to assign the buffer
0376  * @buffer: the preallocated buffer to be used
0377  * @size: the size of the internal buffer, this have to be a power of 2
0378  *
0379  * This macro initializes a fifo using a preallocated buffer.
0380  *
0381  * The number of elements will be rounded-up to a power of 2.
0382  * Return 0 if no error, otherwise an error code.
0383  */
0384 #define kfifo_init(fifo, buffer, size) \
0385 ({ \
0386     typeof((fifo) + 1) __tmp = (fifo); \
0387     struct __kfifo *__kfifo = &__tmp->kfifo; \
0388     __is_kfifo_ptr(__tmp) ? \
0389     __kfifo_init(__kfifo, buffer, size, sizeof(*__tmp->type)) : \
0390     -EINVAL; \
0391 })
0392
0393 /**
0394  * kfifo_put - put data into the fifo
0395  * @fifo: address of the fifo to be used
0396  * @val: the data to be added
0397  *
0398  * This macro copies the given value into the fifo.
0399  * It returns 0 if the fifo was full. Otherwise it returns the number
0400  * processed elements.
0401  *
0402  * Note that with only one concurrent reader and one concurrent
0403  * writer, you don't need extra locking to use these macro.
0404  */
0405 #define kfifo_put(fifo, val) \
0406 ({ \
0407     typeof((fifo) + 1) __tmp = (fifo); \
0408     typeof(*__tmp->const_type) __val = (val); \
0409     unsigned int __ret; \
0410     size_t __recsize = sizeof(*__tmp->rectype); \
0411     struct __kfifo *__kfifo = &__tmp->kfifo; \
0412     if (__recsize) \
0413         __ret = __kfifo_in_r(__kfifo, &__val, sizeof(__val), \
0414             __recsize); \
0415     else { \
0416         __ret = !kfifo_is_full(__tmp); \
0417         if (__ret) { \
0418             (__is_kfifo_ptr(__tmp) ? \
0419             ((typeof(__tmp->type))__kfifo->data) : \
0420             (__tmp->buf) \
0421             )[__kfifo->in & __tmp->kfifo.mask] = \
0422                 *(typeof(__tmp->type))&__val; \
0423             smp_wmb(); \
0424             __kfifo->in++; \
0425         } \
0426     } \
0427     __ret; \
0428 })
0429
0430 /**
0431  * kfifo_get - get data from the fifo
0432  * @fifo: address of the fifo to be used
0433  * @val: address where to store the data
0434  *
0435  * This macro reads the data from the fifo.
0436  * It returns 0 if the fifo was empty. Otherwise it returns the number
0437  * processed elements.
0438  *
0439  * Note that with only one concurrent reader and one concurrent
0440  * writer, you don't need extra locking to use these macro.
0441  */
0442 #define kfifo_get(fifo, val) \
0443 __kfifo_uint_must_check_helper( \
0444 ({ \
0445     typeof((fifo) + 1) __tmp = (fifo); \
0446     typeof(__tmp->ptr) __val = (val); \
0447     unsigned int __ret; \
0448     const size_t __recsize = sizeof(*__tmp->rectype); \
0449     struct __kfifo *__kfifo = &__tmp->kfifo; \
0450     if (__recsize) \
0451         __ret = __kfifo_out_r(__kfifo, __val, sizeof(*__val), \
0452             __recsize); \
0453     else { \
0454         __ret = !kfifo_is_empty(__tmp); \
0455         if (__ret) { \
0456             *(typeof(__tmp->type))__val = \
0457                 (__is_kfifo_ptr(__tmp) ? \
0458                 ((typeof(__tmp->type))__kfifo->data) : \
0459                 (__tmp->buf) \
0460                 )[__kfifo->out & __tmp->kfifo.mask]; \
0461             smp_wmb(); \
0462             __kfifo->out++; \
0463         } \
0464     } \
0465     __ret; \
0466 }) \
0467 )
0468
0469 /**
0470  * kfifo_peek - get data from the fifo without removing
0471  * @fifo: address of the fifo to be used
0472  * @val: address where to store the data
0473  *
0474  * This reads the data from the fifo without removing it from the fifo.
0475  * It returns 0 if the fifo was empty. Otherwise it returns the number
0476  * processed elements.
0477  *
0478  * Note that with only one concurrent reader and one concurrent
0479  * writer, you don't need extra locking to use these macro.
0480  */
0481 #define kfifo_peek(fifo, val) \
0482 __kfifo_uint_must_check_helper( \
0483 ({ \
0484     typeof((fifo) + 1) __tmp = (fifo); \
0485     typeof(__tmp->ptr) __val = (val); \
0486     unsigned int __ret; \
0487     const size_t __recsize = sizeof(*__tmp->rectype); \
0488     struct __kfifo *__kfifo = &__tmp->kfifo; \
0489     if (__recsize) \
0490         __ret = __kfifo_out_peek_r(__kfifo, __val, sizeof(*__val), \
0491             __recsize); \
0492     else { \
0493         __ret = !kfifo_is_empty(__tmp); \
0494         if (__ret) { \
0495             *(typeof(__tmp->type))__val = \
0496                 (__is_kfifo_ptr(__tmp) ? \
0497                 ((typeof(__tmp->type))__kfifo->data) : \
0498                 (__tmp->buf) \
0499                 )[__kfifo->out & __tmp->kfifo.mask]; \
0500             smp_wmb(); \
0501         } \
0502     } \
0503     __ret; \
0504 }) \
0505 )
0506
0507 /**
0508  * kfifo_in - put data into the fifo
0509  * @fifo: address of the fifo to be used
0510  * @buf: the data to be added
0511  * @n: number of elements to be added
0512  *
0513  * This macro copies the given buffer into the fifo and returns the
0514  * number of copied elements.
0515  *
0516  * Note that with only one concurrent reader and one concurrent
0517  * writer, you don't need extra locking to use these macro.
0518  */
0519 #define kfifo_in(fifo, buf, n) \
0520 ({ \
0521     typeof((fifo) + 1) __tmp = (fifo); \
0522     typeof(__tmp->ptr_const) __buf = (buf); \
0523     unsigned long __n = (n); \
0524     const size_t __recsize = sizeof(*__tmp->rectype); \
0525     struct __kfifo *__kfifo = &__tmp->kfifo; \
0526     (__recsize) ?\
0527     __kfifo_in_r(__kfifo, __buf, __n, __recsize) : \
0528     __kfifo_in(__kfifo, __buf, __n); \
0529 })
0530
0531 /**
0532  * kfifo_in_spinlocked - put data into the fifo using a spinlock for locking
0533  * @fifo: address of the fifo to be used
0534  * @buf: the data to be added
0535  * @n: number of elements to be added
0536  * @lock: pointer to the spinlock to use for locking
0537  *
0538  * This macro copies the given values buffer into the fifo and returns the
0539  * number of copied elements.
0540  */
0541 #define kfifo_in_spinlocked(fifo, buf, n, lock) \
0542 ({ \
0543     unsigned long __flags; \
0544     unsigned int __ret; \
0545     spin_lock_irqsave(lock, __flags); \
0546     __ret = kfifo_in(fifo, buf, n); \
0547     spin_unlock_irqrestore(lock, __flags); \
0548     __ret; \
0549 })
0550
0551 /**
0552  * kfifo_in_spinlocked_noirqsave - put data into fifo using a spinlock for
0553  * locking, don't disable interrupts
0554  * @fifo: address of the fifo to be used
0555  * @buf: the data to be added
0556  * @n: number of elements to be added
0557  * @lock: pointer to the spinlock to use for locking
0558  *
0559  * This is a variant of kfifo_in_spinlocked() but uses spin_lock/unlock()
0560  * for locking and doesn't disable interrupts.
0561  */
0562 #define kfifo_in_spinlocked_noirqsave(fifo, buf, n, lock) \
0563 ({ \
0564     unsigned int __ret; \
0565     spin_lock(lock); \
0566     __ret = kfifo_in(fifo, buf, n); \
0567     spin_unlock(lock); \
0568     __ret; \
0569 })
0570
0571 /* alias for kfifo_in_spinlocked, will be removed in a future release */
0572 #define kfifo_in_locked(fifo, buf, n, lock) \
0573         kfifo_in_spinlocked(fifo, buf, n, lock)
0574
0575 /**
0576  * kfifo_out - get data from the fifo
0577  * @fifo: address of the fifo to be used
0578  * @buf: pointer to the storage buffer
0579  * @n: max. number of elements to get
0580  *
0581  * This macro get some data from the fifo and return the numbers of elements
0582  * copied.
0583  *
0584  * Note that with only one concurrent reader and one concurrent
0585  * writer, you don't need extra locking to use these macro.
0586  */
0587 #define kfifo_out(fifo, buf, n) \
0588 __kfifo_uint_must_check_helper( \
0589 ({ \
0590     typeof((fifo) + 1) __tmp = (fifo); \
0591     typeof(__tmp->ptr) __buf = (buf); \
0592     unsigned long __n = (n); \
0593     const size_t __recsize = sizeof(*__tmp->rectype); \
0594     struct __kfifo *__kfifo = &__tmp->kfifo; \
0595     (__recsize) ?\
0596     __kfifo_out_r(__kfifo, __buf, __n, __recsize) : \
0597     __kfifo_out(__kfifo, __buf, __n); \
0598 }) \
0599 )
0600
0601 /**
0602  * kfifo_out_spinlocked - get data from the fifo using a spinlock for locking
0603  * @fifo: address of the fifo to be used
0604  * @buf: pointer to the storage buffer
0605  * @n: max. number of elements to get
0606  * @lock: pointer to the spinlock to use for locking
0607  *
0608  * This macro get the data from the fifo and return the numbers of elements
0609  * copied.
0610  */
0611 #define kfifo_out_spinlocked(fifo, buf, n, lock) \
0612 __kfifo_uint_must_check_helper( \
0613 ({ \
0614     unsigned long __flags; \
0615     unsigned int __ret; \
0616     spin_lock_irqsave(lock, __flags); \
0617     __ret = kfifo_out(fifo, buf, n); \
0618     spin_unlock_irqrestore(lock, __flags); \
0619     __ret; \
0620 }) \
0621 )
0622
0623 /**
0624  * kfifo_out_spinlocked_noirqsave - get data from the fifo using a spinlock
0625  * for locking, don't disable interrupts
0626  * @fifo: address of the fifo to be used
0627  * @buf: pointer to the storage buffer
0628  * @n: max. number of elements to get
0629  * @lock: pointer to the spinlock to use for locking
0630  *
0631  * This is a variant of kfifo_out_spinlocked() which uses spin_lock/unlock()
0632  * for locking and doesn't disable interrupts.
0633  */
0634 #define kfifo_out_spinlocked_noirqsave(fifo, buf, n, lock) \
0635 __kfifo_uint_must_check_helper( \
0636 ({ \
0637     unsigned int __ret; \
0638     spin_lock(lock); \
0639     __ret = kfifo_out(fifo, buf, n); \
0640     spin_unlock(lock); \
0641     __ret; \
0642 }) \
0643 )
0644
0645 /* alias for kfifo_out_spinlocked, will be removed in a future release */
0646 #define kfifo_out_locked(fifo, buf, n, lock) \
0647         kfifo_out_spinlocked(fifo, buf, n, lock)
0648
0649 /**
0650  * kfifo_from_user - puts some data from user space into the fifo
0651  * @fifo: address of the fifo to be used
0652  * @from: pointer to the data to be added
0653  * @len: the length of the data to be added
0654  * @copied: pointer to output variable to store the number of copied bytes
0655  *
0656  * This macro copies at most @len bytes from the @from into the
0657  * fifo, depending of the available space and returns -EFAULT/0.
0658  *
0659  * Note that with only one concurrent reader and one concurrent
0660  * writer, you don't need extra locking to use these macro.
0661  */
0662 #define kfifo_from_user(fifo, from, len, copied) \
0663 __kfifo_uint_must_check_helper( \
0664 ({ \
0665     typeof((fifo) + 1) __tmp = (fifo); \
0666     const void __user *__from = (from); \
0667     unsigned int __len = (len); \
0668     unsigned int *__copied = (copied); \
0669     const size_t __recsize = sizeof(*__tmp->rectype); \
0670     struct __kfifo *__kfifo = &__tmp->kfifo; \
0671     (__recsize) ? \
0672     __kfifo_from_user_r(__kfifo, __from, __len,  __copied, __recsize) : \
0673     __kfifo_from_user(__kfifo, __from, __len, __copied); \
0674 }) \
0675 )
0676
0677 /**
0678  * kfifo_to_user - copies data from the fifo into user space
0679  * @fifo: address of the fifo to be used
0680  * @to: where the data must be copied
0681  * @len: the size of the destination buffer
0682  * @copied: pointer to output variable to store the number of copied bytes
0683  *
0684  * This macro copies at most @len bytes from the fifo into the
0685  * @to buffer and returns -EFAULT/0.
0686  *
0687  * Note that with only one concurrent reader and one concurrent
0688  * writer, you don't need extra locking to use these macro.
0689  */
0690 #define kfifo_to_user(fifo, to, len, copied) \
0691 __kfifo_int_must_check_helper( \
0692 ({ \
0693     typeof((fifo) + 1) __tmp = (fifo); \
0694     void __user *__to = (to); \
0695     unsigned int __len = (len); \
0696     unsigned int *__copied = (copied); \
0697     const size_t __recsize = sizeof(*__tmp->rectype); \
0698     struct __kfifo *__kfifo = &__tmp->kfifo; \
0699     (__recsize) ? \
0700     __kfifo_to_user_r(__kfifo, __to, __len, __copied, __recsize) : \
0701     __kfifo_to_user(__kfifo, __to, __len, __copied); \
0702 }) \
0703 )
0704
0705 /**
0706  * kfifo_dma_in_prepare - setup a scatterlist for DMA input
0707  * @fifo: address of the fifo to be used
0708  * @sgl: pointer to the scatterlist array
0709  * @nents: number of entries in the scatterlist array
0710  * @len: number of elements to transfer
0711  *
0712  * This macro fills a scatterlist for DMA input.
0713  * It returns the number entries in the scatterlist array.
0714  *
0715  * Note that with only one concurrent reader and one concurrent
0716  * writer, you don't need extra locking to use these macros.
0717  */
0718 #define kfifo_dma_in_prepare(fifo, sgl, nents, len) \
0719 ({ \
0720     typeof((fifo) + 1) __tmp = (fifo); \
0721     struct scatterlist *__sgl = (sgl); \
0722     int __nents = (nents); \
0723     unsigned int __len = (len); \
0724     const size_t __recsize = sizeof(*__tmp->rectype); \
0725     struct __kfifo *__kfifo = &__tmp->kfifo; \
0726     (__recsize) ? \
0727     __kfifo_dma_in_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
0728     __kfifo_dma_in_prepare(__kfifo, __sgl, __nents, __len); \
0729 })
0730
0731 /**
0732  * kfifo_dma_in_finish - finish a DMA IN operation
0733  * @fifo: address of the fifo to be used
0734  * @len: number of bytes to received
0735  *
0736  * This macro finish a DMA IN operation. The in counter will be updated by
0737  * the len parameter. No error checking will be done.
0738  *
0739  * Note that with only one concurrent reader and one concurrent
0740  * writer, you don't need extra locking to use these macros.
0741  */
0742 #define kfifo_dma_in_finish(fifo, len) \
0743 (void)({ \
0744     typeof((fifo) + 1) __tmp = (fifo); \
0745     unsigned int __len = (len); \
0746     const size_t __recsize = sizeof(*__tmp->rectype); \
0747     struct __kfifo *__kfifo = &__tmp->kfifo; \
0748     if (__recsize) \
0749         __kfifo_dma_in_finish_r(__kfifo, __len, __recsize); \
0750     else \
0751         __kfifo->in += __len / sizeof(*__tmp->type); \
0752 })
0753
0754 /**
0755  * kfifo_dma_out_prepare - setup a scatterlist for DMA output
0756  * @fifo: address of the fifo to be used
0757  * @sgl: pointer to the scatterlist array
0758  * @nents: number of entries in the scatterlist array
0759  * @len: number of elements to transfer
0760  *
0761  * This macro fills a scatterlist for DMA output which at most @len bytes
0762  * to transfer.
0763  * It returns the number entries in the scatterlist array.
0764  * A zero means there is no space available and the scatterlist is not filled.
0765  *
0766  * Note that with only one concurrent reader and one concurrent
0767  * writer, you don't need extra locking to use these macros.
0768  */
0769 #define kfifo_dma_out_prepare(fifo, sgl, nents, len) \
0770 ({ \
0771     typeof((fifo) + 1) __tmp = (fifo);  \
0772     struct scatterlist *__sgl = (sgl); \
0773     int __nents = (nents); \
0774     unsigned int __len = (len); \
0775     const size_t __recsize = sizeof(*__tmp->rectype); \
0776     struct __kfifo *__kfifo = &__tmp->kfifo; \
0777     (__recsize) ? \
0778     __kfifo_dma_out_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
0779     __kfifo_dma_out_prepare(__kfifo, __sgl, __nents, __len); \
0780 })
0781
0782 /**
0783  * kfifo_dma_out_finish - finish a DMA OUT operation
0784  * @fifo: address of the fifo to be used
0785  * @len: number of bytes transferred
0786  *
0787  * This macro finish a DMA OUT operation. The out counter will be updated by
0788  * the len parameter. No error checking will be done.
0789  *
0790  * Note that with only one concurrent reader and one concurrent
0791  * writer, you don't need extra locking to use these macros.
0792  */
0793 #define kfifo_dma_out_finish(fifo, len) \
0794 (void)({ \
0795     typeof((fifo) + 1) __tmp = (fifo); \
0796     unsigned int __len = (len); \
0797     const size_t __recsize = sizeof(*__tmp->rectype); \
0798     struct __kfifo *__kfifo = &__tmp->kfifo; \
0799     if (__recsize) \
0800         __kfifo_dma_out_finish_r(__kfifo, __recsize); \
0801     else \
0802         __kfifo->out += __len / sizeof(*__tmp->type); \
0803 })
0804
0805 /**
0806  * kfifo_out_peek - gets some data from the fifo
0807  * @fifo: address of the fifo to be used
0808  * @buf: pointer to the storage buffer
0809  * @n: max. number of elements to get
0810  *
0811  * This macro get the data from the fifo and return the numbers of elements
0812  * copied. The data is not removed from the fifo.
0813  *
0814  * Note that with only one concurrent reader and one concurrent
0815  * writer, you don't need extra locking to use these macro.
0816  */
0817 #define kfifo_out_peek(fifo, buf, n) \
0818 __kfifo_uint_must_check_helper( \
0819 ({ \
0820     typeof((fifo) + 1) __tmp = (fifo); \
0821     typeof(__tmp->ptr) __buf = (buf); \
0822     unsigned long __n = (n); \
0823     const size_t __recsize = sizeof(*__tmp->rectype); \
0824     struct __kfifo *__kfifo = &__tmp->kfifo; \
0825     (__recsize) ? \
0826     __kfifo_out_peek_r(__kfifo, __buf, __n, __recsize) : \
0827     __kfifo_out_peek(__kfifo, __buf, __n); \
0828 }) \
0829 )
0830
0831 extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
0832     size_t esize, gfp_t gfp_mask);
0833
0834 extern void __kfifo_free(struct __kfifo *fifo);
0835
0836 extern int __kfifo_init(struct __kfifo *fifo, void *buffer,
0837     unsigned int size, size_t esize);
0838
0839 extern unsigned int __kfifo_in(struct __kfifo *fifo,
0840     const void *buf, unsigned int len);
0841
0842 extern unsigned int __kfifo_out(struct __kfifo *fifo,
0843     void *buf, unsigned int len);
0844
0845 extern int __kfifo_from_user(struct __kfifo *fifo,
0846     const void __user *from, unsigned long len, unsigned int *copied);
0847
0848 extern int __kfifo_to_user(struct __kfifo *fifo,
0849     void __user *to, unsigned long len, unsigned int *copied);
0850
0851 extern unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo,
0852     struct scatterlist *sgl, int nents, unsigned int len);
0853
0854 extern unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo,
0855     struct scatterlist *sgl, int nents, unsigned int len);
0856
0857 extern unsigned int __kfifo_out_peek(struct __kfifo *fifo,
0858     void *buf, unsigned int len);
0859
0860 extern unsigned int __kfifo_in_r(struct __kfifo *fifo,
0861     const void *buf, unsigned int len, size_t recsize);
0862
0863 extern unsigned int __kfifo_out_r(struct __kfifo *fifo,
0864     void *buf, unsigned int len, size_t recsize);
0865
0866 extern int __kfifo_from_user_r(struct __kfifo *fifo,
0867     const void __user *from, unsigned long len, unsigned int *copied,
0868     size_t recsize);
0869
0870 extern int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to,
0871     unsigned long len, unsigned int *copied, size_t recsize);
0872
0873 extern unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo,
0874     struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
0875
0876 extern void __kfifo_dma_in_finish_r(struct __kfifo *fifo,
0877     unsigned int len, size_t recsize);
0878
0879 extern unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo,
0880     struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
0881
0882 extern void __kfifo_dma_out_finish_r(struct __kfifo *fifo, size_t recsize);
0883
0884 extern unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize);
0885
0886 extern void __kfifo_skip_r(struct __kfifo *fifo, size_t recsize);
0887
0888 extern unsigned int __kfifo_out_peek_r(struct __kfifo *fifo,
0889     void *buf, unsigned int len, size_t recsize);
0890
0891 extern unsigned int __kfifo_max_r(unsigned int len, size_t recsize);
0892
0893 #endif