Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * DMA Engine test module
0004  *
0005  * Copyright (C) 2007 Atmel Corporation
0006  * Copyright (C) 2013 Intel Corporation
0007  */
0008 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0009 
0010 #include <linux/err.h>
0011 #include <linux/delay.h>
0012 #include <linux/dma-mapping.h>
0013 #include <linux/dmaengine.h>
0014 #include <linux/freezer.h>
0015 #include <linux/init.h>
0016 #include <linux/kthread.h>
0017 #include <linux/sched/task.h>
0018 #include <linux/module.h>
0019 #include <linux/moduleparam.h>
0020 #include <linux/random.h>
0021 #include <linux/slab.h>
0022 #include <linux/wait.h>
0023 
0024 static unsigned int test_buf_size = 16384;
0025 module_param(test_buf_size, uint, 0644);
0026 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
0027 
0028 static char test_device[32];
0029 module_param_string(device, test_device, sizeof(test_device), 0644);
0030 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
0031 
0032 static unsigned int threads_per_chan = 1;
0033 module_param(threads_per_chan, uint, 0644);
0034 MODULE_PARM_DESC(threads_per_chan,
0035         "Number of threads to start per channel (default: 1)");
0036 
0037 static unsigned int max_channels;
0038 module_param(max_channels, uint, 0644);
0039 MODULE_PARM_DESC(max_channels,
0040         "Maximum number of channels to use (default: all)");
0041 
0042 static unsigned int iterations;
0043 module_param(iterations, uint, 0644);
0044 MODULE_PARM_DESC(iterations,
0045         "Iterations before stopping test (default: infinite)");
0046 
0047 static unsigned int dmatest;
0048 module_param(dmatest, uint, 0644);
0049 MODULE_PARM_DESC(dmatest,
0050         "dmatest 0-memcpy 1-memset (default: 0)");
0051 
0052 static unsigned int xor_sources = 3;
0053 module_param(xor_sources, uint, 0644);
0054 MODULE_PARM_DESC(xor_sources,
0055         "Number of xor source buffers (default: 3)");
0056 
0057 static unsigned int pq_sources = 3;
0058 module_param(pq_sources, uint, 0644);
0059 MODULE_PARM_DESC(pq_sources,
0060         "Number of p+q source buffers (default: 3)");
0061 
0062 static int timeout = 3000;
0063 module_param(timeout, int, 0644);
0064 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
0065          "Pass -1 for infinite timeout");
0066 
0067 static bool noverify;
0068 module_param(noverify, bool, 0644);
0069 MODULE_PARM_DESC(noverify, "Disable data verification (default: verify)");
0070 
0071 static bool norandom;
0072 module_param(norandom, bool, 0644);
0073 MODULE_PARM_DESC(norandom, "Disable random offset setup (default: random)");
0074 
0075 static bool verbose;
0076 module_param(verbose, bool, 0644);
0077 MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
0078 
0079 static int alignment = -1;
0080 module_param(alignment, int, 0644);
0081 MODULE_PARM_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))");
0082 
0083 static unsigned int transfer_size;
0084 module_param(transfer_size, uint, 0644);
0085 MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default: not used (0))");
0086 
0087 static bool polled;
0088 module_param(polled, bool, 0644);
0089 MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts");
0090 
0091 /**
0092  * struct dmatest_params - test parameters.
0093  * @buf_size:       size of the memcpy test buffer
0094  * @channel:        bus ID of the channel to test
0095  * @device:     bus ID of the DMA Engine to test
0096  * @threads_per_chan:   number of threads to start per channel
0097  * @max_channels:   maximum number of channels to use
0098  * @iterations:     iterations before stopping test
0099  * @xor_sources:    number of xor source buffers
0100  * @pq_sources:     number of p+q source buffers
0101  * @timeout:        transfer timeout in msec, -1 for infinite timeout
0102  * @noverify:       disable data verification
0103  * @norandom:       disable random offset setup
0104  * @alignment:      custom data address alignment taken as 2^alignment
0105  * @transfer_size:  custom transfer size in bytes
0106  * @polled:     use polling for completion instead of interrupts
0107  */
0108 struct dmatest_params {
0109     unsigned int    buf_size;
0110     char        channel[20];
0111     char        device[32];
0112     unsigned int    threads_per_chan;
0113     unsigned int    max_channels;
0114     unsigned int    iterations;
0115     unsigned int    xor_sources;
0116     unsigned int    pq_sources;
0117     int     timeout;
0118     bool        noverify;
0119     bool        norandom;
0120     int     alignment;
0121     unsigned int    transfer_size;
0122     bool        polled;
0123 };
0124 
0125 /**
0126  * struct dmatest_info - test information.
0127  * @params:     test parameters
0128  * @channels:       channels under test
0129  * @nr_channels:    number of channels under test
0130  * @lock:       access protection to the fields of this structure
0131  * @did_init:       module has been initialized completely
0132  * @last_error:     test has faced configuration issues
0133  */
0134 static struct dmatest_info {
0135     /* Test parameters */
0136     struct dmatest_params   params;
0137 
0138     /* Internal state */
0139     struct list_head    channels;
0140     unsigned int        nr_channels;
0141     int         last_error;
0142     struct mutex        lock;
0143     bool            did_init;
0144 } test_info = {
0145     .channels = LIST_HEAD_INIT(test_info.channels),
0146     .lock = __MUTEX_INITIALIZER(test_info.lock),
0147 };
0148 
0149 static int dmatest_run_set(const char *val, const struct kernel_param *kp);
0150 static int dmatest_run_get(char *val, const struct kernel_param *kp);
0151 static const struct kernel_param_ops run_ops = {
0152     .set = dmatest_run_set,
0153     .get = dmatest_run_get,
0154 };
0155 static bool dmatest_run;
0156 module_param_cb(run, &run_ops, &dmatest_run, 0644);
0157 MODULE_PARM_DESC(run, "Run the test (default: false)");
0158 
0159 static int dmatest_chan_set(const char *val, const struct kernel_param *kp);
0160 static int dmatest_chan_get(char *val, const struct kernel_param *kp);
0161 static const struct kernel_param_ops multi_chan_ops = {
0162     .set = dmatest_chan_set,
0163     .get = dmatest_chan_get,
0164 };
0165 
0166 static char test_channel[20];
0167 static struct kparam_string newchan_kps = {
0168     .string = test_channel,
0169     .maxlen = 20,
0170 };
0171 module_param_cb(channel, &multi_chan_ops, &newchan_kps, 0644);
0172 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
0173 
0174 static int dmatest_test_list_get(char *val, const struct kernel_param *kp);
0175 static const struct kernel_param_ops test_list_ops = {
0176     .get = dmatest_test_list_get,
0177 };
0178 module_param_cb(test_list, &test_list_ops, NULL, 0444);
0179 MODULE_PARM_DESC(test_list, "Print current test list");
0180 
0181 /* Maximum amount of mismatched bytes in buffer to print */
0182 #define MAX_ERROR_COUNT     32
0183 
0184 /*
0185  * Initialization patterns. All bytes in the source buffer has bit 7
0186  * set, all bytes in the destination buffer has bit 7 cleared.
0187  *
0188  * Bit 6 is set for all bytes which are to be copied by the DMA
0189  * engine. Bit 5 is set for all bytes which are to be overwritten by
0190  * the DMA engine.
0191  *
0192  * The remaining bits are the inverse of a counter which increments by
0193  * one for each byte address.
0194  */
0195 #define PATTERN_SRC     0x80
0196 #define PATTERN_DST     0x00
0197 #define PATTERN_COPY        0x40
0198 #define PATTERN_OVERWRITE   0x20
0199 #define PATTERN_COUNT_MASK  0x1f
0200 #define PATTERN_MEMSET_IDX  0x01
0201 
0202 /* Fixed point arithmetic ops */
0203 #define FIXPT_SHIFT     8
0204 #define FIXPNT_MASK     0xFF
0205 #define FIXPT_TO_INT(a) ((a) >> FIXPT_SHIFT)
0206 #define INT_TO_FIXPT(a) ((a) << FIXPT_SHIFT)
0207 #define FIXPT_GET_FRAC(a)   ((((a) & FIXPNT_MASK) * 100) >> FIXPT_SHIFT)
0208 
0209 /* poor man's completion - we want to use wait_event_freezable() on it */
0210 struct dmatest_done {
0211     bool            done;
0212     wait_queue_head_t   *wait;
0213 };
0214 
0215 struct dmatest_data {
0216     u8      **raw;
0217     u8      **aligned;
0218     unsigned int    cnt;
0219     unsigned int    off;
0220 };
0221 
0222 struct dmatest_thread {
0223     struct list_head    node;
0224     struct dmatest_info *info;
0225     struct task_struct  *task;
0226     struct dma_chan     *chan;
0227     struct dmatest_data src;
0228     struct dmatest_data dst;
0229     enum dma_transaction_type type;
0230     wait_queue_head_t done_wait;
0231     struct dmatest_done test_done;
0232     bool            done;
0233     bool            pending;
0234 };
0235 
0236 struct dmatest_chan {
0237     struct list_head    node;
0238     struct dma_chan     *chan;
0239     struct list_head    threads;
0240 };
0241 
0242 static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
0243 static bool wait;
0244 
0245 static bool is_threaded_test_run(struct dmatest_info *info)
0246 {
0247     struct dmatest_chan *dtc;
0248 
0249     list_for_each_entry(dtc, &info->channels, node) {
0250         struct dmatest_thread *thread;
0251 
0252         list_for_each_entry(thread, &dtc->threads, node) {
0253             if (!thread->done && !thread->pending)
0254                 return true;
0255         }
0256     }
0257 
0258     return false;
0259 }
0260 
0261 static bool is_threaded_test_pending(struct dmatest_info *info)
0262 {
0263     struct dmatest_chan *dtc;
0264 
0265     list_for_each_entry(dtc, &info->channels, node) {
0266         struct dmatest_thread *thread;
0267 
0268         list_for_each_entry(thread, &dtc->threads, node) {
0269             if (thread->pending)
0270                 return true;
0271         }
0272     }
0273 
0274     return false;
0275 }
0276 
0277 static int dmatest_wait_get(char *val, const struct kernel_param *kp)
0278 {
0279     struct dmatest_info *info = &test_info;
0280     struct dmatest_params *params = &info->params;
0281 
0282     if (params->iterations)
0283         wait_event(thread_wait, !is_threaded_test_run(info));
0284     wait = true;
0285     return param_get_bool(val, kp);
0286 }
0287 
0288 static const struct kernel_param_ops wait_ops = {
0289     .get = dmatest_wait_get,
0290     .set = param_set_bool,
0291 };
0292 module_param_cb(wait, &wait_ops, &wait, 0444);
0293 MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
0294 
0295 static bool dmatest_match_channel(struct dmatest_params *params,
0296         struct dma_chan *chan)
0297 {
0298     if (params->channel[0] == '\0')
0299         return true;
0300     return strcmp(dma_chan_name(chan), params->channel) == 0;
0301 }
0302 
0303 static bool dmatest_match_device(struct dmatest_params *params,
0304         struct dma_device *device)
0305 {
0306     if (params->device[0] == '\0')
0307         return true;
0308     return strcmp(dev_name(device->dev), params->device) == 0;
0309 }
0310 
0311 static unsigned long dmatest_random(void)
0312 {
0313     unsigned long buf;
0314 
0315     prandom_bytes(&buf, sizeof(buf));
0316     return buf;
0317 }
0318 
0319 static inline u8 gen_inv_idx(u8 index, bool is_memset)
0320 {
0321     u8 val = is_memset ? PATTERN_MEMSET_IDX : index;
0322 
0323     return ~val & PATTERN_COUNT_MASK;
0324 }
0325 
0326 static inline u8 gen_src_value(u8 index, bool is_memset)
0327 {
0328     return PATTERN_SRC | gen_inv_idx(index, is_memset);
0329 }
0330 
0331 static inline u8 gen_dst_value(u8 index, bool is_memset)
0332 {
0333     return PATTERN_DST | gen_inv_idx(index, is_memset);
0334 }
0335 
0336 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
0337         unsigned int buf_size, bool is_memset)
0338 {
0339     unsigned int i;
0340     u8 *buf;
0341 
0342     for (; (buf = *bufs); bufs++) {
0343         for (i = 0; i < start; i++)
0344             buf[i] = gen_src_value(i, is_memset);
0345         for ( ; i < start + len; i++)
0346             buf[i] = gen_src_value(i, is_memset) | PATTERN_COPY;
0347         for ( ; i < buf_size; i++)
0348             buf[i] = gen_src_value(i, is_memset);
0349         buf++;
0350     }
0351 }
0352 
0353 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
0354         unsigned int buf_size, bool is_memset)
0355 {
0356     unsigned int i;
0357     u8 *buf;
0358 
0359     for (; (buf = *bufs); bufs++) {
0360         for (i = 0; i < start; i++)
0361             buf[i] = gen_dst_value(i, is_memset);
0362         for ( ; i < start + len; i++)
0363             buf[i] = gen_dst_value(i, is_memset) |
0364                         PATTERN_OVERWRITE;
0365         for ( ; i < buf_size; i++)
0366             buf[i] = gen_dst_value(i, is_memset);
0367     }
0368 }
0369 
0370 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
0371         unsigned int counter, bool is_srcbuf, bool is_memset)
0372 {
0373     u8      diff = actual ^ pattern;
0374     u8      expected = pattern | gen_inv_idx(counter, is_memset);
0375     const char  *thread_name = current->comm;
0376 
0377     if (is_srcbuf)
0378         pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
0379             thread_name, index, expected, actual);
0380     else if ((pattern & PATTERN_COPY)
0381             && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
0382         pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
0383             thread_name, index, expected, actual);
0384     else if (diff & PATTERN_SRC)
0385         pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
0386             thread_name, index, expected, actual);
0387     else
0388         pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
0389             thread_name, index, expected, actual);
0390 }
0391 
0392 static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
0393         unsigned int end, unsigned int counter, u8 pattern,
0394         bool is_srcbuf, bool is_memset)
0395 {
0396     unsigned int i;
0397     unsigned int error_count = 0;
0398     u8 actual;
0399     u8 expected;
0400     u8 *buf;
0401     unsigned int counter_orig = counter;
0402 
0403     for (; (buf = *bufs); bufs++) {
0404         counter = counter_orig;
0405         for (i = start; i < end; i++) {
0406             actual = buf[i];
0407             expected = pattern | gen_inv_idx(counter, is_memset);
0408             if (actual != expected) {
0409                 if (error_count < MAX_ERROR_COUNT)
0410                     dmatest_mismatch(actual, pattern, i,
0411                              counter, is_srcbuf,
0412                              is_memset);
0413                 error_count++;
0414             }
0415             counter++;
0416         }
0417     }
0418 
0419     if (error_count > MAX_ERROR_COUNT)
0420         pr_warn("%s: %u errors suppressed\n",
0421             current->comm, error_count - MAX_ERROR_COUNT);
0422 
0423     return error_count;
0424 }
0425 
0426 
0427 static void dmatest_callback(void *arg)
0428 {
0429     struct dmatest_done *done = arg;
0430     struct dmatest_thread *thread =
0431         container_of(done, struct dmatest_thread, test_done);
0432     if (!thread->done) {
0433         done->done = true;
0434         wake_up_all(done->wait);
0435     } else {
0436         /*
0437          * If thread->done, it means that this callback occurred
0438          * after the parent thread has cleaned up. This can
0439          * happen in the case that driver doesn't implement
0440          * the terminate_all() functionality and a dma operation
0441          * did not occur within the timeout period
0442          */
0443         WARN(1, "dmatest: Kernel memory may be corrupted!!\n");
0444     }
0445 }
0446 
0447 static unsigned int min_odd(unsigned int x, unsigned int y)
0448 {
0449     unsigned int val = min(x, y);
0450 
0451     return val % 2 ? val : val - 1;
0452 }
0453 
0454 static void result(const char *err, unsigned int n, unsigned int src_off,
0455            unsigned int dst_off, unsigned int len, unsigned long data)
0456 {
0457     if (IS_ERR_VALUE(data)) {
0458         pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%ld)\n",
0459             current->comm, n, err, src_off, dst_off, len, data);
0460     } else {
0461         pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
0462             current->comm, n, err, src_off, dst_off, len, data);
0463     }
0464 }
0465 
0466 static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
0467                unsigned int dst_off, unsigned int len,
0468                unsigned long data)
0469 {
0470     pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
0471          current->comm, n, err, src_off, dst_off, len, data);
0472 }
0473 
0474 #define verbose_result(err, n, src_off, dst_off, len, data) ({  \
0475     if (verbose)                        \
0476         result(err, n, src_off, dst_off, len, data);    \
0477     else                            \
0478         dbg_result(err, n, src_off, dst_off, len, data);\
0479 })
0480 
0481 static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
0482 {
0483     unsigned long long per_sec = 1000000;
0484 
0485     if (runtime <= 0)
0486         return 0;
0487 
0488     /* drop precision until runtime is 32-bits */
0489     while (runtime > UINT_MAX) {
0490         runtime >>= 1;
0491         per_sec <<= 1;
0492     }
0493 
0494     per_sec *= val;
0495     per_sec = INT_TO_FIXPT(per_sec);
0496     do_div(per_sec, runtime);
0497 
0498     return per_sec;
0499 }
0500 
0501 static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
0502 {
0503     return FIXPT_TO_INT(dmatest_persec(runtime, len >> 10));
0504 }
0505 
0506 static void __dmatest_free_test_data(struct dmatest_data *d, unsigned int cnt)
0507 {
0508     unsigned int i;
0509 
0510     for (i = 0; i < cnt; i++)
0511         kfree(d->raw[i]);
0512 
0513     kfree(d->aligned);
0514     kfree(d->raw);
0515 }
0516 
0517 static void dmatest_free_test_data(struct dmatest_data *d)
0518 {
0519     __dmatest_free_test_data(d, d->cnt);
0520 }
0521 
0522 static int dmatest_alloc_test_data(struct dmatest_data *d,
0523         unsigned int buf_size, u8 align)
0524 {
0525     unsigned int i = 0;
0526 
0527     d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
0528     if (!d->raw)
0529         return -ENOMEM;
0530 
0531     d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
0532     if (!d->aligned)
0533         goto err;
0534 
0535     for (i = 0; i < d->cnt; i++) {
0536         d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL);
0537         if (!d->raw[i])
0538             goto err;
0539 
0540         /* align to alignment restriction */
0541         if (align)
0542             d->aligned[i] = PTR_ALIGN(d->raw[i], align);
0543         else
0544             d->aligned[i] = d->raw[i];
0545     }
0546 
0547     return 0;
0548 err:
0549     __dmatest_free_test_data(d, i);
0550     return -ENOMEM;
0551 }
0552 
0553 /*
0554  * This function repeatedly tests DMA transfers of various lengths and
0555  * offsets for a given operation type until it is told to exit by
0556  * kthread_stop(). There may be multiple threads running this function
0557  * in parallel for a single channel, and there may be multiple channels
0558  * being tested in parallel.
0559  *
0560  * Before each test, the source and destination buffer is initialized
0561  * with a known pattern. This pattern is different depending on
0562  * whether it's in an area which is supposed to be copied or
0563  * overwritten, and different in the source and destination buffers.
0564  * So if the DMA engine doesn't copy exactly what we tell it to copy,
0565  * we'll notice.
0566  */
0567 static int dmatest_func(void *data)
0568 {
0569     struct dmatest_thread   *thread = data;
0570     struct dmatest_done *done = &thread->test_done;
0571     struct dmatest_info *info;
0572     struct dmatest_params   *params;
0573     struct dma_chan     *chan;
0574     struct dma_device   *dev;
0575     struct device       *dma_dev;
0576     unsigned int        error_count;
0577     unsigned int        failed_tests = 0;
0578     unsigned int        total_tests = 0;
0579     dma_cookie_t        cookie;
0580     enum dma_status     status;
0581     enum dma_ctrl_flags flags;
0582     u8          *pq_coefs = NULL;
0583     int         ret;
0584     unsigned int        buf_size;
0585     struct dmatest_data *src;
0586     struct dmatest_data *dst;
0587     int         i;
0588     ktime_t         ktime, start, diff;
0589     ktime_t         filltime = 0;
0590     ktime_t         comparetime = 0;
0591     s64         runtime = 0;
0592     unsigned long long  total_len = 0;
0593     unsigned long long  iops = 0;
0594     u8          align = 0;
0595     bool            is_memset = false;
0596     dma_addr_t      *srcs;
0597     dma_addr_t      *dma_pq;
0598 
0599     set_freezable();
0600 
0601     ret = -ENOMEM;
0602 
0603     smp_rmb();
0604     thread->pending = false;
0605     info = thread->info;
0606     params = &info->params;
0607     chan = thread->chan;
0608     dev = chan->device;
0609     dma_dev = dmaengine_get_dma_device(chan);
0610 
0611     src = &thread->src;
0612     dst = &thread->dst;
0613     if (thread->type == DMA_MEMCPY) {
0614         align = params->alignment < 0 ? dev->copy_align :
0615                         params->alignment;
0616         src->cnt = dst->cnt = 1;
0617     } else if (thread->type == DMA_MEMSET) {
0618         align = params->alignment < 0 ? dev->fill_align :
0619                         params->alignment;
0620         src->cnt = dst->cnt = 1;
0621         is_memset = true;
0622     } else if (thread->type == DMA_XOR) {
0623         /* force odd to ensure dst = src */
0624         src->cnt = min_odd(params->xor_sources | 1, dev->max_xor);
0625         dst->cnt = 1;
0626         align = params->alignment < 0 ? dev->xor_align :
0627                         params->alignment;
0628     } else if (thread->type == DMA_PQ) {
0629         /* force odd to ensure dst = src */
0630         src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
0631         dst->cnt = 2;
0632         align = params->alignment < 0 ? dev->pq_align :
0633                         params->alignment;
0634 
0635         pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);
0636         if (!pq_coefs)
0637             goto err_thread_type;
0638 
0639         for (i = 0; i < src->cnt; i++)
0640             pq_coefs[i] = 1;
0641     } else
0642         goto err_thread_type;
0643 
0644     /* Check if buffer count fits into map count variable (u8) */
0645     if ((src->cnt + dst->cnt) >= 255) {
0646         pr_err("too many buffers (%d of 255 supported)\n",
0647                src->cnt + dst->cnt);
0648         goto err_free_coefs;
0649     }
0650 
0651     buf_size = params->buf_size;
0652     if (1 << align > buf_size) {
0653         pr_err("%u-byte buffer too small for %d-byte alignment\n",
0654                buf_size, 1 << align);
0655         goto err_free_coefs;
0656     }
0657 
0658     if (dmatest_alloc_test_data(src, buf_size, align) < 0)
0659         goto err_free_coefs;
0660 
0661     if (dmatest_alloc_test_data(dst, buf_size, align) < 0)
0662         goto err_src;
0663 
0664     set_user_nice(current, 10);
0665 
0666     srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL);
0667     if (!srcs)
0668         goto err_dst;
0669 
0670     dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL);
0671     if (!dma_pq)
0672         goto err_srcs_array;
0673 
0674     /*
0675      * src and dst buffers are freed by ourselves below
0676      */
0677     if (params->polled)
0678         flags = DMA_CTRL_ACK;
0679     else
0680         flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
0681 
0682     ktime = ktime_get();
0683     while (!(kthread_should_stop() ||
0684            (params->iterations && total_tests >= params->iterations))) {
0685         struct dma_async_tx_descriptor *tx = NULL;
0686         struct dmaengine_unmap_data *um;
0687         dma_addr_t *dsts;
0688         unsigned int len;
0689 
0690         total_tests++;
0691 
0692         if (params->transfer_size) {
0693             if (params->transfer_size >= buf_size) {
0694                 pr_err("%u-byte transfer size must be lower than %u-buffer size\n",
0695                        params->transfer_size, buf_size);
0696                 break;
0697             }
0698             len = params->transfer_size;
0699         } else if (params->norandom) {
0700             len = buf_size;
0701         } else {
0702             len = dmatest_random() % buf_size + 1;
0703         }
0704 
0705         /* Do not alter transfer size explicitly defined by user */
0706         if (!params->transfer_size) {
0707             len = (len >> align) << align;
0708             if (!len)
0709                 len = 1 << align;
0710         }
0711         total_len += len;
0712 
0713         if (params->norandom) {
0714             src->off = 0;
0715             dst->off = 0;
0716         } else {
0717             src->off = dmatest_random() % (buf_size - len + 1);
0718             dst->off = dmatest_random() % (buf_size - len + 1);
0719 
0720             src->off = (src->off >> align) << align;
0721             dst->off = (dst->off >> align) << align;
0722         }
0723 
0724         if (!params->noverify) {
0725             start = ktime_get();
0726             dmatest_init_srcs(src->aligned, src->off, len,
0727                       buf_size, is_memset);
0728             dmatest_init_dsts(dst->aligned, dst->off, len,
0729                       buf_size, is_memset);
0730 
0731             diff = ktime_sub(ktime_get(), start);
0732             filltime = ktime_add(filltime, diff);
0733         }
0734 
0735         um = dmaengine_get_unmap_data(dma_dev, src->cnt + dst->cnt,
0736                           GFP_KERNEL);
0737         if (!um) {
0738             failed_tests++;
0739             result("unmap data NULL", total_tests,
0740                    src->off, dst->off, len, ret);
0741             continue;
0742         }
0743 
0744         um->len = buf_size;
0745         for (i = 0; i < src->cnt; i++) {
0746             void *buf = src->aligned[i];
0747             struct page *pg = virt_to_page(buf);
0748             unsigned long pg_off = offset_in_page(buf);
0749 
0750             um->addr[i] = dma_map_page(dma_dev, pg, pg_off,
0751                            um->len, DMA_TO_DEVICE);
0752             srcs[i] = um->addr[i] + src->off;
0753             ret = dma_mapping_error(dma_dev, um->addr[i]);
0754             if (ret) {
0755                 result("src mapping error", total_tests,
0756                        src->off, dst->off, len, ret);
0757                 goto error_unmap_continue;
0758             }
0759             um->to_cnt++;
0760         }
0761         /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
0762         dsts = &um->addr[src->cnt];
0763         for (i = 0; i < dst->cnt; i++) {
0764             void *buf = dst->aligned[i];
0765             struct page *pg = virt_to_page(buf);
0766             unsigned long pg_off = offset_in_page(buf);
0767 
0768             dsts[i] = dma_map_page(dma_dev, pg, pg_off, um->len,
0769                            DMA_BIDIRECTIONAL);
0770             ret = dma_mapping_error(dma_dev, dsts[i]);
0771             if (ret) {
0772                 result("dst mapping error", total_tests,
0773                        src->off, dst->off, len, ret);
0774                 goto error_unmap_continue;
0775             }
0776             um->bidi_cnt++;
0777         }
0778 
0779         if (thread->type == DMA_MEMCPY)
0780             tx = dev->device_prep_dma_memcpy(chan,
0781                              dsts[0] + dst->off,
0782                              srcs[0], len, flags);
0783         else if (thread->type == DMA_MEMSET)
0784             tx = dev->device_prep_dma_memset(chan,
0785                         dsts[0] + dst->off,
0786                         *(src->aligned[0] + src->off),
0787                         len, flags);
0788         else if (thread->type == DMA_XOR)
0789             tx = dev->device_prep_dma_xor(chan,
0790                               dsts[0] + dst->off,
0791                               srcs, src->cnt,
0792                               len, flags);
0793         else if (thread->type == DMA_PQ) {
0794             for (i = 0; i < dst->cnt; i++)
0795                 dma_pq[i] = dsts[i] + dst->off;
0796             tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
0797                              src->cnt, pq_coefs,
0798                              len, flags);
0799         }
0800 
0801         if (!tx) {
0802             result("prep error", total_tests, src->off,
0803                    dst->off, len, ret);
0804             msleep(100);
0805             goto error_unmap_continue;
0806         }
0807 
0808         done->done = false;
0809         if (!params->polled) {
0810             tx->callback = dmatest_callback;
0811             tx->callback_param = done;
0812         }
0813         cookie = tx->tx_submit(tx);
0814 
0815         if (dma_submit_error(cookie)) {
0816             result("submit error", total_tests, src->off,
0817                    dst->off, len, ret);
0818             msleep(100);
0819             goto error_unmap_continue;
0820         }
0821 
0822         if (params->polled) {
0823             status = dma_sync_wait(chan, cookie);
0824             dmaengine_terminate_sync(chan);
0825             if (status == DMA_COMPLETE)
0826                 done->done = true;
0827         } else {
0828             dma_async_issue_pending(chan);
0829 
0830             wait_event_freezable_timeout(thread->done_wait,
0831                     done->done,
0832                     msecs_to_jiffies(params->timeout));
0833 
0834             status = dma_async_is_tx_complete(chan, cookie, NULL,
0835                               NULL);
0836         }
0837 
0838         if (!done->done) {
0839             result("test timed out", total_tests, src->off, dst->off,
0840                    len, 0);
0841             goto error_unmap_continue;
0842         } else if (status != DMA_COMPLETE &&
0843                !(dma_has_cap(DMA_COMPLETION_NO_ORDER,
0844                      dev->cap_mask) &&
0845                  status == DMA_OUT_OF_ORDER)) {
0846             result(status == DMA_ERROR ?
0847                    "completion error status" :
0848                    "completion busy status", total_tests, src->off,
0849                    dst->off, len, ret);
0850             goto error_unmap_continue;
0851         }
0852 
0853         dmaengine_unmap_put(um);
0854 
0855         if (params->noverify) {
0856             verbose_result("test passed", total_tests, src->off,
0857                        dst->off, len, 0);
0858             continue;
0859         }
0860 
0861         start = ktime_get();
0862         pr_debug("%s: verifying source buffer...\n", current->comm);
0863         error_count = dmatest_verify(src->aligned, 0, src->off,
0864                 0, PATTERN_SRC, true, is_memset);
0865         error_count += dmatest_verify(src->aligned, src->off,
0866                 src->off + len, src->off,
0867                 PATTERN_SRC | PATTERN_COPY, true, is_memset);
0868         error_count += dmatest_verify(src->aligned, src->off + len,
0869                 buf_size, src->off + len,
0870                 PATTERN_SRC, true, is_memset);
0871 
0872         pr_debug("%s: verifying dest buffer...\n", current->comm);
0873         error_count += dmatest_verify(dst->aligned, 0, dst->off,
0874                 0, PATTERN_DST, false, is_memset);
0875 
0876         error_count += dmatest_verify(dst->aligned, dst->off,
0877                 dst->off + len, src->off,
0878                 PATTERN_SRC | PATTERN_COPY, false, is_memset);
0879 
0880         error_count += dmatest_verify(dst->aligned, dst->off + len,
0881                 buf_size, dst->off + len,
0882                 PATTERN_DST, false, is_memset);
0883 
0884         diff = ktime_sub(ktime_get(), start);
0885         comparetime = ktime_add(comparetime, diff);
0886 
0887         if (error_count) {
0888             result("data error", total_tests, src->off, dst->off,
0889                    len, error_count);
0890             failed_tests++;
0891         } else {
0892             verbose_result("test passed", total_tests, src->off,
0893                        dst->off, len, 0);
0894         }
0895 
0896         continue;
0897 
0898 error_unmap_continue:
0899         dmaengine_unmap_put(um);
0900         failed_tests++;
0901     }
0902     ktime = ktime_sub(ktime_get(), ktime);
0903     ktime = ktime_sub(ktime, comparetime);
0904     ktime = ktime_sub(ktime, filltime);
0905     runtime = ktime_to_us(ktime);
0906 
0907     ret = 0;
0908     kfree(dma_pq);
0909 err_srcs_array:
0910     kfree(srcs);
0911 err_dst:
0912     dmatest_free_test_data(dst);
0913 err_src:
0914     dmatest_free_test_data(src);
0915 err_free_coefs:
0916     kfree(pq_coefs);
0917 err_thread_type:
0918     iops = dmatest_persec(runtime, total_tests);
0919     pr_info("%s: summary %u tests, %u failures %llu.%02llu iops %llu KB/s (%d)\n",
0920         current->comm, total_tests, failed_tests,
0921         FIXPT_TO_INT(iops), FIXPT_GET_FRAC(iops),
0922         dmatest_KBs(runtime, total_len), ret);
0923 
0924     /* terminate all transfers on specified channels */
0925     if (ret || failed_tests)
0926         dmaengine_terminate_sync(chan);
0927 
0928     thread->done = true;
0929     wake_up(&thread_wait);
0930 
0931     return ret;
0932 }
0933 
0934 static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
0935 {
0936     struct dmatest_thread   *thread;
0937     struct dmatest_thread   *_thread;
0938     int         ret;
0939 
0940     list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
0941         ret = kthread_stop(thread->task);
0942         pr_debug("thread %s exited with status %d\n",
0943              thread->task->comm, ret);
0944         list_del(&thread->node);
0945         put_task_struct(thread->task);
0946         kfree(thread);
0947     }
0948 
0949     /* terminate all transfers on specified channels */
0950     dmaengine_terminate_sync(dtc->chan);
0951 
0952     kfree(dtc);
0953 }
0954 
0955 static int dmatest_add_threads(struct dmatest_info *info,
0956         struct dmatest_chan *dtc, enum dma_transaction_type type)
0957 {
0958     struct dmatest_params *params = &info->params;
0959     struct dmatest_thread *thread;
0960     struct dma_chan *chan = dtc->chan;
0961     char *op;
0962     unsigned int i;
0963 
0964     if (type == DMA_MEMCPY)
0965         op = "copy";
0966     else if (type == DMA_MEMSET)
0967         op = "set";
0968     else if (type == DMA_XOR)
0969         op = "xor";
0970     else if (type == DMA_PQ)
0971         op = "pq";
0972     else
0973         return -EINVAL;
0974 
0975     for (i = 0; i < params->threads_per_chan; i++) {
0976         thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
0977         if (!thread) {
0978             pr_warn("No memory for %s-%s%u\n",
0979                 dma_chan_name(chan), op, i);
0980             break;
0981         }
0982         thread->info = info;
0983         thread->chan = dtc->chan;
0984         thread->type = type;
0985         thread->test_done.wait = &thread->done_wait;
0986         init_waitqueue_head(&thread->done_wait);
0987         smp_wmb();
0988         thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
0989                 dma_chan_name(chan), op, i);
0990         if (IS_ERR(thread->task)) {
0991             pr_warn("Failed to create thread %s-%s%u\n",
0992                 dma_chan_name(chan), op, i);
0993             kfree(thread);
0994             break;
0995         }
0996 
0997         /* srcbuf and dstbuf are allocated by the thread itself */
0998         get_task_struct(thread->task);
0999         list_add_tail(&thread->node, &dtc->threads);
1000         thread->pending = true;
1001     }
1002 
1003     return i;
1004 }
1005 
1006 static int dmatest_add_channel(struct dmatest_info *info,
1007         struct dma_chan *chan)
1008 {
1009     struct dmatest_chan *dtc;
1010     struct dma_device   *dma_dev = chan->device;
1011     unsigned int        thread_count = 0;
1012     int cnt;
1013 
1014     dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
1015     if (!dtc) {
1016         pr_warn("No memory for %s\n", dma_chan_name(chan));
1017         return -ENOMEM;
1018     }
1019 
1020     dtc->chan = chan;
1021     INIT_LIST_HEAD(&dtc->threads);
1022 
1023     if (dma_has_cap(DMA_COMPLETION_NO_ORDER, dma_dev->cap_mask) &&
1024         info->params.polled) {
1025         info->params.polled = false;
1026         pr_warn("DMA_COMPLETION_NO_ORDER, polled disabled\n");
1027     }
1028 
1029     if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1030         if (dmatest == 0) {
1031             cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
1032             thread_count += cnt > 0 ? cnt : 0;
1033         }
1034     }
1035 
1036     if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
1037         if (dmatest == 1) {
1038             cnt = dmatest_add_threads(info, dtc, DMA_MEMSET);
1039             thread_count += cnt > 0 ? cnt : 0;
1040         }
1041     }
1042 
1043     if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1044         cnt = dmatest_add_threads(info, dtc, DMA_XOR);
1045         thread_count += cnt > 0 ? cnt : 0;
1046     }
1047     if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1048         cnt = dmatest_add_threads(info, dtc, DMA_PQ);
1049         thread_count += cnt > 0 ? cnt : 0;
1050     }
1051 
1052     pr_info("Added %u threads using %s\n",
1053         thread_count, dma_chan_name(chan));
1054 
1055     list_add_tail(&dtc->node, &info->channels);
1056     info->nr_channels++;
1057 
1058     return 0;
1059 }
1060 
1061 static bool filter(struct dma_chan *chan, void *param)
1062 {
1063     return dmatest_match_channel(param, chan) && dmatest_match_device(param, chan->device);
1064 }
1065 
1066 static void request_channels(struct dmatest_info *info,
1067                  enum dma_transaction_type type)
1068 {
1069     dma_cap_mask_t mask;
1070 
1071     dma_cap_zero(mask);
1072     dma_cap_set(type, mask);
1073     for (;;) {
1074         struct dmatest_params *params = &info->params;
1075         struct dma_chan *chan;
1076 
1077         chan = dma_request_channel(mask, filter, params);
1078         if (chan) {
1079             if (dmatest_add_channel(info, chan)) {
1080                 dma_release_channel(chan);
1081                 break; /* add_channel failed, punt */
1082             }
1083         } else
1084             break; /* no more channels available */
1085         if (params->max_channels &&
1086             info->nr_channels >= params->max_channels)
1087             break; /* we have all we need */
1088     }
1089 }
1090 
1091 static void add_threaded_test(struct dmatest_info *info)
1092 {
1093     struct dmatest_params *params = &info->params;
1094 
1095     /* Copy test parameters */
1096     params->buf_size = test_buf_size;
1097     strscpy(params->channel, strim(test_channel), sizeof(params->channel));
1098     strscpy(params->device, strim(test_device), sizeof(params->device));
1099     params->threads_per_chan = threads_per_chan;
1100     params->max_channels = max_channels;
1101     params->iterations = iterations;
1102     params->xor_sources = xor_sources;
1103     params->pq_sources = pq_sources;
1104     params->timeout = timeout;
1105     params->noverify = noverify;
1106     params->norandom = norandom;
1107     params->alignment = alignment;
1108     params->transfer_size = transfer_size;
1109     params->polled = polled;
1110 
1111     request_channels(info, DMA_MEMCPY);
1112     request_channels(info, DMA_MEMSET);
1113     request_channels(info, DMA_XOR);
1114     request_channels(info, DMA_PQ);
1115 }
1116 
1117 static void run_pending_tests(struct dmatest_info *info)
1118 {
1119     struct dmatest_chan *dtc;
1120     unsigned int thread_count = 0;
1121 
1122     list_for_each_entry(dtc, &info->channels, node) {
1123         struct dmatest_thread *thread;
1124 
1125         thread_count = 0;
1126         list_for_each_entry(thread, &dtc->threads, node) {
1127             wake_up_process(thread->task);
1128             thread_count++;
1129         }
1130         pr_info("Started %u threads using %s\n",
1131             thread_count, dma_chan_name(dtc->chan));
1132     }
1133 }
1134 
1135 static void stop_threaded_test(struct dmatest_info *info)
1136 {
1137     struct dmatest_chan *dtc, *_dtc;
1138     struct dma_chan *chan;
1139 
1140     list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
1141         list_del(&dtc->node);
1142         chan = dtc->chan;
1143         dmatest_cleanup_channel(dtc);
1144         pr_debug("dropped channel %s\n", dma_chan_name(chan));
1145         dma_release_channel(chan);
1146     }
1147 
1148     info->nr_channels = 0;
1149 }
1150 
1151 static void start_threaded_tests(struct dmatest_info *info)
1152 {
1153     /* we might be called early to set run=, defer running until all
1154      * parameters have been evaluated
1155      */
1156     if (!info->did_init)
1157         return;
1158 
1159     run_pending_tests(info);
1160 }
1161 
1162 static int dmatest_run_get(char *val, const struct kernel_param *kp)
1163 {
1164     struct dmatest_info *info = &test_info;
1165 
1166     mutex_lock(&info->lock);
1167     if (is_threaded_test_run(info)) {
1168         dmatest_run = true;
1169     } else {
1170         if (!is_threaded_test_pending(info))
1171             stop_threaded_test(info);
1172         dmatest_run = false;
1173     }
1174     mutex_unlock(&info->lock);
1175 
1176     return param_get_bool(val, kp);
1177 }
1178 
1179 static int dmatest_run_set(const char *val, const struct kernel_param *kp)
1180 {
1181     struct dmatest_info *info = &test_info;
1182     int ret;
1183 
1184     mutex_lock(&info->lock);
1185     ret = param_set_bool(val, kp);
1186     if (ret) {
1187         mutex_unlock(&info->lock);
1188         return ret;
1189     } else if (dmatest_run) {
1190         if (!is_threaded_test_pending(info)) {
1191             /*
1192              * We have nothing to run. This can be due to:
1193              */
1194             ret = info->last_error;
1195             if (ret) {
1196                 /* 1) Misconfiguration */
1197                 pr_err("Channel misconfigured, can't continue\n");
1198                 mutex_unlock(&info->lock);
1199                 return ret;
1200             } else {
1201                 /* 2) We rely on defaults */
1202                 pr_info("No channels configured, continue with any\n");
1203                 if (!is_threaded_test_run(info))
1204                     stop_threaded_test(info);
1205                 add_threaded_test(info);
1206             }
1207         }
1208         start_threaded_tests(info);
1209     } else {
1210         stop_threaded_test(info);
1211     }
1212 
1213     mutex_unlock(&info->lock);
1214 
1215     return ret;
1216 }
1217 
1218 static int dmatest_chan_set(const char *val, const struct kernel_param *kp)
1219 {
1220     struct dmatest_info *info = &test_info;
1221     struct dmatest_chan *dtc;
1222     char chan_reset_val[20];
1223     int ret;
1224 
1225     mutex_lock(&info->lock);
1226     ret = param_set_copystring(val, kp);
1227     if (ret) {
1228         mutex_unlock(&info->lock);
1229         return ret;
1230     }
1231     /*Clear any previously run threads */
1232     if (!is_threaded_test_run(info) && !is_threaded_test_pending(info))
1233         stop_threaded_test(info);
1234     /* Reject channels that are already registered */
1235     if (is_threaded_test_pending(info)) {
1236         list_for_each_entry(dtc, &info->channels, node) {
1237             if (strcmp(dma_chan_name(dtc->chan),
1238                    strim(test_channel)) == 0) {
1239                 dtc = list_last_entry(&info->channels,
1240                               struct dmatest_chan,
1241                               node);
1242                 strscpy(chan_reset_val,
1243                     dma_chan_name(dtc->chan),
1244                     sizeof(chan_reset_val));
1245                 ret = -EBUSY;
1246                 goto add_chan_err;
1247             }
1248         }
1249     }
1250 
1251     add_threaded_test(info);
1252 
1253     /* Check if channel was added successfully */
1254     if (!list_empty(&info->channels)) {
1255         /*
1256          * if new channel was not successfully added, revert the
1257          * "test_channel" string to the name of the last successfully
1258          * added channel. exception for when users issues empty string
1259          * to channel parameter.
1260          */
1261         dtc = list_last_entry(&info->channels, struct dmatest_chan, node);
1262         if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0)
1263             && (strcmp("", strim(test_channel)) != 0)) {
1264             ret = -EINVAL;
1265             strscpy(chan_reset_val, dma_chan_name(dtc->chan),
1266                 sizeof(chan_reset_val));
1267             goto add_chan_err;
1268         }
1269 
1270     } else {
1271         /* Clear test_channel if no channels were added successfully */
1272         strscpy(chan_reset_val, "", sizeof(chan_reset_val));
1273         ret = -EBUSY;
1274         goto add_chan_err;
1275     }
1276 
1277     info->last_error = ret;
1278     mutex_unlock(&info->lock);
1279 
1280     return ret;
1281 
1282 add_chan_err:
1283     param_set_copystring(chan_reset_val, kp);
1284     info->last_error = ret;
1285     mutex_unlock(&info->lock);
1286 
1287     return ret;
1288 }
1289 
1290 static int dmatest_chan_get(char *val, const struct kernel_param *kp)
1291 {
1292     struct dmatest_info *info = &test_info;
1293 
1294     mutex_lock(&info->lock);
1295     if (!is_threaded_test_run(info) && !is_threaded_test_pending(info)) {
1296         stop_threaded_test(info);
1297         strscpy(test_channel, "", sizeof(test_channel));
1298     }
1299     mutex_unlock(&info->lock);
1300 
1301     return param_get_string(val, kp);
1302 }
1303 
1304 static int dmatest_test_list_get(char *val, const struct kernel_param *kp)
1305 {
1306     struct dmatest_info *info = &test_info;
1307     struct dmatest_chan *dtc;
1308     unsigned int thread_count = 0;
1309 
1310     list_for_each_entry(dtc, &info->channels, node) {
1311         struct dmatest_thread *thread;
1312 
1313         thread_count = 0;
1314         list_for_each_entry(thread, &dtc->threads, node) {
1315             thread_count++;
1316         }
1317         pr_info("%u threads using %s\n",
1318             thread_count, dma_chan_name(dtc->chan));
1319     }
1320 
1321     return 0;
1322 }
1323 
1324 static int __init dmatest_init(void)
1325 {
1326     struct dmatest_info *info = &test_info;
1327     struct dmatest_params *params = &info->params;
1328 
1329     if (dmatest_run) {
1330         mutex_lock(&info->lock);
1331         add_threaded_test(info);
1332         run_pending_tests(info);
1333         mutex_unlock(&info->lock);
1334     }
1335 
1336     if (params->iterations && wait)
1337         wait_event(thread_wait, !is_threaded_test_run(info));
1338 
1339     /* module parameters are stable, inittime tests are started,
1340      * let userspace take over 'run' control
1341      */
1342     info->did_init = true;
1343 
1344     return 0;
1345 }
1346 /* when compiled-in wait for drivers to load first */
1347 late_initcall(dmatest_init);
1348 
1349 static void __exit dmatest_exit(void)
1350 {
1351     struct dmatest_info *info = &test_info;
1352 
1353     mutex_lock(&info->lock);
1354     stop_threaded_test(info);
1355     mutex_unlock(&info->lock);
1356 }
1357 module_exit(dmatest_exit);
1358 
1359 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1360 MODULE_LICENSE("GPL v2");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!