0001
0002
0003
0004
0005
0006
0007 #include <linux/delay.h>
0008 #include <linux/dma-fence.h>
0009 #include <linux/kernel.h>
0010 #include <linux/kthread.h>
0011 #include <linux/sched/signal.h>
0012 #include <linux/slab.h>
0013 #include <linux/spinlock.h>
0014
0015 #include "selftest.h"
0016
0017 static struct kmem_cache *slab_fences;
0018
0019 static struct mock_fence {
0020 struct dma_fence base;
0021 struct spinlock lock;
0022 } *to_mock_fence(struct dma_fence *f) {
0023 return container_of(f, struct mock_fence, base);
0024 }
0025
0026 static const char *mock_name(struct dma_fence *f)
0027 {
0028 return "mock";
0029 }
0030
0031 static void mock_fence_release(struct dma_fence *f)
0032 {
0033 kmem_cache_free(slab_fences, to_mock_fence(f));
0034 }
0035
0036 struct wait_cb {
0037 struct dma_fence_cb cb;
0038 struct task_struct *task;
0039 };
0040
0041 static void mock_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
0042 {
0043 wake_up_process(container_of(cb, struct wait_cb, cb)->task);
0044 }
0045
0046 static long mock_wait(struct dma_fence *f, bool intr, long timeout)
0047 {
0048 const int state = intr ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
0049 struct wait_cb cb = { .task = current };
0050
0051 if (dma_fence_add_callback(f, &cb.cb, mock_wakeup))
0052 return timeout;
0053
0054 while (timeout) {
0055 set_current_state(state);
0056
0057 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags))
0058 break;
0059
0060 if (signal_pending_state(state, current))
0061 break;
0062
0063 timeout = schedule_timeout(timeout);
0064 }
0065 __set_current_state(TASK_RUNNING);
0066
0067 if (!dma_fence_remove_callback(f, &cb.cb))
0068 return timeout;
0069
0070 if (signal_pending_state(state, current))
0071 return -ERESTARTSYS;
0072
0073 return -ETIME;
0074 }
0075
0076 static const struct dma_fence_ops mock_ops = {
0077 .get_driver_name = mock_name,
0078 .get_timeline_name = mock_name,
0079 .wait = mock_wait,
0080 .release = mock_fence_release,
0081 };
0082
0083 static struct dma_fence *mock_fence(void)
0084 {
0085 struct mock_fence *f;
0086
0087 f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
0088 if (!f)
0089 return NULL;
0090
0091 spin_lock_init(&f->lock);
0092 dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
0093
0094 return &f->base;
0095 }
0096
0097 static int sanitycheck(void *arg)
0098 {
0099 struct dma_fence *f;
0100
0101 f = mock_fence();
0102 if (!f)
0103 return -ENOMEM;
0104
0105 dma_fence_signal(f);
0106 dma_fence_put(f);
0107
0108 return 0;
0109 }
0110
0111 static int test_signaling(void *arg)
0112 {
0113 struct dma_fence *f;
0114 int err = -EINVAL;
0115
0116 f = mock_fence();
0117 if (!f)
0118 return -ENOMEM;
0119
0120 if (dma_fence_is_signaled(f)) {
0121 pr_err("Fence unexpectedly signaled on creation\n");
0122 goto err_free;
0123 }
0124
0125 if (dma_fence_signal(f)) {
0126 pr_err("Fence reported being already signaled\n");
0127 goto err_free;
0128 }
0129
0130 if (!dma_fence_is_signaled(f)) {
0131 pr_err("Fence not reporting signaled\n");
0132 goto err_free;
0133 }
0134
0135 if (!dma_fence_signal(f)) {
0136 pr_err("Fence reported not being already signaled\n");
0137 goto err_free;
0138 }
0139
0140 err = 0;
0141 err_free:
0142 dma_fence_put(f);
0143 return err;
0144 }
0145
0146 struct simple_cb {
0147 struct dma_fence_cb cb;
0148 bool seen;
0149 };
0150
0151 static void simple_callback(struct dma_fence *f, struct dma_fence_cb *cb)
0152 {
0153 smp_store_mb(container_of(cb, struct simple_cb, cb)->seen, true);
0154 }
0155
0156 static int test_add_callback(void *arg)
0157 {
0158 struct simple_cb cb = {};
0159 struct dma_fence *f;
0160 int err = -EINVAL;
0161
0162 f = mock_fence();
0163 if (!f)
0164 return -ENOMEM;
0165
0166 if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
0167 pr_err("Failed to add callback, fence already signaled!\n");
0168 goto err_free;
0169 }
0170
0171 dma_fence_signal(f);
0172 if (!cb.seen) {
0173 pr_err("Callback failed!\n");
0174 goto err_free;
0175 }
0176
0177 err = 0;
0178 err_free:
0179 dma_fence_put(f);
0180 return err;
0181 }
0182
0183 static int test_late_add_callback(void *arg)
0184 {
0185 struct simple_cb cb = {};
0186 struct dma_fence *f;
0187 int err = -EINVAL;
0188
0189 f = mock_fence();
0190 if (!f)
0191 return -ENOMEM;
0192
0193 dma_fence_signal(f);
0194
0195 if (!dma_fence_add_callback(f, &cb.cb, simple_callback)) {
0196 pr_err("Added callback, but fence was already signaled!\n");
0197 goto err_free;
0198 }
0199
0200 dma_fence_signal(f);
0201 if (cb.seen) {
0202 pr_err("Callback called after failed attachment !\n");
0203 goto err_free;
0204 }
0205
0206 err = 0;
0207 err_free:
0208 dma_fence_put(f);
0209 return err;
0210 }
0211
0212 static int test_rm_callback(void *arg)
0213 {
0214 struct simple_cb cb = {};
0215 struct dma_fence *f;
0216 int err = -EINVAL;
0217
0218 f = mock_fence();
0219 if (!f)
0220 return -ENOMEM;
0221
0222 if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
0223 pr_err("Failed to add callback, fence already signaled!\n");
0224 goto err_free;
0225 }
0226
0227 if (!dma_fence_remove_callback(f, &cb.cb)) {
0228 pr_err("Failed to remove callback!\n");
0229 goto err_free;
0230 }
0231
0232 dma_fence_signal(f);
0233 if (cb.seen) {
0234 pr_err("Callback still signaled after removal!\n");
0235 goto err_free;
0236 }
0237
0238 err = 0;
0239 err_free:
0240 dma_fence_put(f);
0241 return err;
0242 }
0243
0244 static int test_late_rm_callback(void *arg)
0245 {
0246 struct simple_cb cb = {};
0247 struct dma_fence *f;
0248 int err = -EINVAL;
0249
0250 f = mock_fence();
0251 if (!f)
0252 return -ENOMEM;
0253
0254 if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
0255 pr_err("Failed to add callback, fence already signaled!\n");
0256 goto err_free;
0257 }
0258
0259 dma_fence_signal(f);
0260 if (!cb.seen) {
0261 pr_err("Callback failed!\n");
0262 goto err_free;
0263 }
0264
0265 if (dma_fence_remove_callback(f, &cb.cb)) {
0266 pr_err("Callback removal succeed after being executed!\n");
0267 goto err_free;
0268 }
0269
0270 err = 0;
0271 err_free:
0272 dma_fence_put(f);
0273 return err;
0274 }
0275
0276 static int test_status(void *arg)
0277 {
0278 struct dma_fence *f;
0279 int err = -EINVAL;
0280
0281 f = mock_fence();
0282 if (!f)
0283 return -ENOMEM;
0284
0285 if (dma_fence_get_status(f)) {
0286 pr_err("Fence unexpectedly has signaled status on creation\n");
0287 goto err_free;
0288 }
0289
0290 dma_fence_signal(f);
0291 if (!dma_fence_get_status(f)) {
0292 pr_err("Fence not reporting signaled status\n");
0293 goto err_free;
0294 }
0295
0296 err = 0;
0297 err_free:
0298 dma_fence_put(f);
0299 return err;
0300 }
0301
0302 static int test_error(void *arg)
0303 {
0304 struct dma_fence *f;
0305 int err = -EINVAL;
0306
0307 f = mock_fence();
0308 if (!f)
0309 return -ENOMEM;
0310
0311 dma_fence_set_error(f, -EIO);
0312
0313 if (dma_fence_get_status(f)) {
0314 pr_err("Fence unexpectedly has error status before signal\n");
0315 goto err_free;
0316 }
0317
0318 dma_fence_signal(f);
0319 if (dma_fence_get_status(f) != -EIO) {
0320 pr_err("Fence not reporting error status, got %d\n",
0321 dma_fence_get_status(f));
0322 goto err_free;
0323 }
0324
0325 err = 0;
0326 err_free:
0327 dma_fence_put(f);
0328 return err;
0329 }
0330
0331 static int test_wait(void *arg)
0332 {
0333 struct dma_fence *f;
0334 int err = -EINVAL;
0335
0336 f = mock_fence();
0337 if (!f)
0338 return -ENOMEM;
0339
0340 if (dma_fence_wait_timeout(f, false, 0) != -ETIME) {
0341 pr_err("Wait reported complete before being signaled\n");
0342 goto err_free;
0343 }
0344
0345 dma_fence_signal(f);
0346
0347 if (dma_fence_wait_timeout(f, false, 0) != 0) {
0348 pr_err("Wait reported incomplete after being signaled\n");
0349 goto err_free;
0350 }
0351
0352 err = 0;
0353 err_free:
0354 dma_fence_signal(f);
0355 dma_fence_put(f);
0356 return err;
0357 }
0358
0359 struct wait_timer {
0360 struct timer_list timer;
0361 struct dma_fence *f;
0362 };
0363
0364 static void wait_timer(struct timer_list *timer)
0365 {
0366 struct wait_timer *wt = from_timer(wt, timer, timer);
0367
0368 dma_fence_signal(wt->f);
0369 }
0370
0371 static int test_wait_timeout(void *arg)
0372 {
0373 struct wait_timer wt;
0374 int err = -EINVAL;
0375
0376 timer_setup_on_stack(&wt.timer, wait_timer, 0);
0377
0378 wt.f = mock_fence();
0379 if (!wt.f)
0380 return -ENOMEM;
0381
0382 if (dma_fence_wait_timeout(wt.f, false, 1) != -ETIME) {
0383 pr_err("Wait reported complete before being signaled\n");
0384 goto err_free;
0385 }
0386
0387 mod_timer(&wt.timer, jiffies + 1);
0388
0389 if (dma_fence_wait_timeout(wt.f, false, 2) == -ETIME) {
0390 if (timer_pending(&wt.timer)) {
0391 pr_notice("Timer did not fire within the jiffie!\n");
0392 err = 0;
0393 } else {
0394 pr_err("Wait reported incomplete after timeout\n");
0395 }
0396 goto err_free;
0397 }
0398
0399 err = 0;
0400 err_free:
0401 del_timer_sync(&wt.timer);
0402 destroy_timer_on_stack(&wt.timer);
0403 dma_fence_signal(wt.f);
0404 dma_fence_put(wt.f);
0405 return err;
0406 }
0407
0408 static int test_stub(void *arg)
0409 {
0410 struct dma_fence *f[64];
0411 int err = -EINVAL;
0412 int i;
0413
0414 for (i = 0; i < ARRAY_SIZE(f); i++) {
0415 f[i] = dma_fence_get_stub();
0416 if (!dma_fence_is_signaled(f[i])) {
0417 pr_err("Obtained unsignaled stub fence!\n");
0418 goto err;
0419 }
0420 }
0421
0422 err = 0;
0423 err:
0424 while (i--)
0425 dma_fence_put(f[i]);
0426 return err;
0427 }
0428
0429
0430
0431 struct race_thread {
0432 struct dma_fence __rcu **fences;
0433 struct task_struct *task;
0434 bool before;
0435 int id;
0436 };
0437
0438 static void __wait_for_callbacks(struct dma_fence *f)
0439 {
0440 spin_lock_irq(f->lock);
0441 spin_unlock_irq(f->lock);
0442 }
0443
0444 static int thread_signal_callback(void *arg)
0445 {
0446 const struct race_thread *t = arg;
0447 unsigned long pass = 0;
0448 unsigned long miss = 0;
0449 int err = 0;
0450
0451 while (!err && !kthread_should_stop()) {
0452 struct dma_fence *f1, *f2;
0453 struct simple_cb cb;
0454
0455 f1 = mock_fence();
0456 if (!f1) {
0457 err = -ENOMEM;
0458 break;
0459 }
0460
0461 rcu_assign_pointer(t->fences[t->id], f1);
0462 smp_wmb();
0463
0464 rcu_read_lock();
0465 do {
0466 f2 = dma_fence_get_rcu_safe(&t->fences[!t->id]);
0467 } while (!f2 && !kthread_should_stop());
0468 rcu_read_unlock();
0469
0470 if (t->before)
0471 dma_fence_signal(f1);
0472
0473 smp_store_mb(cb.seen, false);
0474 if (!f2 ||
0475 dma_fence_add_callback(f2, &cb.cb, simple_callback)) {
0476 miss++;
0477 cb.seen = true;
0478 }
0479
0480 if (!t->before)
0481 dma_fence_signal(f1);
0482
0483 if (!cb.seen) {
0484 dma_fence_wait(f2, false);
0485 __wait_for_callbacks(f2);
0486 }
0487
0488 if (!READ_ONCE(cb.seen)) {
0489 pr_err("Callback not seen on thread %d, pass %lu (%lu misses), signaling %s add_callback; fence signaled? %s\n",
0490 t->id, pass, miss,
0491 t->before ? "before" : "after",
0492 dma_fence_is_signaled(f2) ? "yes" : "no");
0493 err = -EINVAL;
0494 }
0495
0496 dma_fence_put(f2);
0497
0498 rcu_assign_pointer(t->fences[t->id], NULL);
0499 smp_wmb();
0500
0501 dma_fence_put(f1);
0502
0503 pass++;
0504 }
0505
0506 pr_info("%s[%d] completed %lu passes, %lu misses\n",
0507 __func__, t->id, pass, miss);
0508 return err;
0509 }
0510
0511 static int race_signal_callback(void *arg)
0512 {
0513 struct dma_fence __rcu *f[2] = {};
0514 int ret = 0;
0515 int pass;
0516
0517 for (pass = 0; !ret && pass <= 1; pass++) {
0518 struct race_thread t[2];
0519 int i;
0520
0521 for (i = 0; i < ARRAY_SIZE(t); i++) {
0522 t[i].fences = f;
0523 t[i].id = i;
0524 t[i].before = pass;
0525 t[i].task = kthread_run(thread_signal_callback, &t[i],
0526 "dma-fence:%d", i);
0527 get_task_struct(t[i].task);
0528 }
0529
0530 msleep(50);
0531
0532 for (i = 0; i < ARRAY_SIZE(t); i++) {
0533 int err;
0534
0535 err = kthread_stop(t[i].task);
0536 if (err && !ret)
0537 ret = err;
0538
0539 put_task_struct(t[i].task);
0540 }
0541 }
0542
0543 return ret;
0544 }
0545
0546 int dma_fence(void)
0547 {
0548 static const struct subtest tests[] = {
0549 SUBTEST(sanitycheck),
0550 SUBTEST(test_signaling),
0551 SUBTEST(test_add_callback),
0552 SUBTEST(test_late_add_callback),
0553 SUBTEST(test_rm_callback),
0554 SUBTEST(test_late_rm_callback),
0555 SUBTEST(test_status),
0556 SUBTEST(test_error),
0557 SUBTEST(test_wait),
0558 SUBTEST(test_wait_timeout),
0559 SUBTEST(test_stub),
0560 SUBTEST(race_signal_callback),
0561 };
0562 int ret;
0563
0564 pr_info("sizeof(dma_fence)=%zu\n", sizeof(struct dma_fence));
0565
0566 slab_fences = KMEM_CACHE(mock_fence,
0567 SLAB_TYPESAFE_BY_RCU |
0568 SLAB_HWCACHE_ALIGN);
0569 if (!slab_fences)
0570 return -ENOMEM;
0571
0572 ret = subtests(tests, NULL);
0573
0574 kmem_cache_destroy(slab_fences);
0575
0576 return ret;
0577 }
