Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: MIT
0002 /*
0003  * Copyright © 2020 Intel Corporation
0004  */
0005 
0006 #include <linux/sort.h>
0007 
0008 #include "gem/i915_gem_internal.h"
0009 
0010 #include "selftests/i915_random.h"
0011 
0012 static const unsigned int sizes[] = {
0013     SZ_4K,
0014     SZ_64K,
0015     SZ_2M,
0016     CHUNK_SZ - SZ_4K,
0017     CHUNK_SZ,
0018     CHUNK_SZ + SZ_4K,
0019     SZ_64M,
0020 };
0021 
0022 static struct drm_i915_gem_object *
0023 create_lmem_or_internal(struct drm_i915_private *i915, size_t size)
0024 {
0025     struct drm_i915_gem_object *obj;
0026 
0027     obj = i915_gem_object_create_lmem(i915, size, 0);
0028     if (!IS_ERR(obj))
0029         return obj;
0030 
0031     return i915_gem_object_create_internal(i915, size);
0032 }
0033 
0034 static int copy(struct intel_migrate *migrate,
0035         int (*fn)(struct intel_migrate *migrate,
0036               struct i915_gem_ww_ctx *ww,
0037               struct drm_i915_gem_object *src,
0038               struct drm_i915_gem_object *dst,
0039               struct i915_request **out),
0040         u32 sz, struct rnd_state *prng)
0041 {
0042     struct drm_i915_private *i915 = migrate->context->engine->i915;
0043     struct drm_i915_gem_object *src, *dst;
0044     struct i915_request *rq;
0045     struct i915_gem_ww_ctx ww;
0046     u32 *vaddr;
0047     int err = 0;
0048     int i;
0049 
0050     src = create_lmem_or_internal(i915, sz);
0051     if (IS_ERR(src))
0052         return 0;
0053 
0054     sz = src->base.size;
0055     dst = i915_gem_object_create_internal(i915, sz);
0056     if (IS_ERR(dst))
0057         goto err_free_src;
0058 
0059     for_i915_gem_ww(&ww, err, true) {
0060         err = i915_gem_object_lock(src, &ww);
0061         if (err)
0062             continue;
0063 
0064         err = i915_gem_object_lock(dst, &ww);
0065         if (err)
0066             continue;
0067 
0068         vaddr = i915_gem_object_pin_map(src, I915_MAP_WC);
0069         if (IS_ERR(vaddr)) {
0070             err = PTR_ERR(vaddr);
0071             continue;
0072         }
0073 
0074         for (i = 0; i < sz / sizeof(u32); i++)
0075             vaddr[i] = i;
0076         i915_gem_object_flush_map(src);
0077 
0078         vaddr = i915_gem_object_pin_map(dst, I915_MAP_WC);
0079         if (IS_ERR(vaddr)) {
0080             err = PTR_ERR(vaddr);
0081             goto unpin_src;
0082         }
0083 
0084         for (i = 0; i < sz / sizeof(u32); i++)
0085             vaddr[i] = ~i;
0086         i915_gem_object_flush_map(dst);
0087 
0088         err = fn(migrate, &ww, src, dst, &rq);
0089         if (!err)
0090             continue;
0091 
0092         if (err != -EDEADLK && err != -EINTR && err != -ERESTARTSYS)
0093             pr_err("%ps failed, size: %u\n", fn, sz);
0094         if (rq) {
0095             i915_request_wait(rq, 0, HZ);
0096             i915_request_put(rq);
0097         }
0098         i915_gem_object_unpin_map(dst);
0099 unpin_src:
0100         i915_gem_object_unpin_map(src);
0101     }
0102     if (err)
0103         goto err_out;
0104 
0105     if (rq) {
0106         if (i915_request_wait(rq, 0, HZ) < 0) {
0107             pr_err("%ps timed out, size: %u\n", fn, sz);
0108             err = -ETIME;
0109         }
0110         i915_request_put(rq);
0111     }
0112 
0113     for (i = 0; !err && i < sz / PAGE_SIZE; i++) {
0114         int x = i * 1024 + i915_prandom_u32_max_state(1024, prng);
0115 
0116         if (vaddr[x] != x) {
0117             pr_err("%ps failed, size: %u, offset: %zu\n",
0118                    fn, sz, x * sizeof(u32));
0119             igt_hexdump(vaddr + i * 1024, 4096);
0120             err = -EINVAL;
0121         }
0122     }
0123 
0124     i915_gem_object_unpin_map(dst);
0125     i915_gem_object_unpin_map(src);
0126 
0127 err_out:
0128     i915_gem_object_put(dst);
0129 err_free_src:
0130     i915_gem_object_put(src);
0131 
0132     return err;
0133 }
0134 
0135 static int intel_context_copy_ccs(struct intel_context *ce,
0136                   const struct i915_deps *deps,
0137                   struct scatterlist *sg,
0138                   enum i915_cache_level cache_level,
0139                   bool write_to_ccs,
0140                   struct i915_request **out)
0141 {
0142     u8 src_access = write_to_ccs ? DIRECT_ACCESS : INDIRECT_ACCESS;
0143     u8 dst_access = write_to_ccs ? INDIRECT_ACCESS : DIRECT_ACCESS;
0144     struct sgt_dma it = sg_sgt(sg);
0145     struct i915_request *rq;
0146     u32 offset;
0147     int err;
0148 
0149     GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
0150     *out = NULL;
0151 
0152     GEM_BUG_ON(ce->ring->size < SZ_64K);
0153 
0154     offset = 0;
0155     if (HAS_64K_PAGES(ce->engine->i915))
0156         offset = CHUNK_SZ;
0157 
0158     do {
0159         int len;
0160 
0161         rq = i915_request_create(ce);
0162         if (IS_ERR(rq)) {
0163             err = PTR_ERR(rq);
0164             goto out_ce;
0165         }
0166 
0167         if (deps) {
0168             err = i915_request_await_deps(rq, deps);
0169             if (err)
0170                 goto out_rq;
0171 
0172             if (rq->engine->emit_init_breadcrumb) {
0173                 err = rq->engine->emit_init_breadcrumb(rq);
0174                 if (err)
0175                     goto out_rq;
0176             }
0177 
0178             deps = NULL;
0179         }
0180 
0181         /* The PTE updates + clear must not be interrupted. */
0182         err = emit_no_arbitration(rq);
0183         if (err)
0184             goto out_rq;
0185 
0186         len = emit_pte(rq, &it, cache_level, true, offset, CHUNK_SZ);
0187         if (len <= 0) {
0188             err = len;
0189             goto out_rq;
0190         }
0191 
0192         err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
0193         if (err)
0194             goto out_rq;
0195 
0196         err = emit_copy_ccs(rq, offset, dst_access,
0197                     offset, src_access, len);
0198         if (err)
0199             goto out_rq;
0200 
0201         err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
0202 
0203         /* Arbitration is re-enabled between requests. */
0204 out_rq:
0205         if (*out)
0206             i915_request_put(*out);
0207         *out = i915_request_get(rq);
0208         i915_request_add(rq);
0209         if (err || !it.sg || !sg_dma_len(it.sg))
0210             break;
0211 
0212         cond_resched();
0213     } while (1);
0214 
0215 out_ce:
0216     return err;
0217 }
0218 
0219 static int
0220 intel_migrate_ccs_copy(struct intel_migrate *m,
0221                struct i915_gem_ww_ctx *ww,
0222                const struct i915_deps *deps,
0223                struct scatterlist *sg,
0224                enum i915_cache_level cache_level,
0225                bool write_to_ccs,
0226                struct i915_request **out)
0227 {
0228     struct intel_context *ce;
0229     int err;
0230 
0231     *out = NULL;
0232     if (!m->context)
0233         return -ENODEV;
0234 
0235     ce = intel_migrate_create_context(m);
0236     if (IS_ERR(ce))
0237         ce = intel_context_get(m->context);
0238     GEM_BUG_ON(IS_ERR(ce));
0239 
0240     err = intel_context_pin_ww(ce, ww);
0241     if (err)
0242         goto out;
0243 
0244     err = intel_context_copy_ccs(ce, deps, sg, cache_level,
0245                      write_to_ccs, out);
0246 
0247     intel_context_unpin(ce);
0248 out:
0249     intel_context_put(ce);
0250     return err;
0251 }
0252 
0253 static int clear(struct intel_migrate *migrate,
0254          int (*fn)(struct intel_migrate *migrate,
0255                struct i915_gem_ww_ctx *ww,
0256                struct drm_i915_gem_object *obj,
0257                u32 value,
0258                struct i915_request **out),
0259          u32 sz, struct rnd_state *prng)
0260 {
0261     struct drm_i915_private *i915 = migrate->context->engine->i915;
0262     struct drm_i915_gem_object *obj;
0263     struct i915_request *rq;
0264     struct i915_gem_ww_ctx ww;
0265     u32 *vaddr, val = 0;
0266     bool ccs_cap = false;
0267     int err = 0;
0268     int i;
0269 
0270     obj = create_lmem_or_internal(i915, sz);
0271     if (IS_ERR(obj))
0272         return 0;
0273 
0274     /* Consider the rounded up memory too */
0275     sz = obj->base.size;
0276 
0277     if (HAS_FLAT_CCS(i915) && i915_gem_object_is_lmem(obj))
0278         ccs_cap = true;
0279 
0280     for_i915_gem_ww(&ww, err, true) {
0281         int ccs_bytes, ccs_bytes_per_chunk;
0282 
0283         err = i915_gem_object_lock(obj, &ww);
0284         if (err)
0285             continue;
0286 
0287         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
0288         if (IS_ERR(vaddr)) {
0289             err = PTR_ERR(vaddr);
0290             continue;
0291         }
0292 
0293         for (i = 0; i < sz / sizeof(u32); i++)
0294             vaddr[i] = ~i;
0295         i915_gem_object_flush_map(obj);
0296 
0297         if (ccs_cap && !val) {
0298             /* Write the obj data into ccs surface */
0299             err = intel_migrate_ccs_copy(migrate, &ww, NULL,
0300                              obj->mm.pages->sgl,
0301                              obj->cache_level,
0302                              true, &rq);
0303             if (rq && !err) {
0304                 if (i915_request_wait(rq, 0, HZ) < 0) {
0305                     pr_err("%ps timed out, size: %u\n",
0306                            fn, sz);
0307                     err = -ETIME;
0308                 }
0309                 i915_request_put(rq);
0310                 rq = NULL;
0311             }
0312             if (err)
0313                 continue;
0314         }
0315 
0316         err = fn(migrate, &ww, obj, val, &rq);
0317         if (rq && !err) {
0318             if (i915_request_wait(rq, 0, HZ) < 0) {
0319                 pr_err("%ps timed out, size: %u\n", fn, sz);
0320                 err = -ETIME;
0321             }
0322             i915_request_put(rq);
0323             rq = NULL;
0324         }
0325         if (err)
0326             continue;
0327 
0328         i915_gem_object_flush_map(obj);
0329 
0330         /* Verify the set/clear of the obj mem */
0331         for (i = 0; !err && i < sz / PAGE_SIZE; i++) {
0332             int x = i * 1024 +
0333                 i915_prandom_u32_max_state(1024, prng);
0334 
0335             if (vaddr[x] != val) {
0336                 pr_err("%ps failed, (%u != %u), offset: %zu\n",
0337                        fn, vaddr[x], val,  x * sizeof(u32));
0338                 igt_hexdump(vaddr + i * 1024, 4096);
0339                 err = -EINVAL;
0340             }
0341         }
0342         if (err)
0343             continue;
0344 
0345         if (ccs_cap && !val) {
0346             for (i = 0; i < sz / sizeof(u32); i++)
0347                 vaddr[i] = ~i;
0348             i915_gem_object_flush_map(obj);
0349 
0350             err = intel_migrate_ccs_copy(migrate, &ww, NULL,
0351                              obj->mm.pages->sgl,
0352                              obj->cache_level,
0353                              false, &rq);
0354             if (rq && !err) {
0355                 if (i915_request_wait(rq, 0, HZ) < 0) {
0356                     pr_err("%ps timed out, size: %u\n",
0357                            fn, sz);
0358                     err = -ETIME;
0359                 }
0360                 i915_request_put(rq);
0361                 rq = NULL;
0362             }
0363             if (err)
0364                 continue;
0365 
0366             ccs_bytes = GET_CCS_BYTES(i915, sz);
0367             ccs_bytes_per_chunk = GET_CCS_BYTES(i915, CHUNK_SZ);
0368             i915_gem_object_flush_map(obj);
0369 
0370             for (i = 0; !err && i < DIV_ROUND_UP(ccs_bytes, PAGE_SIZE); i++) {
0371                 int offset = ((i * PAGE_SIZE)  /
0372                     ccs_bytes_per_chunk) * CHUNK_SZ / sizeof(u32);
0373                 int ccs_bytes_left = (ccs_bytes - i * PAGE_SIZE) / sizeof(u32);
0374                 int x = i915_prandom_u32_max_state(min_t(int, 1024,
0375                                      ccs_bytes_left), prng);
0376 
0377                 if (vaddr[offset + x]) {
0378                     pr_err("%ps ccs clearing failed, offset: %ld/%d\n",
0379                            fn, i * PAGE_SIZE + x * sizeof(u32), ccs_bytes);
0380                     igt_hexdump(vaddr + offset,
0381                             min_t(int, 4096,
0382                               ccs_bytes_left * sizeof(u32)));
0383                     err = -EINVAL;
0384                 }
0385             }
0386 
0387             if (err)
0388                 continue;
0389         }
0390         i915_gem_object_unpin_map(obj);
0391     }
0392 
0393     if (err) {
0394         if (err != -EDEADLK && err != -EINTR && err != -ERESTARTSYS)
0395             pr_err("%ps failed, size: %u\n", fn, sz);
0396         if (rq && err != -EINVAL) {
0397             i915_request_wait(rq, 0, HZ);
0398             i915_request_put(rq);
0399         }
0400 
0401         i915_gem_object_unpin_map(obj);
0402     }
0403 
0404     i915_gem_object_put(obj);
0405     return err;
0406 }
0407 
0408 static int __migrate_copy(struct intel_migrate *migrate,
0409               struct i915_gem_ww_ctx *ww,
0410               struct drm_i915_gem_object *src,
0411               struct drm_i915_gem_object *dst,
0412               struct i915_request **out)
0413 {
0414     return intel_migrate_copy(migrate, ww, NULL,
0415                   src->mm.pages->sgl, src->cache_level,
0416                   i915_gem_object_is_lmem(src),
0417                   dst->mm.pages->sgl, dst->cache_level,
0418                   i915_gem_object_is_lmem(dst),
0419                   out);
0420 }
0421 
0422 static int __global_copy(struct intel_migrate *migrate,
0423              struct i915_gem_ww_ctx *ww,
0424              struct drm_i915_gem_object *src,
0425              struct drm_i915_gem_object *dst,
0426              struct i915_request **out)
0427 {
0428     return intel_context_migrate_copy(migrate->context, NULL,
0429                       src->mm.pages->sgl, src->cache_level,
0430                       i915_gem_object_is_lmem(src),
0431                       dst->mm.pages->sgl, dst->cache_level,
0432                       i915_gem_object_is_lmem(dst),
0433                       out);
0434 }
0435 
0436 static int
0437 migrate_copy(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng)
0438 {
0439     return copy(migrate, __migrate_copy, sz, prng);
0440 }
0441 
0442 static int
0443 global_copy(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng)
0444 {
0445     return copy(migrate, __global_copy, sz, prng);
0446 }
0447 
0448 static int __migrate_clear(struct intel_migrate *migrate,
0449                struct i915_gem_ww_ctx *ww,
0450                struct drm_i915_gem_object *obj,
0451                u32 value,
0452                struct i915_request **out)
0453 {
0454     return intel_migrate_clear(migrate, ww, NULL,
0455                    obj->mm.pages->sgl,
0456                    obj->cache_level,
0457                    i915_gem_object_is_lmem(obj),
0458                    value, out);
0459 }
0460 
0461 static int __global_clear(struct intel_migrate *migrate,
0462               struct i915_gem_ww_ctx *ww,
0463               struct drm_i915_gem_object *obj,
0464               u32 value,
0465               struct i915_request **out)
0466 {
0467     return intel_context_migrate_clear(migrate->context, NULL,
0468                        obj->mm.pages->sgl,
0469                        obj->cache_level,
0470                        i915_gem_object_is_lmem(obj),
0471                        value, out);
0472 }
0473 
0474 static int
0475 migrate_clear(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng)
0476 {
0477     return clear(migrate, __migrate_clear, sz, prng);
0478 }
0479 
0480 static int
0481 global_clear(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng)
0482 {
0483     return clear(migrate, __global_clear, sz, prng);
0484 }
0485 
0486 static int live_migrate_copy(void *arg)
0487 {
0488     struct intel_migrate *migrate = arg;
0489     struct drm_i915_private *i915 = migrate->context->engine->i915;
0490     I915_RND_STATE(prng);
0491     int i;
0492 
0493     for (i = 0; i < ARRAY_SIZE(sizes); i++) {
0494         int err;
0495 
0496         err = migrate_copy(migrate, sizes[i], &prng);
0497         if (err == 0)
0498             err = global_copy(migrate, sizes[i], &prng);
0499         i915_gem_drain_freed_objects(i915);
0500         if (err)
0501             return err;
0502     }
0503 
0504     return 0;
0505 }
0506 
0507 static int live_migrate_clear(void *arg)
0508 {
0509     struct intel_migrate *migrate = arg;
0510     struct drm_i915_private *i915 = migrate->context->engine->i915;
0511     I915_RND_STATE(prng);
0512     int i;
0513 
0514     for (i = 0; i < ARRAY_SIZE(sizes); i++) {
0515         int err;
0516 
0517         err = migrate_clear(migrate, sizes[i], &prng);
0518         if (err == 0)
0519             err = global_clear(migrate, sizes[i], &prng);
0520 
0521         i915_gem_drain_freed_objects(i915);
0522         if (err)
0523             return err;
0524     }
0525 
0526     return 0;
0527 }
0528 
0529 struct threaded_migrate {
0530     struct intel_migrate *migrate;
0531     struct task_struct *tsk;
0532     struct rnd_state prng;
0533 };
0534 
0535 static int threaded_migrate(struct intel_migrate *migrate,
0536                 int (*fn)(void *arg),
0537                 unsigned int flags)
0538 {
0539     const unsigned int n_cpus = num_online_cpus() + 1;
0540     struct threaded_migrate *thread;
0541     I915_RND_STATE(prng);
0542     unsigned int i;
0543     int err = 0;
0544 
0545     thread = kcalloc(n_cpus, sizeof(*thread), GFP_KERNEL);
0546     if (!thread)
0547         return 0;
0548 
0549     for (i = 0; i < n_cpus; ++i) {
0550         struct task_struct *tsk;
0551 
0552         thread[i].migrate = migrate;
0553         thread[i].prng =
0554             I915_RND_STATE_INITIALIZER(prandom_u32_state(&prng));
0555 
0556         tsk = kthread_run(fn, &thread[i], "igt-%d", i);
0557         if (IS_ERR(tsk)) {
0558             err = PTR_ERR(tsk);
0559             break;
0560         }
0561 
0562         get_task_struct(tsk);
0563         thread[i].tsk = tsk;
0564     }
0565 
0566     msleep(10); /* start all threads before we kthread_stop() */
0567 
0568     for (i = 0; i < n_cpus; ++i) {
0569         struct task_struct *tsk = thread[i].tsk;
0570         int status;
0571 
0572         if (IS_ERR_OR_NULL(tsk))
0573             continue;
0574 
0575         status = kthread_stop(tsk);
0576         if (status && !err)
0577             err = status;
0578 
0579         put_task_struct(tsk);
0580     }
0581 
0582     kfree(thread);
0583     return err;
0584 }
0585 
0586 static int __thread_migrate_copy(void *arg)
0587 {
0588     struct threaded_migrate *tm = arg;
0589 
0590     return migrate_copy(tm->migrate, 2 * CHUNK_SZ, &tm->prng);
0591 }
0592 
0593 static int thread_migrate_copy(void *arg)
0594 {
0595     return threaded_migrate(arg, __thread_migrate_copy, 0);
0596 }
0597 
0598 static int __thread_global_copy(void *arg)
0599 {
0600     struct threaded_migrate *tm = arg;
0601 
0602     return global_copy(tm->migrate, 2 * CHUNK_SZ, &tm->prng);
0603 }
0604 
0605 static int thread_global_copy(void *arg)
0606 {
0607     return threaded_migrate(arg, __thread_global_copy, 0);
0608 }
0609 
0610 static int __thread_migrate_clear(void *arg)
0611 {
0612     struct threaded_migrate *tm = arg;
0613 
0614     return migrate_clear(tm->migrate, 2 * CHUNK_SZ, &tm->prng);
0615 }
0616 
0617 static int __thread_global_clear(void *arg)
0618 {
0619     struct threaded_migrate *tm = arg;
0620 
0621     return global_clear(tm->migrate, 2 * CHUNK_SZ, &tm->prng);
0622 }
0623 
0624 static int thread_migrate_clear(void *arg)
0625 {
0626     return threaded_migrate(arg, __thread_migrate_clear, 0);
0627 }
0628 
0629 static int thread_global_clear(void *arg)
0630 {
0631     return threaded_migrate(arg, __thread_global_clear, 0);
0632 }
0633 
0634 int intel_migrate_live_selftests(struct drm_i915_private *i915)
0635 {
0636     static const struct i915_subtest tests[] = {
0637         SUBTEST(live_migrate_copy),
0638         SUBTEST(live_migrate_clear),
0639         SUBTEST(thread_migrate_copy),
0640         SUBTEST(thread_migrate_clear),
0641         SUBTEST(thread_global_copy),
0642         SUBTEST(thread_global_clear),
0643     };
0644     struct intel_gt *gt = to_gt(i915);
0645 
0646     if (!gt->migrate.context)
0647         return 0;
0648 
0649     return i915_subtests(tests, &gt->migrate);
0650 }
0651 
0652 static struct drm_i915_gem_object *
0653 create_init_lmem_internal(struct intel_gt *gt, size_t sz, bool try_lmem)
0654 {
0655     struct drm_i915_gem_object *obj = NULL;
0656     int err;
0657 
0658     if (try_lmem)
0659         obj = i915_gem_object_create_lmem(gt->i915, sz, 0);
0660 
0661     if (IS_ERR_OR_NULL(obj)) {
0662         obj = i915_gem_object_create_internal(gt->i915, sz);
0663         if (IS_ERR(obj))
0664             return obj;
0665     }
0666 
0667     i915_gem_object_trylock(obj, NULL);
0668     err = i915_gem_object_pin_pages(obj);
0669     if (err) {
0670         i915_gem_object_unlock(obj);
0671         i915_gem_object_put(obj);
0672         return ERR_PTR(err);
0673     }
0674 
0675     return obj;
0676 }
0677 
0678 static int wrap_ktime_compare(const void *A, const void *B)
0679 {
0680     const ktime_t *a = A, *b = B;
0681 
0682     return ktime_compare(*a, *b);
0683 }
0684 
0685 static int __perf_clear_blt(struct intel_context *ce,
0686                 struct scatterlist *sg,
0687                 enum i915_cache_level cache_level,
0688                 bool is_lmem,
0689                 size_t sz)
0690 {
0691     ktime_t t[5];
0692     int pass;
0693     int err = 0;
0694 
0695     for (pass = 0; pass < ARRAY_SIZE(t); pass++) {
0696         struct i915_request *rq;
0697         ktime_t t0, t1;
0698 
0699         t0 = ktime_get();
0700 
0701         err = intel_context_migrate_clear(ce, NULL, sg, cache_level,
0702                           is_lmem, 0, &rq);
0703         if (rq) {
0704             if (i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT) < 0)
0705                 err = -EIO;
0706             i915_request_put(rq);
0707         }
0708         if (err)
0709             break;
0710 
0711         t1 = ktime_get();
0712         t[pass] = ktime_sub(t1, t0);
0713     }
0714     if (err)
0715         return err;
0716 
0717     sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL);
0718     pr_info("%s: %zd KiB fill: %lld MiB/s\n",
0719         ce->engine->name, sz >> 10,
0720         div64_u64(mul_u32_u32(4 * sz,
0721                       1000 * 1000 * 1000),
0722               t[1] + 2 * t[2] + t[3]) >> 20);
0723     return 0;
0724 }
0725 
0726 static int perf_clear_blt(void *arg)
0727 {
0728     struct intel_gt *gt = arg;
0729     static const unsigned long sizes[] = {
0730         SZ_4K,
0731         SZ_64K,
0732         SZ_2M,
0733         SZ_64M
0734     };
0735     int i;
0736 
0737     for (i = 0; i < ARRAY_SIZE(sizes); i++) {
0738         struct drm_i915_gem_object *dst;
0739         int err;
0740 
0741         dst = create_init_lmem_internal(gt, sizes[i], true);
0742         if (IS_ERR(dst))
0743             return PTR_ERR(dst);
0744 
0745         err = __perf_clear_blt(gt->migrate.context,
0746                        dst->mm.pages->sgl,
0747                        I915_CACHE_NONE,
0748                        i915_gem_object_is_lmem(dst),
0749                        sizes[i]);
0750 
0751         i915_gem_object_unlock(dst);
0752         i915_gem_object_put(dst);
0753         if (err)
0754             return err;
0755     }
0756 
0757     return 0;
0758 }
0759 
0760 static int __perf_copy_blt(struct intel_context *ce,
0761                struct scatterlist *src,
0762                enum i915_cache_level src_cache_level,
0763                bool src_is_lmem,
0764                struct scatterlist *dst,
0765                enum i915_cache_level dst_cache_level,
0766                bool dst_is_lmem,
0767                size_t sz)
0768 {
0769     ktime_t t[5];
0770     int pass;
0771     int err = 0;
0772 
0773     for (pass = 0; pass < ARRAY_SIZE(t); pass++) {
0774         struct i915_request *rq;
0775         ktime_t t0, t1;
0776 
0777         t0 = ktime_get();
0778 
0779         err = intel_context_migrate_copy(ce, NULL,
0780                          src, src_cache_level,
0781                          src_is_lmem,
0782                          dst, dst_cache_level,
0783                          dst_is_lmem,
0784                          &rq);
0785         if (rq) {
0786             if (i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT) < 0)
0787                 err = -EIO;
0788             i915_request_put(rq);
0789         }
0790         if (err)
0791             break;
0792 
0793         t1 = ktime_get();
0794         t[pass] = ktime_sub(t1, t0);
0795     }
0796     if (err)
0797         return err;
0798 
0799     sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL);
0800     pr_info("%s: %zd KiB copy: %lld MiB/s\n",
0801         ce->engine->name, sz >> 10,
0802         div64_u64(mul_u32_u32(4 * sz,
0803                       1000 * 1000 * 1000),
0804               t[1] + 2 * t[2] + t[3]) >> 20);
0805     return 0;
0806 }
0807 
0808 static int perf_copy_blt(void *arg)
0809 {
0810     struct intel_gt *gt = arg;
0811     static const unsigned long sizes[] = {
0812         SZ_4K,
0813         SZ_64K,
0814         SZ_2M,
0815         SZ_64M
0816     };
0817     int i;
0818 
0819     for (i = 0; i < ARRAY_SIZE(sizes); i++) {
0820         struct drm_i915_gem_object *src, *dst;
0821         size_t sz;
0822         int err;
0823 
0824         src = create_init_lmem_internal(gt, sizes[i], true);
0825         if (IS_ERR(src))
0826             return PTR_ERR(src);
0827 
0828         sz = src->base.size;
0829         dst = create_init_lmem_internal(gt, sz, false);
0830         if (IS_ERR(dst)) {
0831             err = PTR_ERR(dst);
0832             goto err_src;
0833         }
0834 
0835         err = __perf_copy_blt(gt->migrate.context,
0836                       src->mm.pages->sgl,
0837                       I915_CACHE_NONE,
0838                       i915_gem_object_is_lmem(src),
0839                       dst->mm.pages->sgl,
0840                       I915_CACHE_NONE,
0841                       i915_gem_object_is_lmem(dst),
0842                       sz);
0843 
0844         i915_gem_object_unlock(dst);
0845         i915_gem_object_put(dst);
0846 err_src:
0847         i915_gem_object_unlock(src);
0848         i915_gem_object_put(src);
0849         if (err)
0850             return err;
0851     }
0852 
0853     return 0;
0854 }
0855 
0856 int intel_migrate_perf_selftests(struct drm_i915_private *i915)
0857 {
0858     static const struct i915_subtest tests[] = {
0859         SUBTEST(perf_clear_blt),
0860         SUBTEST(perf_copy_blt),
0861     };
0862     struct intel_gt *gt = to_gt(i915);
0863 
0864     if (intel_gt_is_wedged(gt))
0865         return 0;
0866 
0867     if (!gt->migrate.context)
0868         return 0;
0869 
0870     return intel_gt_live_subtests(tests, gt);
0871 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!