drm/i915/i915_syncmap.c

0001 /*
0002  * Copyright © 2017 Intel Corporation
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice (including the next
0012  * paragraph) shall be included in all copies or substantial portions of the
0013  * Software.
0014  *
0015  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0016  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0017  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0018  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
0019  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
0020  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
0021  * IN THE SOFTWARE.
0022  *
0023  */
0024
0025 #include <linux/slab.h>
0026
0027 #include "i915_syncmap.h"
0028
0029 #include "i915_gem.h" /* GEM_BUG_ON() */
0030 #include "i915_selftest.h"
0031
0032 #define SHIFT ilog2(KSYNCMAP)
0033 #define MASK (KSYNCMAP - 1)
0034
0035 /*
0036  * struct i915_syncmap is a layer of a radixtree that maps a u64 fence
0037  * context id to the last u32 fence seqno waited upon from that context.
0038  * Unlike lib/radixtree it uses a parent pointer that allows traversal back to
0039  * the root. This allows us to access the whole tree via a single pointer
0040  * to the most recently used layer. We expect fence contexts to be dense
0041  * and most reuse to be on the same i915_gem_context but on neighbouring
0042  * engines (i.e. on adjacent contexts) and reuse the same leaf, a very
0043  * effective lookup cache. If the new lookup is not on the same leaf, we
0044  * expect it to be on the neighbouring branch.
0045  *
0046  * A leaf holds an array of u32 seqno, and has height 0. The bitmap field
0047  * allows us to store whether a particular seqno is valid (i.e. allows us
0048  * to distinguish unset from 0).
0049  *
0050  * A branch holds an array of layer pointers, and has height > 0, and always
0051  * has at least 2 layers (either branches or leaves) below it.
0052  *
0053  * For example,
0054  *  for x in
0055  *    0 1 2 0x10 0x11 0x200 0x201
0056  *    0x500000 0x500001 0x503000 0x503001
0057  *    0xE<<60:
0058  *      i915_syncmap_set(&sync, x, lower_32_bits(x));
0059  * will build a tree like:
0060  *  0xXXXXXXXXXXXXXXXX
0061  *  0-> 0x0000000000XXXXXX
0062  *  |   0-> 0x0000000000000XXX
0063  *  |   |   0-> 0x00000000000000XX
0064  *  |   |   |   0-> 0x000000000000000X 0:0, 1:1, 2:2
0065  *  |   |   |   1-> 0x000000000000001X 0:10, 1:11
0066  *  |   |   2-> 0x000000000000020X 0:200, 1:201
0067  *  |   5-> 0x000000000050XXXX
0068  *  |       0-> 0x000000000050000X 0:500000, 1:500001
0069  *  |       3-> 0x000000000050300X 0:503000, 1:503001
0070  *  e-> 0xe00000000000000X e:e
0071  */
0072
0073 struct i915_syncmap {
0074     u64 prefix;
0075     unsigned int height;
0076     unsigned int bitmap;
0077     struct i915_syncmap *parent;
0078     /*
0079      * Following this header is an array of either seqno or child pointers:
0080      * union {
0081      *  u32 seqno[KSYNCMAP];
0082      *  struct i915_syncmap *child[KSYNCMAP];
0083      * };
0084      */
0085 };
0086
0087 /**
0088  * i915_syncmap_init -- initialise the #i915_syncmap
0089  * @root: pointer to the #i915_syncmap
0090  */
0091 void i915_syncmap_init(struct i915_syncmap **root)
0092 {
0093     BUILD_BUG_ON_NOT_POWER_OF_2(KSYNCMAP);
0094     BUILD_BUG_ON_NOT_POWER_OF_2(SHIFT);
0095     BUILD_BUG_ON(KSYNCMAP > BITS_PER_TYPE((*root)->bitmap));
0096     *root = NULL;
0097 }
0098
0099 static inline u32 *__sync_seqno(struct i915_syncmap *p)
0100 {
0101     GEM_BUG_ON(p->height);
0102     return (u32 *)(p + 1);
0103 }
0104
0105 static inline struct i915_syncmap **__sync_child(struct i915_syncmap *p)
0106 {
0107     GEM_BUG_ON(!p->height);
0108     return (struct i915_syncmap **)(p + 1);
0109 }
0110
0111 static inline unsigned int
0112 __sync_branch_idx(const struct i915_syncmap *p, u64 id)
0113 {
0114     return (id >> p->height) & MASK;
0115 }
0116
0117 static inline unsigned int
0118 __sync_leaf_idx(const struct i915_syncmap *p, u64 id)
0119 {
0120     GEM_BUG_ON(p->height);
0121     return id & MASK;
0122 }
0123
0124 static inline u64 __sync_branch_prefix(const struct i915_syncmap *p, u64 id)
0125 {
0126     return id >> p->height >> SHIFT;
0127 }
0128
0129 static inline u64 __sync_leaf_prefix(const struct i915_syncmap *p, u64 id)
0130 {
0131     GEM_BUG_ON(p->height);
0132     return id >> SHIFT;
0133 }
0134
0135 static inline bool seqno_later(u32 a, u32 b)
0136 {
0137     return (s32)(a - b) >= 0;
0138 }
0139
0140 /**
0141  * i915_syncmap_is_later -- compare against the last know sync point
0142  * @root: pointer to the #i915_syncmap
0143  * @id: the context id (other timeline) we are synchronising to
0144  * @seqno: the sequence number along the other timeline
0145  *
0146  * If we have already synchronised this @root timeline with another (@id) then
0147  * we can omit any repeated or earlier synchronisation requests. If the two
0148  * timelines are already coupled, we can also omit the dependency between the
0149  * two as that is already known via the timeline.
0150  *
0151  * Returns true if the two timelines are already synchronised wrt to @seqno,
0152  * false if not and the synchronisation must be emitted.
0153  */
0154 bool i915_syncmap_is_later(struct i915_syncmap **root, u64 id, u32 seqno)
0155 {
0156     struct i915_syncmap *p;
0157     unsigned int idx;
0158
0159     p = *root;
0160     if (!p)
0161         return false;
0162
0163     if (likely(__sync_leaf_prefix(p, id) == p->prefix))
0164         goto found;
0165
0166     /* First climb the tree back to a parent branch */
0167     do {
0168         p = p->parent;
0169         if (!p)
0170             return false;
0171
0172         if (__sync_branch_prefix(p, id) == p->prefix)
0173             break;
0174     } while (1);
0175
0176     /* And then descend again until we find our leaf */
0177     do {
0178         if (!p->height)
0179             break;
0180
0181         p = __sync_child(p)[__sync_branch_idx(p, id)];
0182         if (!p)
0183             return false;
0184
0185         if (__sync_branch_prefix(p, id) != p->prefix)
0186             return false;
0187     } while (1);
0188
0189     *root = p;
0190 found:
0191     idx = __sync_leaf_idx(p, id);
0192     if (!(p->bitmap & BIT(idx)))
0193         return false;
0194
0195     return seqno_later(__sync_seqno(p)[idx], seqno);
0196 }
0197
0198 static struct i915_syncmap *
0199 __sync_alloc_leaf(struct i915_syncmap *parent, u64 id)
0200 {
0201     struct i915_syncmap *p;
0202
0203     p = kmalloc(sizeof(*p) + KSYNCMAP * sizeof(u32), GFP_KERNEL);
0204     if (unlikely(!p))
0205         return NULL;
0206
0207     p->parent = parent;
0208     p->height = 0;
0209     p->bitmap = 0;
0210     p->prefix = __sync_leaf_prefix(p, id);
0211     return p;
0212 }
0213
0214 static inline void __sync_set_seqno(struct i915_syncmap *p, u64 id, u32 seqno)
0215 {
0216     unsigned int idx = __sync_leaf_idx(p, id);
0217
0218     p->bitmap |= BIT(idx);
0219     __sync_seqno(p)[idx] = seqno;
0220 }
0221
0222 static inline void __sync_set_child(struct i915_syncmap *p,
0223                     unsigned int idx,
0224                     struct i915_syncmap *child)
0225 {
0226     p->bitmap |= BIT(idx);
0227     __sync_child(p)[idx] = child;
0228 }
0229
0230 static noinline int __sync_set(struct i915_syncmap **root, u64 id, u32 seqno)
0231 {
0232     struct i915_syncmap *p = *root;
0233     unsigned int idx;
0234
0235     if (!p) {
0236         p = __sync_alloc_leaf(NULL, id);
0237         if (unlikely(!p))
0238             return -ENOMEM;
0239
0240         goto found;
0241     }
0242
0243     /* Caller handled the likely cached case */
0244     GEM_BUG_ON(__sync_leaf_prefix(p, id) == p->prefix);
0245
0246     /* Climb back up the tree until we find a common prefix */
0247     do {
0248         if (!p->parent)
0249             break;
0250
0251         p = p->parent;
0252
0253         if (__sync_branch_prefix(p, id) == p->prefix)
0254             break;
0255     } while (1);
0256
0257     /*
0258      * No shortcut, we have to descend the tree to find the right layer
0259      * containing this fence.
0260      *
0261      * Each layer in the tree holds 16 (KSYNCMAP) pointers, either fences
0262      * or lower layers. Leaf nodes (height = 0) contain the fences, all
0263      * other nodes (height > 0) are internal layers that point to a lower
0264      * node. Each internal layer has at least 2 descendents.
0265      *
0266      * Starting at the top, we check whether the current prefix matches. If
0267      * it doesn't, we have gone past our target and need to insert a join
0268      * into the tree, and a new leaf node for the target as a descendent
0269      * of the join, as well as the original layer.
0270      *
0271      * The matching prefix means we are still following the right branch
0272      * of the tree. If it has height 0, we have found our leaf and just
0273      * need to replace the fence slot with ourselves. If the height is
0274      * not zero, our slot contains the next layer in the tree (unless
0275      * it is empty, in which case we can add ourselves as a new leaf).
0276      * As descend the tree the prefix grows (and height decreases).
0277      */
0278     do {
0279         struct i915_syncmap *next;
0280
0281         if (__sync_branch_prefix(p, id) != p->prefix) {
0282             unsigned int above;
0283
0284             /* Insert a join above the current layer */
0285             next = kzalloc(sizeof(*next) + KSYNCMAP * sizeof(next),
0286                        GFP_KERNEL);
0287             if (unlikely(!next))
0288                 return -ENOMEM;
0289
0290             /* Compute the height at which these two diverge */
0291             above = fls64(__sync_branch_prefix(p, id) ^ p->prefix);
0292             above = round_up(above, SHIFT);
0293             next->height = above + p->height;
0294             next->prefix = __sync_branch_prefix(next, id);
0295
0296             /* Insert the join into the parent */
0297             if (p->parent) {
0298                 idx = __sync_branch_idx(p->parent, id);
0299                 __sync_child(p->parent)[idx] = next;
0300                 GEM_BUG_ON(!(p->parent->bitmap & BIT(idx)));
0301             }
0302             next->parent = p->parent;
0303
0304             /* Compute the idx of the other branch, not our id! */
0305             idx = p->prefix >> (above - SHIFT) & MASK;
0306             __sync_set_child(next, idx, p);
0307             p->parent = next;
0308
0309             /* Ascend to the join */
0310             p = next;
0311         } else {
0312             if (!p->height)
0313                 break;
0314         }
0315
0316         /* Descend into the next layer */
0317         GEM_BUG_ON(!p->height);
0318         idx = __sync_branch_idx(p, id);
0319         next = __sync_child(p)[idx];
0320         if (!next) {
0321             next = __sync_alloc_leaf(p, id);
0322             if (unlikely(!next))
0323                 return -ENOMEM;
0324
0325             __sync_set_child(p, idx, next);
0326             p = next;
0327             break;
0328         }
0329
0330         p = next;
0331     } while (1);
0332
0333 found:
0334     GEM_BUG_ON(p->prefix != __sync_leaf_prefix(p, id));
0335     __sync_set_seqno(p, id, seqno);
0336     *root = p;
0337     return 0;
0338 }
0339
0340 /**
0341  * i915_syncmap_set -- mark the most recent syncpoint between contexts
0342  * @root: pointer to the #i915_syncmap
0343  * @id: the context id (other timeline) we have synchronised to
0344  * @seqno: the sequence number along the other timeline
0345  *
0346  * When we synchronise this @root timeline with another (@id), we also know
0347  * that we have synchronized with all previous seqno along that timeline. If
0348  * we then have a request to synchronise with the same seqno or older, we can
0349  * omit it, see i915_syncmap_is_later()
0350  *
0351  * Returns 0 on success, or a negative error code.
0352  */
0353 int i915_syncmap_set(struct i915_syncmap **root, u64 id, u32 seqno)
0354 {
0355     struct i915_syncmap *p = *root;
0356
0357     /*
0358      * We expect to be called in sequence following is_later(id), which
0359      * should have preloaded the root for us.
0360      */
0361     if (likely(p && __sync_leaf_prefix(p, id) == p->prefix)) {
0362         __sync_set_seqno(p, id, seqno);
0363         return 0;
0364     }
0365
0366     return __sync_set(root, id, seqno);
0367 }
0368
0369 static void __sync_free(struct i915_syncmap *p)
0370 {
0371     if (p->height) {
0372         unsigned int i;
0373
0374         while ((i = ffs(p->bitmap))) {
0375             p->bitmap &= ~0u << i;
0376             __sync_free(__sync_child(p)[i - 1]);
0377         }
0378     }
0379
0380     kfree(p);
0381 }
0382
0383 /**
0384  * i915_syncmap_free -- free all memory associated with the syncmap
0385  * @root: pointer to the #i915_syncmap
0386  *
0387  * Either when the timeline is to be freed and we no longer need the sync
0388  * point tracking, or when the fences are all known to be signaled and the
0389  * sync point tracking is redundant, we can free the #i915_syncmap to recover
0390  * its allocations.
0391  *
0392  * Will reinitialise the @root pointer so that the #i915_syncmap is ready for
0393  * reuse.
0394  */
0395 void i915_syncmap_free(struct i915_syncmap **root)
0396 {
0397     struct i915_syncmap *p;
0398
0399     p = *root;
0400     if (!p)
0401         return;
0402
0403     while (p->parent)
0404         p = p->parent;
0405
0406     __sync_free(p);
0407     *root = NULL;
0408 }
0409
0410 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
0411 #include "selftests/i915_syncmap.c"
0412 #endif