drm/radeon/radeon_sa.c

0001 /*
0002  * Copyright 2011 Red Hat Inc.
0003  * All Rights Reserved.
0004  *
0005  * Permission is hereby granted, free of charge, to any person obtaining a
0006  * copy of this software and associated documentation files (the
0007  * "Software"), to deal in the Software without restriction, including
0008  * without limitation the rights to use, copy, modify, merge, publish,
0009  * distribute, sub license, and/or sell copies of the Software, and to
0010  * permit persons to whom the Software is furnished to do so, subject to
0011  * the following conditions:
0012  *
0013  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0014  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0015  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
0016  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
0017  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
0018  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
0019  * USE OR OTHER DEALINGS IN THE SOFTWARE.
0020  *
0021  * The above copyright notice and this permission notice (including the
0022  * next paragraph) shall be included in all copies or substantial portions
0023  * of the Software.
0024  *
0025  */
0026 /*
0027  * Authors:
0028  *    Jerome Glisse <glisse@freedesktop.org>
0029  */
0030 /* Algorithm:
0031  *
0032  * We store the last allocated bo in "hole", we always try to allocate
0033  * after the last allocated bo. Principle is that in a linear GPU ring
0034  * progression was is after last is the oldest bo we allocated and thus
0035  * the first one that should no longer be in use by the GPU.
0036  *
0037  * If it's not the case we skip over the bo after last to the closest
0038  * done bo if such one exist. If none exist and we are not asked to
0039  * block we report failure to allocate.
0040  *
0041  * If we are asked to block we wait on all the oldest fence of all
0042  * rings. We just wait for any of those fence to complete.
0043  */
0044
0045 #include "radeon.h"
0046
0047 static void radeon_sa_bo_remove_locked(struct radeon_sa_bo *sa_bo);
0048 static void radeon_sa_bo_try_free(struct radeon_sa_manager *sa_manager);
0049
0050 int radeon_sa_bo_manager_init(struct radeon_device *rdev,
0051                   struct radeon_sa_manager *sa_manager,
0052                   unsigned size, u32 align, u32 domain, u32 flags)
0053 {
0054     int i, r;
0055
0056     init_waitqueue_head(&sa_manager->wq);
0057     sa_manager->bo = NULL;
0058     sa_manager->size = size;
0059     sa_manager->domain = domain;
0060     sa_manager->align = align;
0061     sa_manager->hole = &sa_manager->olist;
0062     INIT_LIST_HEAD(&sa_manager->olist);
0063     for (i = 0; i < RADEON_NUM_RINGS; ++i) {
0064         INIT_LIST_HEAD(&sa_manager->flist[i]);
0065     }
0066
0067     r = radeon_bo_create(rdev, size, align, true,
0068                  domain, flags, NULL, NULL, &sa_manager->bo);
0069     if (r) {
0070         dev_err(rdev->dev, "(%d) failed to allocate bo for manager\n", r);
0071         return r;
0072     }
0073
0074     return r;
0075 }
0076
0077 void radeon_sa_bo_manager_fini(struct radeon_device *rdev,
0078                    struct radeon_sa_manager *sa_manager)
0079 {
0080     struct radeon_sa_bo *sa_bo, *tmp;
0081
0082     if (!list_empty(&sa_manager->olist)) {
0083         sa_manager->hole = &sa_manager->olist,
0084         radeon_sa_bo_try_free(sa_manager);
0085         if (!list_empty(&sa_manager->olist)) {
0086             dev_err(rdev->dev, "sa_manager is not empty, clearing anyway\n");
0087         }
0088     }
0089     list_for_each_entry_safe(sa_bo, tmp, &sa_manager->olist, olist) {
0090         radeon_sa_bo_remove_locked(sa_bo);
0091     }
0092     radeon_bo_unref(&sa_manager->bo);
0093     sa_manager->size = 0;
0094 }
0095
0096 int radeon_sa_bo_manager_start(struct radeon_device *rdev,
0097                    struct radeon_sa_manager *sa_manager)
0098 {
0099     int r;
0100
0101     if (sa_manager->bo == NULL) {
0102         dev_err(rdev->dev, "no bo for sa manager\n");
0103         return -EINVAL;
0104     }
0105
0106     /* map the buffer */
0107     r = radeon_bo_reserve(sa_manager->bo, false);
0108     if (r) {
0109         dev_err(rdev->dev, "(%d) failed to reserve manager bo\n", r);
0110         return r;
0111     }
0112     r = radeon_bo_pin(sa_manager->bo, sa_manager->domain, &sa_manager->gpu_addr);
0113     if (r) {
0114         radeon_bo_unreserve(sa_manager->bo);
0115         dev_err(rdev->dev, "(%d) failed to pin manager bo\n", r);
0116         return r;
0117     }
0118     r = radeon_bo_kmap(sa_manager->bo, &sa_manager->cpu_ptr);
0119     radeon_bo_unreserve(sa_manager->bo);
0120     return r;
0121 }
0122
0123 int radeon_sa_bo_manager_suspend(struct radeon_device *rdev,
0124                  struct radeon_sa_manager *sa_manager)
0125 {
0126     int r;
0127
0128     if (sa_manager->bo == NULL) {
0129         dev_err(rdev->dev, "no bo for sa manager\n");
0130         return -EINVAL;
0131     }
0132
0133     r = radeon_bo_reserve(sa_manager->bo, false);
0134     if (!r) {
0135         radeon_bo_kunmap(sa_manager->bo);
0136         radeon_bo_unpin(sa_manager->bo);
0137         radeon_bo_unreserve(sa_manager->bo);
0138     }
0139     return r;
0140 }
0141
0142 static void radeon_sa_bo_remove_locked(struct radeon_sa_bo *sa_bo)
0143 {
0144     struct radeon_sa_manager *sa_manager = sa_bo->manager;
0145     if (sa_manager->hole == &sa_bo->olist) {
0146         sa_manager->hole = sa_bo->olist.prev;
0147     }
0148     list_del_init(&sa_bo->olist);
0149     list_del_init(&sa_bo->flist);
0150     radeon_fence_unref(&sa_bo->fence);
0151     kfree(sa_bo);
0152 }
0153
0154 static void radeon_sa_bo_try_free(struct radeon_sa_manager *sa_manager)
0155 {
0156     struct radeon_sa_bo *sa_bo, *tmp;
0157
0158     if (sa_manager->hole->next == &sa_manager->olist)
0159         return;
0160
0161     sa_bo = list_entry(sa_manager->hole->next, struct radeon_sa_bo, olist);
0162     list_for_each_entry_safe_from(sa_bo, tmp, &sa_manager->olist, olist) {
0163         if (sa_bo->fence == NULL || !radeon_fence_signaled(sa_bo->fence)) {
0164             return;
0165         }
0166         radeon_sa_bo_remove_locked(sa_bo);
0167     }
0168 }
0169
0170 static inline unsigned radeon_sa_bo_hole_soffset(struct radeon_sa_manager *sa_manager)
0171 {
0172     struct list_head *hole = sa_manager->hole;
0173
0174     if (hole != &sa_manager->olist) {
0175         return list_entry(hole, struct radeon_sa_bo, olist)->eoffset;
0176     }
0177     return 0;
0178 }
0179
0180 static inline unsigned radeon_sa_bo_hole_eoffset(struct radeon_sa_manager *sa_manager)
0181 {
0182     struct list_head *hole = sa_manager->hole;
0183
0184     if (hole->next != &sa_manager->olist) {
0185         return list_entry(hole->next, struct radeon_sa_bo, olist)->soffset;
0186     }
0187     return sa_manager->size;
0188 }
0189
0190 static bool radeon_sa_bo_try_alloc(struct radeon_sa_manager *sa_manager,
0191                    struct radeon_sa_bo *sa_bo,
0192                    unsigned size, unsigned align)
0193 {
0194     unsigned soffset, eoffset, wasted;
0195
0196     soffset = radeon_sa_bo_hole_soffset(sa_manager);
0197     eoffset = radeon_sa_bo_hole_eoffset(sa_manager);
0198     wasted = (align - (soffset % align)) % align;
0199
0200     if ((eoffset - soffset) >= (size + wasted)) {
0201         soffset += wasted;
0202
0203         sa_bo->manager = sa_manager;
0204         sa_bo->soffset = soffset;
0205         sa_bo->eoffset = soffset + size;
0206         list_add(&sa_bo->olist, sa_manager->hole);
0207         INIT_LIST_HEAD(&sa_bo->flist);
0208         sa_manager->hole = &sa_bo->olist;
0209         return true;
0210     }
0211     return false;
0212 }
0213
0214 /**
0215  * radeon_sa_event - Check if we can stop waiting
0216  *
0217  * @sa_manager: pointer to the sa_manager
0218  * @size: number of bytes we want to allocate
0219  * @align: alignment we need to match
0220  *
0221  * Check if either there is a fence we can wait for or
0222  * enough free memory to satisfy the allocation directly
0223  */
0224 static bool radeon_sa_event(struct radeon_sa_manager *sa_manager,
0225                 unsigned size, unsigned align)
0226 {
0227     unsigned soffset, eoffset, wasted;
0228     int i;
0229
0230     for (i = 0; i < RADEON_NUM_RINGS; ++i) {
0231         if (!list_empty(&sa_manager->flist[i])) {
0232             return true;
0233         }
0234     }
0235
0236     soffset = radeon_sa_bo_hole_soffset(sa_manager);
0237     eoffset = radeon_sa_bo_hole_eoffset(sa_manager);
0238     wasted = (align - (soffset % align)) % align;
0239
0240     if ((eoffset - soffset) >= (size + wasted)) {
0241         return true;
0242     }
0243
0244     return false;
0245 }
0246
0247 static bool radeon_sa_bo_next_hole(struct radeon_sa_manager *sa_manager,
0248                    struct radeon_fence **fences,
0249                    unsigned *tries)
0250 {
0251     struct radeon_sa_bo *best_bo = NULL;
0252     unsigned i, soffset, best, tmp;
0253
0254     /* if hole points to the end of the buffer */
0255     if (sa_manager->hole->next == &sa_manager->olist) {
0256         /* try again with its beginning */
0257         sa_manager->hole = &sa_manager->olist;
0258         return true;
0259     }
0260
0261     soffset = radeon_sa_bo_hole_soffset(sa_manager);
0262     /* to handle wrap around we add sa_manager->size */
0263     best = sa_manager->size * 2;
0264     /* go over all fence list and try to find the closest sa_bo
0265      * of the current last
0266      */
0267     for (i = 0; i < RADEON_NUM_RINGS; ++i) {
0268         struct radeon_sa_bo *sa_bo;
0269
0270         fences[i] = NULL;
0271
0272         if (list_empty(&sa_manager->flist[i])) {
0273             continue;
0274         }
0275
0276         sa_bo = list_first_entry(&sa_manager->flist[i],
0277                      struct radeon_sa_bo, flist);
0278
0279         if (!radeon_fence_signaled(sa_bo->fence)) {
0280             fences[i] = sa_bo->fence;
0281             continue;
0282         }
0283
0284         /* limit the number of tries each ring gets */
0285         if (tries[i] > 2) {
0286             continue;
0287         }
0288
0289         tmp = sa_bo->soffset;
0290         if (tmp < soffset) {
0291             /* wrap around, pretend it's after */
0292             tmp += sa_manager->size;
0293         }
0294         tmp -= soffset;
0295         if (tmp < best) {
0296             /* this sa bo is the closest one */
0297             best = tmp;
0298             best_bo = sa_bo;
0299         }
0300     }
0301
0302     if (best_bo) {
0303         ++tries[best_bo->fence->ring];
0304         sa_manager->hole = best_bo->olist.prev;
0305
0306         /* we knew that this one is signaled,
0307            so it's save to remote it */
0308         radeon_sa_bo_remove_locked(best_bo);
0309         return true;
0310     }
0311     return false;
0312 }
0313
0314 int radeon_sa_bo_new(struct radeon_device *rdev,
0315              struct radeon_sa_manager *sa_manager,
0316              struct radeon_sa_bo **sa_bo,
0317              unsigned size, unsigned align)
0318 {
0319     struct radeon_fence *fences[RADEON_NUM_RINGS];
0320     unsigned tries[RADEON_NUM_RINGS];
0321     int i, r;
0322
0323     BUG_ON(align > sa_manager->align);
0324     BUG_ON(size > sa_manager->size);
0325
0326     *sa_bo = kmalloc(sizeof(struct radeon_sa_bo), GFP_KERNEL);
0327     if ((*sa_bo) == NULL) {
0328         return -ENOMEM;
0329     }
0330     (*sa_bo)->manager = sa_manager;
0331     (*sa_bo)->fence = NULL;
0332     INIT_LIST_HEAD(&(*sa_bo)->olist);
0333     INIT_LIST_HEAD(&(*sa_bo)->flist);
0334
0335     spin_lock(&sa_manager->wq.lock);
0336     do {
0337         for (i = 0; i < RADEON_NUM_RINGS; ++i)
0338             tries[i] = 0;
0339
0340         do {
0341             radeon_sa_bo_try_free(sa_manager);
0342
0343             if (radeon_sa_bo_try_alloc(sa_manager, *sa_bo,
0344                            size, align)) {
0345                 spin_unlock(&sa_manager->wq.lock);
0346                 return 0;
0347             }
0348
0349             /* see if we can skip over some allocations */
0350         } while (radeon_sa_bo_next_hole(sa_manager, fences, tries));
0351
0352         for (i = 0; i < RADEON_NUM_RINGS; ++i)
0353             radeon_fence_ref(fences[i]);
0354
0355         spin_unlock(&sa_manager->wq.lock);
0356         r = radeon_fence_wait_any(rdev, fences, false);
0357         for (i = 0; i < RADEON_NUM_RINGS; ++i)
0358             radeon_fence_unref(&fences[i]);
0359         spin_lock(&sa_manager->wq.lock);
0360         /* if we have nothing to wait for block */
0361         if (r == -ENOENT) {
0362             r = wait_event_interruptible_locked(
0363                 sa_manager->wq,
0364                 radeon_sa_event(sa_manager, size, align)
0365             );
0366         }
0367
0368     } while (!r);
0369
0370     spin_unlock(&sa_manager->wq.lock);
0371     kfree(*sa_bo);
0372     *sa_bo = NULL;
0373     return r;
0374 }
0375
0376 void radeon_sa_bo_free(struct radeon_device *rdev, struct radeon_sa_bo **sa_bo,
0377                struct radeon_fence *fence)
0378 {
0379     struct radeon_sa_manager *sa_manager;
0380
0381     if (sa_bo == NULL || *sa_bo == NULL) {
0382         return;
0383     }
0384
0385     sa_manager = (*sa_bo)->manager;
0386     spin_lock(&sa_manager->wq.lock);
0387     if (fence && !radeon_fence_signaled(fence)) {
0388         (*sa_bo)->fence = radeon_fence_ref(fence);
0389         list_add_tail(&(*sa_bo)->flist,
0390                   &sa_manager->flist[fence->ring]);
0391     } else {
0392         radeon_sa_bo_remove_locked(*sa_bo);
0393     }
0394     wake_up_all_locked(&sa_manager->wq);
0395     spin_unlock(&sa_manager->wq.lock);
0396     *sa_bo = NULL;
0397 }
0398
0399 #if defined(CONFIG_DEBUG_FS)
0400 void radeon_sa_bo_dump_debug_info(struct radeon_sa_manager *sa_manager,
0401                   struct seq_file *m)
0402 {
0403     struct radeon_sa_bo *i;
0404
0405     spin_lock(&sa_manager->wq.lock);
0406     list_for_each_entry(i, &sa_manager->olist, olist) {
0407         uint64_t soffset = i->soffset + sa_manager->gpu_addr;
0408         uint64_t eoffset = i->eoffset + sa_manager->gpu_addr;
0409         if (&i->olist == sa_manager->hole) {
0410             seq_printf(m, ">");
0411         } else {
0412             seq_printf(m, " ");
0413         }
0414         seq_printf(m, "[0x%010llx 0x%010llx] size %8lld",
0415                soffset, eoffset, eoffset - soffset);
0416         if (i->fence) {
0417             seq_printf(m, " protected by 0x%016llx on ring %d",
0418                    i->fence->seq, i->fence->ring);
0419         }
0420         seq_printf(m, "\n");
0421     }
0422     spin_unlock(&sa_manager->wq.lock);
0423 }
0424 #endif