drm/vmwgfx/vmwgfx_page_dirty.c

0001 // SPDX-License-Identifier: GPL-2.0 OR MIT
0002 /**************************************************************************
0003  *
0004  * Copyright 2019 VMware, Inc., Palo Alto, CA., USA
0005  *
0006  * Permission is hereby granted, free of charge, to any person obtaining a
0007  * copy of this software and associated documentation files (the
0008  * "Software"), to deal in the Software without restriction, including
0009  * without limitation the rights to use, copy, modify, merge, publish,
0010  * distribute, sub license, and/or sell copies of the Software, and to
0011  * permit persons to whom the Software is furnished to do so, subject to
0012  * the following conditions:
0013  *
0014  * The above copyright notice and this permission notice (including the
0015  * next paragraph) shall be included in all copies or substantial portions
0016  * of the Software.
0017  *
0018  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0019  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0020  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
0021  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
0022  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
0023  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
0024  * USE OR OTHER DEALINGS IN THE SOFTWARE.
0025  *
0026  **************************************************************************/
0027 #include "vmwgfx_drv.h"
0028
0029 /*
0030  * Different methods for tracking dirty:
0031  * VMW_BO_DIRTY_PAGETABLE - Scan the pagetable for hardware dirty bits
0032  * VMW_BO_DIRTY_MKWRITE - Write-protect page table entries and record write-
0033  * accesses in the VM mkwrite() callback
0034  */
0035 enum vmw_bo_dirty_method {
0036     VMW_BO_DIRTY_PAGETABLE,
0037     VMW_BO_DIRTY_MKWRITE,
0038 };
0039
0040 /*
0041  * No dirtied pages at scan trigger a transition to the _MKWRITE method,
0042  * similarly a certain percentage of dirty pages trigger a transition to
0043  * the _PAGETABLE method. How many triggers should we wait for before
0044  * changing method?
0045  */
0046 #define VMW_DIRTY_NUM_CHANGE_TRIGGERS 2
0047
0048 /* Percentage to trigger a transition to the _PAGETABLE method */
0049 #define VMW_DIRTY_PERCENTAGE 10
0050
0051 /**
0052  * struct vmw_bo_dirty - Dirty information for buffer objects
0053  * @start: First currently dirty bit
0054  * @end: Last currently dirty bit + 1
0055  * @method: The currently used dirty method
0056  * @change_count: Number of consecutive method change triggers
0057  * @ref_count: Reference count for this structure
0058  * @bitmap_size: The size of the bitmap in bits. Typically equal to the
0059  * nuber of pages in the bo.
0060  * @bitmap: A bitmap where each bit represents a page. A set bit means a
0061  * dirty page.
0062  */
0063 struct vmw_bo_dirty {
0064     unsigned long start;
0065     unsigned long end;
0066     enum vmw_bo_dirty_method method;
0067     unsigned int change_count;
0068     unsigned int ref_count;
0069     unsigned long bitmap_size;
0070     unsigned long bitmap[];
0071 };
0072
0073 /**
0074  * vmw_bo_dirty_scan_pagetable - Perform a pagetable scan for dirty bits
0075  * @vbo: The buffer object to scan
0076  *
0077  * Scans the pagetable for dirty bits. Clear those bits and modify the
0078  * dirty structure with the results. This function may change the
0079  * dirty-tracking method.
0080  */
0081 static void vmw_bo_dirty_scan_pagetable(struct vmw_buffer_object *vbo)
0082 {
0083     struct vmw_bo_dirty *dirty = vbo->dirty;
0084     pgoff_t offset = drm_vma_node_start(&vbo->base.base.vma_node);
0085     struct address_space *mapping = vbo->base.bdev->dev_mapping;
0086     pgoff_t num_marked;
0087
0088     num_marked = clean_record_shared_mapping_range
0089         (mapping,
0090          offset, dirty->bitmap_size,
0091          offset, &dirty->bitmap[0],
0092          &dirty->start, &dirty->end);
0093     if (num_marked == 0)
0094         dirty->change_count++;
0095     else
0096         dirty->change_count = 0;
0097
0098     if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
0099         dirty->change_count = 0;
0100         dirty->method = VMW_BO_DIRTY_MKWRITE;
0101         wp_shared_mapping_range(mapping,
0102                     offset, dirty->bitmap_size);
0103         clean_record_shared_mapping_range(mapping,
0104                           offset, dirty->bitmap_size,
0105                           offset, &dirty->bitmap[0],
0106                           &dirty->start, &dirty->end);
0107     }
0108 }
0109
0110 /**
0111  * vmw_bo_dirty_scan_mkwrite - Reset the mkwrite dirty-tracking method
0112  * @vbo: The buffer object to scan
0113  *
0114  * Write-protect pages written to so that consecutive write accesses will
0115  * trigger a call to mkwrite.
0116  *
0117  * This function may change the dirty-tracking method.
0118  */
0119 static void vmw_bo_dirty_scan_mkwrite(struct vmw_buffer_object *vbo)
0120 {
0121     struct vmw_bo_dirty *dirty = vbo->dirty;
0122     unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node);
0123     struct address_space *mapping = vbo->base.bdev->dev_mapping;
0124     pgoff_t num_marked;
0125
0126     if (dirty->end <= dirty->start)
0127         return;
0128
0129     num_marked = wp_shared_mapping_range(vbo->base.bdev->dev_mapping,
0130                     dirty->start + offset,
0131                     dirty->end - dirty->start);
0132
0133     if (100UL * num_marked / dirty->bitmap_size >
0134         VMW_DIRTY_PERCENTAGE) {
0135         dirty->change_count++;
0136     } else {
0137         dirty->change_count = 0;
0138     }
0139
0140     if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
0141         pgoff_t start = 0;
0142         pgoff_t end = dirty->bitmap_size;
0143
0144         dirty->method = VMW_BO_DIRTY_PAGETABLE;
0145         clean_record_shared_mapping_range(mapping, offset, end, offset,
0146                           &dirty->bitmap[0],
0147                           &start, &end);
0148         bitmap_clear(&dirty->bitmap[0], 0, dirty->bitmap_size);
0149         if (dirty->start < dirty->end)
0150             bitmap_set(&dirty->bitmap[0], dirty->start,
0151                    dirty->end - dirty->start);
0152         dirty->change_count = 0;
0153     }
0154 }
0155
0156 /**
0157  * vmw_bo_dirty_scan - Scan for dirty pages and add them to the dirty
0158  * tracking structure
0159  * @vbo: The buffer object to scan
0160  *
0161  * This function may change the dirty tracking method.
0162  */
0163 void vmw_bo_dirty_scan(struct vmw_buffer_object *vbo)
0164 {
0165     struct vmw_bo_dirty *dirty = vbo->dirty;
0166
0167     if (dirty->method == VMW_BO_DIRTY_PAGETABLE)
0168         vmw_bo_dirty_scan_pagetable(vbo);
0169     else
0170         vmw_bo_dirty_scan_mkwrite(vbo);
0171 }
0172
0173 /**
0174  * vmw_bo_dirty_pre_unmap - write-protect and pick up dirty pages before
0175  * an unmap_mapping_range operation.
0176  * @vbo: The buffer object,
0177  * @start: First page of the range within the buffer object.
0178  * @end: Last page of the range within the buffer object + 1.
0179  *
0180  * If we're using the _PAGETABLE scan method, we may leak dirty pages
0181  * when calling unmap_mapping_range(). This function makes sure we pick
0182  * up all dirty pages.
0183  */
0184 static void vmw_bo_dirty_pre_unmap(struct vmw_buffer_object *vbo,
0185                    pgoff_t start, pgoff_t end)
0186 {
0187     struct vmw_bo_dirty *dirty = vbo->dirty;
0188     unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node);
0189     struct address_space *mapping = vbo->base.bdev->dev_mapping;
0190
0191     if (dirty->method != VMW_BO_DIRTY_PAGETABLE || start >= end)
0192         return;
0193
0194     wp_shared_mapping_range(mapping, start + offset, end - start);
0195     clean_record_shared_mapping_range(mapping, start + offset,
0196                       end - start, offset,
0197                       &dirty->bitmap[0], &dirty->start,
0198                       &dirty->end);
0199 }
0200
0201 /**
0202  * vmw_bo_dirty_unmap - Clear all ptes pointing to a range within a bo
0203  * @vbo: The buffer object,
0204  * @start: First page of the range within the buffer object.
0205  * @end: Last page of the range within the buffer object + 1.
0206  *
0207  * This is similar to ttm_bo_unmap_virtual() except it takes a subrange.
0208  */
0209 void vmw_bo_dirty_unmap(struct vmw_buffer_object *vbo,
0210             pgoff_t start, pgoff_t end)
0211 {
0212     unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node);
0213     struct address_space *mapping = vbo->base.bdev->dev_mapping;
0214
0215     vmw_bo_dirty_pre_unmap(vbo, start, end);
0216     unmap_shared_mapping_range(mapping, (offset + start) << PAGE_SHIFT,
0217                    (loff_t) (end - start) << PAGE_SHIFT);
0218 }
0219
0220 /**
0221  * vmw_bo_dirty_add - Add a dirty-tracking user to a buffer object
0222  * @vbo: The buffer object
0223  *
0224  * This function registers a dirty-tracking user to a buffer object.
0225  * A user can be for example a resource or a vma in a special user-space
0226  * mapping.
0227  *
0228  * Return: Zero on success, -ENOMEM on memory allocation failure.
0229  */
0230 int vmw_bo_dirty_add(struct vmw_buffer_object *vbo)
0231 {
0232     struct vmw_bo_dirty *dirty = vbo->dirty;
0233     pgoff_t num_pages = vbo->base.resource->num_pages;
0234     size_t size;
0235     int ret;
0236
0237     if (dirty) {
0238         dirty->ref_count++;
0239         return 0;
0240     }
0241
0242     size = sizeof(*dirty) + BITS_TO_LONGS(num_pages) * sizeof(long);
0243     dirty = kvzalloc(size, GFP_KERNEL);
0244     if (!dirty) {
0245         ret = -ENOMEM;
0246         goto out_no_dirty;
0247     }
0248
0249     dirty->bitmap_size = num_pages;
0250     dirty->start = dirty->bitmap_size;
0251     dirty->end = 0;
0252     dirty->ref_count = 1;
0253     if (num_pages < PAGE_SIZE / sizeof(pte_t)) {
0254         dirty->method = VMW_BO_DIRTY_PAGETABLE;
0255     } else {
0256         struct address_space *mapping = vbo->base.bdev->dev_mapping;
0257         pgoff_t offset = drm_vma_node_start(&vbo->base.base.vma_node);
0258
0259         dirty->method = VMW_BO_DIRTY_MKWRITE;
0260
0261         /* Write-protect and then pick up already dirty bits */
0262         wp_shared_mapping_range(mapping, offset, num_pages);
0263         clean_record_shared_mapping_range(mapping, offset, num_pages,
0264                           offset,
0265                           &dirty->bitmap[0],
0266                           &dirty->start, &dirty->end);
0267     }
0268
0269     vbo->dirty = dirty;
0270
0271     return 0;
0272
0273 out_no_dirty:
0274     return ret;
0275 }
0276
0277 /**
0278  * vmw_bo_dirty_release - Release a dirty-tracking user from a buffer object
0279  * @vbo: The buffer object
0280  *
0281  * This function releases a dirty-tracking user from a buffer object.
0282  * If the reference count reaches zero, then the dirty-tracking object is
0283  * freed and the pointer to it cleared.
0284  *
0285  * Return: Zero on success, -ENOMEM on memory allocation failure.
0286  */
0287 void vmw_bo_dirty_release(struct vmw_buffer_object *vbo)
0288 {
0289     struct vmw_bo_dirty *dirty = vbo->dirty;
0290
0291     if (dirty && --dirty->ref_count == 0) {
0292         kvfree(dirty);
0293         vbo->dirty = NULL;
0294     }
0295 }
0296
0297 /**
0298  * vmw_bo_dirty_transfer_to_res - Pick up a resource's dirty region from
0299  * its backing mob.
0300  * @res: The resource
0301  *
0302  * This function will pick up all dirty ranges affecting the resource from
0303  * it's backup mob, and call vmw_resource_dirty_update() once for each
0304  * range. The transferred ranges will be cleared from the backing mob's
0305  * dirty tracking.
0306  */
0307 void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res)
0308 {
0309     struct vmw_buffer_object *vbo = res->backup;
0310     struct vmw_bo_dirty *dirty = vbo->dirty;
0311     pgoff_t start, cur, end;
0312     unsigned long res_start = res->backup_offset;
0313     unsigned long res_end = res->backup_offset + res->backup_size;
0314
0315     WARN_ON_ONCE(res_start & ~PAGE_MASK);
0316     res_start >>= PAGE_SHIFT;
0317     res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
0318
0319     if (res_start >= dirty->end || res_end <= dirty->start)
0320         return;
0321
0322     cur = max(res_start, dirty->start);
0323     res_end = max(res_end, dirty->end);
0324     while (cur < res_end) {
0325         unsigned long num;
0326
0327         start = find_next_bit(&dirty->bitmap[0], res_end, cur);
0328         if (start >= res_end)
0329             break;
0330
0331         end = find_next_zero_bit(&dirty->bitmap[0], res_end, start + 1);
0332         cur = end + 1;
0333         num = end - start;
0334         bitmap_clear(&dirty->bitmap[0], start, num);
0335         vmw_resource_dirty_update(res, start, end);
0336     }
0337
0338     if (res_start <= dirty->start && res_end > dirty->start)
0339         dirty->start = res_end;
0340     if (res_start < dirty->end && res_end >= dirty->end)
0341         dirty->end = res_start;
0342 }
0343
0344 /**
0345  * vmw_bo_dirty_clear_res - Clear a resource's dirty region from
0346  * its backing mob.
0347  * @res: The resource
0348  *
0349  * This function will clear all dirty ranges affecting the resource from
0350  * it's backup mob's dirty tracking.
0351  */
0352 void vmw_bo_dirty_clear_res(struct vmw_resource *res)
0353 {
0354     unsigned long res_start = res->backup_offset;
0355     unsigned long res_end = res->backup_offset + res->backup_size;
0356     struct vmw_buffer_object *vbo = res->backup;
0357     struct vmw_bo_dirty *dirty = vbo->dirty;
0358
0359     res_start >>= PAGE_SHIFT;
0360     res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
0361
0362     if (res_start >= dirty->end || res_end <= dirty->start)
0363         return;
0364
0365     res_start = max(res_start, dirty->start);
0366     res_end = min(res_end, dirty->end);
0367     bitmap_clear(&dirty->bitmap[0], res_start, res_end - res_start);
0368
0369     if (res_start <= dirty->start && res_end > dirty->start)
0370         dirty->start = res_end;
0371     if (res_start < dirty->end && res_end >= dirty->end)
0372         dirty->end = res_start;
0373 }
0374
0375 vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf)
0376 {
0377     struct vm_area_struct *vma = vmf->vma;
0378     struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
0379         vma->vm_private_data;
0380     vm_fault_t ret;
0381     unsigned long page_offset;
0382     unsigned int save_flags;
0383     struct vmw_buffer_object *vbo =
0384         container_of(bo, typeof(*vbo), base);
0385
0386     /*
0387      * mkwrite() doesn't handle the VM_FAULT_RETRY return value correctly.
0388      * So make sure the TTM helpers are aware.
0389      */
0390     save_flags = vmf->flags;
0391     vmf->flags &= ~FAULT_FLAG_ALLOW_RETRY;
0392     ret = ttm_bo_vm_reserve(bo, vmf);
0393     vmf->flags = save_flags;
0394     if (ret)
0395         return ret;
0396
0397     page_offset = vmf->pgoff - drm_vma_node_start(&bo->base.vma_node);
0398     if (unlikely(page_offset >= bo->resource->num_pages)) {
0399         ret = VM_FAULT_SIGBUS;
0400         goto out_unlock;
0401     }
0402
0403     if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE &&
0404         !test_bit(page_offset, &vbo->dirty->bitmap[0])) {
0405         struct vmw_bo_dirty *dirty = vbo->dirty;
0406
0407         __set_bit(page_offset, &dirty->bitmap[0]);
0408         dirty->start = min(dirty->start, page_offset);
0409         dirty->end = max(dirty->end, page_offset + 1);
0410     }
0411
0412 out_unlock:
0413     dma_resv_unlock(bo->base.resv);
0414     return ret;
0415 }
0416
0417 vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf)
0418 {
0419     struct vm_area_struct *vma = vmf->vma;
0420     struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
0421         vma->vm_private_data;
0422     struct vmw_buffer_object *vbo =
0423         container_of(bo, struct vmw_buffer_object, base);
0424     pgoff_t num_prefault;
0425     pgprot_t prot;
0426     vm_fault_t ret;
0427
0428     ret = ttm_bo_vm_reserve(bo, vmf);
0429     if (ret)
0430         return ret;
0431
0432     num_prefault = (vma->vm_flags & VM_RAND_READ) ? 1 :
0433         TTM_BO_VM_NUM_PREFAULT;
0434
0435     if (vbo->dirty) {
0436         pgoff_t allowed_prefault;
0437         unsigned long page_offset;
0438
0439         page_offset = vmf->pgoff -
0440             drm_vma_node_start(&bo->base.vma_node);
0441         if (page_offset >= bo->resource->num_pages ||
0442             vmw_resources_clean(vbo, page_offset,
0443                     page_offset + PAGE_SIZE,
0444                     &allowed_prefault)) {
0445             ret = VM_FAULT_SIGBUS;
0446             goto out_unlock;
0447         }
0448
0449         num_prefault = min(num_prefault, allowed_prefault);
0450     }
0451
0452     /*
0453      * If we don't track dirty using the MKWRITE method, make sure
0454      * sure the page protection is write-enabled so we don't get
0455      * a lot of unnecessary write faults.
0456      */
0457     if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE)
0458         prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED);
0459     else
0460         prot = vm_get_page_prot(vma->vm_flags);
0461
0462     ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault);
0463     if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
0464         return ret;
0465
0466 out_unlock:
0467     dma_resv_unlock(bo->base.resv);
0468
0469     return ret;
0470 }