net/rds/page.c

0001 /*
0002  * Copyright (c) 2006 Oracle.  All rights reserved.
0003  *
0004  * This software is available to you under a choice of one of two
0005  * licenses.  You may choose to be licensed under the terms of the GNU
0006  * General Public License (GPL) Version 2, available from the file
0007  * COPYING in the main directory of this source tree, or the
0008  * OpenIB.org BSD license below:
0009  *
0010  *     Redistribution and use in source and binary forms, with or
0011  *     without modification, are permitted provided that the following
0012  *     conditions are met:
0013  *
0014  *      - Redistributions of source code must retain the above
0015  *        copyright notice, this list of conditions and the following
0016  *        disclaimer.
0017  *
0018  *      - Redistributions in binary form must reproduce the above
0019  *        copyright notice, this list of conditions and the following
0020  *        disclaimer in the documentation and/or other materials
0021  *        provided with the distribution.
0022  *
0023  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
0024  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
0025  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
0026  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
0027  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
0028  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
0029  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
0030  * SOFTWARE.
0031  *
0032  */
0033 #include <linux/highmem.h>
0034 #include <linux/gfp.h>
0035 #include <linux/cpu.h>
0036 #include <linux/export.h>
0037
0038 #include "rds.h"
0039
0040 struct rds_page_remainder {
0041     struct page *r_page;
0042     unsigned long   r_offset;
0043 };
0044
0045 static
0046 DEFINE_PER_CPU_SHARED_ALIGNED(struct rds_page_remainder, rds_page_remainders);
0047
0048 /**
0049  * rds_page_remainder_alloc - build up regions of a message.
0050  *
0051  * @scat: Scatter list for message
0052  * @bytes: the number of bytes needed.
0053  * @gfp: the waiting behaviour of the allocation
0054  *
0055  * @gfp is always ored with __GFP_HIGHMEM.  Callers must be prepared to
0056  * kmap the pages, etc.
0057  *
0058  * If @bytes is at least a full page then this just returns a page from
0059  * alloc_page().
0060  *
0061  * If @bytes is a partial page then this stores the unused region of the
0062  * page in a per-cpu structure.  Future partial-page allocations may be
0063  * satisfied from that cached region.  This lets us waste less memory on
0064  * small allocations with minimal complexity.  It works because the transmit
0065  * path passes read-only page regions down to devices.  They hold a page
0066  * reference until they are done with the region.
0067  */
0068 int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
0069                  gfp_t gfp)
0070 {
0071     struct rds_page_remainder *rem;
0072     unsigned long flags;
0073     struct page *page;
0074     int ret;
0075
0076     gfp |= __GFP_HIGHMEM;
0077
0078     /* jump straight to allocation if we're trying for a huge page */
0079     if (bytes >= PAGE_SIZE) {
0080         page = alloc_page(gfp);
0081         if (!page) {
0082             ret = -ENOMEM;
0083         } else {
0084             sg_set_page(scat, page, PAGE_SIZE, 0);
0085             ret = 0;
0086         }
0087         goto out;
0088     }
0089
0090     rem = &per_cpu(rds_page_remainders, get_cpu());
0091     local_irq_save(flags);
0092
0093     while (1) {
0094         /* avoid a tiny region getting stuck by tossing it */
0095         if (rem->r_page && bytes > (PAGE_SIZE - rem->r_offset)) {
0096             rds_stats_inc(s_page_remainder_miss);
0097             __free_page(rem->r_page);
0098             rem->r_page = NULL;
0099         }
0100
0101         /* hand out a fragment from the cached page */
0102         if (rem->r_page && bytes <= (PAGE_SIZE - rem->r_offset)) {
0103             sg_set_page(scat, rem->r_page, bytes, rem->r_offset);
0104             get_page(sg_page(scat));
0105
0106             if (rem->r_offset != 0)
0107                 rds_stats_inc(s_page_remainder_hit);
0108
0109             rem->r_offset += ALIGN(bytes, 8);
0110             if (rem->r_offset >= PAGE_SIZE) {
0111                 __free_page(rem->r_page);
0112                 rem->r_page = NULL;
0113             }
0114             ret = 0;
0115             break;
0116         }
0117
0118         /* alloc if there is nothing for us to use */
0119         local_irq_restore(flags);
0120         put_cpu();
0121
0122         page = alloc_page(gfp);
0123
0124         rem = &per_cpu(rds_page_remainders, get_cpu());
0125         local_irq_save(flags);
0126
0127         if (!page) {
0128             ret = -ENOMEM;
0129             break;
0130         }
0131
0132         /* did someone race to fill the remainder before us? */
0133         if (rem->r_page) {
0134             __free_page(page);
0135             continue;
0136         }
0137
0138         /* otherwise install our page and loop around to alloc */
0139         rem->r_page = page;
0140         rem->r_offset = 0;
0141     }
0142
0143     local_irq_restore(flags);
0144     put_cpu();
0145 out:
0146     rdsdebug("bytes %lu ret %d %p %u %u\n", bytes, ret,
0147          ret ? NULL : sg_page(scat), ret ? 0 : scat->offset,
0148          ret ? 0 : scat->length);
0149     return ret;
0150 }
0151 EXPORT_SYMBOL_GPL(rds_page_remainder_alloc);
0152
0153 void rds_page_exit(void)
0154 {
0155     unsigned int cpu;
0156
0157     for_each_possible_cpu(cpu) {
0158         struct rds_page_remainder *rem;
0159
0160         rem = &per_cpu(rds_page_remainders, cpu);
0161         rdsdebug("cpu %u\n", cpu);
0162
0163         if (rem->r_page)
0164             __free_page(rem->r_page);
0165         rem->r_page = NULL;
0166     }
0167 }