Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2006 Oracle. All rights reserved. |
| 3 | * |
| 4 | * This software is available to you under a choice of one of two |
| 5 | * licenses. You may choose to be licensed under the terms of the GNU |
| 6 | * General Public License (GPL) Version 2, available from the file |
| 7 | * COPYING in the main directory of this source tree, or the |
| 8 | * OpenIB.org BSD license below: |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or |
| 11 | * without modification, are permitted provided that the following |
| 12 | * conditions are met: |
| 13 | * |
| 14 | * - Redistributions of source code must retain the above |
| 15 | * copyright notice, this list of conditions and the following |
| 16 | * disclaimer. |
| 17 | * |
| 18 | * - Redistributions in binary form must reproduce the above |
| 19 | * copyright notice, this list of conditions and the following |
| 20 | * disclaimer in the documentation and/or other materials |
| 21 | * provided with the distribution. |
| 22 | * |
| 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 30 | * SOFTWARE. |
| 31 | * |
| 32 | */ |
| 33 | #include <linux/highmem.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 34 | #include <linux/gfp.h> |
Amerigo Wang | 80f1ff9 | 2011-07-25 17:13:08 -0700 | [diff] [blame] | 35 | #include <linux/cpu.h> |
Paul Gortmaker | bc3b2d7 | 2011-07-15 11:47:34 -0400 | [diff] [blame] | 36 | #include <linux/export.h> |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 37 | |
| 38 | #include "rds.h" |
| 39 | |
| 40 | struct rds_page_remainder { |
| 41 | struct page *r_page; |
| 42 | unsigned long r_offset; |
| 43 | }; |
| 44 | |
Peter Zijlstra | 2552821 | 2016-03-15 14:52:49 -0700 | [diff] [blame] | 45 | static |
| 46 | DEFINE_PER_CPU_SHARED_ALIGNED(struct rds_page_remainder, rds_page_remainders); |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 47 | |
Ben Hutchings | 2c53040 | 2012-07-10 10:55:09 +0000 | [diff] [blame] | 48 | /** |
| 49 | * rds_page_remainder_alloc - build up regions of a message. |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 50 | * |
Ben Hutchings | 2c53040 | 2012-07-10 10:55:09 +0000 | [diff] [blame] | 51 | * @scat: Scatter list for message |
| 52 | * @bytes: the number of bytes needed. |
| 53 | * @gfp: the waiting behaviour of the allocation |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 54 | * |
| 55 | * @gfp is always ored with __GFP_HIGHMEM. Callers must be prepared to |
| 56 | * kmap the pages, etc. |
| 57 | * |
| 58 | * If @bytes is at least a full page then this just returns a page from |
| 59 | * alloc_page(). |
| 60 | * |
| 61 | * If @bytes is a partial page then this stores the unused region of the |
| 62 | * page in a per-cpu structure. Future partial-page allocations may be |
| 63 | * satisfied from that cached region. This lets us waste less memory on |
| 64 | * small allocations with minimal complexity. It works because the transmit |
| 65 | * path passes read-only page regions down to devices. They hold a page |
| 66 | * reference until they are done with the region. |
| 67 | */ |
| 68 | int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes, |
| 69 | gfp_t gfp) |
| 70 | { |
| 71 | struct rds_page_remainder *rem; |
| 72 | unsigned long flags; |
| 73 | struct page *page; |
| 74 | int ret; |
| 75 | |
| 76 | gfp |= __GFP_HIGHMEM; |
| 77 | |
| 78 | /* jump straight to allocation if we're trying for a huge page */ |
| 79 | if (bytes >= PAGE_SIZE) { |
| 80 | page = alloc_page(gfp); |
Andy Grover | 8690bfa | 2010-01-12 11:56:44 -0800 | [diff] [blame] | 81 | if (!page) { |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 82 | ret = -ENOMEM; |
| 83 | } else { |
| 84 | sg_set_page(scat, page, PAGE_SIZE, 0); |
| 85 | ret = 0; |
| 86 | } |
| 87 | goto out; |
| 88 | } |
| 89 | |
| 90 | rem = &per_cpu(rds_page_remainders, get_cpu()); |
| 91 | local_irq_save(flags); |
| 92 | |
| 93 | while (1) { |
| 94 | /* avoid a tiny region getting stuck by tossing it */ |
| 95 | if (rem->r_page && bytes > (PAGE_SIZE - rem->r_offset)) { |
| 96 | rds_stats_inc(s_page_remainder_miss); |
| 97 | __free_page(rem->r_page); |
| 98 | rem->r_page = NULL; |
| 99 | } |
| 100 | |
| 101 | /* hand out a fragment from the cached page */ |
| 102 | if (rem->r_page && bytes <= (PAGE_SIZE - rem->r_offset)) { |
| 103 | sg_set_page(scat, rem->r_page, bytes, rem->r_offset); |
| 104 | get_page(sg_page(scat)); |
| 105 | |
| 106 | if (rem->r_offset != 0) |
| 107 | rds_stats_inc(s_page_remainder_hit); |
| 108 | |
shamir rabinovitch | e98499a | 2016-04-07 07:57:35 -0400 | [diff] [blame] | 109 | rem->r_offset += ALIGN(bytes, 8); |
| 110 | if (rem->r_offset >= PAGE_SIZE) { |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 111 | __free_page(rem->r_page); |
| 112 | rem->r_page = NULL; |
| 113 | } |
| 114 | ret = 0; |
| 115 | break; |
| 116 | } |
| 117 | |
| 118 | /* alloc if there is nothing for us to use */ |
| 119 | local_irq_restore(flags); |
| 120 | put_cpu(); |
| 121 | |
| 122 | page = alloc_page(gfp); |
| 123 | |
| 124 | rem = &per_cpu(rds_page_remainders, get_cpu()); |
| 125 | local_irq_save(flags); |
| 126 | |
Andy Grover | 8690bfa | 2010-01-12 11:56:44 -0800 | [diff] [blame] | 127 | if (!page) { |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 128 | ret = -ENOMEM; |
| 129 | break; |
| 130 | } |
| 131 | |
| 132 | /* did someone race to fill the remainder before us? */ |
| 133 | if (rem->r_page) { |
| 134 | __free_page(page); |
| 135 | continue; |
| 136 | } |
| 137 | |
| 138 | /* otherwise install our page and loop around to alloc */ |
| 139 | rem->r_page = page; |
| 140 | rem->r_offset = 0; |
| 141 | } |
| 142 | |
| 143 | local_irq_restore(flags); |
| 144 | put_cpu(); |
| 145 | out: |
| 146 | rdsdebug("bytes %lu ret %d %p %u %u\n", bytes, ret, |
| 147 | ret ? NULL : sg_page(scat), ret ? 0 : scat->offset, |
| 148 | ret ? 0 : scat->length); |
| 149 | return ret; |
| 150 | } |
Andy Grover | 0b088e0 | 2010-05-24 20:12:41 -0700 | [diff] [blame] | 151 | EXPORT_SYMBOL_GPL(rds_page_remainder_alloc); |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 152 | |
Sebastian Andrzej Siewior | f2830d0 | 2015-12-19 12:55:43 -0800 | [diff] [blame] | 153 | void rds_page_exit(void) |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 154 | { |
Sebastian Andrzej Siewior | f2830d0 | 2015-12-19 12:55:43 -0800 | [diff] [blame] | 155 | unsigned int cpu; |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 156 | |
Sebastian Andrzej Siewior | f2830d0 | 2015-12-19 12:55:43 -0800 | [diff] [blame] | 157 | for_each_possible_cpu(cpu) { |
| 158 | struct rds_page_remainder *rem; |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 159 | |
Sebastian Andrzej Siewior | f2830d0 | 2015-12-19 12:55:43 -0800 | [diff] [blame] | 160 | rem = &per_cpu(rds_page_remainders, cpu); |
| 161 | rdsdebug("cpu %u\n", cpu); |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 162 | |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 163 | if (rem->r_page) |
| 164 | __free_page(rem->r_page); |
| 165 | rem->r_page = NULL; |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 166 | } |
Andy Grover | 7875e18 | 2009-02-24 15:30:26 +0000 | [diff] [blame] | 167 | } |