Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 1 | #ifndef _LINUX_SLUB_DEF_H |
| 2 | #define _LINUX_SLUB_DEF_H |
| 3 | |
| 4 | /* |
| 5 | * SLUB : A Slab allocator without object queues. |
| 6 | * |
Christoph Lameter | cde5353 | 2008-07-04 09:59:22 -0700 | [diff] [blame] | 7 | * (C) 2007 SGI, Christoph Lameter |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 8 | */ |
| 9 | #include <linux/types.h> |
| 10 | #include <linux/gfp.h> |
Paul Gortmaker | 187f188 | 2011-11-23 20:12:59 -0500 | [diff] [blame] | 11 | #include <linux/bug.h> |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 12 | #include <linux/workqueue.h> |
| 13 | #include <linux/kobject.h> |
| 14 | |
Richard Kennedy | 4a92379 | 2010-10-21 10:29:19 +0100 | [diff] [blame] | 15 | #include <linux/kmemleak.h> |
Li Zefan | 039ca4e | 2010-05-26 17:22:17 +0800 | [diff] [blame] | 16 | |
Christoph Lameter | 8ff12cf | 2008-02-07 17:47:41 -0800 | [diff] [blame] | 17 | enum stat_item { |
| 18 | ALLOC_FASTPATH, /* Allocation from cpu slab */ |
| 19 | ALLOC_SLOWPATH, /* Allocation by getting a new cpu slab */ |
| 20 | FREE_FASTPATH, /* Free to cpu slub */ |
| 21 | FREE_SLOWPATH, /* Freeing not to cpu slab */ |
| 22 | FREE_FROZEN, /* Freeing to frozen slab */ |
| 23 | FREE_ADD_PARTIAL, /* Freeing moves slab to partial list */ |
| 24 | FREE_REMOVE_PARTIAL, /* Freeing removes last object */ |
Alex Shi | 8028dce | 2012-02-03 23:34:56 +0800 | [diff] [blame] | 25 | ALLOC_FROM_PARTIAL, /* Cpu slab acquired from node partial list */ |
Christoph Lameter | 8ff12cf | 2008-02-07 17:47:41 -0800 | [diff] [blame] | 26 | ALLOC_SLAB, /* Cpu slab acquired from page allocator */ |
| 27 | ALLOC_REFILL, /* Refill cpu slab from slab freelist */ |
Christoph Lameter | e36a265 | 2011-06-01 12:25:57 -0500 | [diff] [blame] | 28 | ALLOC_NODE_MISMATCH, /* Switching cpu slab */ |
Christoph Lameter | 8ff12cf | 2008-02-07 17:47:41 -0800 | [diff] [blame] | 29 | FREE_SLAB, /* Slab freed to the page allocator */ |
| 30 | CPUSLAB_FLUSH, /* Abandoning of the cpu slab */ |
| 31 | DEACTIVATE_FULL, /* Cpu slab was full when deactivated */ |
| 32 | DEACTIVATE_EMPTY, /* Cpu slab was empty when deactivated */ |
| 33 | DEACTIVATE_TO_HEAD, /* Cpu slab was moved to the head of partials */ |
| 34 | DEACTIVATE_TO_TAIL, /* Cpu slab was moved to the tail of partials */ |
| 35 | DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */ |
Christoph Lameter | 03e404a | 2011-06-01 12:25:58 -0500 | [diff] [blame] | 36 | DEACTIVATE_BYPASS, /* Implicit deactivation */ |
Christoph Lameter | 65c3376 | 2008-04-14 19:11:40 +0300 | [diff] [blame] | 37 | ORDER_FALLBACK, /* Number of times fallback was necessary */ |
Christoph Lameter | 4fdccdf | 2011-03-22 13:35:00 -0500 | [diff] [blame] | 38 | CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */ |
Christoph Lameter | b789ef5 | 2011-06-01 12:25:49 -0500 | [diff] [blame] | 39 | CMPXCHG_DOUBLE_FAIL, /* Number of times that cmpxchg double did not match */ |
Christoph Lameter | 49e2258 | 2011-08-09 16:12:27 -0500 | [diff] [blame] | 40 | CPU_PARTIAL_ALLOC, /* Used cpu partial on alloc */ |
Alex Shi | 8028dce | 2012-02-03 23:34:56 +0800 | [diff] [blame] | 41 | CPU_PARTIAL_FREE, /* Refill cpu partial on free */ |
| 42 | CPU_PARTIAL_NODE, /* Refill cpu partial from node partial */ |
| 43 | CPU_PARTIAL_DRAIN, /* Drain cpu partial to node partial */ |
Christoph Lameter | 8ff12cf | 2008-02-07 17:47:41 -0800 | [diff] [blame] | 44 | NR_SLUB_STAT_ITEMS }; |
| 45 | |
Christoph Lameter | dfb4f09 | 2007-10-16 01:26:05 -0700 | [diff] [blame] | 46 | struct kmem_cache_cpu { |
Christoph Lameter | 8a5ec0b | 2011-02-25 11:38:54 -0600 | [diff] [blame] | 47 | void **freelist; /* Pointer to next available object */ |
Christoph Lameter | 8a5ec0b | 2011-02-25 11:38:54 -0600 | [diff] [blame] | 48 | unsigned long tid; /* Globally unique transaction id */ |
Christoph Lameter | da89b79 | 2008-01-07 23:20:31 -0800 | [diff] [blame] | 49 | struct page *page; /* The slab from which we are allocating */ |
Christoph Lameter | 49e2258 | 2011-08-09 16:12:27 -0500 | [diff] [blame] | 50 | struct page *partial; /* Partially allocated frozen slabs */ |
Christoph Lameter | 8ff12cf | 2008-02-07 17:47:41 -0800 | [diff] [blame] | 51 | #ifdef CONFIG_SLUB_STATS |
| 52 | unsigned stat[NR_SLUB_STAT_ITEMS]; |
| 53 | #endif |
Christoph Lameter | 4c93c355 | 2007-10-16 01:26:08 -0700 | [diff] [blame] | 54 | }; |
Christoph Lameter | dfb4f09 | 2007-10-16 01:26:05 -0700 | [diff] [blame] | 55 | |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 56 | /* |
Christoph Lameter | 834f3d1 | 2008-04-14 19:11:31 +0300 | [diff] [blame] | 57 | * Word size structure that can be atomically updated or read and that |
| 58 | * contains both the order and the number of objects that a slab of the |
| 59 | * given order would contain. |
| 60 | */ |
| 61 | struct kmem_cache_order_objects { |
| 62 | unsigned long x; |
| 63 | }; |
| 64 | |
| 65 | /* |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 66 | * Slab cache management. |
| 67 | */ |
| 68 | struct kmem_cache { |
Namhyung Kim | 1b5ad24 | 2010-08-07 14:29:22 +0200 | [diff] [blame] | 69 | struct kmem_cache_cpu __percpu *cpu_slab; |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 70 | /* Used for retriving partial slabs etc */ |
| 71 | unsigned long flags; |
Christoph Lameter | 1a757fe | 2011-02-25 11:38:51 -0600 | [diff] [blame] | 72 | unsigned long min_partial; |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 73 | int size; /* The size of an object including meta data */ |
Christoph Lameter | 3b0efdf | 2012-06-13 10:24:57 -0500 | [diff] [blame] | 74 | int object_size; /* The size of an object without meta data */ |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 75 | int offset; /* Free pointer offset. */ |
Alex Shi | 9f26490 | 2011-09-01 11:32:18 +0800 | [diff] [blame] | 76 | int cpu_partial; /* Number of per cpu partial objects to keep around */ |
Christoph Lameter | 834f3d1 | 2008-04-14 19:11:31 +0300 | [diff] [blame] | 77 | struct kmem_cache_order_objects oo; |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 78 | |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 79 | /* Allocation and freeing of slabs */ |
Christoph Lameter | 205ab99 | 2008-04-14 19:11:40 +0300 | [diff] [blame] | 80 | struct kmem_cache_order_objects max; |
Christoph Lameter | 65c3376 | 2008-04-14 19:11:40 +0300 | [diff] [blame] | 81 | struct kmem_cache_order_objects min; |
Christoph Lameter | b7a49f0 | 2008-02-14 14:21:32 -0800 | [diff] [blame] | 82 | gfp_t allocflags; /* gfp flags to use on each alloc */ |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 83 | int refcount; /* Refcount for slab cache destroy */ |
Alexey Dobriyan | 51cc506 | 2008-07-25 19:45:34 -0700 | [diff] [blame] | 84 | void (*ctor)(void *); |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 85 | int inuse; /* Offset to metadata */ |
| 86 | int align; /* Alignment */ |
Lai Jiangshan | ab9a0f1 | 2011-03-10 15:21:48 +0800 | [diff] [blame] | 87 | int reserved; /* Reserved bytes at the end of slabs */ |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 88 | const char *name; /* Name (only for display!) */ |
| 89 | struct list_head list; /* List of slab caches */ |
Christoph Lameter | ab4d5ed | 2010-10-05 13:57:26 -0500 | [diff] [blame] | 90 | #ifdef CONFIG_SYSFS |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 91 | struct kobject kobj; /* For sysfs */ |
Christoph Lameter | 0c71001 | 2007-07-17 04:03:24 -0700 | [diff] [blame] | 92 | #endif |
Glauber Costa | ba6c496 | 2012-12-18 14:22:27 -0800 | [diff] [blame] | 93 | #ifdef CONFIG_MEMCG_KMEM |
| 94 | struct memcg_cache_params *memcg_params; |
Glauber Costa | 107dab5 | 2012-12-18 14:23:05 -0800 | [diff] [blame] | 95 | int max_attr_size; /* for propagation, maximum size of a stored attr */ |
Glauber Costa | ba6c496 | 2012-12-18 14:22:27 -0800 | [diff] [blame] | 96 | #endif |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 97 | |
| 98 | #ifdef CONFIG_NUMA |
Christoph Lameter | 9824601 | 2008-01-07 23:20:26 -0800 | [diff] [blame] | 99 | /* |
| 100 | * Defragmentation by allocating from a remote node. |
| 101 | */ |
| 102 | int remote_node_defrag_ratio; |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 103 | #endif |
Christoph Lameter | 7340cc8 | 2010-09-28 08:10:26 -0500 | [diff] [blame] | 104 | struct kmem_cache_node *node[MAX_NUMNODES]; |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 105 | }; |
| 106 | |
Paul Mundt | 6193a2f | 2007-07-15 23:38:22 -0700 | [diff] [blame] | 107 | void *kmem_cache_alloc(struct kmem_cache *, gfp_t); |
| 108 | void *__kmalloc(size_t size, gfp_t flags); |
| 109 | |
Richard Kennedy | 4a92379 | 2010-10-21 10:29:19 +0100 | [diff] [blame] | 110 | static __always_inline void * |
| 111 | kmalloc_order(size_t size, gfp_t flags, unsigned int order) |
| 112 | { |
Glauber Costa | d79923f | 2012-12-18 14:22:48 -0800 | [diff] [blame] | 113 | void *ret; |
| 114 | |
| 115 | flags |= (__GFP_COMP | __GFP_KMEMCG); |
| 116 | ret = (void *) __get_free_pages(flags, order); |
Richard Kennedy | 4a92379 | 2010-10-21 10:29:19 +0100 | [diff] [blame] | 117 | kmemleak_alloc(ret, size, 1, flags); |
| 118 | return ret; |
| 119 | } |
| 120 | |
Ben Greear | d18a90d | 2011-07-07 11:36:37 -0700 | [diff] [blame] | 121 | /** |
| 122 | * Calling this on allocated memory will check that the memory |
| 123 | * is expected to be in use, and print warnings if not. |
| 124 | */ |
| 125 | #ifdef CONFIG_SLUB_DEBUG |
| 126 | extern bool verify_mem_not_deleted(const void *x); |
| 127 | #else |
| 128 | static inline bool verify_mem_not_deleted(const void *x) |
| 129 | { |
| 130 | return true; |
| 131 | } |
| 132 | #endif |
| 133 | |
Li Zefan | 0f24f12 | 2009-12-11 15:45:30 +0800 | [diff] [blame] | 134 | #ifdef CONFIG_TRACING |
Richard Kennedy | 4a92379 | 2010-10-21 10:29:19 +0100 | [diff] [blame] | 135 | extern void * |
| 136 | kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size); |
| 137 | extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order); |
Eduard - Gabriel Munteanu | 5b882be | 2008-08-19 20:43:26 +0300 | [diff] [blame] | 138 | #else |
| 139 | static __always_inline void * |
Richard Kennedy | 4a92379 | 2010-10-21 10:29:19 +0100 | [diff] [blame] | 140 | kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) |
Eduard - Gabriel Munteanu | 5b882be | 2008-08-19 20:43:26 +0300 | [diff] [blame] | 141 | { |
| 142 | return kmem_cache_alloc(s, gfpflags); |
| 143 | } |
Richard Kennedy | 4a92379 | 2010-10-21 10:29:19 +0100 | [diff] [blame] | 144 | |
| 145 | static __always_inline void * |
| 146 | kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) |
| 147 | { |
| 148 | return kmalloc_order(size, flags, order); |
| 149 | } |
Eduard - Gabriel Munteanu | 5b882be | 2008-08-19 20:43:26 +0300 | [diff] [blame] | 150 | #endif |
| 151 | |
Pekka Enberg | eada35e | 2008-02-11 22:47:46 +0200 | [diff] [blame] | 152 | static __always_inline void *kmalloc_large(size_t size, gfp_t flags) |
| 153 | { |
Eduard - Gabriel Munteanu | 5b882be | 2008-08-19 20:43:26 +0300 | [diff] [blame] | 154 | unsigned int order = get_order(size); |
Richard Kennedy | 4a92379 | 2010-10-21 10:29:19 +0100 | [diff] [blame] | 155 | return kmalloc_order_trace(size, flags, order); |
Pekka Enberg | eada35e | 2008-02-11 22:47:46 +0200 | [diff] [blame] | 156 | } |
| 157 | |
Christoph Lameter | aa137f9 | 2007-08-31 00:48:45 -0700 | [diff] [blame] | 158 | static __always_inline void *kmalloc(size_t size, gfp_t flags) |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 159 | { |
Christoph Lameter | aadb4bc | 2007-10-16 01:24:38 -0700 | [diff] [blame] | 160 | if (__builtin_constant_p(size)) { |
Christoph Lameter | 95a05b4 | 2013-01-10 19:14:19 +0000 | [diff] [blame] | 161 | if (size > KMALLOC_MAX_CACHE_SIZE) |
Pekka Enberg | eada35e | 2008-02-11 22:47:46 +0200 | [diff] [blame] | 162 | return kmalloc_large(size, flags); |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 163 | |
Christoph Lameter | 2c59dd6 | 2013-01-10 19:14:19 +0000 | [diff] [blame] | 164 | if (!(flags & GFP_DMA)) { |
| 165 | int index = kmalloc_index(size); |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 166 | |
Christoph Lameter | 2c59dd6 | 2013-01-10 19:14:19 +0000 | [diff] [blame] | 167 | if (!index) |
Christoph Lameter | aadb4bc | 2007-10-16 01:24:38 -0700 | [diff] [blame] | 168 | return ZERO_SIZE_PTR; |
| 169 | |
Christoph Lameter | 2c59dd6 | 2013-01-10 19:14:19 +0000 | [diff] [blame] | 170 | return kmem_cache_alloc_trace(kmalloc_caches[index], |
| 171 | flags, size); |
Christoph Lameter | aadb4bc | 2007-10-16 01:24:38 -0700 | [diff] [blame] | 172 | } |
| 173 | } |
| 174 | return __kmalloc(size, flags); |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 175 | } |
| 176 | |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 177 | #ifdef CONFIG_NUMA |
Paul Mundt | 6193a2f | 2007-07-15 23:38:22 -0700 | [diff] [blame] | 178 | void *__kmalloc_node(size_t size, gfp_t flags, int node); |
| 179 | void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 180 | |
Li Zefan | 0f24f12 | 2009-12-11 15:45:30 +0800 | [diff] [blame] | 181 | #ifdef CONFIG_TRACING |
Richard Kennedy | 4a92379 | 2010-10-21 10:29:19 +0100 | [diff] [blame] | 182 | extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, |
Eduard - Gabriel Munteanu | 5b882be | 2008-08-19 20:43:26 +0300 | [diff] [blame] | 183 | gfp_t gfpflags, |
Richard Kennedy | 4a92379 | 2010-10-21 10:29:19 +0100 | [diff] [blame] | 184 | int node, size_t size); |
Eduard - Gabriel Munteanu | 5b882be | 2008-08-19 20:43:26 +0300 | [diff] [blame] | 185 | #else |
| 186 | static __always_inline void * |
Richard Kennedy | 4a92379 | 2010-10-21 10:29:19 +0100 | [diff] [blame] | 187 | kmem_cache_alloc_node_trace(struct kmem_cache *s, |
Eduard - Gabriel Munteanu | 5b882be | 2008-08-19 20:43:26 +0300 | [diff] [blame] | 188 | gfp_t gfpflags, |
Richard Kennedy | 4a92379 | 2010-10-21 10:29:19 +0100 | [diff] [blame] | 189 | int node, size_t size) |
Eduard - Gabriel Munteanu | 5b882be | 2008-08-19 20:43:26 +0300 | [diff] [blame] | 190 | { |
| 191 | return kmem_cache_alloc_node(s, gfpflags, node); |
| 192 | } |
| 193 | #endif |
| 194 | |
Christoph Lameter | aa137f9 | 2007-08-31 00:48:45 -0700 | [diff] [blame] | 195 | static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 196 | { |
Christoph Lameter | aadb4bc | 2007-10-16 01:24:38 -0700 | [diff] [blame] | 197 | if (__builtin_constant_p(size) && |
Christoph Lameter | 2c59dd6 | 2013-01-10 19:14:19 +0000 | [diff] [blame] | 198 | size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) { |
| 199 | int index = kmalloc_index(size); |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 200 | |
Christoph Lameter | 2c59dd6 | 2013-01-10 19:14:19 +0000 | [diff] [blame] | 201 | if (!index) |
Christoph Lameter | 272c1d2 | 2007-06-08 13:46:49 -0700 | [diff] [blame] | 202 | return ZERO_SIZE_PTR; |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 203 | |
Christoph Lameter | 2c59dd6 | 2013-01-10 19:14:19 +0000 | [diff] [blame] | 204 | return kmem_cache_alloc_node_trace(kmalloc_caches[index], |
| 205 | flags, node, size); |
Christoph Lameter | aadb4bc | 2007-10-16 01:24:38 -0700 | [diff] [blame] | 206 | } |
| 207 | return __kmalloc_node(size, flags, node); |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 208 | } |
| 209 | #endif |
| 210 | |
Christoph Lameter | 81819f0 | 2007-05-06 14:49:36 -0700 | [diff] [blame] | 211 | #endif /* _LINUX_SLUB_DEF_H */ |