Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 1 | /* |
| 2 | * jump label support |
| 3 | * |
| 4 | * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> |
| 5 | * |
| 6 | */ |
| 7 | #include <linux/jump_label.h> |
| 8 | #include <linux/memory.h> |
| 9 | #include <linux/uaccess.h> |
| 10 | #include <linux/module.h> |
| 11 | #include <linux/list.h> |
| 12 | #include <linux/jhash.h> |
| 13 | #include <linux/slab.h> |
| 14 | #include <linux/sort.h> |
| 15 | #include <linux/err.h> |
| 16 | |
| 17 | #ifdef HAVE_JUMP_LABEL |
| 18 | |
| 19 | #define JUMP_LABEL_HASH_BITS 6 |
| 20 | #define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS) |
| 21 | static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE]; |
| 22 | |
| 23 | /* mutex to protect coming/going of the the jump_label table */ |
| 24 | static DEFINE_MUTEX(jump_label_mutex); |
| 25 | |
| 26 | struct jump_label_entry { |
| 27 | struct hlist_node hlist; |
| 28 | struct jump_entry *table; |
| 29 | int nr_entries; |
| 30 | /* hang modules off here */ |
| 31 | struct hlist_head modules; |
| 32 | unsigned long key; |
| 33 | }; |
| 34 | |
| 35 | struct jump_label_module_entry { |
| 36 | struct hlist_node hlist; |
| 37 | struct jump_entry *table; |
| 38 | int nr_entries; |
| 39 | struct module *mod; |
| 40 | }; |
| 41 | |
Jason Baron | 91bad2f | 2010-10-01 17:23:48 -0400 | [diff] [blame] | 42 | void jump_label_lock(void) |
| 43 | { |
| 44 | mutex_lock(&jump_label_mutex); |
| 45 | } |
| 46 | |
| 47 | void jump_label_unlock(void) |
| 48 | { |
| 49 | mutex_unlock(&jump_label_mutex); |
| 50 | } |
| 51 | |
Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 52 | static int jump_label_cmp(const void *a, const void *b) |
| 53 | { |
| 54 | const struct jump_entry *jea = a; |
| 55 | const struct jump_entry *jeb = b; |
| 56 | |
| 57 | if (jea->key < jeb->key) |
| 58 | return -1; |
| 59 | |
| 60 | if (jea->key > jeb->key) |
| 61 | return 1; |
| 62 | |
| 63 | return 0; |
| 64 | } |
| 65 | |
| 66 | static void |
| 67 | sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop) |
| 68 | { |
| 69 | unsigned long size; |
| 70 | |
| 71 | size = (((unsigned long)stop - (unsigned long)start) |
| 72 | / sizeof(struct jump_entry)); |
| 73 | sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); |
| 74 | } |
| 75 | |
| 76 | static struct jump_label_entry *get_jump_label_entry(jump_label_t key) |
| 77 | { |
| 78 | struct hlist_head *head; |
| 79 | struct hlist_node *node; |
| 80 | struct jump_label_entry *e; |
| 81 | u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0); |
| 82 | |
| 83 | head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)]; |
| 84 | hlist_for_each_entry(e, node, head, hlist) { |
| 85 | if (key == e->key) |
| 86 | return e; |
| 87 | } |
| 88 | return NULL; |
| 89 | } |
| 90 | |
| 91 | static struct jump_label_entry * |
| 92 | add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table) |
| 93 | { |
| 94 | struct hlist_head *head; |
| 95 | struct jump_label_entry *e; |
| 96 | u32 hash; |
| 97 | |
| 98 | e = get_jump_label_entry(key); |
| 99 | if (e) |
| 100 | return ERR_PTR(-EEXIST); |
| 101 | |
| 102 | e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL); |
| 103 | if (!e) |
| 104 | return ERR_PTR(-ENOMEM); |
| 105 | |
| 106 | hash = jhash((void *)&key, sizeof(jump_label_t), 0); |
| 107 | head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)]; |
| 108 | e->key = key; |
| 109 | e->table = table; |
| 110 | e->nr_entries = nr_entries; |
| 111 | INIT_HLIST_HEAD(&(e->modules)); |
| 112 | hlist_add_head(&e->hlist, head); |
| 113 | return e; |
| 114 | } |
| 115 | |
| 116 | static int |
| 117 | build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop) |
| 118 | { |
| 119 | struct jump_entry *iter, *iter_begin; |
| 120 | struct jump_label_entry *entry; |
| 121 | int count; |
| 122 | |
| 123 | sort_jump_label_entries(start, stop); |
| 124 | iter = start; |
| 125 | while (iter < stop) { |
| 126 | entry = get_jump_label_entry(iter->key); |
| 127 | if (!entry) { |
| 128 | iter_begin = iter; |
| 129 | count = 0; |
| 130 | while ((iter < stop) && |
| 131 | (iter->key == iter_begin->key)) { |
| 132 | iter++; |
| 133 | count++; |
| 134 | } |
| 135 | entry = add_jump_label_entry(iter_begin->key, |
| 136 | count, iter_begin); |
| 137 | if (IS_ERR(entry)) |
| 138 | return PTR_ERR(entry); |
| 139 | } else { |
| 140 | WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n"); |
| 141 | return -1; |
| 142 | } |
| 143 | } |
| 144 | return 0; |
| 145 | } |
| 146 | |
| 147 | /*** |
| 148 | * jump_label_update - update jump label text |
| 149 | * @key - key value associated with a a jump label |
| 150 | * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE |
| 151 | * |
| 152 | * Will enable/disable the jump for jump label @key, depending on the |
| 153 | * value of @type. |
| 154 | * |
| 155 | */ |
| 156 | |
| 157 | void jump_label_update(unsigned long key, enum jump_label_type type) |
| 158 | { |
| 159 | struct jump_entry *iter; |
| 160 | struct jump_label_entry *entry; |
| 161 | struct hlist_node *module_node; |
| 162 | struct jump_label_module_entry *e_module; |
| 163 | int count; |
| 164 | |
Jason Baron | 91bad2f | 2010-10-01 17:23:48 -0400 | [diff] [blame] | 165 | jump_label_lock(); |
Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 166 | entry = get_jump_label_entry((jump_label_t)key); |
| 167 | if (entry) { |
| 168 | count = entry->nr_entries; |
| 169 | iter = entry->table; |
| 170 | while (count--) { |
| 171 | if (kernel_text_address(iter->code)) |
| 172 | arch_jump_label_transform(iter, type); |
| 173 | iter++; |
| 174 | } |
| 175 | /* eanble/disable jump labels in modules */ |
| 176 | hlist_for_each_entry(e_module, module_node, &(entry->modules), |
| 177 | hlist) { |
| 178 | count = e_module->nr_entries; |
| 179 | iter = e_module->table; |
| 180 | while (count--) { |
Jason Baron | b842f8f | 2010-10-01 17:23:41 -0400 | [diff] [blame] | 181 | if (iter->key && |
| 182 | kernel_text_address(iter->code)) |
Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 183 | arch_jump_label_transform(iter, type); |
| 184 | iter++; |
| 185 | } |
| 186 | } |
| 187 | } |
Jason Baron | 91bad2f | 2010-10-01 17:23:48 -0400 | [diff] [blame] | 188 | jump_label_unlock(); |
Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 189 | } |
| 190 | |
Jason Baron | 4c3ef6d | 2010-09-17 11:09:08 -0400 | [diff] [blame] | 191 | static int addr_conflict(struct jump_entry *entry, void *start, void *end) |
| 192 | { |
| 193 | if (entry->code <= (unsigned long)end && |
| 194 | entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start) |
| 195 | return 1; |
| 196 | |
| 197 | return 0; |
| 198 | } |
| 199 | |
| 200 | #ifdef CONFIG_MODULES |
| 201 | |
| 202 | static int module_conflict(void *start, void *end) |
| 203 | { |
| 204 | struct hlist_head *head; |
| 205 | struct hlist_node *node, *node_next, *module_node, *module_node_next; |
| 206 | struct jump_label_entry *e; |
| 207 | struct jump_label_module_entry *e_module; |
| 208 | struct jump_entry *iter; |
| 209 | int i, count; |
| 210 | int conflict = 0; |
| 211 | |
| 212 | for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { |
| 213 | head = &jump_label_table[i]; |
| 214 | hlist_for_each_entry_safe(e, node, node_next, head, hlist) { |
| 215 | hlist_for_each_entry_safe(e_module, module_node, |
| 216 | module_node_next, |
| 217 | &(e->modules), hlist) { |
| 218 | count = e_module->nr_entries; |
| 219 | iter = e_module->table; |
| 220 | while (count--) { |
| 221 | if (addr_conflict(iter, start, end)) { |
| 222 | conflict = 1; |
| 223 | goto out; |
| 224 | } |
| 225 | iter++; |
| 226 | } |
| 227 | } |
| 228 | } |
| 229 | } |
| 230 | out: |
| 231 | return conflict; |
| 232 | } |
| 233 | |
| 234 | #endif |
| 235 | |
| 236 | /*** |
| 237 | * jump_label_text_reserved - check if addr range is reserved |
| 238 | * @start: start text addr |
| 239 | * @end: end text addr |
| 240 | * |
| 241 | * checks if the text addr located between @start and @end |
| 242 | * overlaps with any of the jump label patch addresses. Code |
| 243 | * that wants to modify kernel text should first verify that |
| 244 | * it does not overlap with any of the jump label addresses. |
Jason Baron | 91bad2f | 2010-10-01 17:23:48 -0400 | [diff] [blame] | 245 | * Caller must hold jump_label_mutex. |
Jason Baron | 4c3ef6d | 2010-09-17 11:09:08 -0400 | [diff] [blame] | 246 | * |
| 247 | * returns 1 if there is an overlap, 0 otherwise |
| 248 | */ |
| 249 | int jump_label_text_reserved(void *start, void *end) |
| 250 | { |
| 251 | struct jump_entry *iter; |
| 252 | struct jump_entry *iter_start = __start___jump_table; |
| 253 | struct jump_entry *iter_stop = __start___jump_table; |
| 254 | int conflict = 0; |
| 255 | |
Jason Baron | 4c3ef6d | 2010-09-17 11:09:08 -0400 | [diff] [blame] | 256 | iter = iter_start; |
| 257 | while (iter < iter_stop) { |
| 258 | if (addr_conflict(iter, start, end)) { |
| 259 | conflict = 1; |
| 260 | goto out; |
| 261 | } |
| 262 | iter++; |
| 263 | } |
| 264 | |
| 265 | /* now check modules */ |
| 266 | #ifdef CONFIG_MODULES |
| 267 | conflict = module_conflict(start, end); |
| 268 | #endif |
| 269 | out: |
Jason Baron | 4c3ef6d | 2010-09-17 11:09:08 -0400 | [diff] [blame] | 270 | return conflict; |
| 271 | } |
| 272 | |
Steven Rostedt | 95bcd68 | 2010-10-29 11:02:43 -0400 | [diff] [blame] | 273 | /* |
| 274 | * Not all archs need this. |
| 275 | */ |
| 276 | void __weak arch_jump_label_text_poke_early(jump_label_t addr) |
| 277 | { |
| 278 | } |
| 279 | |
Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 280 | static __init int init_jump_label(void) |
| 281 | { |
| 282 | int ret; |
| 283 | struct jump_entry *iter_start = __start___jump_table; |
| 284 | struct jump_entry *iter_stop = __stop___jump_table; |
| 285 | struct jump_entry *iter; |
| 286 | |
Jason Baron | 91bad2f | 2010-10-01 17:23:48 -0400 | [diff] [blame] | 287 | jump_label_lock(); |
Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 288 | ret = build_jump_label_hashtable(__start___jump_table, |
| 289 | __stop___jump_table); |
| 290 | iter = iter_start; |
| 291 | while (iter < iter_stop) { |
| 292 | arch_jump_label_text_poke_early(iter->code); |
| 293 | iter++; |
| 294 | } |
Jason Baron | 91bad2f | 2010-10-01 17:23:48 -0400 | [diff] [blame] | 295 | jump_label_unlock(); |
Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 296 | return ret; |
| 297 | } |
| 298 | early_initcall(init_jump_label); |
| 299 | |
| 300 | #ifdef CONFIG_MODULES |
| 301 | |
| 302 | static struct jump_label_module_entry * |
| 303 | add_jump_label_module_entry(struct jump_label_entry *entry, |
| 304 | struct jump_entry *iter_begin, |
| 305 | int count, struct module *mod) |
| 306 | { |
| 307 | struct jump_label_module_entry *e; |
| 308 | |
| 309 | e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL); |
| 310 | if (!e) |
| 311 | return ERR_PTR(-ENOMEM); |
| 312 | e->mod = mod; |
| 313 | e->nr_entries = count; |
| 314 | e->table = iter_begin; |
| 315 | hlist_add_head(&e->hlist, &entry->modules); |
| 316 | return e; |
| 317 | } |
| 318 | |
| 319 | static int add_jump_label_module(struct module *mod) |
| 320 | { |
| 321 | struct jump_entry *iter, *iter_begin; |
| 322 | struct jump_label_entry *entry; |
| 323 | struct jump_label_module_entry *module_entry; |
| 324 | int count; |
| 325 | |
| 326 | /* if the module doesn't have jump label entries, just return */ |
| 327 | if (!mod->num_jump_entries) |
| 328 | return 0; |
| 329 | |
| 330 | sort_jump_label_entries(mod->jump_entries, |
| 331 | mod->jump_entries + mod->num_jump_entries); |
| 332 | iter = mod->jump_entries; |
| 333 | while (iter < mod->jump_entries + mod->num_jump_entries) { |
| 334 | entry = get_jump_label_entry(iter->key); |
| 335 | iter_begin = iter; |
| 336 | count = 0; |
| 337 | while ((iter < mod->jump_entries + mod->num_jump_entries) && |
| 338 | (iter->key == iter_begin->key)) { |
| 339 | iter++; |
| 340 | count++; |
| 341 | } |
| 342 | if (!entry) { |
| 343 | entry = add_jump_label_entry(iter_begin->key, 0, NULL); |
| 344 | if (IS_ERR(entry)) |
| 345 | return PTR_ERR(entry); |
| 346 | } |
| 347 | module_entry = add_jump_label_module_entry(entry, iter_begin, |
| 348 | count, mod); |
| 349 | if (IS_ERR(module_entry)) |
| 350 | return PTR_ERR(module_entry); |
| 351 | } |
| 352 | return 0; |
| 353 | } |
| 354 | |
| 355 | static void remove_jump_label_module(struct module *mod) |
| 356 | { |
| 357 | struct hlist_head *head; |
| 358 | struct hlist_node *node, *node_next, *module_node, *module_node_next; |
| 359 | struct jump_label_entry *e; |
| 360 | struct jump_label_module_entry *e_module; |
| 361 | int i; |
| 362 | |
| 363 | /* if the module doesn't have jump label entries, just return */ |
| 364 | if (!mod->num_jump_entries) |
| 365 | return; |
| 366 | |
| 367 | for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { |
| 368 | head = &jump_label_table[i]; |
| 369 | hlist_for_each_entry_safe(e, node, node_next, head, hlist) { |
| 370 | hlist_for_each_entry_safe(e_module, module_node, |
| 371 | module_node_next, |
| 372 | &(e->modules), hlist) { |
| 373 | if (e_module->mod == mod) { |
| 374 | hlist_del(&e_module->hlist); |
| 375 | kfree(e_module); |
| 376 | } |
| 377 | } |
| 378 | if (hlist_empty(&e->modules) && (e->nr_entries == 0)) { |
| 379 | hlist_del(&e->hlist); |
| 380 | kfree(e); |
| 381 | } |
| 382 | } |
| 383 | } |
| 384 | } |
| 385 | |
Jason Baron | b842f8f | 2010-10-01 17:23:41 -0400 | [diff] [blame] | 386 | static void remove_jump_label_module_init(struct module *mod) |
| 387 | { |
| 388 | struct hlist_head *head; |
| 389 | struct hlist_node *node, *node_next, *module_node, *module_node_next; |
| 390 | struct jump_label_entry *e; |
| 391 | struct jump_label_module_entry *e_module; |
| 392 | struct jump_entry *iter; |
| 393 | int i, count; |
| 394 | |
| 395 | /* if the module doesn't have jump label entries, just return */ |
| 396 | if (!mod->num_jump_entries) |
| 397 | return; |
| 398 | |
| 399 | for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { |
| 400 | head = &jump_label_table[i]; |
| 401 | hlist_for_each_entry_safe(e, node, node_next, head, hlist) { |
| 402 | hlist_for_each_entry_safe(e_module, module_node, |
| 403 | module_node_next, |
| 404 | &(e->modules), hlist) { |
| 405 | if (e_module->mod != mod) |
| 406 | continue; |
| 407 | count = e_module->nr_entries; |
| 408 | iter = e_module->table; |
| 409 | while (count--) { |
| 410 | if (within_module_init(iter->code, mod)) |
| 411 | iter->key = 0; |
| 412 | iter++; |
| 413 | } |
| 414 | } |
| 415 | } |
| 416 | } |
| 417 | } |
| 418 | |
Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 419 | static int |
| 420 | jump_label_module_notify(struct notifier_block *self, unsigned long val, |
| 421 | void *data) |
| 422 | { |
| 423 | struct module *mod = data; |
| 424 | int ret = 0; |
| 425 | |
| 426 | switch (val) { |
| 427 | case MODULE_STATE_COMING: |
Jason Baron | 91bad2f | 2010-10-01 17:23:48 -0400 | [diff] [blame] | 428 | jump_label_lock(); |
Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 429 | ret = add_jump_label_module(mod); |
| 430 | if (ret) |
| 431 | remove_jump_label_module(mod); |
Jason Baron | 91bad2f | 2010-10-01 17:23:48 -0400 | [diff] [blame] | 432 | jump_label_unlock(); |
Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 433 | break; |
| 434 | case MODULE_STATE_GOING: |
Jason Baron | 91bad2f | 2010-10-01 17:23:48 -0400 | [diff] [blame] | 435 | jump_label_lock(); |
Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 436 | remove_jump_label_module(mod); |
Jason Baron | 91bad2f | 2010-10-01 17:23:48 -0400 | [diff] [blame] | 437 | jump_label_unlock(); |
Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 438 | break; |
Jason Baron | b842f8f | 2010-10-01 17:23:41 -0400 | [diff] [blame] | 439 | case MODULE_STATE_LIVE: |
Jason Baron | 91bad2f | 2010-10-01 17:23:48 -0400 | [diff] [blame] | 440 | jump_label_lock(); |
Jason Baron | b842f8f | 2010-10-01 17:23:41 -0400 | [diff] [blame] | 441 | remove_jump_label_module_init(mod); |
Jason Baron | 91bad2f | 2010-10-01 17:23:48 -0400 | [diff] [blame] | 442 | jump_label_unlock(); |
Jason Baron | b842f8f | 2010-10-01 17:23:41 -0400 | [diff] [blame] | 443 | break; |
Jason Baron | bf5438fc | 2010-09-17 11:09:00 -0400 | [diff] [blame] | 444 | } |
| 445 | return ret; |
| 446 | } |
| 447 | |
| 448 | /*** |
| 449 | * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() |
| 450 | * @mod: module to patch |
| 451 | * |
| 452 | * Allow for run-time selection of the optimal nops. Before the module |
| 453 | * loads patch these with arch_get_jump_label_nop(), which is specified by |
| 454 | * the arch specific jump label code. |
| 455 | */ |
| 456 | void jump_label_apply_nops(struct module *mod) |
| 457 | { |
| 458 | struct jump_entry *iter; |
| 459 | |
| 460 | /* if the module doesn't have jump label entries, just return */ |
| 461 | if (!mod->num_jump_entries) |
| 462 | return; |
| 463 | |
| 464 | iter = mod->jump_entries; |
| 465 | while (iter < mod->jump_entries + mod->num_jump_entries) { |
| 466 | arch_jump_label_text_poke_early(iter->code); |
| 467 | iter++; |
| 468 | } |
| 469 | } |
| 470 | |
| 471 | struct notifier_block jump_label_module_nb = { |
| 472 | .notifier_call = jump_label_module_notify, |
| 473 | .priority = 0, |
| 474 | }; |
| 475 | |
| 476 | static __init int init_jump_label_module(void) |
| 477 | { |
| 478 | return register_module_notifier(&jump_label_module_nb); |
| 479 | } |
| 480 | early_initcall(init_jump_label_module); |
| 481 | |
| 482 | #endif /* CONFIG_MODULES */ |
| 483 | |
| 484 | #endif |