Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 1 | /* |
| 2 | * This file is subject to the terms and conditions of the GNU General Public |
| 3 | * License. See the file "COPYING" in the main directory of this archive |
| 4 | * for more details. |
| 5 | * |
| 6 | * Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org) |
| 7 | * Copyright (C) 2007 Maciej W. Rozycki |
| 8 | * Copyright (C) 2008 Thiemo Seufer |
Steven J. Hill | c022630 | 2012-07-06 21:56:01 +0200 | [diff] [blame] | 9 | * Copyright (C) 2012 MIPS Technologies, Inc. |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 10 | */ |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 11 | #include <linux/kernel.h> |
| 12 | #include <linux/sched.h> |
Ralf Baechle | 631330f | 2009-06-19 14:05:26 +0100 | [diff] [blame] | 13 | #include <linux/smp.h> |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 14 | #include <linux/mm.h> |
| 15 | #include <linux/module.h> |
| 16 | #include <linux/proc_fs.h> |
| 17 | |
| 18 | #include <asm/bugs.h> |
| 19 | #include <asm/cacheops.h> |
Ralf Baechle | 69f24d1 | 2013-09-17 10:25:47 +0200 | [diff] [blame] | 20 | #include <asm/cpu-type.h> |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 21 | #include <asm/inst.h> |
| 22 | #include <asm/io.h> |
| 23 | #include <asm/page.h> |
| 24 | #include <asm/pgtable.h> |
| 25 | #include <asm/prefetch.h> |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 26 | #include <asm/bootinfo.h> |
| 27 | #include <asm/mipsregs.h> |
| 28 | #include <asm/mmu_context.h> |
| 29 | #include <asm/cpu.h> |
| 30 | #include <asm/war.h> |
| 31 | |
| 32 | #ifdef CONFIG_SIBYTE_DMA_PAGEOPS |
| 33 | #include <asm/sibyte/sb1250.h> |
| 34 | #include <asm/sibyte/sb1250_regs.h> |
| 35 | #include <asm/sibyte/sb1250_dma.h> |
| 36 | #endif |
| 37 | |
Florian Fainelli | 3482d71 | 2010-01-28 15:21:24 +0100 | [diff] [blame] | 38 | #include <asm/uasm.h> |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 39 | |
| 40 | /* Registers used in the assembled routines. */ |
| 41 | #define ZERO 0 |
| 42 | #define AT 2 |
| 43 | #define A0 4 |
| 44 | #define A1 5 |
| 45 | #define A2 6 |
| 46 | #define T0 8 |
| 47 | #define T1 9 |
| 48 | #define T2 10 |
| 49 | #define T3 11 |
| 50 | #define T9 25 |
| 51 | #define RA 31 |
| 52 | |
| 53 | /* Handle labels (which must be positive integers). */ |
| 54 | enum label_id { |
| 55 | label_clear_nopref = 1, |
| 56 | label_clear_pref, |
| 57 | label_copy_nopref, |
| 58 | label_copy_pref_both, |
| 59 | label_copy_pref_store, |
| 60 | }; |
| 61 | |
| 62 | UASM_L_LA(_clear_nopref) |
| 63 | UASM_L_LA(_clear_pref) |
| 64 | UASM_L_LA(_copy_nopref) |
| 65 | UASM_L_LA(_copy_pref_both) |
| 66 | UASM_L_LA(_copy_pref_store) |
| 67 | |
| 68 | /* We need one branch and therefore one relocation per target label. */ |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 69 | static struct uasm_label labels[5]; |
| 70 | static struct uasm_reloc relocs[5]; |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 71 | |
| 72 | #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) |
| 73 | #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) |
| 74 | |
Markos Chandras | d2e6d30 | 2014-11-19 09:39:56 +0000 | [diff] [blame] | 75 | /* |
| 76 | * R6 has a limited offset of the pref instruction. |
| 77 | * Skip it if the offset is more than 9 bits. |
| 78 | */ |
| 79 | #define _uasm_i_pref(a, b, c, d) \ |
| 80 | do { \ |
| 81 | if (cpu_has_mips_r6) { \ |
| 82 | if (c <= 0xff && c >= -0x100) \ |
| 83 | uasm_i_pref(a, b, c, d);\ |
| 84 | } else { \ |
| 85 | uasm_i_pref(a, b, c, d); \ |
| 86 | } \ |
| 87 | } while(0) |
| 88 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 89 | static int pref_bias_clear_store; |
| 90 | static int pref_bias_copy_load; |
| 91 | static int pref_bias_copy_store; |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 92 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 93 | static u32 pref_src_mode; |
| 94 | static u32 pref_dst_mode; |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 95 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 96 | static int clear_word_size; |
| 97 | static int copy_word_size; |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 98 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 99 | static int half_clear_loop_size; |
| 100 | static int half_copy_loop_size; |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 101 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 102 | static int cache_line_size; |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 103 | #define cache_line_mask() (cache_line_size - 1) |
| 104 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 105 | static inline void |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 106 | pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off) |
| 107 | { |
| 108 | if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) { |
| 109 | if (off > 0x7fff) { |
| 110 | uasm_i_lui(buf, T9, uasm_rel_hi(off)); |
| 111 | uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off)); |
| 112 | } else |
| 113 | uasm_i_addiu(buf, T9, ZERO, off); |
| 114 | uasm_i_daddu(buf, reg1, reg2, T9); |
| 115 | } else { |
| 116 | if (off > 0x7fff) { |
| 117 | uasm_i_lui(buf, T9, uasm_rel_hi(off)); |
| 118 | uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off)); |
| 119 | UASM_i_ADDU(buf, reg1, reg2, T9); |
| 120 | } else |
| 121 | UASM_i_ADDIU(buf, reg1, reg2, off); |
| 122 | } |
| 123 | } |
| 124 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 125 | static void set_prefetch_parameters(void) |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 126 | { |
| 127 | if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) |
| 128 | clear_word_size = 8; |
| 129 | else |
| 130 | clear_word_size = 4; |
| 131 | |
| 132 | if (cpu_has_64bit_gp_regs) |
| 133 | copy_word_size = 8; |
| 134 | else |
| 135 | copy_word_size = 4; |
| 136 | |
| 137 | /* |
| 138 | * The pref's used here are using "streaming" hints, which cause the |
| 139 | * copied data to be kicked out of the cache sooner. A page copy often |
| 140 | * ends up copying a lot more data than is commonly used, so this seems |
| 141 | * to make sense in terms of reducing cache pollution, but I've no real |
| 142 | * performance data to back this up. |
| 143 | */ |
| 144 | if (cpu_has_prefetch) { |
| 145 | /* |
| 146 | * XXX: Most prefetch bias values in here are based on |
| 147 | * guesswork. |
| 148 | */ |
| 149 | cache_line_size = cpu_dcache_line_size(); |
| 150 | switch (current_cpu_type()) { |
Shinya Kuribayashi | a644b27 | 2009-03-03 18:05:51 +0900 | [diff] [blame] | 151 | case CPU_R5500: |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 152 | case CPU_TX49XX: |
Shinya Kuribayashi | a644b27 | 2009-03-03 18:05:51 +0900 | [diff] [blame] | 153 | /* These processors only support the Pref_Load. */ |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 154 | pref_bias_copy_load = 256; |
| 155 | break; |
| 156 | |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 157 | case CPU_R10000: |
| 158 | case CPU_R12000: |
| 159 | case CPU_R14000: |
Joshua Kinard | 3057739 | 2015-01-21 07:59:45 -0500 | [diff] [blame] | 160 | case CPU_R16000: |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 161 | /* |
| 162 | * Those values have been experimentally tuned for an |
| 163 | * Origin 200. |
| 164 | */ |
| 165 | pref_bias_clear_store = 512; |
| 166 | pref_bias_copy_load = 256; |
| 167 | pref_bias_copy_store = 256; |
| 168 | pref_src_mode = Pref_LoadStreamed; |
| 169 | pref_dst_mode = Pref_StoreStreamed; |
| 170 | break; |
| 171 | |
| 172 | case CPU_SB1: |
| 173 | case CPU_SB1A: |
| 174 | pref_bias_clear_store = 128; |
| 175 | pref_bias_copy_load = 128; |
| 176 | pref_bias_copy_store = 128; |
| 177 | /* |
| 178 | * SB1 pass1 Pref_LoadStreamed/Pref_StoreStreamed |
| 179 | * hints are broken. |
| 180 | */ |
| 181 | if (current_cpu_type() == CPU_SB1 && |
| 182 | (current_cpu_data.processor_id & 0xff) < 0x02) { |
| 183 | pref_src_mode = Pref_Load; |
| 184 | pref_dst_mode = Pref_Store; |
| 185 | } else { |
| 186 | pref_src_mode = Pref_LoadStreamed; |
| 187 | pref_dst_mode = Pref_StoreStreamed; |
| 188 | } |
| 189 | break; |
| 190 | |
| 191 | default: |
| 192 | pref_bias_clear_store = 128; |
| 193 | pref_bias_copy_load = 256; |
| 194 | pref_bias_copy_store = 128; |
| 195 | pref_src_mode = Pref_LoadStreamed; |
Markos Chandras | d2e6d30 | 2014-11-19 09:39:56 +0000 | [diff] [blame] | 196 | if (cpu_has_mips_r6) |
| 197 | /* |
| 198 | * Bit 30 (Pref_PrepareForStore) has been |
| 199 | * removed from MIPS R6. Use bit 5 |
| 200 | * (Pref_StoreStreamed). |
| 201 | */ |
| 202 | pref_dst_mode = Pref_StoreStreamed; |
| 203 | else |
| 204 | pref_dst_mode = Pref_PrepareForStore; |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 205 | break; |
| 206 | } |
| 207 | } else { |
| 208 | if (cpu_has_cache_cdex_s) |
| 209 | cache_line_size = cpu_scache_line_size(); |
| 210 | else if (cpu_has_cache_cdex_p) |
| 211 | cache_line_size = cpu_dcache_line_size(); |
| 212 | } |
| 213 | /* |
| 214 | * Too much unrolling will overflow the available space in |
Thomas Bogendoerfer | 14defd9 | 2008-07-08 14:46:34 +0200 | [diff] [blame] | 215 | * clear_space_array / copy_page_array. |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 216 | */ |
Thomas Bogendoerfer | 14defd9 | 2008-07-08 14:46:34 +0200 | [diff] [blame] | 217 | half_clear_loop_size = min(16 * clear_word_size, |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 218 | max(cache_line_size >> 1, |
| 219 | 4 * clear_word_size)); |
Thomas Bogendoerfer | 14defd9 | 2008-07-08 14:46:34 +0200 | [diff] [blame] | 220 | half_copy_loop_size = min(16 * copy_word_size, |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 221 | max(cache_line_size >> 1, |
| 222 | 4 * copy_word_size)); |
| 223 | } |
| 224 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 225 | static void build_clear_store(u32 **buf, int off) |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 226 | { |
| 227 | if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) { |
| 228 | uasm_i_sd(buf, ZERO, off, A0); |
| 229 | } else { |
| 230 | uasm_i_sw(buf, ZERO, off, A0); |
| 231 | } |
| 232 | } |
| 233 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 234 | static inline void build_clear_pref(u32 **buf, int off) |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 235 | { |
| 236 | if (off & cache_line_mask()) |
| 237 | return; |
| 238 | |
| 239 | if (pref_bias_clear_store) { |
Markos Chandras | d2e6d30 | 2014-11-19 09:39:56 +0000 | [diff] [blame] | 240 | _uasm_i_pref(buf, pref_dst_mode, pref_bias_clear_store + off, |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 241 | A0); |
Thomas Bogendoerfer | 14defd9 | 2008-07-08 14:46:34 +0200 | [diff] [blame] | 242 | } else if (cache_line_size == (half_clear_loop_size << 1)) { |
| 243 | if (cpu_has_cache_cdex_s) { |
| 244 | uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0); |
| 245 | } else if (cpu_has_cache_cdex_p) { |
| 246 | if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) { |
| 247 | uasm_i_nop(buf); |
| 248 | uasm_i_nop(buf); |
| 249 | uasm_i_nop(buf); |
| 250 | uasm_i_nop(buf); |
| 251 | } |
| 252 | |
| 253 | if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) |
| 254 | uasm_i_lw(buf, ZERO, ZERO, AT); |
| 255 | |
| 256 | uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0); |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 257 | } |
Tony Wu | fc192e5 | 2013-06-21 10:10:46 +0000 | [diff] [blame] | 258 | } |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 259 | } |
| 260 | |
Steven J. Hill | c022630 | 2012-07-06 21:56:01 +0200 | [diff] [blame] | 261 | extern u32 __clear_page_start; |
| 262 | extern u32 __clear_page_end; |
| 263 | extern u32 __copy_page_start; |
| 264 | extern u32 __copy_page_end; |
| 265 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 266 | void build_clear_page(void) |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 267 | { |
| 268 | int off; |
Steven J. Hill | c022630 | 2012-07-06 21:56:01 +0200 | [diff] [blame] | 269 | u32 *buf = &__clear_page_start; |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 270 | struct uasm_label *l = labels; |
| 271 | struct uasm_reloc *r = relocs; |
| 272 | int i; |
Huacai Chen | 8759934 | 2013-03-17 11:49:38 +0000 | [diff] [blame] | 273 | static atomic_t run_once = ATOMIC_INIT(0); |
| 274 | |
| 275 | if (atomic_xchg(&run_once, 1)) { |
| 276 | return; |
| 277 | } |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 278 | |
| 279 | memset(labels, 0, sizeof(labels)); |
| 280 | memset(relocs, 0, sizeof(relocs)); |
| 281 | |
| 282 | set_prefetch_parameters(); |
| 283 | |
| 284 | /* |
| 285 | * This algorithm makes the following assumptions: |
| 286 | * - The prefetch bias is a multiple of 2 words. |
| 287 | * - The prefetch bias is less than one page. |
| 288 | */ |
| 289 | BUG_ON(pref_bias_clear_store % (2 * clear_word_size)); |
| 290 | BUG_ON(PAGE_SIZE < pref_bias_clear_store); |
| 291 | |
| 292 | off = PAGE_SIZE - pref_bias_clear_store; |
| 293 | if (off > 0xffff || !pref_bias_clear_store) |
| 294 | pg_addiu(&buf, A2, A0, off); |
| 295 | else |
| 296 | uasm_i_ori(&buf, A2, A0, off); |
| 297 | |
| 298 | if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) |
Thomas Bogendoerfer | f3f0d95 | 2014-04-08 08:58:01 +0200 | [diff] [blame] | 299 | uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000)); |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 300 | |
Yoichi Yuasa | cd9da13 | 2008-05-07 23:38:15 +0900 | [diff] [blame] | 301 | off = cache_line_size ? min(8, pref_bias_clear_store / cache_line_size) |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 302 | * cache_line_size : 0; |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 303 | while (off) { |
| 304 | build_clear_pref(&buf, -off); |
| 305 | off -= cache_line_size; |
| 306 | } |
| 307 | uasm_l_clear_pref(&l, buf); |
| 308 | do { |
| 309 | build_clear_pref(&buf, off); |
| 310 | build_clear_store(&buf, off); |
| 311 | off += clear_word_size; |
| 312 | } while (off < half_clear_loop_size); |
| 313 | pg_addiu(&buf, A0, A0, 2 * off); |
| 314 | off = -off; |
| 315 | do { |
| 316 | build_clear_pref(&buf, off); |
| 317 | if (off == -clear_word_size) |
| 318 | uasm_il_bne(&buf, &r, A0, A2, label_clear_pref); |
| 319 | build_clear_store(&buf, off); |
| 320 | off += clear_word_size; |
| 321 | } while (off < 0); |
| 322 | |
| 323 | if (pref_bias_clear_store) { |
| 324 | pg_addiu(&buf, A2, A0, pref_bias_clear_store); |
| 325 | uasm_l_clear_nopref(&l, buf); |
| 326 | off = 0; |
| 327 | do { |
| 328 | build_clear_store(&buf, off); |
| 329 | off += clear_word_size; |
| 330 | } while (off < half_clear_loop_size); |
| 331 | pg_addiu(&buf, A0, A0, 2 * off); |
| 332 | off = -off; |
| 333 | do { |
| 334 | if (off == -clear_word_size) |
| 335 | uasm_il_bne(&buf, &r, A0, A2, |
| 336 | label_clear_nopref); |
| 337 | build_clear_store(&buf, off); |
| 338 | off += clear_word_size; |
| 339 | } while (off < 0); |
| 340 | } |
| 341 | |
| 342 | uasm_i_jr(&buf, RA); |
| 343 | uasm_i_nop(&buf); |
| 344 | |
Steven J. Hill | c022630 | 2012-07-06 21:56:01 +0200 | [diff] [blame] | 345 | BUG_ON(buf > &__clear_page_end); |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 346 | |
| 347 | uasm_resolve_relocs(relocs, labels); |
| 348 | |
| 349 | pr_debug("Synthesized clear page handler (%u instructions).\n", |
Steven J. Hill | c022630 | 2012-07-06 21:56:01 +0200 | [diff] [blame] | 350 | (u32)(buf - &__clear_page_start)); |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 351 | |
| 352 | pr_debug("\t.set push\n"); |
| 353 | pr_debug("\t.set noreorder\n"); |
Steven J. Hill | c022630 | 2012-07-06 21:56:01 +0200 | [diff] [blame] | 354 | for (i = 0; i < (buf - &__clear_page_start); i++) |
| 355 | pr_debug("\t.word 0x%08x\n", (&__clear_page_start)[i]); |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 356 | pr_debug("\t.set pop\n"); |
| 357 | } |
| 358 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 359 | static void build_copy_load(u32 **buf, int reg, int off) |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 360 | { |
| 361 | if (cpu_has_64bit_gp_regs) { |
| 362 | uasm_i_ld(buf, reg, off, A1); |
| 363 | } else { |
| 364 | uasm_i_lw(buf, reg, off, A1); |
| 365 | } |
| 366 | } |
| 367 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 368 | static void build_copy_store(u32 **buf, int reg, int off) |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 369 | { |
| 370 | if (cpu_has_64bit_gp_regs) { |
| 371 | uasm_i_sd(buf, reg, off, A0); |
| 372 | } else { |
| 373 | uasm_i_sw(buf, reg, off, A0); |
| 374 | } |
| 375 | } |
| 376 | |
| 377 | static inline void build_copy_load_pref(u32 **buf, int off) |
| 378 | { |
| 379 | if (off & cache_line_mask()) |
| 380 | return; |
| 381 | |
| 382 | if (pref_bias_copy_load) |
Markos Chandras | d2e6d30 | 2014-11-19 09:39:56 +0000 | [diff] [blame] | 383 | _uasm_i_pref(buf, pref_src_mode, pref_bias_copy_load + off, A1); |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 384 | } |
| 385 | |
| 386 | static inline void build_copy_store_pref(u32 **buf, int off) |
| 387 | { |
| 388 | if (off & cache_line_mask()) |
| 389 | return; |
| 390 | |
| 391 | if (pref_bias_copy_store) { |
Markos Chandras | d2e6d30 | 2014-11-19 09:39:56 +0000 | [diff] [blame] | 392 | _uasm_i_pref(buf, pref_dst_mode, pref_bias_copy_store + off, |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 393 | A0); |
Thomas Bogendoerfer | 14defd9 | 2008-07-08 14:46:34 +0200 | [diff] [blame] | 394 | } else if (cache_line_size == (half_copy_loop_size << 1)) { |
| 395 | if (cpu_has_cache_cdex_s) { |
| 396 | uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0); |
| 397 | } else if (cpu_has_cache_cdex_p) { |
| 398 | if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) { |
| 399 | uasm_i_nop(buf); |
| 400 | uasm_i_nop(buf); |
| 401 | uasm_i_nop(buf); |
| 402 | uasm_i_nop(buf); |
| 403 | } |
| 404 | |
| 405 | if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) |
| 406 | uasm_i_lw(buf, ZERO, ZERO, AT); |
| 407 | |
| 408 | uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0); |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 409 | } |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 410 | } |
| 411 | } |
| 412 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 413 | void build_copy_page(void) |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 414 | { |
| 415 | int off; |
Steven J. Hill | c022630 | 2012-07-06 21:56:01 +0200 | [diff] [blame] | 416 | u32 *buf = &__copy_page_start; |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 417 | struct uasm_label *l = labels; |
| 418 | struct uasm_reloc *r = relocs; |
| 419 | int i; |
Huacai Chen | 8759934 | 2013-03-17 11:49:38 +0000 | [diff] [blame] | 420 | static atomic_t run_once = ATOMIC_INIT(0); |
| 421 | |
| 422 | if (atomic_xchg(&run_once, 1)) { |
| 423 | return; |
| 424 | } |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 425 | |
| 426 | memset(labels, 0, sizeof(labels)); |
| 427 | memset(relocs, 0, sizeof(relocs)); |
| 428 | |
| 429 | set_prefetch_parameters(); |
| 430 | |
| 431 | /* |
| 432 | * This algorithm makes the following assumptions: |
| 433 | * - All prefetch biases are multiples of 8 words. |
| 434 | * - The prefetch biases are less than one page. |
| 435 | * - The store prefetch bias isn't greater than the load |
| 436 | * prefetch bias. |
| 437 | */ |
| 438 | BUG_ON(pref_bias_copy_load % (8 * copy_word_size)); |
| 439 | BUG_ON(pref_bias_copy_store % (8 * copy_word_size)); |
| 440 | BUG_ON(PAGE_SIZE < pref_bias_copy_load); |
| 441 | BUG_ON(pref_bias_copy_store > pref_bias_copy_load); |
| 442 | |
| 443 | off = PAGE_SIZE - pref_bias_copy_load; |
| 444 | if (off > 0xffff || !pref_bias_copy_load) |
| 445 | pg_addiu(&buf, A2, A0, off); |
| 446 | else |
| 447 | uasm_i_ori(&buf, A2, A0, off); |
| 448 | |
| 449 | if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) |
Thomas Bogendoerfer | f3f0d95 | 2014-04-08 08:58:01 +0200 | [diff] [blame] | 450 | uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000)); |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 451 | |
Yoichi Yuasa | cd9da13 | 2008-05-07 23:38:15 +0900 | [diff] [blame] | 452 | off = cache_line_size ? min(8, pref_bias_copy_load / cache_line_size) * |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 453 | cache_line_size : 0; |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 454 | while (off) { |
| 455 | build_copy_load_pref(&buf, -off); |
| 456 | off -= cache_line_size; |
| 457 | } |
Atsushi Nemoto | 7bd0fea | 2008-05-30 13:07:21 +0900 | [diff] [blame] | 458 | off = cache_line_size ? min(8, pref_bias_copy_store / cache_line_size) * |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 459 | cache_line_size : 0; |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 460 | while (off) { |
| 461 | build_copy_store_pref(&buf, -off); |
| 462 | off -= cache_line_size; |
| 463 | } |
| 464 | uasm_l_copy_pref_both(&l, buf); |
| 465 | do { |
| 466 | build_copy_load_pref(&buf, off); |
| 467 | build_copy_load(&buf, T0, off); |
| 468 | build_copy_load_pref(&buf, off + copy_word_size); |
| 469 | build_copy_load(&buf, T1, off + copy_word_size); |
| 470 | build_copy_load_pref(&buf, off + 2 * copy_word_size); |
| 471 | build_copy_load(&buf, T2, off + 2 * copy_word_size); |
| 472 | build_copy_load_pref(&buf, off + 3 * copy_word_size); |
| 473 | build_copy_load(&buf, T3, off + 3 * copy_word_size); |
| 474 | build_copy_store_pref(&buf, off); |
| 475 | build_copy_store(&buf, T0, off); |
| 476 | build_copy_store_pref(&buf, off + copy_word_size); |
| 477 | build_copy_store(&buf, T1, off + copy_word_size); |
| 478 | build_copy_store_pref(&buf, off + 2 * copy_word_size); |
| 479 | build_copy_store(&buf, T2, off + 2 * copy_word_size); |
| 480 | build_copy_store_pref(&buf, off + 3 * copy_word_size); |
| 481 | build_copy_store(&buf, T3, off + 3 * copy_word_size); |
| 482 | off += 4 * copy_word_size; |
| 483 | } while (off < half_copy_loop_size); |
| 484 | pg_addiu(&buf, A1, A1, 2 * off); |
| 485 | pg_addiu(&buf, A0, A0, 2 * off); |
| 486 | off = -off; |
| 487 | do { |
| 488 | build_copy_load_pref(&buf, off); |
| 489 | build_copy_load(&buf, T0, off); |
| 490 | build_copy_load_pref(&buf, off + copy_word_size); |
| 491 | build_copy_load(&buf, T1, off + copy_word_size); |
| 492 | build_copy_load_pref(&buf, off + 2 * copy_word_size); |
| 493 | build_copy_load(&buf, T2, off + 2 * copy_word_size); |
| 494 | build_copy_load_pref(&buf, off + 3 * copy_word_size); |
| 495 | build_copy_load(&buf, T3, off + 3 * copy_word_size); |
| 496 | build_copy_store_pref(&buf, off); |
| 497 | build_copy_store(&buf, T0, off); |
| 498 | build_copy_store_pref(&buf, off + copy_word_size); |
| 499 | build_copy_store(&buf, T1, off + copy_word_size); |
| 500 | build_copy_store_pref(&buf, off + 2 * copy_word_size); |
| 501 | build_copy_store(&buf, T2, off + 2 * copy_word_size); |
| 502 | build_copy_store_pref(&buf, off + 3 * copy_word_size); |
| 503 | if (off == -(4 * copy_word_size)) |
| 504 | uasm_il_bne(&buf, &r, A2, A0, label_copy_pref_both); |
| 505 | build_copy_store(&buf, T3, off + 3 * copy_word_size); |
| 506 | off += 4 * copy_word_size; |
| 507 | } while (off < 0); |
| 508 | |
| 509 | if (pref_bias_copy_load - pref_bias_copy_store) { |
| 510 | pg_addiu(&buf, A2, A0, |
| 511 | pref_bias_copy_load - pref_bias_copy_store); |
| 512 | uasm_l_copy_pref_store(&l, buf); |
| 513 | off = 0; |
| 514 | do { |
| 515 | build_copy_load(&buf, T0, off); |
| 516 | build_copy_load(&buf, T1, off + copy_word_size); |
| 517 | build_copy_load(&buf, T2, off + 2 * copy_word_size); |
| 518 | build_copy_load(&buf, T3, off + 3 * copy_word_size); |
| 519 | build_copy_store_pref(&buf, off); |
| 520 | build_copy_store(&buf, T0, off); |
| 521 | build_copy_store_pref(&buf, off + copy_word_size); |
| 522 | build_copy_store(&buf, T1, off + copy_word_size); |
| 523 | build_copy_store_pref(&buf, off + 2 * copy_word_size); |
| 524 | build_copy_store(&buf, T2, off + 2 * copy_word_size); |
| 525 | build_copy_store_pref(&buf, off + 3 * copy_word_size); |
| 526 | build_copy_store(&buf, T3, off + 3 * copy_word_size); |
| 527 | off += 4 * copy_word_size; |
| 528 | } while (off < half_copy_loop_size); |
| 529 | pg_addiu(&buf, A1, A1, 2 * off); |
| 530 | pg_addiu(&buf, A0, A0, 2 * off); |
| 531 | off = -off; |
| 532 | do { |
| 533 | build_copy_load(&buf, T0, off); |
| 534 | build_copy_load(&buf, T1, off + copy_word_size); |
| 535 | build_copy_load(&buf, T2, off + 2 * copy_word_size); |
| 536 | build_copy_load(&buf, T3, off + 3 * copy_word_size); |
| 537 | build_copy_store_pref(&buf, off); |
| 538 | build_copy_store(&buf, T0, off); |
| 539 | build_copy_store_pref(&buf, off + copy_word_size); |
| 540 | build_copy_store(&buf, T1, off + copy_word_size); |
| 541 | build_copy_store_pref(&buf, off + 2 * copy_word_size); |
| 542 | build_copy_store(&buf, T2, off + 2 * copy_word_size); |
| 543 | build_copy_store_pref(&buf, off + 3 * copy_word_size); |
| 544 | if (off == -(4 * copy_word_size)) |
| 545 | uasm_il_bne(&buf, &r, A2, A0, |
| 546 | label_copy_pref_store); |
| 547 | build_copy_store(&buf, T3, off + 3 * copy_word_size); |
| 548 | off += 4 * copy_word_size; |
| 549 | } while (off < 0); |
| 550 | } |
| 551 | |
| 552 | if (pref_bias_copy_store) { |
| 553 | pg_addiu(&buf, A2, A0, pref_bias_copy_store); |
| 554 | uasm_l_copy_nopref(&l, buf); |
| 555 | off = 0; |
| 556 | do { |
| 557 | build_copy_load(&buf, T0, off); |
| 558 | build_copy_load(&buf, T1, off + copy_word_size); |
| 559 | build_copy_load(&buf, T2, off + 2 * copy_word_size); |
| 560 | build_copy_load(&buf, T3, off + 3 * copy_word_size); |
| 561 | build_copy_store(&buf, T0, off); |
| 562 | build_copy_store(&buf, T1, off + copy_word_size); |
| 563 | build_copy_store(&buf, T2, off + 2 * copy_word_size); |
| 564 | build_copy_store(&buf, T3, off + 3 * copy_word_size); |
| 565 | off += 4 * copy_word_size; |
| 566 | } while (off < half_copy_loop_size); |
| 567 | pg_addiu(&buf, A1, A1, 2 * off); |
| 568 | pg_addiu(&buf, A0, A0, 2 * off); |
| 569 | off = -off; |
| 570 | do { |
| 571 | build_copy_load(&buf, T0, off); |
| 572 | build_copy_load(&buf, T1, off + copy_word_size); |
| 573 | build_copy_load(&buf, T2, off + 2 * copy_word_size); |
| 574 | build_copy_load(&buf, T3, off + 3 * copy_word_size); |
| 575 | build_copy_store(&buf, T0, off); |
| 576 | build_copy_store(&buf, T1, off + copy_word_size); |
| 577 | build_copy_store(&buf, T2, off + 2 * copy_word_size); |
| 578 | if (off == -(4 * copy_word_size)) |
| 579 | uasm_il_bne(&buf, &r, A2, A0, |
| 580 | label_copy_nopref); |
| 581 | build_copy_store(&buf, T3, off + 3 * copy_word_size); |
| 582 | off += 4 * copy_word_size; |
| 583 | } while (off < 0); |
| 584 | } |
| 585 | |
| 586 | uasm_i_jr(&buf, RA); |
| 587 | uasm_i_nop(&buf); |
| 588 | |
Steven J. Hill | c022630 | 2012-07-06 21:56:01 +0200 | [diff] [blame] | 589 | BUG_ON(buf > &__copy_page_end); |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 590 | |
| 591 | uasm_resolve_relocs(relocs, labels); |
| 592 | |
| 593 | pr_debug("Synthesized copy page handler (%u instructions).\n", |
Steven J. Hill | c022630 | 2012-07-06 21:56:01 +0200 | [diff] [blame] | 594 | (u32)(buf - &__copy_page_start)); |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 595 | |
| 596 | pr_debug("\t.set push\n"); |
| 597 | pr_debug("\t.set noreorder\n"); |
Steven J. Hill | c022630 | 2012-07-06 21:56:01 +0200 | [diff] [blame] | 598 | for (i = 0; i < (buf - &__copy_page_start); i++) |
| 599 | pr_debug("\t.word 0x%08x\n", (&__copy_page_start)[i]); |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 600 | pr_debug("\t.set pop\n"); |
| 601 | } |
| 602 | |
| 603 | #ifdef CONFIG_SIBYTE_DMA_PAGEOPS |
Steven J. Hill | c022630 | 2012-07-06 21:56:01 +0200 | [diff] [blame] | 604 | extern void clear_page_cpu(void *page); |
| 605 | extern void copy_page_cpu(void *to, void *from); |
Thiemo Seufer | fb2a27e7 | 2008-02-18 19:32:49 +0000 | [diff] [blame] | 606 | |
| 607 | /* |
| 608 | * Pad descriptors to cacheline, since each is exclusively owned by a |
| 609 | * particular CPU. |
| 610 | */ |
| 611 | struct dmadscr { |
| 612 | u64 dscr_a; |
| 613 | u64 dscr_b; |
| 614 | u64 pad_a; |
| 615 | u64 pad_b; |
| 616 | } ____cacheline_aligned_in_smp page_descr[DM_NUM_CHANNELS]; |
| 617 | |
| 618 | void sb1_dma_init(void) |
| 619 | { |
| 620 | int i; |
| 621 | |
| 622 | for (i = 0; i < DM_NUM_CHANNELS; i++) { |
| 623 | const u64 base_val = CPHYSADDR((unsigned long)&page_descr[i]) | |
| 624 | V_DM_DSCR_BASE_RINGSZ(1); |
| 625 | void *base_reg = IOADDR(A_DM_REGISTER(i, R_DM_DSCR_BASE)); |
| 626 | |
| 627 | __raw_writeq(base_val, base_reg); |
| 628 | __raw_writeq(base_val | M_DM_DSCR_BASE_RESET, base_reg); |
| 629 | __raw_writeq(base_val | M_DM_DSCR_BASE_ENABL, base_reg); |
| 630 | } |
| 631 | } |
| 632 | |
| 633 | void clear_page(void *page) |
| 634 | { |
| 635 | u64 to_phys = CPHYSADDR((unsigned long)page); |
| 636 | unsigned int cpu = smp_processor_id(); |
| 637 | |
| 638 | /* if the page is not in KSEG0, use old way */ |
| 639 | if ((long)KSEGX((unsigned long)page) != (long)CKSEG0) |
| 640 | return clear_page_cpu(page); |
| 641 | |
| 642 | page_descr[cpu].dscr_a = to_phys | M_DM_DSCRA_ZERO_MEM | |
| 643 | M_DM_DSCRA_L2C_DEST | M_DM_DSCRA_INTERRUPT; |
| 644 | page_descr[cpu].dscr_b = V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE); |
| 645 | __raw_writeq(1, IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT))); |
| 646 | |
| 647 | /* |
| 648 | * Don't really want to do it this way, but there's no |
| 649 | * reliable way to delay completion detection. |
| 650 | */ |
| 651 | while (!(__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG))) |
| 652 | & M_DM_DSCR_BASE_INTERRUPT)) |
| 653 | ; |
| 654 | __raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE))); |
| 655 | } |
| 656 | |
| 657 | void copy_page(void *to, void *from) |
| 658 | { |
| 659 | u64 from_phys = CPHYSADDR((unsigned long)from); |
| 660 | u64 to_phys = CPHYSADDR((unsigned long)to); |
| 661 | unsigned int cpu = smp_processor_id(); |
| 662 | |
| 663 | /* if any page is not in KSEG0, use old way */ |
| 664 | if ((long)KSEGX((unsigned long)to) != (long)CKSEG0 |
| 665 | || (long)KSEGX((unsigned long)from) != (long)CKSEG0) |
| 666 | return copy_page_cpu(to, from); |
| 667 | |
| 668 | page_descr[cpu].dscr_a = to_phys | M_DM_DSCRA_L2C_DEST | |
| 669 | M_DM_DSCRA_INTERRUPT; |
| 670 | page_descr[cpu].dscr_b = from_phys | V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE); |
| 671 | __raw_writeq(1, IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT))); |
| 672 | |
| 673 | /* |
| 674 | * Don't really want to do it this way, but there's no |
| 675 | * reliable way to delay completion detection. |
| 676 | */ |
| 677 | while (!(__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG))) |
| 678 | & M_DM_DSCR_BASE_INTERRUPT)) |
| 679 | ; |
| 680 | __raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE))); |
| 681 | } |
| 682 | |
| 683 | #endif /* CONFIG_SIBYTE_DMA_PAGEOPS */ |