Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 1 | /* |
| 2 | * Memory copy functions for 32-bit PowerPC. |
| 3 | * |
| 4 | * Copyright (C) 1996-2005 Paul Mackerras. |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License |
| 8 | * as published by the Free Software Foundation; either version |
| 9 | * 2 of the License, or (at your option) any later version. |
| 10 | */ |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 11 | #include <asm/processor.h> |
| 12 | #include <asm/cache.h> |
| 13 | #include <asm/errno.h> |
| 14 | #include <asm/ppc_asm.h> |
Al Viro | 9445aa1 | 2016-01-13 23:33:46 -0500 | [diff] [blame] | 15 | #include <asm/export.h> |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 16 | |
| 17 | #define COPY_16_BYTES \ |
| 18 | lwz r7,4(r4); \ |
| 19 | lwz r8,8(r4); \ |
| 20 | lwz r9,12(r4); \ |
| 21 | lwzu r10,16(r4); \ |
| 22 | stw r7,4(r6); \ |
| 23 | stw r8,8(r6); \ |
| 24 | stw r9,12(r6); \ |
| 25 | stwu r10,16(r6) |
| 26 | |
| 27 | #define COPY_16_BYTES_WITHEX(n) \ |
| 28 | 8 ## n ## 0: \ |
| 29 | lwz r7,4(r4); \ |
| 30 | 8 ## n ## 1: \ |
| 31 | lwz r8,8(r4); \ |
| 32 | 8 ## n ## 2: \ |
| 33 | lwz r9,12(r4); \ |
| 34 | 8 ## n ## 3: \ |
| 35 | lwzu r10,16(r4); \ |
| 36 | 8 ## n ## 4: \ |
| 37 | stw r7,4(r6); \ |
| 38 | 8 ## n ## 5: \ |
| 39 | stw r8,8(r6); \ |
| 40 | 8 ## n ## 6: \ |
| 41 | stw r9,12(r6); \ |
| 42 | 8 ## n ## 7: \ |
| 43 | stwu r10,16(r6) |
| 44 | |
| 45 | #define COPY_16_BYTES_EXCODE(n) \ |
| 46 | 9 ## n ## 0: \ |
| 47 | addi r5,r5,-(16 * n); \ |
| 48 | b 104f; \ |
| 49 | 9 ## n ## 1: \ |
| 50 | addi r5,r5,-(16 * n); \ |
| 51 | b 105f; \ |
Nicholas Piggin | 24bfa6a | 2016-10-13 16:42:53 +1100 | [diff] [blame] | 52 | EX_TABLE(8 ## n ## 0b,9 ## n ## 0b); \ |
| 53 | EX_TABLE(8 ## n ## 1b,9 ## n ## 0b); \ |
| 54 | EX_TABLE(8 ## n ## 2b,9 ## n ## 0b); \ |
| 55 | EX_TABLE(8 ## n ## 3b,9 ## n ## 0b); \ |
| 56 | EX_TABLE(8 ## n ## 4b,9 ## n ## 1b); \ |
| 57 | EX_TABLE(8 ## n ## 5b,9 ## n ## 1b); \ |
| 58 | EX_TABLE(8 ## n ## 6b,9 ## n ## 1b); \ |
| 59 | EX_TABLE(8 ## n ## 7b,9 ## n ## 1b) |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 60 | |
| 61 | .text |
| 62 | .stabs "arch/powerpc/lib/",N_SO,0,0,0f |
Sean MacLennan | 025c018 | 2010-09-01 07:21:21 +0000 | [diff] [blame] | 63 | .stabs "copy_32.S",N_SO,0,0,0f |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 64 | 0: |
| 65 | |
Stephen Rothwell | 7dffb72 | 2005-10-17 11:50:32 +1000 | [diff] [blame] | 66 | CACHELINE_BYTES = L1_CACHE_BYTES |
| 67 | LG_CACHELINE_BYTES = L1_CACHE_SHIFT |
| 68 | CACHELINE_MASK = (L1_CACHE_BYTES-1) |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 69 | |
LEROY Christophe | df087e4 | 2015-05-19 12:07:48 +0200 | [diff] [blame] | 70 | /* |
| 71 | * Use dcbz on the complete cache lines in the destination |
| 72 | * to set them to zero. This requires that the destination |
| 73 | * area is cacheable. -- paulus |
LEROY Christophe | 400c47d | 2015-09-16 12:04:53 +0200 | [diff] [blame] | 74 | * |
| 75 | * During early init, cache might not be active yet, so dcbz cannot be used. |
| 76 | * We therefore skip the optimised bloc that uses dcbz. This jump is |
| 77 | * replaced by a nop once cache is active. This is done in machine_init() |
LEROY Christophe | df087e4 | 2015-05-19 12:07:48 +0200 | [diff] [blame] | 78 | */ |
LEROY Christophe | 5b2a32e | 2015-05-19 12:07:50 +0200 | [diff] [blame] | 79 | _GLOBAL(memset) |
LEROY Christophe | c152f14 | 2015-05-19 12:07:52 +0200 | [diff] [blame] | 80 | rlwimi r4,r4,8,16,23 |
| 81 | rlwimi r4,r4,16,0,15 |
| 82 | |
LEROY Christophe | df087e4 | 2015-05-19 12:07:48 +0200 | [diff] [blame] | 83 | addi r6,r3,-4 |
| 84 | cmplwi 0,r5,4 |
| 85 | blt 7f |
| 86 | stwu r4,4(r6) |
| 87 | beqlr |
| 88 | andi. r0,r6,3 |
| 89 | add r5,r0,r5 |
| 90 | subf r6,r0,r6 |
LEROY Christophe | c152f14 | 2015-05-19 12:07:52 +0200 | [diff] [blame] | 91 | cmplwi 0,r4,0 |
| 92 | bne 2f /* Use normal procedure if r4 is not zero */ |
Al Viro | 9445aa1 | 2016-01-13 23:33:46 -0500 | [diff] [blame] | 93 | EXPORT_SYMBOL(memset) |
LEROY Christophe | 400c47d | 2015-09-16 12:04:53 +0200 | [diff] [blame] | 94 | _GLOBAL(memset_nocache_branch) |
| 95 | b 2f /* Skip optimised bloc until cache is enabled */ |
LEROY Christophe | c152f14 | 2015-05-19 12:07:52 +0200 | [diff] [blame] | 96 | |
LEROY Christophe | df087e4 | 2015-05-19 12:07:48 +0200 | [diff] [blame] | 97 | clrlwi r7,r6,32-LG_CACHELINE_BYTES |
| 98 | add r8,r7,r5 |
| 99 | srwi r9,r8,LG_CACHELINE_BYTES |
| 100 | addic. r9,r9,-1 /* total number of complete cachelines */ |
| 101 | ble 2f |
| 102 | xori r0,r7,CACHELINE_MASK & ~3 |
| 103 | srwi. r0,r0,2 |
| 104 | beq 3f |
| 105 | mtctr r0 |
| 106 | 4: stwu r4,4(r6) |
| 107 | bdnz 4b |
| 108 | 3: mtctr r9 |
| 109 | li r7,4 |
| 110 | 10: dcbz r7,r6 |
| 111 | addi r6,r6,CACHELINE_BYTES |
| 112 | bdnz 10b |
| 113 | clrlwi r5,r8,32-LG_CACHELINE_BYTES |
| 114 | addi r5,r5,4 |
LEROY Christophe | df087e4 | 2015-05-19 12:07:48 +0200 | [diff] [blame] | 115 | |
LEROY Christophe | c152f14 | 2015-05-19 12:07:52 +0200 | [diff] [blame] | 116 | 2: srwi r0,r5,2 |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 117 | mtctr r0 |
| 118 | bdz 6f |
| 119 | 1: stwu r4,4(r6) |
| 120 | bdnz 1b |
| 121 | 6: andi. r5,r5,3 |
| 122 | 7: cmpwi 0,r5,0 |
| 123 | beqlr |
| 124 | mtctr r5 |
| 125 | addi r6,r6,3 |
| 126 | 8: stbu r4,1(r6) |
| 127 | bdnz 8b |
| 128 | blr |
| 129 | |
LEROY Christophe | df087e4 | 2015-05-19 12:07:48 +0200 | [diff] [blame] | 130 | /* |
| 131 | * This version uses dcbz on the complete cache lines in the |
| 132 | * destination area to reduce memory traffic. This requires that |
| 133 | * the destination area is cacheable. |
| 134 | * We only use this version if the source and dest don't overlap. |
| 135 | * -- paulus. |
LEROY Christophe | 1cd0389 | 2015-09-16 12:04:51 +0200 | [diff] [blame] | 136 | * |
| 137 | * During early init, cache might not be active yet, so dcbz cannot be used. |
| 138 | * We therefore jump to generic_memcpy which doesn't use dcbz. This jump is |
| 139 | * replaced by a nop once cache is active. This is done in machine_init() |
LEROY Christophe | df087e4 | 2015-05-19 12:07:48 +0200 | [diff] [blame] | 140 | */ |
LEROY Christophe | 0b05e2d | 2015-05-19 12:07:55 +0200 | [diff] [blame] | 141 | _GLOBAL(memmove) |
| 142 | cmplw 0,r3,r4 |
| 143 | bgt backwards_memcpy |
| 144 | /* fall through */ |
| 145 | |
| 146 | _GLOBAL(memcpy) |
LEROY Christophe | 1cd0389 | 2015-09-16 12:04:51 +0200 | [diff] [blame] | 147 | b generic_memcpy |
LEROY Christophe | df087e4 | 2015-05-19 12:07:48 +0200 | [diff] [blame] | 148 | add r7,r3,r5 /* test if the src & dst overlap */ |
| 149 | add r8,r4,r5 |
| 150 | cmplw 0,r4,r7 |
| 151 | cmplw 1,r3,r8 |
| 152 | crand 0,0,4 /* cr0.lt &= cr1.lt */ |
LEROY Christophe | 0b05e2d | 2015-05-19 12:07:55 +0200 | [diff] [blame] | 153 | blt generic_memcpy /* if regions overlap */ |
LEROY Christophe | df087e4 | 2015-05-19 12:07:48 +0200 | [diff] [blame] | 154 | |
| 155 | addi r4,r4,-4 |
| 156 | addi r6,r3,-4 |
| 157 | neg r0,r3 |
| 158 | andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */ |
| 159 | beq 58f |
| 160 | |
| 161 | cmplw 0,r5,r0 /* is this more than total to do? */ |
| 162 | blt 63f /* if not much to do */ |
| 163 | andi. r8,r0,3 /* get it word-aligned first */ |
| 164 | subf r5,r0,r5 |
| 165 | mtctr r8 |
| 166 | beq+ 61f |
| 167 | 70: lbz r9,4(r4) /* do some bytes */ |
LEROY Christophe | df087e4 | 2015-05-19 12:07:48 +0200 | [diff] [blame] | 168 | addi r4,r4,1 |
| 169 | addi r6,r6,1 |
LEROY Christophe | 295ffb4 | 2015-05-19 12:07:57 +0200 | [diff] [blame] | 170 | stb r9,3(r6) |
LEROY Christophe | df087e4 | 2015-05-19 12:07:48 +0200 | [diff] [blame] | 171 | bdnz 70b |
| 172 | 61: srwi. r0,r0,2 |
| 173 | mtctr r0 |
| 174 | beq 58f |
| 175 | 72: lwzu r9,4(r4) /* do some words */ |
| 176 | stwu r9,4(r6) |
| 177 | bdnz 72b |
| 178 | |
| 179 | 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */ |
| 180 | clrlwi r5,r5,32-LG_CACHELINE_BYTES |
| 181 | li r11,4 |
| 182 | mtctr r0 |
| 183 | beq 63f |
| 184 | 53: |
| 185 | dcbz r11,r6 |
| 186 | COPY_16_BYTES |
| 187 | #if L1_CACHE_BYTES >= 32 |
| 188 | COPY_16_BYTES |
| 189 | #if L1_CACHE_BYTES >= 64 |
| 190 | COPY_16_BYTES |
| 191 | COPY_16_BYTES |
| 192 | #if L1_CACHE_BYTES >= 128 |
| 193 | COPY_16_BYTES |
| 194 | COPY_16_BYTES |
| 195 | COPY_16_BYTES |
| 196 | COPY_16_BYTES |
| 197 | #endif |
| 198 | #endif |
| 199 | #endif |
| 200 | bdnz 53b |
| 201 | |
| 202 | 63: srwi. r0,r5,2 |
| 203 | mtctr r0 |
| 204 | beq 64f |
| 205 | 30: lwzu r0,4(r4) |
| 206 | stwu r0,4(r6) |
| 207 | bdnz 30b |
| 208 | |
| 209 | 64: andi. r0,r5,3 |
| 210 | mtctr r0 |
| 211 | beq+ 65f |
LEROY Christophe | 295ffb4 | 2015-05-19 12:07:57 +0200 | [diff] [blame] | 212 | addi r4,r4,3 |
| 213 | addi r6,r6,3 |
| 214 | 40: lbzu r0,1(r4) |
| 215 | stbu r0,1(r6) |
LEROY Christophe | df087e4 | 2015-05-19 12:07:48 +0200 | [diff] [blame] | 216 | bdnz 40b |
| 217 | 65: blr |
Al Viro | 9445aa1 | 2016-01-13 23:33:46 -0500 | [diff] [blame] | 218 | EXPORT_SYMBOL(memcpy) |
| 219 | EXPORT_SYMBOL(memmove) |
LEROY Christophe | df087e4 | 2015-05-19 12:07:48 +0200 | [diff] [blame] | 220 | |
Michael Ellerman | b4c6afd | 2016-03-16 21:36:06 +1100 | [diff] [blame] | 221 | generic_memcpy: |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 222 | srwi. r7,r5,3 |
| 223 | addi r6,r3,-4 |
| 224 | addi r4,r4,-4 |
| 225 | beq 2f /* if less than 8 bytes to do */ |
| 226 | andi. r0,r6,3 /* get dest word aligned */ |
| 227 | mtctr r7 |
| 228 | bne 5f |
| 229 | 1: lwz r7,4(r4) |
| 230 | lwzu r8,8(r4) |
| 231 | stw r7,4(r6) |
| 232 | stwu r8,8(r6) |
| 233 | bdnz 1b |
| 234 | andi. r5,r5,7 |
| 235 | 2: cmplwi 0,r5,4 |
| 236 | blt 3f |
| 237 | lwzu r0,4(r4) |
| 238 | addi r5,r5,-4 |
| 239 | stwu r0,4(r6) |
| 240 | 3: cmpwi 0,r5,0 |
| 241 | beqlr |
| 242 | mtctr r5 |
| 243 | addi r4,r4,3 |
| 244 | addi r6,r6,3 |
| 245 | 4: lbzu r0,1(r4) |
| 246 | stbu r0,1(r6) |
| 247 | bdnz 4b |
| 248 | blr |
| 249 | 5: subfic r0,r0,4 |
| 250 | mtctr r0 |
| 251 | 6: lbz r7,4(r4) |
| 252 | addi r4,r4,1 |
| 253 | stb r7,4(r6) |
| 254 | addi r6,r6,1 |
| 255 | bdnz 6b |
| 256 | subf r5,r0,r5 |
| 257 | rlwinm. r7,r5,32-3,3,31 |
| 258 | beq 2b |
| 259 | mtctr r7 |
| 260 | b 1b |
| 261 | |
| 262 | _GLOBAL(backwards_memcpy) |
| 263 | rlwinm. r7,r5,32-3,3,31 /* r0 = r5 >> 3 */ |
| 264 | add r6,r3,r5 |
| 265 | add r4,r4,r5 |
| 266 | beq 2f |
| 267 | andi. r0,r6,3 |
| 268 | mtctr r7 |
| 269 | bne 5f |
| 270 | 1: lwz r7,-4(r4) |
| 271 | lwzu r8,-8(r4) |
| 272 | stw r7,-4(r6) |
| 273 | stwu r8,-8(r6) |
| 274 | bdnz 1b |
| 275 | andi. r5,r5,7 |
| 276 | 2: cmplwi 0,r5,4 |
| 277 | blt 3f |
| 278 | lwzu r0,-4(r4) |
| 279 | subi r5,r5,4 |
| 280 | stwu r0,-4(r6) |
| 281 | 3: cmpwi 0,r5,0 |
| 282 | beqlr |
| 283 | mtctr r5 |
| 284 | 4: lbzu r0,-1(r4) |
| 285 | stbu r0,-1(r6) |
| 286 | bdnz 4b |
| 287 | blr |
| 288 | 5: mtctr r0 |
| 289 | 6: lbzu r7,-1(r4) |
| 290 | stbu r7,-1(r6) |
| 291 | bdnz 6b |
| 292 | subf r5,r0,r5 |
| 293 | rlwinm. r7,r5,32-3,3,31 |
| 294 | beq 2b |
| 295 | mtctr r7 |
| 296 | b 1b |
| 297 | |
| 298 | _GLOBAL(__copy_tofrom_user) |
| 299 | addi r4,r4,-4 |
| 300 | addi r6,r3,-4 |
| 301 | neg r0,r3 |
| 302 | andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */ |
| 303 | beq 58f |
| 304 | |
| 305 | cmplw 0,r5,r0 /* is this more than total to do? */ |
| 306 | blt 63f /* if not much to do */ |
| 307 | andi. r8,r0,3 /* get it word-aligned first */ |
| 308 | mtctr r8 |
| 309 | beq+ 61f |
| 310 | 70: lbz r9,4(r4) /* do some bytes */ |
| 311 | 71: stb r9,4(r6) |
| 312 | addi r4,r4,1 |
| 313 | addi r6,r6,1 |
| 314 | bdnz 70b |
| 315 | 61: subf r5,r0,r5 |
| 316 | srwi. r0,r0,2 |
| 317 | mtctr r0 |
| 318 | beq 58f |
| 319 | 72: lwzu r9,4(r4) /* do some words */ |
| 320 | 73: stwu r9,4(r6) |
| 321 | bdnz 72b |
| 322 | |
Nicholas Piggin | 24bfa6a | 2016-10-13 16:42:53 +1100 | [diff] [blame] | 323 | EX_TABLE(70b,100f) |
| 324 | EX_TABLE(71b,101f) |
| 325 | EX_TABLE(72b,102f) |
| 326 | EX_TABLE(73b,103f) |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 327 | |
| 328 | 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */ |
| 329 | clrlwi r5,r5,32-LG_CACHELINE_BYTES |
| 330 | li r11,4 |
| 331 | beq 63f |
| 332 | |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 333 | /* Here we decide how far ahead to prefetch the source */ |
| 334 | li r3,4 |
| 335 | cmpwi r0,1 |
| 336 | li r7,0 |
| 337 | ble 114f |
| 338 | li r7,1 |
| 339 | #if MAX_COPY_PREFETCH > 1 |
| 340 | /* Heuristically, for large transfers we prefetch |
| 341 | MAX_COPY_PREFETCH cachelines ahead. For small transfers |
| 342 | we prefetch 1 cacheline ahead. */ |
| 343 | cmpwi r0,MAX_COPY_PREFETCH |
| 344 | ble 112f |
| 345 | li r7,MAX_COPY_PREFETCH |
| 346 | 112: mtctr r7 |
| 347 | 111: dcbt r3,r4 |
| 348 | addi r3,r3,CACHELINE_BYTES |
| 349 | bdnz 111b |
| 350 | #else |
| 351 | dcbt r3,r4 |
| 352 | addi r3,r3,CACHELINE_BYTES |
| 353 | #endif /* MAX_COPY_PREFETCH > 1 */ |
| 354 | |
| 355 | 114: subf r8,r7,r0 |
| 356 | mr r0,r7 |
| 357 | mtctr r8 |
| 358 | |
| 359 | 53: dcbt r3,r4 |
| 360 | 54: dcbz r11,r6 |
Nicholas Piggin | 24bfa6a | 2016-10-13 16:42:53 +1100 | [diff] [blame] | 361 | EX_TABLE(54b,105f) |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 362 | /* the main body of the cacheline loop */ |
| 363 | COPY_16_BYTES_WITHEX(0) |
Stephen Rothwell | 7dffb72 | 2005-10-17 11:50:32 +1000 | [diff] [blame] | 364 | #if L1_CACHE_BYTES >= 32 |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 365 | COPY_16_BYTES_WITHEX(1) |
Stephen Rothwell | 7dffb72 | 2005-10-17 11:50:32 +1000 | [diff] [blame] | 366 | #if L1_CACHE_BYTES >= 64 |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 367 | COPY_16_BYTES_WITHEX(2) |
| 368 | COPY_16_BYTES_WITHEX(3) |
Stephen Rothwell | 7dffb72 | 2005-10-17 11:50:32 +1000 | [diff] [blame] | 369 | #if L1_CACHE_BYTES >= 128 |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 370 | COPY_16_BYTES_WITHEX(4) |
| 371 | COPY_16_BYTES_WITHEX(5) |
| 372 | COPY_16_BYTES_WITHEX(6) |
| 373 | COPY_16_BYTES_WITHEX(7) |
| 374 | #endif |
| 375 | #endif |
| 376 | #endif |
| 377 | bdnz 53b |
| 378 | cmpwi r0,0 |
| 379 | li r3,4 |
| 380 | li r7,0 |
| 381 | bne 114b |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 382 | |
| 383 | 63: srwi. r0,r5,2 |
| 384 | mtctr r0 |
| 385 | beq 64f |
| 386 | 30: lwzu r0,4(r4) |
| 387 | 31: stwu r0,4(r6) |
| 388 | bdnz 30b |
| 389 | |
| 390 | 64: andi. r0,r5,3 |
| 391 | mtctr r0 |
| 392 | beq+ 65f |
| 393 | 40: lbz r0,4(r4) |
| 394 | 41: stb r0,4(r6) |
| 395 | addi r4,r4,1 |
| 396 | addi r6,r6,1 |
| 397 | bdnz 40b |
| 398 | 65: li r3,0 |
| 399 | blr |
| 400 | |
| 401 | /* read fault, initial single-byte copy */ |
| 402 | 100: li r9,0 |
| 403 | b 90f |
| 404 | /* write fault, initial single-byte copy */ |
| 405 | 101: li r9,1 |
| 406 | 90: subf r5,r8,r5 |
| 407 | li r3,0 |
| 408 | b 99f |
| 409 | /* read fault, initial word copy */ |
| 410 | 102: li r9,0 |
| 411 | b 91f |
| 412 | /* write fault, initial word copy */ |
| 413 | 103: li r9,1 |
| 414 | 91: li r3,2 |
| 415 | b 99f |
| 416 | |
| 417 | /* |
| 418 | * this stuff handles faults in the cacheline loop and branches to either |
| 419 | * 104f (if in read part) or 105f (if in write part), after updating r5 |
| 420 | */ |
| 421 | COPY_16_BYTES_EXCODE(0) |
Stephen Rothwell | 7dffb72 | 2005-10-17 11:50:32 +1000 | [diff] [blame] | 422 | #if L1_CACHE_BYTES >= 32 |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 423 | COPY_16_BYTES_EXCODE(1) |
Stephen Rothwell | 7dffb72 | 2005-10-17 11:50:32 +1000 | [diff] [blame] | 424 | #if L1_CACHE_BYTES >= 64 |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 425 | COPY_16_BYTES_EXCODE(2) |
| 426 | COPY_16_BYTES_EXCODE(3) |
Stephen Rothwell | 7dffb72 | 2005-10-17 11:50:32 +1000 | [diff] [blame] | 427 | #if L1_CACHE_BYTES >= 128 |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 428 | COPY_16_BYTES_EXCODE(4) |
| 429 | COPY_16_BYTES_EXCODE(5) |
| 430 | COPY_16_BYTES_EXCODE(6) |
| 431 | COPY_16_BYTES_EXCODE(7) |
| 432 | #endif |
| 433 | #endif |
| 434 | #endif |
| 435 | |
| 436 | /* read fault in cacheline loop */ |
| 437 | 104: li r9,0 |
| 438 | b 92f |
| 439 | /* fault on dcbz (effectively a write fault) */ |
| 440 | /* or write fault in cacheline loop */ |
| 441 | 105: li r9,1 |
| 442 | 92: li r3,LG_CACHELINE_BYTES |
| 443 | mfctr r8 |
| 444 | add r0,r0,r8 |
| 445 | b 106f |
| 446 | /* read fault in final word loop */ |
| 447 | 108: li r9,0 |
| 448 | b 93f |
| 449 | /* write fault in final word loop */ |
| 450 | 109: li r9,1 |
| 451 | 93: andi. r5,r5,3 |
| 452 | li r3,2 |
| 453 | b 99f |
| 454 | /* read fault in final byte loop */ |
| 455 | 110: li r9,0 |
| 456 | b 94f |
| 457 | /* write fault in final byte loop */ |
| 458 | 111: li r9,1 |
| 459 | 94: li r5,0 |
| 460 | li r3,0 |
| 461 | /* |
| 462 | * At this stage the number of bytes not copied is |
| 463 | * r5 + (ctr << r3), and r9 is 0 for read or 1 for write. |
| 464 | */ |
| 465 | 99: mfctr r0 |
| 466 | 106: slw r3,r0,r3 |
| 467 | add. r3,r3,r5 |
| 468 | beq 120f /* shouldn't happen */ |
| 469 | cmpwi 0,r9,0 |
| 470 | bne 120f |
| 471 | /* for a read fault, first try to continue the copy one byte at a time */ |
| 472 | mtctr r3 |
| 473 | 130: lbz r0,4(r4) |
| 474 | 131: stb r0,4(r6) |
| 475 | addi r4,r4,1 |
| 476 | addi r6,r6,1 |
| 477 | bdnz 130b |
| 478 | /* then clear out the destination: r3 bytes starting at 4(r6) */ |
| 479 | 132: mfctr r3 |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 480 | 120: blr |
| 481 | |
Nicholas Piggin | 24bfa6a | 2016-10-13 16:42:53 +1100 | [diff] [blame] | 482 | EX_TABLE(30b,108b) |
| 483 | EX_TABLE(31b,109b) |
| 484 | EX_TABLE(40b,110b) |
| 485 | EX_TABLE(41b,111b) |
| 486 | EX_TABLE(130b,132b) |
| 487 | EX_TABLE(131b,120b) |
Nicholas Piggin | 24bfa6a | 2016-10-13 16:42:53 +1100 | [diff] [blame] | 488 | |
Al Viro | 9445aa1 | 2016-01-13 23:33:46 -0500 | [diff] [blame] | 489 | EXPORT_SYMBOL(__copy_tofrom_user) |