blob: aa8214f30c920e05c6bf62349c6ea52e0ca68dc9 [file] [log] [blame]
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001/*
2 * This file contains assembly-language implementations
3 * of IP-style 1's complement checksum routines.
4 *
5 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 *
12 * Severely hacked about by Paul Mackerras (paulus@cs.anu.edu.au).
13 */
14
15#include <linux/sys.h>
16#include <asm/processor.h>
Christophe Leroy7aef41362015-09-22 16:34:27 +020017#include <asm/cache.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100018#include <asm/errno.h>
19#include <asm/ppc_asm.h>
20
21 .text
22
23/*
Paul Mackerras14cf11a2005-09-26 16:04:21 +100024 * computes the checksum of a memory block at buff, length len,
25 * and adds in "sum" (32-bit)
26 *
Christophe Leroy7e393222016-03-07 18:44:37 +010027 * __csum_partial(buff, len, sum)
Paul Mackerras14cf11a2005-09-26 16:04:21 +100028 */
Christophe Leroy7e393222016-03-07 18:44:37 +010029_GLOBAL(__csum_partial)
Paul Mackerras14cf11a2005-09-26 16:04:21 +100030 subi r3,r3,4
Christophe Leroy48821a32015-09-22 16:34:29 +020031 srawi. r6,r4,2 /* Divide len by 4 and also clear carry */
Paul Mackerras14cf11a2005-09-26 16:04:21 +100032 beq 3f /* if we're doing < 4 bytes */
Christophe Leroy48821a32015-09-22 16:34:29 +020033 andi. r0,r3,2 /* Align buffer to longword boundary */
Paul Mackerras14cf11a2005-09-26 16:04:21 +100034 beq+ 1f
Christophe Leroy48821a32015-09-22 16:34:29 +020035 lhz r0,4(r3) /* do 2 bytes to get aligned */
Paul Mackerras14cf11a2005-09-26 16:04:21 +100036 subi r4,r4,2
Christophe Leroy48821a32015-09-22 16:34:29 +020037 addi r3,r3,2
Paul Mackerras14cf11a2005-09-26 16:04:21 +100038 srwi. r6,r4,2 /* # words to do */
Christophe Leroy48821a32015-09-22 16:34:29 +020039 adde r5,r5,r0
Paul Mackerras14cf11a2005-09-26 16:04:21 +100040 beq 3f
Christophe Leroyf867d552015-09-22 16:34:32 +0200411: andi. r6,r6,3 /* Prepare to handle words 4 by 4 */
42 beq 21f
43 mtctr r6
Christophe Leroy48821a32015-09-22 16:34:29 +0200442: lwzu r0,4(r3)
45 adde r5,r5,r0
Paul Mackerras14cf11a2005-09-26 16:04:21 +100046 bdnz 2b
Christophe Leroyf867d552015-09-22 16:34:32 +02004721: srwi. r6,r4,4 /* # blocks of 4 words to do */
48 beq 3f
49 mtctr r6
5022: lwz r0,4(r3)
51 lwz r6,8(r3)
52 lwz r7,12(r3)
53 lwzu r8,16(r3)
54 adde r5,r5,r0
55 adde r5,r5,r6
56 adde r5,r5,r7
57 adde r5,r5,r8
58 bdnz 22b
Christophe Leroy48821a32015-09-22 16:34:29 +0200593: andi. r0,r4,2
60 beq+ 4f
61 lhz r0,4(r3)
Paul Mackerras14cf11a2005-09-26 16:04:21 +100062 addi r3,r3,2
Christophe Leroy48821a32015-09-22 16:34:29 +020063 adde r5,r5,r0
644: andi. r0,r4,1
65 beq+ 5f
66 lbz r0,4(r3)
67 slwi r0,r0,8 /* Upper byte of word */
68 adde r5,r5,r0
695: addze r3,r5 /* add in final carry */
Paul Mackerras14cf11a2005-09-26 16:04:21 +100070 blr
71
72/*
73 * Computes the checksum of a memory block at src, length len,
74 * and adds in "sum" (32-bit), while copying the block to dst.
75 * If an access exception occurs on src or dst, it stores -EFAULT
76 * to *src_err or *dst_err respectively, and (for an error on
77 * src) zeroes the rest of dst.
78 *
79 * csum_partial_copy_generic(src, dst, len, sum, src_err, dst_err)
80 */
Christophe Leroy7aef41362015-09-22 16:34:27 +020081#define CSUM_COPY_16_BYTES_WITHEX(n) \
828 ## n ## 0: \
83 lwz r7,4(r4); \
848 ## n ## 1: \
85 lwz r8,8(r4); \
868 ## n ## 2: \
87 lwz r9,12(r4); \
888 ## n ## 3: \
89 lwzu r10,16(r4); \
908 ## n ## 4: \
91 stw r7,4(r6); \
92 adde r12,r12,r7; \
938 ## n ## 5: \
94 stw r8,8(r6); \
95 adde r12,r12,r8; \
968 ## n ## 6: \
97 stw r9,12(r6); \
98 adde r12,r12,r9; \
998 ## n ## 7: \
100 stwu r10,16(r6); \
101 adde r12,r12,r10
102
103#define CSUM_COPY_16_BYTES_EXCODE(n) \
104.section __ex_table,"a"; \
105 .align 2; \
106 .long 8 ## n ## 0b,src_error; \
107 .long 8 ## n ## 1b,src_error; \
108 .long 8 ## n ## 2b,src_error; \
109 .long 8 ## n ## 3b,src_error; \
110 .long 8 ## n ## 4b,dst_error; \
111 .long 8 ## n ## 5b,dst_error; \
112 .long 8 ## n ## 6b,dst_error; \
113 .long 8 ## n ## 7b,dst_error; \
114 .text
115
116 .text
117 .stabs "arch/powerpc/lib/",N_SO,0,0,0f
118 .stabs "checksum_32.S",N_SO,0,0,0f
1190:
120
121CACHELINE_BYTES = L1_CACHE_BYTES
122LG_CACHELINE_BYTES = L1_CACHE_SHIFT
123CACHELINE_MASK = (L1_CACHE_BYTES-1)
124
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000125_GLOBAL(csum_partial_copy_generic)
Christophe Leroy7aef41362015-09-22 16:34:27 +0200126 stwu r1,-16(r1)
127 stw r7,12(r1)
128 stw r8,8(r1)
129
Christophe Leroy7aef41362015-09-22 16:34:27 +0200130 addic r12,r6,0
131 addi r6,r4,-4
132 neg r0,r4
133 addi r4,r3,-4
134 andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
Christophe Leroy85405712016-08-26 16:45:13 +0200135 crset 4*cr7+eq
Christophe Leroy7aef41362015-09-22 16:34:27 +0200136 beq 58f
137
138 cmplw 0,r5,r0 /* is this more than total to do? */
139 blt 63f /* if not much to do */
Christophe Leroy85405712016-08-26 16:45:13 +0200140 rlwinm r7,r6,3,0x8
141 rlwnm r12,r12,r7,0,31 /* odd destination address: rotate one byte */
142 cmplwi cr7,r7,0 /* is destination address even ? */
Christophe Leroy7aef41362015-09-22 16:34:27 +0200143 andi. r8,r0,3 /* get it word-aligned first */
144 mtctr r8
145 beq+ 61f
146 li r3,0
14770: lbz r9,4(r4) /* do some bytes */
148 addi r4,r4,1
149 slwi r3,r3,8
150 rlwimi r3,r9,0,24,31
15171: stb r9,4(r6)
152 addi r6,r6,1
153 bdnz 70b
154 adde r12,r12,r3
15561: subf r5,r0,r5
156 srwi. r0,r0,2
157 mtctr r0
158 beq 58f
15972: lwzu r9,4(r4) /* do some words */
160 adde r12,r12,r9
16173: stwu r9,4(r6)
162 bdnz 72b
163
16458: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
165 clrlwi r5,r5,32-LG_CACHELINE_BYTES
166 li r11,4
167 beq 63f
168
169 /* Here we decide how far ahead to prefetch the source */
170 li r3,4
171 cmpwi r0,1
172 li r7,0
173 ble 114f
174 li r7,1
175#if MAX_COPY_PREFETCH > 1
176 /* Heuristically, for large transfers we prefetch
177 MAX_COPY_PREFETCH cachelines ahead. For small transfers
178 we prefetch 1 cacheline ahead. */
179 cmpwi r0,MAX_COPY_PREFETCH
180 ble 112f
181 li r7,MAX_COPY_PREFETCH
182112: mtctr r7
183111: dcbt r3,r4
184 addi r3,r3,CACHELINE_BYTES
185 bdnz 111b
186#else
187 dcbt r3,r4
188 addi r3,r3,CACHELINE_BYTES
189#endif /* MAX_COPY_PREFETCH > 1 */
190
191114: subf r8,r7,r0
192 mr r0,r7
193 mtctr r8
194
19553: dcbt r3,r4
19654: dcbz r11,r6
197/* the main body of the cacheline loop */
198 CSUM_COPY_16_BYTES_WITHEX(0)
199#if L1_CACHE_BYTES >= 32
200 CSUM_COPY_16_BYTES_WITHEX(1)
201#if L1_CACHE_BYTES >= 64
202 CSUM_COPY_16_BYTES_WITHEX(2)
203 CSUM_COPY_16_BYTES_WITHEX(3)
204#if L1_CACHE_BYTES >= 128
205 CSUM_COPY_16_BYTES_WITHEX(4)
206 CSUM_COPY_16_BYTES_WITHEX(5)
207 CSUM_COPY_16_BYTES_WITHEX(6)
208 CSUM_COPY_16_BYTES_WITHEX(7)
209#endif
210#endif
211#endif
212 bdnz 53b
213 cmpwi r0,0
214 li r3,4
215 li r7,0
216 bne 114b
217
21863: srwi. r0,r5,2
219 mtctr r0
220 beq 64f
22130: lwzu r0,4(r4)
222 adde r12,r12,r0
22331: stwu r0,4(r6)
224 bdnz 30b
225
22664: andi. r0,r5,2
227 beq+ 65f
22840: lhz r0,4(r4)
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000229 addi r4,r4,2
Christophe Leroy7aef41362015-09-22 16:34:27 +020023041: sth r0,4(r6)
231 adde r12,r12,r0
232 addi r6,r6,2
23365: andi. r0,r5,1
234 beq+ 66f
23550: lbz r0,4(r4)
23651: stb r0,4(r6)
237 slwi r0,r0,8
238 adde r12,r12,r0
23966: addze r3,r12
240 addi r1,r1,16
241 beqlr+ cr7
Christophe Leroy1bc8b812016-08-02 10:07:05 +0200242 rlwinm r3,r3,8,0,31 /* odd destination address: rotate one byte */
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000243 blr
244
Christophe Leroy7aef41362015-09-22 16:34:27 +0200245/* read fault */
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000246src_error:
Christophe Leroy7aef41362015-09-22 16:34:27 +0200247 lwz r7,12(r1)
248 addi r1,r1,16
249 cmpwi cr0,r7,0
250 beqlr
251 li r0,-EFAULT
252 stw r0,0(r7)
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000253 blr
Christophe Leroy7aef41362015-09-22 16:34:27 +0200254/* write fault */
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000255dst_error:
Christophe Leroy7aef41362015-09-22 16:34:27 +0200256 lwz r8,8(r1)
257 addi r1,r1,16
258 cmpwi cr0,r8,0
259 beqlr
260 li r0,-EFAULT
261 stw r0,0(r8)
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000262 blr
263
Christophe Leroy7aef41362015-09-22 16:34:27 +0200264 .section __ex_table,"a"
265 .align 2
266 .long 70b,src_error
267 .long 71b,dst_error
268 .long 72b,src_error
269 .long 73b,dst_error
270 .long 54b,dst_error
271 .text
272
273/*
274 * this stuff handles faults in the cacheline loop and branches to either
275 * src_error (if in read part) or dst_error (if in write part)
276 */
277 CSUM_COPY_16_BYTES_EXCODE(0)
278#if L1_CACHE_BYTES >= 32
279 CSUM_COPY_16_BYTES_EXCODE(1)
280#if L1_CACHE_BYTES >= 64
281 CSUM_COPY_16_BYTES_EXCODE(2)
282 CSUM_COPY_16_BYTES_EXCODE(3)
283#if L1_CACHE_BYTES >= 128
284 CSUM_COPY_16_BYTES_EXCODE(4)
285 CSUM_COPY_16_BYTES_EXCODE(5)
286 CSUM_COPY_16_BYTES_EXCODE(6)
287 CSUM_COPY_16_BYTES_EXCODE(7)
288#endif
289#endif
290#endif
291
292 .section __ex_table,"a"
293 .align 2
294 .long 30b,src_error
295 .long 31b,dst_error
296 .long 40b,src_error
297 .long 41b,dst_error
298 .long 50b,src_error
299 .long 51b,dst_error