blob: f5cc9eb1d51bc02ef817bd664c7810f4f5b928b1 [file] [log] [blame]
Luca Barbieria7e926a2010-02-24 10:54:25 +01001/*
2 * atomic64_t for 586+
3 *
4 * Copyright © 2010 Luca Barbieri
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 */
11
12#include <linux/linkage.h>
13#include <asm/alternative-asm.h>
14#include <asm/dwarf2.h>
15
16.macro SAVE reg
Jan Beulich60cf6372011-02-28 15:54:40 +000017 pushl_cfi %\reg
Luca Barbieria7e926a2010-02-24 10:54:25 +010018 CFI_REL_OFFSET \reg, 0
19.endm
20
21.macro RESTORE reg
Jan Beulich60cf6372011-02-28 15:54:40 +000022 popl_cfi %\reg
Luca Barbieria7e926a2010-02-24 10:54:25 +010023 CFI_RESTORE \reg
24.endm
25
26.macro read64 reg
27 movl %ebx, %eax
28 movl %ecx, %edx
29/* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */
30 LOCK_PREFIX
31 cmpxchg8b (\reg)
32.endm
33
34ENTRY(atomic64_read_cx8)
35 CFI_STARTPROC
36
37 read64 %ecx
38 ret
39 CFI_ENDPROC
40ENDPROC(atomic64_read_cx8)
41
42ENTRY(atomic64_set_cx8)
43 CFI_STARTPROC
44
451:
46/* we don't need LOCK_PREFIX since aligned 64-bit writes
47 * are atomic on 586 and newer */
48 cmpxchg8b (%esi)
49 jne 1b
50
51 ret
52 CFI_ENDPROC
53ENDPROC(atomic64_set_cx8)
54
55ENTRY(atomic64_xchg_cx8)
56 CFI_STARTPROC
57
Luca Barbieria7e926a2010-02-24 10:54:25 +0100581:
59 LOCK_PREFIX
60 cmpxchg8b (%esi)
61 jne 1b
62
63 ret
64 CFI_ENDPROC
65ENDPROC(atomic64_xchg_cx8)
66
67.macro addsub_return func ins insc
68ENTRY(atomic64_\func\()_return_cx8)
69 CFI_STARTPROC
70 SAVE ebp
71 SAVE ebx
72 SAVE esi
73 SAVE edi
74
75 movl %eax, %esi
76 movl %edx, %edi
77 movl %ecx, %ebp
78
Jan Beulichcb8095b2012-01-20 16:22:04 +000079 read64 %ecx
Luca Barbieria7e926a2010-02-24 10:54:25 +0100801:
81 movl %eax, %ebx
82 movl %edx, %ecx
83 \ins\()l %esi, %ebx
84 \insc\()l %edi, %ecx
85 LOCK_PREFIX
86 cmpxchg8b (%ebp)
87 jne 1b
88
8910:
90 movl %ebx, %eax
91 movl %ecx, %edx
92 RESTORE edi
93 RESTORE esi
94 RESTORE ebx
95 RESTORE ebp
96 ret
97 CFI_ENDPROC
98ENDPROC(atomic64_\func\()_return_cx8)
99.endm
100
101addsub_return add add adc
102addsub_return sub sub sbb
103
104.macro incdec_return func ins insc
105ENTRY(atomic64_\func\()_return_cx8)
106 CFI_STARTPROC
107 SAVE ebx
108
109 read64 %esi
1101:
111 movl %eax, %ebx
112 movl %edx, %ecx
113 \ins\()l $1, %ebx
114 \insc\()l $0, %ecx
115 LOCK_PREFIX
116 cmpxchg8b (%esi)
117 jne 1b
118
11910:
120 movl %ebx, %eax
121 movl %ecx, %edx
122 RESTORE ebx
123 ret
124 CFI_ENDPROC
125ENDPROC(atomic64_\func\()_return_cx8)
126.endm
127
128incdec_return inc add adc
129incdec_return dec sub sbb
130
131ENTRY(atomic64_dec_if_positive_cx8)
132 CFI_STARTPROC
133 SAVE ebx
134
135 read64 %esi
1361:
137 movl %eax, %ebx
138 movl %edx, %ecx
139 subl $1, %ebx
140 sbb $0, %ecx
141 js 2f
142 LOCK_PREFIX
143 cmpxchg8b (%esi)
144 jne 1b
145
1462:
147 movl %ebx, %eax
148 movl %ecx, %edx
149 RESTORE ebx
150 ret
151 CFI_ENDPROC
152ENDPROC(atomic64_dec_if_positive_cx8)
153
154ENTRY(atomic64_add_unless_cx8)
155 CFI_STARTPROC
156 SAVE ebp
157 SAVE ebx
158/* these just push these two parameters on the stack */
159 SAVE edi
Jan Beulichcb8095b2012-01-20 16:22:04 +0000160 SAVE ecx
Luca Barbieria7e926a2010-02-24 10:54:25 +0100161
Jan Beulichcb8095b2012-01-20 16:22:04 +0000162 movl %eax, %ebp
Luca Barbieria7e926a2010-02-24 10:54:25 +0100163 movl %edx, %edi
164
Jan Beulichcb8095b2012-01-20 16:22:04 +0000165 read64 %esi
Luca Barbieria7e926a2010-02-24 10:54:25 +01001661:
167 cmpl %eax, 0(%esp)
168 je 4f
1692:
170 movl %eax, %ebx
171 movl %edx, %ecx
Jan Beulichcb8095b2012-01-20 16:22:04 +0000172 addl %ebp, %ebx
Luca Barbieria7e926a2010-02-24 10:54:25 +0100173 adcl %edi, %ecx
174 LOCK_PREFIX
Jan Beulichcb8095b2012-01-20 16:22:04 +0000175 cmpxchg8b (%esi)
Luca Barbieria7e926a2010-02-24 10:54:25 +0100176 jne 1b
177
Luca Barbieri6e6104f2010-03-01 19:55:46 +0100178 movl $1, %eax
Luca Barbieria7e926a2010-02-24 10:54:25 +01001793:
180 addl $8, %esp
181 CFI_ADJUST_CFA_OFFSET -8
182 RESTORE ebx
183 RESTORE ebp
184 ret
1854:
186 cmpl %edx, 4(%esp)
187 jne 2b
Luca Barbieri6e6104f2010-03-01 19:55:46 +0100188 xorl %eax, %eax
Luca Barbieria7e926a2010-02-24 10:54:25 +0100189 jmp 3b
190 CFI_ENDPROC
191ENDPROC(atomic64_add_unless_cx8)
192
193ENTRY(atomic64_inc_not_zero_cx8)
194 CFI_STARTPROC
195 SAVE ebx
196
197 read64 %esi
1981:
Jan Beulichcb8095b2012-01-20 16:22:04 +0000199 movl %eax, %ecx
200 orl %edx, %ecx
201 jz 3f
Luca Barbieria7e926a2010-02-24 10:54:25 +0100202 movl %eax, %ebx
Jan Beulichcb8095b2012-01-20 16:22:04 +0000203 xorl %ecx, %ecx
Luca Barbieria7e926a2010-02-24 10:54:25 +0100204 addl $1, %ebx
Jan Beulichcb8095b2012-01-20 16:22:04 +0000205 adcl %edx, %ecx
Luca Barbieria7e926a2010-02-24 10:54:25 +0100206 LOCK_PREFIX
207 cmpxchg8b (%esi)
208 jne 1b
209
Luca Barbierif3e83132010-03-01 19:55:49 +0100210 movl $1, %eax
Luca Barbieria7e926a2010-02-24 10:54:25 +01002113:
212 RESTORE ebx
213 ret
Luca Barbieria7e926a2010-02-24 10:54:25 +0100214 CFI_ENDPROC
215ENDPROC(atomic64_inc_not_zero_cx8)