blob: 5d1cbfe4ae58eea4276bbbd9fc342188cafeeb95 [file] [log] [blame]
Srikar Dronamraju2b144492012-02-09 14:56:42 +05301/*
Ingo Molnar7b2d81d2012-02-17 09:27:41 +01002 * User-space Probes (UProbes) for x86
Srikar Dronamraju2b144492012-02-09 14:56:42 +05303 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * Copyright (C) IBM Corporation, 2008-2011
19 * Authors:
20 * Srikar Dronamraju
21 * Jim Keniston
22 */
Srikar Dronamraju2b144492012-02-09 14:56:42 +053023#include <linux/kernel.h>
24#include <linux/sched.h>
25#include <linux/ptrace.h>
26#include <linux/uprobes.h>
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +053027#include <linux/uaccess.h>
Srikar Dronamraju2b144492012-02-09 14:56:42 +053028
29#include <linux/kdebug.h>
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +053030#include <asm/processor.h>
Srikar Dronamraju2b144492012-02-09 14:56:42 +053031#include <asm/insn.h>
32
33/* Post-execution fixups. */
34
Srikar Dronamraju2b144492012-02-09 14:56:42 +053035/* Adjust IP back to vicinity of actual insn */
Oleg Nesterov78d9af42014-04-24 18:52:37 +020036#define UPROBE_FIX_IP 0x01
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +053037
Srikar Dronamraju2b144492012-02-09 14:56:42 +053038/* Adjust the return address of a call insn */
Oleg Nesterov78d9af42014-04-24 18:52:37 +020039#define UPROBE_FIX_CALL 0x02
Srikar Dronamraju2b144492012-02-09 14:56:42 +053040
Sebastian Andrzej Siewiorbdc1e472012-08-20 12:47:34 +020041/* Instruction will modify TF, don't change it */
Oleg Nesterov78d9af42014-04-24 18:52:37 +020042#define UPROBE_FIX_SETF 0x04
Sebastian Andrzej Siewiorbdc1e472012-08-20 12:47:34 +020043
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +020044#define UPROBE_FIX_RIP_SI 0x08
45#define UPROBE_FIX_RIP_DI 0x10
46#define UPROBE_FIX_RIP_BX 0x20
47#define UPROBE_FIX_RIP_MASK \
48 (UPROBE_FIX_RIP_SI | UPROBE_FIX_RIP_DI | UPROBE_FIX_RIP_BX)
Srikar Dronamraju2b144492012-02-09 14:56:42 +053049
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +053050#define UPROBE_TRAP_NR UINT_MAX
51
Srikar Dronamraju2b144492012-02-09 14:56:42 +053052/* Adaptations for mhiramat x86 decoder v14. */
Ingo Molnar7b2d81d2012-02-17 09:27:41 +010053#define OPCODE1(insn) ((insn)->opcode.bytes[0])
54#define OPCODE2(insn) ((insn)->opcode.bytes[1])
55#define OPCODE3(insn) ((insn)->opcode.bytes[2])
Oleg Nesterovddb69f22014-03-31 15:16:22 +020056#define MODRM_REG(insn) X86_MODRM_REG((insn)->modrm.value)
Srikar Dronamraju2b144492012-02-09 14:56:42 +053057
58#define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
59 (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \
60 (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \
61 (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \
62 (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \
63 << (row % 32))
64
Srikar Dronamraju04a3d982012-02-22 14:45:35 +053065/*
66 * Good-instruction tables for 32-bit apps. This is non-const and volatile
67 * to keep gcc from statically optimizing it out, as variable_test_bit makes
68 * some versions of gcc to think only *(unsigned long*) is used.
69 */
Oleg Nesterov8dbacad2014-04-19 16:07:15 +020070#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
Srikar Dronamraju04a3d982012-02-22 14:45:35 +053071static volatile u32 good_insns_32[256 / 32] = {
Srikar Dronamraju2b144492012-02-09 14:56:42 +053072 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
73 /* ---------------------------------------------- */
74 W(0x00, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0) | /* 00 */
75 W(0x10, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0) , /* 10 */
76 W(0x20, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1) | /* 20 */
77 W(0x30, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1) , /* 30 */
78 W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
79 W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
80 W(0x60, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) | /* 60 */
81 W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 70 */
82 W(0x80, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
83 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
84 W(0xa0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* a0 */
85 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
86 W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* c0 */
87 W(0xd0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
88 W(0xe0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) | /* e0 */
89 W(0xf0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1) /* f0 */
90 /* ---------------------------------------------- */
91 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
92};
Oleg Nesterov8dbacad2014-04-19 16:07:15 +020093#else
94#define good_insns_32 NULL
95#endif
96
97/* Good-instruction tables for 64-bit apps */
98#if defined(CONFIG_X86_64)
99static volatile u32 good_insns_64[256 / 32] = {
100 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
101 /* ---------------------------------------------- */
102 W(0x00, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0) | /* 00 */
103 W(0x10, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0) , /* 10 */
104 W(0x20, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0) | /* 20 */
105 W(0x30, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0) , /* 30 */
106 W(0x40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 40 */
107 W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
108 W(0x60, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) | /* 60 */
109 W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 70 */
110 W(0x80, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
111 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
112 W(0xa0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* a0 */
113 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
114 W(0xc0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* c0 */
115 W(0xd0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
116 W(0xe0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) | /* e0 */
117 W(0xf0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1) /* f0 */
118 /* ---------------------------------------------- */
119 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
120};
121#else
122#define good_insns_64 NULL
123#endif
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530124
125/* Using this for both 64-bit and 32-bit apps */
Srikar Dronamraju04a3d982012-02-22 14:45:35 +0530126static volatile u32 good_2byte_insns[256 / 32] = {
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530127 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
128 /* ---------------------------------------------- */
129 W(0x00, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1) | /* 00 */
130 W(0x10, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1) , /* 10 */
131 W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* 20 */
132 W(0x30, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */
133 W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
134 W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
135 W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 60 */
136 W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */
137 W(0x80, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
138 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
139 W(0xa0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1) | /* a0 */
140 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
141 W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
142 W(0xd0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
143 W(0xe0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* e0 */
144 W(0xf0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0) /* f0 */
145 /* ---------------------------------------------- */
146 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
147};
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530148#undef W
149
150/*
151 * opcodes we'll probably never support:
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100152 *
153 * 6c-6d, e4-e5, ec-ed - in
154 * 6e-6f, e6-e7, ee-ef - out
155 * cc, cd - int3, int
156 * cf - iret
157 * d6 - illegal instruction
158 * f1 - int1/icebp
159 * f4 - hlt
160 * fa, fb - cli, sti
161 * 0f - lar, lsl, syscall, clts, sysret, sysenter, sysexit, invd, wbinvd, ud2
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530162 *
163 * invalid opcodes in 64-bit mode:
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530164 *
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100165 * 06, 0e, 16, 1e, 27, 2f, 37, 3f, 60-62, 82, c4-c5, d4-d5
166 * 63 - we support this opcode in x86_64 but not in i386.
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530167 *
168 * opcodes we may need to refine support for:
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100169 *
170 * 0f - 2-byte instructions: For many of these instructions, the validity
171 * depends on the prefix and/or the reg field. On such instructions, we
172 * just consider the opcode combination valid if it corresponds to any
173 * valid instruction.
174 *
175 * 8f - Group 1 - only reg = 0 is OK
176 * c6-c7 - Group 11 - only reg = 0 is OK
177 * d9-df - fpu insns with some illegal encodings
178 * f2, f3 - repnz, repz prefixes. These are also the first byte for
179 * certain floating-point instructions, such as addsd.
180 *
181 * fe - Group 4 - only reg = 0 or 1 is OK
182 * ff - Group 5 - only reg = 0-6 is OK
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530183 *
184 * others -- Do we need to support these?
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100185 *
186 * 0f - (floating-point?) prefetch instructions
187 * 07, 17, 1f - pop es, pop ss, pop ds
188 * 26, 2e, 36, 3e - es:, cs:, ss:, ds: segment prefixes --
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530189 * but 64 and 65 (fs: and gs:) seem to be used, so we support them
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100190 * 67 - addr16 prefix
191 * ce - into
192 * f0 - lock prefix
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530193 */
194
195/*
196 * TODO:
197 * - Where necessary, examine the modrm byte and allow only valid instructions
198 * in the different Groups and fpu instructions.
199 */
200
201static bool is_prefix_bad(struct insn *insn)
202{
203 int i;
204
205 for (i = 0; i < insn->prefixes.nbytes; i++) {
206 switch (insn->prefixes.bytes[i]) {
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100207 case 0x26: /* INAT_PFX_ES */
208 case 0x2E: /* INAT_PFX_CS */
209 case 0x36: /* INAT_PFX_DS */
210 case 0x3E: /* INAT_PFX_SS */
211 case 0xF0: /* INAT_PFX_LOCK */
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530212 return true;
213 }
214 }
215 return false;
216}
217
Oleg Nesterov73175d02014-04-19 12:34:02 +0200218static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64)
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530219{
Oleg Nesterov73175d02014-04-19 12:34:02 +0200220 u32 volatile *good_insns;
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530221
Oleg Nesterov73175d02014-04-19 12:34:02 +0200222 insn_init(insn, auprobe->insn, x86_64);
Oleg Nesterovff261962014-04-19 14:15:27 +0200223 /* has the side-effect of processing the entire instruction */
224 insn_get_length(insn);
225 if (WARN_ON_ONCE(!insn_complete(insn)))
226 return -ENOEXEC;
Oleg Nesterov73175d02014-04-19 12:34:02 +0200227
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530228 if (is_prefix_bad(insn))
229 return -ENOTSUPP;
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100230
Oleg Nesterov73175d02014-04-19 12:34:02 +0200231 if (x86_64)
232 good_insns = good_insns_64;
233 else
234 good_insns = good_insns_32;
235
236 if (test_bit(OPCODE1(insn), (unsigned long *)good_insns))
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530237 return 0;
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100238
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530239 if (insn->opcode.nbytes == 2) {
240 if (test_bit(OPCODE2(insn), (unsigned long *)good_2byte_insns))
241 return 0;
242 }
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100243
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530244 return -ENOTSUPP;
245}
246
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530247#ifdef CONFIG_X86_64
Oleg Nesterov2ae1f492014-04-19 14:03:05 +0200248static inline bool is_64bit_mm(struct mm_struct *mm)
249{
250 return !config_enabled(CONFIG_IA32_EMULATION) ||
Oleg Nesterovb24dc8d2014-04-19 18:10:09 +0200251 !(mm->context.ia32_compat == TIF_IA32);
Oleg Nesterov2ae1f492014-04-19 14:03:05 +0200252}
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530253/*
Srikar Dronamraju3ff54ef2012-02-22 14:46:02 +0530254 * If arch_uprobe->insn doesn't use rip-relative addressing, return
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530255 * immediately. Otherwise, rewrite the instruction so that it accesses
256 * its memory operand indirectly through a scratch register. Set
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200257 * defparam->fixups accordingly. (The contents of the scratch register
Denys Vlasenko50204c62014-05-01 16:52:46 +0200258 * will be saved before we single-step the modified instruction,
259 * and restored afterward).
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530260 *
261 * We do this because a rip-relative instruction can access only a
262 * relatively small area (+/- 2 GB from the instruction), and the XOL
263 * area typically lies beyond that area. At least for instructions
264 * that store to memory, we can't execute the original instruction
265 * and "fix things up" later, because the misdirected store could be
266 * disastrous.
267 *
268 * Some useful facts about rip-relative instructions:
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100269 *
Denys Vlasenko50204c62014-05-01 16:52:46 +0200270 * - There's always a modrm byte with bit layout "00 reg 101".
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100271 * - There's never a SIB byte.
272 * - The displacement is always 4 bytes.
Denys Vlasenko50204c62014-05-01 16:52:46 +0200273 * - REX.B=1 bit in REX prefix, which normally extends r/m field,
274 * has no effect on rip-relative mode. It doesn't make modrm byte
275 * with r/m=101 refer to register 1101 = R13.
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530276 */
Oleg Nesterov1475ee72014-04-27 16:31:59 +0200277static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530278{
279 u8 *cursor;
280 u8 reg;
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +0200281 u8 reg2;
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530282
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530283 if (!insn_rip_relative(insn))
284 return;
285
286 /*
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +0200287 * insn_rip_relative() would have decoded rex_prefix, vex_prefix, modrm.
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530288 * Clear REX.b bit (extension of MODRM.rm field):
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +0200289 * we want to encode low numbered reg, not r8+.
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530290 */
291 if (insn->rex_prefix.nbytes) {
Srikar Dronamraju3ff54ef2012-02-22 14:46:02 +0530292 cursor = auprobe->insn + insn_offset_rex_prefix(insn);
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +0200293 /* REX byte has 0100wrxb layout, clearing REX.b bit */
294 *cursor &= 0xfe;
295 }
296 /*
297 * Similar treatment for VEX3 prefix.
298 * TODO: add XOP/EVEX treatment when insn decoder supports them
299 */
300 if (insn->vex_prefix.nbytes == 3) {
301 /*
302 * vex2: c5 rvvvvLpp (has no b bit)
303 * vex3/xop: c4/8f rxbmmmmm wvvvvLpp
304 * evex: 62 rxbR00mm wvvvv1pp zllBVaaa
305 * (evex will need setting of both b and x since
306 * in non-sib encoding evex.x is 4th bit of MODRM.rm)
307 * Setting VEX3.b (setting because it has inverted meaning):
308 */
309 cursor = auprobe->insn + insn_offset_vex_prefix(insn) + 1;
310 *cursor |= 0x20;
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530311 }
312
313 /*
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +0200314 * Convert from rip-relative addressing to register-relative addressing
315 * via a scratch register.
316 *
317 * This is tricky since there are insns with modrm byte
318 * which also use registers not encoded in modrm byte:
319 * [i]div/[i]mul: implicitly use dx:ax
320 * shift ops: implicitly use cx
321 * cmpxchg: implicitly uses ax
322 * cmpxchg8/16b: implicitly uses dx:ax and bx:cx
323 * Encoding: 0f c7/1 modrm
324 * The code below thinks that reg=1 (cx), chooses si as scratch.
325 * mulx: implicitly uses dx: mulx r/m,r1,r2 does r1:r2 = dx * r/m.
326 * First appeared in Haswell (BMI2 insn). It is vex-encoded.
327 * Example where none of bx,cx,dx can be used as scratch reg:
328 * c4 e2 63 f6 0d disp32 mulx disp32(%rip),%ebx,%ecx
329 * [v]pcmpistri: implicitly uses cx, xmm0
330 * [v]pcmpistrm: implicitly uses xmm0
331 * [v]pcmpestri: implicitly uses ax, dx, cx, xmm0
332 * [v]pcmpestrm: implicitly uses ax, dx, xmm0
333 * Evil SSE4.2 string comparison ops from hell.
334 * maskmovq/[v]maskmovdqu: implicitly uses (ds:rdi) as destination.
335 * Encoding: 0f f7 modrm, 66 0f f7 modrm, vex-encoded: c5 f9 f7 modrm.
336 * Store op1, byte-masked by op2 msb's in each byte, to (ds:rdi).
337 * AMD says it has no 3-operand form (vex.vvvv must be 1111)
338 * and that it can have only register operands, not mem
339 * (its modrm byte must have mode=11).
340 * If these restrictions will ever be lifted,
341 * we'll need code to prevent selection of di as scratch reg!
342 *
343 * Summary: I don't know any insns with modrm byte which
344 * use SI register implicitly. DI register is used only
345 * by one insn (maskmovq) and BX register is used
346 * only by one too (cmpxchg8b).
347 * BP is stack-segment based (may be a problem?).
348 * AX, DX, CX are off-limits (many implicit users).
349 * SP is unusable (it's stack pointer - think about "pop mem";
350 * also, rsp+disp32 needs sib encoding -> insn length change).
351 */
352
353 reg = MODRM_REG(insn); /* Fetch modrm.reg */
354 reg2 = 0xff; /* Fetch vex.vvvv */
355 if (insn->vex_prefix.nbytes == 2)
356 reg2 = insn->vex_prefix.bytes[1];
357 else if (insn->vex_prefix.nbytes == 3)
358 reg2 = insn->vex_prefix.bytes[2];
359 /*
360 * TODO: add XOP, EXEV vvvv reading.
361 *
362 * vex.vvvv field is in bits 6-3, bits are inverted.
363 * But in 32-bit mode, high-order bit may be ignored.
364 * Therefore, let's consider only 3 low-order bits.
365 */
366 reg2 = ((reg2 >> 3) & 0x7) ^ 0x7;
367 /*
368 * Register numbering is ax,cx,dx,bx, sp,bp,si,di, r8..r15.
369 *
370 * Choose scratch reg. Order is important: must not select bx
371 * if we can use si (cmpxchg8b case!)
372 */
373 if (reg != 6 && reg2 != 6) {
374 reg2 = 6;
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200375 auprobe->defparam.fixups |= UPROBE_FIX_RIP_SI;
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +0200376 } else if (reg != 7 && reg2 != 7) {
377 reg2 = 7;
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200378 auprobe->defparam.fixups |= UPROBE_FIX_RIP_DI;
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +0200379 /* TODO (paranoia): force maskmovq to not use di */
380 } else {
381 reg2 = 3;
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200382 auprobe->defparam.fixups |= UPROBE_FIX_RIP_BX;
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +0200383 }
384 /*
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530385 * Point cursor at the modrm byte. The next 4 bytes are the
386 * displacement. Beyond the displacement, for some instructions,
387 * is the immediate operand.
388 */
Srikar Dronamraju3ff54ef2012-02-22 14:46:02 +0530389 cursor = auprobe->insn + insn_offset_modrm(insn);
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530390 /*
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +0200391 * Change modrm from "00 reg 101" to "10 reg reg2". Example:
392 * 89 05 disp32 mov %eax,disp32(%rip) becomes
393 * 89 86 disp32 mov %eax,disp32(%rsi)
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530394 */
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +0200395 *cursor = 0x80 | (reg << 3) | reg2;
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530396}
397
Oleg Nesterovc90a6952014-04-27 18:13:31 +0200398static inline unsigned long *
399scratch_reg(struct arch_uprobe *auprobe, struct pt_regs *regs)
400{
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200401 if (auprobe->defparam.fixups & UPROBE_FIX_RIP_SI)
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +0200402 return &regs->si;
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200403 if (auprobe->defparam.fixups & UPROBE_FIX_RIP_DI)
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +0200404 return &regs->di;
405 return &regs->bx;
Oleg Nesterovc90a6952014-04-27 18:13:31 +0200406}
407
Oleg Nesterovd20737c2014-03-31 18:35:09 +0200408/*
409 * If we're emulating a rip-relative instruction, save the contents
410 * of the scratch register and store the target address in that register.
411 */
Oleg Nesterov7f55e822014-04-27 17:00:46 +0200412static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
Oleg Nesterovd20737c2014-03-31 18:35:09 +0200413{
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200414 if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {
Oleg Nesterovc90a6952014-04-27 18:13:31 +0200415 struct uprobe_task *utask = current->utask;
416 unsigned long *sr = scratch_reg(auprobe, regs);
Oleg Nesterov7f55e822014-04-27 17:00:46 +0200417
Oleg Nesterovc90a6952014-04-27 18:13:31 +0200418 utask->autask.saved_scratch_register = *sr;
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200419 *sr = utask->vaddr + auprobe->defparam.ilen;
Oleg Nesterovd20737c2014-03-31 18:35:09 +0200420 }
421}
422
Denys Vlasenko50204c62014-05-01 16:52:46 +0200423static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
Oleg Nesterovd20737c2014-03-31 18:35:09 +0200424{
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200425 if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {
Oleg Nesterovc90a6952014-04-27 18:13:31 +0200426 struct uprobe_task *utask = current->utask;
427 unsigned long *sr = scratch_reg(auprobe, regs);
Oleg Nesterovd20737c2014-03-31 18:35:09 +0200428
Oleg Nesterovc90a6952014-04-27 18:13:31 +0200429 *sr = utask->autask.saved_scratch_register;
Oleg Nesterovd20737c2014-03-31 18:35:09 +0200430 }
431}
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100432#else /* 32-bit: */
Oleg Nesterov2ae1f492014-04-19 14:03:05 +0200433static inline bool is_64bit_mm(struct mm_struct *mm)
434{
435 return false;
436}
Oleg Nesterovd20737c2014-03-31 18:35:09 +0200437/*
438 * No RIP-relative addressing on 32-bit
439 */
Oleg Nesterov1475ee72014-04-27 16:31:59 +0200440static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530441{
Oleg Nesterovd20737c2014-03-31 18:35:09 +0200442}
Oleg Nesterov7f55e822014-04-27 17:00:46 +0200443static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
Oleg Nesterovd20737c2014-03-31 18:35:09 +0200444{
445}
Denys Vlasenko50204c62014-05-01 16:52:46 +0200446static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
Oleg Nesterovd20737c2014-03-31 18:35:09 +0200447{
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530448}
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530449#endif /* CONFIG_X86_64 */
450
Oleg Nesterov8ad8e9d2014-03-31 21:01:31 +0200451struct uprobe_xol_ops {
452 bool (*emulate)(struct arch_uprobe *, struct pt_regs *);
453 int (*pre_xol)(struct arch_uprobe *, struct pt_regs *);
454 int (*post_xol)(struct arch_uprobe *, struct pt_regs *);
Oleg Nesterov588fbd62014-04-21 16:58:17 +0200455 void (*abort)(struct arch_uprobe *, struct pt_regs *);
Oleg Nesterov8ad8e9d2014-03-31 21:01:31 +0200456};
457
Oleg Nesterov8faaed12014-04-06 17:16:10 +0200458static inline int sizeof_long(void)
459{
460 return is_ia32_task() ? 4 : 8;
461}
462
Oleg Nesterov8ad8e9d2014-03-31 21:01:31 +0200463static int default_pre_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
464{
Oleg Nesterov7f55e822014-04-27 17:00:46 +0200465 riprel_pre_xol(auprobe, regs);
Oleg Nesterov8ad8e9d2014-03-31 21:01:31 +0200466 return 0;
467}
468
Oleg Nesterov2b82cad2014-04-24 19:21:38 +0200469static int push_ret_address(struct pt_regs *regs, unsigned long ip)
470{
471 unsigned long new_sp = regs->sp - sizeof_long();
472
473 if (copy_to_user((void __user *)new_sp, &ip, sizeof_long()))
474 return -EFAULT;
475
476 regs->sp = new_sp;
477 return 0;
478}
479
Denys Vlasenko1ea30fb2014-05-02 17:04:00 +0200480/*
481 * We have to fix things up as follows:
482 *
483 * Typically, the new ip is relative to the copied instruction. We need
484 * to make it relative to the original instruction (FIX_IP). Exceptions
485 * are return instructions and absolute or indirect jump or call instructions.
486 *
487 * If the single-stepped instruction was a call, the return address that
488 * is atop the stack is the address following the copied instruction. We
489 * need to make it the address following the original instruction (FIX_CALL).
490 *
491 * If the original instruction was a rip-relative instruction such as
492 * "movl %edx,0xnnnn(%rip)", we have instead executed an equivalent
493 * instruction using a scratch register -- e.g., "movl %edx,0xnnnn(%rsi)".
494 * We need to restore the contents of the scratch register
495 * (FIX_RIP_reg).
496 */
Oleg Nesterov8ad8e9d2014-03-31 21:01:31 +0200497static int default_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
498{
499 struct uprobe_task *utask = current->utask;
Oleg Nesterov8ad8e9d2014-03-31 21:01:31 +0200500
Denys Vlasenko50204c62014-05-01 16:52:46 +0200501 riprel_post_xol(auprobe, regs);
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200502 if (auprobe->defparam.fixups & UPROBE_FIX_IP) {
Denys Vlasenko50204c62014-05-01 16:52:46 +0200503 long correction = utask->vaddr - utask->xol_vaddr;
Oleg Nesterov8ad8e9d2014-03-31 21:01:31 +0200504 regs->ip += correction;
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200505 } else if (auprobe->defparam.fixups & UPROBE_FIX_CALL) {
506 regs->sp += sizeof_long(); /* Pop incorrect return address */
507 if (push_ret_address(regs, utask->vaddr + auprobe->defparam.ilen))
Oleg Nesterov75f9ef02014-04-03 20:52:19 +0200508 return -ERESTART;
Oleg Nesterov75f9ef02014-04-03 20:52:19 +0200509 }
Oleg Nesterov220ef8d2014-04-21 20:39:56 +0200510 /* popf; tell the caller to not touch TF */
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200511 if (auprobe->defparam.fixups & UPROBE_FIX_SETF)
Oleg Nesterov220ef8d2014-04-21 20:39:56 +0200512 utask->autask.saved_tf = true;
Oleg Nesterov8ad8e9d2014-03-31 21:01:31 +0200513
Oleg Nesterov75f9ef02014-04-03 20:52:19 +0200514 return 0;
Oleg Nesterov8ad8e9d2014-03-31 21:01:31 +0200515}
516
Oleg Nesterov588fbd62014-04-21 16:58:17 +0200517static void default_abort_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
518{
Denys Vlasenko50204c62014-05-01 16:52:46 +0200519 riprel_post_xol(auprobe, regs);
Oleg Nesterov588fbd62014-04-21 16:58:17 +0200520}
521
Oleg Nesterov8ad8e9d2014-03-31 21:01:31 +0200522static struct uprobe_xol_ops default_xol_ops = {
523 .pre_xol = default_pre_xol_op,
524 .post_xol = default_post_xol_op,
Oleg Nesterov588fbd62014-04-21 16:58:17 +0200525 .abort = default_abort_op,
Oleg Nesterov8ad8e9d2014-03-31 21:01:31 +0200526};
527
Oleg Nesterov8e89c0b2014-04-06 18:11:02 +0200528static bool branch_is_call(struct arch_uprobe *auprobe)
529{
530 return auprobe->branch.opc1 == 0xe8;
531}
532
Oleg Nesterov8f955052014-04-06 21:53:47 +0200533#define CASE_COND \
534 COND(70, 71, XF(OF)) \
535 COND(72, 73, XF(CF)) \
536 COND(74, 75, XF(ZF)) \
537 COND(78, 79, XF(SF)) \
538 COND(7a, 7b, XF(PF)) \
539 COND(76, 77, XF(CF) || XF(ZF)) \
540 COND(7c, 7d, XF(SF) != XF(OF)) \
541 COND(7e, 7f, XF(ZF) || XF(SF) != XF(OF))
542
543#define COND(op_y, op_n, expr) \
544 case 0x ## op_y: DO((expr) != 0) \
545 case 0x ## op_n: DO((expr) == 0)
546
547#define XF(xf) (!!(flags & X86_EFLAGS_ ## xf))
548
549static bool is_cond_jmp_opcode(u8 opcode)
550{
551 switch (opcode) {
552 #define DO(expr) \
553 return true;
554 CASE_COND
555 #undef DO
556
557 default:
558 return false;
559 }
560}
561
562static bool check_jmp_cond(struct arch_uprobe *auprobe, struct pt_regs *regs)
563{
564 unsigned long flags = regs->flags;
565
566 switch (auprobe->branch.opc1) {
567 #define DO(expr) \
568 return expr;
569 CASE_COND
570 #undef DO
571
572 default: /* not a conditional jmp */
573 return true;
574 }
575}
576
577#undef XF
578#undef COND
579#undef CASE_COND
580
Oleg Nesterov7ba6db22014-04-05 20:05:02 +0200581static bool branch_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
582{
Oleg Nesterov8e89c0b2014-04-06 18:11:02 +0200583 unsigned long new_ip = regs->ip += auprobe->branch.ilen;
Oleg Nesterov8f955052014-04-06 21:53:47 +0200584 unsigned long offs = (long)auprobe->branch.offs;
Oleg Nesterov8e89c0b2014-04-06 18:11:02 +0200585
586 if (branch_is_call(auprobe)) {
Oleg Nesterov8e89c0b2014-04-06 18:11:02 +0200587 /*
588 * If it fails we execute this (mangled, see the comment in
589 * branch_clear_offset) insn out-of-line. In the likely case
590 * this should trigger the trap, and the probed application
591 * should die or restart the same insn after it handles the
592 * signal, arch_uprobe_post_xol() won't be even called.
593 *
594 * But there is corner case, see the comment in ->post_xol().
595 */
Oleg Nesterov2b82cad2014-04-24 19:21:38 +0200596 if (push_ret_address(regs, new_ip))
Oleg Nesterov8e89c0b2014-04-06 18:11:02 +0200597 return false;
Oleg Nesterov8f955052014-04-06 21:53:47 +0200598 } else if (!check_jmp_cond(auprobe, regs)) {
599 offs = 0;
Oleg Nesterov8e89c0b2014-04-06 18:11:02 +0200600 }
601
Oleg Nesterov8f955052014-04-06 21:53:47 +0200602 regs->ip = new_ip + offs;
Oleg Nesterov7ba6db22014-04-05 20:05:02 +0200603 return true;
604}
605
Oleg Nesterov8e89c0b2014-04-06 18:11:02 +0200606static int branch_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
607{
608 BUG_ON(!branch_is_call(auprobe));
609 /*
610 * We can only get here if branch_emulate_op() failed to push the ret
611 * address _and_ another thread expanded our stack before the (mangled)
612 * "call" insn was executed out-of-line. Just restore ->sp and restart.
613 * We could also restore ->ip and try to call branch_emulate_op() again.
614 */
615 regs->sp += sizeof_long();
616 return -ERESTART;
617}
618
619static void branch_clear_offset(struct arch_uprobe *auprobe, struct insn *insn)
620{
621 /*
622 * Turn this insn into "call 1f; 1:", this is what we will execute
623 * out-of-line if ->emulate() fails. We only need this to generate
624 * a trap, so that the probed task receives the correct signal with
625 * the properly filled siginfo.
626 *
627 * But see the comment in ->post_xol(), in the unlikely case it can
628 * succeed. So we need to ensure that the new ->ip can not fall into
629 * the non-canonical area and trigger #GP.
630 *
631 * We could turn it into (say) "pushf", but then we would need to
632 * divorce ->insn[] and ->ixol[]. We need to preserve the 1st byte
633 * of ->insn[] for set_orig_insn().
634 */
635 memset(auprobe->insn + insn_offset_immediate(insn),
636 0, insn->immediate.nbytes);
637}
638
Oleg Nesterov7ba6db22014-04-05 20:05:02 +0200639static struct uprobe_xol_ops branch_xol_ops = {
640 .emulate = branch_emulate_op,
Oleg Nesterov8e89c0b2014-04-06 18:11:02 +0200641 .post_xol = branch_post_xol_op,
Oleg Nesterov7ba6db22014-04-05 20:05:02 +0200642};
643
644/* Returns -ENOSYS if branch_xol_ops doesn't handle this insn */
645static int branch_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)
646{
Oleg Nesterov8e89c0b2014-04-06 18:11:02 +0200647 u8 opc1 = OPCODE1(insn);
Denys Vlasenko250bbd12014-04-24 19:08:24 +0200648 int i;
Oleg Nesterov7ba6db22014-04-05 20:05:02 +0200649
Oleg Nesterov8e89c0b2014-04-06 18:11:02 +0200650 switch (opc1) {
651 case 0xeb: /* jmp 8 */
652 case 0xe9: /* jmp 32 */
653 case 0x90: /* prefix* + nop; same as jmp with .offs = 0 */
654 break;
655
656 case 0xe8: /* call relative */
657 branch_clear_offset(auprobe, insn);
658 break;
Oleg Nesterov8f955052014-04-06 21:53:47 +0200659
Oleg Nesterov6cc5e7f2014-04-07 16:22:58 +0200660 case 0x0f:
661 if (insn->opcode.nbytes != 2)
662 return -ENOSYS;
663 /*
664 * If it is a "near" conditional jmp, OPCODE2() - 0x10 matches
665 * OPCODE1() of the "short" jmp which checks the same condition.
666 */
667 opc1 = OPCODE2(insn) - 0x10;
Oleg Nesterov8e89c0b2014-04-06 18:11:02 +0200668 default:
Oleg Nesterov8f955052014-04-06 21:53:47 +0200669 if (!is_cond_jmp_opcode(opc1))
670 return -ENOSYS;
Oleg Nesterov8e89c0b2014-04-06 18:11:02 +0200671 }
672
Denys Vlasenko250bbd12014-04-24 19:08:24 +0200673 /*
674 * 16-bit overrides such as CALLW (66 e8 nn nn) are not supported.
675 * Intel and AMD behavior differ in 64-bit mode: Intel ignores 66 prefix.
676 * No one uses these insns, reject any branch insns with such prefix.
677 */
678 for (i = 0; i < insn->prefixes.nbytes; i++) {
679 if (insn->prefixes.bytes[i] == 0x66)
680 return -ENOTSUPP;
681 }
682
Oleg Nesterov8e89c0b2014-04-06 18:11:02 +0200683 auprobe->branch.opc1 = opc1;
Oleg Nesterov7ba6db22014-04-05 20:05:02 +0200684 auprobe->branch.ilen = insn->length;
685 auprobe->branch.offs = insn->immediate.value;
686
687 auprobe->ops = &branch_xol_ops;
688 return 0;
689}
690
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530691/**
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530692 * arch_uprobe_analyze_insn - instruction analysis including validity and fixups.
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530693 * @mm: the probed address space.
Srikar Dronamraju3ff54ef2012-02-22 14:46:02 +0530694 * @arch_uprobe: the probepoint information.
Ananth N Mavinakayanahalli7eb9ba52012-06-08 15:02:57 +0530695 * @addr: virtual address at which to install the probepoint
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530696 * Return 0 on success or a -ve number on error.
697 */
Ananth N Mavinakayanahalli7eb9ba52012-06-08 15:02:57 +0530698int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr)
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530699{
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530700 struct insn insn;
Oleg Nesterov83cd5912014-04-25 18:53:32 +0200701 u8 fix_ip_or_call = UPROBE_FIX_IP;
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200702 int ret;
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530703
Oleg Nesterov2ae1f492014-04-19 14:03:05 +0200704 ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm));
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200705 if (ret)
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530706 return ret;
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100707
Oleg Nesterov7ba6db22014-04-05 20:05:02 +0200708 ret = branch_setup_xol_ops(auprobe, &insn);
709 if (ret != -ENOSYS)
710 return ret;
711
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200712 /*
Oleg Nesterov97aa5cd2014-04-22 16:20:55 +0200713 * Figure out which fixups default_post_xol_op() will need to perform,
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200714 * and annotate defparam->fixups accordingly.
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200715 */
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200716 switch (OPCODE1(&insn)) {
717 case 0x9d: /* popf */
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200718 auprobe->defparam.fixups |= UPROBE_FIX_SETF;
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200719 break;
720 case 0xc3: /* ret or lret -- ip is correct */
721 case 0xcb:
722 case 0xc2:
723 case 0xca:
Oleg Nesterov83cd5912014-04-25 18:53:32 +0200724 case 0xea: /* jmp absolute -- ip is correct */
725 fix_ip_or_call = 0;
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200726 break;
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200727 case 0x9a: /* call absolute - Fix return addr, not ip */
Oleg Nesterov83cd5912014-04-25 18:53:32 +0200728 fix_ip_or_call = UPROBE_FIX_CALL;
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200729 break;
730 case 0xff:
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200731 switch (MODRM_REG(&insn)) {
732 case 2: case 3: /* call or lcall, indirect */
Oleg Nesterov83cd5912014-04-25 18:53:32 +0200733 fix_ip_or_call = UPROBE_FIX_CALL;
734 break;
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200735 case 4: case 5: /* jmp or ljmp, indirect */
Oleg Nesterov83cd5912014-04-25 18:53:32 +0200736 fix_ip_or_call = 0;
737 break;
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200738 }
Oleg Nesterove55848a2014-03-31 17:24:14 +0200739 /* fall through */
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200740 default:
Oleg Nesterov1475ee72014-04-27 16:31:59 +0200741 riprel_analyze(auprobe, &insn);
Oleg Nesterovddb69f22014-03-31 15:16:22 +0200742 }
743
Oleg Nesterov5cdb76d2014-06-01 21:13:46 +0200744 auprobe->defparam.ilen = insn.length;
745 auprobe->defparam.fixups |= fix_ip_or_call;
Ingo Molnar7b2d81d2012-02-17 09:27:41 +0100746
Oleg Nesterov8ad8e9d2014-03-31 21:01:31 +0200747 auprobe->ops = &default_xol_ops;
Srikar Dronamraju2b144492012-02-09 14:56:42 +0530748 return 0;
749}
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530750
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530751/*
752 * arch_uprobe_pre_xol - prepare to execute out of line.
753 * @auprobe: the probepoint information.
754 * @regs: reflects the saved user state of current task.
755 */
756int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
757{
Oleg Nesterov34e73172014-03-31 19:38:09 +0200758 struct uprobe_task *utask = current->utask;
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530759
Oleg Nesterovdd910162014-04-22 15:20:07 +0200760 if (auprobe->ops->pre_xol) {
761 int err = auprobe->ops->pre_xol(auprobe, regs);
762 if (err)
763 return err;
764 }
765
Oleg Nesterov34e73172014-03-31 19:38:09 +0200766 regs->ip = utask->xol_vaddr;
767 utask->autask.saved_trap_nr = current->thread.trap_nr;
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530768 current->thread.trap_nr = UPROBE_TRAP_NR;
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530769
Oleg Nesterov34e73172014-03-31 19:38:09 +0200770 utask->autask.saved_tf = !!(regs->flags & X86_EFLAGS_TF);
Oleg Nesterov4dc316c2012-10-28 17:57:30 +0100771 regs->flags |= X86_EFLAGS_TF;
772 if (test_tsk_thread_flag(current, TIF_BLOCKSTEP))
773 set_task_blockstep(current, false);
774
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530775 return 0;
776}
777
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530778/*
779 * If xol insn itself traps and generates a signal(Say,
780 * SIGILL/SIGSEGV/etc), then detect the case where a singlestepped
781 * instruction jumps back to its own address. It is assumed that anything
782 * like do_page_fault/do_trap/etc sets thread.trap_nr != -1.
783 *
784 * arch_uprobe_pre_xol/arch_uprobe_post_xol save/restore thread.trap_nr,
785 * arch_uprobe_xol_was_trapped() simply checks that ->trap_nr is not equal to
786 * UPROBE_TRAP_NR == -1 set by arch_uprobe_pre_xol().
787 */
788bool arch_uprobe_xol_was_trapped(struct task_struct *t)
789{
790 if (t->thread.trap_nr != UPROBE_TRAP_NR)
791 return true;
792
793 return false;
794}
795
796/*
797 * Called after single-stepping. To avoid the SMP problems that can
798 * occur when we temporarily put back the original opcode to
799 * single-step, we single-stepped a copy of the instruction.
800 *
801 * This function prepares to resume execution after the single-step.
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530802 */
803int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
804{
Oleg Nesterov34e73172014-03-31 19:38:09 +0200805 struct uprobe_task *utask = current->utask;
Oleg Nesterov220ef8d2014-04-21 20:39:56 +0200806 bool send_sigtrap = utask->autask.saved_tf;
807 int err = 0;
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530808
809 WARN_ON_ONCE(current->thread.trap_nr != UPROBE_TRAP_NR);
Oleg Nesterov6ded5f32014-04-21 18:28:02 +0200810 current->thread.trap_nr = utask->autask.saved_trap_nr;
Oleg Nesterov014940b2014-04-03 20:20:10 +0200811
812 if (auprobe->ops->post_xol) {
Oleg Nesterov220ef8d2014-04-21 20:39:56 +0200813 err = auprobe->ops->post_xol(auprobe, regs);
Oleg Nesterov014940b2014-04-03 20:20:10 +0200814 if (err) {
Oleg Nesterov75f9ef02014-04-03 20:52:19 +0200815 /*
Oleg Nesterov6ded5f32014-04-21 18:28:02 +0200816 * Restore ->ip for restart or post mortem analysis.
817 * ->post_xol() must not return -ERESTART unless this
818 * is really possible.
Oleg Nesterov75f9ef02014-04-03 20:52:19 +0200819 */
Oleg Nesterov6ded5f32014-04-21 18:28:02 +0200820 regs->ip = utask->vaddr;
Oleg Nesterov75f9ef02014-04-03 20:52:19 +0200821 if (err == -ERESTART)
Oleg Nesterov220ef8d2014-04-21 20:39:56 +0200822 err = 0;
823 send_sigtrap = false;
Oleg Nesterov014940b2014-04-03 20:20:10 +0200824 }
825 }
Oleg Nesterov4dc316c2012-10-28 17:57:30 +0100826 /*
827 * arch_uprobe_pre_xol() doesn't save the state of TIF_BLOCKSTEP
828 * so we can get an extra SIGTRAP if we do not clear TF. We need
829 * to examine the opcode to make it right.
830 */
Oleg Nesterov220ef8d2014-04-21 20:39:56 +0200831 if (send_sigtrap)
Oleg Nesterov4dc316c2012-10-28 17:57:30 +0100832 send_sig(SIGTRAP, current, 0);
Oleg Nesterov220ef8d2014-04-21 20:39:56 +0200833
834 if (!utask->autask.saved_tf)
Oleg Nesterov4dc316c2012-10-28 17:57:30 +0100835 regs->flags &= ~X86_EFLAGS_TF;
836
Oleg Nesterov220ef8d2014-04-21 20:39:56 +0200837 return err;
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530838}
839
840/* callback routine for handling exceptions. */
841int arch_uprobe_exception_notify(struct notifier_block *self, unsigned long val, void *data)
842{
843 struct die_args *args = data;
844 struct pt_regs *regs = args->regs;
845 int ret = NOTIFY_DONE;
846
847 /* We are only interested in userspace traps */
848 if (regs && !user_mode_vm(regs))
849 return NOTIFY_DONE;
850
851 switch (val) {
852 case DIE_INT3:
853 if (uprobe_pre_sstep_notifier(regs))
854 ret = NOTIFY_STOP;
855
856 break;
857
858 case DIE_DEBUG:
859 if (uprobe_post_sstep_notifier(regs))
860 ret = NOTIFY_STOP;
861
862 default:
863 break;
864 }
865
866 return ret;
867}
868
869/*
870 * This function gets called when XOL instruction either gets trapped or
Oleg Nesterov6ded5f32014-04-21 18:28:02 +0200871 * the thread has a fatal signal. Reset the instruction pointer to its
872 * probed address for the potential restart or for post mortem analysis.
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530873 */
874void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
875{
876 struct uprobe_task *utask = current->utask;
877
Oleg Nesterov588fbd62014-04-21 16:58:17 +0200878 if (auprobe->ops->abort)
879 auprobe->ops->abort(auprobe, regs);
Oleg Nesterov4dc316c2012-10-28 17:57:30 +0100880
Oleg Nesterov588fbd62014-04-21 16:58:17 +0200881 current->thread.trap_nr = utask->autask.saved_trap_nr;
882 regs->ip = utask->vaddr;
Oleg Nesterov4dc316c2012-10-28 17:57:30 +0100883 /* clear TF if it was set by us in arch_uprobe_pre_xol() */
884 if (!utask->autask.saved_tf)
885 regs->flags &= ~X86_EFLAGS_TF;
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530886}
887
Oleg Nesterov3a4664a2012-09-03 16:05:10 +0200888static bool __skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530889{
Oleg Nesterov8ad8e9d2014-03-31 21:01:31 +0200890 if (auprobe->ops->emulate)
891 return auprobe->ops->emulate(auprobe, regs);
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530892 return false;
893}
Sebastian Andrzej Siewiorbdc1e472012-08-20 12:47:34 +0200894
Oleg Nesterov3a4664a2012-09-03 16:05:10 +0200895bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
896{
897 bool ret = __skip_sstep(auprobe, regs);
898 if (ret && (regs->flags & X86_EFLAGS_TF))
899 send_sig(SIGTRAP, current, 0);
900 return ret;
901}
Anton Arapov791eca12013-04-03 18:00:33 +0200902
903unsigned long
904arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, struct pt_regs *regs)
905{
Oleg Nesterov8faaed12014-04-06 17:16:10 +0200906 int rasize = sizeof_long(), nleft;
Anton Arapov791eca12013-04-03 18:00:33 +0200907 unsigned long orig_ret_vaddr = 0; /* clear high bits for 32-bit apps */
908
Oleg Nesterov8faaed12014-04-06 17:16:10 +0200909 if (copy_from_user(&orig_ret_vaddr, (void __user *)regs->sp, rasize))
Anton Arapov791eca12013-04-03 18:00:33 +0200910 return -1;
911
912 /* check whether address has been already hijacked */
913 if (orig_ret_vaddr == trampoline_vaddr)
914 return orig_ret_vaddr;
915
Oleg Nesterov8faaed12014-04-06 17:16:10 +0200916 nleft = copy_to_user((void __user *)regs->sp, &trampoline_vaddr, rasize);
917 if (likely(!nleft))
Anton Arapov791eca12013-04-03 18:00:33 +0200918 return orig_ret_vaddr;
919
Oleg Nesterov8faaed12014-04-06 17:16:10 +0200920 if (nleft != rasize) {
Anton Arapov791eca12013-04-03 18:00:33 +0200921 pr_err("uprobe: return address clobbered: pid=%d, %%sp=%#lx, "
922 "%%ip=%#lx\n", current->pid, regs->sp, regs->ip);
923
924 force_sig_info(SIGSEGV, SEND_SIG_FORCED, current);
925 }
926
927 return -1;
928}