LLVM API Documentation

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
X86InstrInfo.cpp
Go to the documentation of this file.
1 //===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the X86 implementation of the TargetInstrInfo class.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "X86InstrInfo.h"
15 #include "X86.h"
16 #include "X86InstrBuilder.h"
17 #include "X86MachineFunctionInfo.h"
18 #include "X86Subtarget.h"
19 #include "X86TargetMachine.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/CodeGen/LiveVariables.h"
22 #include "llvm/CodeGen/MachineConstantPool.h"
23 #include "llvm/CodeGen/MachineDominators.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineInstrBuilder.h"
26 #include "llvm/CodeGen/MachineRegisterInfo.h"
27 #include "llvm/CodeGen/StackMaps.h"
28 #include "llvm/IR/DerivedTypes.h"
29 #include "llvm/IR/LLVMContext.h"
30 #include "llvm/MC/MCAsmInfo.h"
31 #include "llvm/MC/MCInst.h"
32 #include "llvm/Support/CommandLine.h"
33 #include "llvm/Support/Debug.h"
34 #include "llvm/Support/ErrorHandling.h"
35 #include "llvm/Support/raw_ostream.h"
36 #include "llvm/Target/TargetOptions.h"
37 #include <limits>
38 
39 #define GET_INSTRINFO_CTOR_DTOR
40 #include "X86GenInstrInfo.inc"
41 
42 using namespace llvm;
43 
44 static cl::opt<bool>
45 NoFusing("disable-spill-fusing",
46  cl::desc("Disable fusing of spill code into instructions"));
47 static cl::opt<bool>
48 PrintFailedFusing("print-failed-fuse-candidates",
49  cl::desc("Print instructions that the allocator wants to"
50  " fuse, but the X86 backend currently can't"),
51  cl::Hidden);
52 static cl::opt<bool>
53 ReMatPICStubLoad("remat-pic-stub-load",
54  cl::desc("Re-materialize load from stub in PIC mode"),
55  cl::init(false), cl::Hidden);
56 
57 enum {
58  // Select which memory operand is being unfolded.
59  // (stored in bits 0 - 3)
60  TB_INDEX_0 = 0,
61  TB_INDEX_1 = 1,
62  TB_INDEX_2 = 2,
63  TB_INDEX_3 = 3,
64  TB_INDEX_MASK = 0xf,
65 
66  // Do not insert the reverse map (MemOp -> RegOp) into the table.
67  // This may be needed because there is a many -> one mapping.
68  TB_NO_REVERSE = 1 << 4,
69 
70  // Do not insert the forward map (RegOp -> MemOp) into the table.
71  // This is needed for Native Client, which prohibits branch
72  // instructions from using a memory operand.
73  TB_NO_FORWARD = 1 << 5,
74 
75  TB_FOLDED_LOAD = 1 << 6,
76  TB_FOLDED_STORE = 1 << 7,
77 
78  // Minimum alignment required for load/store.
79  // Used for RegOp->MemOp conversion.
80  // (stored in bits 8 - 15)
81  TB_ALIGN_SHIFT = 8,
82  TB_ALIGN_NONE = 0 << TB_ALIGN_SHIFT,
83  TB_ALIGN_16 = 16 << TB_ALIGN_SHIFT,
84  TB_ALIGN_32 = 32 << TB_ALIGN_SHIFT,
85  TB_ALIGN_64 = 64 << TB_ALIGN_SHIFT,
86  TB_ALIGN_MASK = 0xff << TB_ALIGN_SHIFT
87 };
88 
89 struct X86OpTblEntry {
90  uint16_t RegOp;
91  uint16_t MemOp;
92  uint16_t Flags;
93 };
94 
95 // Pin the vtable to this file.
96 void X86InstrInfo::anchor() {}
97 
98 X86InstrInfo::X86InstrInfo(X86TargetMachine &tm)
99  : X86GenInstrInfo((tm.getSubtarget<X86Subtarget>().is64Bit()
100  ? X86::ADJCALLSTACKDOWN64
101  : X86::ADJCALLSTACKDOWN32),
102  (tm.getSubtarget<X86Subtarget>().is64Bit()
103  ? X86::ADJCALLSTACKUP64
104  : X86::ADJCALLSTACKUP32)),
105  TM(tm), RI(tm) {
106 
107  static const X86OpTblEntry OpTbl2Addr[] = {
108  { X86::ADC32ri, X86::ADC32mi, 0 },
109  { X86::ADC32ri8, X86::ADC32mi8, 0 },
110  { X86::ADC32rr, X86::ADC32mr, 0 },
111  { X86::ADC64ri32, X86::ADC64mi32, 0 },
112  { X86::ADC64ri8, X86::ADC64mi8, 0 },
113  { X86::ADC64rr, X86::ADC64mr, 0 },
114  { X86::ADD16ri, X86::ADD16mi, 0 },
115  { X86::ADD16ri8, X86::ADD16mi8, 0 },
116  { X86::ADD16ri_DB, X86::ADD16mi, TB_NO_REVERSE },
117  { X86::ADD16ri8_DB, X86::ADD16mi8, TB_NO_REVERSE },
118  { X86::ADD16rr, X86::ADD16mr, 0 },
119  { X86::ADD16rr_DB, X86::ADD16mr, TB_NO_REVERSE },
120  { X86::ADD32ri, X86::ADD32mi, 0 },
121  { X86::ADD32ri8, X86::ADD32mi8, 0 },
122  { X86::ADD32ri_DB, X86::ADD32mi, TB_NO_REVERSE },
123  { X86::ADD32ri8_DB, X86::ADD32mi8, TB_NO_REVERSE },
124  { X86::ADD32rr, X86::ADD32mr, 0 },
125  { X86::ADD32rr_DB, X86::ADD32mr, TB_NO_REVERSE },
126  { X86::ADD64ri32, X86::ADD64mi32, 0 },
127  { X86::ADD64ri8, X86::ADD64mi8, 0 },
128  { X86::ADD64ri32_DB,X86::ADD64mi32, TB_NO_REVERSE },
129  { X86::ADD64ri8_DB, X86::ADD64mi8, TB_NO_REVERSE },
130  { X86::ADD64rr, X86::ADD64mr, 0 },
131  { X86::ADD64rr_DB, X86::ADD64mr, TB_NO_REVERSE },
132  { X86::ADD8ri, X86::ADD8mi, 0 },
133  { X86::ADD8rr, X86::ADD8mr, 0 },
134  { X86::AND16ri, X86::AND16mi, 0 },
135  { X86::AND16ri8, X86::AND16mi8, 0 },
136  { X86::AND16rr, X86::AND16mr, 0 },
137  { X86::AND32ri, X86::AND32mi, 0 },
138  { X86::AND32ri8, X86::AND32mi8, 0 },
139  { X86::AND32rr, X86::AND32mr, 0 },
140  { X86::AND64ri32, X86::AND64mi32, 0 },
141  { X86::AND64ri8, X86::AND64mi8, 0 },
142  { X86::AND64rr, X86::AND64mr, 0 },
143  { X86::AND8ri, X86::AND8mi, 0 },
144  { X86::AND8rr, X86::AND8mr, 0 },
145  { X86::DEC16r, X86::DEC16m, 0 },
146  { X86::DEC32r, X86::DEC32m, 0 },
147  { X86::DEC64_16r, X86::DEC64_16m, 0 },
148  { X86::DEC64_32r, X86::DEC64_32m, 0 },
149  { X86::DEC64r, X86::DEC64m, 0 },
150  { X86::DEC8r, X86::DEC8m, 0 },
151  { X86::INC16r, X86::INC16m, 0 },
152  { X86::INC32r, X86::INC32m, 0 },
153  { X86::INC64_16r, X86::INC64_16m, 0 },
154  { X86::INC64_32r, X86::INC64_32m, 0 },
155  { X86::INC64r, X86::INC64m, 0 },
156  { X86::INC8r, X86::INC8m, 0 },
157  { X86::NEG16r, X86::NEG16m, 0 },
158  { X86::NEG32r, X86::NEG32m, 0 },
159  { X86::NEG64r, X86::NEG64m, 0 },
160  { X86::NEG8r, X86::NEG8m, 0 },
161  { X86::NOT16r, X86::NOT16m, 0 },
162  { X86::NOT32r, X86::NOT32m, 0 },
163  { X86::NOT64r, X86::NOT64m, 0 },
164  { X86::NOT8r, X86::NOT8m, 0 },
165  { X86::OR16ri, X86::OR16mi, 0 },
166  { X86::OR16ri8, X86::OR16mi8, 0 },
167  { X86::OR16rr, X86::OR16mr, 0 },
168  { X86::OR32ri, X86::OR32mi, 0 },
169  { X86::OR32ri8, X86::OR32mi8, 0 },
170  { X86::OR32rr, X86::OR32mr, 0 },
171  { X86::OR64ri32, X86::OR64mi32, 0 },
172  { X86::OR64ri8, X86::OR64mi8, 0 },
173  { X86::OR64rr, X86::OR64mr, 0 },
174  { X86::OR8ri, X86::OR8mi, 0 },
175  { X86::OR8rr, X86::OR8mr, 0 },
176  { X86::ROL16r1, X86::ROL16m1, 0 },
177  { X86::ROL16rCL, X86::ROL16mCL, 0 },
178  { X86::ROL16ri, X86::ROL16mi, 0 },
179  { X86::ROL32r1, X86::ROL32m1, 0 },
180  { X86::ROL32rCL, X86::ROL32mCL, 0 },
181  { X86::ROL32ri, X86::ROL32mi, 0 },
182  { X86::ROL64r1, X86::ROL64m1, 0 },
183  { X86::ROL64rCL, X86::ROL64mCL, 0 },
184  { X86::ROL64ri, X86::ROL64mi, 0 },
185  { X86::ROL8r1, X86::ROL8m1, 0 },
186  { X86::ROL8rCL, X86::ROL8mCL, 0 },
187  { X86::ROL8ri, X86::ROL8mi, 0 },
188  { X86::ROR16r1, X86::ROR16m1, 0 },
189  { X86::ROR16rCL, X86::ROR16mCL, 0 },
190  { X86::ROR16ri, X86::ROR16mi, 0 },
191  { X86::ROR32r1, X86::ROR32m1, 0 },
192  { X86::ROR32rCL, X86::ROR32mCL, 0 },
193  { X86::ROR32ri, X86::ROR32mi, 0 },
194  { X86::ROR64r1, X86::ROR64m1, 0 },
195  { X86::ROR64rCL, X86::ROR64mCL, 0 },
196  { X86::ROR64ri, X86::ROR64mi, 0 },
197  { X86::ROR8r1, X86::ROR8m1, 0 },
198  { X86::ROR8rCL, X86::ROR8mCL, 0 },
199  { X86::ROR8ri, X86::ROR8mi, 0 },
200  { X86::SAR16r1, X86::SAR16m1, 0 },
201  { X86::SAR16rCL, X86::SAR16mCL, 0 },
202  { X86::SAR16ri, X86::SAR16mi, 0 },
203  { X86::SAR32r1, X86::SAR32m1, 0 },
204  { X86::SAR32rCL, X86::SAR32mCL, 0 },
205  { X86::SAR32ri, X86::SAR32mi, 0 },
206  { X86::SAR64r1, X86::SAR64m1, 0 },
207  { X86::SAR64rCL, X86::SAR64mCL, 0 },
208  { X86::SAR64ri, X86::SAR64mi, 0 },
209  { X86::SAR8r1, X86::SAR8m1, 0 },
210  { X86::SAR8rCL, X86::SAR8mCL, 0 },
211  { X86::SAR8ri, X86::SAR8mi, 0 },
212  { X86::SBB32ri, X86::SBB32mi, 0 },
213  { X86::SBB32ri8, X86::SBB32mi8, 0 },
214  { X86::SBB32rr, X86::SBB32mr, 0 },
215  { X86::SBB64ri32, X86::SBB64mi32, 0 },
216  { X86::SBB64ri8, X86::SBB64mi8, 0 },
217  { X86::SBB64rr, X86::SBB64mr, 0 },
218  { X86::SHL16rCL, X86::SHL16mCL, 0 },
219  { X86::SHL16ri, X86::SHL16mi, 0 },
220  { X86::SHL32rCL, X86::SHL32mCL, 0 },
221  { X86::SHL32ri, X86::SHL32mi, 0 },
222  { X86::SHL64rCL, X86::SHL64mCL, 0 },
223  { X86::SHL64ri, X86::SHL64mi, 0 },
224  { X86::SHL8rCL, X86::SHL8mCL, 0 },
225  { X86::SHL8ri, X86::SHL8mi, 0 },
226  { X86::SHLD16rrCL, X86::SHLD16mrCL, 0 },
227  { X86::SHLD16rri8, X86::SHLD16mri8, 0 },
228  { X86::SHLD32rrCL, X86::SHLD32mrCL, 0 },
229  { X86::SHLD32rri8, X86::SHLD32mri8, 0 },
230  { X86::SHLD64rrCL, X86::SHLD64mrCL, 0 },
231  { X86::SHLD64rri8, X86::SHLD64mri8, 0 },
232  { X86::SHR16r1, X86::SHR16m1, 0 },
233  { X86::SHR16rCL, X86::SHR16mCL, 0 },
234  { X86::SHR16ri, X86::SHR16mi, 0 },
235  { X86::SHR32r1, X86::SHR32m1, 0 },
236  { X86::SHR32rCL, X86::SHR32mCL, 0 },
237  { X86::SHR32ri, X86::SHR32mi, 0 },
238  { X86::SHR64r1, X86::SHR64m1, 0 },
239  { X86::SHR64rCL, X86::SHR64mCL, 0 },
240  { X86::SHR64ri, X86::SHR64mi, 0 },
241  { X86::SHR8r1, X86::SHR8m1, 0 },
242  { X86::SHR8rCL, X86::SHR8mCL, 0 },
243  { X86::SHR8ri, X86::SHR8mi, 0 },
244  { X86::SHRD16rrCL, X86::SHRD16mrCL, 0 },
245  { X86::SHRD16rri8, X86::SHRD16mri8, 0 },
246  { X86::SHRD32rrCL, X86::SHRD32mrCL, 0 },
247  { X86::SHRD32rri8, X86::SHRD32mri8, 0 },
248  { X86::SHRD64rrCL, X86::SHRD64mrCL, 0 },
249  { X86::SHRD64rri8, X86::SHRD64mri8, 0 },
250  { X86::SUB16ri, X86::SUB16mi, 0 },
251  { X86::SUB16ri8, X86::SUB16mi8, 0 },
252  { X86::SUB16rr, X86::SUB16mr, 0 },
253  { X86::SUB32ri, X86::SUB32mi, 0 },
254  { X86::SUB32ri8, X86::SUB32mi8, 0 },
255  { X86::SUB32rr, X86::SUB32mr, 0 },
256  { X86::SUB64ri32, X86::SUB64mi32, 0 },
257  { X86::SUB64ri8, X86::SUB64mi8, 0 },
258  { X86::SUB64rr, X86::SUB64mr, 0 },
259  { X86::SUB8ri, X86::SUB8mi, 0 },
260  { X86::SUB8rr, X86::SUB8mr, 0 },
261  { X86::XOR16ri, X86::XOR16mi, 0 },
262  { X86::XOR16ri8, X86::XOR16mi8, 0 },
263  { X86::XOR16rr, X86::XOR16mr, 0 },
264  { X86::XOR32ri, X86::XOR32mi, 0 },
265  { X86::XOR32ri8, X86::XOR32mi8, 0 },
266  { X86::XOR32rr, X86::XOR32mr, 0 },
267  { X86::XOR64ri32, X86::XOR64mi32, 0 },
268  { X86::XOR64ri8, X86::XOR64mi8, 0 },
269  { X86::XOR64rr, X86::XOR64mr, 0 },
270  { X86::XOR8ri, X86::XOR8mi, 0 },
271  { X86::XOR8rr, X86::XOR8mr, 0 }
272  };
273 
274  for (unsigned i = 0, e = array_lengthof(OpTbl2Addr); i != e; ++i) {
275  unsigned RegOp = OpTbl2Addr[i].RegOp;
276  unsigned MemOp = OpTbl2Addr[i].MemOp;
277  unsigned Flags = OpTbl2Addr[i].Flags;
278  AddTableEntry(RegOp2MemOpTable2Addr, MemOp2RegOpTable,
279  RegOp, MemOp,
280  // Index 0, folded load and store, no alignment requirement.
281  Flags | TB_INDEX_0 | TB_FOLDED_LOAD | TB_FOLDED_STORE);
282  }
283 
284  static const X86OpTblEntry OpTbl0[] = {
285  { X86::BT16ri8, X86::BT16mi8, TB_FOLDED_LOAD },
286  { X86::BT32ri8, X86::BT32mi8, TB_FOLDED_LOAD },
287  { X86::BT64ri8, X86::BT64mi8, TB_FOLDED_LOAD },
288  { X86::CALL32r, X86::CALL32m, TB_FOLDED_LOAD },
289  { X86::CALL64r, X86::CALL64m, TB_FOLDED_LOAD },
290  { X86::CMP16ri, X86::CMP16mi, TB_FOLDED_LOAD },
291  { X86::CMP16ri8, X86::CMP16mi8, TB_FOLDED_LOAD },
292  { X86::CMP16rr, X86::CMP16mr, TB_FOLDED_LOAD },
293  { X86::CMP32ri, X86::CMP32mi, TB_FOLDED_LOAD },
294  { X86::CMP32ri8, X86::CMP32mi8, TB_FOLDED_LOAD },
295  { X86::CMP32rr, X86::CMP32mr, TB_FOLDED_LOAD },
296  { X86::CMP64ri32, X86::CMP64mi32, TB_FOLDED_LOAD },
297  { X86::CMP64ri8, X86::CMP64mi8, TB_FOLDED_LOAD },
298  { X86::CMP64rr, X86::CMP64mr, TB_FOLDED_LOAD },
299  { X86::CMP8ri, X86::CMP8mi, TB_FOLDED_LOAD },
300  { X86::CMP8rr, X86::CMP8mr, TB_FOLDED_LOAD },
301  { X86::DIV16r, X86::DIV16m, TB_FOLDED_LOAD },
302  { X86::DIV32r, X86::DIV32m, TB_FOLDED_LOAD },
303  { X86::DIV64r, X86::DIV64m, TB_FOLDED_LOAD },
304  { X86::DIV8r, X86::DIV8m, TB_FOLDED_LOAD },
305  { X86::EXTRACTPSrr, X86::EXTRACTPSmr, TB_FOLDED_STORE },
306  { X86::IDIV16r, X86::IDIV16m, TB_FOLDED_LOAD },
307  { X86::IDIV32r, X86::IDIV32m, TB_FOLDED_LOAD },
308  { X86::IDIV64r, X86::IDIV64m, TB_FOLDED_LOAD },
309  { X86::IDIV8r, X86::IDIV8m, TB_FOLDED_LOAD },
310  { X86::IMUL16r, X86::IMUL16m, TB_FOLDED_LOAD },
311  { X86::IMUL32r, X86::IMUL32m, TB_FOLDED_LOAD },
312  { X86::IMUL64r, X86::IMUL64m, TB_FOLDED_LOAD },
313  { X86::IMUL8r, X86::IMUL8m, TB_FOLDED_LOAD },
314  { X86::JMP32r, X86::JMP32m, TB_FOLDED_LOAD },
315  { X86::JMP64r, X86::JMP64m, TB_FOLDED_LOAD },
316  { X86::MOV16ri, X86::MOV16mi, TB_FOLDED_STORE },
317  { X86::MOV16rr, X86::MOV16mr, TB_FOLDED_STORE },
318  { X86::MOV32ri, X86::MOV32mi, TB_FOLDED_STORE },
319  { X86::MOV32rr, X86::MOV32mr, TB_FOLDED_STORE },
320  { X86::MOV64ri32, X86::MOV64mi32, TB_FOLDED_STORE },
321  { X86::MOV64rr, X86::MOV64mr, TB_FOLDED_STORE },
322  { X86::MOV8ri, X86::MOV8mi, TB_FOLDED_STORE },
323  { X86::MOV8rr, X86::MOV8mr, TB_FOLDED_STORE },
324  { X86::MOV8rr_NOREX, X86::MOV8mr_NOREX, TB_FOLDED_STORE },
325  { X86::MOVAPDrr, X86::MOVAPDmr, TB_FOLDED_STORE | TB_ALIGN_16 },
326  { X86::MOVAPSrr, X86::MOVAPSmr, TB_FOLDED_STORE | TB_ALIGN_16 },
327  { X86::MOVDQArr, X86::MOVDQAmr, TB_FOLDED_STORE | TB_ALIGN_16 },
328  { X86::MOVPDI2DIrr, X86::MOVPDI2DImr, TB_FOLDED_STORE },
329  { X86::MOVPQIto64rr,X86::MOVPQI2QImr, TB_FOLDED_STORE },
330  { X86::MOVSDto64rr, X86::MOVSDto64mr, TB_FOLDED_STORE },
331  { X86::MOVSS2DIrr, X86::MOVSS2DImr, TB_FOLDED_STORE },
332  { X86::MOVUPDrr, X86::MOVUPDmr, TB_FOLDED_STORE },
333  { X86::MOVUPSrr, X86::MOVUPSmr, TB_FOLDED_STORE },
334  { X86::MUL16r, X86::MUL16m, TB_FOLDED_LOAD },
335  { X86::MUL32r, X86::MUL32m, TB_FOLDED_LOAD },
336  { X86::MUL64r, X86::MUL64m, TB_FOLDED_LOAD },
337  { X86::MUL8r, X86::MUL8m, TB_FOLDED_LOAD },
338  { X86::SETAEr, X86::SETAEm, TB_FOLDED_STORE },
339  { X86::SETAr, X86::SETAm, TB_FOLDED_STORE },
340  { X86::SETBEr, X86::SETBEm, TB_FOLDED_STORE },
341  { X86::SETBr, X86::SETBm, TB_FOLDED_STORE },
342  { X86::SETEr, X86::SETEm, TB_FOLDED_STORE },
343  { X86::SETGEr, X86::SETGEm, TB_FOLDED_STORE },
344  { X86::SETGr, X86::SETGm, TB_FOLDED_STORE },
345  { X86::SETLEr, X86::SETLEm, TB_FOLDED_STORE },
346  { X86::SETLr, X86::SETLm, TB_FOLDED_STORE },
347  { X86::SETNEr, X86::SETNEm, TB_FOLDED_STORE },
348  { X86::SETNOr, X86::SETNOm, TB_FOLDED_STORE },
349  { X86::SETNPr, X86::SETNPm, TB_FOLDED_STORE },
350  { X86::SETNSr, X86::SETNSm, TB_FOLDED_STORE },
351  { X86::SETOr, X86::SETOm, TB_FOLDED_STORE },
352  { X86::SETPr, X86::SETPm, TB_FOLDED_STORE },
353  { X86::SETSr, X86::SETSm, TB_FOLDED_STORE },
354  { X86::TAILJMPr, X86::TAILJMPm, TB_FOLDED_LOAD },
355  { X86::TAILJMPr64, X86::TAILJMPm64, TB_FOLDED_LOAD },
356  { X86::TEST16ri, X86::TEST16mi, TB_FOLDED_LOAD },
357  { X86::TEST32ri, X86::TEST32mi, TB_FOLDED_LOAD },
358  { X86::TEST64ri32, X86::TEST64mi32, TB_FOLDED_LOAD },
359  { X86::TEST8ri, X86::TEST8mi, TB_FOLDED_LOAD },
360  // AVX 128-bit versions of foldable instructions
361  { X86::VEXTRACTPSrr,X86::VEXTRACTPSmr, TB_FOLDED_STORE },
362  { X86::VEXTRACTF128rr, X86::VEXTRACTF128mr, TB_FOLDED_STORE | TB_ALIGN_16 },
363  { X86::VMOVAPDrr, X86::VMOVAPDmr, TB_FOLDED_STORE | TB_ALIGN_16 },
364  { X86::VMOVAPSrr, X86::VMOVAPSmr, TB_FOLDED_STORE | TB_ALIGN_16 },
365  { X86::VMOVDQArr, X86::VMOVDQAmr, TB_FOLDED_STORE | TB_ALIGN_16 },
366  { X86::VMOVPDI2DIrr,X86::VMOVPDI2DImr, TB_FOLDED_STORE },
367  { X86::VMOVPQIto64rr, X86::VMOVPQI2QImr,TB_FOLDED_STORE },
368  { X86::VMOVSDto64rr,X86::VMOVSDto64mr, TB_FOLDED_STORE },
369  { X86::VMOVSS2DIrr, X86::VMOVSS2DImr, TB_FOLDED_STORE },
370  { X86::VMOVUPDrr, X86::VMOVUPDmr, TB_FOLDED_STORE },
371  { X86::VMOVUPSrr, X86::VMOVUPSmr, TB_FOLDED_STORE },
372  // AVX 256-bit foldable instructions
373  { X86::VEXTRACTI128rr, X86::VEXTRACTI128mr, TB_FOLDED_STORE | TB_ALIGN_16 },
374  { X86::VMOVAPDYrr, X86::VMOVAPDYmr, TB_FOLDED_STORE | TB_ALIGN_32 },
375  { X86::VMOVAPSYrr, X86::VMOVAPSYmr, TB_FOLDED_STORE | TB_ALIGN_32 },
376  { X86::VMOVDQAYrr, X86::VMOVDQAYmr, TB_FOLDED_STORE | TB_ALIGN_32 },
377  { X86::VMOVUPDYrr, X86::VMOVUPDYmr, TB_FOLDED_STORE },
378  { X86::VMOVUPSYrr, X86::VMOVUPSYmr, TB_FOLDED_STORE },
379  // AVX-512 foldable instructions
380  { X86::VMOVPDI2DIZrr,X86::VMOVPDI2DIZmr, TB_FOLDED_STORE }
381  };
382 
383  for (unsigned i = 0, e = array_lengthof(OpTbl0); i != e; ++i) {
384  unsigned RegOp = OpTbl0[i].RegOp;
385  unsigned MemOp = OpTbl0[i].MemOp;
386  unsigned Flags = OpTbl0[i].Flags;
387  AddTableEntry(RegOp2MemOpTable0, MemOp2RegOpTable,
388  RegOp, MemOp, TB_INDEX_0 | Flags);
389  }
390 
391  static const X86OpTblEntry OpTbl1[] = {
392  { X86::CMP16rr, X86::CMP16rm, 0 },
393  { X86::CMP32rr, X86::CMP32rm, 0 },
394  { X86::CMP64rr, X86::CMP64rm, 0 },
395  { X86::CMP8rr, X86::CMP8rm, 0 },
396  { X86::CVTSD2SSrr, X86::CVTSD2SSrm, 0 },
397  { X86::CVTSI2SD64rr, X86::CVTSI2SD64rm, 0 },
398  { X86::CVTSI2SDrr, X86::CVTSI2SDrm, 0 },
399  { X86::CVTSI2SS64rr, X86::CVTSI2SS64rm, 0 },
400  { X86::CVTSI2SSrr, X86::CVTSI2SSrm, 0 },
401  { X86::CVTSS2SDrr, X86::CVTSS2SDrm, 0 },
402  { X86::CVTTSD2SI64rr, X86::CVTTSD2SI64rm, 0 },
403  { X86::CVTTSD2SIrr, X86::CVTTSD2SIrm, 0 },
404  { X86::CVTTSS2SI64rr, X86::CVTTSS2SI64rm, 0 },
405  { X86::CVTTSS2SIrr, X86::CVTTSS2SIrm, 0 },
406  { X86::IMUL16rri, X86::IMUL16rmi, 0 },
407  { X86::IMUL16rri8, X86::IMUL16rmi8, 0 },
408  { X86::IMUL32rri, X86::IMUL32rmi, 0 },
409  { X86::IMUL32rri8, X86::IMUL32rmi8, 0 },
410  { X86::IMUL64rri32, X86::IMUL64rmi32, 0 },
411  { X86::IMUL64rri8, X86::IMUL64rmi8, 0 },
412  { X86::Int_COMISDrr, X86::Int_COMISDrm, 0 },
413  { X86::Int_COMISSrr, X86::Int_COMISSrm, 0 },
414  { X86::CVTSD2SI64rr, X86::CVTSD2SI64rm, 0 },
415  { X86::CVTSD2SIrr, X86::CVTSD2SIrm, 0 },
416  { X86::CVTSS2SI64rr, X86::CVTSS2SI64rm, 0 },
417  { X86::CVTSS2SIrr, X86::CVTSS2SIrm, 0 },
418  { X86::CVTTPD2DQrr, X86::CVTTPD2DQrm, TB_ALIGN_16 },
419  { X86::CVTTPS2DQrr, X86::CVTTPS2DQrm, TB_ALIGN_16 },
420  { X86::Int_CVTTSD2SI64rr,X86::Int_CVTTSD2SI64rm, 0 },
421  { X86::Int_CVTTSD2SIrr, X86::Int_CVTTSD2SIrm, 0 },
422  { X86::Int_CVTTSS2SI64rr,X86::Int_CVTTSS2SI64rm, 0 },
423  { X86::Int_CVTTSS2SIrr, X86::Int_CVTTSS2SIrm, 0 },
424  { X86::Int_UCOMISDrr, X86::Int_UCOMISDrm, 0 },
425  { X86::Int_UCOMISSrr, X86::Int_UCOMISSrm, 0 },
426  { X86::MOV16rr, X86::MOV16rm, 0 },
427  { X86::MOV32rr, X86::MOV32rm, 0 },
428  { X86::MOV64rr, X86::MOV64rm, 0 },
429  { X86::MOV64toPQIrr, X86::MOVQI2PQIrm, 0 },
430  { X86::MOV64toSDrr, X86::MOV64toSDrm, 0 },
431  { X86::MOV8rr, X86::MOV8rm, 0 },
432  { X86::MOVAPDrr, X86::MOVAPDrm, TB_ALIGN_16 },
433  { X86::MOVAPSrr, X86::MOVAPSrm, TB_ALIGN_16 },
434  { X86::MOVDDUPrr, X86::MOVDDUPrm, 0 },
435  { X86::MOVDI2PDIrr, X86::MOVDI2PDIrm, 0 },
436  { X86::MOVDI2SSrr, X86::MOVDI2SSrm, 0 },
437  { X86::MOVDQArr, X86::MOVDQArm, TB_ALIGN_16 },
438  { X86::MOVSHDUPrr, X86::MOVSHDUPrm, TB_ALIGN_16 },
439  { X86::MOVSLDUPrr, X86::MOVSLDUPrm, TB_ALIGN_16 },
440  { X86::MOVSX16rr8, X86::MOVSX16rm8, 0 },
441  { X86::MOVSX32rr16, X86::MOVSX32rm16, 0 },
442  { X86::MOVSX32rr8, X86::MOVSX32rm8, 0 },
443  { X86::MOVSX64rr16, X86::MOVSX64rm16, 0 },
444  { X86::MOVSX64rr32, X86::MOVSX64rm32, 0 },
445  { X86::MOVSX64rr8, X86::MOVSX64rm8, 0 },
446  { X86::MOVUPDrr, X86::MOVUPDrm, TB_ALIGN_16 },
447  { X86::MOVUPSrr, X86::MOVUPSrm, 0 },
448  { X86::MOVZQI2PQIrr, X86::MOVZQI2PQIrm, 0 },
449  { X86::MOVZPQILo2PQIrr, X86::MOVZPQILo2PQIrm, TB_ALIGN_16 },
450  { X86::MOVZX16rr8, X86::MOVZX16rm8, 0 },
451  { X86::MOVZX32rr16, X86::MOVZX32rm16, 0 },
452  { X86::MOVZX32_NOREXrr8, X86::MOVZX32_NOREXrm8, 0 },
453  { X86::MOVZX32rr8, X86::MOVZX32rm8, 0 },
454  { X86::PABSBrr128, X86::PABSBrm128, TB_ALIGN_16 },
455  { X86::PABSDrr128, X86::PABSDrm128, TB_ALIGN_16 },
456  { X86::PABSWrr128, X86::PABSWrm128, TB_ALIGN_16 },
457  { X86::PSHUFDri, X86::PSHUFDmi, TB_ALIGN_16 },
458  { X86::PSHUFHWri, X86::PSHUFHWmi, TB_ALIGN_16 },
459  { X86::PSHUFLWri, X86::PSHUFLWmi, TB_ALIGN_16 },
460  { X86::RCPPSr, X86::RCPPSm, TB_ALIGN_16 },
461  { X86::RCPPSr_Int, X86::RCPPSm_Int, TB_ALIGN_16 },
462  { X86::RSQRTPSr, X86::RSQRTPSm, TB_ALIGN_16 },
463  { X86::RSQRTPSr_Int, X86::RSQRTPSm_Int, TB_ALIGN_16 },
464  { X86::RSQRTSSr, X86::RSQRTSSm, 0 },
465  { X86::RSQRTSSr_Int, X86::RSQRTSSm_Int, 0 },
466  { X86::SQRTPDr, X86::SQRTPDm, TB_ALIGN_16 },
467  { X86::SQRTPSr, X86::SQRTPSm, TB_ALIGN_16 },
468  { X86::SQRTSDr, X86::SQRTSDm, 0 },
469  { X86::SQRTSDr_Int, X86::SQRTSDm_Int, 0 },
470  { X86::SQRTSSr, X86::SQRTSSm, 0 },
471  { X86::SQRTSSr_Int, X86::SQRTSSm_Int, 0 },
472  { X86::TEST16rr, X86::TEST16rm, 0 },
473  { X86::TEST32rr, X86::TEST32rm, 0 },
474  { X86::TEST64rr, X86::TEST64rm, 0 },
475  { X86::TEST8rr, X86::TEST8rm, 0 },
476  // FIXME: TEST*rr EAX,EAX ---> CMP [mem], 0
477  { X86::UCOMISDrr, X86::UCOMISDrm, 0 },
478  { X86::UCOMISSrr, X86::UCOMISSrm, 0 },
479  // AVX 128-bit versions of foldable instructions
480  { X86::Int_VCOMISDrr, X86::Int_VCOMISDrm, 0 },
481  { X86::Int_VCOMISSrr, X86::Int_VCOMISSrm, 0 },
482  { X86::Int_VUCOMISDrr, X86::Int_VUCOMISDrm, 0 },
483  { X86::Int_VUCOMISSrr, X86::Int_VUCOMISSrm, 0 },
484  { X86::VCVTTSD2SI64rr, X86::VCVTTSD2SI64rm, 0 },
485  { X86::Int_VCVTTSD2SI64rr,X86::Int_VCVTTSD2SI64rm,0 },
486  { X86::VCVTTSD2SIrr, X86::VCVTTSD2SIrm, 0 },
487  { X86::Int_VCVTTSD2SIrr,X86::Int_VCVTTSD2SIrm, 0 },
488  { X86::VCVTTSS2SI64rr, X86::VCVTTSS2SI64rm, 0 },
489  { X86::Int_VCVTTSS2SI64rr,X86::Int_VCVTTSS2SI64rm,0 },
490  { X86::VCVTTSS2SIrr, X86::VCVTTSS2SIrm, 0 },
491  { X86::Int_VCVTTSS2SIrr,X86::Int_VCVTTSS2SIrm, 0 },
492  { X86::VCVTSD2SI64rr, X86::VCVTSD2SI64rm, 0 },
493  { X86::VCVTSD2SIrr, X86::VCVTSD2SIrm, 0 },
494  { X86::VCVTSS2SI64rr, X86::VCVTSS2SI64rm, 0 },
495  { X86::VCVTSS2SIrr, X86::VCVTSS2SIrm, 0 },
496  { X86::VMOV64toPQIrr, X86::VMOVQI2PQIrm, 0 },
497  { X86::VMOV64toSDrr, X86::VMOV64toSDrm, 0 },
498  { X86::VMOVAPDrr, X86::VMOVAPDrm, TB_ALIGN_16 },
499  { X86::VMOVAPSrr, X86::VMOVAPSrm, TB_ALIGN_16 },
500  { X86::VMOVDDUPrr, X86::VMOVDDUPrm, 0 },
501  { X86::VMOVDI2PDIrr, X86::VMOVDI2PDIrm, 0 },
502  { X86::VMOVDI2SSrr, X86::VMOVDI2SSrm, 0 },
503  { X86::VMOVDQArr, X86::VMOVDQArm, TB_ALIGN_16 },
504  { X86::VMOVSLDUPrr, X86::VMOVSLDUPrm, TB_ALIGN_16 },
505  { X86::VMOVSHDUPrr, X86::VMOVSHDUPrm, TB_ALIGN_16 },
506  { X86::VMOVUPDrr, X86::VMOVUPDrm, 0 },
507  { X86::VMOVUPSrr, X86::VMOVUPSrm, 0 },
508  { X86::VMOVZQI2PQIrr, X86::VMOVZQI2PQIrm, 0 },
509  { X86::VMOVZPQILo2PQIrr,X86::VMOVZPQILo2PQIrm, TB_ALIGN_16 },
510  { X86::VPABSBrr128, X86::VPABSBrm128, 0 },
511  { X86::VPABSDrr128, X86::VPABSDrm128, 0 },
512  { X86::VPABSWrr128, X86::VPABSWrm128, 0 },
513  { X86::VPERMILPDri, X86::VPERMILPDmi, 0 },
514  { X86::VPERMILPSri, X86::VPERMILPSmi, 0 },
515  { X86::VPSHUFDri, X86::VPSHUFDmi, 0 },
516  { X86::VPSHUFHWri, X86::VPSHUFHWmi, 0 },
517  { X86::VPSHUFLWri, X86::VPSHUFLWmi, 0 },
518  { X86::VRCPPSr, X86::VRCPPSm, 0 },
519  { X86::VRCPPSr_Int, X86::VRCPPSm_Int, 0 },
520  { X86::VRSQRTPSr, X86::VRSQRTPSm, 0 },
521  { X86::VRSQRTPSr_Int, X86::VRSQRTPSm_Int, 0 },
522  { X86::VSQRTPDr, X86::VSQRTPDm, 0 },
523  { X86::VSQRTPSr, X86::VSQRTPSm, 0 },
524  { X86::VUCOMISDrr, X86::VUCOMISDrm, 0 },
525  { X86::VUCOMISSrr, X86::VUCOMISSrm, 0 },
526  { X86::VBROADCASTSSrr, X86::VBROADCASTSSrm, TB_NO_REVERSE },
527 
528  // AVX 256-bit foldable instructions
529  { X86::VMOVAPDYrr, X86::VMOVAPDYrm, TB_ALIGN_32 },
530  { X86::VMOVAPSYrr, X86::VMOVAPSYrm, TB_ALIGN_32 },
531  { X86::VMOVDQAYrr, X86::VMOVDQAYrm, TB_ALIGN_32 },
532  { X86::VMOVUPDYrr, X86::VMOVUPDYrm, 0 },
533  { X86::VMOVUPSYrr, X86::VMOVUPSYrm, 0 },
534  { X86::VPERMILPDYri, X86::VPERMILPDYmi, 0 },
535  { X86::VPERMILPSYri, X86::VPERMILPSYmi, 0 },
536 
537  // AVX2 foldable instructions
538  { X86::VPABSBrr256, X86::VPABSBrm256, 0 },
539  { X86::VPABSDrr256, X86::VPABSDrm256, 0 },
540  { X86::VPABSWrr256, X86::VPABSWrm256, 0 },
541  { X86::VPSHUFDYri, X86::VPSHUFDYmi, 0 },
542  { X86::VPSHUFHWYri, X86::VPSHUFHWYmi, 0 },
543  { X86::VPSHUFLWYri, X86::VPSHUFLWYmi, 0 },
544  { X86::VRCPPSYr, X86::VRCPPSYm, 0 },
545  { X86::VRCPPSYr_Int, X86::VRCPPSYm_Int, 0 },
546  { X86::VRSQRTPSYr, X86::VRSQRTPSYm, 0 },
547  { X86::VSQRTPDYr, X86::VSQRTPDYm, 0 },
548  { X86::VSQRTPSYr, X86::VSQRTPSYm, 0 },
549  { X86::VBROADCASTSSYrr, X86::VBROADCASTSSYrm, TB_NO_REVERSE },
550  { X86::VBROADCASTSDYrr, X86::VBROADCASTSDYrm, TB_NO_REVERSE },
551 
552  // BMI/BMI2/LZCNT/POPCNT/TBM foldable instructions
553  { X86::BEXTR32rr, X86::BEXTR32rm, 0 },
554  { X86::BEXTR64rr, X86::BEXTR64rm, 0 },
555  { X86::BEXTRI32ri, X86::BEXTRI32mi, 0 },
556  { X86::BEXTRI64ri, X86::BEXTRI64mi, 0 },
557  { X86::BLCFILL32rr, X86::BLCFILL32rm, 0 },
558  { X86::BLCFILL64rr, X86::BLCFILL64rm, 0 },
559  { X86::BLCI32rr, X86::BLCI32rm, 0 },
560  { X86::BLCI64rr, X86::BLCI64rm, 0 },
561  { X86::BLCIC32rr, X86::BLCIC32rm, 0 },
562  { X86::BLCIC64rr, X86::BLCIC64rm, 0 },
563  { X86::BLCMSK32rr, X86::BLCMSK32rm, 0 },
564  { X86::BLCMSK64rr, X86::BLCMSK64rm, 0 },
565  { X86::BLCS32rr, X86::BLCS32rm, 0 },
566  { X86::BLCS64rr, X86::BLCS64rm, 0 },
567  { X86::BLSFILL32rr, X86::BLSFILL32rm, 0 },
568  { X86::BLSFILL64rr, X86::BLSFILL64rm, 0 },
569  { X86::BLSI32rr, X86::BLSI32rm, 0 },
570  { X86::BLSI64rr, X86::BLSI64rm, 0 },
571  { X86::BLSIC32rr, X86::BLSIC32rm, 0 },
572  { X86::BLSIC64rr, X86::BLSIC64rm, 0 },
573  { X86::BLSMSK32rr, X86::BLSMSK32rm, 0 },
574  { X86::BLSMSK64rr, X86::BLSMSK64rm, 0 },
575  { X86::BLSR32rr, X86::BLSR32rm, 0 },
576  { X86::BLSR64rr, X86::BLSR64rm, 0 },
577  { X86::BZHI32rr, X86::BZHI32rm, 0 },
578  { X86::BZHI64rr, X86::BZHI64rm, 0 },
579  { X86::LZCNT16rr, X86::LZCNT16rm, 0 },
580  { X86::LZCNT32rr, X86::LZCNT32rm, 0 },
581  { X86::LZCNT64rr, X86::LZCNT64rm, 0 },
582  { X86::POPCNT16rr, X86::POPCNT16rm, 0 },
583  { X86::POPCNT32rr, X86::POPCNT32rm, 0 },
584  { X86::POPCNT64rr, X86::POPCNT64rm, 0 },
585  { X86::RORX32ri, X86::RORX32mi, 0 },
586  { X86::RORX64ri, X86::RORX64mi, 0 },
587  { X86::SARX32rr, X86::SARX32rm, 0 },
588  { X86::SARX64rr, X86::SARX64rm, 0 },
589  { X86::SHRX32rr, X86::SHRX32rm, 0 },
590  { X86::SHRX64rr, X86::SHRX64rm, 0 },
591  { X86::SHLX32rr, X86::SHLX32rm, 0 },
592  { X86::SHLX64rr, X86::SHLX64rm, 0 },
593  { X86::T1MSKC32rr, X86::T1MSKC32rm, 0 },
594  { X86::T1MSKC64rr, X86::T1MSKC64rm, 0 },
595  { X86::TZCNT16rr, X86::TZCNT16rm, 0 },
596  { X86::TZCNT32rr, X86::TZCNT32rm, 0 },
597  { X86::TZCNT64rr, X86::TZCNT64rm, 0 },
598  { X86::TZMSK32rr, X86::TZMSK32rm, 0 },
599  { X86::TZMSK64rr, X86::TZMSK64rm, 0 },
600 
601  // AVX-512 foldable instructions
602  { X86::VMOV64toPQIZrr, X86::VMOVQI2PQIZrm, 0 },
603  { X86::VMOVDI2SSZrr, X86::VMOVDI2SSZrm, 0 },
604  { X86::VMOVDQA32rr, X86::VMOVDQA32rm, TB_ALIGN_64 },
605  { X86::VMOVDQA64rr, X86::VMOVDQA64rm, TB_ALIGN_64 },
606  { X86::VMOVDQU32rr, X86::VMOVDQU32rm, 0 },
607  { X86::VMOVDQU64rr, X86::VMOVDQU64rm, 0 },
608 
609  // AES foldable instructions
610  { X86::AESIMCrr, X86::AESIMCrm, TB_ALIGN_16 },
611  { X86::AESKEYGENASSIST128rr, X86::AESKEYGENASSIST128rm, TB_ALIGN_16 },
612  { X86::VAESIMCrr, X86::VAESIMCrm, TB_ALIGN_16 },
613  { X86::VAESKEYGENASSIST128rr, X86::VAESKEYGENASSIST128rm, TB_ALIGN_16 },
614  };
615 
616  for (unsigned i = 0, e = array_lengthof(OpTbl1); i != e; ++i) {
617  unsigned RegOp = OpTbl1[i].RegOp;
618  unsigned MemOp = OpTbl1[i].MemOp;
619  unsigned Flags = OpTbl1[i].Flags;
620  AddTableEntry(RegOp2MemOpTable1, MemOp2RegOpTable,
621  RegOp, MemOp,
622  // Index 1, folded load
623  Flags | TB_INDEX_1 | TB_FOLDED_LOAD);
624  }
625 
626  static const X86OpTblEntry OpTbl2[] = {
627  { X86::ADC32rr, X86::ADC32rm, 0 },
628  { X86::ADC64rr, X86::ADC64rm, 0 },
629  { X86::ADD16rr, X86::ADD16rm, 0 },
630  { X86::ADD16rr_DB, X86::ADD16rm, TB_NO_REVERSE },
631  { X86::ADD32rr, X86::ADD32rm, 0 },
632  { X86::ADD32rr_DB, X86::ADD32rm, TB_NO_REVERSE },
633  { X86::ADD64rr, X86::ADD64rm, 0 },
634  { X86::ADD64rr_DB, X86::ADD64rm, TB_NO_REVERSE },
635  { X86::ADD8rr, X86::ADD8rm, 0 },
636  { X86::ADDPDrr, X86::ADDPDrm, TB_ALIGN_16 },
637  { X86::ADDPSrr, X86::ADDPSrm, TB_ALIGN_16 },
638  { X86::ADDSDrr, X86::ADDSDrm, 0 },
639  { X86::ADDSSrr, X86::ADDSSrm, 0 },
640  { X86::ADDSUBPDrr, X86::ADDSUBPDrm, TB_ALIGN_16 },
641  { X86::ADDSUBPSrr, X86::ADDSUBPSrm, TB_ALIGN_16 },
642  { X86::AND16rr, X86::AND16rm, 0 },
643  { X86::AND32rr, X86::AND32rm, 0 },
644  { X86::AND64rr, X86::AND64rm, 0 },
645  { X86::AND8rr, X86::AND8rm, 0 },
646  { X86::ANDNPDrr, X86::ANDNPDrm, TB_ALIGN_16 },
647  { X86::ANDNPSrr, X86::ANDNPSrm, TB_ALIGN_16 },
648  { X86::ANDPDrr, X86::ANDPDrm, TB_ALIGN_16 },
649  { X86::ANDPSrr, X86::ANDPSrm, TB_ALIGN_16 },
650  { X86::BLENDPDrri, X86::BLENDPDrmi, TB_ALIGN_16 },
651  { X86::BLENDPSrri, X86::BLENDPSrmi, TB_ALIGN_16 },
652  { X86::BLENDVPDrr0, X86::BLENDVPDrm0, TB_ALIGN_16 },
653  { X86::BLENDVPSrr0, X86::BLENDVPSrm0, TB_ALIGN_16 },
654  { X86::CMOVA16rr, X86::CMOVA16rm, 0 },
655  { X86::CMOVA32rr, X86::CMOVA32rm, 0 },
656  { X86::CMOVA64rr, X86::CMOVA64rm, 0 },
657  { X86::CMOVAE16rr, X86::CMOVAE16rm, 0 },
658  { X86::CMOVAE32rr, X86::CMOVAE32rm, 0 },
659  { X86::CMOVAE64rr, X86::CMOVAE64rm, 0 },
660  { X86::CMOVB16rr, X86::CMOVB16rm, 0 },
661  { X86::CMOVB32rr, X86::CMOVB32rm, 0 },
662  { X86::CMOVB64rr, X86::CMOVB64rm, 0 },
663  { X86::CMOVBE16rr, X86::CMOVBE16rm, 0 },
664  { X86::CMOVBE32rr, X86::CMOVBE32rm, 0 },
665  { X86::CMOVBE64rr, X86::CMOVBE64rm, 0 },
666  { X86::CMOVE16rr, X86::CMOVE16rm, 0 },
667  { X86::CMOVE32rr, X86::CMOVE32rm, 0 },
668  { X86::CMOVE64rr, X86::CMOVE64rm, 0 },
669  { X86::CMOVG16rr, X86::CMOVG16rm, 0 },
670  { X86::CMOVG32rr, X86::CMOVG32rm, 0 },
671  { X86::CMOVG64rr, X86::CMOVG64rm, 0 },
672  { X86::CMOVGE16rr, X86::CMOVGE16rm, 0 },
673  { X86::CMOVGE32rr, X86::CMOVGE32rm, 0 },
674  { X86::CMOVGE64rr, X86::CMOVGE64rm, 0 },
675  { X86::CMOVL16rr, X86::CMOVL16rm, 0 },
676  { X86::CMOVL32rr, X86::CMOVL32rm, 0 },
677  { X86::CMOVL64rr, X86::CMOVL64rm, 0 },
678  { X86::CMOVLE16rr, X86::CMOVLE16rm, 0 },
679  { X86::CMOVLE32rr, X86::CMOVLE32rm, 0 },
680  { X86::CMOVLE64rr, X86::CMOVLE64rm, 0 },
681  { X86::CMOVNE16rr, X86::CMOVNE16rm, 0 },
682  { X86::CMOVNE32rr, X86::CMOVNE32rm, 0 },
683  { X86::CMOVNE64rr, X86::CMOVNE64rm, 0 },
684  { X86::CMOVNO16rr, X86::CMOVNO16rm, 0 },
685  { X86::CMOVNO32rr, X86::CMOVNO32rm, 0 },
686  { X86::CMOVNO64rr, X86::CMOVNO64rm, 0 },
687  { X86::CMOVNP16rr, X86::CMOVNP16rm, 0 },
688  { X86::CMOVNP32rr, X86::CMOVNP32rm, 0 },
689  { X86::CMOVNP64rr, X86::CMOVNP64rm, 0 },
690  { X86::CMOVNS16rr, X86::CMOVNS16rm, 0 },
691  { X86::CMOVNS32rr, X86::CMOVNS32rm, 0 },
692  { X86::CMOVNS64rr, X86::CMOVNS64rm, 0 },
693  { X86::CMOVO16rr, X86::CMOVO16rm, 0 },
694  { X86::CMOVO32rr, X86::CMOVO32rm, 0 },
695  { X86::CMOVO64rr, X86::CMOVO64rm, 0 },
696  { X86::CMOVP16rr, X86::CMOVP16rm, 0 },
697  { X86::CMOVP32rr, X86::CMOVP32rm, 0 },
698  { X86::CMOVP64rr, X86::CMOVP64rm, 0 },
699  { X86::CMOVS16rr, X86::CMOVS16rm, 0 },
700  { X86::CMOVS32rr, X86::CMOVS32rm, 0 },
701  { X86::CMOVS64rr, X86::CMOVS64rm, 0 },
702  { X86::CMPPDrri, X86::CMPPDrmi, TB_ALIGN_16 },
703  { X86::CMPPSrri, X86::CMPPSrmi, TB_ALIGN_16 },
704  { X86::CMPSDrr, X86::CMPSDrm, 0 },
705  { X86::CMPSSrr, X86::CMPSSrm, 0 },
706  { X86::DIVPDrr, X86::DIVPDrm, TB_ALIGN_16 },
707  { X86::DIVPSrr, X86::DIVPSrm, TB_ALIGN_16 },
708  { X86::DIVSDrr, X86::DIVSDrm, 0 },
709  { X86::DIVSSrr, X86::DIVSSrm, 0 },
710  { X86::FsANDNPDrr, X86::FsANDNPDrm, TB_ALIGN_16 },
711  { X86::FsANDNPSrr, X86::FsANDNPSrm, TB_ALIGN_16 },
712  { X86::FsANDPDrr, X86::FsANDPDrm, TB_ALIGN_16 },
713  { X86::FsANDPSrr, X86::FsANDPSrm, TB_ALIGN_16 },
714  { X86::FsORPDrr, X86::FsORPDrm, TB_ALIGN_16 },
715  { X86::FsORPSrr, X86::FsORPSrm, TB_ALIGN_16 },
716  { X86::FsXORPDrr, X86::FsXORPDrm, TB_ALIGN_16 },
717  { X86::FsXORPSrr, X86::FsXORPSrm, TB_ALIGN_16 },
718  { X86::HADDPDrr, X86::HADDPDrm, TB_ALIGN_16 },
719  { X86::HADDPSrr, X86::HADDPSrm, TB_ALIGN_16 },
720  { X86::HSUBPDrr, X86::HSUBPDrm, TB_ALIGN_16 },
721  { X86::HSUBPSrr, X86::HSUBPSrm, TB_ALIGN_16 },
722  { X86::IMUL16rr, X86::IMUL16rm, 0 },
723  { X86::IMUL32rr, X86::IMUL32rm, 0 },
724  { X86::IMUL64rr, X86::IMUL64rm, 0 },
725  { X86::Int_CMPSDrr, X86::Int_CMPSDrm, 0 },
726  { X86::Int_CMPSSrr, X86::Int_CMPSSrm, 0 },
727  { X86::Int_CVTSD2SSrr, X86::Int_CVTSD2SSrm, 0 },
728  { X86::Int_CVTSI2SD64rr,X86::Int_CVTSI2SD64rm, 0 },
729  { X86::Int_CVTSI2SDrr, X86::Int_CVTSI2SDrm, 0 },
730  { X86::Int_CVTSI2SS64rr,X86::Int_CVTSI2SS64rm, 0 },
731  { X86::Int_CVTSI2SSrr, X86::Int_CVTSI2SSrm, 0 },
732  { X86::Int_CVTSS2SDrr, X86::Int_CVTSS2SDrm, 0 },
733  { X86::MAXPDrr, X86::MAXPDrm, TB_ALIGN_16 },
734  { X86::MAXPSrr, X86::MAXPSrm, TB_ALIGN_16 },
735  { X86::MAXSDrr, X86::MAXSDrm, 0 },
736  { X86::MAXSSrr, X86::MAXSSrm, 0 },
737  { X86::MINPDrr, X86::MINPDrm, TB_ALIGN_16 },
738  { X86::MINPSrr, X86::MINPSrm, TB_ALIGN_16 },
739  { X86::MINSDrr, X86::MINSDrm, 0 },
740  { X86::MINSSrr, X86::MINSSrm, 0 },
741  { X86::MPSADBWrri, X86::MPSADBWrmi, TB_ALIGN_16 },
742  { X86::MULPDrr, X86::MULPDrm, TB_ALIGN_16 },
743  { X86::MULPSrr, X86::MULPSrm, TB_ALIGN_16 },
744  { X86::MULSDrr, X86::MULSDrm, 0 },
745  { X86::MULSSrr, X86::MULSSrm, 0 },
746  { X86::OR16rr, X86::OR16rm, 0 },
747  { X86::OR32rr, X86::OR32rm, 0 },
748  { X86::OR64rr, X86::OR64rm, 0 },
749  { X86::OR8rr, X86::OR8rm, 0 },
750  { X86::ORPDrr, X86::ORPDrm, TB_ALIGN_16 },
751  { X86::ORPSrr, X86::ORPSrm, TB_ALIGN_16 },
752  { X86::PACKSSDWrr, X86::PACKSSDWrm, TB_ALIGN_16 },
753  { X86::PACKSSWBrr, X86::PACKSSWBrm, TB_ALIGN_16 },
754  { X86::PACKUSDWrr, X86::PACKUSDWrm, TB_ALIGN_16 },
755  { X86::PACKUSWBrr, X86::PACKUSWBrm, TB_ALIGN_16 },
756  { X86::PADDBrr, X86::PADDBrm, TB_ALIGN_16 },
757  { X86::PADDDrr, X86::PADDDrm, TB_ALIGN_16 },
758  { X86::PADDQrr, X86::PADDQrm, TB_ALIGN_16 },
759  { X86::PADDSBrr, X86::PADDSBrm, TB_ALIGN_16 },
760  { X86::PADDSWrr, X86::PADDSWrm, TB_ALIGN_16 },
761  { X86::PADDUSBrr, X86::PADDUSBrm, TB_ALIGN_16 },
762  { X86::PADDUSWrr, X86::PADDUSWrm, TB_ALIGN_16 },
763  { X86::PADDWrr, X86::PADDWrm, TB_ALIGN_16 },
764  { X86::PALIGNR128rr, X86::PALIGNR128rm, TB_ALIGN_16 },
765  { X86::PANDNrr, X86::PANDNrm, TB_ALIGN_16 },
766  { X86::PANDrr, X86::PANDrm, TB_ALIGN_16 },
767  { X86::PAVGBrr, X86::PAVGBrm, TB_ALIGN_16 },
768  { X86::PAVGWrr, X86::PAVGWrm, TB_ALIGN_16 },
769  { X86::PBLENDWrri, X86::PBLENDWrmi, TB_ALIGN_16 },
770  { X86::PCMPEQBrr, X86::PCMPEQBrm, TB_ALIGN_16 },
771  { X86::PCMPEQDrr, X86::PCMPEQDrm, TB_ALIGN_16 },
772  { X86::PCMPEQQrr, X86::PCMPEQQrm, TB_ALIGN_16 },
773  { X86::PCMPEQWrr, X86::PCMPEQWrm, TB_ALIGN_16 },
774  { X86::PCMPGTBrr, X86::PCMPGTBrm, TB_ALIGN_16 },
775  { X86::PCMPGTDrr, X86::PCMPGTDrm, TB_ALIGN_16 },
776  { X86::PCMPGTQrr, X86::PCMPGTQrm, TB_ALIGN_16 },
777  { X86::PCMPGTWrr, X86::PCMPGTWrm, TB_ALIGN_16 },
778  { X86::PHADDDrr, X86::PHADDDrm, TB_ALIGN_16 },
779  { X86::PHADDWrr, X86::PHADDWrm, TB_ALIGN_16 },
780  { X86::PHADDSWrr128, X86::PHADDSWrm128, TB_ALIGN_16 },
781  { X86::PHSUBDrr, X86::PHSUBDrm, TB_ALIGN_16 },
782  { X86::PHSUBSWrr128, X86::PHSUBSWrm128, TB_ALIGN_16 },
783  { X86::PHSUBWrr, X86::PHSUBWrm, TB_ALIGN_16 },
784  { X86::PINSRWrri, X86::PINSRWrmi, TB_ALIGN_16 },
785  { X86::PMADDUBSWrr128, X86::PMADDUBSWrm128, TB_ALIGN_16 },
786  { X86::PMADDWDrr, X86::PMADDWDrm, TB_ALIGN_16 },
787  { X86::PMAXSWrr, X86::PMAXSWrm, TB_ALIGN_16 },
788  { X86::PMAXUBrr, X86::PMAXUBrm, TB_ALIGN_16 },
789  { X86::PMINSWrr, X86::PMINSWrm, TB_ALIGN_16 },
790  { X86::PMINUBrr, X86::PMINUBrm, TB_ALIGN_16 },
791  { X86::PMINSBrr, X86::PMINSBrm, TB_ALIGN_16 },
792  { X86::PMINSDrr, X86::PMINSDrm, TB_ALIGN_16 },
793  { X86::PMINUDrr, X86::PMINUDrm, TB_ALIGN_16 },
794  { X86::PMINUWrr, X86::PMINUWrm, TB_ALIGN_16 },
795  { X86::PMAXSBrr, X86::PMAXSBrm, TB_ALIGN_16 },
796  { X86::PMAXSDrr, X86::PMAXSDrm, TB_ALIGN_16 },
797  { X86::PMAXUDrr, X86::PMAXUDrm, TB_ALIGN_16 },
798  { X86::PMAXUWrr, X86::PMAXUWrm, TB_ALIGN_16 },
799  { X86::PMULDQrr, X86::PMULDQrm, TB_ALIGN_16 },
800  { X86::PMULHRSWrr128, X86::PMULHRSWrm128, TB_ALIGN_16 },
801  { X86::PMULHUWrr, X86::PMULHUWrm, TB_ALIGN_16 },
802  { X86::PMULHWrr, X86::PMULHWrm, TB_ALIGN_16 },
803  { X86::PMULLDrr, X86::PMULLDrm, TB_ALIGN_16 },
804  { X86::PMULLWrr, X86::PMULLWrm, TB_ALIGN_16 },
805  { X86::PMULUDQrr, X86::PMULUDQrm, TB_ALIGN_16 },
806  { X86::PORrr, X86::PORrm, TB_ALIGN_16 },
807  { X86::PSADBWrr, X86::PSADBWrm, TB_ALIGN_16 },
808  { X86::PSHUFBrr, X86::PSHUFBrm, TB_ALIGN_16 },
809  { X86::PSIGNBrr, X86::PSIGNBrm, TB_ALIGN_16 },
810  { X86::PSIGNWrr, X86::PSIGNWrm, TB_ALIGN_16 },
811  { X86::PSIGNDrr, X86::PSIGNDrm, TB_ALIGN_16 },
812  { X86::PSLLDrr, X86::PSLLDrm, TB_ALIGN_16 },
813  { X86::PSLLQrr, X86::PSLLQrm, TB_ALIGN_16 },
814  { X86::PSLLWrr, X86::PSLLWrm, TB_ALIGN_16 },
815  { X86::PSRADrr, X86::PSRADrm, TB_ALIGN_16 },
816  { X86::PSRAWrr, X86::PSRAWrm, TB_ALIGN_16 },
817  { X86::PSRLDrr, X86::PSRLDrm, TB_ALIGN_16 },
818  { X86::PSRLQrr, X86::PSRLQrm, TB_ALIGN_16 },
819  { X86::PSRLWrr, X86::PSRLWrm, TB_ALIGN_16 },
820  { X86::PSUBBrr, X86::PSUBBrm, TB_ALIGN_16 },
821  { X86::PSUBDrr, X86::PSUBDrm, TB_ALIGN_16 },
822  { X86::PSUBSBrr, X86::PSUBSBrm, TB_ALIGN_16 },
823  { X86::PSUBSWrr, X86::PSUBSWrm, TB_ALIGN_16 },
824  { X86::PSUBWrr, X86::PSUBWrm, TB_ALIGN_16 },
825  { X86::PUNPCKHBWrr, X86::PUNPCKHBWrm, TB_ALIGN_16 },
826  { X86::PUNPCKHDQrr, X86::PUNPCKHDQrm, TB_ALIGN_16 },
827  { X86::PUNPCKHQDQrr, X86::PUNPCKHQDQrm, TB_ALIGN_16 },
828  { X86::PUNPCKHWDrr, X86::PUNPCKHWDrm, TB_ALIGN_16 },
829  { X86::PUNPCKLBWrr, X86::PUNPCKLBWrm, TB_ALIGN_16 },
830  { X86::PUNPCKLDQrr, X86::PUNPCKLDQrm, TB_ALIGN_16 },
831  { X86::PUNPCKLQDQrr, X86::PUNPCKLQDQrm, TB_ALIGN_16 },
832  { X86::PUNPCKLWDrr, X86::PUNPCKLWDrm, TB_ALIGN_16 },
833  { X86::PXORrr, X86::PXORrm, TB_ALIGN_16 },
834  { X86::SBB32rr, X86::SBB32rm, 0 },
835  { X86::SBB64rr, X86::SBB64rm, 0 },
836  { X86::SHUFPDrri, X86::SHUFPDrmi, TB_ALIGN_16 },
837  { X86::SHUFPSrri, X86::SHUFPSrmi, TB_ALIGN_16 },
838  { X86::SUB16rr, X86::SUB16rm, 0 },
839  { X86::SUB32rr, X86::SUB32rm, 0 },
840  { X86::SUB64rr, X86::SUB64rm, 0 },
841  { X86::SUB8rr, X86::SUB8rm, 0 },
842  { X86::SUBPDrr, X86::SUBPDrm, TB_ALIGN_16 },
843  { X86::SUBPSrr, X86::SUBPSrm, TB_ALIGN_16 },
844  { X86::SUBSDrr, X86::SUBSDrm, 0 },
845  { X86::SUBSSrr, X86::SUBSSrm, 0 },
846  // FIXME: TEST*rr -> swapped operand of TEST*mr.
847  { X86::UNPCKHPDrr, X86::UNPCKHPDrm, TB_ALIGN_16 },
848  { X86::UNPCKHPSrr, X86::UNPCKHPSrm, TB_ALIGN_16 },
849  { X86::UNPCKLPDrr, X86::UNPCKLPDrm, TB_ALIGN_16 },
850  { X86::UNPCKLPSrr, X86::UNPCKLPSrm, TB_ALIGN_16 },
851  { X86::XOR16rr, X86::XOR16rm, 0 },
852  { X86::XOR32rr, X86::XOR32rm, 0 },
853  { X86::XOR64rr, X86::XOR64rm, 0 },
854  { X86::XOR8rr, X86::XOR8rm, 0 },
855  { X86::XORPDrr, X86::XORPDrm, TB_ALIGN_16 },
856  { X86::XORPSrr, X86::XORPSrm, TB_ALIGN_16 },
857  // AVX 128-bit versions of foldable instructions
858  { X86::VCVTSD2SSrr, X86::VCVTSD2SSrm, 0 },
859  { X86::Int_VCVTSD2SSrr, X86::Int_VCVTSD2SSrm, 0 },
860  { X86::VCVTSI2SD64rr, X86::VCVTSI2SD64rm, 0 },
861  { X86::Int_VCVTSI2SD64rr, X86::Int_VCVTSI2SD64rm, 0 },
862  { X86::VCVTSI2SDrr, X86::VCVTSI2SDrm, 0 },
863  { X86::Int_VCVTSI2SDrr, X86::Int_VCVTSI2SDrm, 0 },
864  { X86::VCVTSI2SS64rr, X86::VCVTSI2SS64rm, 0 },
865  { X86::Int_VCVTSI2SS64rr, X86::Int_VCVTSI2SS64rm, 0 },
866  { X86::VCVTSI2SSrr, X86::VCVTSI2SSrm, 0 },
867  { X86::Int_VCVTSI2SSrr, X86::Int_VCVTSI2SSrm, 0 },
868  { X86::VCVTSS2SDrr, X86::VCVTSS2SDrm, 0 },
869  { X86::Int_VCVTSS2SDrr, X86::Int_VCVTSS2SDrm, 0 },
870  { X86::VCVTTPD2DQrr, X86::VCVTTPD2DQXrm, 0 },
871  { X86::VCVTTPS2DQrr, X86::VCVTTPS2DQrm, 0 },
872  { X86::VRSQRTSSr, X86::VRSQRTSSm, 0 },
873  { X86::VSQRTSDr, X86::VSQRTSDm, 0 },
874  { X86::VSQRTSSr, X86::VSQRTSSm, 0 },
875  { X86::VADDPDrr, X86::VADDPDrm, 0 },
876  { X86::VADDPSrr, X86::VADDPSrm, 0 },
877  { X86::VADDSDrr, X86::VADDSDrm, 0 },
878  { X86::VADDSSrr, X86::VADDSSrm, 0 },
879  { X86::VADDSUBPDrr, X86::VADDSUBPDrm, 0 },
880  { X86::VADDSUBPSrr, X86::VADDSUBPSrm, 0 },
881  { X86::VANDNPDrr, X86::VANDNPDrm, 0 },
882  { X86::VANDNPSrr, X86::VANDNPSrm, 0 },
883  { X86::VANDPDrr, X86::VANDPDrm, 0 },
884  { X86::VANDPSrr, X86::VANDPSrm, 0 },
885  { X86::VBLENDPDrri, X86::VBLENDPDrmi, 0 },
886  { X86::VBLENDPSrri, X86::VBLENDPSrmi, 0 },
887  { X86::VBLENDVPDrr, X86::VBLENDVPDrm, 0 },
888  { X86::VBLENDVPSrr, X86::VBLENDVPSrm, 0 },
889  { X86::VCMPPDrri, X86::VCMPPDrmi, 0 },
890  { X86::VCMPPSrri, X86::VCMPPSrmi, 0 },
891  { X86::VCMPSDrr, X86::VCMPSDrm, 0 },
892  { X86::VCMPSSrr, X86::VCMPSSrm, 0 },
893  { X86::VDIVPDrr, X86::VDIVPDrm, 0 },
894  { X86::VDIVPSrr, X86::VDIVPSrm, 0 },
895  { X86::VDIVSDrr, X86::VDIVSDrm, 0 },
896  { X86::VDIVSSrr, X86::VDIVSSrm, 0 },
897  { X86::VFsANDNPDrr, X86::VFsANDNPDrm, TB_ALIGN_16 },
898  { X86::VFsANDNPSrr, X86::VFsANDNPSrm, TB_ALIGN_16 },
899  { X86::VFsANDPDrr, X86::VFsANDPDrm, TB_ALIGN_16 },
900  { X86::VFsANDPSrr, X86::VFsANDPSrm, TB_ALIGN_16 },
901  { X86::VFsORPDrr, X86::VFsORPDrm, TB_ALIGN_16 },
902  { X86::VFsORPSrr, X86::VFsORPSrm, TB_ALIGN_16 },
903  { X86::VFsXORPDrr, X86::VFsXORPDrm, TB_ALIGN_16 },
904  { X86::VFsXORPSrr, X86::VFsXORPSrm, TB_ALIGN_16 },
905  { X86::VHADDPDrr, X86::VHADDPDrm, 0 },
906  { X86::VHADDPSrr, X86::VHADDPSrm, 0 },
907  { X86::VHSUBPDrr, X86::VHSUBPDrm, 0 },
908  { X86::VHSUBPSrr, X86::VHSUBPSrm, 0 },
909  { X86::Int_VCMPSDrr, X86::Int_VCMPSDrm, 0 },
910  { X86::Int_VCMPSSrr, X86::Int_VCMPSSrm, 0 },
911  { X86::VMAXPDrr, X86::VMAXPDrm, 0 },
912  { X86::VMAXPSrr, X86::VMAXPSrm, 0 },
913  { X86::VMAXSDrr, X86::VMAXSDrm, 0 },
914  { X86::VMAXSSrr, X86::VMAXSSrm, 0 },
915  { X86::VMINPDrr, X86::VMINPDrm, 0 },
916  { X86::VMINPSrr, X86::VMINPSrm, 0 },
917  { X86::VMINSDrr, X86::VMINSDrm, 0 },
918  { X86::VMINSSrr, X86::VMINSSrm, 0 },
919  { X86::VMPSADBWrri, X86::VMPSADBWrmi, 0 },
920  { X86::VMULPDrr, X86::VMULPDrm, 0 },
921  { X86::VMULPSrr, X86::VMULPSrm, 0 },
922  { X86::VMULSDrr, X86::VMULSDrm, 0 },
923  { X86::VMULSSrr, X86::VMULSSrm, 0 },
924  { X86::VORPDrr, X86::VORPDrm, 0 },
925  { X86::VORPSrr, X86::VORPSrm, 0 },
926  { X86::VPACKSSDWrr, X86::VPACKSSDWrm, 0 },
927  { X86::VPACKSSWBrr, X86::VPACKSSWBrm, 0 },
928  { X86::VPACKUSDWrr, X86::VPACKUSDWrm, 0 },
929  { X86::VPACKUSWBrr, X86::VPACKUSWBrm, 0 },
930  { X86::VPADDBrr, X86::VPADDBrm, 0 },
931  { X86::VPADDDrr, X86::VPADDDrm, 0 },
932  { X86::VPADDQrr, X86::VPADDQrm, 0 },
933  { X86::VPADDSBrr, X86::VPADDSBrm, 0 },
934  { X86::VPADDSWrr, X86::VPADDSWrm, 0 },
935  { X86::VPADDUSBrr, X86::VPADDUSBrm, 0 },
936  { X86::VPADDUSWrr, X86::VPADDUSWrm, 0 },
937  { X86::VPADDWrr, X86::VPADDWrm, 0 },
938  { X86::VPALIGNR128rr, X86::VPALIGNR128rm, 0 },
939  { X86::VPANDNrr, X86::VPANDNrm, 0 },
940  { X86::VPANDrr, X86::VPANDrm, 0 },
941  { X86::VPAVGBrr, X86::VPAVGBrm, 0 },
942  { X86::VPAVGWrr, X86::VPAVGWrm, 0 },
943  { X86::VPBLENDWrri, X86::VPBLENDWrmi, 0 },
944  { X86::VPCMPEQBrr, X86::VPCMPEQBrm, 0 },
945  { X86::VPCMPEQDrr, X86::VPCMPEQDrm, 0 },
946  { X86::VPCMPEQQrr, X86::VPCMPEQQrm, 0 },
947  { X86::VPCMPEQWrr, X86::VPCMPEQWrm, 0 },
948  { X86::VPCMPGTBrr, X86::VPCMPGTBrm, 0 },
949  { X86::VPCMPGTDrr, X86::VPCMPGTDrm, 0 },
950  { X86::VPCMPGTQrr, X86::VPCMPGTQrm, 0 },
951  { X86::VPCMPGTWrr, X86::VPCMPGTWrm, 0 },
952  { X86::VPHADDDrr, X86::VPHADDDrm, 0 },
953  { X86::VPHADDSWrr128, X86::VPHADDSWrm128, 0 },
954  { X86::VPHADDWrr, X86::VPHADDWrm, 0 },
955  { X86::VPHSUBDrr, X86::VPHSUBDrm, 0 },
956  { X86::VPHSUBSWrr128, X86::VPHSUBSWrm128, 0 },
957  { X86::VPHSUBWrr, X86::VPHSUBWrm, 0 },
958  { X86::VPERMILPDrr, X86::VPERMILPDrm, 0 },
959  { X86::VPERMILPSrr, X86::VPERMILPSrm, 0 },
960  { X86::VPINSRWrri, X86::VPINSRWrmi, 0 },
961  { X86::VPMADDUBSWrr128, X86::VPMADDUBSWrm128, 0 },
962  { X86::VPMADDWDrr, X86::VPMADDWDrm, 0 },
963  { X86::VPMAXSWrr, X86::VPMAXSWrm, 0 },
964  { X86::VPMAXUBrr, X86::VPMAXUBrm, 0 },
965  { X86::VPMINSWrr, X86::VPMINSWrm, 0 },
966  { X86::VPMINUBrr, X86::VPMINUBrm, 0 },
967  { X86::VPMINSBrr, X86::VPMINSBrm, 0 },
968  { X86::VPMINSDrr, X86::VPMINSDrm, 0 },
969  { X86::VPMINUDrr, X86::VPMINUDrm, 0 },
970  { X86::VPMINUWrr, X86::VPMINUWrm, 0 },
971  { X86::VPMAXSBrr, X86::VPMAXSBrm, 0 },
972  { X86::VPMAXSDrr, X86::VPMAXSDrm, 0 },
973  { X86::VPMAXUDrr, X86::VPMAXUDrm, 0 },
974  { X86::VPMAXUWrr, X86::VPMAXUWrm, 0 },
975  { X86::VPMULDQrr, X86::VPMULDQrm, 0 },
976  { X86::VPMULHRSWrr128, X86::VPMULHRSWrm128, 0 },
977  { X86::VPMULHUWrr, X86::VPMULHUWrm, 0 },
978  { X86::VPMULHWrr, X86::VPMULHWrm, 0 },
979  { X86::VPMULLDrr, X86::VPMULLDrm, 0 },
980  { X86::VPMULLWrr, X86::VPMULLWrm, 0 },
981  { X86::VPMULUDQrr, X86::VPMULUDQrm, 0 },
982  { X86::VPORrr, X86::VPORrm, 0 },
983  { X86::VPSADBWrr, X86::VPSADBWrm, 0 },
984  { X86::VPSHUFBrr, X86::VPSHUFBrm, 0 },
985  { X86::VPSIGNBrr, X86::VPSIGNBrm, 0 },
986  { X86::VPSIGNWrr, X86::VPSIGNWrm, 0 },
987  { X86::VPSIGNDrr, X86::VPSIGNDrm, 0 },
988  { X86::VPSLLDrr, X86::VPSLLDrm, 0 },
989  { X86::VPSLLQrr, X86::VPSLLQrm, 0 },
990  { X86::VPSLLWrr, X86::VPSLLWrm, 0 },
991  { X86::VPSRADrr, X86::VPSRADrm, 0 },
992  { X86::VPSRAWrr, X86::VPSRAWrm, 0 },
993  { X86::VPSRLDrr, X86::VPSRLDrm, 0 },
994  { X86::VPSRLQrr, X86::VPSRLQrm, 0 },
995  { X86::VPSRLWrr, X86::VPSRLWrm, 0 },
996  { X86::VPSUBBrr, X86::VPSUBBrm, 0 },
997  { X86::VPSUBDrr, X86::VPSUBDrm, 0 },
998  { X86::VPSUBSBrr, X86::VPSUBSBrm, 0 },
999  { X86::VPSUBSWrr, X86::VPSUBSWrm, 0 },
1000  { X86::VPSUBWrr, X86::VPSUBWrm, 0 },
1001  { X86::VPUNPCKHBWrr, X86::VPUNPCKHBWrm, 0 },
1002  { X86::VPUNPCKHDQrr, X86::VPUNPCKHDQrm, 0 },
1003  { X86::VPUNPCKHQDQrr, X86::VPUNPCKHQDQrm, 0 },
1004  { X86::VPUNPCKHWDrr, X86::VPUNPCKHWDrm, 0 },
1005  { X86::VPUNPCKLBWrr, X86::VPUNPCKLBWrm, 0 },
1006  { X86::VPUNPCKLDQrr, X86::VPUNPCKLDQrm, 0 },
1007  { X86::VPUNPCKLQDQrr, X86::VPUNPCKLQDQrm, 0 },
1008  { X86::VPUNPCKLWDrr, X86::VPUNPCKLWDrm, 0 },
1009  { X86::VPXORrr, X86::VPXORrm, 0 },
1010  { X86::VSHUFPDrri, X86::VSHUFPDrmi, 0 },
1011  { X86::VSHUFPSrri, X86::VSHUFPSrmi, 0 },
1012  { X86::VSUBPDrr, X86::VSUBPDrm, 0 },
1013  { X86::VSUBPSrr, X86::VSUBPSrm, 0 },
1014  { X86::VSUBSDrr, X86::VSUBSDrm, 0 },
1015  { X86::VSUBSSrr, X86::VSUBSSrm, 0 },
1016  { X86::VUNPCKHPDrr, X86::VUNPCKHPDrm, 0 },
1017  { X86::VUNPCKHPSrr, X86::VUNPCKHPSrm, 0 },
1018  { X86::VUNPCKLPDrr, X86::VUNPCKLPDrm, 0 },
1019  { X86::VUNPCKLPSrr, X86::VUNPCKLPSrm, 0 },
1020  { X86::VXORPDrr, X86::VXORPDrm, 0 },
1021  { X86::VXORPSrr, X86::VXORPSrm, 0 },
1022  // AVX 256-bit foldable instructions
1023  { X86::VADDPDYrr, X86::VADDPDYrm, 0 },
1024  { X86::VADDPSYrr, X86::VADDPSYrm, 0 },
1025  { X86::VADDSUBPDYrr, X86::VADDSUBPDYrm, 0 },
1026  { X86::VADDSUBPSYrr, X86::VADDSUBPSYrm, 0 },
1027  { X86::VANDNPDYrr, X86::VANDNPDYrm, 0 },
1028  { X86::VANDNPSYrr, X86::VANDNPSYrm, 0 },
1029  { X86::VANDPDYrr, X86::VANDPDYrm, 0 },
1030  { X86::VANDPSYrr, X86::VANDPSYrm, 0 },
1031  { X86::VBLENDPDYrri, X86::VBLENDPDYrmi, 0 },
1032  { X86::VBLENDPSYrri, X86::VBLENDPSYrmi, 0 },
1033  { X86::VBLENDVPDYrr, X86::VBLENDVPDYrm, 0 },
1034  { X86::VBLENDVPSYrr, X86::VBLENDVPSYrm, 0 },
1035  { X86::VCMPPDYrri, X86::VCMPPDYrmi, 0 },
1036  { X86::VCMPPSYrri, X86::VCMPPSYrmi, 0 },
1037  { X86::VDIVPDYrr, X86::VDIVPDYrm, 0 },
1038  { X86::VDIVPSYrr, X86::VDIVPSYrm, 0 },
1039  { X86::VHADDPDYrr, X86::VHADDPDYrm, 0 },
1040  { X86::VHADDPSYrr, X86::VHADDPSYrm, 0 },
1041  { X86::VHSUBPDYrr, X86::VHSUBPDYrm, 0 },
1042  { X86::VHSUBPSYrr, X86::VHSUBPSYrm, 0 },
1043  { X86::VINSERTF128rr, X86::VINSERTF128rm, 0 },
1044  { X86::VMAXPDYrr, X86::VMAXPDYrm, 0 },
1045  { X86::VMAXPSYrr, X86::VMAXPSYrm, 0 },
1046  { X86::VMINPDYrr, X86::VMINPDYrm, 0 },
1047  { X86::VMINPSYrr, X86::VMINPSYrm, 0 },
1048  { X86::VMULPDYrr, X86::VMULPDYrm, 0 },
1049  { X86::VMULPSYrr, X86::VMULPSYrm, 0 },
1050  { X86::VORPDYrr, X86::VORPDYrm, 0 },
1051  { X86::VORPSYrr, X86::VORPSYrm, 0 },
1052  { X86::VPERM2F128rr, X86::VPERM2F128rm, 0 },
1053  { X86::VPERMILPDYrr, X86::VPERMILPDYrm, 0 },
1054  { X86::VPERMILPSYrr, X86::VPERMILPSYrm, 0 },
1055  { X86::VSHUFPDYrri, X86::VSHUFPDYrmi, 0 },
1056  { X86::VSHUFPSYrri, X86::VSHUFPSYrmi, 0 },
1057  { X86::VSUBPDYrr, X86::VSUBPDYrm, 0 },
1058  { X86::VSUBPSYrr, X86::VSUBPSYrm, 0 },
1059  { X86::VUNPCKHPDYrr, X86::VUNPCKHPDYrm, 0 },
1060  { X86::VUNPCKHPSYrr, X86::VUNPCKHPSYrm, 0 },
1061  { X86::VUNPCKLPDYrr, X86::VUNPCKLPDYrm, 0 },
1062  { X86::VUNPCKLPSYrr, X86::VUNPCKLPSYrm, 0 },
1063  { X86::VXORPDYrr, X86::VXORPDYrm, 0 },
1064  { X86::VXORPSYrr, X86::VXORPSYrm, 0 },
1065  // AVX2 foldable instructions
1066  { X86::VINSERTI128rr, X86::VINSERTI128rm, 0 },
1067  { X86::VPACKSSDWYrr, X86::VPACKSSDWYrm, 0 },
1068  { X86::VPACKSSWBYrr, X86::VPACKSSWBYrm, 0 },
1069  { X86::VPACKUSDWYrr, X86::VPACKUSDWYrm, 0 },
1070  { X86::VPACKUSWBYrr, X86::VPACKUSWBYrm, 0 },
1071  { X86::VPADDBYrr, X86::VPADDBYrm, 0 },
1072  { X86::VPADDDYrr, X86::VPADDDYrm, 0 },
1073  { X86::VPADDQYrr, X86::VPADDQYrm, 0 },
1074  { X86::VPADDSBYrr, X86::VPADDSBYrm, 0 },
1075  { X86::VPADDSWYrr, X86::VPADDSWYrm, 0 },
1076  { X86::VPADDUSBYrr, X86::VPADDUSBYrm, 0 },
1077  { X86::VPADDUSWYrr, X86::VPADDUSWYrm, 0 },
1078  { X86::VPADDWYrr, X86::VPADDWYrm, 0 },
1079  { X86::VPALIGNR256rr, X86::VPALIGNR256rm, 0 },
1080  { X86::VPANDNYrr, X86::VPANDNYrm, 0 },
1081  { X86::VPANDYrr, X86::VPANDYrm, 0 },
1082  { X86::VPAVGBYrr, X86::VPAVGBYrm, 0 },
1083  { X86::VPAVGWYrr, X86::VPAVGWYrm, 0 },
1084  { X86::VPBLENDDrri, X86::VPBLENDDrmi, 0 },
1085  { X86::VPBLENDDYrri, X86::VPBLENDDYrmi, 0 },
1086  { X86::VPBLENDWYrri, X86::VPBLENDWYrmi, 0 },
1087  { X86::VPCMPEQBYrr, X86::VPCMPEQBYrm, 0 },
1088  { X86::VPCMPEQDYrr, X86::VPCMPEQDYrm, 0 },
1089  { X86::VPCMPEQQYrr, X86::VPCMPEQQYrm, 0 },
1090  { X86::VPCMPEQWYrr, X86::VPCMPEQWYrm, 0 },
1091  { X86::VPCMPGTBYrr, X86::VPCMPGTBYrm, 0 },
1092  { X86::VPCMPGTDYrr, X86::VPCMPGTDYrm, 0 },
1093  { X86::VPCMPGTQYrr, X86::VPCMPGTQYrm, 0 },
1094  { X86::VPCMPGTWYrr, X86::VPCMPGTWYrm, 0 },
1095  { X86::VPERM2I128rr, X86::VPERM2I128rm, 0 },
1096  { X86::VPERMDYrr, X86::VPERMDYrm, 0 },
1097  { X86::VPERMPDYri, X86::VPERMPDYmi, 0 },
1098  { X86::VPERMPSYrr, X86::VPERMPSYrm, 0 },
1099  { X86::VPERMQYri, X86::VPERMQYmi, 0 },
1100  { X86::VPHADDDYrr, X86::VPHADDDYrm, 0 },
1101  { X86::VPHADDSWrr256, X86::VPHADDSWrm256, 0 },
1102  { X86::VPHADDWYrr, X86::VPHADDWYrm, 0 },
1103  { X86::VPHSUBDYrr, X86::VPHSUBDYrm, 0 },
1104  { X86::VPHSUBSWrr256, X86::VPHSUBSWrm256, 0 },
1105  { X86::VPHSUBWYrr, X86::VPHSUBWYrm, 0 },
1106  { X86::VPMADDUBSWrr256, X86::VPMADDUBSWrm256, 0 },
1107  { X86::VPMADDWDYrr, X86::VPMADDWDYrm, 0 },
1108  { X86::VPMAXSWYrr, X86::VPMAXSWYrm, 0 },
1109  { X86::VPMAXUBYrr, X86::VPMAXUBYrm, 0 },
1110  { X86::VPMINSWYrr, X86::VPMINSWYrm, 0 },
1111  { X86::VPMINUBYrr, X86::VPMINUBYrm, 0 },
1112  { X86::VPMINSBYrr, X86::VPMINSBYrm, 0 },
1113  { X86::VPMINSDYrr, X86::VPMINSDYrm, 0 },
1114  { X86::VPMINUDYrr, X86::VPMINUDYrm, 0 },
1115  { X86::VPMINUWYrr, X86::VPMINUWYrm, 0 },
1116  { X86::VPMAXSBYrr, X86::VPMAXSBYrm, 0 },
1117  { X86::VPMAXSDYrr, X86::VPMAXSDYrm, 0 },
1118  { X86::VPMAXUDYrr, X86::VPMAXUDYrm, 0 },
1119  { X86::VPMAXUWYrr, X86::VPMAXUWYrm, 0 },
1120  { X86::VMPSADBWYrri, X86::VMPSADBWYrmi, 0 },
1121  { X86::VPMULDQYrr, X86::VPMULDQYrm, 0 },
1122  { X86::VPMULHRSWrr256, X86::VPMULHRSWrm256, 0 },
1123  { X86::VPMULHUWYrr, X86::VPMULHUWYrm, 0 },
1124  { X86::VPMULHWYrr, X86::VPMULHWYrm, 0 },
1125  { X86::VPMULLDYrr, X86::VPMULLDYrm, 0 },
1126  { X86::VPMULLWYrr, X86::VPMULLWYrm, 0 },
1127  { X86::VPMULUDQYrr, X86::VPMULUDQYrm, 0 },
1128  { X86::VPORYrr, X86::VPORYrm, 0 },
1129  { X86::VPSADBWYrr, X86::VPSADBWYrm, 0 },
1130  { X86::VPSHUFBYrr, X86::VPSHUFBYrm, 0 },
1131  { X86::VPSIGNBYrr, X86::VPSIGNBYrm, 0 },
1132  { X86::VPSIGNWYrr, X86::VPSIGNWYrm, 0 },
1133  { X86::VPSIGNDYrr, X86::VPSIGNDYrm, 0 },
1134  { X86::VPSLLDYrr, X86::VPSLLDYrm, 0 },
1135  { X86::VPSLLQYrr, X86::VPSLLQYrm, 0 },
1136  { X86::VPSLLWYrr, X86::VPSLLWYrm, 0 },
1137  { X86::VPSLLVDrr, X86::VPSLLVDrm, 0 },
1138  { X86::VPSLLVDYrr, X86::VPSLLVDYrm, 0 },
1139  { X86::VPSLLVQrr, X86::VPSLLVQrm, 0 },
1140  { X86::VPSLLVQYrr, X86::VPSLLVQYrm, 0 },
1141  { X86::VPSRADYrr, X86::VPSRADYrm, 0 },
1142  { X86::VPSRAWYrr, X86::VPSRAWYrm, 0 },
1143  { X86::VPSRAVDrr, X86::VPSRAVDrm, 0 },
1144  { X86::VPSRAVDYrr, X86::VPSRAVDYrm, 0 },
1145  { X86::VPSRLDYrr, X86::VPSRLDYrm, 0 },
1146  { X86::VPSRLQYrr, X86::VPSRLQYrm, 0 },
1147  { X86::VPSRLWYrr, X86::VPSRLWYrm, 0 },
1148  { X86::VPSRLVDrr, X86::VPSRLVDrm, 0 },
1149  { X86::VPSRLVDYrr, X86::VPSRLVDYrm, 0 },
1150  { X86::VPSRLVQrr, X86::VPSRLVQrm, 0 },
1151  { X86::VPSRLVQYrr, X86::VPSRLVQYrm, 0 },
1152  { X86::VPSUBBYrr, X86::VPSUBBYrm, 0 },
1153  { X86::VPSUBDYrr, X86::VPSUBDYrm, 0 },
1154  { X86::VPSUBSBYrr, X86::VPSUBSBYrm, 0 },
1155  { X86::VPSUBSWYrr, X86::VPSUBSWYrm, 0 },
1156  { X86::VPSUBWYrr, X86::VPSUBWYrm, 0 },
1157  { X86::VPUNPCKHBWYrr, X86::VPUNPCKHBWYrm, 0 },
1158  { X86::VPUNPCKHDQYrr, X86::VPUNPCKHDQYrm, 0 },
1159  { X86::VPUNPCKHQDQYrr, X86::VPUNPCKHQDQYrm, 0 },
1160  { X86::VPUNPCKHWDYrr, X86::VPUNPCKHWDYrm, 0 },
1161  { X86::VPUNPCKLBWYrr, X86::VPUNPCKLBWYrm, 0 },
1162  { X86::VPUNPCKLDQYrr, X86::VPUNPCKLDQYrm, 0 },
1163  { X86::VPUNPCKLQDQYrr, X86::VPUNPCKLQDQYrm, 0 },
1164  { X86::VPUNPCKLWDYrr, X86::VPUNPCKLWDYrm, 0 },
1165  { X86::VPXORYrr, X86::VPXORYrm, 0 },
1166  // FIXME: add AVX 256-bit foldable instructions
1167 
1168  // FMA4 foldable patterns
1169  { X86::VFMADDSS4rr, X86::VFMADDSS4mr, 0 },
1170  { X86::VFMADDSD4rr, X86::VFMADDSD4mr, 0 },
1171  { X86::VFMADDPS4rr, X86::VFMADDPS4mr, TB_ALIGN_16 },
1172  { X86::VFMADDPD4rr, X86::VFMADDPD4mr, TB_ALIGN_16 },
1173  { X86::VFMADDPS4rrY, X86::VFMADDPS4mrY, TB_ALIGN_32 },
1174  { X86::VFMADDPD4rrY, X86::VFMADDPD4mrY, TB_ALIGN_32 },
1175  { X86::VFNMADDSS4rr, X86::VFNMADDSS4mr, 0 },
1176  { X86::VFNMADDSD4rr, X86::VFNMADDSD4mr, 0 },
1177  { X86::VFNMADDPS4rr, X86::VFNMADDPS4mr, TB_ALIGN_16 },
1178  { X86::VFNMADDPD4rr, X86::VFNMADDPD4mr, TB_ALIGN_16 },
1179  { X86::VFNMADDPS4rrY, X86::VFNMADDPS4mrY, TB_ALIGN_32 },
1180  { X86::VFNMADDPD4rrY, X86::VFNMADDPD4mrY, TB_ALIGN_32 },
1181  { X86::VFMSUBSS4rr, X86::VFMSUBSS4mr, 0 },
1182  { X86::VFMSUBSD4rr, X86::VFMSUBSD4mr, 0 },
1183  { X86::VFMSUBPS4rr, X86::VFMSUBPS4mr, TB_ALIGN_16 },
1184  { X86::VFMSUBPD4rr, X86::VFMSUBPD4mr, TB_ALIGN_16 },
1185  { X86::VFMSUBPS4rrY, X86::VFMSUBPS4mrY, TB_ALIGN_32 },
1186  { X86::VFMSUBPD4rrY, X86::VFMSUBPD4mrY, TB_ALIGN_32 },
1187  { X86::VFNMSUBSS4rr, X86::VFNMSUBSS4mr, 0 },
1188  { X86::VFNMSUBSD4rr, X86::VFNMSUBSD4mr, 0 },
1189  { X86::VFNMSUBPS4rr, X86::VFNMSUBPS4mr, TB_ALIGN_16 },
1190  { X86::VFNMSUBPD4rr, X86::VFNMSUBPD4mr, TB_ALIGN_16 },
1191  { X86::VFNMSUBPS4rrY, X86::VFNMSUBPS4mrY, TB_ALIGN_32 },
1192  { X86::VFNMSUBPD4rrY, X86::VFNMSUBPD4mrY, TB_ALIGN_32 },
1193  { X86::VFMADDSUBPS4rr, X86::VFMADDSUBPS4mr, TB_ALIGN_16 },
1194  { X86::VFMADDSUBPD4rr, X86::VFMADDSUBPD4mr, TB_ALIGN_16 },
1195  { X86::VFMADDSUBPS4rrY, X86::VFMADDSUBPS4mrY, TB_ALIGN_32 },
1196  { X86::VFMADDSUBPD4rrY, X86::VFMADDSUBPD4mrY, TB_ALIGN_32 },
1197  { X86::VFMSUBADDPS4rr, X86::VFMSUBADDPS4mr, TB_ALIGN_16 },
1198  { X86::VFMSUBADDPD4rr, X86::VFMSUBADDPD4mr, TB_ALIGN_16 },
1199  { X86::VFMSUBADDPS4rrY, X86::VFMSUBADDPS4mrY, TB_ALIGN_32 },
1200  { X86::VFMSUBADDPD4rrY, X86::VFMSUBADDPD4mrY, TB_ALIGN_32 },
1201 
1202  // BMI/BMI2 foldable instructions
1203  { X86::ANDN32rr, X86::ANDN32rm, 0 },
1204  { X86::ANDN64rr, X86::ANDN64rm, 0 },
1205  { X86::MULX32rr, X86::MULX32rm, 0 },
1206  { X86::MULX64rr, X86::MULX64rm, 0 },
1207  { X86::PDEP32rr, X86::PDEP32rm, 0 },
1208  { X86::PDEP64rr, X86::PDEP64rm, 0 },
1209  { X86::PEXT32rr, X86::PEXT32rm, 0 },
1210  { X86::PEXT64rr, X86::PEXT64rm, 0 },
1211 
1212  // AVX-512 foldable instructions
1213  { X86::VPADDDZrr, X86::VPADDDZrm, 0 },
1214  { X86::VPADDQZrr, X86::VPADDQZrm, 0 },
1215  { X86::VADDPSZrr, X86::VADDPSZrm, 0 },
1216  { X86::VADDPDZrr, X86::VADDPDZrm, 0 },
1217  { X86::VSUBPSZrr, X86::VSUBPSZrm, 0 },
1218  { X86::VSUBPDZrr, X86::VSUBPDZrm, 0 },
1219  { X86::VMULPSZrr, X86::VMULPSZrm, 0 },
1220  { X86::VMULPDZrr, X86::VMULPDZrm, 0 },
1221  { X86::VDIVPSZrr, X86::VDIVPSZrm, 0 },
1222  { X86::VDIVPDZrr, X86::VDIVPDZrm, 0 },
1223  { X86::VMINPSZrr, X86::VMINPSZrm, 0 },
1224  { X86::VMINPDZrr, X86::VMINPDZrm, 0 },
1225  { X86::VMAXPSZrr, X86::VMAXPSZrm, 0 },
1226  { X86::VMAXPDZrr, X86::VMAXPDZrm, 0 },
1227  { X86::VPERMPDZri, X86::VPERMPDZmi, 0 },
1228  { X86::VPERMPSZrr, X86::VPERMPSZrm, 0 },
1229  { X86::VPSLLVDZrr, X86::VPSLLVDZrm, 0 },
1230  { X86::VPSLLVQZrr, X86::VPSLLVQZrm, 0 },
1231  { X86::VPSRAVDZrr, X86::VPSRAVDZrm, 0 },
1232  { X86::VPSRLVDZrr, X86::VPSRLVDZrm, 0 },
1233  { X86::VPSRLVQZrr, X86::VPSRLVQZrm, 0 },
1234  { X86::VSHUFPDZrri, X86::VSHUFPDZrmi, 0 },
1235  { X86::VSHUFPSZrri, X86::VSHUFPSZrmi, 0 },
1236  { X86::VALIGNQrri, X86::VALIGNQrmi, 0 },
1237  { X86::VALIGNDrri, X86::VALIGNDrmi, 0 },
1238 
1239  // AES foldable instructions
1240  { X86::AESDECLASTrr, X86::AESDECLASTrm, TB_ALIGN_16 },
1241  { X86::AESDECrr, X86::AESDECrm, TB_ALIGN_16 },
1242  { X86::AESENCLASTrr, X86::AESENCLASTrm, TB_ALIGN_16 },
1243  { X86::AESENCrr, X86::AESENCrm, TB_ALIGN_16 },
1244  { X86::VAESDECLASTrr, X86::VAESDECLASTrm, TB_ALIGN_16 },
1245  { X86::VAESDECrr, X86::VAESDECrm, TB_ALIGN_16 },
1246  { X86::VAESENCLASTrr, X86::VAESENCLASTrm, TB_ALIGN_16 },
1247  { X86::VAESENCrr, X86::VAESENCrm, TB_ALIGN_16 },
1248 
1249  // SHA foldable instructions
1250  { X86::SHA1MSG1rr, X86::SHA1MSG1rm, TB_ALIGN_16 },
1251  { X86::SHA1MSG2rr, X86::SHA1MSG2rm, TB_ALIGN_16 },
1252  { X86::SHA1NEXTErr, X86::SHA1NEXTErm, TB_ALIGN_16 },
1253  { X86::SHA1RNDS4rri, X86::SHA1RNDS4rmi, TB_ALIGN_16 },
1254  { X86::SHA256MSG1rr, X86::SHA256MSG1rm, TB_ALIGN_16 },
1255  { X86::SHA256MSG2rr, X86::SHA256MSG2rm, TB_ALIGN_16 },
1256  { X86::SHA256RNDS2rr, X86::SHA256RNDS2rm, TB_ALIGN_16 },
1257  };
1258 
1259  for (unsigned i = 0, e = array_lengthof(OpTbl2); i != e; ++i) {
1260  unsigned RegOp = OpTbl2[i].RegOp;
1261  unsigned MemOp = OpTbl2[i].MemOp;
1262  unsigned Flags = OpTbl2[i].Flags;
1263  AddTableEntry(RegOp2MemOpTable2, MemOp2RegOpTable,
1264  RegOp, MemOp,
1265  // Index 2, folded load
1266  Flags | TB_INDEX_2 | TB_FOLDED_LOAD);
1267  }
1268 
1269  static const X86OpTblEntry OpTbl3[] = {
1270  // FMA foldable instructions
1271  { X86::VFMADDSSr231r, X86::VFMADDSSr231m, 0 },
1272  { X86::VFMADDSDr231r, X86::VFMADDSDr231m, 0 },
1273  { X86::VFMADDSSr132r, X86::VFMADDSSr132m, 0 },
1274  { X86::VFMADDSDr132r, X86::VFMADDSDr132m, 0 },
1275  { X86::VFMADDSSr213r, X86::VFMADDSSr213m, 0 },
1276  { X86::VFMADDSDr213r, X86::VFMADDSDr213m, 0 },
1277  { X86::VFMADDSSr213r_Int, X86::VFMADDSSr213m_Int, 0 },
1278  { X86::VFMADDSDr213r_Int, X86::VFMADDSDr213m_Int, 0 },
1279 
1280  { X86::VFMADDPSr231r, X86::VFMADDPSr231m, TB_ALIGN_16 },
1281  { X86::VFMADDPDr231r, X86::VFMADDPDr231m, TB_ALIGN_16 },
1282  { X86::VFMADDPSr132r, X86::VFMADDPSr132m, TB_ALIGN_16 },
1283  { X86::VFMADDPDr132r, X86::VFMADDPDr132m, TB_ALIGN_16 },
1284  { X86::VFMADDPSr213r, X86::VFMADDPSr213m, TB_ALIGN_16 },
1285  { X86::VFMADDPDr213r, X86::VFMADDPDr213m, TB_ALIGN_16 },
1286  { X86::VFMADDPSr231rY, X86::VFMADDPSr231mY, TB_ALIGN_32 },
1287  { X86::VFMADDPDr231rY, X86::VFMADDPDr231mY, TB_ALIGN_32 },
1288  { X86::VFMADDPSr132rY, X86::VFMADDPSr132mY, TB_ALIGN_32 },
1289  { X86::VFMADDPDr132rY, X86::VFMADDPDr132mY, TB_ALIGN_32 },
1290  { X86::VFMADDPSr213rY, X86::VFMADDPSr213mY, TB_ALIGN_32 },
1291  { X86::VFMADDPDr213rY, X86::VFMADDPDr213mY, TB_ALIGN_32 },
1292 
1293  { X86::VFNMADDSSr231r, X86::VFNMADDSSr231m, 0 },
1294  { X86::VFNMADDSDr231r, X86::VFNMADDSDr231m, 0 },
1295  { X86::VFNMADDSSr132r, X86::VFNMADDSSr132m, 0 },
1296  { X86::VFNMADDSDr132r, X86::VFNMADDSDr132m, 0 },
1297  { X86::VFNMADDSSr213r, X86::VFNMADDSSr213m, 0 },
1298  { X86::VFNMADDSDr213r, X86::VFNMADDSDr213m, 0 },
1299  { X86::VFNMADDSSr213r_Int, X86::VFNMADDSSr213m_Int, 0 },
1300  { X86::VFNMADDSDr213r_Int, X86::VFNMADDSDr213m_Int, 0 },
1301 
1302  { X86::VFNMADDPSr231r, X86::VFNMADDPSr231m, TB_ALIGN_16 },
1303  { X86::VFNMADDPDr231r, X86::VFNMADDPDr231m, TB_ALIGN_16 },
1304  { X86::VFNMADDPSr132r, X86::VFNMADDPSr132m, TB_ALIGN_16 },
1305  { X86::VFNMADDPDr132r, X86::VFNMADDPDr132m, TB_ALIGN_16 },
1306  { X86::VFNMADDPSr213r, X86::VFNMADDPSr213m, TB_ALIGN_16 },
1307  { X86::VFNMADDPDr213r, X86::VFNMADDPDr213m, TB_ALIGN_16 },
1308  { X86::VFNMADDPSr231rY, X86::VFNMADDPSr231mY, TB_ALIGN_32 },
1309  { X86::VFNMADDPDr231rY, X86::VFNMADDPDr231mY, TB_ALIGN_32 },
1310  { X86::VFNMADDPSr132rY, X86::VFNMADDPSr132mY, TB_ALIGN_32 },
1311  { X86::VFNMADDPDr132rY, X86::VFNMADDPDr132mY, TB_ALIGN_32 },
1312  { X86::VFNMADDPSr213rY, X86::VFNMADDPSr213mY, TB_ALIGN_32 },
1313  { X86::VFNMADDPDr213rY, X86::VFNMADDPDr213mY, TB_ALIGN_32 },
1314 
1315  { X86::VFMSUBSSr231r, X86::VFMSUBSSr231m, 0 },
1316  { X86::VFMSUBSDr231r, X86::VFMSUBSDr231m, 0 },
1317  { X86::VFMSUBSSr132r, X86::VFMSUBSSr132m, 0 },
1318  { X86::VFMSUBSDr132r, X86::VFMSUBSDr132m, 0 },
1319  { X86::VFMSUBSSr213r, X86::VFMSUBSSr213m, 0 },
1320  { X86::VFMSUBSDr213r, X86::VFMSUBSDr213m, 0 },
1321  { X86::VFMSUBSSr213r_Int, X86::VFMSUBSSr213m_Int, 0 },
1322  { X86::VFMSUBSDr213r_Int, X86::VFMSUBSDr213m_Int, 0 },
1323 
1324  { X86::VFMSUBPSr231r, X86::VFMSUBPSr231m, TB_ALIGN_16 },
1325  { X86::VFMSUBPDr231r, X86::VFMSUBPDr231m, TB_ALIGN_16 },
1326  { X86::VFMSUBPSr132r, X86::VFMSUBPSr132m, TB_ALIGN_16 },
1327  { X86::VFMSUBPDr132r, X86::VFMSUBPDr132m, TB_ALIGN_16 },
1328  { X86::VFMSUBPSr213r, X86::VFMSUBPSr213m, TB_ALIGN_16 },
1329  { X86::VFMSUBPDr213r, X86::VFMSUBPDr213m, TB_ALIGN_16 },
1330  { X86::VFMSUBPSr231rY, X86::VFMSUBPSr231mY, TB_ALIGN_32 },
1331  { X86::VFMSUBPDr231rY, X86::VFMSUBPDr231mY, TB_ALIGN_32 },
1332  { X86::VFMSUBPSr132rY, X86::VFMSUBPSr132mY, TB_ALIGN_32 },
1333  { X86::VFMSUBPDr132rY, X86::VFMSUBPDr132mY, TB_ALIGN_32 },
1334  { X86::VFMSUBPSr213rY, X86::VFMSUBPSr213mY, TB_ALIGN_32 },
1335  { X86::VFMSUBPDr213rY, X86::VFMSUBPDr213mY, TB_ALIGN_32 },
1336 
1337  { X86::VFNMSUBSSr231r, X86::VFNMSUBSSr231m, 0 },
1338  { X86::VFNMSUBSDr231r, X86::VFNMSUBSDr231m, 0 },
1339  { X86::VFNMSUBSSr132r, X86::VFNMSUBSSr132m, 0 },
1340  { X86::VFNMSUBSDr132r, X86::VFNMSUBSDr132m, 0 },
1341  { X86::VFNMSUBSSr213r, X86::VFNMSUBSSr213m, 0 },
1342  { X86::VFNMSUBSDr213r, X86::VFNMSUBSDr213m, 0 },
1343  { X86::VFNMSUBSSr213r_Int, X86::VFNMSUBSSr213m_Int, 0 },
1344  { X86::VFNMSUBSDr213r_Int, X86::VFNMSUBSDr213m_Int, 0 },
1345 
1346  { X86::VFNMSUBPSr231r, X86::VFNMSUBPSr231m, TB_ALIGN_16 },
1347  { X86::VFNMSUBPDr231r, X86::VFNMSUBPDr231m, TB_ALIGN_16 },
1348  { X86::VFNMSUBPSr132r, X86::VFNMSUBPSr132m, TB_ALIGN_16 },
1349  { X86::VFNMSUBPDr132r, X86::VFNMSUBPDr132m, TB_ALIGN_16 },
1350  { X86::VFNMSUBPSr213r, X86::VFNMSUBPSr213m, TB_ALIGN_16 },
1351  { X86::VFNMSUBPDr213r, X86::VFNMSUBPDr213m, TB_ALIGN_16 },
1352  { X86::VFNMSUBPSr231rY, X86::VFNMSUBPSr231mY, TB_ALIGN_32 },
1353  { X86::VFNMSUBPDr231rY, X86::VFNMSUBPDr231mY, TB_ALIGN_32 },
1354  { X86::VFNMSUBPSr132rY, X86::VFNMSUBPSr132mY, TB_ALIGN_32 },
1355  { X86::VFNMSUBPDr132rY, X86::VFNMSUBPDr132mY, TB_ALIGN_32 },
1356  { X86::VFNMSUBPSr213rY, X86::VFNMSUBPSr213mY, TB_ALIGN_32 },
1357  { X86::VFNMSUBPDr213rY, X86::VFNMSUBPDr213mY, TB_ALIGN_32 },
1358 
1359  { X86::VFMADDSUBPSr231r, X86::VFMADDSUBPSr231m, TB_ALIGN_16 },
1360  { X86::VFMADDSUBPDr231r, X86::VFMADDSUBPDr231m, TB_ALIGN_16 },
1361  { X86::VFMADDSUBPSr132r, X86::VFMADDSUBPSr132m, TB_ALIGN_16 },
1362  { X86::VFMADDSUBPDr132r, X86::VFMADDSUBPDr132m, TB_ALIGN_16 },
1363  { X86::VFMADDSUBPSr213r, X86::VFMADDSUBPSr213m, TB_ALIGN_16 },
1364  { X86::VFMADDSUBPDr213r, X86::VFMADDSUBPDr213m, TB_ALIGN_16 },
1365  { X86::VFMADDSUBPSr231rY, X86::VFMADDSUBPSr231mY, TB_ALIGN_32 },
1366  { X86::VFMADDSUBPDr231rY, X86::VFMADDSUBPDr231mY, TB_ALIGN_32 },
1367  { X86::VFMADDSUBPSr132rY, X86::VFMADDSUBPSr132mY, TB_ALIGN_32 },
1368  { X86::VFMADDSUBPDr132rY, X86::VFMADDSUBPDr132mY, TB_ALIGN_32 },
1369  { X86::VFMADDSUBPSr213rY, X86::VFMADDSUBPSr213mY, TB_ALIGN_32 },
1370  { X86::VFMADDSUBPDr213rY, X86::VFMADDSUBPDr213mY, TB_ALIGN_32 },
1371 
1372  { X86::VFMSUBADDPSr231r, X86::VFMSUBADDPSr231m, TB_ALIGN_16 },
1373  { X86::VFMSUBADDPDr231r, X86::VFMSUBADDPDr231m, TB_ALIGN_16 },
1374  { X86::VFMSUBADDPSr132r, X86::VFMSUBADDPSr132m, TB_ALIGN_16 },
1375  { X86::VFMSUBADDPDr132r, X86::VFMSUBADDPDr132m, TB_ALIGN_16 },
1376  { X86::VFMSUBADDPSr213r, X86::VFMSUBADDPSr213m, TB_ALIGN_16 },
1377  { X86::VFMSUBADDPDr213r, X86::VFMSUBADDPDr213m, TB_ALIGN_16 },
1378  { X86::VFMSUBADDPSr231rY, X86::VFMSUBADDPSr231mY, TB_ALIGN_32 },
1379  { X86::VFMSUBADDPDr231rY, X86::VFMSUBADDPDr231mY, TB_ALIGN_32 },
1380  { X86::VFMSUBADDPSr132rY, X86::VFMSUBADDPSr132mY, TB_ALIGN_32 },
1381  { X86::VFMSUBADDPDr132rY, X86::VFMSUBADDPDr132mY, TB_ALIGN_32 },
1382  { X86::VFMSUBADDPSr213rY, X86::VFMSUBADDPSr213mY, TB_ALIGN_32 },
1383  { X86::VFMSUBADDPDr213rY, X86::VFMSUBADDPDr213mY, TB_ALIGN_32 },
1384 
1385  // FMA4 foldable patterns
1386  { X86::VFMADDSS4rr, X86::VFMADDSS4rm, 0 },
1387  { X86::VFMADDSD4rr, X86::VFMADDSD4rm, 0 },
1388  { X86::VFMADDPS4rr, X86::VFMADDPS4rm, TB_ALIGN_16 },
1389  { X86::VFMADDPD4rr, X86::VFMADDPD4rm, TB_ALIGN_16 },
1390  { X86::VFMADDPS4rrY, X86::VFMADDPS4rmY, TB_ALIGN_32 },
1391  { X86::VFMADDPD4rrY, X86::VFMADDPD4rmY, TB_ALIGN_32 },
1392  { X86::VFNMADDSS4rr, X86::VFNMADDSS4rm, 0 },
1393  { X86::VFNMADDSD4rr, X86::VFNMADDSD4rm, 0 },
1394  { X86::VFNMADDPS4rr, X86::VFNMADDPS4rm, TB_ALIGN_16 },
1395  { X86::VFNMADDPD4rr, X86::VFNMADDPD4rm, TB_ALIGN_16 },
1396  { X86::VFNMADDPS4rrY, X86::VFNMADDPS4rmY, TB_ALIGN_32 },
1397  { X86::VFNMADDPD4rrY, X86::VFNMADDPD4rmY, TB_ALIGN_32 },
1398  { X86::VFMSUBSS4rr, X86::VFMSUBSS4rm, 0 },
1399  { X86::VFMSUBSD4rr, X86::VFMSUBSD4rm, 0 },
1400  { X86::VFMSUBPS4rr, X86::VFMSUBPS4rm, TB_ALIGN_16 },
1401  { X86::VFMSUBPD4rr, X86::VFMSUBPD4rm, TB_ALIGN_16 },
1402  { X86::VFMSUBPS4rrY, X86::VFMSUBPS4rmY, TB_ALIGN_32 },
1403  { X86::VFMSUBPD4rrY, X86::VFMSUBPD4rmY, TB_ALIGN_32 },
1404  { X86::VFNMSUBSS4rr, X86::VFNMSUBSS4rm, 0 },
1405  { X86::VFNMSUBSD4rr, X86::VFNMSUBSD4rm, 0 },
1406  { X86::VFNMSUBPS4rr, X86::VFNMSUBPS4rm, TB_ALIGN_16 },
1407  { X86::VFNMSUBPD4rr, X86::VFNMSUBPD4rm, TB_ALIGN_16 },
1408  { X86::VFNMSUBPS4rrY, X86::VFNMSUBPS4rmY, TB_ALIGN_32 },
1409  { X86::VFNMSUBPD4rrY, X86::VFNMSUBPD4rmY, TB_ALIGN_32 },
1410  { X86::VFMADDSUBPS4rr, X86::VFMADDSUBPS4rm, TB_ALIGN_16 },
1411  { X86::VFMADDSUBPD4rr, X86::VFMADDSUBPD4rm, TB_ALIGN_16 },
1412  { X86::VFMADDSUBPS4rrY, X86::VFMADDSUBPS4rmY, TB_ALIGN_32 },
1413  { X86::VFMADDSUBPD4rrY, X86::VFMADDSUBPD4rmY, TB_ALIGN_32 },
1414  { X86::VFMSUBADDPS4rr, X86::VFMSUBADDPS4rm, TB_ALIGN_16 },
1415  { X86::VFMSUBADDPD4rr, X86::VFMSUBADDPD4rm, TB_ALIGN_16 },
1416  { X86::VFMSUBADDPS4rrY, X86::VFMSUBADDPS4rmY, TB_ALIGN_32 },
1417  { X86::VFMSUBADDPD4rrY, X86::VFMSUBADDPD4rmY, TB_ALIGN_32 },
1418  // AVX-512 VPERMI instructions with 3 source operands.
1419  { X86::VPERMI2Drr, X86::VPERMI2Drm, 0 },
1420  { X86::VPERMI2Qrr, X86::VPERMI2Qrm, 0 },
1421  { X86::VPERMI2PSrr, X86::VPERMI2PSrm, 0 },
1422  { X86::VPERMI2PDrr, X86::VPERMI2PDrm, 0 },
1423  };
1424 
1425  for (unsigned i = 0, e = array_lengthof(OpTbl3); i != e; ++i) {
1426  unsigned RegOp = OpTbl3[i].RegOp;
1427  unsigned MemOp = OpTbl3[i].MemOp;
1428  unsigned Flags = OpTbl3[i].Flags;
1429  AddTableEntry(RegOp2MemOpTable3, MemOp2RegOpTable,
1430  RegOp, MemOp,
1431  // Index 3, folded load
1432  Flags | TB_INDEX_3 | TB_FOLDED_LOAD);
1433  }
1434 
1435 }
1436 
1437 void
1438 X86InstrInfo::AddTableEntry(RegOp2MemOpTableType &R2MTable,
1439  MemOp2RegOpTableType &M2RTable,
1440  unsigned RegOp, unsigned MemOp, unsigned Flags) {
1441  if ((Flags & TB_NO_FORWARD) == 0) {
1442  assert(!R2MTable.count(RegOp) && "Duplicate entry!");
1443  R2MTable[RegOp] = std::make_pair(MemOp, Flags);
1444  }
1445  if ((Flags & TB_NO_REVERSE) == 0) {
1446  assert(!M2RTable.count(MemOp) &&
1447  "Duplicated entries in unfolding maps?");
1448  M2RTable[MemOp] = std::make_pair(RegOp, Flags);
1449  }
1450 }
1451 
1452 bool
1453 X86InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
1454  unsigned &SrcReg, unsigned &DstReg,
1455  unsigned &SubIdx) const {
1456  switch (MI.getOpcode()) {
1457  default: break;
1458  case X86::MOVSX16rr8:
1459  case X86::MOVZX16rr8:
1460  case X86::MOVSX32rr8:
1461  case X86::MOVZX32rr8:
1462  case X86::MOVSX64rr8:
1463  if (!TM.getSubtarget<X86Subtarget>().is64Bit())
1464  // It's not always legal to reference the low 8-bit of the larger
1465  // register in 32-bit mode.
1466  return false;
1467  case X86::MOVSX32rr16:
1468  case X86::MOVZX32rr16:
1469  case X86::MOVSX64rr16:
1470  case X86::MOVSX64rr32: {
1471  if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg())
1472  // Be conservative.
1473  return false;
1474  SrcReg = MI.getOperand(1).getReg();
1475  DstReg = MI.getOperand(0).getReg();
1476  switch (MI.getOpcode()) {
1477  default: llvm_unreachable("Unreachable!");
1478  case X86::MOVSX16rr8:
1479  case X86::MOVZX16rr8:
1480  case X86::MOVSX32rr8:
1481  case X86::MOVZX32rr8:
1482  case X86::MOVSX64rr8:
1483  SubIdx = X86::sub_8bit;
1484  break;
1485  case X86::MOVSX32rr16:
1486  case X86::MOVZX32rr16:
1487  case X86::MOVSX64rr16:
1488  SubIdx = X86::sub_16bit;
1489  break;
1490  case X86::MOVSX64rr32:
1491  SubIdx = X86::sub_32bit;
1492  break;
1493  }
1494  return true;
1495  }
1496  }
1497  return false;
1498 }
1499 
1500 /// isFrameOperand - Return true and the FrameIndex if the specified
1501 /// operand and follow operands form a reference to the stack frame.
1502 bool X86InstrInfo::isFrameOperand(const MachineInstr *MI, unsigned int Op,
1503  int &FrameIndex) const {
1504  if (MI->getOperand(Op).isFI() && MI->getOperand(Op+1).isImm() &&
1505  MI->getOperand(Op+2).isReg() && MI->getOperand(Op+3).isImm() &&
1506  MI->getOperand(Op+1).getImm() == 1 &&
1507  MI->getOperand(Op+2).getReg() == 0 &&
1508  MI->getOperand(Op+3).getImm() == 0) {
1509  FrameIndex = MI->getOperand(Op).getIndex();
1510  return true;
1511  }
1512  return false;
1513 }
1514 
1515 static bool isFrameLoadOpcode(int Opcode) {
1516  switch (Opcode) {
1517  default:
1518  return false;
1519  case X86::MOV8rm:
1520  case X86::MOV16rm:
1521  case X86::MOV32rm:
1522  case X86::MOV64rm:
1523  case X86::LD_Fp64m:
1524  case X86::MOVSSrm:
1525  case X86::MOVSDrm:
1526  case X86::MOVAPSrm:
1527  case X86::MOVAPDrm:
1528  case X86::MOVDQArm:
1529  case X86::VMOVSSrm:
1530  case X86::VMOVSDrm:
1531  case X86::VMOVAPSrm:
1532  case X86::VMOVAPDrm:
1533  case X86::VMOVDQArm:
1534  case X86::VMOVAPSYrm:
1535  case X86::VMOVAPDYrm:
1536  case X86::VMOVDQAYrm:
1537  case X86::MMX_MOVD64rm:
1538  case X86::MMX_MOVQ64rm:
1539  case X86::VMOVDQA32rm:
1540  case X86::VMOVDQA64rm:
1541  return true;
1542  }
1543 }
1544 
1545 static bool isFrameStoreOpcode(int Opcode) {
1546  switch (Opcode) {
1547  default: break;
1548  case X86::MOV8mr:
1549  case X86::MOV16mr:
1550  case X86::MOV32mr:
1551  case X86::MOV64mr:
1552  case X86::ST_FpP64m:
1553  case X86::MOVSSmr:
1554  case X86::MOVSDmr:
1555  case X86::MOVAPSmr:
1556  case X86::MOVAPDmr:
1557  case X86::MOVDQAmr:
1558  case X86::VMOVSSmr:
1559  case X86::VMOVSDmr:
1560  case X86::VMOVAPSmr:
1561  case X86::VMOVAPDmr:
1562  case X86::VMOVDQAmr:
1563  case X86::VMOVAPSYmr:
1564  case X86::VMOVAPDYmr:
1565  case X86::VMOVDQAYmr:
1566  case X86::MMX_MOVD64mr:
1567  case X86::MMX_MOVQ64mr:
1568  case X86::MMX_MOVNTQmr:
1569  return true;
1570  }
1571  return false;
1572 }
1573 
1574 unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr *MI,
1575  int &FrameIndex) const {
1576  if (isFrameLoadOpcode(MI->getOpcode()))
1577  if (MI->getOperand(0).getSubReg() == 0 && isFrameOperand(MI, 1, FrameIndex))
1578  return MI->getOperand(0).getReg();
1579  return 0;
1580 }
1581 
1582 unsigned X86InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr *MI,
1583  int &FrameIndex) const {
1584  if (isFrameLoadOpcode(MI->getOpcode())) {
1585  unsigned Reg;
1586  if ((Reg = isLoadFromStackSlot(MI, FrameIndex)))
1587  return Reg;
1588  // Check for post-frame index elimination operations
1589  const MachineMemOperand *Dummy;
1590  return hasLoadFromStackSlot(MI, Dummy, FrameIndex);
1591  }
1592  return 0;
1593 }
1594 
1595 unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr *MI,
1596  int &FrameIndex) const {
1597  if (isFrameStoreOpcode(MI->getOpcode()))
1598  if (MI->getOperand(X86::AddrNumOperands).getSubReg() == 0 &&
1599  isFrameOperand(MI, 0, FrameIndex))
1600  return MI->getOperand(X86::AddrNumOperands).getReg();
1601  return 0;
1602 }
1603 
1604 unsigned X86InstrInfo::isStoreToStackSlotPostFE(const MachineInstr *MI,
1605  int &FrameIndex) const {
1606  if (isFrameStoreOpcode(MI->getOpcode())) {
1607  unsigned Reg;
1608  if ((Reg = isStoreToStackSlot(MI, FrameIndex)))
1609  return Reg;
1610  // Check for post-frame index elimination operations
1611  const MachineMemOperand *Dummy;
1612  return hasStoreToStackSlot(MI, Dummy, FrameIndex);
1613  }
1614  return 0;
1615 }
1616 
1617 /// regIsPICBase - Return true if register is PIC base (i.e.g defined by
1618 /// X86::MOVPC32r.
1619 static bool regIsPICBase(unsigned BaseReg, const MachineRegisterInfo &MRI) {
1620  // Don't waste compile time scanning use-def chains of physregs.
1621  if (!TargetRegisterInfo::isVirtualRegister(BaseReg))
1622  return false;
1623  bool isPICBase = false;
1624  for (MachineRegisterInfo::def_iterator I = MRI.def_begin(BaseReg),
1625  E = MRI.def_end(); I != E; ++I) {
1626  MachineInstr *DefMI = I.getOperand().getParent();
1627  if (DefMI->getOpcode() != X86::MOVPC32r)
1628  return false;
1629  assert(!isPICBase && "More than one PIC base?");
1630  isPICBase = true;
1631  }
1632  return isPICBase;
1633 }
1634 
1635 bool
1636 X86InstrInfo::isReallyTriviallyReMaterializable(const MachineInstr *MI,
1637  AliasAnalysis *AA) const {
1638  switch (MI->getOpcode()) {
1639  default: break;
1640  case X86::MOV8rm:
1641  case X86::MOV16rm:
1642  case X86::MOV32rm:
1643  case X86::MOV64rm:
1644  case X86::LD_Fp64m:
1645  case X86::MOVSSrm:
1646  case X86::MOVSDrm:
1647  case X86::MOVAPSrm:
1648  case X86::MOVUPSrm:
1649  case X86::MOVAPDrm:
1650  case X86::MOVDQArm:
1651  case X86::MOVDQUrm:
1652  case X86::VMOVSSrm:
1653  case X86::VMOVSDrm:
1654  case X86::VMOVAPSrm:
1655  case X86::VMOVUPSrm:
1656  case X86::VMOVAPDrm:
1657  case X86::VMOVDQArm:
1658  case X86::VMOVDQUrm:
1659  case X86::VMOVAPSYrm:
1660  case X86::VMOVUPSYrm:
1661  case X86::VMOVAPDYrm:
1662  case X86::VMOVDQAYrm:
1663  case X86::VMOVDQUYrm:
1664  case X86::MMX_MOVD64rm:
1665  case X86::MMX_MOVQ64rm:
1666  case X86::FsVMOVAPSrm:
1667  case X86::FsVMOVAPDrm:
1668  case X86::FsMOVAPSrm:
1669  case X86::FsMOVAPDrm: {
1670  // Loads from constant pools are trivially rematerializable.
1671  if (MI->getOperand(1).isReg() &&
1672  MI->getOperand(2).isImm() &&
1673  MI->getOperand(3).isReg() && MI->getOperand(3).getReg() == 0 &&
1674  MI->isInvariantLoad(AA)) {
1675  unsigned BaseReg = MI->getOperand(1).getReg();
1676  if (BaseReg == 0 || BaseReg == X86::RIP)
1677  return true;
1678  // Allow re-materialization of PIC load.
1679  if (!ReMatPICStubLoad && MI->getOperand(4).isGlobal())
1680  return false;
1681  const MachineFunction &MF = *MI->getParent()->getParent();
1682  const MachineRegisterInfo &MRI = MF.getRegInfo();
1683  return regIsPICBase(BaseReg, MRI);
1684  }
1685  return false;
1686  }
1687 
1688  case X86::LEA32r:
1689  case X86::LEA64r: {
1690  if (MI->getOperand(2).isImm() &&
1691  MI->getOperand(3).isReg() && MI->getOperand(3).getReg() == 0 &&
1692  !MI->getOperand(4).isReg()) {
1693  // lea fi#, lea GV, etc. are all rematerializable.
1694  if (!MI->getOperand(1).isReg())
1695  return true;
1696  unsigned BaseReg = MI->getOperand(1).getReg();
1697  if (BaseReg == 0)
1698  return true;
1699  // Allow re-materialization of lea PICBase + x.
1700  const MachineFunction &MF = *MI->getParent()->getParent();
1701  const MachineRegisterInfo &MRI = MF.getRegInfo();
1702  return regIsPICBase(BaseReg, MRI);
1703  }
1704  return false;
1705  }
1706  }
1707 
1708  // All other instructions marked M_REMATERIALIZABLE are always trivially
1709  // rematerializable.
1710  return true;
1711 }
1712 
1713 /// isSafeToClobberEFLAGS - Return true if it's safe insert an instruction that
1714 /// would clobber the EFLAGS condition register. Note the result may be
1715 /// conservative. If it cannot definitely determine the safety after visiting
1716 /// a few instructions in each direction it assumes it's not safe.
1717 static bool isSafeToClobberEFLAGS(MachineBasicBlock &MBB,
1718  MachineBasicBlock::iterator I) {
1719  MachineBasicBlock::iterator E = MBB.end();
1720 
1721  // For compile time consideration, if we are not able to determine the
1722  // safety after visiting 4 instructions in each direction, we will assume
1723  // it's not safe.
1724  MachineBasicBlock::iterator Iter = I;
1725  for (unsigned i = 0; Iter != E && i < 4; ++i) {
1726  bool SeenDef = false;
1727  for (unsigned j = 0, e = Iter->getNumOperands(); j != e; ++j) {
1728  MachineOperand &MO = Iter->getOperand(j);
1729  if (MO.isRegMask() && MO.clobbersPhysReg(X86::EFLAGS))
1730  SeenDef = true;
1731  if (!MO.isReg())
1732  continue;
1733  if (MO.getReg() == X86::EFLAGS) {
1734  if (MO.isUse())
1735  return false;
1736  SeenDef = true;
1737  }
1738  }
1739 
1740  if (SeenDef)
1741  // This instruction defines EFLAGS, no need to look any further.
1742  return true;
1743  ++Iter;
1744  // Skip over DBG_VALUE.
1745  while (Iter != E && Iter->isDebugValue())
1746  ++Iter;
1747  }
1748 
1749  // It is safe to clobber EFLAGS at the end of a block of no successor has it
1750  // live in.
1751  if (Iter == E) {
1752  for (MachineBasicBlock::succ_iterator SI = MBB.succ_begin(),
1753  SE = MBB.succ_end(); SI != SE; ++SI)
1754  if ((*SI)->isLiveIn(X86::EFLAGS))
1755  return false;
1756  return true;
1757  }
1758 
1759  MachineBasicBlock::iterator B = MBB.begin();
1760  Iter = I;
1761  for (unsigned i = 0; i < 4; ++i) {
1762  // If we make it to the beginning of the block, it's safe to clobber
1763  // EFLAGS iff EFLAGS is not live-in.
1764  if (Iter == B)
1765  return !MBB.isLiveIn(X86::EFLAGS);
1766 
1767  --Iter;
1768  // Skip over DBG_VALUE.
1769  while (Iter != B && Iter->isDebugValue())
1770  --Iter;
1771 
1772  bool SawKill = false;
1773  for (unsigned j = 0, e = Iter->getNumOperands(); j != e; ++j) {
1774  MachineOperand &MO = Iter->getOperand(j);
1775  // A register mask may clobber EFLAGS, but we should still look for a
1776  // live EFLAGS def.
1777  if (MO.isRegMask() && MO.clobbersPhysReg(X86::EFLAGS))
1778  SawKill = true;
1779  if (MO.isReg() && MO.getReg() == X86::EFLAGS) {
1780  if (MO.isDef()) return MO.isDead();
1781  if (MO.isKill()) SawKill = true;
1782  }
1783  }
1784 
1785  if (SawKill)
1786  // This instruction kills EFLAGS and doesn't redefine it, so
1787  // there's no need to look further.
1788  return true;
1789  }
1790 
1791  // Conservative answer.
1792  return false;
1793 }
1794 
1795 void X86InstrInfo::reMaterialize(MachineBasicBlock &MBB,
1796  MachineBasicBlock::iterator I,
1797  unsigned DestReg, unsigned SubIdx,
1798  const MachineInstr *Orig,
1799  const TargetRegisterInfo &TRI) const {
1800  // MOV32r0 is implemented with a xor which clobbers condition code.
1801  // Re-materialize it as movri instructions to avoid side effects.
1802  unsigned Opc = Orig->getOpcode();
1803  if (Opc == X86::MOV32r0 && !isSafeToClobberEFLAGS(MBB, I)) {
1804  DebugLoc DL = Orig->getDebugLoc();
1805  BuildMI(MBB, I, DL, get(X86::MOV32ri)).addOperand(Orig->getOperand(0))
1806  .addImm(0);
1807  } else {
1808  MachineInstr *MI = MBB.getParent()->CloneMachineInstr(Orig);
1809  MBB.insert(I, MI);
1810  }
1811 
1812  MachineInstr *NewMI = prior(I);
1813  NewMI->substituteRegister(Orig->getOperand(0).getReg(), DestReg, SubIdx, TRI);
1814 }
1815 
1816 /// hasLiveCondCodeDef - True if MI has a condition code def, e.g. EFLAGS, that
1817 /// is not marked dead.
1818 static bool hasLiveCondCodeDef(MachineInstr *MI) {
1819  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
1820  MachineOperand &MO = MI->getOperand(i);
1821  if (MO.isReg() && MO.isDef() &&
1822  MO.getReg() == X86::EFLAGS && !MO.isDead()) {
1823  return true;
1824  }
1825  }
1826  return false;
1827 }
1828 
1829 /// getTruncatedShiftCount - check whether the shift count for a machine operand
1830 /// is non-zero.
1831 inline static unsigned getTruncatedShiftCount(MachineInstr *MI,
1832  unsigned ShiftAmtOperandIdx) {
1833  // The shift count is six bits with the REX.W prefix and five bits without.
1834  unsigned ShiftCountMask = (MI->getDesc().TSFlags & X86II::REX_W) ? 63 : 31;
1835  unsigned Imm = MI->getOperand(ShiftAmtOperandIdx).getImm();
1836  return Imm & ShiftCountMask;
1837 }
1838 
1839 /// isTruncatedShiftCountForLEA - check whether the given shift count is appropriate
1840 /// can be represented by a LEA instruction.
1841 inline static bool isTruncatedShiftCountForLEA(unsigned ShAmt) {
1842  // Left shift instructions can be transformed into load-effective-address
1843  // instructions if we can encode them appropriately.
1844  // A LEA instruction utilizes a SIB byte to encode it's scale factor.
1845  // The SIB.scale field is two bits wide which means that we can encode any
1846  // shift amount less than 4.
1847  return ShAmt < 4 && ShAmt > 0;
1848 }
1849 
1850 bool X86InstrInfo::classifyLEAReg(MachineInstr *MI, const MachineOperand &Src,
1851  unsigned Opc, bool AllowSP,
1852  unsigned &NewSrc, bool &isKill, bool &isUndef,
1853  MachineOperand &ImplicitOp) const {
1854  MachineFunction &MF = *MI->getParent()->getParent();
1855  const TargetRegisterClass *RC;
1856  if (AllowSP) {
1857  RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass;
1858  } else {
1859  RC = Opc != X86::LEA32r ?
1860  &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass;
1861  }
1862  unsigned SrcReg = Src.getReg();
1863 
1864  // For both LEA64 and LEA32 the register already has essentially the right
1865  // type (32-bit or 64-bit) we may just need to forbid SP.
1866  if (Opc != X86::LEA64_32r) {
1867  NewSrc = SrcReg;
1868  isKill = Src.isKill();
1869  isUndef = Src.isUndef();
1870 
1871  if (TargetRegisterInfo::isVirtualRegister(NewSrc) &&
1872  !MF.getRegInfo().constrainRegClass(NewSrc, RC))
1873  return false;
1874 
1875  return true;
1876  }
1877 
1878  // This is for an LEA64_32r and incoming registers are 32-bit. One way or
1879  // another we need to add 64-bit registers to the final MI.
1880  if (TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
1881  ImplicitOp = Src;
1882  ImplicitOp.setImplicit();
1883 
1884  NewSrc = getX86SubSuperRegister(Src.getReg(), MVT::i64);
1885  MachineBasicBlock::LivenessQueryResult LQR =
1886  MI->getParent()->computeRegisterLiveness(&getRegisterInfo(), NewSrc, MI);
1887 
1888  switch (LQR) {
1889  case MachineBasicBlock::LQR_Unknown:
1890  // We can't give sane liveness flags to the instruction, abandon LEA
1891  // formation.
1892  return false;
1893  case MachineBasicBlock::LQR_Live:
1894  isKill = MI->killsRegister(SrcReg);
1895  isUndef = false;
1896  break;
1897  default:
1898  // The physreg itself is dead, so we have to use it as an <undef>.
1899  isKill = false;
1900  isUndef = true;
1901  break;
1902  }
1903  } else {
1904  // Virtual register of the wrong class, we have to create a temporary 64-bit
1905  // vreg to feed into the LEA.
1906  NewSrc = MF.getRegInfo().createVirtualRegister(RC);
1907  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1908  get(TargetOpcode::COPY))
1909  .addReg(NewSrc, RegState::Define | RegState::Undef, X86::sub_32bit)
1910  .addOperand(Src);
1911 
1912  // Which is obviously going to be dead after we're done with it.
1913  isKill = true;
1914  isUndef = false;
1915  }
1916 
1917  // We've set all the parameters without issue.
1918  return true;
1919 }
1920 
1921 /// convertToThreeAddressWithLEA - Helper for convertToThreeAddress when
1922 /// 16-bit LEA is disabled, use 32-bit LEA to form 3-address code by promoting
1923 /// to a 32-bit superregister and then truncating back down to a 16-bit
1924 /// subregister.
1925 MachineInstr *
1926 X86InstrInfo::convertToThreeAddressWithLEA(unsigned MIOpc,
1927  MachineFunction::iterator &MFI,
1928  MachineBasicBlock::iterator &MBBI,
1929  LiveVariables *LV) const {
1930  MachineInstr *MI = MBBI;
1931  unsigned Dest = MI->getOperand(0).getReg();
1932  unsigned Src = MI->getOperand(1).getReg();
1933  bool isDead = MI->getOperand(0).isDead();
1934  bool isKill = MI->getOperand(1).isKill();
1935 
1936  MachineRegisterInfo &RegInfo = MFI->getParent()->getRegInfo();
1937  unsigned leaOutReg = RegInfo.createVirtualRegister(&X86::GR32RegClass);
1938  unsigned Opc, leaInReg;
1939  if (TM.getSubtarget<X86Subtarget>().is64Bit()) {
1940  Opc = X86::LEA64_32r;
1941  leaInReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
1942  } else {
1943  Opc = X86::LEA32r;
1944  leaInReg = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
1945  }
1946 
1947  // Build and insert into an implicit UNDEF value. This is OK because
1948  // well be shifting and then extracting the lower 16-bits.
1949  // This has the potential to cause partial register stall. e.g.
1950  // movw (%rbp,%rcx,2), %dx
1951  // leal -65(%rdx), %esi
1952  // But testing has shown this *does* help performance in 64-bit mode (at
1953  // least on modern x86 machines).
1954  BuildMI(*MFI, MBBI, MI->getDebugLoc(), get(X86::IMPLICIT_DEF), leaInReg);
1955  MachineInstr *InsMI =
1956  BuildMI(*MFI, MBBI, MI->getDebugLoc(), get(TargetOpcode::COPY))
1957  .addReg(leaInReg, RegState::Define, X86::sub_16bit)
1958  .addReg(Src, getKillRegState(isKill));
1959 
1960  MachineInstrBuilder MIB = BuildMI(*MFI, MBBI, MI->getDebugLoc(),
1961  get(Opc), leaOutReg);
1962  switch (MIOpc) {
1963  default: llvm_unreachable("Unreachable!");
1964  case X86::SHL16ri: {
1965  unsigned ShAmt = MI->getOperand(2).getImm();
1966  MIB.addReg(0).addImm(1 << ShAmt)
1967  .addReg(leaInReg, RegState::Kill).addImm(0).addReg(0);
1968  break;
1969  }
1970  case X86::INC16r:
1971  case X86::INC64_16r:
1972  addRegOffset(MIB, leaInReg, true, 1);
1973  break;
1974  case X86::DEC16r:
1975  case X86::DEC64_16r:
1976  addRegOffset(MIB, leaInReg, true, -1);
1977  break;
1978  case X86::ADD16ri:
1979  case X86::ADD16ri8:
1980  case X86::ADD16ri_DB:
1981  case X86::ADD16ri8_DB:
1982  addRegOffset(MIB, leaInReg, true, MI->getOperand(2).getImm());
1983  break;
1984  case X86::ADD16rr:
1985  case X86::ADD16rr_DB: {
1986  unsigned Src2 = MI->getOperand(2).getReg();
1987  bool isKill2 = MI->getOperand(2).isKill();
1988  unsigned leaInReg2 = 0;
1989  MachineInstr *InsMI2 = 0;
1990  if (Src == Src2) {
1991  // ADD16rr %reg1028<kill>, %reg1028
1992  // just a single insert_subreg.
1993  addRegReg(MIB, leaInReg, true, leaInReg, false);
1994  } else {
1995  if (TM.getSubtarget<X86Subtarget>().is64Bit())
1996  leaInReg2 = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
1997  else
1998  leaInReg2 = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
1999  // Build and insert into an implicit UNDEF value. This is OK because
2000  // well be shifting and then extracting the lower 16-bits.
2001  BuildMI(*MFI, &*MIB, MI->getDebugLoc(), get(X86::IMPLICIT_DEF),leaInReg2);
2002  InsMI2 =
2003  BuildMI(*MFI, &*MIB, MI->getDebugLoc(), get(TargetOpcode::COPY))
2004  .addReg(leaInReg2, RegState::Define, X86::sub_16bit)
2005  .addReg(Src2, getKillRegState(isKill2));
2006  addRegReg(MIB, leaInReg, true, leaInReg2, true);
2007  }
2008  if (LV && isKill2 && InsMI2)
2009  LV->replaceKillInstruction(Src2, MI, InsMI2);
2010  break;
2011  }
2012  }
2013 
2014  MachineInstr *NewMI = MIB;
2015  MachineInstr *ExtMI =
2016  BuildMI(*MFI, MBBI, MI->getDebugLoc(), get(TargetOpcode::COPY))
2017  .addReg(Dest, RegState::Define | getDeadRegState(isDead))
2018  .addReg(leaOutReg, RegState::Kill, X86::sub_16bit);
2019 
2020  if (LV) {
2021  // Update live variables
2022  LV->getVarInfo(leaInReg).Kills.push_back(NewMI);
2023  LV->getVarInfo(leaOutReg).Kills.push_back(ExtMI);
2024  if (isKill)
2025  LV->replaceKillInstruction(Src, MI, InsMI);
2026  if (isDead)
2027  LV->replaceKillInstruction(Dest, MI, ExtMI);
2028  }
2029 
2030  return ExtMI;
2031 }
2032 
2033 /// convertToThreeAddress - This method must be implemented by targets that
2034 /// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target
2035 /// may be able to convert a two-address instruction into a true
2036 /// three-address instruction on demand. This allows the X86 target (for
2037 /// example) to convert ADD and SHL instructions into LEA instructions if they
2038 /// would require register copies due to two-addressness.
2039 ///
2040 /// This method returns a null pointer if the transformation cannot be
2041 /// performed, otherwise it returns the new instruction.
2042 ///
2043 MachineInstr *
2044 X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
2045  MachineBasicBlock::iterator &MBBI,
2046  LiveVariables *LV) const {
2047  MachineInstr *MI = MBBI;
2048 
2049  // The following opcodes also sets the condition code register(s). Only
2050  // convert them to equivalent lea if the condition code register def's
2051  // are dead!
2052  if (hasLiveCondCodeDef(MI))
2053  return 0;
2054 
2055  MachineFunction &MF = *MI->getParent()->getParent();
2056  // All instructions input are two-addr instructions. Get the known operands.
2057  const MachineOperand &Dest = MI->getOperand(0);
2058  const MachineOperand &Src = MI->getOperand(1);
2059 
2060  MachineInstr *NewMI = NULL;
2061  // FIXME: 16-bit LEA's are really slow on Athlons, but not bad on P4's. When
2062  // we have better subtarget support, enable the 16-bit LEA generation here.
2063  // 16-bit LEA is also slow on Core2.
2064  bool DisableLEA16 = true;
2065  bool is64Bit = TM.getSubtarget<X86Subtarget>().is64Bit();
2066 
2067  unsigned MIOpc = MI->getOpcode();
2068  switch (MIOpc) {
2069  case X86::SHUFPSrri: {
2070  assert(MI->getNumOperands() == 4 && "Unknown shufps instruction!");
2071  if (!TM.getSubtarget<X86Subtarget>().hasSSE2()) return 0;
2072 
2073  unsigned B = MI->getOperand(1).getReg();
2074  unsigned C = MI->getOperand(2).getReg();
2075  if (B != C) return 0;
2076  unsigned M = MI->getOperand(3).getImm();
2077  NewMI = BuildMI(MF, MI->getDebugLoc(), get(X86::PSHUFDri))
2078  .addOperand(Dest).addOperand(Src).addImm(M);
2079  break;
2080  }
2081  case X86::SHUFPDrri: {
2082  assert(MI->getNumOperands() == 4 && "Unknown shufpd instruction!");
2083  if (!TM.getSubtarget<X86Subtarget>().hasSSE2()) return 0;
2084 
2085  unsigned B = MI->getOperand(1).getReg();
2086  unsigned C = MI->getOperand(2).getReg();
2087  if (B != C) return 0;
2088  unsigned M = MI->getOperand(3).getImm();
2089 
2090  // Convert to PSHUFD mask.
2091  M = ((M & 1) << 1) | ((M & 1) << 3) | ((M & 2) << 4) | ((M & 2) << 6)| 0x44;
2092 
2093  NewMI = BuildMI(MF, MI->getDebugLoc(), get(X86::PSHUFDri))
2094  .addOperand(Dest).addOperand(Src).addImm(M);
2095  break;
2096  }
2097  case X86::SHL64ri: {
2098  assert(MI->getNumOperands() >= 3 && "Unknown shift instruction!");
2099  unsigned ShAmt = getTruncatedShiftCount(MI, 2);
2100  if (!isTruncatedShiftCountForLEA(ShAmt)) return 0;
2101 
2102  // LEA can't handle RSP.
2103  if (TargetRegisterInfo::isVirtualRegister(Src.getReg()) &&
2104  !MF.getRegInfo().constrainRegClass(Src.getReg(),
2105  &X86::GR64_NOSPRegClass))
2106  return 0;
2107 
2108  NewMI = BuildMI(MF, MI->getDebugLoc(), get(X86::LEA64r))
2109  .addOperand(Dest)
2110  .addReg(0).addImm(1 << ShAmt).addOperand(Src).addImm(0).addReg(0);
2111  break;
2112  }
2113  case X86::SHL32ri: {
2114  assert(MI->getNumOperands() >= 3 && "Unknown shift instruction!");
2115  unsigned ShAmt = getTruncatedShiftCount(MI, 2);
2116  if (!isTruncatedShiftCountForLEA(ShAmt)) return 0;
2117 
2118  unsigned Opc = is64Bit ? X86::LEA64_32r : X86::LEA32r;
2119 
2120  // LEA can't handle ESP.
2121  bool isKill, isUndef;
2122  unsigned SrcReg;
2123  MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
2124  if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ false,
2125  SrcReg, isKill, isUndef, ImplicitOp))
2126  return 0;
2127 
2128  MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), get(Opc))
2129  .addOperand(Dest)
2130  .addReg(0).addImm(1 << ShAmt)
2131  .addReg(SrcReg, getKillRegState(isKill) | getUndefRegState(isUndef))
2132  .addImm(0).addReg(0);
2133  if (ImplicitOp.getReg() != 0)
2134  MIB.addOperand(ImplicitOp);
2135  NewMI = MIB;
2136 
2137  break;
2138  }
2139  case X86::SHL16ri: {
2140  assert(MI->getNumOperands() >= 3 && "Unknown shift instruction!");
2141  unsigned ShAmt = getTruncatedShiftCount(MI, 2);
2142  if (!isTruncatedShiftCountForLEA(ShAmt)) return 0;
2143 
2144  if (DisableLEA16)
2145  return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MBBI, LV) : 0;
2146  NewMI = BuildMI(MF, MI->getDebugLoc(), get(X86::LEA16r))
2147  .addOperand(Dest)
2148  .addReg(0).addImm(1 << ShAmt).addOperand(Src).addImm(0).addReg(0);
2149  break;
2150  }
2151  default: {
2152 
2153  switch (MIOpc) {
2154  default: return 0;
2155  case X86::INC64r:
2156  case X86::INC32r:
2157  case X86::INC64_32r: {
2158  assert(MI->getNumOperands() >= 2 && "Unknown inc instruction!");
2159  unsigned Opc = MIOpc == X86::INC64r ? X86::LEA64r
2160  : (is64Bit ? X86::LEA64_32r : X86::LEA32r);
2161  bool isKill, isUndef;
2162  unsigned SrcReg;
2163  MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
2164  if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ false,
2165  SrcReg, isKill, isUndef, ImplicitOp))
2166  return 0;
2167 
2168  MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), get(Opc))
2169  .addOperand(Dest)
2170  .addReg(SrcReg, getKillRegState(isKill) | getUndefRegState(isUndef));
2171  if (ImplicitOp.getReg() != 0)
2172  MIB.addOperand(ImplicitOp);
2173 
2174  NewMI = addOffset(MIB, 1);
2175  break;
2176  }
2177  case X86::INC16r:
2178  case X86::INC64_16r:
2179  if (DisableLEA16)
2180  return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MBBI, LV) : 0;
2181  assert(MI->getNumOperands() >= 2 && "Unknown inc instruction!");
2182  NewMI = addOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA16r))
2183  .addOperand(Dest).addOperand(Src), 1);
2184  break;
2185  case X86::DEC64r:
2186  case X86::DEC32r:
2187  case X86::DEC64_32r: {
2188  assert(MI->getNumOperands() >= 2 && "Unknown dec instruction!");
2189  unsigned Opc = MIOpc == X86::DEC64r ? X86::LEA64r
2190  : (is64Bit ? X86::LEA64_32r : X86::LEA32r);
2191 
2192  bool isKill, isUndef;
2193  unsigned SrcReg;
2194  MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
2195  if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ false,
2196  SrcReg, isKill, isUndef, ImplicitOp))
2197  return 0;
2198 
2199  MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), get(Opc))
2200  .addOperand(Dest)
2201  .addReg(SrcReg, getUndefRegState(isUndef) | getKillRegState(isKill));
2202  if (ImplicitOp.getReg() != 0)
2203  MIB.addOperand(ImplicitOp);
2204 
2205  NewMI = addOffset(MIB, -1);
2206 
2207  break;
2208  }
2209  case X86::DEC16r:
2210  case X86::DEC64_16r:
2211  if (DisableLEA16)
2212  return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MBBI, LV) : 0;
2213  assert(MI->getNumOperands() >= 2 && "Unknown dec instruction!");
2214  NewMI = addOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA16r))
2215  .addOperand(Dest).addOperand(Src), -1);
2216  break;
2217  case X86::ADD64rr:
2218  case X86::ADD64rr_DB:
2219  case X86::ADD32rr:
2220  case X86::ADD32rr_DB: {
2221  assert(MI->getNumOperands() >= 3 && "Unknown add instruction!");
2222  unsigned Opc;
2223  if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_DB)
2224  Opc = X86::LEA64r;
2225  else
2226  Opc = is64Bit ? X86::LEA64_32r : X86::LEA32r;
2227 
2228  bool isKill, isUndef;
2229  unsigned SrcReg;
2230  MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
2231  if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ true,
2232  SrcReg, isKill, isUndef, ImplicitOp))
2233  return 0;
2234 
2235  const MachineOperand &Src2 = MI->getOperand(2);
2236  bool isKill2, isUndef2;
2237  unsigned SrcReg2;
2238  MachineOperand ImplicitOp2 = MachineOperand::CreateReg(0, false);
2239  if (!classifyLEAReg(MI, Src2, Opc, /*AllowSP=*/ false,
2240  SrcReg2, isKill2, isUndef2, ImplicitOp2))
2241  return 0;
2242 
2243  MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), get(Opc))
2244  .addOperand(Dest);
2245  if (ImplicitOp.getReg() != 0)
2246  MIB.addOperand(ImplicitOp);
2247  if (ImplicitOp2.getReg() != 0)
2248  MIB.addOperand(ImplicitOp2);
2249 
2250  NewMI = addRegReg(MIB, SrcReg, isKill, SrcReg2, isKill2);
2251 
2252  // Preserve undefness of the operands.
2253  NewMI->getOperand(1).setIsUndef(isUndef);
2254  NewMI->getOperand(3).setIsUndef(isUndef2);
2255 
2256  if (LV && Src2.isKill())
2257  LV->replaceKillInstruction(SrcReg2, MI, NewMI);
2258  break;
2259  }
2260  case X86::ADD16rr:
2261  case X86::ADD16rr_DB: {
2262  if (DisableLEA16)
2263  return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MBBI, LV) : 0;
2264  assert(MI->getNumOperands() >= 3 && "Unknown add instruction!");
2265  unsigned Src2 = MI->getOperand(2).getReg();
2266  bool isKill2 = MI->getOperand(2).isKill();
2267  NewMI = addRegReg(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA16r))
2268  .addOperand(Dest),
2269  Src.getReg(), Src.isKill(), Src2, isKill2);
2270 
2271  // Preserve undefness of the operands.
2272  bool isUndef = MI->getOperand(1).isUndef();
2273  bool isUndef2 = MI->getOperand(2).isUndef();
2274  NewMI->getOperand(1).setIsUndef(isUndef);
2275  NewMI->getOperand(3).setIsUndef(isUndef2);
2276 
2277  if (LV && isKill2)
2278  LV->replaceKillInstruction(Src2, MI, NewMI);
2279  break;
2280  }
2281  case X86::ADD64ri32:
2282  case X86::ADD64ri8:
2283  case X86::ADD64ri32_DB:
2284  case X86::ADD64ri8_DB:
2285  assert(MI->getNumOperands() >= 3 && "Unknown add instruction!");
2286  NewMI = addOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA64r))
2287  .addOperand(Dest).addOperand(Src),
2288  MI->getOperand(2).getImm());
2289  break;
2290  case X86::ADD32ri:
2291  case X86::ADD32ri8:
2292  case X86::ADD32ri_DB:
2293  case X86::ADD32ri8_DB: {
2294  assert(MI->getNumOperands() >= 3 && "Unknown add instruction!");
2295  unsigned Opc = is64Bit ? X86::LEA64_32r : X86::LEA32r;
2296 
2297  bool isKill, isUndef;
2298  unsigned SrcReg;
2299  MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
2300  if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ true,
2301  SrcReg, isKill, isUndef, ImplicitOp))
2302  return 0;
2303 
2304  MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), get(Opc))
2305  .addOperand(Dest)
2306  .addReg(SrcReg, getUndefRegState(isUndef) | getKillRegState(isKill));
2307  if (ImplicitOp.getReg() != 0)
2308  MIB.addOperand(ImplicitOp);
2309 
2310  NewMI = addOffset(MIB, MI->getOperand(2).getImm());
2311  break;
2312  }
2313  case X86::ADD16ri:
2314  case X86::ADD16ri8:
2315  case X86::ADD16ri_DB:
2316  case X86::ADD16ri8_DB:
2317  if (DisableLEA16)
2318  return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MBBI, LV) : 0;
2319  assert(MI->getNumOperands() >= 3 && "Unknown add instruction!");
2320  NewMI = addOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA16r))
2321  .addOperand(Dest).addOperand(Src),
2322  MI->getOperand(2).getImm());
2323  break;
2324  }
2325  }
2326  }
2327 
2328  if (!NewMI) return 0;
2329 
2330  if (LV) { // Update live variables
2331  if (Src.isKill())
2332  LV->replaceKillInstruction(Src.getReg(), MI, NewMI);
2333  if (Dest.isDead())
2334  LV->replaceKillInstruction(Dest.getReg(), MI, NewMI);
2335  }
2336 
2337  MFI->insert(MBBI, NewMI); // Insert the new inst
2338  return NewMI;
2339 }
2340 
2341 /// commuteInstruction - We have a few instructions that must be hacked on to
2342 /// commute them.
2343 ///
2344 MachineInstr *
2345 X86InstrInfo::commuteInstruction(MachineInstr *MI, bool NewMI) const {
2346  switch (MI->getOpcode()) {
2347  case X86::SHRD16rri8: // A = SHRD16rri8 B, C, I -> A = SHLD16rri8 C, B, (16-I)
2348  case X86::SHLD16rri8: // A = SHLD16rri8 B, C, I -> A = SHRD16rri8 C, B, (16-I)
2349  case X86::SHRD32rri8: // A = SHRD32rri8 B, C, I -> A = SHLD32rri8 C, B, (32-I)
2350  case X86::SHLD32rri8: // A = SHLD32rri8 B, C, I -> A = SHRD32rri8 C, B, (32-I)
2351  case X86::SHRD64rri8: // A = SHRD64rri8 B, C, I -> A = SHLD64rri8 C, B, (64-I)
2352  case X86::SHLD64rri8:{// A = SHLD64rri8 B, C, I -> A = SHRD64rri8 C, B, (64-I)
2353  unsigned Opc;
2354  unsigned Size;
2355  switch (MI->getOpcode()) {
2356  default: llvm_unreachable("Unreachable!");
2357  case X86::SHRD16rri8: Size = 16; Opc = X86::SHLD16rri8; break;
2358  case X86::SHLD16rri8: Size = 16; Opc = X86::SHRD16rri8; break;
2359  case X86::SHRD32rri8: Size = 32; Opc = X86::SHLD32rri8; break;
2360  case X86::SHLD32rri8: Size = 32; Opc = X86::SHRD32rri8; break;
2361  case X86::SHRD64rri8: Size = 64; Opc = X86::SHLD64rri8; break;
2362  case X86::SHLD64rri8: Size = 64; Opc = X86::SHRD64rri8; break;
2363  }
2364  unsigned Amt = MI->getOperand(3).getImm();
2365  if (NewMI) {
2366  MachineFunction &MF = *MI->getParent()->getParent();
2367  MI = MF.CloneMachineInstr(MI);
2368  NewMI = false;
2369  }
2370  MI->setDesc(get(Opc));
2371  MI->getOperand(3).setImm(Size-Amt);
2372  return TargetInstrInfo::commuteInstruction(MI, NewMI);
2373  }
2374  case X86::CMOVB16rr: case X86::CMOVB32rr: case X86::CMOVB64rr:
2375  case X86::CMOVAE16rr: case X86::CMOVAE32rr: case X86::CMOVAE64rr:
2376  case X86::CMOVE16rr: case X86::CMOVE32rr: case X86::CMOVE64rr:
2377  case X86::CMOVNE16rr: case X86::CMOVNE32rr: case X86::CMOVNE64rr:
2378  case X86::CMOVBE16rr: case X86::CMOVBE32rr: case X86::CMOVBE64rr:
2379  case X86::CMOVA16rr: case X86::CMOVA32rr: case X86::CMOVA64rr:
2380  case X86::CMOVL16rr: case X86::CMOVL32rr: case X86::CMOVL64rr:
2381  case X86::CMOVGE16rr: case X86::CMOVGE32rr: case X86::CMOVGE64rr:
2382  case X86::CMOVLE16rr: case X86::CMOVLE32rr: case X86::CMOVLE64rr:
2383  case X86::CMOVG16rr: case X86::CMOVG32rr: case X86::CMOVG64rr:
2384  case X86::CMOVS16rr: case X86::CMOVS32rr: case X86::CMOVS64rr:
2385  case X86::CMOVNS16rr: case X86::CMOVNS32rr: case X86::CMOVNS64rr:
2386  case X86::CMOVP16rr: case X86::CMOVP32rr: case X86::CMOVP64rr:
2387  case X86::CMOVNP16rr: case X86::CMOVNP32rr: case X86::CMOVNP64rr:
2388  case X86::CMOVO16rr: case X86::CMOVO32rr: case X86::CMOVO64rr:
2389  case X86::CMOVNO16rr: case X86::CMOVNO32rr: case X86::CMOVNO64rr: {
2390  unsigned Opc;
2391  switch (MI->getOpcode()) {
2392  default: llvm_unreachable("Unreachable!");
2393  case X86::CMOVB16rr: Opc = X86::CMOVAE16rr; break;
2394  case X86::CMOVB32rr: Opc = X86::CMOVAE32rr; break;
2395  case X86::CMOVB64rr: Opc = X86::CMOVAE64rr; break;
2396  case X86::CMOVAE16rr: Opc = X86::CMOVB16rr; break;
2397  case X86::CMOVAE32rr: Opc = X86::CMOVB32rr; break;
2398  case X86::CMOVAE64rr: Opc = X86::CMOVB64rr; break;
2399  case X86::CMOVE16rr: Opc = X86::CMOVNE16rr; break;
2400  case X86::CMOVE32rr: Opc = X86::CMOVNE32rr; break;
2401  case X86::CMOVE64rr: Opc = X86::CMOVNE64rr; break;
2402  case X86::CMOVNE16rr: Opc = X86::CMOVE16rr; break;
2403  case X86::CMOVNE32rr: Opc = X86::CMOVE32rr; break;
2404  case X86::CMOVNE64rr: Opc = X86::CMOVE64rr; break;
2405  case X86::CMOVBE16rr: Opc = X86::CMOVA16rr; break;
2406  case X86::CMOVBE32rr: Opc = X86::CMOVA32rr; break;
2407  case X86::CMOVBE64rr: Opc = X86::CMOVA64rr; break;
2408  case X86::CMOVA16rr: Opc = X86::CMOVBE16rr; break;
2409  case X86::CMOVA32rr: Opc = X86::CMOVBE32rr; break;
2410  case X86::CMOVA64rr: Opc = X86::CMOVBE64rr; break;
2411  case X86::CMOVL16rr: Opc = X86::CMOVGE16rr; break;
2412  case X86::CMOVL32rr: Opc = X86::CMOVGE32rr; break;
2413  case X86::CMOVL64rr: Opc = X86::CMOVGE64rr; break;
2414  case X86::CMOVGE16rr: Opc = X86::CMOVL16rr; break;
2415  case X86::CMOVGE32rr: Opc = X86::CMOVL32rr; break;
2416  case X86::CMOVGE64rr: Opc = X86::CMOVL64rr; break;
2417  case X86::CMOVLE16rr: Opc = X86::CMOVG16rr; break;
2418  case X86::CMOVLE32rr: Opc = X86::CMOVG32rr; break;
2419  case X86::CMOVLE64rr: Opc = X86::CMOVG64rr; break;
2420  case X86::CMOVG16rr: Opc = X86::CMOVLE16rr; break;
2421  case X86::CMOVG32rr: Opc = X86::CMOVLE32rr; break;
2422  case X86::CMOVG64rr: Opc = X86::CMOVLE64rr; break;
2423  case X86::CMOVS16rr: Opc = X86::CMOVNS16rr; break;
2424  case X86::CMOVS32rr: Opc = X86::CMOVNS32rr; break;
2425  case X86::CMOVS64rr: Opc = X86::CMOVNS64rr; break;
2426  case X86::CMOVNS16rr: Opc = X86::CMOVS16rr; break;
2427  case X86::CMOVNS32rr: Opc = X86::CMOVS32rr; break;
2428  case X86::CMOVNS64rr: Opc = X86::CMOVS64rr; break;
2429  case X86::CMOVP16rr: Opc = X86::CMOVNP16rr; break;
2430  case X86::CMOVP32rr: Opc = X86::CMOVNP32rr; break;
2431  case X86::CMOVP64rr: Opc = X86::CMOVNP64rr; break;
2432  case X86::CMOVNP16rr: Opc = X86::CMOVP16rr; break;
2433  case X86::CMOVNP32rr: Opc = X86::CMOVP32rr; break;
2434  case X86::CMOVNP64rr: Opc = X86::CMOVP64rr; break;
2435  case X86::CMOVO16rr: Opc = X86::CMOVNO16rr; break;
2436  case X86::CMOVO32rr: Opc = X86::CMOVNO32rr; break;
2437  case X86::CMOVO64rr: Opc = X86::CMOVNO64rr; break;
2438  case X86::CMOVNO16rr: Opc = X86::CMOVO16rr; break;
2439  case X86::CMOVNO32rr: Opc = X86::CMOVO32rr; break;
2440  case X86::CMOVNO64rr: Opc = X86::CMOVO64rr; break;
2441  }
2442  if (NewMI) {
2443  MachineFunction &MF = *MI->getParent()->getParent();
2444  MI = MF.CloneMachineInstr(MI);
2445  NewMI = false;
2446  }
2447  MI->setDesc(get(Opc));
2448  // Fallthrough intended.
2449  }
2450  default:
2451  return TargetInstrInfo::commuteInstruction(MI, NewMI);
2452  }
2453 }
2454 
2455 static X86::CondCode getCondFromBranchOpc(unsigned BrOpc) {
2456  switch (BrOpc) {
2457  default: return X86::COND_INVALID;
2458  case X86::JE_4: return X86::COND_E;
2459  case X86::JNE_4: return X86::COND_NE;
2460  case X86::JL_4: return X86::COND_L;
2461  case X86::JLE_4: return X86::COND_LE;
2462  case X86::JG_4: return X86::COND_G;
2463  case X86::JGE_4: return X86::COND_GE;
2464  case X86::JB_4: return X86::COND_B;
2465  case X86::JBE_4: return X86::COND_BE;
2466  case X86::JA_4: return X86::COND_A;
2467  case X86::JAE_4: return X86::COND_AE;
2468  case X86::JS_4: return X86::COND_S;
2469  case X86::JNS_4: return X86::COND_NS;
2470  case X86::JP_4: return X86::COND_P;
2471  case X86::JNP_4: return X86::COND_NP;
2472  case X86::JO_4: return X86::COND_O;
2473  case X86::JNO_4: return X86::COND_NO;
2474  }
2475 }
2476 
2477 /// getCondFromSETOpc - return condition code of a SET opcode.
2478 static X86::CondCode getCondFromSETOpc(unsigned Opc) {
2479  switch (Opc) {
2480  default: return X86::COND_INVALID;
2481  case X86::SETAr: case X86::SETAm: return X86::COND_A;
2482  case X86::SETAEr: case X86::SETAEm: return X86::COND_AE;
2483  case X86::SETBr: case X86::SETBm: return X86::COND_B;
2484  case X86::SETBEr: case X86::SETBEm: return X86::COND_BE;
2485  case X86::SETEr: case X86::SETEm: return X86::COND_E;
2486  case X86::SETGr: case X86::SETGm: return X86::COND_G;
2487  case X86::SETGEr: case X86::SETGEm: return X86::COND_GE;
2488  case X86::SETLr: case X86::SETLm: return X86::COND_L;
2489  case X86::SETLEr: case X86::SETLEm: return X86::COND_LE;
2490  case X86::SETNEr: case X86::SETNEm: return X86::COND_NE;
2491  case X86::SETNOr: case X86::SETNOm: return X86::COND_NO;
2492  case X86::SETNPr: case X86::SETNPm: return X86::COND_NP;
2493  case X86::SETNSr: case X86::SETNSm: return X86::COND_NS;
2494  case X86::SETOr: case X86::SETOm: return X86::COND_O;
2495  case X86::SETPr: case X86::SETPm: return X86::COND_P;
2496  case X86::SETSr: case X86::SETSm: return X86::COND_S;
2497  }
2498 }
2499 
2500 /// getCondFromCmovOpc - return condition code of a CMov opcode.
2501 X86::CondCode X86::getCondFromCMovOpc(unsigned Opc) {
2502  switch (Opc) {
2503  default: return X86::COND_INVALID;
2504  case X86::CMOVA16rm: case X86::CMOVA16rr: case X86::CMOVA32rm:
2505  case X86::CMOVA32rr: case X86::CMOVA64rm: case X86::CMOVA64rr:
2506  return X86::COND_A;
2507  case X86::CMOVAE16rm: case X86::CMOVAE16rr: case X86::CMOVAE32rm:
2508  case X86::CMOVAE32rr: case X86::CMOVAE64rm: case X86::CMOVAE64rr:
2509  return X86::COND_AE;
2510  case X86::CMOVB16rm: case X86::CMOVB16rr: case X86::CMOVB32rm:
2511  case X86::CMOVB32rr: case X86::CMOVB64rm: case X86::CMOVB64rr:
2512  return X86::COND_B;
2513  case X86::CMOVBE16rm: case X86::CMOVBE16rr: case X86::CMOVBE32rm:
2514  case X86::CMOVBE32rr: case X86::CMOVBE64rm: case X86::CMOVBE64rr:
2515  return X86::COND_BE;
2516  case X86::CMOVE16rm: case X86::CMOVE16rr: case X86::CMOVE32rm:
2517  case X86::CMOVE32rr: case X86::CMOVE64rm: case X86::CMOVE64rr:
2518  return X86::COND_E;
2519  case X86::CMOVG16rm: case X86::CMOVG16rr: case X86::CMOVG32rm:
2520  case X86::CMOVG32rr: case X86::CMOVG64rm: case X86::CMOVG64rr:
2521  return X86::COND_G;
2522  case X86::CMOVGE16rm: case X86::CMOVGE16rr: case X86::CMOVGE32rm:
2523  case X86::CMOVGE32rr: case X86::CMOVGE64rm: case X86::CMOVGE64rr:
2524  return X86::COND_GE;
2525  case X86::CMOVL16rm: case X86::CMOVL16rr: case X86::CMOVL32rm:
2526  case X86::CMOVL32rr: case X86::CMOVL64rm: case X86::CMOVL64rr:
2527  return X86::COND_L;
2528  case X86::CMOVLE16rm: case X86::CMOVLE16rr: case X86::CMOVLE32rm:
2529  case X86::CMOVLE32rr: case X86::CMOVLE64rm: case X86::CMOVLE64rr:
2530  return X86::COND_LE;
2531  case X86::CMOVNE16rm: case X86::CMOVNE16rr: case X86::CMOVNE32rm:
2532  case X86::CMOVNE32rr: case X86::CMOVNE64rm: case X86::CMOVNE64rr:
2533  return X86::COND_NE;
2534  case X86::CMOVNO16rm: case X86::CMOVNO16rr: case X86::CMOVNO32rm:
2535  case X86::CMOVNO32rr: case X86::CMOVNO64rm: case X86::CMOVNO64rr:
2536  return X86::COND_NO;
2537  case X86::CMOVNP16rm: case X86::CMOVNP16rr: case X86::CMOVNP32rm:
2538  case X86::CMOVNP32rr: case X86::CMOVNP64rm: case X86::CMOVNP64rr:
2539  return X86::COND_NP;
2540  case X86::CMOVNS16rm: case X86::CMOVNS16rr: case X86::CMOVNS32rm:
2541  case X86::CMOVNS32rr: case X86::CMOVNS64rm: case X86::CMOVNS64rr:
2542  return X86::COND_NS;
2543  case X86::CMOVO16rm: case X86::CMOVO16rr: case X86::CMOVO32rm:
2544  case X86::CMOVO32rr: case X86::CMOVO64rm: case X86::CMOVO64rr:
2545  return X86::COND_O;
2546  case X86::CMOVP16rm: case X86::CMOVP16rr: case X86::CMOVP32rm:
2547  case X86::CMOVP32rr: case X86::CMOVP64rm: case X86::CMOVP64rr:
2548  return X86::COND_P;
2549  case X86::CMOVS16rm: case X86::CMOVS16rr: case X86::CMOVS32rm:
2550  case X86::CMOVS32rr: case X86::CMOVS64rm: case X86::CMOVS64rr:
2551  return X86::COND_S;
2552  }
2553 }
2554 
2555 unsigned X86::GetCondBranchFromCond(X86::CondCode CC) {
2556  switch (CC) {
2557  default: llvm_unreachable("Illegal condition code!");
2558  case X86::COND_E: return X86::JE_4;
2559  case X86::COND_NE: return X86::JNE_4;
2560  case X86::COND_L: return X86::JL_4;
2561  case X86::COND_LE: return X86::JLE_4;
2562  case X86::COND_G: return X86::JG_4;
2563  case X86::COND_GE: return X86::JGE_4;
2564  case X86::COND_B: return X86::JB_4;
2565  case X86::COND_BE: return X86::JBE_4;
2566  case X86::COND_A: return X86::JA_4;
2567  case X86::COND_AE: return X86::JAE_4;
2568  case X86::COND_S: return X86::JS_4;
2569  case X86::COND_NS: return X86::JNS_4;
2570  case X86::COND_P: return X86::JP_4;
2571  case X86::COND_NP: return X86::JNP_4;
2572  case X86::COND_O: return X86::JO_4;
2573  case X86::COND_NO: return X86::JNO_4;
2574  }
2575 }
2576 
2577 /// GetOppositeBranchCondition - Return the inverse of the specified condition,
2578 /// e.g. turning COND_E to COND_NE.
2579 X86::CondCode X86::GetOppositeBranchCondition(X86::CondCode CC) {
2580  switch (CC) {
2581  default: llvm_unreachable("Illegal condition code!");
2582  case X86::COND_E: return X86::COND_NE;
2583  case X86::COND_NE: return X86::COND_E;
2584  case X86::COND_L: return X86::COND_GE;
2585  case X86::COND_LE: return X86::COND_G;
2586  case X86::COND_G: return X86::COND_LE;
2587  case X86::COND_GE: return X86::COND_L;
2588  case X86::COND_B: return X86::COND_AE;
2589  case X86::COND_BE: return X86::COND_A;
2590  case X86::COND_A: return X86::COND_BE;
2591  case X86::COND_AE: return X86::COND_B;
2592  case X86::COND_S: return X86::COND_NS;
2593  case X86::COND_NS: return X86::COND_S;
2594  case X86::COND_P: return X86::COND_NP;
2595  case X86::COND_NP: return X86::COND_P;
2596  case X86::COND_O: return X86::COND_NO;
2597  case X86::COND_NO: return X86::COND_O;
2598  }
2599 }
2600 
2601 /// getSwappedCondition - assume the flags are set by MI(a,b), return
2602 /// the condition code if we modify the instructions such that flags are
2603 /// set by MI(b,a).
2604 static X86::CondCode getSwappedCondition(X86::CondCode CC) {
2605  switch (CC) {
2606  default: return X86::COND_INVALID;
2607  case X86::COND_E: return X86::COND_E;
2608  case X86::COND_NE: return X86::COND_NE;
2609  case X86::COND_L: return X86::COND_G;
2610  case X86::COND_LE: return X86::COND_GE;
2611  case X86::COND_G: return X86::COND_L;
2612  case X86::COND_GE: return X86::COND_LE;
2613  case X86::COND_B: return X86::COND_A;
2614  case X86::COND_BE: return X86::COND_AE;
2615  case X86::COND_A: return X86::COND_B;
2616  case X86::COND_AE: return X86::COND_BE;
2617  }
2618 }
2619 
2620 /// getSETFromCond - Return a set opcode for the given condition and
2621 /// whether it has memory operand.
2622 static unsigned getSETFromCond(X86::CondCode CC,
2623  bool HasMemoryOperand) {
2624  static const uint16_t Opc[16][2] = {
2625  { X86::SETAr, X86::SETAm },
2626  { X86::SETAEr, X86::SETAEm },
2627  { X86::SETBr, X86::SETBm },
2628  { X86::SETBEr, X86::SETBEm },
2629  { X86::SETEr, X86::SETEm },
2630  { X86::SETGr, X86::SETGm },
2631  { X86::SETGEr, X86::SETGEm },
2632  { X86::SETLr, X86::SETLm },
2633  { X86::SETLEr, X86::SETLEm },
2634  { X86::SETNEr, X86::SETNEm },
2635  { X86::SETNOr, X86::SETNOm },
2636  { X86::SETNPr, X86::SETNPm },
2637  { X86::SETNSr, X86::SETNSm },
2638  { X86::SETOr, X86::SETOm },
2639  { X86::SETPr, X86::SETPm },
2640  { X86::SETSr, X86::SETSm }
2641  };
2642 
2643  assert(CC < 16 && "Can only handle standard cond codes");
2644  return Opc[CC][HasMemoryOperand ? 1 : 0];
2645 }
2646 
2647 /// getCMovFromCond - Return a cmov opcode for the given condition,
2648 /// register size in bytes, and operand type.
2649 static unsigned getCMovFromCond(X86::CondCode CC, unsigned RegBytes,
2650  bool HasMemoryOperand) {
2651  static const uint16_t Opc[32][3] = {
2652  { X86::CMOVA16rr, X86::CMOVA32rr, X86::CMOVA64rr },
2653  { X86::CMOVAE16rr, X86::CMOVAE32rr, X86::CMOVAE64rr },
2654  { X86::CMOVB16rr, X86::CMOVB32rr, X86::CMOVB64rr },
2655  { X86::CMOVBE16rr, X86::CMOVBE32rr, X86::CMOVBE64rr },
2656  { X86::CMOVE16rr, X86::CMOVE32rr, X86::CMOVE64rr },
2657  { X86::CMOVG16rr, X86::CMOVG32rr, X86::CMOVG64rr },
2658  { X86::CMOVGE16rr, X86::CMOVGE32rr, X86::CMOVGE64rr },
2659  { X86::CMOVL16rr, X86::CMOVL32rr, X86::CMOVL64rr },
2660  { X86::CMOVLE16rr, X86::CMOVLE32rr, X86::CMOVLE64rr },
2661  { X86::CMOVNE16rr, X86::CMOVNE32rr, X86::CMOVNE64rr },
2662  { X86::CMOVNO16rr, X86::CMOVNO32rr, X86::CMOVNO64rr },
2663  { X86::CMOVNP16rr, X86::CMOVNP32rr, X86::CMOVNP64rr },
2664  { X86::CMOVNS16rr, X86::CMOVNS32rr, X86::CMOVNS64rr },
2665  { X86::CMOVO16rr, X86::CMOVO32rr, X86::CMOVO64rr },
2666  { X86::CMOVP16rr, X86::CMOVP32rr, X86::CMOVP64rr },
2667  { X86::CMOVS16rr, X86::CMOVS32rr, X86::CMOVS64rr },
2668  { X86::CMOVA16rm, X86::CMOVA32rm, X86::CMOVA64rm },
2669  { X86::CMOVAE16rm, X86::CMOVAE32rm, X86::CMOVAE64rm },
2670  { X86::CMOVB16rm, X86::CMOVB32rm, X86::CMOVB64rm },
2671  { X86::CMOVBE16rm, X86::CMOVBE32rm, X86::CMOVBE64rm },
2672  { X86::CMOVE16rm, X86::CMOVE32rm, X86::CMOVE64rm },
2673  { X86::CMOVG16rm, X86::CMOVG32rm, X86::CMOVG64rm },
2674  { X86::CMOVGE16rm, X86::CMOVGE32rm, X86::CMOVGE64rm },
2675  { X86::CMOVL16rm, X86::CMOVL32rm, X86::CMOVL64rm },
2676  { X86::CMOVLE16rm, X86::CMOVLE32rm, X86::CMOVLE64rm },
2677  { X86::CMOVNE16rm, X86::CMOVNE32rm, X86::CMOVNE64rm },
2678  { X86::CMOVNO16rm, X86::CMOVNO32rm, X86::CMOVNO64rm },
2679  { X86::CMOVNP16rm, X86::CMOVNP32rm, X86::CMOVNP64rm },
2680  { X86::CMOVNS16rm, X86::CMOVNS32rm, X86::CMOVNS64rm },
2681  { X86::CMOVO16rm, X86::CMOVO32rm, X86::CMOVO64rm },
2682  { X86::CMOVP16rm, X86::CMOVP32rm, X86::CMOVP64rm },
2683  { X86::CMOVS16rm, X86::CMOVS32rm, X86::CMOVS64rm }
2684  };
2685 
2686  assert(CC < 16 && "Can only handle standard cond codes");
2687  unsigned Idx = HasMemoryOperand ? 16+CC : CC;
2688  switch(RegBytes) {
2689  default: llvm_unreachable("Illegal register size!");
2690  case 2: return Opc[Idx][0];
2691  case 4: return Opc[Idx][1];
2692  case 8: return Opc[Idx][2];
2693  }
2694 }
2695 
2696 bool X86InstrInfo::isUnpredicatedTerminator(const MachineInstr *MI) const {
2697  if (!MI->isTerminator()) return false;
2698 
2699  // Conditional branch is a special case.
2700  if (MI->isBranch() && !MI->isBarrier())
2701  return true;
2702  if (!MI->isPredicable())
2703  return true;
2704  return !isPredicated(MI);
2705 }
2706 
2707 bool X86InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
2708  MachineBasicBlock *&TBB,
2709  MachineBasicBlock *&FBB,
2710  SmallVectorImpl<MachineOperand> &Cond,
2711  bool AllowModify) const {
2712  // Start from the bottom of the block and work up, examining the
2713  // terminator instructions.
2714  MachineBasicBlock::iterator I = MBB.end();
2715  MachineBasicBlock::iterator UnCondBrIter = MBB.end();
2716  while (I != MBB.begin()) {
2717  --I;
2718  if (I->isDebugValue())
2719  continue;
2720 
2721  // Working from the bottom, when we see a non-terminator instruction, we're
2722  // done.
2723  if (!isUnpredicatedTerminator(I))
2724  break;
2725 
2726  // A terminator that isn't a branch can't easily be handled by this
2727  // analysis.
2728  if (!I->isBranch())
2729  return true;
2730 
2731  // Handle unconditional branches.
2732  if (I->getOpcode() == X86::JMP_4) {
2733  UnCondBrIter = I;
2734 
2735  if (!AllowModify) {
2736  TBB = I->getOperand(0).getMBB();
2737  continue;
2738  }
2739 
2740  // If the block has any instructions after a JMP, delete them.
2741  while (llvm::next(I) != MBB.end())
2742  llvm::next(I)->eraseFromParent();
2743 
2744  Cond.clear();
2745  FBB = 0;
2746 
2747  // Delete the JMP if it's equivalent to a fall-through.
2748  if (MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
2749  TBB = 0;
2750  I->eraseFromParent();
2751  I = MBB.end();
2752  UnCondBrIter = MBB.end();
2753  continue;
2754  }
2755 
2756  // TBB is used to indicate the unconditional destination.
2757  TBB = I->getOperand(0).getMBB();
2758  continue;
2759  }
2760 
2761  // Handle conditional branches.
2762  X86::CondCode BranchCode = getCondFromBranchOpc(I->getOpcode());
2763  if (BranchCode == X86::COND_INVALID)
2764  return true; // Can't handle indirect branch.
2765 
2766  // Working from the bottom, handle the first conditional branch.
2767  if (Cond.empty()) {
2768  MachineBasicBlock *TargetBB = I->getOperand(0).getMBB();
2769  if (AllowModify && UnCondBrIter != MBB.end() &&
2770  MBB.isLayoutSuccessor(TargetBB)) {
2771  // If we can modify the code and it ends in something like:
2772  //
2773  // jCC L1
2774  // jmp L2
2775  // L1:
2776  // ...
2777  // L2:
2778  //
2779  // Then we can change this to:
2780  //
2781  // jnCC L2
2782  // L1:
2783  // ...
2784  // L2:
2785  //
2786  // Which is a bit more efficient.
2787  // We conditionally jump to the fall-through block.
2788  BranchCode = GetOppositeBranchCondition(BranchCode);
2789  unsigned JNCC = GetCondBranchFromCond(BranchCode);
2790  MachineBasicBlock::iterator OldInst = I;
2791 
2792  BuildMI(MBB, UnCondBrIter, MBB.findDebugLoc(I), get(JNCC))
2793  .addMBB(UnCondBrIter->getOperand(0).getMBB());
2794  BuildMI(MBB, UnCondBrIter, MBB.findDebugLoc(I), get(X86::JMP_4))
2795  .addMBB(TargetBB);
2796 
2797  OldInst->eraseFromParent();
2798  UnCondBrIter->eraseFromParent();
2799 
2800  // Restart the analysis.
2801  UnCondBrIter = MBB.end();
2802  I = MBB.end();
2803  continue;
2804  }
2805 
2806  FBB = TBB;
2807  TBB = I->getOperand(0).getMBB();
2808  Cond.push_back(MachineOperand::CreateImm(BranchCode));
2809  continue;
2810  }
2811 
2812  // Handle subsequent conditional branches. Only handle the case where all
2813  // conditional branches branch to the same destination and their condition
2814  // opcodes fit one of the special multi-branch idioms.
2815  assert(Cond.size() == 1);
2816  assert(TBB);
2817 
2818  // Only handle the case where all conditional branches branch to the same
2819  // destination.
2820  if (TBB != I->getOperand(0).getMBB())
2821  return true;
2822 
2823  // If the conditions are the same, we can leave them alone.
2824  X86::CondCode OldBranchCode = (X86::CondCode)Cond[0].getImm();
2825  if (OldBranchCode == BranchCode)
2826  continue;
2827 
2828  // If they differ, see if they fit one of the known patterns. Theoretically,
2829  // we could handle more patterns here, but we shouldn't expect to see them
2830  // if instruction selection has done a reasonable job.
2831  if ((OldBranchCode == X86::COND_NP &&
2832  BranchCode == X86::COND_E) ||
2833  (OldBranchCode == X86::COND_E &&
2834  BranchCode == X86::COND_NP))
2835  BranchCode = X86::COND_NP_OR_E;
2836  else if ((OldBranchCode == X86::COND_P &&
2837  BranchCode == X86::COND_NE) ||
2838  (OldBranchCode == X86::COND_NE &&
2839  BranchCode == X86::COND_P))
2840  BranchCode = X86::COND_NE_OR_P;
2841  else
2842  return true;
2843 
2844  // Update the MachineOperand.
2845  Cond[0].setImm(BranchCode);
2846  }
2847 
2848  return false;
2849 }
2850 
2851 unsigned X86InstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
2852  MachineBasicBlock::iterator I = MBB.end();
2853  unsigned Count = 0;
2854 
2855  while (I != MBB.begin()) {
2856  --I;
2857  if (I->isDebugValue())
2858  continue;
2859  if (I->getOpcode() != X86::JMP_4 &&
2860  getCondFromBranchOpc(I->getOpcode()) == X86::COND_INVALID)
2861  break;
2862  // Remove the branch.
2863  I->eraseFromParent();
2864  I = MBB.end();
2865  ++Count;
2866  }
2867 
2868  return Count;
2869 }
2870 
2871 unsigned
2872 X86InstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
2873  MachineBasicBlock *FBB,
2874  const SmallVectorImpl<MachineOperand> &Cond,
2875  DebugLoc DL) const {
2876  // Shouldn't be a fall through.
2877  assert(TBB && "InsertBranch must not be told to insert a fallthrough");
2878  assert((Cond.size() == 1 || Cond.size() == 0) &&
2879  "X86 branch conditions have one component!");
2880 
2881  if (Cond.empty()) {
2882  // Unconditional branch?
2883  assert(!FBB && "Unconditional branch with multiple successors!");
2884  BuildMI(&MBB, DL, get(X86::JMP_4)).addMBB(TBB);
2885  return 1;
2886  }
2887 
2888  // Conditional branch.
2889  unsigned Count = 0;
2890  X86::CondCode CC = (X86::CondCode)Cond[0].getImm();
2891  switch (CC) {
2892  case X86::COND_NP_OR_E:
2893  // Synthesize NP_OR_E with two branches.
2894  BuildMI(&MBB, DL, get(X86::JNP_4)).addMBB(TBB);
2895  ++Count;
2896  BuildMI(&MBB, DL, get(X86::JE_4)).addMBB(TBB);
2897  ++Count;
2898  break;
2899  case X86::COND_NE_OR_P:
2900  // Synthesize NE_OR_P with two branches.
2901  BuildMI(&MBB, DL, get(X86::JNE_4)).addMBB(TBB);
2902  ++Count;
2903  BuildMI(&MBB, DL, get(X86::JP_4)).addMBB(TBB);
2904  ++Count;
2905  break;
2906  default: {
2907  unsigned Opc = GetCondBranchFromCond(CC);
2908  BuildMI(&MBB, DL, get(Opc)).addMBB(TBB);
2909  ++Count;
2910  }
2911  }
2912  if (FBB) {
2913  // Two-way Conditional branch. Insert the second branch.
2914  BuildMI(&MBB, DL, get(X86::JMP_4)).addMBB(FBB);
2915  ++Count;
2916  }
2917  return Count;
2918 }
2919 
2920 bool X86InstrInfo::
2921 canInsertSelect(const MachineBasicBlock &MBB,
2922  const SmallVectorImpl<MachineOperand> &Cond,
2923  unsigned TrueReg, unsigned FalseReg,
2924  int &CondCycles, int &TrueCycles, int &FalseCycles) const {
2925  // Not all subtargets have cmov instructions.
2926  if (!TM.getSubtarget<X86Subtarget>().hasCMov())
2927  return false;
2928  if (Cond.size() != 1)
2929  return false;
2930  // We cannot do the composite conditions, at least not in SSA form.
2931  if ((X86::CondCode)Cond[0].getImm() > X86::COND_S)
2932  return false;
2933 
2934  // Check register classes.
2935  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2936  const TargetRegisterClass *RC =
2937  RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
2938  if (!RC)
2939  return false;
2940 
2941  // We have cmov instructions for 16, 32, and 64 bit general purpose registers.
2942  if (X86::GR16RegClass.hasSubClassEq(RC) ||
2943  X86::GR32RegClass.hasSubClassEq(RC) ||
2944  X86::GR64RegClass.hasSubClassEq(RC)) {
2945  // This latency applies to Pentium M, Merom, Wolfdale, Nehalem, and Sandy
2946  // Bridge. Probably Ivy Bridge as well.
2947  CondCycles = 2;
2948  TrueCycles = 2;
2949  FalseCycles = 2;
2950  return true;
2951  }
2952 
2953  // Can't do vectors.
2954  return false;
2955 }
2956 
2957 void X86InstrInfo::insertSelect(MachineBasicBlock &MBB,
2958  MachineBasicBlock::iterator I, DebugLoc DL,
2959  unsigned DstReg,
2960  const SmallVectorImpl<MachineOperand> &Cond,
2961  unsigned TrueReg, unsigned FalseReg) const {
2962  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2963  assert(Cond.size() == 1 && "Invalid Cond array");
2964  unsigned Opc = getCMovFromCond((X86::CondCode)Cond[0].getImm(),
2965  MRI.getRegClass(DstReg)->getSize(),
2966  false/*HasMemoryOperand*/);
2967  BuildMI(MBB, I, DL, get(Opc), DstReg).addReg(FalseReg).addReg(TrueReg);
2968 }
2969 
2970 /// isHReg - Test if the given register is a physical h register.
2971 static bool isHReg(unsigned Reg) {
2972  return X86::GR8_ABCD_HRegClass.contains(Reg);
2973 }
2974 
2975 // Try and copy between VR128/VR64 and GR64 registers.
2976 static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg,
2977  const X86Subtarget& Subtarget) {
2978 
2979 
2980  // SrcReg(VR128) -> DestReg(GR64)
2981  // SrcReg(VR64) -> DestReg(GR64)
2982  // SrcReg(GR64) -> DestReg(VR128)
2983  // SrcReg(GR64) -> DestReg(VR64)
2984 
2985  bool HasAVX = Subtarget.hasAVX();
2986  bool HasAVX512 = Subtarget.hasAVX512();
2987  if (X86::GR64RegClass.contains(DestReg)) {
2988  if (X86::VR128XRegClass.contains(SrcReg))
2989  // Copy from a VR128 register to a GR64 register.
2990  return HasAVX512 ? X86::VMOVPQIto64Zrr: (HasAVX ? X86::VMOVPQIto64rr :
2991  X86::MOVPQIto64rr);
2992  if (X86::VR64RegClass.contains(SrcReg))
2993  // Copy from a VR64 register to a GR64 register.
2994  return X86::MOVSDto64rr;
2995  } else if (X86::GR64RegClass.contains(SrcReg)) {
2996  // Copy from a GR64 register to a VR128 register.
2997  if (X86::VR128XRegClass.contains(DestReg))
2998  return HasAVX512 ? X86::VMOV64toPQIZrr: (HasAVX ? X86::VMOV64toPQIrr :
2999  X86::MOV64toPQIrr);
3000  // Copy from a GR64 register to a VR64 register.
3001  if (X86::VR64RegClass.contains(DestReg))
3002  return X86::MOV64toSDrr;
3003  }
3004 
3005  // SrcReg(FR32) -> DestReg(GR32)
3006  // SrcReg(GR32) -> DestReg(FR32)
3007 
3008  if (X86::GR32RegClass.contains(DestReg) && X86::FR32XRegClass.contains(SrcReg))
3009  // Copy from a FR32 register to a GR32 register.
3010  return HasAVX512 ? X86::VMOVSS2DIZrr : (HasAVX ? X86::VMOVSS2DIrr : X86::MOVSS2DIrr);
3011 
3012  if (X86::FR32XRegClass.contains(DestReg) && X86::GR32RegClass.contains(SrcReg))
3013  // Copy from a GR32 register to a FR32 register.
3014  return HasAVX512 ? X86::VMOVDI2SSZrr : (HasAVX ? X86::VMOVDI2SSrr : X86::MOVDI2SSrr);
3015  return 0;
3016 }
3017 
3018 static
3019 unsigned copyPhysRegOpcode_AVX512(unsigned& DestReg, unsigned& SrcReg) {
3020  if (X86::VR128XRegClass.contains(DestReg, SrcReg) ||
3021  X86::VR256XRegClass.contains(DestReg, SrcReg) ||
3022  X86::VR512RegClass.contains(DestReg, SrcReg)) {
3023  DestReg = get512BitSuperRegister(DestReg);
3024  SrcReg = get512BitSuperRegister(SrcReg);
3025  return X86::VMOVAPSZrr;
3026  }
3027  if ((X86::VK8RegClass.contains(DestReg) ||
3028  X86::VK16RegClass.contains(DestReg)) &&
3029  (X86::VK8RegClass.contains(SrcReg) ||
3030  X86::VK16RegClass.contains(SrcReg)))
3031  return X86::KMOVWkk;
3032  return 0;
3033 }
3034 
3035 void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
3036  MachineBasicBlock::iterator MI, DebugLoc DL,
3037  unsigned DestReg, unsigned SrcReg,
3038  bool KillSrc) const {
3039  // First deal with the normal symmetric copies.
3040  bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX();
3041  bool HasAVX512 = TM.getSubtarget<X86Subtarget>().hasAVX512();
3042  unsigned Opc = 0;
3043  if (X86::GR64RegClass.contains(DestReg, SrcReg))
3044  Opc = X86::MOV64rr;
3045  else if (X86::GR32RegClass.contains(DestReg, SrcReg))
3046  Opc = X86::MOV32rr;
3047  else if (X86::GR16RegClass.contains(DestReg, SrcReg))
3048  Opc = X86::MOV16rr;
3049  else if (X86::GR8RegClass.contains(DestReg, SrcReg)) {
3050  // Copying to or from a physical H register on x86-64 requires a NOREX
3051  // move. Otherwise use a normal move.
3052  if ((isHReg(DestReg) || isHReg(SrcReg)) &&
3053  TM.getSubtarget<X86Subtarget>().is64Bit()) {
3054  Opc = X86::MOV8rr_NOREX;
3055  // Both operands must be encodable without an REX prefix.
3056  assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) &&
3057  "8-bit H register can not be copied outside GR8_NOREX");
3058  } else
3059  Opc = X86::MOV8rr;
3060  }
3061  else if (X86::VR64RegClass.contains(DestReg, SrcReg))
3062  Opc = X86::MMX_MOVQ64rr;
3063  else if (HasAVX512)
3064  Opc = copyPhysRegOpcode_AVX512(DestReg, SrcReg);
3065  else if (X86::VR128RegClass.contains(DestReg, SrcReg))
3066  Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr;
3067  else if (X86::VR256RegClass.contains(DestReg, SrcReg))
3068  Opc = X86::VMOVAPSYrr;
3069  if (!Opc)
3070  Opc = CopyToFromAsymmetricReg(DestReg, SrcReg, TM.getSubtarget<X86Subtarget>());
3071 
3072  if (Opc) {
3073  BuildMI(MBB, MI, DL, get(Opc), DestReg)
3074  .addReg(SrcReg, getKillRegState(KillSrc));
3075  return;
3076  }
3077 
3078  // Moving EFLAGS to / from another register requires a push and a pop.
3079  // Notice that we have to adjust the stack if we don't want to clobber the
3080  // first frame index. See X86FrameLowering.cpp - colobbersTheStack.
3081  if (SrcReg == X86::EFLAGS) {
3082  if (X86::GR64RegClass.contains(DestReg)) {
3083  BuildMI(MBB, MI, DL, get(X86::PUSHF64));
3084  BuildMI(MBB, MI, DL, get(X86::POP64r), DestReg);
3085  return;
3086  }
3087  if (X86::GR32RegClass.contains(DestReg)) {
3088  BuildMI(MBB, MI, DL, get(X86::PUSHF32));
3089  BuildMI(MBB, MI, DL, get(X86::POP32r), DestReg);
3090  return;
3091  }
3092  }
3093  if (DestReg == X86::EFLAGS) {
3094  if (X86::GR64RegClass.contains(SrcReg)) {
3095  BuildMI(MBB, MI, DL, get(X86::PUSH64r))
3096  .addReg(SrcReg, getKillRegState(KillSrc));
3097  BuildMI(MBB, MI, DL, get(X86::POPF64));
3098  return;
3099  }
3100  if (X86::GR32RegClass.contains(SrcReg)) {
3101  BuildMI(MBB, MI, DL, get(X86::PUSH32r))
3102  .addReg(SrcReg, getKillRegState(KillSrc));
3103  BuildMI(MBB, MI, DL, get(X86::POPF32));
3104  return;
3105  }
3106  }
3107 
3108  DEBUG(dbgs() << "Cannot copy " << RI.getName(SrcReg)
3109  << " to " << RI.getName(DestReg) << '\n');
3110  llvm_unreachable("Cannot emit physreg copy instruction");
3111 }
3112 
3113 static unsigned getLoadStoreRegOpcode(unsigned Reg,
3114  const TargetRegisterClass *RC,
3115  bool isStackAligned,
3116  const TargetMachine &TM,
3117  bool load) {
3118  if (TM.getSubtarget<X86Subtarget>().hasAVX512()) {
3119  if (X86::VK8RegClass.hasSubClassEq(RC) ||
3120  X86::VK16RegClass.hasSubClassEq(RC))
3121  return load ? X86::KMOVWkm : X86::KMOVWmk;
3122  if (RC->getSize() == 4 && X86::FR32XRegClass.hasSubClassEq(RC))
3123  return load ? X86::VMOVSSZrm : X86::VMOVSSZmr;
3124  if (RC->getSize() == 8 && X86::FR64XRegClass.hasSubClassEq(RC))
3125  return load ? X86::VMOVSDZrm : X86::VMOVSDZmr;
3126  if (X86::VR512RegClass.hasSubClassEq(RC))
3127  return load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
3128  }
3129 
3130  bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX();
3131  switch (RC->getSize()) {
3132  default:
3133  llvm_unreachable("Unknown spill size");
3134  case 1:
3135  assert(X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass");
3136  if (TM.getSubtarget<X86Subtarget>().is64Bit())
3137  // Copying to or from a physical H register on x86-64 requires a NOREX
3138  // move. Otherwise use a normal move.
3139  if (isHReg(Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC))
3140  return load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX;
3141  return load ? X86::MOV8rm : X86::MOV8mr;
3142  case 2:
3143  assert(X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass");
3144  return load ? X86::MOV16rm : X86::MOV16mr;
3145  case 4:
3146  if (X86::GR32RegClass.hasSubClassEq(RC))
3147  return load ? X86::MOV32rm : X86::MOV32mr;
3148  if (X86::FR32RegClass.hasSubClassEq(RC))
3149  return load ?
3150  (HasAVX ? X86::VMOVSSrm : X86::MOVSSrm) :
3151  (HasAVX ? X86::VMOVSSmr : X86::MOVSSmr);
3152  if (X86::RFP32RegClass.hasSubClassEq(RC))
3153  return load ? X86::LD_Fp32m : X86::ST_Fp32m;
3154  llvm_unreachable("Unknown 4-byte regclass");
3155  case 8:
3156  if (X86::GR64RegClass.hasSubClassEq(RC))
3157  return load ? X86::MOV64rm : X86::MOV64mr;
3158  if (X86::FR64RegClass.hasSubClassEq(RC))
3159  return load ?
3160  (HasAVX ? X86::VMOVSDrm : X86::MOVSDrm) :
3161  (HasAVX ? X86::VMOVSDmr : X86::MOVSDmr);
3162  if (X86::VR64RegClass.hasSubClassEq(RC))
3163  return load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr;
3164  if (X86::RFP64RegClass.hasSubClassEq(RC))
3165  return load ? X86::LD_Fp64m : X86::ST_Fp64m;
3166  llvm_unreachable("Unknown 8-byte regclass");
3167  case 10:
3168  assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass");
3169  return load ? X86::LD_Fp80m : X86::ST_FpP80m;
3170  case 16: {
3171  assert((X86::VR128RegClass.hasSubClassEq(RC) ||
3172  X86::VR128XRegClass.hasSubClassEq(RC))&& "Unknown 16-byte regclass");
3173  // If stack is realigned we can use aligned stores.
3174  if (isStackAligned)
3175  return load ?
3176  (HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm) :
3177  (HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
3178  else
3179  return load ?
3180  (HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm) :
3181  (HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
3182  }
3183  case 32:
3184  assert((X86::VR256RegClass.hasSubClassEq(RC) ||
3185  X86::VR256XRegClass.hasSubClassEq(RC)) && "Unknown 32-byte regclass");
3186  // If stack is realigned we can use aligned stores.
3187  if (isStackAligned)
3188  return load ? X86::VMOVAPSYrm : X86::VMOVAPSYmr;
3189  else
3190  return load ? X86::VMOVUPSYrm : X86::VMOVUPSYmr;
3191  case 64:
3192  assert(X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass");
3193  if (isStackAligned)
3194  return load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
3195  else
3196  return load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
3197  }
3198 }
3199 
3200 static unsigned getStoreRegOpcode(unsigned SrcReg,
3201  const TargetRegisterClass *RC,
3202  bool isStackAligned,
3203  TargetMachine &TM) {
3204  return getLoadStoreRegOpcode(SrcReg, RC, isStackAligned, TM, false);
3205 }
3206 
3207 
3208 static unsigned getLoadRegOpcode(unsigned DestReg,
3209  const TargetRegisterClass *RC,
3210  bool isStackAligned,
3211  const TargetMachine &TM) {
3212  return getLoadStoreRegOpcode(DestReg, RC, isStackAligned, TM, true);
3213 }
3214 
3215 void X86InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
3216  MachineBasicBlock::iterator MI,
3217  unsigned SrcReg, bool isKill, int FrameIdx,
3218  const TargetRegisterClass *RC,
3219  const TargetRegisterInfo *TRI) const {
3220  const MachineFunction &MF = *MBB.getParent();
3221  assert(MF.getFrameInfo()->getObjectSize(FrameIdx) >= RC->getSize() &&
3222  "Stack slot too small for store");
3223  unsigned Alignment = std::max<uint32_t>(RC->getSize(), 16);
3224  bool isAligned = (TM.getFrameLowering()->getStackAlignment() >= Alignment) ||
3225  RI.canRealignStack(MF);
3226  unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, TM);
3227  DebugLoc DL = MBB.findDebugLoc(MI);
3228  addFrameReference(BuildMI(MBB, MI, DL, get(Opc)), FrameIdx)
3229  .addReg(SrcReg, getKillRegState(isKill));
3230 }
3231 
3232 void X86InstrInfo::storeRegToAddr(MachineFunction &MF, unsigned SrcReg,
3233  bool isKill,
3234  SmallVectorImpl<MachineOperand> &Addr,
3235  const TargetRegisterClass *RC,
3236  MachineInstr::mmo_iterator MMOBegin,
3237  MachineInstr::mmo_iterator MMOEnd,
3238  SmallVectorImpl<MachineInstr*> &NewMIs) const {
3239  unsigned Alignment = std::max<uint32_t>(RC->getSize(), 16);
3240  bool isAligned = MMOBegin != MMOEnd &&
3241  (*MMOBegin)->getAlignment() >= Alignment;
3242  unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, TM);
3243  DebugLoc DL;
3244  MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc));
3245  for (unsigned i = 0, e = Addr.size(); i != e; ++i)
3246  MIB.addOperand(Addr[i]);
3247  MIB.addReg(SrcReg, getKillRegState(isKill));
3248  (*MIB).setMemRefs(MMOBegin, MMOEnd);
3249  NewMIs.push_back(MIB);
3250 }
3251 
3252 
3253 void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
3254  MachineBasicBlock::iterator MI,
3255  unsigned DestReg, int FrameIdx,
3256  const TargetRegisterClass *RC,
3257  const TargetRegisterInfo *TRI) const {
3258  const MachineFunction &MF = *MBB.getParent();
3259  unsigned Alignment = std::max<uint32_t>(RC->getSize(), 16);
3260  bool isAligned = (TM.getFrameLowering()->getStackAlignment() >= Alignment) ||
3261  RI.canRealignStack(MF);
3262  unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, TM);
3263  DebugLoc DL = MBB.findDebugLoc(MI);
3264  addFrameReference(BuildMI(MBB, MI, DL, get(Opc), DestReg), FrameIdx);
3265 }
3266 
3267 void X86InstrInfo::loadRegFromAddr(MachineFunction &MF, unsigned DestReg,
3268  SmallVectorImpl<MachineOperand> &Addr,
3269  const TargetRegisterClass *RC,
3270  MachineInstr::mmo_iterator MMOBegin,
3271  MachineInstr::mmo_iterator MMOEnd,
3272  SmallVectorImpl<MachineInstr*> &NewMIs) const {
3273  unsigned Alignment = std::max<uint32_t>(RC->getSize(), 16);
3274  bool isAligned = MMOBegin != MMOEnd &&
3275  (*MMOBegin)->getAlignment() >= Alignment;
3276  unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, TM);
3277  DebugLoc DL;
3278  MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc), DestReg);
3279  for (unsigned i = 0, e = Addr.size(); i != e; ++i)
3280  MIB.addOperand(Addr[i]);
3281  (*MIB).setMemRefs(MMOBegin, MMOEnd);
3282  NewMIs.push_back(MIB);
3283 }
3284 
3285 bool X86InstrInfo::
3286 analyzeCompare(const MachineInstr *MI, unsigned &SrcReg, unsigned &SrcReg2,
3287  int &CmpMask, int &CmpValue) const {
3288  switch (MI->getOpcode()) {
3289  default: break;
3290  case X86::CMP64ri32:
3291  case X86::CMP64ri8:
3292  case X86::CMP32ri:
3293  case X86::CMP32ri8:
3294  case X86::CMP16ri:
3295  case X86::CMP16ri8:
3296  case X86::CMP8ri:
3297  SrcReg = MI->getOperand(0).getReg();
3298  SrcReg2 = 0;
3299  CmpMask = ~0;
3300  CmpValue = MI->getOperand(1).getImm();
3301  return true;
3302  // A SUB can be used to perform comparison.
3303  case X86::SUB64rm:
3304  case X86::SUB32rm:
3305  case X86::SUB16rm:
3306  case X86::SUB8rm:
3307  SrcReg = MI->getOperand(1).getReg();
3308  SrcReg2 = 0;
3309  CmpMask = ~0;
3310  CmpValue = 0;
3311  return true;
3312  case X86::SUB64rr:
3313  case X86::SUB32rr:
3314  case X86::SUB16rr:
3315  case X86::SUB8rr:
3316  SrcReg = MI->getOperand(1).getReg();
3317  SrcReg2 = MI->getOperand(2).getReg();
3318  CmpMask = ~0;
3319  CmpValue = 0;
3320  return true;
3321  case X86::SUB64ri32:
3322  case X86::SUB64ri8:
3323  case X86::SUB32ri:
3324  case X86::SUB32ri8:
3325  case X86::SUB16ri:
3326  case X86::SUB16ri8:
3327  case X86::SUB8ri:
3328  SrcReg = MI->getOperand(1).getReg();
3329  SrcReg2 = 0;
3330  CmpMask = ~0;
3331  CmpValue = MI->getOperand(2).getImm();
3332  return true;
3333  case X86::CMP64rr:
3334  case X86::CMP32rr:
3335  case X86::CMP16rr:
3336  case X86::CMP8rr:
3337  SrcReg = MI->getOperand(0).getReg();
3338  SrcReg2 = MI->getOperand(1).getReg();
3339  CmpMask = ~0;
3340  CmpValue = 0;
3341  return true;
3342  case X86::TEST8rr:
3343  case X86::TEST16rr:
3344  case X86::TEST32rr:
3345  case X86::TEST64rr:
3346  SrcReg = MI->getOperand(0).getReg();
3347  if (MI->getOperand(1).getReg() != SrcReg) return false;
3348  // Compare against zero.
3349  SrcReg2 = 0;
3350  CmpMask = ~0;
3351  CmpValue = 0;
3352  return true;
3353  }
3354  return false;
3355 }
3356 
3357 /// isRedundantFlagInstr - check whether the first instruction, whose only
3358 /// purpose is to update flags, can be made redundant.
3359 /// CMPrr can be made redundant by SUBrr if the operands are the same.
3360 /// This function can be extended later on.
3361 /// SrcReg, SrcRegs: register operands for FlagI.
3362 /// ImmValue: immediate for FlagI if it takes an immediate.
3363 inline static bool isRedundantFlagInstr(MachineInstr *FlagI, unsigned SrcReg,
3364  unsigned SrcReg2, int ImmValue,
3365  MachineInstr *OI) {
3366  if (((FlagI->getOpcode() == X86::CMP64rr &&
3367  OI->getOpcode() == X86::SUB64rr) ||
3368  (FlagI->getOpcode() == X86::CMP32rr &&
3369  OI->getOpcode() == X86::SUB32rr)||
3370  (FlagI->getOpcode() == X86::CMP16rr &&
3371  OI->getOpcode() == X86::SUB16rr)||
3372  (FlagI->getOpcode() == X86::CMP8rr &&
3373  OI->getOpcode() == X86::SUB8rr)) &&
3374  ((OI->getOperand(1).getReg() == SrcReg &&
3375  OI->getOperand(2).getReg() == SrcReg2) ||
3376  (OI->getOperand(1).getReg() == SrcReg2 &&
3377  OI->getOperand(2).getReg() == SrcReg)))
3378  return true;
3379 
3380  if (((FlagI->getOpcode() == X86::CMP64ri32 &&
3381  OI->getOpcode() == X86::SUB64ri32) ||
3382  (FlagI->getOpcode() == X86::CMP64ri8 &&
3383  OI->getOpcode() == X86::SUB64ri8) ||
3384  (FlagI->getOpcode() == X86::CMP32ri &&
3385  OI->getOpcode() == X86::SUB32ri) ||
3386  (FlagI->getOpcode() == X86::CMP32ri8 &&
3387  OI->getOpcode() == X86::SUB32ri8) ||
3388  (FlagI->getOpcode() == X86::CMP16ri &&
3389  OI->getOpcode() == X86::SUB16ri) ||
3390  (FlagI->getOpcode() == X86::CMP16ri8 &&
3391  OI->getOpcode() == X86::SUB16ri8) ||
3392  (FlagI->getOpcode() == X86::CMP8ri &&
3393  OI->getOpcode() == X86::SUB8ri)) &&
3394  OI->getOperand(1).getReg() == SrcReg &&
3395  OI->getOperand(2).getImm() == ImmValue)
3396  return true;
3397  return false;
3398 }
3399 
3400 /// isDefConvertible - check whether the definition can be converted
3401 /// to remove a comparison against zero.
3402 inline static bool isDefConvertible(MachineInstr *MI) {
3403  switch (MI->getOpcode()) {
3404  default: return false;
3405 
3406  // The shift instructions only modify ZF if their shift count is non-zero.
3407  // N.B.: The processor truncates the shift count depending on the encoding.
3408  case X86::SAR8ri: case X86::SAR16ri: case X86::SAR32ri:case X86::SAR64ri:
3409  case X86::SHR8ri: case X86::SHR16ri: case X86::SHR32ri:case X86::SHR64ri:
3410  return getTruncatedShiftCount(MI, 2) != 0;
3411 
3412  // Some left shift instructions can be turned into LEA instructions but only
3413  // if their flags aren't used. Avoid transforming such instructions.
3414  case X86::SHL8ri: case X86::SHL16ri: case X86::SHL32ri:case X86::SHL64ri:{
3415  unsigned ShAmt = getTruncatedShiftCount(MI, 2);
3416  if (isTruncatedShiftCountForLEA(ShAmt)) return false;
3417  return ShAmt != 0;
3418  }
3419 
3420  case X86::SHRD16rri8:case X86::SHRD32rri8:case X86::SHRD64rri8:
3421  case X86::SHLD16rri8:case X86::SHLD32rri8:case X86::SHLD64rri8:
3422  return getTruncatedShiftCount(MI, 3) != 0;
3423 
3424  case X86::SUB64ri32: case X86::SUB64ri8: case X86::SUB32ri:
3425  case X86::SUB32ri8: case X86::SUB16ri: case X86::SUB16ri8:
3426  case X86::SUB8ri: case X86::SUB64rr: case X86::SUB32rr:
3427  case X86::SUB16rr: case X86::SUB8rr: case X86::SUB64rm:
3428  case X86::SUB32rm: case X86::SUB16rm: case X86::SUB8rm:
3429  case X86::DEC64r: case X86::DEC32r: case X86::DEC16r: case X86::DEC8r:
3430  case X86::DEC64_32r: case X86::DEC64_16r:
3431  case X86::ADD64ri32: case X86::ADD64ri8: case X86::ADD32ri:
3432  case X86::ADD32ri8: case X86::ADD16ri: case X86::ADD16ri8:
3433  case X86::ADD8ri: case X86::ADD64rr: case X86::ADD32rr:
3434  case X86::ADD16rr: case X86::ADD8rr: case X86::ADD64rm:
3435  case X86::ADD32rm: case X86::ADD16rm: case X86::ADD8rm:
3436  case X86::INC64r: case X86::INC32r: case X86::INC16r: case X86::INC8r:
3437  case X86::INC64_32r: case X86::INC64_16r:
3438  case X86::AND64ri32: case X86::AND64ri8: case X86::AND32ri:
3439  case X86::AND32ri8: case X86::AND16ri: case X86::AND16ri8:
3440  case X86::AND8ri: case X86::AND64rr: case X86::AND32rr:
3441  case X86::AND16rr: case X86::AND8rr: case X86::AND64rm:
3442  case X86::AND32rm: case X86::AND16rm: case X86::AND8rm:
3443  case X86::XOR64ri32: case X86::XOR64ri8: case X86::XOR32ri:
3444  case X86::XOR32ri8: case X86::XOR16ri: case X86::XOR16ri8:
3445  case X86::XOR8ri: case X86::XOR64rr: case X86::XOR32rr:
3446  case X86::XOR16rr: case X86::XOR8rr: case X86::XOR64rm:
3447  case X86::XOR32rm: case X86::XOR16rm: case X86::XOR8rm:
3448  case X86::OR64ri32: case X86::OR64ri8: case X86::OR32ri:
3449  case X86::OR32ri8: case X86::OR16ri: case X86::OR16ri8:
3450  case X86::OR8ri: case X86::OR64rr: case X86::OR32rr:
3451  case X86::OR16rr: case X86::OR8rr: case X86::OR64rm:
3452  case X86::OR32rm: case X86::OR16rm: case X86::OR8rm:
3453  case X86::NEG8r: case X86::NEG16r: case X86::NEG32r: case X86::NEG64r:
3454  case X86::SAR8r1: case X86::SAR16r1: case X86::SAR32r1:case X86::SAR64r1:
3455  case X86::SHR8r1: case X86::SHR16r1: case X86::SHR32r1:case X86::SHR64r1:
3456  case X86::SHL8r1: case X86::SHL16r1: case X86::SHL32r1:case X86::SHL64r1:
3457  case X86::ADC32ri: case X86::ADC32ri8:
3458  case X86::ADC32rr: case X86::ADC64ri32:
3459  case X86::ADC64ri8: case X86::ADC64rr:
3460  case X86::SBB32ri: case X86::SBB32ri8:
3461  case X86::SBB32rr: case X86::SBB64ri32:
3462  case X86::SBB64ri8: case X86::SBB64rr:
3463  case X86::ANDN32rr: case X86::ANDN32rm:
3464  case X86::ANDN64rr: case X86::ANDN64rm:
3465  case X86::BEXTR32rr: case X86::BEXTR64rr:
3466  case X86::BEXTR32rm: case X86::BEXTR64rm:
3467  case X86::BLSI32rr: case X86::BLSI32rm:
3468  case X86::BLSI64rr: case X86::BLSI64rm:
3469  case X86::BLSMSK32rr:case X86::BLSMSK32rm:
3470  case X86::BLSMSK64rr:case X86::BLSMSK64rm:
3471  case X86::BLSR32rr: case X86::BLSR32rm:
3472  case X86::BLSR64rr: case X86::BLSR64rm:
3473  case X86::BZHI32rr: case X86::BZHI32rm:
3474  case X86::BZHI64rr: case X86::BZHI64rm:
3475  case X86::LZCNT16rr: case X86::LZCNT16rm:
3476  case X86::LZCNT32rr: case X86::LZCNT32rm:
3477  case X86::LZCNT64rr: case X86::LZCNT64rm:
3478  case X86::POPCNT16rr:case X86::POPCNT16rm:
3479  case X86::POPCNT32rr:case X86::POPCNT32rm:
3480  case X86::POPCNT64rr:case X86::POPCNT64rm:
3481  case X86::TZCNT16rr: case X86::TZCNT16rm:
3482  case X86::TZCNT32rr: case X86::TZCNT32rm:
3483  case X86::TZCNT64rr: case X86::TZCNT64rm:
3484  return true;
3485  }
3486 }
3487 
3488 /// optimizeCompareInstr - Check if there exists an earlier instruction that
3489 /// operates on the same source operands and sets flags in the same way as
3490 /// Compare; remove Compare if possible.
3491 bool X86InstrInfo::
3492 optimizeCompareInstr(MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2,
3493  int CmpMask, int CmpValue,
3494  const MachineRegisterInfo *MRI) const {
3495  // Check whether we can replace SUB with CMP.
3496  unsigned NewOpcode = 0;
3497  switch (CmpInstr->getOpcode()) {
3498  default: break;
3499  case X86::SUB64ri32:
3500  case X86::SUB64ri8:
3501  case X86::SUB32ri:
3502  case X86::SUB32ri8:
3503  case X86::SUB16ri:
3504  case X86::SUB16ri8:
3505  case X86::SUB8ri:
3506  case X86::SUB64rm:
3507  case X86::SUB32rm:
3508  case X86::SUB16rm:
3509  case X86::SUB8rm:
3510  case X86::SUB64rr:
3511  case X86::SUB32rr:
3512  case X86::SUB16rr:
3513  case X86::SUB8rr: {
3514  if (!MRI->use_nodbg_empty(CmpInstr->getOperand(0).getReg()))
3515  return false;
3516  // There is no use of the destination register, we can replace SUB with CMP.
3517  switch (CmpInstr->getOpcode()) {
3518  default: llvm_unreachable("Unreachable!");
3519  case X86::SUB64rm: NewOpcode = X86::CMP64rm; break;
3520  case X86::SUB32rm: NewOpcode = X86::CMP32rm; break;
3521  case X86::SUB16rm: NewOpcode = X86::CMP16rm; break;
3522  case X86::SUB8rm: NewOpcode = X86::CMP8rm; break;
3523  case X86::SUB64rr: NewOpcode = X86::CMP64rr; break;
3524  case X86::SUB32rr: NewOpcode = X86::CMP32rr; break;
3525  case X86::SUB16rr: NewOpcode = X86::CMP16rr; break;
3526  case X86::SUB8rr: NewOpcode = X86::CMP8rr; break;
3527  case X86::SUB64ri32: NewOpcode = X86::CMP64ri32; break;
3528  case X86::SUB64ri8: NewOpcode = X86::CMP64ri8; break;
3529  case X86::SUB32ri: NewOpcode = X86::CMP32ri; break;
3530  case X86::SUB32ri8: NewOpcode = X86::CMP32ri8; break;
3531  case X86::SUB16ri: NewOpcode = X86::CMP16ri; break;
3532  case X86::SUB16ri8: NewOpcode = X86::CMP16ri8; break;
3533  case X86::SUB8ri: NewOpcode = X86::CMP8ri; break;
3534  }
3535  CmpInstr->setDesc(get(NewOpcode));
3536  CmpInstr->RemoveOperand(0);
3537  // Fall through to optimize Cmp if Cmp is CMPrr or CMPri.
3538  if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
3539  NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
3540  return false;
3541  }
3542  }
3543 
3544  // Get the unique definition of SrcReg.
3545  MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
3546  if (!MI) return false;
3547 
3548  // CmpInstr is the first instruction of the BB.
3549  MachineBasicBlock::iterator I = CmpInstr, Def = MI;
3550 
3551  // If we are comparing against zero, check whether we can use MI to update
3552  // EFLAGS. If MI is not in the same BB as CmpInstr, do not optimize.
3553  bool IsCmpZero = (SrcReg2 == 0 && CmpValue == 0);
3554  if (IsCmpZero && (MI->getParent() != CmpInstr->getParent() ||
3555  !isDefConvertible(MI)))
3556  return false;
3557 
3558  // We are searching for an earlier instruction that can make CmpInstr
3559  // redundant and that instruction will be saved in Sub.
3560  MachineInstr *Sub = NULL;
3561  const TargetRegisterInfo *TRI = &getRegisterInfo();
3562 
3563  // We iterate backward, starting from the instruction before CmpInstr and
3564  // stop when reaching the definition of a source register or done with the BB.
3565  // RI points to the instruction before CmpInstr.
3566  // If the definition is in this basic block, RE points to the definition;
3567  // otherwise, RE is the rend of the basic block.
3568  MachineBasicBlock::reverse_iterator
3569  RI = MachineBasicBlock::reverse_iterator(I),
3570  RE = CmpInstr->getParent() == MI->getParent() ?
3571  MachineBasicBlock::reverse_iterator(++Def) /* points to MI */ :
3572  CmpInstr->getParent()->rend();
3573  MachineInstr *Movr0Inst = 0;
3574  for (; RI != RE; ++RI) {
3575  MachineInstr *Instr = &*RI;
3576  // Check whether CmpInstr can be made redundant by the current instruction.
3577  if (!IsCmpZero &&
3578  isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpValue, Instr)) {
3579  Sub = Instr;
3580  break;
3581  }
3582 
3583  if (Instr->modifiesRegister(X86::EFLAGS, TRI) ||
3584  Instr->readsRegister(X86::EFLAGS, TRI)) {
3585  // This instruction modifies or uses EFLAGS.
3586 
3587  // MOV32r0 etc. are implemented with xor which clobbers condition code.
3588  // They are safe to move up, if the definition to EFLAGS is dead and
3589  // earlier instructions do not read or write EFLAGS.
3590  if (!Movr0Inst && Instr->getOpcode() == X86::MOV32r0 &&
3591  Instr->registerDefIsDead(X86::EFLAGS, TRI)) {
3592  Movr0Inst = Instr;
3593  continue;
3594  }
3595 
3596  // We can't remove CmpInstr.
3597  return false;
3598  }
3599  }
3600 
3601  // Return false if no candidates exist.
3602  if (!IsCmpZero && !Sub)
3603  return false;
3604 
3605  bool IsSwapped = (SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
3606  Sub->getOperand(2).getReg() == SrcReg);
3607 
3608  // Scan forward from the instruction after CmpInstr for uses of EFLAGS.
3609  // It is safe to remove CmpInstr if EFLAGS is redefined or killed.
3610  // If we are done with the basic block, we need to check whether EFLAGS is
3611  // live-out.
3612  bool IsSafe = false;
3613  SmallVector<std::pair<MachineInstr*, unsigned /*NewOpc*/>, 4> OpsToUpdate;
3614  MachineBasicBlock::iterator E = CmpInstr->getParent()->end();
3615  for (++I; I != E; ++I) {
3616  const MachineInstr &Instr = *I;
3617  bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS, TRI);
3618  bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS, TRI);
3619  // We should check the usage if this instruction uses and updates EFLAGS.
3620  if (!UseEFLAGS && ModifyEFLAGS) {
3621  // It is safe to remove CmpInstr if EFLAGS is updated again.
3622  IsSafe = true;
3623  break;
3624  }
3625  if (!UseEFLAGS && !ModifyEFLAGS)
3626  continue;
3627 
3628  // EFLAGS is used by this instruction.
3629  X86::CondCode OldCC;
3630  bool OpcIsSET = false;
3631  if (IsCmpZero || IsSwapped) {
3632  // We decode the condition code from opcode.
3633  if (Instr.isBranch())
3634  OldCC = getCondFromBranchOpc(Instr.getOpcode());
3635  else {
3636  OldCC = getCondFromSETOpc(Instr.getOpcode());
3637  if (OldCC != X86::COND_INVALID)
3638  OpcIsSET = true;
3639  else
3640  OldCC = X86::getCondFromCMovOpc(Instr.getOpcode());
3641  }
3642  if (OldCC == X86::COND_INVALID) return false;
3643  }
3644  if (IsCmpZero) {
3645  switch (OldCC) {
3646  default: break;
3647  case X86::COND_A: case X86::COND_AE:
3648  case X86::COND_B: case X86::COND_BE:
3649  case X86::COND_G: case X86::COND_GE:
3650  case X86::COND_L: case X86::COND_LE:
3651  case X86::COND_O: case X86::COND_NO:
3652  // CF and OF are used, we can't perform this optimization.
3653  return false;
3654  }
3655  } else if (IsSwapped) {
3656  // If we have SUB(r1, r2) and CMP(r2, r1), the condition code needs
3657  // to be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
3658  // We swap the condition code and synthesize the new opcode.
3659  X86::CondCode NewCC = getSwappedCondition(OldCC);
3660  if (NewCC == X86::COND_INVALID) return false;
3661 
3662  // Synthesize the new opcode.
3663  bool HasMemoryOperand = Instr.hasOneMemOperand();
3664  unsigned NewOpc;
3665  if (Instr.isBranch())
3666  NewOpc = GetCondBranchFromCond(NewCC);
3667  else if(OpcIsSET)
3668  NewOpc = getSETFromCond(NewCC, HasMemoryOperand);
3669  else {
3670  unsigned DstReg = Instr.getOperand(0).getReg();
3671  NewOpc = getCMovFromCond(NewCC, MRI->getRegClass(DstReg)->getSize(),
3672  HasMemoryOperand);
3673  }
3674 
3675  // Push the MachineInstr to OpsToUpdate.
3676  // If it is safe to remove CmpInstr, the condition code of these
3677  // instructions will be modified.
3678  OpsToUpdate.push_back(std::make_pair(&*I, NewOpc));
3679  }
3680  if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS, TRI)) {
3681  // It is safe to remove CmpInstr if EFLAGS is updated again or killed.
3682  IsSafe = true;
3683  break;
3684  }
3685  }
3686 
3687  // If EFLAGS is not killed nor re-defined, we should check whether it is
3688  // live-out. If it is live-out, do not optimize.
3689  if ((IsCmpZero || IsSwapped) && !IsSafe) {
3690  MachineBasicBlock *MBB = CmpInstr->getParent();
3691  for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(),
3692  SE = MBB->succ_end(); SI != SE; ++SI)
3693  if ((*SI)->isLiveIn(X86::EFLAGS))
3694  return false;
3695  }
3696 
3697  // The instruction to be updated is either Sub or MI.
3698  Sub = IsCmpZero ? MI : Sub;
3699  // Move Movr0Inst to the appropriate place before Sub.
3700  if (Movr0Inst) {
3701  // Look backwards until we find a def that doesn't use the current EFLAGS.
3702  Def = Sub;
3703  MachineBasicBlock::reverse_iterator
3704  InsertI = MachineBasicBlock::reverse_iterator(++Def),
3705  InsertE = Sub->getParent()->rend();
3706  for (; InsertI != InsertE; ++InsertI) {
3707  MachineInstr *Instr = &*InsertI;
3708  if (!Instr->readsRegister(X86::EFLAGS, TRI) &&
3709  Instr->modifiesRegister(X86::EFLAGS, TRI)) {
3710  Sub->getParent()->remove(Movr0Inst);
3711  Instr->getParent()->insert(MachineBasicBlock::iterator(Instr),
3712  Movr0Inst);
3713  break;
3714  }
3715  }
3716  if (InsertI == InsertE)
3717  return false;
3718  }
3719 
3720  // Make sure Sub instruction defines EFLAGS and mark the def live.
3721  unsigned i = 0, e = Sub->getNumOperands();
3722  for (; i != e; ++i) {
3723  MachineOperand &MO = Sub->getOperand(i);
3724  if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS) {
3725  MO.setIsDead(false);
3726  break;
3727  }
3728  }
3729  assert(i != e && "Unable to locate a def EFLAGS operand");
3730 
3731  CmpInstr->eraseFromParent();
3732 
3733  // Modify the condition code of instructions in OpsToUpdate.
3734  for (unsigned i = 0, e = OpsToUpdate.size(); i < e; i++)
3735  OpsToUpdate[i].first->setDesc(get(OpsToUpdate[i].second));
3736  return true;
3737 }
3738 
3739 /// optimizeLoadInstr - Try to remove the load by folding it to a register
3740 /// operand at the use. We fold the load instructions if load defines a virtual
3741 /// register, the virtual register is used once in the same BB, and the
3742 /// instructions in-between do not load or store, and have no side effects.
3743 MachineInstr* X86InstrInfo::
3744 optimizeLoadInstr(MachineInstr *MI, const MachineRegisterInfo *MRI,
3745  unsigned &FoldAsLoadDefReg,
3746  MachineInstr *&DefMI) const {
3747  if (FoldAsLoadDefReg == 0)
3748  return 0;
3749  // To be conservative, if there exists another load, clear the load candidate.
3750  if (MI->mayLoad()) {
3751  FoldAsLoadDefReg = 0;
3752  return 0;
3753  }
3754 
3755  // Check whether we can move DefMI here.
3756  DefMI = MRI->getVRegDef(FoldAsLoadDefReg);
3757  assert(DefMI);
3758  bool SawStore = false;
3759  if (!DefMI->isSafeToMove(this, 0, SawStore))
3760  return 0;
3761 
3762  // We try to commute MI if possible.
3763  unsigned IdxEnd = (MI->isCommutable()) ? 2 : 1;
3764  for (unsigned Idx = 0; Idx < IdxEnd; Idx++) {
3765  // Collect information about virtual register operands of MI.
3766  unsigned SrcOperandId = 0;
3767  bool FoundSrcOperand = false;
3768  for (unsigned i = 0, e = MI->getDesc().getNumOperands(); i != e; ++i) {
3769  MachineOperand &MO = MI->getOperand(i);
3770  if (!MO.isReg())
3771  continue;
3772  unsigned Reg = MO.getReg();
3773  if (Reg != FoldAsLoadDefReg)
3774  continue;
3775  // Do not fold if we have a subreg use or a def or multiple uses.
3776  if (MO.getSubReg() || MO.isDef() || FoundSrcOperand)
3777  return 0;
3778 
3779  SrcOperandId = i;
3780  FoundSrcOperand = true;
3781  }
3782  if (!FoundSrcOperand) return 0;
3783 
3784  // Check whether we can fold the def into SrcOperandId.
3785  SmallVector<unsigned, 8> Ops;
3786  Ops.push_back(SrcOperandId);
3787  MachineInstr *FoldMI = foldMemoryOperand(MI, Ops, DefMI);
3788  if (FoldMI) {
3789  FoldAsLoadDefReg = 0;
3790  return FoldMI;
3791  }
3792 
3793  if (Idx == 1) {
3794  // MI was changed but it didn't help, commute it back!
3795  commuteInstruction(MI, false);
3796  return 0;
3797  }
3798 
3799  // Check whether we can commute MI and enable folding.
3800  if (MI->isCommutable()) {
3801  MachineInstr *NewMI = commuteInstruction(MI, false);
3802  // Unable to commute.
3803  if (!NewMI) return 0;
3804  if (NewMI != MI) {
3805  // New instruction. It doesn't need to be kept.
3806  NewMI->eraseFromParent();
3807  return 0;
3808  }
3809  }
3810  }
3811  return 0;
3812 }
3813 
3814 /// Expand2AddrUndef - Expand a single-def pseudo instruction to a two-addr
3815 /// instruction with two undef reads of the register being defined. This is
3816 /// used for mapping:
3817 /// %xmm4 = V_SET0
3818 /// to:
3819 /// %xmm4 = PXORrr %xmm4<undef>, %xmm4<undef>
3820 ///
3821 static bool Expand2AddrUndef(MachineInstrBuilder &MIB,
3822  const MCInstrDesc &Desc) {
3823  assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction.");
3824  unsigned Reg = MIB->getOperand(0).getReg();
3825  MIB->setDesc(Desc);
3826 
3827  // MachineInstr::addOperand() will insert explicit operands before any
3828  // implicit operands.
3829  MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef);
3830  // But we don't trust that.
3831  assert(MIB->getOperand(1).getReg() == Reg &&
3832  MIB->getOperand(2).getReg() == Reg && "Misplaced operand");
3833  return true;
3834 }
3835 
3836 bool X86InstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
3837  bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX();
3838  MachineInstrBuilder MIB(*MI->getParent()->getParent(), MI);
3839  switch (MI->getOpcode()) {
3840  case X86::SETB_C8r:
3841  return Expand2AddrUndef(MIB, get(X86::SBB8rr));
3842  case X86::SETB_C16r:
3843  return Expand2AddrUndef(MIB, get(X86::SBB16rr));
3844  case X86::SETB_C32r:
3845  return Expand2AddrUndef(MIB, get(X86::SBB32rr));
3846  case X86::SETB_C64r:
3847  return Expand2AddrUndef(MIB, get(X86::SBB64rr));
3848  case X86::V_SET0:
3849  case X86::FsFLD0SS:
3850  case X86::FsFLD0SD:
3851  return Expand2AddrUndef(MIB, get(HasAVX ? X86::VXORPSrr : X86::XORPSrr));
3852  case X86::AVX_SET0:
3853  assert(HasAVX && "AVX not supported");
3854  return Expand2AddrUndef(MIB, get(X86::VXORPSYrr));
3855  case X86::AVX512_512_SET0:
3856  return Expand2AddrUndef(MIB, get(X86::VPXORDZrr));
3857  case X86::V_SETALLONES:
3858  return Expand2AddrUndef(MIB, get(HasAVX ? X86::VPCMPEQDrr : X86::PCMPEQDrr));
3859  case X86::AVX2_SETALLONES:
3860  return Expand2AddrUndef(MIB, get(X86::VPCMPEQDYrr));
3861  case X86::TEST8ri_NOREX:
3862  MI->setDesc(get(X86::TEST8ri));
3863  return true;
3864  case X86::KSET0W: return Expand2AddrUndef(MIB, get(X86::KXORWrr));
3865  case X86::KSET1B:
3866  case X86::KSET1W: return Expand2AddrUndef(MIB, get(X86::KXNORWrr));
3867  }
3868  return false;
3869 }
3870 
3871 static MachineInstr *FuseTwoAddrInst(MachineFunction &MF, unsigned Opcode,
3872  const SmallVectorImpl<MachineOperand> &MOs,
3873  MachineInstr *MI,
3874  const TargetInstrInfo &TII) {
3875  // Create the base instruction with the memory operand as the first part.
3876  // Omit the implicit operands, something BuildMI can't do.
3877  MachineInstr *NewMI = MF.CreateMachineInstr(TII.get(Opcode),
3878  MI->getDebugLoc(), true);
3879  MachineInstrBuilder MIB(MF, NewMI);
3880  unsigned NumAddrOps = MOs.size();
3881  for (unsigned i = 0; i != NumAddrOps; ++i)
3882  MIB.addOperand(MOs[i]);
3883  if (NumAddrOps < 4) // FrameIndex only
3884  addOffset(MIB, 0);
3885 
3886  // Loop over the rest of the ri operands, converting them over.
3887  unsigned NumOps = MI->getDesc().getNumOperands()-2;
3888  for (unsigned i = 0; i != NumOps; ++i) {
3889  MachineOperand &MO = MI->getOperand(i+2);
3890  MIB.addOperand(MO);
3891  }
3892  for (unsigned i = NumOps+2, e = MI->getNumOperands(); i != e; ++i) {
3893  MachineOperand &MO = MI->getOperand(i);
3894  MIB.addOperand(MO);
3895  }
3896  return MIB;
3897 }
3898 
3899 static MachineInstr *FuseInst(MachineFunction &MF,
3900  unsigned Opcode, unsigned OpNo,
3901  const SmallVectorImpl<MachineOperand> &MOs,
3902  MachineInstr *MI, const TargetInstrInfo &TII) {
3903  // Omit the implicit operands, something BuildMI can't do.
3904  MachineInstr *NewMI = MF.CreateMachineInstr(TII.get(Opcode),
3905  MI->getDebugLoc(), true);
3906  MachineInstrBuilder MIB(MF, NewMI);
3907 
3908  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
3909  MachineOperand &MO = MI->getOperand(i);
3910  if (i == OpNo) {
3911  assert(MO.isReg() && "Expected to fold into reg operand!");
3912  unsigned NumAddrOps = MOs.size();
3913  for (unsigned i = 0; i != NumAddrOps; ++i)
3914  MIB.addOperand(MOs[i]);
3915  if (NumAddrOps < 4) // FrameIndex only
3916  addOffset(MIB, 0);
3917  } else {
3918  MIB.addOperand(MO);
3919  }
3920  }
3921  return MIB;
3922 }
3923 
3924 static MachineInstr *MakeM0Inst(const TargetInstrInfo &TII, unsigned Opcode,
3925  const SmallVectorImpl<MachineOperand> &MOs,
3926  MachineInstr *MI) {
3927  MachineFunction &MF = *MI->getParent()->getParent();
3928  MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), TII.get(Opcode));
3929 
3930  unsigned NumAddrOps = MOs.size();
3931  for (unsigned i = 0; i != NumAddrOps; ++i)
3932  MIB.addOperand(MOs[i]);
3933  if (NumAddrOps < 4) // FrameIndex only
3934  addOffset(MIB, 0);
3935  return MIB.addImm(0);
3936 }
3937 
3938 MachineInstr*
3939 X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
3940  MachineInstr *MI, unsigned i,
3941  const SmallVectorImpl<MachineOperand> &MOs,
3942  unsigned Size, unsigned Align) const {
3943  const DenseMap<unsigned, std::pair<unsigned,unsigned> > *OpcodeTablePtr = 0;
3944  bool isCallRegIndirect = TM.getSubtarget<X86Subtarget>().callRegIndirect();
3945  bool isTwoAddrFold = false;
3946 
3947  // Atom favors register form of call. So, we do not fold loads into calls
3948  // when X86Subtarget is Atom.
3949  if (isCallRegIndirect &&
3950  (MI->getOpcode() == X86::CALL32r || MI->getOpcode() == X86::CALL64r)) {
3951  return NULL;
3952  }
3953 
3954  unsigned NumOps = MI->getDesc().getNumOperands();
3955  bool isTwoAddr = NumOps > 1 &&
3956  MI->getDesc().getOperandConstraint(1, MCOI::TIED_TO) != -1;
3957 
3958  // FIXME: AsmPrinter doesn't know how to handle
3959  // X86II::MO_GOT_ABSOLUTE_ADDRESS after folding.
3960  if (MI->getOpcode() == X86::ADD32ri &&
3961  MI->getOperand(2).getTargetFlags() == X86II::MO_GOT_ABSOLUTE_ADDRESS)
3962  return NULL;
3963 
3964  MachineInstr *NewMI = NULL;
3965  // Folding a memory location into the two-address part of a two-address
3966  // instruction is different than folding it other places. It requires
3967  // replacing the *two* registers with the memory location.
3968  if (isTwoAddr && NumOps >= 2 && i < 2 &&
3969  MI->getOperand(0).isReg() &&
3970  MI->getOperand(1).isReg() &&
3971  MI->getOperand(0).getReg() == MI->getOperand(1).getReg()) {
3972  OpcodeTablePtr = &RegOp2MemOpTable2Addr;
3973  isTwoAddrFold = true;
3974  } else if (i == 0) { // If operand 0
3975  if (MI->getOpcode() == X86::MOV32r0) {
3976  NewMI = MakeM0Inst(*this, X86::MOV32mi, MOs, MI);
3977  if (NewMI)
3978  return NewMI;
3979  }
3980 
3981  OpcodeTablePtr = &RegOp2MemOpTable0;
3982  } else if (i == 1) {
3983  OpcodeTablePtr = &RegOp2MemOpTable1;
3984  } else if (i == 2) {
3985  OpcodeTablePtr = &RegOp2MemOpTable2;
3986  } else if (i == 3) {
3987  OpcodeTablePtr = &RegOp2MemOpTable3;
3988  }
3989 
3990  // If table selected...
3991  if (OpcodeTablePtr) {
3992  // Find the Opcode to fuse
3993  DenseMap<unsigned, std::pair<unsigned,unsigned> >::const_iterator I =
3994  OpcodeTablePtr->find(MI->getOpcode());
3995  if (I != OpcodeTablePtr->end()) {
3996  unsigned Opcode = I->second.first;
3997  unsigned MinAlign = (I->second.second & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT;
3998  if (Align < MinAlign)
3999  return NULL;
4000  bool NarrowToMOV32rm = false;
4001  if (Size) {
4002  unsigned RCSize = getRegClass(MI->getDesc(), i, &RI, MF)->getSize();
4003  if (Size < RCSize) {
4004  // Check if it's safe to fold the load. If the size of the object is
4005  // narrower than the load width, then it's not.
4006  if (Opcode != X86::MOV64rm || RCSize != 8 || Size != 4)
4007  return NULL;
4008  // If this is a 64-bit load, but the spill slot is 32, then we can do
4009  // a 32-bit load which is implicitly zero-extended. This likely is due
4010  // to liveintervalanalysis remat'ing a load from stack slot.
4011  if (MI->getOperand(0).getSubReg() || MI->getOperand(1).getSubReg())
4012  return NULL;
4013  Opcode = X86::MOV32rm;
4014  NarrowToMOV32rm = true;
4015  }
4016  }
4017 
4018  if (isTwoAddrFold)
4019  NewMI = FuseTwoAddrInst(MF, Opcode, MOs, MI, *this);
4020  else
4021  NewMI = FuseInst(MF, Opcode, i, MOs, MI, *this);
4022 
4023  if (NarrowToMOV32rm) {
4024  // If this is the special case where we use a MOV32rm to load a 32-bit
4025  // value and zero-extend the top bits. Change the destination register
4026  // to a 32-bit one.
4027  unsigned DstReg = NewMI->getOperand(0).getReg();
4028  if (TargetRegisterInfo::isPhysicalRegister(DstReg))
4029  NewMI->getOperand(0).setReg(RI.getSubReg(DstReg,
4030  X86::sub_32bit));
4031  else
4032  NewMI->getOperand(0).setSubReg(X86::sub_32bit);
4033  }
4034  return NewMI;
4035  }
4036  }
4037 
4038  // No fusion
4039  if (PrintFailedFusing && !MI->isCopy())
4040  dbgs() << "We failed to fuse operand " << i << " in " << *MI;
4041  return NULL;
4042 }
4043 
4044 /// hasPartialRegUpdate - Return true for all instructions that only update
4045 /// the first 32 or 64-bits of the destination register and leave the rest
4046 /// unmodified. This can be used to avoid folding loads if the instructions
4047 /// only update part of the destination register, and the non-updated part is
4048 /// not needed. e.g. cvtss2sd, sqrtss. Unfolding the load from these
4049 /// instructions breaks the partial register dependency and it can improve
4050 /// performance. e.g.:
4051 ///
4052 /// movss (%rdi), %xmm0
4053 /// cvtss2sd %xmm0, %xmm0
4054 ///
4055 /// Instead of
4056 /// cvtss2sd (%rdi), %xmm0
4057 ///
4058 /// FIXME: This should be turned into a TSFlags.
4059 ///
4060 static bool hasPartialRegUpdate(unsigned Opcode) {
4061  switch (Opcode) {
4062  case X86::CVTSI2SSrr:
4063  case X86::CVTSI2SS64rr:
4064  case X86::CVTSI2SDrr:
4065  case X86::CVTSI2SD64rr:
4066  case X86::CVTSD2SSrr:
4067  case X86::Int_CVTSD2SSrr:
4068  case X86::CVTSS2SDrr:
4069  case X86::Int_CVTSS2SDrr:
4070  case X86::RCPSSr:
4071  case X86::RCPSSr_Int:
4072  case X86::ROUNDSDr:
4073  case X86::ROUNDSDr_Int:
4074  case X86::ROUNDSSr:
4075  case X86::ROUNDSSr_Int:
4076  case X86::RSQRTSSr:
4077  case X86::RSQRTSSr_Int:
4078  case X86::SQRTSSr:
4079  case X86::SQRTSSr_Int:
4080  return true;
4081  }
4082 
4083  return false;
4084 }
4085 
4086 /// getPartialRegUpdateClearance - Inform the ExeDepsFix pass how many idle
4087 /// instructions we would like before a partial register update.
4088 unsigned X86InstrInfo::
4089 getPartialRegUpdateClearance(const MachineInstr *MI, unsigned OpNum,
4090  const TargetRegisterInfo *TRI) const {
4091  if (OpNum != 0 || !hasPartialRegUpdate(MI->getOpcode()))
4092  return 0;
4093 
4094  // If MI is marked as reading Reg, the partial register update is wanted.
4095  const MachineOperand &MO = MI->getOperand(0);
4096  unsigned Reg = MO.getReg();
4097  if (TargetRegisterInfo::isVirtualRegister(Reg)) {
4098  if (MO.readsReg() || MI->readsVirtualRegister(Reg))
4099  return 0;
4100  } else {
4101  if (MI->readsRegister(Reg, TRI))
4102  return 0;
4103  }
4104 
4105  // If any of the preceding 16 instructions are reading Reg, insert a
4106  // dependency breaking instruction. The magic number is based on a few
4107  // Nehalem experiments.
4108  return 16;
4109 }
4110 
4111 // Return true for any instruction the copies the high bits of the first source
4112 // operand into the unused high bits of the destination operand.
4113 static bool hasUndefRegUpdate(unsigned Opcode) {
4114  switch (Opcode) {
4115  case X86::VCVTSI2SSrr:
4116  case X86::Int_VCVTSI2SSrr:
4117  case X86::VCVTSI2SS64rr:
4118  case X86::Int_VCVTSI2SS64rr:
4119  case X86::VCVTSI2SDrr:
4120  case X86::Int_VCVTSI2SDrr:
4121  case X86::VCVTSI2SD64rr:
4122  case X86::Int_VCVTSI2SD64rr:
4123  case X86::VCVTSD2SSrr:
4124  case X86::Int_VCVTSD2SSrr:
4125  case X86::VCVTSS2SDrr:
4126  case X86::Int_VCVTSS2SDrr:
4127  case X86::VRCPSSr:
4128  case X86::VROUNDSDr:
4129  case X86::VROUNDSDr_Int:
4130  case X86::VROUNDSSr:
4131  case X86::VROUNDSSr_Int:
4132  case X86::VRSQRTSSr:
4133  case X86::VSQRTSSr:
4134 
4135  // AVX-512
4136  case X86::VCVTSD2SSZrr:
4137  case X86::VCVTSS2SDZrr:
4138  return true;
4139  }
4140 
4141  return false;
4142 }
4143 
4144 /// Inform the ExeDepsFix pass how many idle instructions we would like before
4145 /// certain undef register reads.
4146 ///
4147 /// This catches the VCVTSI2SD family of instructions:
4148 ///
4149 /// vcvtsi2sdq %rax, %xmm0<undef>, %xmm14
4150 ///
4151 /// We should to be careful *not* to catch VXOR idioms which are presumably
4152 /// handled specially in the pipeline:
4153 ///
4154 /// vxorps %xmm1<undef>, %xmm1<undef>, %xmm1
4155 ///
4156 /// Like getPartialRegUpdateClearance, this makes a strong assumption that the
4157 /// high bits that are passed-through are not live.
4158 unsigned X86InstrInfo::
4159 getUndefRegClearance(const MachineInstr *MI, unsigned &OpNum,
4160  const TargetRegisterInfo *TRI) const {
4161  if (!hasUndefRegUpdate(MI->getOpcode()))
4162  return 0;
4163 
4164  // Set the OpNum parameter to the first source operand.
4165  OpNum = 1;
4166 
4167  const MachineOperand &MO = MI->getOperand(OpNum);
4168  if (MO.isUndef() && TargetRegisterInfo::isPhysicalRegister(MO.getReg())) {
4169  // Use the same magic number as getPartialRegUpdateClearance.
4170  return 16;
4171  }
4172  return 0;
4173 }
4174 
4175 void X86InstrInfo::
4176 breakPartialRegDependency(MachineBasicBlock::iterator MI, unsigned OpNum,
4177  const TargetRegisterInfo *TRI) const {
4178  unsigned Reg = MI->getOperand(OpNum).getReg();
4179  // If MI kills this register, the false dependence is already broken.
4180  if (MI->killsRegister(Reg, TRI))
4181  return;
4182  if (X86::VR128RegClass.contains(Reg)) {
4183  // These instructions are all floating point domain, so xorps is the best
4184  // choice.
4185  bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX();
4186  unsigned Opc = HasAVX ? X86::VXORPSrr : X86::XORPSrr;
4187  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), get(Opc), Reg)
4188  .addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef);
4189  } else if (X86::VR256RegClass.contains(Reg)) {
4190  // Use vxorps to clear the full ymm register.
4191  // It wants to read and write the xmm sub-register.
4192  unsigned XReg = TRI->getSubReg(Reg, X86::sub_xmm);
4193  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), get(X86::VXORPSrr), XReg)
4194  .addReg(XReg, RegState::Undef).addReg(XReg, RegState::Undef)
4195  .addReg(Reg, RegState::ImplicitDefine);
4196  } else
4197  return;
4198  MI->addRegisterKilled(Reg, TRI, true);
4199 }
4200 
4201 static MachineInstr* foldPatchpoint(MachineFunction &MF,
4202  MachineInstr *MI,
4203  const SmallVectorImpl<unsigned> &Ops,
4204  int FrameIndex,
4205  const TargetInstrInfo &TII) {
4206  unsigned StartIdx = 0;
4207  switch (MI->getOpcode()) {
4208  case TargetOpcode::STACKMAP:
4209  StartIdx = 2; // Skip ID, nShadowBytes.
4210  break;
4211  case TargetOpcode::PATCHPOINT: {
4212  // For PatchPoint, the call args are not foldable.
4213  PatchPointOpers opers(MI);
4214  StartIdx = opers.getVarIdx();
4215  break;
4216  }
4217  default:
4218  llvm_unreachable("unexpected stackmap opcode");
4219  }
4220 
4221  // Return false if any operands requested for folding are not foldable (not
4222  // part of the stackmap's live values).
4223  for (SmallVectorImpl<unsigned>::const_iterator I = Ops.begin(), E = Ops.end();
4224  I != E; ++I) {
4225  if (*I < StartIdx)
4226  return 0;
4227  }
4228 
4229  MachineInstr *NewMI =
4230  MF.CreateMachineInstr(TII.get(MI->getOpcode()), MI->getDebugLoc(), true);
4231  MachineInstrBuilder MIB(MF, NewMI);
4232 
4233  // No need to fold return, the meta data, and function arguments
4234  for (unsigned i = 0; i < StartIdx; ++i)
4235  MIB.addOperand(MI->getOperand(i));
4236 
4237  for (unsigned i = StartIdx; i < MI->getNumOperands(); ++i) {
4238  MachineOperand &MO = MI->getOperand(i);
4239  if (std::find(Ops.begin(), Ops.end(), i) != Ops.end()) {
4240  assert(MO.getReg() && "patchpoint can only fold a vreg operand");
4241  // Compute the spill slot size and offset.
4242  const TargetRegisterClass *RC = MF.getRegInfo().getRegClass(MO.getReg());
4243  unsigned SpillSize;
4244  unsigned SpillOffset;
4245  bool Valid = TII.getStackSlotRange(RC, MO.getSubReg(), SpillSize,
4246  SpillOffset, &MF.getTarget());
4247  if (!Valid)
4248  report_fatal_error("cannot spill patchpoint subregister operand");
4249 
4250  MIB.addOperand(MachineOperand::CreateImm(StackMaps::IndirectMemRefOp));
4251  MIB.addOperand(MachineOperand::CreateImm(SpillSize));
4252  MIB.addOperand(MachineOperand::CreateFI(FrameIndex));
4253  addOffset(MIB, SpillOffset);
4254  }
4255  else
4256  MIB.addOperand(MO);
4257  }
4258  return NewMI;
4259 }
4260 
4261 MachineInstr*
4262 X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
4263  const SmallVectorImpl<unsigned> &Ops,
4264  int FrameIndex) const {
4265  // Special case stack map and patch point intrinsics.
4266  if (MI->getOpcode() == TargetOpcode::STACKMAP
4267  || MI->getOpcode() == TargetOpcode::PATCHPOINT) {
4268  return foldPatchpoint(MF, MI, Ops, FrameIndex, *this);
4269  }
4270  // Check switch flag
4271  if (NoFusing) return NULL;
4272 
4273  // Unless optimizing for size, don't fold to avoid partial
4274  // register update stalls
4275  if (!MF.getFunction()->getAttributes().
4276  hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize) &&
4277  hasPartialRegUpdate(MI->getOpcode()))
4278  return 0;
4279 
4280  const MachineFrameInfo *MFI = MF.getFrameInfo();
4281  unsigned Size = MFI->getObjectSize(FrameIndex);
4282  unsigned Alignment = MFI->getObjectAlignment(FrameIndex);
4283  // If the function stack isn't realigned we don't want to fold instructions
4284  // that need increased alignment.
4285  if (!RI.needsStackRealignment(MF))
4286  Alignment = std::min(Alignment, TM.getFrameLowering()->getStackAlignment());
4287  if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) {
4288  unsigned NewOpc = 0;
4289  unsigned RCSize = 0;
4290  switch (MI->getOpcode()) {
4291  default: return NULL;
4292  case X86::TEST8rr: NewOpc = X86::CMP8ri; RCSize = 1; break;
4293  case X86::TEST16rr: NewOpc = X86::CMP16ri8; RCSize = 2; break;
4294  case X86::TEST32rr: NewOpc = X86::CMP32ri8; RCSize = 4; break;
4295  case X86::TEST64rr: NewOpc = X86::CMP64ri8; RCSize = 8; break;
4296  }
4297  // Check if it's safe to fold the load. If the size of the object is
4298  // narrower than the load width, then it's not.
4299  if (Size < RCSize)
4300  return NULL;
4301  // Change to CMPXXri r, 0 first.
4302  MI->setDesc(get(NewOpc));
4303  MI->getOperand(1).ChangeToImmediate(0);
4304  } else if (Ops.size() != 1)
4305  return NULL;
4306 
4307  SmallVector<MachineOperand,4> MOs;
4308  MOs.push_back(MachineOperand::CreateFI(FrameIndex));
4309  return foldMemoryOperandImpl(MF, MI, Ops[0], MOs, Size, Alignment);
4310 }
4311 
4312 MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
4313  MachineInstr *MI,
4314  const SmallVectorImpl<unsigned> &Ops,
4315  MachineInstr *LoadMI) const {
4316  // If loading from a FrameIndex, fold directly from the FrameIndex.
4317  unsigned NumOps = LoadMI->getDesc().getNumOperands();
4318  int FrameIndex;
4319  if (isLoadFromStackSlot(LoadMI, FrameIndex))
4320  return foldMemoryOperandImpl(MF, MI, Ops, FrameIndex);
4321 
4322  // Check switch flag
4323  if (NoFusing) return NULL;
4324 
4325  // Unless optimizing for size, don't fold to avoid partial
4326  // register update stalls
4327  if (!MF.getFunction()->getAttributes().
4328  hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize) &&
4329  hasPartialRegUpdate(MI->getOpcode()))
4330  return 0;
4331 
4332  // Determine the alignment of the load.
4333  unsigned Alignment = 0;
4334  if (LoadMI->hasOneMemOperand())
4335  Alignment = (*LoadMI->memoperands_begin())->getAlignment();
4336  else
4337  switch (LoadMI->getOpcode()) {
4338  case X86::AVX2_SETALLONES:
4339  case X86::AVX_SET0:
4340  Alignment = 32;
4341  break;
4342  case X86::V_SET0:
4343  case X86::V_SETALLONES:
4344  Alignment = 16;
4345  break;
4346  case X86::FsFLD0SD:
4347  Alignment = 8;
4348  break;
4349  case X86::FsFLD0SS:
4350  Alignment = 4;
4351  break;
4352  default:
4353  return 0;
4354  }
4355  if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) {
4356  unsigned NewOpc = 0;
4357  switch (MI->getOpcode()) {
4358  default: return NULL;
4359  case X86::TEST8rr: NewOpc = X86::CMP8ri; break;
4360  case X86::TEST16rr: NewOpc = X86::CMP16ri8; break;
4361  case X86::TEST32rr: NewOpc = X86::CMP32ri8; break;
4362  case X86::TEST64rr: NewOpc = X86::CMP64ri8; break;
4363  }
4364  // Change to CMPXXri r, 0 first.
4365  MI->setDesc(get(NewOpc));
4366  MI->getOperand(1).ChangeToImmediate(0);
4367  } else if (Ops.size() != 1)
4368  return NULL;
4369 
4370  // Make sure the subregisters match.
4371  // Otherwise we risk changing the size of the load.
4372  if (LoadMI->getOperand(0).getSubReg() != MI->getOperand(Ops[0]).getSubReg())
4373  return NULL;
4374 
4375  SmallVector<MachineOperand,X86::AddrNumOperands> MOs;
4376  switch (LoadMI->getOpcode()) {
4377  case X86::V_SET0:
4378  case X86::V_SETALLONES:
4379  case X86::AVX2_SETALLONES:
4380  case X86::AVX_SET0:
4381  case X86::FsFLD0SD:
4382  case X86::FsFLD0SS: {
4383  // Folding a V_SET0 or V_SETALLONES as a load, to ease register pressure.
4384  // Create a constant-pool entry and operands to load from it.
4385 
4386  // Medium and large mode can't fold loads this way.
4387  if (TM.getCodeModel() != CodeModel::Small &&
4388  TM.getCodeModel() != CodeModel::Kernel)
4389  return NULL;
4390 
4391  // x86-32 PIC requires a PIC base register for constant pools.
4392  unsigned PICBase = 0;
4393  if (TM.getRelocationModel() == Reloc::PIC_) {
4394  if (TM.getSubtarget<X86Subtarget>().is64Bit())
4395  PICBase = X86::RIP;
4396  else
4397  // FIXME: PICBase = getGlobalBaseReg(&MF);
4398  // This doesn't work for several reasons.
4399  // 1. GlobalBaseReg may have been spilled.
4400  // 2. It may not be live at MI.
4401  return NULL;
4402  }
4403 
4404  // Create a constant-pool entry.
4405  MachineConstantPool &MCP = *MF.getConstantPool();
4406  Type *Ty;
4407  unsigned Opc = LoadMI->getOpcode();
4408  if (Opc == X86::FsFLD0SS)
4409  Ty = Type::getFloatTy(MF.getFunction()->getContext());
4410  else if (Opc == X86::FsFLD0SD)
4411  Ty = Type::getDoubleTy(MF.getFunction()->getContext());
4412  else if (Opc == X86::AVX2_SETALLONES || Opc == X86::AVX_SET0)
4413  Ty = VectorType::get(Type::getInt32Ty(MF.getFunction()->getContext()), 8);
4414  else
4415  Ty = VectorType::get(Type::getInt32Ty(MF.getFunction()->getContext()), 4);
4416 
4417  bool IsAllOnes = (Opc == X86::V_SETALLONES || Opc == X86::AVX2_SETALLONES);
4418  const Constant *C = IsAllOnes ? Constant::getAllOnesValue(Ty) :
4419  Constant::getNullValue(Ty);
4420  unsigned CPI = MCP.getConstantPoolIndex(C, Alignment);
4421 
4422  // Create operands to load from the constant pool entry.
4423  MOs.push_back(MachineOperand::CreateReg(PICBase, false));
4424  MOs.push_back(MachineOperand::CreateImm(1));
4425  MOs.push_back(MachineOperand::CreateReg(0, false));
4426  MOs.push_back(MachineOperand::CreateCPI(CPI, 0));
4427  MOs.push_back(MachineOperand::CreateReg(0, false));
4428  break;
4429  }
4430  default: {
4431  if ((LoadMI->getOpcode() == X86::MOVSSrm ||
4432  LoadMI->getOpcode() == X86::VMOVSSrm) &&
4433  MF.getRegInfo().getRegClass(LoadMI->getOperand(0).getReg())->getSize()
4434  > 4)
4435  // These instructions only load 32 bits, we can't fold them if the
4436  // destination register is wider than 32 bits (4 bytes).
4437  return NULL;
4438  if ((LoadMI->getOpcode() == X86::MOVSDrm ||
4439  LoadMI->getOpcode() == X86::VMOVSDrm) &&
4440  MF.getRegInfo().getRegClass(LoadMI->getOperand(0).getReg())->getSize()
4441  > 8)
4442  // These instructions only load 64 bits, we can't fold them if the
4443  // destination register is wider than 64 bits (8 bytes).
4444  return NULL;
4445 
4446  // Folding a normal load. Just copy the load's address operands.
4447  for (unsigned i = NumOps - X86::AddrNumOperands; i != NumOps; ++i)
4448  MOs.push_back(LoadMI->getOperand(i));
4449  break;
4450  }
4451  }
4452  return foldMemoryOperandImpl(MF, MI, Ops[0], MOs, 0, Alignment);
4453 }
4454 
4455 
4456 bool X86InstrInfo::canFoldMemoryOperand(const MachineInstr *MI,
4457  const SmallVectorImpl<unsigned> &Ops) const {
4458  // Check switch flag
4459  if (NoFusing) return 0;
4460 
4461  if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) {
4462  switch (MI->getOpcode()) {
4463  default: return false;
4464  case X86::TEST8rr:
4465  case X86::TEST16rr:
4466  case X86::TEST32rr:
4467  case X86::TEST64rr:
4468  return true;
4469  case X86::ADD32ri:
4470  // FIXME: AsmPrinter doesn't know how to handle
4471  // X86II::MO_GOT_ABSOLUTE_ADDRESS after folding.
4472  if (MI->getOperand(2).getTargetFlags() == X86II::MO_GOT_ABSOLUTE_ADDRESS)
4473  return false;
4474  break;
4475  }
4476  }
4477 
4478  if (Ops.size() != 1)
4479  return false;
4480 
4481  unsigned OpNum = Ops[0];
4482  unsigned Opc = MI->getOpcode();
4483  unsigned NumOps = MI->getDesc().getNumOperands();
4484  bool isTwoAddr = NumOps > 1 &&
4485  MI->getDesc().getOperandConstraint(1, MCOI::TIED_TO) != -1;
4486 
4487  // Folding a memory location into the two-address part of a two-address
4488  // instruction is different than folding it other places. It requires
4489  // replacing the *two* registers with the memory location.
4490  const DenseMap<unsigned, std::pair<unsigned,unsigned> > *OpcodeTablePtr = 0;
4491  if (isTwoAddr && NumOps >= 2 && OpNum < 2) {
4492  OpcodeTablePtr = &RegOp2MemOpTable2Addr;
4493  } else if (OpNum == 0) { // If operand 0
4494  if (Opc == X86::MOV32r0)
4495  return true;
4496 
4497  OpcodeTablePtr = &RegOp2MemOpTable0;
4498  } else if (OpNum == 1) {
4499  OpcodeTablePtr = &RegOp2MemOpTable1;
4500  } else if (OpNum == 2) {
4501  OpcodeTablePtr = &RegOp2MemOpTable2;
4502  } else if (OpNum == 3) {
4503  OpcodeTablePtr = &RegOp2MemOpTable3;
4504  }
4505 
4506  if (OpcodeTablePtr && OpcodeTablePtr->count(Opc))
4507  return true;
4508  return TargetInstrInfo::canFoldMemoryOperand(MI, Ops);
4509 }
4510 
4511 bool X86InstrInfo::unfoldMemoryOperand(MachineFunction &MF, MachineInstr *MI,
4512  unsigned Reg, bool UnfoldLoad, bool UnfoldStore,
4513  SmallVectorImpl<MachineInstr*> &NewMIs) const {
4514  DenseMap<unsigned, std::pair<unsigned,unsigned> >::const_iterator I =
4515  MemOp2RegOpTable.find(MI->getOpcode());
4516  if (I == MemOp2RegOpTable.end())
4517  return false;
4518  unsigned Opc = I->second.first;
4519  unsigned Index = I->second.second & TB_INDEX_MASK;
4520  bool FoldedLoad = I->second.second & TB_FOLDED_LOAD;
4521  bool FoldedStore = I->second.second & TB_FOLDED_STORE;
4522  if (UnfoldLoad && !FoldedLoad)
4523  return false;
4524  UnfoldLoad &= FoldedLoad;
4525  if (UnfoldStore && !FoldedStore)
4526  return false;
4527  UnfoldStore &= FoldedStore;
4528 
4529  const MCInstrDesc &MCID = get(Opc);
4530  const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF);
4531  if (!MI->hasOneMemOperand() &&
4532  RC == &X86::VR128RegClass &&
4533  !TM.getSubtarget<X86Subtarget>().isUnalignedMemAccessFast())
4534  // Without memoperands, loadRegFromAddr and storeRegToStackSlot will
4535  // conservatively assume the address is unaligned. That's bad for
4536  // performance.
4537  return false;
4538  SmallVector<MachineOperand, X86::AddrNumOperands> AddrOps;
4539  SmallVector<MachineOperand,2> BeforeOps;
4540  SmallVector<MachineOperand,2> AfterOps;
4541  SmallVector<MachineOperand,4> ImpOps;
4542  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
4543  MachineOperand &Op = MI->getOperand(i);
4544  if (i >= Index && i < Index + X86::AddrNumOperands)
4545  AddrOps.push_back(Op);
4546  else if (Op.isReg() && Op.isImplicit())
4547  ImpOps.push_back(Op);
4548  else if (i < Index)
4549  BeforeOps.push_back(Op);
4550  else if (i > Index)
4551  AfterOps.push_back(Op);
4552  }
4553 
4554  // Emit the load instruction.
4555  if (UnfoldLoad) {
4556  std::pair<MachineInstr::mmo_iterator,
4557  MachineInstr::mmo_iterator> MMOs =
4558  MF.extractLoadMemRefs(MI->memoperands_begin(),
4559  MI->memoperands_end());
4560  loadRegFromAddr(MF, Reg, AddrOps, RC, MMOs.first, MMOs.second, NewMIs);
4561  if (UnfoldStore) {
4562  // Address operands cannot be marked isKill.
4563  for (unsigned i = 1; i != 1 + X86::AddrNumOperands; ++i) {
4564  MachineOperand &MO = NewMIs[0]->getOperand(i);
4565  if (MO.isReg())
4566  MO.setIsKill(false);
4567  }
4568  }
4569  }
4570 
4571  // Emit the data processing instruction.
4572  MachineInstr *DataMI = MF.CreateMachineInstr(MCID, MI->getDebugLoc(), true);
4573  MachineInstrBuilder MIB(MF, DataMI);
4574 
4575  if (FoldedStore)
4576  MIB.addReg(Reg, RegState::Define);
4577  for (unsigned i = 0, e = BeforeOps.size(); i != e; ++i)
4578  MIB.addOperand(BeforeOps[i]);
4579  if (FoldedLoad)
4580  MIB.addReg(Reg);
4581  for (unsigned i = 0, e = AfterOps.size(); i != e; ++i)
4582  MIB.addOperand(AfterOps[i]);
4583  for (unsigned i = 0, e = ImpOps.size(); i != e; ++i) {
4584  MachineOperand &MO = ImpOps[i];
4585  MIB.addReg(MO.getReg(),
4586  getDefRegState(MO.isDef()) |
4587  RegState::Implicit |
4588  getKillRegState(MO.isKill()) |
4589  getDeadRegState(MO.isDead()) |
4590  getUndefRegState(MO.isUndef()));
4591  }
4592  // Change CMP32ri r, 0 back to TEST32rr r, r, etc.
4593  switch (DataMI->getOpcode()) {
4594  default: break;
4595  case X86::CMP64ri32:
4596  case X86::CMP64ri8:
4597  case X86::CMP32ri:
4598  case X86::CMP32ri8:
4599  case X86::CMP16ri:
4600  case X86::CMP16ri8:
4601  case X86::CMP8ri: {
4602  MachineOperand &MO0 = DataMI->getOperand(0);
4603  MachineOperand &MO1 = DataMI->getOperand(1);
4604  if (MO1.getImm() == 0) {
4605  unsigned NewOpc;
4606  switch (DataMI->getOpcode()) {
4607  default: llvm_unreachable("Unreachable!");
4608  case X86::CMP64ri8:
4609  case X86::CMP64ri32: NewOpc = X86::TEST64rr; break;
4610  case X86::CMP32ri8:
4611  case X86::CMP32ri: NewOpc = X86::TEST32rr; break;
4612  case X86::CMP16ri8:
4613  case X86::CMP16ri: NewOpc = X86::TEST16rr; break;
4614  case X86::CMP8ri: NewOpc = X86::TEST8rr; break;
4615  }
4616  DataMI->setDesc(get(NewOpc));
4617  MO1.ChangeToRegister(MO0.getReg(), false);
4618  }
4619  }
4620  }
4621  NewMIs.push_back(DataMI);
4622 
4623  // Emit the store instruction.
4624  if (UnfoldStore) {
4625  const TargetRegisterClass *DstRC = getRegClass(MCID, 0, &RI, MF);
4626  std::pair<MachineInstr::mmo_iterator,
4627  MachineInstr::mmo_iterator> MMOs =
4628  MF.extractStoreMemRefs(MI->memoperands_begin(),
4629  MI->memoperands_end());
4630  storeRegToAddr(MF, Reg, true, AddrOps, DstRC, MMOs.first, MMOs.second, NewMIs);
4631  }
4632 
4633  return true;
4634 }
4635 
4636 bool
4637 X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N,
4638  SmallVectorImpl<SDNode*> &NewNodes) const {
4639  if (!N->isMachineOpcode())
4640  return false;
4641 
4642  DenseMap<unsigned, std::pair<unsigned,unsigned> >::const_iterator I =
4643  MemOp2RegOpTable.find(N->getMachineOpcode());
4644  if (I == MemOp2RegOpTable.end())
4645  return false;
4646  unsigned Opc = I->second.first;
4647  unsigned Index = I->second.second & TB_INDEX_MASK;
4648  bool FoldedLoad = I->second.second & TB_FOLDED_LOAD;
4649  bool FoldedStore = I->second.second & TB_FOLDED_STORE;
4650  const MCInstrDesc &MCID = get(Opc);
4651  MachineFunction &MF = DAG.getMachineFunction();
4652  const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF);
4653  unsigned NumDefs = MCID.NumDefs;
4654  std::vector<SDValue> AddrOps;
4655  std::vector<SDValue> BeforeOps;
4656  std::vector<SDValue> AfterOps;
4657  SDLoc dl(N);
4658  unsigned NumOps = N->getNumOperands();
4659  for (unsigned i = 0; i != NumOps-1; ++i) {
4660  SDValue Op = N->getOperand(i);
4661  if (i >= Index-NumDefs && i < Index-NumDefs + X86::AddrNumOperands)
4662  AddrOps.push_back(Op);
4663  else if (i < Index-NumDefs)
4664  BeforeOps.push_back(Op);
4665  else if (i > Index-NumDefs)
4666  AfterOps.push_back(Op);
4667  }
4668  SDValue Chain = N->getOperand(NumOps-1);
4669  AddrOps.push_back(Chain);
4670 
4671  // Emit the load instruction.
4672  SDNode *Load = 0;
4673  if (FoldedLoad) {
4674  EVT VT = *RC->vt_begin();
4675  std::pair<MachineInstr::mmo_iterator,
4676  MachineInstr::mmo_iterator> MMOs =
4677  MF.extractLoadMemRefs(cast<MachineSDNode>(N)->memoperands_begin(),
4678  cast<MachineSDNode>(N)->memoperands_end());
4679  if (!(*MMOs.first) &&
4680  RC == &X86::VR128RegClass &&
4681  !TM.getSubtarget<X86Subtarget>().isUnalignedMemAccessFast())
4682  // Do not introduce a slow unaligned load.
4683  return false;
4684  unsigned Alignment = RC->getSize() == 32 ? 32 : 16;
4685  bool isAligned = (*MMOs.first) &&
4686  (*MMOs.first)->getAlignment() >= Alignment;
4687  Load = DAG.getMachineNode(getLoadRegOpcode(0, RC, isAligned, TM), dl,
4688  VT, MVT::Other, AddrOps);
4689  NewNodes.push_back(Load);
4690 
4691  // Preserve memory reference information.
4692  cast<MachineSDNode>(Load)->setMemRefs(MMOs.first, MMOs.second);
4693  }
4694 
4695  // Emit the data processing instruction.
4696  std::vector<EVT> VTs;
4697  const TargetRegisterClass *DstRC = 0;
4698  if (MCID.getNumDefs() > 0) {
4699  DstRC = getRegClass(MCID, 0, &RI, MF);
4700  VTs.push_back(*DstRC->vt_begin());
4701  }
4702  for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
4703  EVT VT = N->getValueType(i);
4704  if (VT != MVT::Other && i >= (unsigned)MCID.getNumDefs())
4705  VTs.push_back(VT);
4706  }
4707  if (Load)
4708  BeforeOps.push_back(SDValue(Load, 0));
4709  std::copy(AfterOps.begin(), AfterOps.end(), std::back_inserter(BeforeOps));
4710  SDNode *NewNode= DAG.getMachineNode(Opc, dl, VTs, BeforeOps);
4711  NewNodes.push_back(NewNode);
4712 
4713  // Emit the store instruction.
4714  if (FoldedStore) {
4715  AddrOps.pop_back();
4716  AddrOps.push_back(SDValue(NewNode, 0));
4717  AddrOps.push_back(Chain);
4718  std::pair<MachineInstr::mmo_iterator,
4719  MachineInstr::mmo_iterator> MMOs =
4720  MF.extractStoreMemRefs(cast<MachineSDNode>(N)->memoperands_begin(),
4721  cast<MachineSDNode>(N)->memoperands_end());
4722  if (!(*MMOs.first) &&
4723  RC == &X86::VR128RegClass &&
4724  !TM.getSubtarget<X86Subtarget>().isUnalignedMemAccessFast())
4725  // Do not introduce a slow unaligned store.
4726  return false;
4727  unsigned Alignment = RC->getSize() == 32 ? 32 : 16;
4728  bool isAligned = (*MMOs.first) &&
4729  (*MMOs.first)->getAlignment() >= Alignment;
4730  SDNode *Store = DAG.getMachineNode(getStoreRegOpcode(0, DstRC,
4731  isAligned, TM),
4732  dl, MVT::Other, AddrOps);
4733  NewNodes.push_back(Store);
4734 
4735  // Preserve memory reference information.
4736  cast<MachineSDNode>(Load)->setMemRefs(MMOs.first, MMOs.second);
4737  }
4738 
4739  return true;
4740 }
4741 
4742 unsigned X86InstrInfo::getOpcodeAfterMemoryUnfold(unsigned Opc,
4743  bool UnfoldLoad, bool UnfoldStore,
4744  unsigned *LoadRegIndex) const {
4745  DenseMap<unsigned, std::pair<unsigned,unsigned> >::const_iterator I =
4746  MemOp2RegOpTable.find(Opc);
4747  if (I == MemOp2RegOpTable.end())
4748  return 0;
4749  bool FoldedLoad = I->second.second & TB_FOLDED_LOAD;
4750  bool FoldedStore = I->second.second & TB_FOLDED_STORE;
4751  if (UnfoldLoad && !FoldedLoad)
4752  return 0;
4753  if (UnfoldStore && !FoldedStore)
4754  return 0;
4755  if (LoadRegIndex)
4756  *LoadRegIndex = I->second.second & TB_INDEX_MASK;
4757  return I->second.first;
4758 }
4759 
4760 bool
4761 X86InstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
4762  int64_t &Offset1, int64_t &Offset2) const {
4763  if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode())
4764  return false;
4765  unsigned Opc1 = Load1->getMachineOpcode();
4766  unsigned Opc2 = Load2->getMachineOpcode();
4767  switch (Opc1) {
4768  default: return false;
4769  case X86::MOV8rm:
4770  case X86::MOV16rm:
4771  case X86::MOV32rm:
4772  case X86::MOV64rm:
4773  case X86::LD_Fp32m:
4774  case X86::LD_Fp64m:
4775  case X86::LD_Fp80m:
4776  case X86::MOVSSrm:
4777  case X86::MOVSDrm:
4778  case X86::MMX_MOVD64rm:
4779  case X86::MMX_MOVQ64rm:
4780  case X86::FsMOVAPSrm:
4781  case X86::FsMOVAPDrm:
4782  case X86::MOVAPSrm:
4783  case X86::MOVUPSrm:
4784  case X86::MOVAPDrm:
4785  case X86::MOVDQArm:
4786  case X86::MOVDQUrm:
4787  // AVX load instructions
4788  case X86::VMOVSSrm:
4789  case X86::VMOVSDrm:
4790  case X86::FsVMOVAPSrm:
4791  case X86::FsVMOVAPDrm:
4792  case X86::VMOVAPSrm:
4793  case X86::VMOVUPSrm:
4794  case X86::VMOVAPDrm:
4795  case X86::VMOVDQArm:
4796  case X86::VMOVDQUrm:
4797  case X86::VMOVAPSYrm:
4798  case X86::VMOVUPSYrm:
4799  case X86::VMOVAPDYrm:
4800  case X86::VMOVDQAYrm:
4801  case X86::VMOVDQUYrm:
4802  break;
4803  }
4804  switch (Opc2) {
4805  default: return false;
4806  case X86::MOV8rm:
4807  case X86::MOV16rm:
4808  case X86::MOV32rm:
4809  case X86::MOV64rm:
4810  case X86::LD_Fp32m:
4811  case X86::LD_Fp64m:
4812  case X86::LD_Fp80m:
4813  case X86::MOVSSrm:
4814  case X86::MOVSDrm:
4815  case X86::MMX_MOVD64rm:
4816  case X86::MMX_MOVQ64rm:
4817  case X86::FsMOVAPSrm:
4818  case X86::FsMOVAPDrm:
4819  case X86::MOVAPSrm:
4820  case X86::MOVUPSrm:
4821  case X86::MOVAPDrm:
4822  case X86::MOVDQArm:
4823  case X86::MOVDQUrm:
4824  // AVX load instructions
4825  case X86::VMOVSSrm:
4826  case X86::VMOVSDrm:
4827  case X86::FsVMOVAPSrm:
4828  case X86::FsVMOVAPDrm:
4829  case X86::VMOVAPSrm:
4830  case X86::VMOVUPSrm:
4831  case X86::VMOVAPDrm:
4832  case X86::VMOVDQArm:
4833  case X86::VMOVDQUrm:
4834  case X86::VMOVAPSYrm:
4835  case X86::VMOVUPSYrm:
4836  case X86::VMOVAPDYrm:
4837  case X86::VMOVDQAYrm:
4838  case X86::VMOVDQUYrm:
4839  break;
4840  }
4841 
4842  // Check if chain operands and base addresses match.
4843  if (Load1->getOperand(0) != Load2->getOperand(0) ||
4844  Load1->getOperand(5) != Load2->getOperand(5))
4845  return false;
4846  // Segment operands should match as well.
4847  if (Load1->getOperand(4) != Load2->getOperand(4))
4848  return false;
4849  // Scale should be 1, Index should be Reg0.
4850  if (Load1->getOperand(1) == Load2->getOperand(1) &&
4851  Load1->getOperand(2) == Load2->getOperand(2)) {
4852  if (cast<ConstantSDNode>(Load1->getOperand(1))->getZExtValue() != 1)
4853  return false;
4854 
4855  // Now let's examine the displacements.
4856  if (isa<ConstantSDNode>(Load1->getOperand(3)) &&
4857  isa<ConstantSDNode>(Load2->getOperand(3))) {
4858  Offset1 = cast<ConstantSDNode>(Load1->getOperand(3))->getSExtValue();
4859  Offset2 = cast<ConstantSDNode>(Load2->getOperand(3))->getSExtValue();
4860  return true;
4861  }
4862  }
4863  return false;
4864 }
4865 
4866 bool X86InstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2,
4867  int64_t Offset1, int64_t Offset2,
4868  unsigned NumLoads) const {
4869  assert(Offset2 > Offset1);
4870  if ((Offset2 - Offset1) / 8 > 64)
4871  return false;
4872 
4873  unsigned Opc1 = Load1->getMachineOpcode();
4874  unsigned Opc2 = Load2->getMachineOpcode();
4875  if (Opc1 != Opc2)
4876  return false; // FIXME: overly conservative?
4877 
4878  switch (Opc1) {
4879  default: break;
4880  case X86::LD_Fp32m:
4881  case X86::LD_Fp64m:
4882  case X86::LD_Fp80m:
4883  case X86::MMX_MOVD64rm:
4884  case X86::MMX_MOVQ64rm:
4885  return false;
4886  }
4887 
4888  EVT VT = Load1->getValueType(0);
4889  switch (VT.getSimpleVT().SimpleTy) {
4890  default:
4891  // XMM registers. In 64-bit mode we can be a bit more aggressive since we
4892  // have 16 of them to play with.
4893  if (TM.getSubtargetImpl()->is64Bit()) {
4894  if (NumLoads >= 3)
4895  return false;
4896  } else if (NumLoads) {
4897  return false;
4898  }
4899  break;
4900  case MVT::i8:
4901  case MVT::i16:
4902  case MVT::i32:
4903  case MVT::i64:
4904  case MVT::f32:
4905  case MVT::f64:
4906  if (NumLoads)
4907  return false;
4908  break;
4909  }
4910 
4911  return true;
4912 }
4913 
4914 bool X86InstrInfo::shouldScheduleAdjacent(MachineInstr* First,
4915  MachineInstr *Second) const {
4916  // Check if this processor supports macro-fusion. Since this is a minor
4917  // heuristic, we haven't specifically reserved a feature. hasAVX is a decent
4918  // proxy for SandyBridge+.
4919  if (!TM.getSubtarget<X86Subtarget>().hasAVX())
4920  return false;
4921 
4922  enum {
4923  FuseTest,
4924  FuseCmp,
4925  FuseInc
4926  } FuseKind;
4927 
4928  switch(Second->getOpcode()) {
4929  default:
4930  return false;
4931  case X86::JE_4:
4932  case X86::JNE_4:
4933  case X86::JL_4:
4934  case X86::JLE_4:
4935  case X86::JG_4:
4936  case X86::JGE_4:
4937  FuseKind = FuseInc;
4938  break;
4939  case X86::JB_4:
4940  case X86::JBE_4:
4941  case X86::JA_4:
4942  case X86::JAE_4:
4943  FuseKind = FuseCmp;
4944  break;
4945  case X86::JS_4:
4946  case X86::JNS_4:
4947  case X86::JP_4:
4948  case X86::JNP_4:
4949  case X86::JO_4:
4950  case X86::JNO_4:
4951  FuseKind = FuseTest;
4952  break;
4953  }
4954  switch (First->getOpcode()) {
4955  default:
4956  return false;
4957  case X86::TEST8rr:
4958  case X86::TEST16rr:
4959  case X86::TEST32rr:
4960  case X86::TEST64rr:
4961  case X86::TEST8ri:
4962  case X86::TEST16ri:
4963  case X86::TEST32ri:
4964  case X86::TEST32i32:
4965  case X86::TEST64i32:
4966  case X86::TEST64ri32:
4967  case X86::TEST8rm:
4968  case X86::TEST16rm:
4969  case X86::TEST32rm:
4970  case X86::TEST64rm:
4971  case X86::AND16i16:
4972  case X86::AND16ri:
4973  case X86::AND16ri8:
4974  case X86::AND16rm:
4975  case X86::AND16rr:
4976  case X86::AND32i32:
4977  case X86::AND32ri:
4978  case X86::AND32ri8:
4979  case X86::AND32rm:
4980  case X86::AND32rr:
4981  case X86::AND64i32:
4982  case X86::AND64ri32:
4983  case X86::AND64ri8:
4984  case X86::AND64rm:
4985  case X86::AND64rr:
4986  case X86::AND8i8:
4987  case X86::AND8ri:
4988  case X86::AND8rm:
4989  case X86::AND8rr:
4990  return true;
4991  case X86::CMP16i16:
4992  case X86::CMP16ri:
4993  case X86::CMP16ri8:
4994  case X86::CMP16rm:
4995  case X86::CMP16rr:
4996  case X86::CMP32i32:
4997  case X86::CMP32ri:
4998  case X86::CMP32ri8:
4999  case X86::CMP32rm:
5000  case X86::CMP32rr:
5001  case X86::CMP64i32:
5002  case X86::CMP64ri32:
5003  case X86::CMP64ri8:
5004  case X86::CMP64rm:
5005  case X86::CMP64rr:
5006  case X86::CMP8i8:
5007  case X86::CMP8ri:
5008  case X86::CMP8rm:
5009  case X86::CMP8rr:
5010  case X86::ADD16i16:
5011  case X86::ADD16ri:
5012  case X86::ADD16ri8:
5013  case X86::ADD16ri8_DB:
5014  case X86::ADD16ri_DB:
5015  case X86::ADD16rm:
5016  case X86::ADD16rr:
5017  case X86::ADD16rr_DB:
5018  case X86::ADD32i32:
5019  case X86::ADD32ri:
5020  case X86::ADD32ri8:
5021  case X86::ADD32ri8_DB:
5022  case X86::ADD32ri_DB:
5023  case X86::ADD32rm:
5024  case X86::ADD32rr:
5025  case X86::ADD32rr_DB:
5026  case X86::ADD64i32:
5027  case X86::ADD64ri32:
5028  case X86::ADD64ri32_DB:
5029  case X86::ADD64ri8:
5030  case X86::ADD64ri8_DB:
5031  case X86::ADD64rm:
5032  case X86::ADD64rr:
5033  case X86::ADD64rr_DB:
5034  case X86::ADD8i8:
5035  case X86::ADD8mi:
5036  case X86::ADD8mr:
5037  case X86::ADD8ri:
5038  case X86::ADD8rm:
5039  case X86::ADD8rr:
5040  case X86::SUB16i16:
5041  case X86::SUB16ri:
5042  case X86::SUB16ri8:
5043  case X86::SUB16rm:
5044  case X86::SUB16rr:
5045  case X86::SUB32i32:
5046  case X86::SUB32ri:
5047  case X86::SUB32ri8:
5048  case X86::SUB32rm:
5049  case X86::SUB32rr:
5050  case X86::SUB64i32:
5051  case X86::SUB64ri32:
5052  case X86::SUB64ri8:
5053  case X86::SUB64rm:
5054  case X86::SUB64rr:
5055  case X86::SUB8i8:
5056  case X86::SUB8ri:
5057  case X86::SUB8rm:
5058  case X86::SUB8rr:
5059  return FuseKind == FuseCmp || FuseKind == FuseInc;
5060  case X86::INC16r:
5061  case X86::INC32r:
5062  case X86::INC64_16r:
5063  case X86::INC64_32r:
5064  case X86::INC64r:
5065  case X86::INC8r:
5066  case X86::DEC16r:
5067  case X86::DEC32r:
5068  case X86::DEC64_16r:
5069  case X86::DEC64_32r:
5070  case X86::DEC64r:
5071  case X86::DEC8r:
5072  return FuseKind == FuseInc;
5073  }
5074 }
5075 
5076 bool X86InstrInfo::
5077 ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
5078  assert(Cond.size() == 1 && "Invalid X86 branch condition!");
5079  X86::CondCode CC = static_cast<X86::CondCode>(Cond[0].getImm());
5080  if (CC == X86::COND_NE_OR_P || CC == X86::COND_NP_OR_E)
5081  return true;
5082  Cond[0].setImm(GetOppositeBranchCondition(CC));
5083  return false;
5084 }
5085 
5086 bool X86InstrInfo::
5087 isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const {
5088  // FIXME: Return false for x87 stack register classes for now. We can't
5089  // allow any loads of these registers before FpGet_ST0_80.
5090  return !(RC == &X86::CCRRegClass || RC == &X86::RFP32RegClass ||
5091  RC == &X86::RFP64RegClass || RC == &X86::RFP80RegClass);
5092 }
5093 
5094 /// getGlobalBaseReg - Return a virtual register initialized with the
5095 /// the global base register value. Output instructions required to
5096 /// initialize the register in the function entry block, if necessary.
5097 ///
5098 /// TODO: Eliminate this and move the code to X86MachineFunctionInfo.
5099 ///
5100 unsigned X86InstrInfo::getGlobalBaseReg(MachineFunction *MF) const {
5101  assert(!TM.getSubtarget<X86Subtarget>().is64Bit() &&
5102  "X86-64 PIC uses RIP relative addressing");
5103 
5104  X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>();
5105  unsigned GlobalBaseReg = X86FI->getGlobalBaseReg();
5106  if (GlobalBaseReg != 0)
5107  return GlobalBaseReg;
5108 
5109  // Create the register. The code to initialize it is inserted
5110  // later, by the CGBR pass (below).
5111  MachineRegisterInfo &RegInfo = MF->getRegInfo();
5112  GlobalBaseReg = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
5113  X86FI->setGlobalBaseReg(GlobalBaseReg);
5114  return GlobalBaseReg;
5115 }
5116 
5117 // These are the replaceable SSE instructions. Some of these have Int variants
5118 // that we don't include here. We don't want to replace instructions selected
5119 // by intrinsics.
5120 static const uint16_t ReplaceableInstrs[][3] = {
5121  //PackedSingle PackedDouble PackedInt
5122  { X86::MOVAPSmr, X86::MOVAPDmr, X86::MOVDQAmr },
5123  { X86::MOVAPSrm, X86::MOVAPDrm, X86::MOVDQArm },
5124  { X86::MOVAPSrr, X86::MOVAPDrr, X86::MOVDQArr },
5125  { X86::MOVUPSmr, X86::MOVUPDmr, X86::MOVDQUmr },
5126  { X86::MOVUPSrm, X86::MOVUPDrm, X86::MOVDQUrm },
5127  { X86::MOVNTPSmr, X86::MOVNTPDmr, X86::MOVNTDQmr },
5128  { X86::ANDNPSrm, X86::ANDNPDrm, X86::PANDNrm },
5129  { X86::ANDNPSrr, X86::ANDNPDrr, X86::PANDNrr },
5130  { X86::ANDPSrm, X86::ANDPDrm, X86::PANDrm },
5131  { X86::ANDPSrr, X86::ANDPDrr, X86::PANDrr },
5132  { X86::ORPSrm, X86::ORPDrm, X86::PORrm },
5133  { X86::ORPSrr, X86::ORPDrr, X86::PORrr },
5134  { X86::XORPSrm, X86::XORPDrm, X86::PXORrm },
5135  { X86::XORPSrr, X86::XORPDrr, X86::PXORrr },
5136  // AVX 128-bit support
5137  { X86::VMOVAPSmr, X86::VMOVAPDmr, X86::VMOVDQAmr },
5138  { X86::VMOVAPSrm, X86::VMOVAPDrm, X86::VMOVDQArm },
5139  { X86::VMOVAPSrr, X86::VMOVAPDrr, X86::VMOVDQArr },
5140  { X86::VMOVUPSmr, X86::VMOVUPDmr, X86::VMOVDQUmr },
5141  { X86::VMOVUPSrm, X86::VMOVUPDrm, X86::VMOVDQUrm },
5142  { X86::VMOVNTPSmr, X86::VMOVNTPDmr, X86::VMOVNTDQmr },
5143  { X86::VANDNPSrm, X86::VANDNPDrm, X86::VPANDNrm },
5144  { X86::VANDNPSrr, X86::VANDNPDrr, X86::VPANDNrr },
5145  { X86::VANDPSrm, X86::VANDPDrm, X86::VPANDrm },
5146  { X86::VANDPSrr, X86::VANDPDrr, X86::VPANDrr },
5147  { X86::VORPSrm, X86::VORPDrm, X86::VPORrm },
5148  { X86::VORPSrr, X86::VORPDrr, X86::VPORrr },
5149  { X86::VXORPSrm, X86::VXORPDrm, X86::VPXORrm },
5150  { X86::VXORPSrr, X86::VXORPDrr, X86::VPXORrr },
5151  // AVX 256-bit support
5152  { X86::VMOVAPSYmr, X86::VMOVAPDYmr, X86::VMOVDQAYmr },
5153  { X86::VMOVAPSYrm, X86::VMOVAPDYrm, X86::VMOVDQAYrm },
5154  { X86::VMOVAPSYrr, X86::VMOVAPDYrr, X86::VMOVDQAYrr },
5155  { X86::VMOVUPSYmr, X86::VMOVUPDYmr, X86::VMOVDQUYmr },
5156  { X86::VMOVUPSYrm, X86::VMOVUPDYrm, X86::VMOVDQUYrm },
5157  { X86::VMOVNTPSYmr, X86::VMOVNTPDYmr, X86::VMOVNTDQYmr }
5158 };
5159 
5160 static const uint16_t ReplaceableInstrsAVX2[][3] = {
5161  //PackedSingle PackedDouble PackedInt
5162  { X86::VANDNPSYrm, X86::VANDNPDYrm, X86::VPANDNYrm },
5163  { X86::VANDNPSYrr, X86::VANDNPDYrr, X86::VPANDNYrr },
5164  { X86::VANDPSYrm, X86::VANDPDYrm, X86::VPANDYrm },
5165  { X86::VANDPSYrr, X86::VANDPDYrr, X86::VPANDYrr },
5166  { X86::VORPSYrm, X86::VORPDYrm, X86::VPORYrm },
5167  { X86::VORPSYrr, X86::VORPDYrr, X86::VPORYrr },
5168  { X86::VXORPSYrm, X86::VXORPDYrm, X86::VPXORYrm },
5169  { X86::VXORPSYrr, X86::VXORPDYrr, X86::VPXORYrr },
5170  { X86::VEXTRACTF128mr, X86::VEXTRACTF128mr, X86::VEXTRACTI128mr },
5171  { X86::VEXTRACTF128rr, X86::VEXTRACTF128rr, X86::VEXTRACTI128rr },
5172  { X86::VINSERTF128rm, X86::VINSERTF128rm, X86::VINSERTI128rm },
5173  { X86::VINSERTF128rr, X86::VINSERTF128rr, X86::VINSERTI128rr },
5174  { X86::VPERM2F128rm, X86::VPERM2F128rm, X86::VPERM2I128rm },
5175  { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr }
5176 };
5177 
5178 // FIXME: Some shuffle and unpack instructions have equivalents in different
5179 // domains, but they require a bit more work than just switching opcodes.
5180 
5181 static const uint16_t *lookup(unsigned opcode, unsigned domain) {
5182  for (unsigned i = 0, e = array_lengthof(ReplaceableInstrs); i != e; ++i)
5183  if (ReplaceableInstrs[i][domain-1] == opcode)
5184  return ReplaceableInstrs[i];
5185  return 0;
5186 }
5187 
5188 static const uint16_t *lookupAVX2(unsigned opcode, unsigned domain) {
5189  for (unsigned i = 0, e = array_lengthof(ReplaceableInstrsAVX2); i != e; ++i)
5190  if (ReplaceableInstrsAVX2[i][domain-1] == opcode)
5191  return ReplaceableInstrsAVX2[i];
5192  return 0;
5193 }
5194 
5195 std::pair<uint16_t, uint16_t>
5196 X86InstrInfo::getExecutionDomain(const MachineInstr *MI) const {
5197  uint16_t domain = (MI->getDesc().TSFlags >> X86II::SSEDomainShift) & 3;
5198  bool hasAVX2 = TM.getSubtarget<X86Subtarget>().hasAVX2();
5199  uint16_t validDomains = 0;
5200  if (domain && lookup(MI->getOpcode(), domain))
5201  validDomains = 0xe;
5202  else if (domain && lookupAVX2(MI->getOpcode(), domain))
5203  validDomains = hasAVX2 ? 0xe : 0x6;
5204  return std::make_pair(domain, validDomains);
5205 }
5206 
5207 void X86InstrInfo::setExecutionDomain(MachineInstr *MI, unsigned Domain) const {
5208  assert(Domain>0 && Domain<4 && "Invalid execution domain");
5209  uint16_t dom = (MI->getDesc().TSFlags >> X86II::SSEDomainShift) & 3;
5210  assert(dom && "Not an SSE instruction");
5211  const uint16_t *table = lookup(MI->getOpcode(), dom);
5212  if (!table) { // try the other table
5213  assert((TM.getSubtarget<X86Subtarget>().hasAVX2() || Domain < 3) &&
5214  "256-bit vector operations only available in AVX2");
5215  table = lookupAVX2(MI->getOpcode(), dom);
5216  }
5217  assert(table && "Cannot change domain");
5218  MI->setDesc(get(table[Domain-1]));
5219 }
5220 
5221 /// getNoopForMachoTarget - Return the noop instruction to use for a noop.
5222 void X86InstrInfo::getNoopForMachoTarget(MCInst &NopInst) const {
5223  NopInst.setOpcode(X86::NOOP);
5224 }
5225 
5226 bool X86InstrInfo::isHighLatencyDef(int opc) const {
5227  switch (opc) {
5228  default: return false;
5229  case X86::DIVSDrm:
5230  case X86::DIVSDrm_Int:
5231  case X86::DIVSDrr:
5232  case X86::DIVSDrr_Int:
5233  case X86::DIVSSrm:
5234  case X86::DIVSSrm_Int:
5235  case X86::DIVSSrr:
5236  case X86::DIVSSrr_Int:
5237  case X86::SQRTPDm:
5238  case X86::SQRTPDr:
5239  case X86::SQRTPSm:
5240  case X86::SQRTPSr:
5241  case X86::SQRTSDm:
5242  case X86::SQRTSDm_Int:
5243  case X86::SQRTSDr:
5244  case X86::SQRTSDr_Int:
5245  case X86::SQRTSSm:
5246  case X86::SQRTSSm_Int:
5247  case X86::SQRTSSr:
5248  case X86::SQRTSSr_Int:
5249  // AVX instructions with high latency
5250  case X86::VDIVSDrm:
5251  case X86::VDIVSDrm_Int:
5252  case X86::VDIVSDrr:
5253  case X86::VDIVSDrr_Int:
5254  case X86::VDIVSSrm:
5255  case X86::VDIVSSrm_Int:
5256  case X86::VDIVSSrr:
5257  case X86::VDIVSSrr_Int:
5258  case X86::VSQRTPDm:
5259  case X86::VSQRTPDr:
5260  case X86::VSQRTPSm:
5261  case X86::VSQRTPSr:
5262  case X86::VSQRTSDm:
5263  case X86::VSQRTSDm_Int:
5264  case X86::VSQRTSDr:
5265  case X86::VSQRTSSm:
5266  case X86::VSQRTSSm_Int:
5267  case X86::VSQRTSSr:
5268  case X86::VSQRTPDZrm:
5269  case X86::VSQRTPDZrr:
5270  case X86::VSQRTPSZrm:
5271  case X86::VSQRTPSZrr:
5272  case X86::VSQRTSDZm:
5273  case X86::VSQRTSDZm_Int:
5274  case X86::VSQRTSDZr:
5275  case X86::VSQRTSSZm_Int:
5276  case X86::VSQRTSSZr:
5277  case X86::VSQRTSSZm:
5278  case X86::VDIVSDZrm:
5279  case X86::VDIVSDZrr:
5280  case X86::VDIVSSZrm:
5281  case X86::VDIVSSZrr:
5282 
5283  case X86::VGATHERQPSZrm:
5284  case X86::VGATHERQPDZrm:
5285  case X86::VGATHERDPDZrm:
5286  case X86::VGATHERDPSZrm:
5287  case X86::VPGATHERQDZrm:
5288  case X86::VPGATHERQQZrm:
5289  case X86::VPGATHERDDZrm:
5290  case X86::VPGATHERDQZrm:
5291  case X86::VSCATTERQPDZmr:
5292  case X86::VSCATTERQPSZmr:
5293  case X86::VSCATTERDPDZmr:
5294  case X86::VSCATTERDPSZmr:
5295  case X86::VPSCATTERQDZmr:
5296  case X86::VPSCATTERQQZmr:
5297  case X86::VPSCATTERDDZmr:
5298  case X86::VPSCATTERDQZmr:
5299  return true;
5300  }
5301 }
5302 
5303 bool X86InstrInfo::
5304 hasHighOperandLatency(const InstrItineraryData *ItinData,
5305  const MachineRegisterInfo *MRI,
5306  const MachineInstr *DefMI, unsigned DefIdx,
5307  const MachineInstr *UseMI, unsigned UseIdx) const {
5308  return isHighLatencyDef(DefMI->getOpcode());
5309 }
5310 
5311 namespace {
5312  /// CGBR - Create Global Base Reg pass. This initializes the PIC
5313  /// global base register for x86-32.
5314  struct CGBR : public MachineFunctionPass {
5315  static char ID;
5316  CGBR() : MachineFunctionPass(ID) {}
5317 
5318  virtual bool runOnMachineFunction(MachineFunction &MF) {
5319  const X86TargetMachine *TM =
5320  static_cast<const X86TargetMachine *>(&MF.getTarget());
5321 
5322  assert(!TM->getSubtarget<X86Subtarget>().is64Bit() &&
5323  "X86-64 PIC uses RIP relative addressing");
5324 
5325  // Only emit a global base reg in PIC mode.
5326  if (TM->getRelocationModel() != Reloc::PIC_)
5327  return false;
5328 
5329  X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
5330  unsigned GlobalBaseReg = X86FI->getGlobalBaseReg();
5331 
5332  // If we didn't need a GlobalBaseReg, don't insert code.
5333  if (GlobalBaseReg == 0)
5334  return false;
5335 
5336  // Insert the set of GlobalBaseReg into the first MBB of the function
5337  MachineBasicBlock &FirstMBB = MF.front();
5338  MachineBasicBlock::iterator MBBI = FirstMBB.begin();
5339  DebugLoc DL = FirstMBB.findDebugLoc(MBBI);
5340  MachineRegisterInfo &RegInfo = MF.getRegInfo();
5341  const X86InstrInfo *TII = TM->getInstrInfo();
5342 
5343  unsigned PC;
5344  if (TM->getSubtarget<X86Subtarget>().isPICStyleGOT())
5345  PC = RegInfo.createVirtualRegister(&X86::GR32RegClass);
5346  else
5347  PC = GlobalBaseReg;
5348 
5349  // Operand of MovePCtoStack is completely ignored by asm printer. It's
5350  // only used in JIT code emission as displacement to pc.
5351  BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOVPC32r), PC).addImm(0);
5352 
5353  // If we're using vanilla 'GOT' PIC style, we should use relative addressing
5354  // not to pc, but to _GLOBAL_OFFSET_TABLE_ external.
5355  if (TM->getSubtarget<X86Subtarget>().isPICStyleGOT()) {
5356  // Generate addl $__GLOBAL_OFFSET_TABLE_ + [.-piclabel], %some_register
5357  BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD32ri), GlobalBaseReg)
5358  .addReg(PC).addExternalSymbol("_GLOBAL_OFFSET_TABLE_",
5359  X86II::MO_GOT_ABSOLUTE_ADDRESS);
5360  }
5361 
5362  return true;
5363  }
5364 
5365  virtual const char *getPassName() const {
5366  return "X86 PIC Global Base Reg Initialization";
5367  }
5368 
5369  virtual void getAnalysisUsage(AnalysisUsage &AU) const {
5370  AU.setPreservesCFG();
5371  MachineFunctionPass::getAnalysisUsage(AU);
5372  }
5373  };
5374 }
5375 
5376 char CGBR::ID = 0;
5377 FunctionPass*
5378 llvm::createGlobalBaseRegPass() { return new CGBR(); }
5379 
5380 namespace {
5381  struct LDTLSCleanup : public MachineFunctionPass {
5382  static char ID;
5383  LDTLSCleanup() : MachineFunctionPass(ID) {}
5384 
5385  virtual bool runOnMachineFunction(MachineFunction &MF) {
5386  X86MachineFunctionInfo* MFI = MF.getInfo<X86MachineFunctionInfo>();
5387  if (MFI->getNumLocalDynamicTLSAccesses() < 2) {
5388  // No point folding accesses if there isn't at least two.
5389  return false;
5390  }
5391 
5392  MachineDominatorTree *DT = &getAnalysis<MachineDominatorTree>();
5393  return VisitNode(DT->getRootNode(), 0);
5394  }
5395 
5396  // Visit the dominator subtree rooted at Node in pre-order.
5397  // If TLSBaseAddrReg is non-null, then use that to replace any
5398  // TLS_base_addr instructions. Otherwise, create the register
5399  // when the first such instruction is seen, and then use it
5400  // as we encounter more instructions.
5401  bool VisitNode(MachineDomTreeNode *Node, unsigned TLSBaseAddrReg) {
5402  MachineBasicBlock *BB = Node->getBlock();
5403  bool Changed = false;
5404 
5405  // Traverse the current block.
5406  for (MachineBasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;
5407  ++I) {
5408  switch (I->getOpcode()) {
5409  case X86::TLS_base_addr32:
5410  case X86::TLS_base_addr64:
5411  if (TLSBaseAddrReg)
5412  I = ReplaceTLSBaseAddrCall(I, TLSBaseAddrReg);
5413  else
5414  I = SetRegister(I, &TLSBaseAddrReg);
5415  Changed = true;
5416  break;
5417  default:
5418  break;
5419  }
5420  }
5421 
5422  // Visit the children of this block in the dominator tree.
5423  for (MachineDomTreeNode::iterator I = Node->begin(), E = Node->end();
5424  I != E; ++I) {
5425  Changed |= VisitNode(*I, TLSBaseAddrReg);
5426  }
5427 
5428  return Changed;
5429  }
5430 
5431  // Replace the TLS_base_addr instruction I with a copy from
5432  // TLSBaseAddrReg, returning the new instruction.
5433  MachineInstr *ReplaceTLSBaseAddrCall(MachineInstr *I,
5434  unsigned TLSBaseAddrReg) {
5435  MachineFunction *MF = I->getParent()->getParent();
5436  const X86TargetMachine *TM =
5437  static_cast<const X86TargetMachine *>(&MF->getTarget());
5438  const bool is64Bit = TM->getSubtarget<X86Subtarget>().is64Bit();
5439  const X86InstrInfo *TII = TM->getInstrInfo();
5440 
5441  // Insert a Copy from TLSBaseAddrReg to RAX/EAX.
5442  MachineInstr *Copy = BuildMI(*I->getParent(), I, I->getDebugLoc(),
5443  TII->get(TargetOpcode::COPY),
5444  is64Bit ? X86::RAX : X86::EAX)
5445  .addReg(TLSBaseAddrReg);
5446 
5447  // Erase the TLS_base_addr instruction.
5448  I->eraseFromParent();
5449 
5450  return Copy;
5451  }
5452 
5453  // Create a virtal register in *TLSBaseAddrReg, and populate it by
5454  // inserting a copy instruction after I. Returns the new instruction.
5455  MachineInstr *SetRegister(MachineInstr *I, unsigned *TLSBaseAddrReg) {
5456  MachineFunction *MF = I->getParent()->getParent();
5457  const X86TargetMachine *TM =
5458  static_cast<const X86TargetMachine *>(&MF->getTarget());
5459  const bool is64Bit = TM->getSubtarget<X86Subtarget>().is64Bit();
5460  const X86InstrInfo *TII = TM->getInstrInfo();
5461 
5462  // Create a virtual register for the TLS base address.
5463  MachineRegisterInfo &RegInfo = MF->getRegInfo();
5464  *TLSBaseAddrReg = RegInfo.createVirtualRegister(is64Bit
5465  ? &X86::GR64RegClass
5466  : &X86::GR32RegClass);
5467 
5468  // Insert a copy from RAX/EAX to TLSBaseAddrReg.
5469  MachineInstr *Next = I->getNextNode();
5470  MachineInstr *Copy = BuildMI(*I->getParent(), Next, I->getDebugLoc(),
5471  TII->get(TargetOpcode::COPY),
5472  *TLSBaseAddrReg)
5473  .addReg(is64Bit ? X86::RAX : X86::EAX);
5474 
5475  return Copy;
5476  }
5477 
5478  virtual const char *getPassName() const {
5479  return "Local Dynamic TLS Access Clean-up";
5480  }
5481 
5482  virtual void getAnalysisUsage(AnalysisUsage &AU) const {
5483  AU.setPreservesCFG();
5484  AU.addRequired<MachineDominatorTree>();
5485  MachineFunctionPass::getAnalysisUsage(AU);
5486  }
5487  };
5488 }
5489 
5490 char LDTLSCleanup::ID = 0;
5491 FunctionPass*
5492 llvm::createCleanupLocalDynamicTLSPass() { return new LDTLSCleanup(); }
llvm::MachineOperand::isImplicit
bool isImplicit() const
Definition: MachineOperand.h:279
llvm::TargetFrameLowering::getStackAlignment
unsigned getStackAlignment() const
Definition: TargetFrameLowering.h:70
llvm::X86::GetCondBranchFromCond
unsigned GetCondBranchFromCond(CondCode CC)
Definition: X86InstrInfo.cpp:2555
llvm::X86InstrInfo::getNoopForMachoTarget
virtual void getNoopForMachoTarget(MCInst &NopInst) const
getNoopForMachoTarget - Return the noop instruction to use for a noop.
Definition: X86InstrInfo.cpp:5222
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Definition: MachineBasicBlock.h:137
llvm::MachineConstantPool
The machine constant pool.
Definition: MachineConstantPool.h:134
hasUndefRegUpdate
static bool hasUndefRegUpdate(unsigned Opcode)
Definition: X86InstrInfo.cpp:4113
llvm::Type::getDoubleTy
static Type * getDoubleTy(LLVMContext &C)
Definition: Type.cpp:231
llvm::MachineFunction::CreateMachineInstr
MachineInstr * CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL, bool NoImp=false)
Definition: MachineFunction.cpp:173
X86OpTblEntry::RegOp
uint16_t RegOp
Definition: X86InstrInfo.cpp:90
llvm::Function::getContext
LLVMContext & getContext() const
Definition: Function.cpp:167
regIsPICBase
static bool regIsPICBase(unsigned BaseReg, const MachineRegisterInfo &MRI)
Definition: X86InstrInfo.cpp:1619
llvm::MachineInstr::isBranch
bool isBranch(QueryType Type=AnyInBundle) const
Definition: MachineInstr.h:374
llvm::TargetMachine::getRelocationModel
Reloc::Model getRelocationModel() const
Definition: TargetMachine.cpp:91
llvm::X86InstrInfo::storeRegToStackSlot
virtual void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const
Definition: X86InstrInfo.cpp:3215
llvm::X86InstrInfo::storeRegToAddr
virtual void storeRegToAddr(MachineFunction &MF, unsigned SrcReg, bool isKill, SmallVectorImpl< MachineOperand > &Addr, const TargetRegisterClass *RC, MachineInstr::mmo_iterator MMOBegin, MachineInstr::mmo_iterator MMOEnd, SmallVectorImpl< MachineInstr * > &NewMIs) const
Definition: X86InstrInfo.cpp:3232
CopyToFromAsymmetricReg
static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg, const X86Subtarget &Subtarget)
Definition: X86InstrInfo.cpp:2976
llvm::MachineOperand::ChangeToRegister
void ChangeToRegister(unsigned Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isDebug=false)
Definition: MachineInstr.cpp:129
llvm::X86InstrInfo::AnalyzeBranch
virtual bool AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const
Definition: X86InstrInfo.cpp:2707
llvm::MCInstrDesc::getNumDefs
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions. Register definitions always occur...
Definition: MCInstrDesc.h:198
llvm::MachineRegisterInfo::createVirtualRegister
unsigned createVirtualRegister(const TargetRegisterClass *RegClass)
Definition: MachineRegisterInfo.cpp:104
llvm::X86::getCondFromCMovOpc
CondCode getCondFromCMovOpc(unsigned Opc)
getCondFromCmovOpc - return condition code of a CMov opcode.
Definition: X86InstrInfo.cpp:2501
llvm::MachineDominatorTree::getRootNode
MachineDomTreeNode * getRootNode() const
Definition: MachineDominators.h:65
llvm::X86InstrInfo::isHighLatencyDef
bool isHighLatencyDef(int opc) const
Definition: X86InstrInfo.cpp:5226
llvm::X86InstrInfo::getOpcodeAfterMemoryUnfold
virtual unsigned getOpcodeAfterMemoryUnfold(unsigned Opc, bool UnfoldLoad, bool UnfoldStore, unsigned *LoadRegIndex=0) const
Definition: X86InstrInfo.cpp:4742
llvm::MachineOperand::isDead
bool isDead() const
Definition: MachineOperand.h:284
llvm::TargetRegisterInfo::isVirtualRegister
static bool isVirtualRegister(unsigned Reg)
Definition: TargetRegisterInfo.h:287
llvm::MachineInstr::isPredicable
bool isPredicable(QueryType Type=AllInBundle) const
Definition: MachineInstr.h:404
llvm::MachineInstr::readsVirtualRegister
bool readsVirtualRegister(unsigned Reg) const
Definition: MachineInstr.h:731
ReMatPICStubLoad
static cl::opt< bool > ReMatPICStubLoad("remat-pic-stub-load", cl::desc("Re-materialize load from stub in PIC mode"), cl::init(false), cl::Hidden)
llvm::MachineRegisterInfo::use_nodbg_empty
bool use_nodbg_empty(unsigned RegNo) const
Definition: MachineRegisterInfo.h:285
getCondFromBranchOpc
static X86::CondCode getCondFromBranchOpc(unsigned BrOpc)
Definition: X86InstrInfo.cpp:2455
llvm::X86InstrInfo::foldMemoryOperandImpl
virtual MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI, const SmallVectorImpl< unsigned > &Ops, int FrameIndex) const
Definition: X86InstrInfo.cpp:4262
llvm::MachineBasicBlock::computeRegisterLiveness
LivenessQueryResult computeRegisterLiveness(const TargetRegisterInfo *TRI, unsigned Reg, MachineInstr *MI, unsigned Neighborhood=10)
Definition: MachineBasicBlock.cpp:1144
llvm::MachineInstr::getDesc
const MCInstrDesc & getDesc() const
Definition: MachineInstr.h:257
llvm::SDNode::getNumOperands
unsigned getNumOperands() const
Definition: SelectionDAGNodes.h:548
llvm::MachineInstr::registerDefIsDead
bool registerDefIsDead(unsigned Reg, const TargetRegisterInfo *TRI=NULL) const
Definition: MachineInstr.h:767
llvm::SDNode::getOperand
const SDValue & getOperand(unsigned Num) const
Definition: SelectionDAGNodes.h:554
llvm::MachineOperand::setIsDead
void setIsDead(bool Val=true)
Definition: MachineOperand.h:373
llvm::MachineFunction::getFunction
const Function * getFunction() const
Definition: MachineFunction.h:151
llvm::X86InstrInfo::canInsertSelect
virtual bool canInsertSelect(const MachineBasicBlock &, const SmallVectorImpl< MachineOperand > &Cond, unsigned, unsigned, int &, int &, int &) const
Definition: X86InstrInfo.cpp:2921
llvm::PatchPointOpers::getVarIdx
unsigned getVarIdx() const
Definition: StackMaps.h:64
llvm::MachineInstr::isTerminator
bool isTerminator(QueryType Type=AnyInBundle) const
Definition: MachineInstr.h:366
llvm::report_fatal_error
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const char *reason, bool gen_crash_diag=true)
Definition: ErrorHandling.cpp:53
llvm::Constant::getNullValue
static Constant * getNullValue(Type *Ty)
Definition: Constants.cpp:111
llvm::X86InstrInfo::getRegisterInfo
virtual const X86RegisterInfo & getRegisterInfo() const
Definition: X86InstrInfo.h:164
llvm::X86InstrInfo::commuteInstruction
virtual MachineInstr * commuteInstruction(MachineInstr *MI, bool NewMI) const
Definition: X86InstrInfo.cpp:2345
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:55
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:115
llvm::Type::getFloatTy
static Type * getFloatTy(LLVMContext &C)
Definition: Type.cpp:230
llvm::ilist_node::getNextNode
NodeTy * getNextNode()
Get the next node, or 0 for the list tail.
Definition: ilist_node.h:80
llvm::MachineOperand::CreateReg
static MachineOperand CreateReg(unsigned Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false)
Definition: MachineOperand.h:560
SawStore
INITIALIZE_PASS(DeadMachineInstructionElim,"dead-mi-elimination","Remove dead machine instructions", false, false) bool DeadMachineInstructionElim bool SawStore
Definition: DeadMachineInstructionElim.cpp:61
llvm::MachineOperand::isImm
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
Definition: MachineOperand.h:226
llvm::X86InstrInfo::isLoadFromStackSlot
unsigned isLoadFromStackSlot(const MachineInstr *MI, int &FrameIndex) const
Definition: X86InstrInfo.cpp:1574
llvm_unreachable
#define llvm_unreachable(msg)
Definition: ErrorHandling.h:103
llvm::SDNode::getValueType
EVT getValueType(unsigned ResNo) const
Definition: SelectionDAGNodes.h:609
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:280
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition: MachineOperand.h:224
llvm::MachineOperand::setImplicit
void setImplicit(bool Val=true)
Definition: MachineOperand.h:362
hasPartialRegUpdate
static bool hasPartialRegUpdate(unsigned Opcode)
Definition: X86InstrInfo.cpp:4060
llvm::MachineInstr::mayLoad
bool mayLoad(QueryType Type=AnyInBundle) const
Definition: MachineInstr.h:465
llvm::MachineRegisterInfo::getRegClass
const TargetRegisterClass * getRegClass(unsigned Reg) const
Definition: MachineRegisterInfo.h:342
llvm::MachineBasicBlock::succ_iterator
std::vector< MachineBasicBlock * >::iterator succ_iterator
Definition: MachineBasicBlock.h:249
llvm::X86InstrInfo::shouldScheduleAdjacent
virtual bool shouldScheduleAdjacent(MachineInstr *First, MachineInstr *Second) const LLVM_OVERRIDE
Definition: X86InstrInfo.cpp:4914
llvm::MVT::SimpleTy
SimpleValueType SimpleTy
Definition: ValueTypes.h:161
llvm::MachineInstr::mmo_iterator
MachineMemOperand ** mmo_iterator
Definition: MachineInstr.h:52
llvm::LiveVariables::replaceKillInstruction
void replaceKillInstruction(unsigned Reg, MachineInstr *OldMI, MachineInstr *NewMI)
Definition: LiveVariables.cpp:676
llvm::X86InstrInfo::getExecutionDomain
std::pair< uint16_t, uint16_t > getExecutionDomain(const MachineInstr *MI) const
Definition: X86InstrInfo.cpp:5196
llvm::X86MachineFunctionInfo::getNumLocalDynamicTLSAccesses
unsigned getNumLocalDynamicTLSAccesses() const
Definition: X86MachineFunctionInfo.h:138
llvm::MachineFrameInfo
Abstract Stack Frame Information.
Definition: MachineFrameInfo.h:80
llvm::MachineFunction::extractLoadMemRefs
std::pair< MachineInstr::mmo_iterator, MachineInstr::mmo_iterator > extractLoadMemRefs(MachineInstr::mmo_iterator Begin, MachineInstr::mmo_iterator End)
Definition: MachineFunction.cpp:248
llvm::MachineOperand::isUndef
bool isUndef() const
Definition: MachineOperand.h:294
llvm::createGlobalBaseRegPass
FunctionPass * createGlobalBaseRegPass()
Definition: X86InstrInfo.cpp:5378
llvm::X86RegisterInfo::needsStackRealignment
bool needsStackRealignment(const MachineFunction &MF) const
Definition: X86RegisterInfo.cpp:439
llvm::CallingConv::ID
ID
LLVM Calling Convention Representation.
Definition: CallingConv.h:26
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Definition: MachineInstrBuilder.h:83
llvm::X86InstrInfo::loadRegFromStackSlot
virtual void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const
Definition: X86InstrInfo.cpp:3253
llvm::MachineInstr::getNumOperands
unsigned getNumOperands() const
Definition: MachineInstr.h:265
llvm::MachineOperand::isGlobal
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.
Definition: MachineOperand.h:242
llvm::MachineInstr::RemoveOperand
void RemoveOperand(unsigned i)
Definition: MachineInstr.cpp:717
llvm::MachineFunction::front
const MachineBasicBlock & front() const
Definition: MachineFunction.h:327
llvm::MCInstrDesc::NumDefs
unsigned short NumDefs
Definition: MCInstrDesc.h:141
llvm::MachineOperand::isKill
bool isKill() const
Definition: MachineOperand.h:289
llvm::MachineOperand::isFI
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
Definition: MachineOperand.h:234
llvm::array_lengthof
size_t array_lengthof(T(&)[N])
Find the length of an array.
Definition: STLExtras.h:250
llvm::MachineInstr::readsRegister
bool readsRegister(unsigned Reg, const TargetRegisterInfo *TRI=NULL) const
Definition: MachineInstr.h:724
llvm::SDNode::getNumValues
unsigned getNumValues() const
Definition: SelectionDAGNodes.h:605
llvm::SmallVectorBase::empty
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:56
getCondFromSETOpc
static X86::CondCode getCondFromSETOpc(unsigned Opc)
getCondFromSETOpc - return condition code of a SET opcode.
Definition: X86InstrInfo.cpp:2478
llvm::X86RegisterInfo::canRealignStack
bool canRealignStack(const MachineFunction &MF) const
Definition: X86RegisterInfo.cpp:420
llvm::TargetInstrInfo::commuteInstruction
virtual MachineInstr * commuteInstruction(MachineInstr *MI, bool NewMI=false) const
Definition: TargetInstrInfo.cpp:119
llvm::MachineInstr::getOpcode
int getOpcode() const
Definition: MachineInstr.h:261
llvm::X86Subtarget::is64Bit
bool is64Bit() const
Is this x86_64? (disregarding specific ABI / programming model)
Definition: X86Subtarget.h:240
llvm::X86InstrInfo::breakPartialRegDependency
void breakPartialRegDependency(MachineBasicBlock::iterator MI, unsigned OpNum, const TargetRegisterInfo *TRI) const
Definition: X86InstrInfo.cpp:4176
llvm::TargetInstrInfo::getStackSlotRange
virtual bool getStackSlotRange(const TargetRegisterClass *RC, unsigned SubIdx, unsigned &Size, unsigned &Offset, const TargetMachine *TM) const
Definition: TargetInstrInfo.cpp:280
llvm::MachineMemOperand::getAlignment
uint64_t getAlignment() const
Definition: MachineInstr.cpp:460
llvm::X86InstrInfo::getUndefRegClearance
unsigned getUndefRegClearance(const MachineInstr *MI, unsigned &OpNum, const TargetRegisterInfo *TRI) const
Definition: X86InstrInfo.cpp:4159
llvm::MachineOperand::getImm
int64_t getImm() const
Definition: MachineOperand.h:402
llvm::getUndefRegState
unsigned getUndefRegState(bool B)
Definition: MachineInstrBuilder.h:395
llvm::MachineBasicBlock::rend
reverse_iterator rend()
Definition: MachineBasicBlock.h:242
llvm::DomTreeNodeBase::getBlock
NodeT * getBlock() const
Definition: Dominators.h:82
llvm::MachineRegisterInfo::constrainRegClass
const TargetRegisterClass * constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
Definition: MachineRegisterInfo.cpp:52
llvm::getKillRegState
unsigned getKillRegState(bool B)
Definition: MachineInstrBuilder.h:389
llvm::X86Subtarget::hasSSE2
bool hasSSE2() const
Definition: X86Subtarget.h:260
MakeM0Inst
static MachineInstr * MakeM0Inst(const TargetInstrInfo &TII, unsigned Opcode, const SmallVectorImpl< MachineOperand > &MOs, MachineInstr *MI)
Definition: X86InstrInfo.cpp:3924
llvm::MachineOperand::ChangeToImmediate
void ChangeToImmediate(int64_t ImmVal)
Definition: MachineInstr.cpp:112
llvm::getX86SubSuperRegister
unsigned getX86SubSuperRegister(unsigned Reg, MVT::SimpleValueType VT, bool High)
Definition: X86RegisterInfo.cpp:523
llvm::MachineInstr::getParent
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:119
getLoadStoreRegOpcode
static unsigned getLoadStoreRegOpcode(unsigned Reg, const TargetRegisterClass *RC, bool isStackAligned, const TargetMachine &TM, bool load)
Definition: X86InstrInfo.cpp:3113
llvm::getDeadRegState
unsigned getDeadRegState(bool B)
Definition: MachineInstrBuilder.h:392
llvm::MachineInstr::memoperands_end
mmo_iterator memoperands_end() const
Definition: MachineInstr.h:292
getSETFromCond
static unsigned getSETFromCond(X86::CondCode CC, bool HasMemoryOperand)
Definition: X86InstrInfo.cpp:2622
llvm::getDefRegState
unsigned getDefRegState(bool B)
Definition: MachineInstrBuilder.h:383
llvm::X86::GetOppositeBranchCondition
CondCode GetOppositeBranchCondition(X86::CondCode CC)
Definition: X86InstrInfo.cpp:2579
llvm::MachineBasicBlock::iterator
bundle_iterator< MachineInstr, instr_iterator > iterator
Definition: MachineBasicBlock.h:212
llvm::X86InstrInfo::hasHighOperandLatency
bool hasHighOperandLatency(const InstrItineraryData *ItinData, const MachineRegisterInfo *MRI, const MachineInstr *DefMI, unsigned DefIdx, const MachineInstr *UseMI, unsigned UseIdx) const
Definition: X86InstrInfo.cpp:5304
isFrameLoadOpcode
static bool isFrameLoadOpcode(int Opcode)
Definition: X86InstrInfo.cpp:1515
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:314
llvm::MachineOperand::getTargetFlags
unsigned getTargetFlags() const
Definition: MachineOperand.h:187
llvm::MachineInstrBuilder::setMemRefs
const MachineInstrBuilder & setMemRefs(MachineInstr::mmo_iterator b, MachineInstr::mmo_iterator e) const
Definition: MachineInstrBuilder.h:159
isDefConvertible
static bool isDefConvertible(MachineInstr *MI)
Definition: X86InstrInfo.cpp:3402
isTruncatedShiftCountForLEA
static bool isTruncatedShiftCountForLEA(unsigned ShAmt)
Definition: X86InstrInfo.cpp:1841
llvm::MachineOperand::CreateCPI
static MachineOperand CreateCPI(unsigned Idx, int Offset, unsigned char TargetFlags=0)
Definition: MachineOperand.h:595
llvm::TargetMachine::getCodeModel
CodeModel::Model getCodeModel() const
Definition: TargetMachine.cpp:99
foldPatchpoint
static MachineInstr * foldPatchpoint(MachineFunction &MF, MachineInstr *MI, const SmallVectorImpl< unsigned > &Ops, int FrameIndex, const TargetInstrInfo &TII)
Definition: X86InstrInfo.cpp:4201
ReplaceableInstrs
static const uint16_t ReplaceableInstrs[][3]
Definition: X86InstrInfo.cpp:5120
llvm::Constant
LLVM Constant Representation.
Definition: Constant.h:41
llvm::X86InstrInfo::expandPostRAPseudo
virtual bool expandPostRAPseudo(MachineBasicBlock::iterator MI) const
Definition: X86InstrInfo.cpp:3836
llvm::X86Subtarget::hasAVX2
bool hasAVX2() const
Definition: X86Subtarget.h:266
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:267
llvm::get512BitSuperRegister
unsigned get512BitSuperRegister(unsigned Reg)
Definition: X86RegisterInfo.cpp:696
llvm::MCRegisterInfo::getSubReg
unsigned getSubReg(unsigned Reg, unsigned Idx) const
Returns the physical register number of sub-register "Index" for physical register RegNo...
Definition: MCRegisterInfo.cpp:26
PrintFailedFusing
static cl::opt< bool > PrintFailedFusing("print-failed-fuse-candidates", cl::desc("Print instructions that the allocator wants to"" fuse, but the X86 backend currently can't"), cl::Hidden)
llvm::MachineInstr::isCopy
bool isCopy() const
Definition: MachineInstr.h:669
llvm::addRegOffset
static const MachineInstrBuilder & addRegOffset(const MachineInstrBuilder &MIB, unsigned Reg, bool isKill, int Offset)
Definition: X86InstrBuilder.h:107
llvm::next
ItTy next(ItTy it, Dist n)
Definition: STLExtras.h:154
llvm::X86InstrInfo::canFoldMemoryOperand
virtual bool canFoldMemoryOperand(const MachineInstr *, const SmallVectorImpl< unsigned > &) const
Definition: X86InstrInfo.cpp:4456
llvm::MachineOperand::setImm
void setImm(int64_t immVal)
Definition: MachineOperand.h:488
llvm::MachineInstr::hasOneMemOperand
bool hasOneMemOperand() const
Definition: MachineInstr.h:297
llvm::PatchPointOpers
MI-level patchpoint operands.
Definition: StackMaps.h:37
llvm::X86InstrInfo::isStoreToStackSlot
unsigned isStoreToStackSlot(const MachineInstr *MI, int &FrameIndex) const
Definition: X86InstrInfo.cpp:1595
llvm::X86::AddrNumOperands
AddrNumOperands - Total number of operands in a memory reference.
Definition: X86BaseInfo.h:42
llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT >, KeyT, ValueT, KeyInfoT >::count
bool count(const KeyT &Val) const
count - Return true if the specified key is in the map.
Definition: DenseMap.h:103
getTruncatedShiftCount
static unsigned getTruncatedShiftCount(MachineInstr *MI, unsigned ShiftAmtOperandIdx)
Definition: X86InstrInfo.cpp:1831
llvm::MachineInstr::isInvariantLoad
bool isInvariantLoad(AliasAnalysis *AA) const
Definition: MachineInstr.cpp:1289
llvm::MachineFunction::getConstantPool
MachineConstantPool * getConstantPool()
Definition: MachineFunction.h:192
llvm::MachineBasicBlock::LivenessQueryResult
LivenessQueryResult
Possible outcome of a register liveness query to computeRegisterLiveness()
Definition: MachineBasicBlock.h:586
llvm::Constant::getAllOnesValue
static Constant * getAllOnesValue(Type *Ty)
Get the all ones value.
Definition: Constants.cpp:163
llvm::X86InstrInfo::classifyLEAReg
bool classifyLEAReg(MachineInstr *MI, const MachineOperand &Src, unsigned LEAOpcode, bool AllowSP, unsigned &NewSrc, bool &isKill, bool &isUndef, MachineOperand &ImplicitOp) const
Definition: X86InstrInfo.cpp:1850
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, const MCInstrDesc &MCID)
Definition: MachineInstrBuilder.h:223
llvm::MachineBasicBlock::LQR_Live
Register is known to be live.
Definition: MachineBasicBlock.h:587
llvm::X86Subtarget::isPICStyleGOT
bool isPICStyleGOT() const
Definition: X86Subtarget.h:347
GetCondBranchFromCond
static unsigned GetCondBranchFromCond(XCore::CondCode CC)
Definition: XCoreInstrInfo.cpp:147
llvm::MachineOperand::getSubReg
unsigned getSubReg() const
Definition: MachineOperand.h:264
llvm::LiveVariables::getVarInfo
VarInfo & getVarInfo(unsigned RegIdx)
getVarInfo - Get (possibly creating) a VarInfo object for the given vreg.
Definition: LiveVariables.cpp:85
llvm::MCInstrDesc::getOperandConstraint
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specific constraint if it is set. Returns -1 if it is not set...
Definition: MCInstrDesc.h:156
llvm::MCInstrInfo::get
const MCInstrDesc & get(unsigned Opcode) const
Definition: MCInstrInfo.h:48
llvm::X86InstrInfo::InsertBranch
virtual unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, const SmallVectorImpl< MachineOperand > &Cond, DebugLoc DL) const
Definition: X86InstrInfo.cpp:2872
isHReg
static bool isHReg(unsigned Reg)
isHReg - Test if the given register is a physical h register.
Definition: X86InstrInfo.cpp:2971
llvm::MachineOperand::setIsKill
void setIsKill(bool Val=true)
Definition: MachineOperand.h:367
llvm::addFrameReference
static const MachineInstrBuilder & addFrameReference(const MachineInstrBuilder &MIB, int FI, int Offset=0, bool mem=true)
Definition: PPCInstrBuilder.h:33
llvm::MachineOperand::isRegMask
bool isRegMask() const
isRegMask - Tests if this is a MO_RegisterMask operand.
Definition: MachineOperand.h:248
llvm::MachineBasicBlock::findDebugLoc
DebugLoc findDebugLoc(instr_iterator MBBI)
isSafeToClobberEFLAGS
static bool isSafeToClobberEFLAGS(MachineBasicBlock &MBB, MachineBasicBlock::iterator I)
Definition: X86InstrInfo.cpp:1717
llvm::X86InstrInfo::isCoalescableExtInstr
virtual bool isCoalescableExtInstr(const MachineInstr &MI, unsigned &SrcReg, unsigned &DstReg, unsigned &SubIdx) const
Definition: X86InstrInfo.cpp:1453
llvm::LiveVariables::VarInfo::Kills
std::vector< MachineInstr * > Kills
Definition: LiveVariables.h:89
isRedundantFlagInstr
static bool isRedundantFlagInstr(MachineInstr *FlagI, unsigned SrcReg, unsigned SrcReg2, int ImmValue, MachineInstr *OI)
Definition: X86InstrInfo.cpp:3363
llvm::X86InstrInfo::areLoadsFromSameBasePtr
virtual bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, int64_t &Offset1, int64_t &Offset2) const
Definition: X86InstrInfo.cpp:4761
llvm::MCInst::setOpcode
void setOpcode(unsigned Op)
Definition: MCInst.h:157
llvm::X86InstrInfo::insertSelect
virtual void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, DebugLoc DL, unsigned DstReg, const SmallVectorImpl< MachineOperand > &Cond, unsigned TrueReg, unsigned FalseReg) const
Definition: X86InstrInfo.cpp:2957
llvm::MachineInstr::setDesc
void setDesc(const MCInstrDesc &tid)
Definition: MachineInstr.h:984
llvm::TargetMachine::getSubtarget
const STC & getSubtarget() const
Definition: Target/TargetMachine.h:132
llvm::MachineInstr::substituteRegister
void substituteRegister(unsigned FromReg, unsigned ToReg, unsigned SubIdx, const TargetRegisterInfo &RegInfo)
Definition: MachineInstr.cpp:1202
llvm::MachineFunction::CloneMachineInstr
MachineInstr * CloneMachineInstr(const MachineInstr *Orig)
Definition: MachineFunction.cpp:184
copyPhysRegOpcode_AVX512
static unsigned copyPhysRegOpcode_AVX512(unsigned &DestReg, unsigned &SrcReg)
Definition: X86InstrInfo.cpp:3019
llvm::MachineFrameInfo::getObjectAlignment
unsigned getObjectAlignment(int ObjectIdx) const
getObjectAlignment - Return the alignment of the specified stack object.
Definition: MachineFrameInfo.h:361
llvm::addRegReg
static const MachineInstrBuilder & addRegReg(const MachineInstrBuilder &MIB, unsigned Reg1, bool isKill1, unsigned Reg2, bool isKill2)
Definition: X86InstrBuilder.h:114
llvm::X86InstrInfo::isStoreToStackSlotPostFE
unsigned isStoreToStackSlotPostFE(const MachineInstr *MI, int &FrameIndex) const
Definition: X86InstrInfo.cpp:1604
llvm::X86InstrInfo::getGlobalBaseReg
unsigned getGlobalBaseReg(MachineFunction *MF) const
Definition: X86InstrInfo.cpp:5100
llvm::MachineInstr::killsRegister
bool killsRegister(unsigned Reg, const TargetRegisterInfo *TRI=NULL) const
Definition: MachineInstr.h:745
llvm::MachineFunction::getFrameInfo
MachineFrameInfo * getFrameInfo()
Definition: MachineFunction.h:174
getStoreRegOpcode
static unsigned getStoreRegOpcode(unsigned SrcReg, const TargetRegisterClass *RC, bool isStackAligned, TargetMachine &TM)
Definition: X86InstrInfo.cpp:3200
llvm::AnalysisUsage::setPreservesCFG
void setPreservesCFG()
Definition: Pass.cpp:249
llvm::MachineBasicBlock::remove
MachineInstr * remove(MachineInstr *I)
Definition: MachineBasicBlock.h:519
llvm::X86InstrInfo::copyPhysReg
virtual void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, DebugLoc DL, unsigned DestReg, unsigned SrcReg, bool KillSrc) const
Definition: X86InstrInfo.cpp:3035
llvm::dbgs
raw_ostream & dbgs()
dbgs - Return a circular-buffered debug stream.
Definition: Debug.cpp:101
llvm::Function::getAttributes
AttributeSet getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:170
llvm::MachineOperand::clobbersPhysReg
static bool clobbersPhysReg(const uint32_t *RegMask, unsigned PhysReg)
Definition: MachineOperand.h:461
llvm::X86InstrInfo::isLoadFromStackSlotPostFE
unsigned isLoadFromStackSlotPostFE(const MachineInstr *MI, int &FrameIndex) const
Definition: X86InstrInfo.cpp:1582
llvm::MachineInstr::isSafeToMove
bool isSafeToMove(const TargetInstrInfo *TII, AliasAnalysis *AA, bool &SawStore) const
Definition: MachineInstr.cpp:1228
llvm::X86Subtarget::isUnalignedMemAccessFast
bool isUnalignedMemAccessFast() const
Definition: X86Subtarget.h:295
llvm::MachineRegisterInfo::getUniqueVRegDef
MachineInstr * getUniqueVRegDef(unsigned Reg) const
Definition: MachineRegisterInfo.cpp:319
Align
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(DefaultAlign), cl::values(clEnumValN(DefaultAlign,"arm-default-align","Generate unaligned accesses only on hardware/OS ""combinations that are known to support them"), clEnumValN(StrictAlign,"arm-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"arm-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
llvm::X86Subtarget::hasCMov
bool hasCMov() const
Definition: X86Subtarget.h:257
llvm::MachineRegisterInfo::def_begin
def_iterator def_begin(unsigned RegNo) const
Definition: MachineRegisterInfo.h:237
llvm::X86InstrInfo::RemoveBranch
virtual unsigned RemoveBranch(MachineBasicBlock &MBB) const
Definition: X86InstrInfo.cpp:2851
llvm::X86TargetMachine::getFrameLowering
virtual const TargetFrameLowering * getFrameLowering() const
Definition: X86TargetMachine.h:47
llvm::X86InstrInfo::optimizeCompareInstr
virtual bool optimizeCompareInstr(MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2, int CmpMask, int CmpValue, const MachineRegisterInfo *MRI) const
Definition: X86InstrInfo.cpp:3492
llvm::MinAlign
uint64_t MinAlign(uint64_t A, uint64_t B)
Definition: MathExtras.h:535
lookup
static const uint16_t * lookup(unsigned opcode, unsigned domain)
Definition: X86InstrInfo.cpp:5181
FuseInst
static MachineInstr * FuseInst(MachineFunction &MF, unsigned Opcode, unsigned OpNo, const SmallVectorImpl< MachineOperand > &MOs, MachineInstr *MI, const TargetInstrInfo &TII)
Definition: X86InstrInfo.cpp:3899
llvm::TargetRegisterInfo::isPhysicalRegister
static bool isPhysicalRegister(unsigned Reg)
Definition: TargetRegisterInfo.h:280
llvm::X86InstrInfo::analyzeCompare
virtual bool analyzeCompare(const MachineInstr *MI, unsigned &SrcReg, unsigned &SrcReg2, int &CmpMask, int &CmpValue) const
Definition: X86InstrInfo.cpp:3286
llvm::X86InstrInfo::isSafeToMoveRegClassDefs
bool isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const
Definition: X86InstrInfo.cpp:5087
GetOppositeBranchCondition
static SPCC::CondCodes GetOppositeBranchCondition(SPCC::CondCodes CC)
Definition: SparcInstrInfo.cpp:89
llvm::MachineBasicBlock::isLiveIn
bool isLiveIn(unsigned Reg) const
Definition: MachineBasicBlock.cpp:340
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
Definition: MachineFunction.h:167
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:241
llvm::addOffset
static const MachineInstrBuilder & addOffset(const MachineInstrBuilder &MIB, int Offset)
Definition: X86InstrBuilder.h:98
llvm::MachineFunctionPass::getAnalysisUsage
virtual void getAnalysisUsage(AnalysisUsage &AU) const
Definition: MachineFunctionPass.cpp:36
llvm::MachineInstrBuilder::addExternalSymbol
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned char TargetFlags=0) const
Definition: MachineInstrBuilder.h:136
llvm::TargetOpcode::IMPLICIT_DEF
IMPLICIT_DEF - This is the MachineInstr-level equivalent of undef.
Definition: TargetOpcodes.h:52
llvm::X86InstrInfo::unfoldMemoryOperand
virtual bool unfoldMemoryOperand(MachineFunction &MF, MachineInstr *MI, unsigned Reg, bool UnfoldLoad, bool UnfoldStore, SmallVectorImpl< MachineInstr * > &NewMIs) const
Definition: X86InstrInfo.cpp:4511
llvm::X86InstrInfo::setExecutionDomain
void setExecutionDomain(MachineInstr *MI, unsigned Domain) const
Definition: X86InstrInfo.cpp:5207
llvm::MachineOperand::setReg
void setReg(unsigned Reg)
Definition: MachineInstr.cpp:49
llvm::MachineOperand::CreateImm
static MachineOperand CreateImm(int64_t Val)
Definition: MachineOperand.h:542
I
#define I(x, y, z)
Definition: MD5.cpp:54
N
#define N
llvm::MachineOperand::setSubReg
void setSubReg(unsigned subReg)
Definition: MachineOperand.h:339
llvm::TargetInstrInfo::canFoldMemoryOperand
virtual bool canFoldMemoryOperand(const MachineInstr *MI, const SmallVectorImpl< unsigned > &Ops) const
Definition: TargetInstrInfo.cpp:370
llvm::X86Subtarget::hasAVX512
bool hasAVX512() const
Definition: X86Subtarget.h:267
llvm::MachineFunction::getTarget
const TargetMachine & getTarget() const
Definition: MachineFunction.h:163
llvm::MachineBasicBlock::insert
instr_iterator insert(instr_iterator I, MachineInstr *M)
llvm::SelectionDAG::getMachineNode
MachineSDNode * getMachineNode(unsigned Opcode, SDLoc dl, EVT VT)
Definition: SelectionDAG.cpp:5411
llvm::X86InstrInfo::loadRegFromAddr
virtual void loadRegFromAddr(MachineFunction &MF, unsigned DestReg, SmallVectorImpl< MachineOperand > &Addr, const TargetRegisterClass *RC, MachineInstr::mmo_iterator MMOBegin, MachineInstr::mmo_iterator MMOEnd, SmallVectorImpl< MachineInstr * > &NewMIs) const
Definition: X86InstrInfo.cpp:3267
llvm::X86InstrInfo::isUnpredicatedTerminator
virtual bool isUnpredicatedTerminator(const MachineInstr *MI) const
Definition: X86InstrInfo.cpp:2696
llvm::MachineRegisterInfo::getVRegDef
MachineInstr * getVRegDef(unsigned Reg) const
Definition: MachineRegisterInfo.cpp:308
llvm::MachineOperand::getReg
unsigned getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:259
llvm::MachineFunction::extractStoreMemRefs
std::pair< MachineInstr::mmo_iterator, MachineInstr::mmo_iterator > extractStoreMemRefs(MachineInstr::mmo_iterator Begin, MachineInstr::mmo_iterator End)
Definition: MachineFunction.cpp:280
FuseTwoAddrInst
static MachineInstr * FuseTwoAddrInst(MachineFunction &MF, unsigned Opcode, const SmallVectorImpl< MachineOperand > &MOs, MachineInstr *MI, const TargetInstrInfo &TII)
Definition: X86InstrInfo.cpp:3871
llvm::MachineInstr::isCommutable
bool isCommutable(QueryType Type=IgnoreBundle) const
Definition: MachineInstr.h:502
getLoadRegOpcode
static unsigned getLoadRegOpcode(unsigned DestReg, const TargetRegisterClass *RC, bool isStackAligned, const TargetMachine &TM)
Definition: X86InstrInfo.cpp:3208
llvm::MachineRegisterInfo::def_end
static def_iterator def_end()
Definition: MachineRegisterInfo.h:240
llvm::DomTreeNodeBase::iterator
std::vector< DomTreeNodeBase< NodeT > * >::iterator iterator
Definition: Dominators.h:73
ReplaceableInstrsAVX2
static const uint16_t ReplaceableInstrsAVX2[][3]
Definition: X86InstrInfo.cpp:5160
llvm::X86InstrInfo::isReallyTriviallyReMaterializable
bool isReallyTriviallyReMaterializable(const MachineInstr *MI, AliasAnalysis *AA) const
Definition: X86InstrInfo.cpp:1636
llvm::X86InstrInfo::shouldScheduleLoadsNear
virtual bool shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, int64_t Offset1, int64_t Offset2, unsigned NumLoads) const
Definition: X86InstrInfo.cpp:4866
llvm::MachineBasicBlock::reverse_iterator
std::reverse_iterator< iterator > reverse_iterator
Definition: MachineBasicBlock.h:216
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
Definition: MachineInstrBuilder.h:98
llvm::VectorType::get
static VectorType * get(Type *ElementType, unsigned NumElements)
Definition: Type.cpp:706
llvm::MCInstrDesc::getNumOperands
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction. Note that variadic (isVari...
Definition: MCInstrDesc.h:190
hasLiveCondCodeDef
static bool hasLiveCondCodeDef(MachineInstr *MI)
Definition: X86InstrInfo.cpp:1818
llvm::MachineInstrBuilder::addOperand
const MachineInstrBuilder & addOperand(const MachineOperand &MO) const
Definition: MachineInstrBuilder.h:166
llvm::MachineFunction::iterator
BasicBlockListType::iterator iterator
Definition: MachineFunction.h:303
llvm::prior
ItTy prior(ItTy it, Dist n)
Definition: STLExtras.h:167
DEBUG
#define DEBUG(X)
Definition: Debug.h:97
llvm::X86InstrInfo::optimizeLoadInstr
virtual MachineInstr * optimizeLoadInstr(MachineInstr *MI, const MachineRegisterInfo *MRI, unsigned &FoldAsLoadDefReg, MachineInstr *&DefMI) const
Definition: X86InstrInfo.cpp:3744
llvm::MachineInstr::addRegisterKilled
bool addRegisterKilled(unsigned IncomingReg, const TargetRegisterInfo *RegInfo, bool AddIfNotFound=false)
Definition: MachineInstr.cpp:1635
llvm::X86InstrInfo::reMaterialize
void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg, unsigned SubIdx, const MachineInstr *Orig, const TargetRegisterInfo &TRI) const
Definition: X86InstrInfo.cpp:1795
getCMovFromCond
static unsigned getCMovFromCond(X86::CondCode CC, unsigned RegBytes, bool HasMemoryOperand)
Definition: X86InstrInfo.cpp:2649
llvm::TargetRegisterClass::vt_begin
vt_iterator vt_begin() const
Definition: TargetRegisterInfo.h:113
llvm::X86TargetMachine::getSubtargetImpl
virtual const X86Subtarget * getSubtargetImpl() const
Definition: X86TargetMachine.h:53
X86OpTblEntry::Flags
uint16_t Flags
Definition: X86InstrInfo.cpp:92
MRI
const MCRegisterInfo & MRI
Definition: MCModuleYAML.cpp:56
isFrameStoreOpcode
static bool isFrameStoreOpcode(int Opcode)
Definition: X86InstrInfo.cpp:1545
llvm::MachineOperand::readsReg
bool readsReg() const
Definition: MachineOperand.h:326
raw_ostream.h
lookupAVX2
static const uint16_t * lookupAVX2(unsigned opcode, unsigned domain)
Definition: X86InstrInfo.cpp:5188
X86OpTblEntry::MemOp
uint16_t MemOp
Definition: X86InstrInfo.cpp:91
llvm::X86InstrInfo::convertToThreeAddress
virtual MachineInstr * convertToThreeAddress(MachineFunction::iterator &MFI, MachineBasicBlock::iterator &MBBI, LiveVariables *LV) const
Definition: X86InstrInfo.cpp:2044
llvm::X86InstrInfo::ReverseBranchCondition
virtual bool ReverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const
Definition: X86InstrInfo.cpp:5077
getSwappedCondition
static X86::CondCode getSwappedCondition(X86::CondCode CC)
Definition: X86InstrInfo.cpp:2604
llvm::MachineBasicBlock::isLayoutSuccessor
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Definition: MachineBasicBlock.cpp:623
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Definition: MachineInstrBuilder.h:64
llvm::X86InstrInfo::X86InstrInfo
X86InstrInfo(X86TargetMachine &tm)
Definition: X86InstrInfo.cpp:98
llvm::MachineRegisterInfo::defusechain_iterator
Definition: MachineRegisterInfo.h:204
llvm::createCleanupLocalDynamicTLSPass
FunctionPass * createCleanupLocalDynamicTLSPass()
Definition: X86InstrInfo.cpp:5492
Expand2AddrUndef
static bool Expand2AddrUndef(MachineInstrBuilder &MIB, const MCInstrDesc &Desc)
Definition: X86InstrInfo.cpp:3821
llvm::MachineOperand::CreateFI
static MachineOperand CreateFI(int Idx)
Definition: MachineOperand.h:590
llvm::MachineFrameInfo::getObjectSize
int64_t getObjectSize(int ObjectIdx) const
Definition: MachineFrameInfo.h:347
llvm::MachineConstantPool::getConstantPoolIndex
unsigned getConstantPoolIndex(const Constant *C, unsigned Alignment)
Definition: MachineFunction.cpp:865
llvm::MachineInstr::modifiesRegister
bool modifiesRegister(unsigned Reg, const TargetRegisterInfo *TRI) const
Definition: MachineInstr.h:760
llvm::MachineInstr::isBarrier
bool isBarrier(QueryType Type=AnyInBundle) const
Definition: MachineInstr.h:356
llvm::X86InstrInfo::getPartialRegUpdateClearance
unsigned getPartialRegUpdateClearance(const MachineInstr *MI, unsigned OpNum, const TargetRegisterInfo *TRI) const
Definition: X86InstrInfo.cpp:4089
llvm::X86Subtarget::hasAVX
bool hasAVX() const
Definition: X86Subtarget.h:265
llvm::EVT::getSimpleVT
MVT getSimpleVT() const
Definition: ValueTypes.h:749
llvm::MachineInstr::getDebugLoc
DebugLoc getDebugLoc() const
Definition: MachineInstr.h:244
llvm::X86TargetMachine::getInstrInfo
virtual const X86InstrInfo * getInstrInfo() const
Definition: X86TargetMachine.h:44
NoFusing
static cl::opt< bool > NoFusing("disable-spill-fusing", cl::desc("Disable fusing of spill code into instructions"))
llvm::SDNode::isMachineOpcode
bool isMachineOpcode() const
Definition: SelectionDAGNodes.h:384
llvm::SDNode::getMachineOpcode
unsigned getMachineOpcode() const
Definition: SelectionDAGNodes.h:389
llvm::MachineInstr::memoperands_begin
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:291

Generated on Tue Mar 18 2014 10:39:24 for LLVM by Doxygen1.8.6
Copyright © 2003-2013 University of Illinois at Urbana-Champaign. All Rights Reserved.