clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizeDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 1 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Analysis -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ASMParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/BinaryFormat -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitstream -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /include/llvm/CodeGen -I /include/llvm/CodeGen/PBQP -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Coroutines -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData/Coverage -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/CodeView -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/DWARF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/MSF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/PDB -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Demangle -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/JITLink -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/Orc -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenACC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenMP -I /include/llvm/CodeGen/GlobalISel -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IRReader -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/LTO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Linker -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC/MCParser -I /include/llvm/CodeGen/MIRParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Object -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Option -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Passes -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Scalar -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ADT -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Support -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/Symbolize -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Target -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Utils -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Vectorize -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/IPO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include -I /usr/src/gnu/usr.bin/clang/libLLVM/../include -I /usr/src/gnu/usr.bin/clang/libLLVM/obj -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include -D NDEBUG -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D LLVM_PREFIX="/usr" -D PIC -internal-isystem /usr/include/c++/v1 -internal-isystem /usr/local/lib/clang/13.0.0/include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -ferror-limit 19 -fvisibility-inlines-hidden -fwrapv -D_RET_PROTECTOR -ret-protector -fno-rtti -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/ben/Projects/vmm/scan-build/2022-01-12-194120-40624-1 -x c++ /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
1 | |
2 | |
3 | |
4 | |
5 | |
6 | |
7 | |
8 | |
9 | |
10 | |
11 | |
12 | |
13 | #include "llvm/ADT/APFloat.h" |
14 | #include "llvm/ADT/APInt.h" |
15 | #include "llvm/ADT/ArrayRef.h" |
16 | #include "llvm/ADT/SetVector.h" |
17 | #include "llvm/ADT/SmallPtrSet.h" |
18 | #include "llvm/ADT/SmallSet.h" |
19 | #include "llvm/ADT/SmallVector.h" |
20 | #include "llvm/Analysis/TargetLibraryInfo.h" |
21 | #include "llvm/CodeGen/ISDOpcodes.h" |
22 | #include "llvm/CodeGen/MachineFunction.h" |
23 | #include "llvm/CodeGen/MachineJumpTableInfo.h" |
24 | #include "llvm/CodeGen/MachineMemOperand.h" |
25 | #include "llvm/CodeGen/RuntimeLibcalls.h" |
26 | #include "llvm/CodeGen/SelectionDAG.h" |
27 | #include "llvm/CodeGen/SelectionDAGNodes.h" |
28 | #include "llvm/CodeGen/TargetFrameLowering.h" |
29 | #include "llvm/CodeGen/TargetLowering.h" |
30 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
31 | #include "llvm/CodeGen/ValueTypes.h" |
32 | #include "llvm/IR/CallingConv.h" |
33 | #include "llvm/IR/Constants.h" |
34 | #include "llvm/IR/DataLayout.h" |
35 | #include "llvm/IR/DerivedTypes.h" |
36 | #include "llvm/IR/Function.h" |
37 | #include "llvm/IR/Metadata.h" |
38 | #include "llvm/IR/Type.h" |
39 | #include "llvm/Support/Casting.h" |
40 | #include "llvm/Support/Compiler.h" |
41 | #include "llvm/Support/Debug.h" |
42 | #include "llvm/Support/ErrorHandling.h" |
43 | #include "llvm/Support/MachineValueType.h" |
44 | #include "llvm/Support/MathExtras.h" |
45 | #include "llvm/Support/raw_ostream.h" |
46 | #include "llvm/Target/TargetMachine.h" |
47 | #include "llvm/Target/TargetOptions.h" |
48 | #include <algorithm> |
49 | #include <cassert> |
50 | #include <cstdint> |
51 | #include <tuple> |
52 | #include <utility> |
53 | |
54 | using namespace llvm; |
55 | |
56 | #define DEBUG_TYPE "legalizedag" |
57 | |
58 | namespace { |
59 | |
60 | |
61 | |
62 | struct FloatSignAsInt { |
63 | EVT FloatVT; |
64 | SDValue Chain; |
65 | SDValue FloatPtr; |
66 | SDValue IntPtr; |
67 | MachinePointerInfo IntPointerInfo; |
68 | MachinePointerInfo FloatPointerInfo; |
69 | SDValue IntValue; |
70 | APInt SignMask; |
71 | uint8_t SignBit; |
72 | }; |
73 | |
74 | |
75 | |
76 | |
77 | |
78 | |
79 | |
80 | |
81 | |
82 | |
83 | |
84 | |
85 | class SelectionDAGLegalize { |
86 | const TargetMachine &TM; |
87 | const TargetLowering &TLI; |
88 | SelectionDAG &DAG; |
89 | |
90 | |
91 | |
92 | SmallPtrSetImpl<SDNode *> &LegalizedNodes; |
93 | |
94 | |
95 | SmallSetVector<SDNode *, 16> *UpdatedNodes; |
96 | |
97 | EVT getSetCCResultType(EVT VT) const { |
98 | return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); |
99 | } |
100 | |
101 | |
102 | |
103 | public: |
104 | SelectionDAGLegalize(SelectionDAG &DAG, |
105 | SmallPtrSetImpl<SDNode *> &LegalizedNodes, |
106 | SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr) |
107 | : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG), |
108 | LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {} |
109 | |
110 | |
111 | void LegalizeOp(SDNode *Node); |
112 | |
113 | private: |
114 | SDValue OptimizeFloatStore(StoreSDNode *ST); |
115 | |
116 | void LegalizeLoadOps(SDNode *Node); |
117 | void LegalizeStoreOps(SDNode *Node); |
118 | |
119 | |
120 | |
121 | |
122 | |
123 | SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, |
124 | const SDLoc &dl); |
125 | SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, |
126 | const SDLoc &dl); |
127 | |
128 | |
129 | |
130 | |
131 | |
132 | SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl, |
133 | SDValue N1, SDValue N2, |
134 | ArrayRef<int> Mask) const; |
135 | |
136 | SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); |
137 | |
138 | void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC, |
139 | SmallVectorImpl<SDValue> &Results); |
140 | void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, |
141 | RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, |
142 | RTLIB::Libcall Call_F128, |
143 | RTLIB::Libcall Call_PPCF128, |
144 | SmallVectorImpl<SDValue> &Results); |
145 | SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, |
146 | RTLIB::Libcall Call_I8, |
147 | RTLIB::Libcall Call_I16, |
148 | RTLIB::Libcall Call_I32, |
149 | RTLIB::Libcall Call_I64, |
150 | RTLIB::Libcall Call_I128); |
151 | void ExpandArgFPLibCall(SDNode *Node, |
152 | RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, |
153 | RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, |
154 | RTLIB::Libcall Call_PPCF128, |
155 | SmallVectorImpl<SDValue> &Results); |
156 | void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); |
157 | void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); |
158 | |
159 | SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, |
160 | const SDLoc &dl); |
161 | SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, |
162 | const SDLoc &dl, SDValue ChainIn); |
163 | SDValue ExpandBUILD_VECTOR(SDNode *Node); |
164 | SDValue ExpandSPLAT_VECTOR(SDNode *Node); |
165 | SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); |
166 | void ExpandDYNAMIC_STACKALLOC(SDNode *Node, |
167 | SmallVectorImpl<SDValue> &Results); |
168 | void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL, |
169 | SDValue Value) const; |
170 | SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL, |
171 | SDValue NewIntValue) const; |
172 | SDValue ExpandFCOPYSIGN(SDNode *Node) const; |
173 | SDValue ExpandFABS(SDNode *Node) const; |
174 | SDValue ExpandFNEG(SDNode *Node) const; |
175 | SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain); |
176 | void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl, |
177 | SmallVectorImpl<SDValue> &Results); |
178 | void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl, |
179 | SmallVectorImpl<SDValue> &Results); |
180 | SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl); |
181 | |
182 | SDValue ExpandPARITY(SDValue Op, const SDLoc &dl); |
183 | |
184 | SDValue ExpandExtractFromVectorThroughStack(SDValue Op); |
185 | SDValue ExpandInsertToVectorThroughStack(SDValue Op); |
186 | SDValue ExpandVectorBuildThroughStack(SDNode* Node); |
187 | |
188 | SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); |
189 | SDValue ExpandConstant(ConstantSDNode *CP); |
190 | |
191 | |
192 | bool ExpandNode(SDNode *Node); |
193 | void ConvertNodeToLibcall(SDNode *Node); |
194 | void PromoteNode(SDNode *Node); |
195 | |
196 | public: |
197 | |
198 | |
199 | void ReplacedNode(SDNode *N) { |
200 | LegalizedNodes.erase(N); |
201 | if (UpdatedNodes) |
202 | UpdatedNodes->insert(N); |
203 | } |
204 | |
205 | void ReplaceNode(SDNode *Old, SDNode *New) { |
206 | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); |
207 | dbgs() << " with: "; New->dump(&DAG)); |
208 | |
209 | assert(Old->getNumValues() == New->getNumValues() && |
210 | "Replacing one node with another that produces a different number " |
211 | "of values!"); |
212 | DAG.ReplaceAllUsesWith(Old, New); |
213 | if (UpdatedNodes) |
214 | UpdatedNodes->insert(New); |
215 | ReplacedNode(Old); |
216 | } |
217 | |
218 | void ReplaceNode(SDValue Old, SDValue New) { |
219 | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); |
220 | dbgs() << " with: "; New->dump(&DAG)); |
221 | |
222 | DAG.ReplaceAllUsesWith(Old, New); |
223 | if (UpdatedNodes) |
224 | UpdatedNodes->insert(New.getNode()); |
225 | ReplacedNode(Old.getNode()); |
226 | } |
227 | |
228 | void ReplaceNode(SDNode *Old, const SDValue *New) { |
229 | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG)); |
230 | |
231 | DAG.ReplaceAllUsesWith(Old, New); |
232 | for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) { |
233 | LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: "); |
234 | New[i]->dump(&DAG)); |
235 | if (UpdatedNodes) |
236 | UpdatedNodes->insert(New[i].getNode()); |
237 | } |
238 | ReplacedNode(Old); |
239 | } |
240 | |
241 | void ReplaceNodeWithValue(SDValue Old, SDValue New) { |
242 | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); |
243 | dbgs() << " with: "; New->dump(&DAG)); |
244 | |
245 | DAG.ReplaceAllUsesOfValueWith(Old, New); |
246 | if (UpdatedNodes) |
247 | UpdatedNodes->insert(New.getNode()); |
248 | ReplacedNode(Old.getNode()); |
249 | } |
250 | }; |
251 | |
252 | } |
253 | |
254 | |
255 | |
256 | |
257 | |
258 | SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType( |
259 | EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, |
260 | ArrayRef<int> Mask) const { |
261 | unsigned NumMaskElts = VT.getVectorNumElements(); |
262 | unsigned NumDestElts = NVT.getVectorNumElements(); |
263 | unsigned NumEltsGrowth = NumDestElts / NumMaskElts; |
264 | |
265 | assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); |
266 | |
267 | if (NumEltsGrowth == 1) |
268 | return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask); |
269 | |
270 | SmallVector<int, 8> NewMask; |
271 | for (unsigned i = 0; i != NumMaskElts; ++i) { |
272 | int Idx = Mask[i]; |
273 | for (unsigned j = 0; j != NumEltsGrowth; ++j) { |
274 | if (Idx < 0) |
275 | NewMask.push_back(-1); |
276 | else |
277 | NewMask.push_back(Idx * NumEltsGrowth + j); |
278 | } |
279 | } |
280 | assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); |
281 | assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"); |
282 | return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask); |
283 | } |
284 | |
285 | |
286 | |
287 | SDValue |
288 | SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { |
289 | bool Extend = false; |
290 | SDLoc dl(CFP); |
291 | |
292 | |
293 | |
294 | |
295 | |
296 | |
297 | |
298 | EVT VT = CFP->getValueType(0); |
299 | ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); |
300 | if (!UseCP) { |
301 | assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"); |
302 | return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl, |
303 | (VT == MVT::f64) ? MVT::i64 : MVT::i32); |
304 | } |
305 | |
306 | APFloat APF = CFP->getValueAPF(); |
307 | EVT OrigVT = VT; |
308 | EVT SVT = VT; |
309 | |
310 | |
311 | |
312 | if (!APF.isSignaling()) { |
313 | while (SVT != MVT::f32 && SVT != MVT::f16) { |
314 | SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); |
315 | if (ConstantFPSDNode::isValueValidForType(SVT, APF) && |
316 | |
317 | |
318 | TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && |
319 | TLI.ShouldShrinkFPConstant(OrigVT)) { |
320 | Type *SType = SVT.getTypeForEVT(*DAG.getContext()); |
321 | LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); |
322 | VT = SVT; |
323 | Extend = true; |
324 | } |
325 | } |
326 | } |
327 | |
328 | SDValue CPIdx = |
329 | DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout())); |
330 | Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); |
331 | if (Extend) { |
332 | SDValue Result = DAG.getExtLoad( |
333 | ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx, |
334 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT, |
335 | Alignment); |
336 | return Result; |
337 | } |
338 | SDValue Result = DAG.getLoad( |
339 | OrigVT, dl, DAG.getEntryNode(), CPIdx, |
340 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); |
341 | return Result; |
342 | } |
343 | |
344 | |
345 | SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) { |
346 | SDLoc dl(CP); |
347 | EVT VT = CP->getValueType(0); |
348 | SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(), |
349 | TLI.getPointerTy(DAG.getDataLayout())); |
350 | Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); |
351 | SDValue Result = DAG.getLoad( |
352 | VT, dl, DAG.getEntryNode(), CPIdx, |
353 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); |
354 | return Result; |
355 | } |
356 | |
357 | |
358 | |
359 | |
360 | |
361 | SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec, |
362 | SDValue Val, |
363 | SDValue Idx, |
364 | const SDLoc &dl) { |
365 | SDValue Tmp1 = Vec; |
366 | SDValue Tmp2 = Val; |
367 | SDValue Tmp3 = Idx; |
368 | |
369 | |
370 | |
371 | |
372 | |
373 | |
374 | |
375 | EVT VT = Tmp1.getValueType(); |
376 | EVT EltVT = VT.getVectorElementType(); |
377 | SDValue StackPtr = DAG.CreateStackTemporary(VT); |
378 | |
379 | int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); |
380 | |
381 | |
382 | SDValue Ch = DAG.getStore( |
383 | DAG.getEntryNode(), dl, Tmp1, StackPtr, |
384 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); |
385 | |
386 | SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3); |
387 | |
388 | |
389 | Ch = DAG.getTruncStore( |
390 | Ch, dl, Tmp2, StackPtr2, |
391 | MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT); |
392 | |
393 | return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack( |
394 | DAG.getMachineFunction(), SPFI)); |
395 | } |
396 | |
397 | SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, |
398 | SDValue Idx, |
399 | const SDLoc &dl) { |
400 | if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) { |
401 | |
402 | |
403 | |
404 | EVT EltVT = Vec.getValueType().getVectorElementType(); |
405 | if (Val.getValueType() == EltVT || |
406 | (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) { |
407 | SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, |
408 | Vec.getValueType(), Val); |
409 | |
410 | unsigned NumElts = Vec.getValueType().getVectorNumElements(); |
411 | |
412 | |
413 | |
414 | SmallVector<int, 8> ShufOps; |
415 | for (unsigned i = 0; i != NumElts; ++i) |
416 | ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); |
417 | |
418 | return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps); |
419 | } |
420 | } |
421 | return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); |
422 | } |
423 | |
424 | SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { |
425 | if (!ISD::isNormalStore(ST)) |
426 | return SDValue(); |
427 | |
428 | LLVM_DEBUG(dbgs() << "Optimizing float store operations\n"); |
429 | |
430 | |
431 | |
432 | |
433 | |
434 | |
435 | SDValue Chain = ST->getChain(); |
436 | SDValue Ptr = ST->getBasePtr(); |
437 | SDValue Value = ST->getValue(); |
438 | MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); |
439 | AAMDNodes AAInfo = ST->getAAInfo(); |
440 | SDLoc dl(ST); |
441 | |
442 | |
443 | if (Value.getOpcode() == ISD::TargetConstantFP) |
444 | return SDValue(); |
445 | |
446 | if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) { |
447 | if (CFP->getValueType(0) == MVT::f32 && |
448 | TLI.isTypeLegal(MVT::i32)) { |
449 | SDValue Con = DAG.getConstant(CFP->getValueAPF(). |
450 | bitcastToAPInt().zextOrTrunc(32), |
451 | SDLoc(CFP), MVT::i32); |
452 | return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), |
453 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
454 | } |
455 | |
456 | if (CFP->getValueType(0) == MVT::f64) { |
457 | |
458 | if (TLI.isTypeLegal(MVT::i64)) { |
459 | SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). |
460 | zextOrTrunc(64), SDLoc(CFP), MVT::i64); |
461 | return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), |
462 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
463 | } |
464 | |
465 | if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) { |
466 | |
467 | |
468 | |
469 | const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt(); |
470 | SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32); |
471 | SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32); |
472 | if (DAG.getDataLayout().isBigEndian()) |
473 | std::swap(Lo, Hi); |
474 | |
475 | Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), |
476 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
477 | Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(4), dl); |
478 | Hi = DAG.getStore(Chain, dl, Hi, Ptr, |
479 | ST->getPointerInfo().getWithOffset(4), |
480 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
481 | |
482 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
483 | } |
484 | } |
485 | } |
486 | return SDValue(); |
487 | } |
488 | |
489 | void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { |
490 | StoreSDNode *ST = cast<StoreSDNode>(Node); |
| 1 | 'Node' is a 'StoreSDNode' | |
|
491 | SDValue Chain = ST->getChain(); |
492 | SDValue Ptr = ST->getBasePtr(); |
493 | SDLoc dl(Node); |
494 | |
495 | MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); |
496 | AAMDNodes AAInfo = ST->getAAInfo(); |
497 | |
498 | if (!ST->isTruncatingStore()) { |
| 2 | | Assuming the condition is false | |
|
| |
499 | LLVM_DEBUG(dbgs() << "Legalizing store operation\n"); |
500 | if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) { |
501 | ReplaceNode(ST, OptStore); |
502 | return; |
503 | } |
504 | |
505 | SDValue Value = ST->getValue(); |
506 | MVT VT = Value.getSimpleValueType(); |
507 | switch (TLI.getOperationAction(ISD::STORE, VT)) { |
508 | default: llvm_unreachable("This action is not supported yet!"); |
509 | case TargetLowering::Legal: { |
510 | |
511 | |
512 | EVT MemVT = ST->getMemoryVT(); |
513 | const DataLayout &DL = DAG.getDataLayout(); |
514 | if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT, |
515 | *ST->getMemOperand())) { |
516 | LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n"); |
517 | SDValue Result = TLI.expandUnalignedStore(ST, DAG); |
518 | ReplaceNode(SDValue(ST, 0), Result); |
519 | } else |
520 | LLVM_DEBUG(dbgs() << "Legal store\n"); |
521 | break; |
522 | } |
523 | case TargetLowering::Custom: { |
524 | LLVM_DEBUG(dbgs() << "Trying custom lowering\n"); |
525 | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); |
526 | if (Res && Res != SDValue(Node, 0)) |
527 | ReplaceNode(SDValue(Node, 0), Res); |
528 | return; |
529 | } |
530 | case TargetLowering::Promote: { |
531 | MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); |
532 | assert(NVT.getSizeInBits() == VT.getSizeInBits() && |
533 | "Can only promote stores to same size type"); |
534 | Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); |
535 | SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), |
536 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
537 | ReplaceNode(SDValue(Node, 0), Result); |
538 | break; |
539 | } |
540 | } |
541 | return; |
542 | } |
543 | |
544 | LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n"); |
| 4 | | Loop condition is false. Exiting loop | |
|
545 | SDValue Value = ST->getValue(); |
546 | EVT StVT = ST->getMemoryVT(); |
547 | TypeSize StWidth = StVT.getSizeInBits(); |
548 | TypeSize StSize = StVT.getStoreSizeInBits(); |
549 | auto &DL = DAG.getDataLayout(); |
550 | |
551 | if (StWidth != StSize) { |
| 5 | | Calling 'UnivariateLinearPolyBase::operator!=' | |
|
| 12 | | Returning from 'UnivariateLinearPolyBase::operator!=' | |
|
| |
552 | |
553 | |
554 | |
555 | EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedSize()); |
556 | Value = DAG.getZeroExtendInReg(Value, dl, StVT); |
557 | SDValue Result = |
558 | DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT, |
559 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
560 | ReplaceNode(SDValue(Node, 0), Result); |
561 | } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedSize())) { |
| 14 | | Calling 'EVT::isVector' | |
|
| 17 | | Returning from 'EVT::isVector' | |
|
| 18 | | Assuming the condition is true | |
|
| 19 | | Calling 'isPowerOf2_64' | |
|
| 22 | | Returning from 'isPowerOf2_64' | |
|
| |
562 | |
563 | assert(!StVT.isVector() && "Unsupported truncstore!"); |
564 | unsigned StWidthBits = StWidth.getFixedSize(); |
565 | unsigned LogStWidth = Log2_32(StWidthBits); |
| |
| 26 | | Returning from 'Log2_32' | |
|
| 27 | | 'LogStWidth' initialized to 4294967295 | |
|
566 | assert(LogStWidth < 32); |
567 | unsigned RoundWidth = 1 << LogStWidth; |
| 28 | | The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int' |
|
568 | assert(RoundWidth < StWidthBits); |
569 | unsigned ExtraWidth = StWidthBits - RoundWidth; |
570 | assert(ExtraWidth < RoundWidth); |
571 | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && |
572 | "Store size not an integral number of bytes!"); |
573 | EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); |
574 | EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); |
575 | SDValue Lo, Hi; |
576 | unsigned IncrementSize; |
577 | |
578 | if (DL.isLittleEndian()) { |
579 | |
580 | |
581 | Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), |
582 | RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo); |
583 | |
584 | |
585 | IncrementSize = RoundWidth / 8; |
586 | Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); |
587 | Hi = DAG.getNode( |
588 | ISD::SRL, dl, Value.getValueType(), Value, |
589 | DAG.getConstant(RoundWidth, dl, |
590 | TLI.getShiftAmountTy(Value.getValueType(), DL))); |
591 | Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, |
592 | ST->getPointerInfo().getWithOffset(IncrementSize), |
593 | ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo); |
594 | } else { |
595 | |
596 | |
597 | |
598 | Hi = DAG.getNode( |
599 | ISD::SRL, dl, Value.getValueType(), Value, |
600 | DAG.getConstant(ExtraWidth, dl, |
601 | TLI.getShiftAmountTy(Value.getValueType(), DL))); |
602 | Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT, |
603 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
604 | |
605 | |
606 | IncrementSize = RoundWidth / 8; |
607 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
608 | DAG.getConstant(IncrementSize, dl, |
609 | Ptr.getValueType())); |
610 | Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, |
611 | ST->getPointerInfo().getWithOffset(IncrementSize), |
612 | ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo); |
613 | } |
614 | |
615 | |
616 | SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
617 | ReplaceNode(SDValue(Node, 0), Result); |
618 | } else { |
619 | switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { |
620 | default: llvm_unreachable("This action is not supported yet!"); |
621 | case TargetLowering::Legal: { |
622 | EVT MemVT = ST->getMemoryVT(); |
623 | |
624 | |
625 | if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT, |
626 | *ST->getMemOperand())) { |
627 | SDValue Result = TLI.expandUnalignedStore(ST, DAG); |
628 | ReplaceNode(SDValue(ST, 0), Result); |
629 | } |
630 | break; |
631 | } |
632 | case TargetLowering::Custom: { |
633 | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); |
634 | if (Res && Res != SDValue(Node, 0)) |
635 | ReplaceNode(SDValue(Node, 0), Res); |
636 | return; |
637 | } |
638 | case TargetLowering::Expand: |
639 | assert(!StVT.isVector() && |
640 | "Vector Stores are handled in LegalizeVectorOps"); |
641 | |
642 | SDValue Result; |
643 | |
644 | |
645 | if (TLI.isTypeLegal(StVT)) { |
646 | Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); |
647 | Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), |
648 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
649 | } else { |
650 | |
651 | |
652 | Value = DAG.getNode(ISD::TRUNCATE, dl, |
653 | TLI.getTypeToTransformTo(*DAG.getContext(), StVT), |
654 | Value); |
655 | Result = |
656 | DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT, |
657 | ST->getOriginalAlign(), MMOFlags, AAInfo); |
658 | } |
659 | |
660 | ReplaceNode(SDValue(Node, 0), Result); |
661 | break; |
662 | } |
663 | } |
664 | } |
665 | |
666 | void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { |
667 | LoadSDNode *LD = cast<LoadSDNode>(Node); |
668 | SDValue Chain = LD->getChain(); |
669 | SDValue Ptr = LD->getBasePtr(); |
670 | SDValue Value; |
671 | SDLoc dl(Node); |
672 | |
673 | ISD::LoadExtType ExtType = LD->getExtensionType(); |
674 | if (ExtType == ISD::NON_EXTLOAD) { |
675 | LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n"); |
676 | MVT VT = Node->getSimpleValueType(0); |
677 | SDValue RVal = SDValue(Node, 0); |
678 | SDValue RChain = SDValue(Node, 1); |
679 | |
680 | switch (TLI.getOperationAction(Node->getOpcode(), VT)) { |
681 | default: llvm_unreachable("This action is not supported yet!"); |
682 | case TargetLowering::Legal: { |
683 | EVT MemVT = LD->getMemoryVT(); |
684 | const DataLayout &DL = DAG.getDataLayout(); |
685 | |
686 | |
687 | if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT, |
688 | *LD->getMemOperand())) { |
689 | std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG); |
690 | } |
691 | break; |
692 | } |
693 | case TargetLowering::Custom: |
694 | if (SDValue Res = TLI.LowerOperation(RVal, DAG)) { |
695 | RVal = Res; |
696 | RChain = Res.getValue(1); |
697 | } |
698 | break; |
699 | |
700 | case TargetLowering::Promote: { |
701 | MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); |
702 | assert(NVT.getSizeInBits() == VT.getSizeInBits() && |
703 | "Can only promote loads to same size type"); |
704 | |
705 | SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand()); |
706 | RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); |
707 | RChain = Res.getValue(1); |
708 | break; |
709 | } |
710 | } |
711 | if (RChain.getNode() != Node) { |
712 | assert(RVal.getNode() != Node && "Load must be completely replaced"); |
713 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); |
714 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); |
715 | if (UpdatedNodes) { |
716 | UpdatedNodes->insert(RVal.getNode()); |
717 | UpdatedNodes->insert(RChain.getNode()); |
718 | } |
719 | ReplacedNode(Node); |
720 | } |
721 | return; |
722 | } |
723 | |
724 | LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n"); |
725 | EVT SrcVT = LD->getMemoryVT(); |
726 | TypeSize SrcWidth = SrcVT.getSizeInBits(); |
727 | MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); |
728 | AAMDNodes AAInfo = LD->getAAInfo(); |
729 | |
730 | if (SrcWidth != SrcVT.getStoreSizeInBits() && |
731 | |
732 | |
733 | |
734 | |
735 | |
736 | |
737 | |
738 | (SrcVT != MVT::i1 || |
739 | TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == |
740 | TargetLowering::Promote)) { |
741 | |
742 | |
743 | unsigned NewWidth = SrcVT.getStoreSizeInBits(); |
744 | EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); |
745 | SDValue Ch; |
746 | |
747 | |
748 | |
749 | |
750 | ISD::LoadExtType NewExtType = |
751 | ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; |
752 | |
753 | SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), |
754 | Chain, Ptr, LD->getPointerInfo(), NVT, |
755 | LD->getOriginalAlign(), MMOFlags, AAInfo); |
756 | |
757 | Ch = Result.getValue(1); |
758 | |
759 | if (ExtType == ISD::SEXTLOAD) |
760 | |
761 | Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, |
762 | Result.getValueType(), |
763 | Result, DAG.getValueType(SrcVT)); |
764 | else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) |
765 | |
766 | Result = DAG.getNode(ISD::AssertZext, dl, |
767 | Result.getValueType(), Result, |
768 | DAG.getValueType(SrcVT)); |
769 | |
770 | Value = Result; |
771 | Chain = Ch; |
772 | } else if (!isPowerOf2_64(SrcWidth.getKnownMinSize())) { |
773 | |
774 | assert(!SrcVT.isVector() && "Unsupported extload!"); |
775 | unsigned SrcWidthBits = SrcWidth.getFixedSize(); |
776 | unsigned LogSrcWidth = Log2_32(SrcWidthBits); |
777 | assert(LogSrcWidth < 32); |
778 | unsigned RoundWidth = 1 << LogSrcWidth; |
779 | assert(RoundWidth < SrcWidthBits); |
780 | unsigned ExtraWidth = SrcWidthBits - RoundWidth; |
781 | assert(ExtraWidth < RoundWidth); |
782 | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && |
783 | "Load size not an integral number of bytes!"); |
784 | EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); |
785 | EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); |
786 | SDValue Lo, Hi, Ch; |
787 | unsigned IncrementSize; |
788 | auto &DL = DAG.getDataLayout(); |
789 | |
790 | if (DL.isLittleEndian()) { |
791 | |
792 | |
793 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, |
794 | LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(), |
795 | MMOFlags, AAInfo); |
796 | |
797 | |
798 | IncrementSize = RoundWidth / 8; |
799 | Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); |
800 | Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, |
801 | LD->getPointerInfo().getWithOffset(IncrementSize), |
802 | ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo); |
803 | |
804 | |
805 | |
806 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
807 | Hi.getValue(1)); |
808 | |
809 | |
810 | Hi = DAG.getNode( |
811 | ISD::SHL, dl, Hi.getValueType(), Hi, |
812 | DAG.getConstant(RoundWidth, dl, |
813 | TLI.getShiftAmountTy(Hi.getValueType(), DL))); |
814 | |
815 | |
816 | Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); |
817 | } else { |
818 | |
819 | |
820 | |
821 | Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, |
822 | LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(), |
823 | MMOFlags, AAInfo); |
824 | |
825 | |
826 | IncrementSize = RoundWidth / 8; |
827 | Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); |
828 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, |
829 | LD->getPointerInfo().getWithOffset(IncrementSize), |
830 | ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo); |
831 | |
832 | |
833 | |
834 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
835 | Hi.getValue(1)); |
836 | |
837 | |
838 | Hi = DAG.getNode( |
839 | ISD::SHL, dl, Hi.getValueType(), Hi, |
840 | DAG.getConstant(ExtraWidth, dl, |
841 | TLI.getShiftAmountTy(Hi.getValueType(), DL))); |
842 | |
843 | |
844 | Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); |
845 | } |
846 | |
847 | Chain = Ch; |
848 | } else { |
849 | bool isCustom = false; |
850 | switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), |
851 | SrcVT.getSimpleVT())) { |
852 | default: llvm_unreachable("This action is not supported yet!"); |
853 | case TargetLowering::Custom: |
854 | isCustom = true; |
855 | LLVM_FALLTHROUGH; |
856 | case TargetLowering::Legal: |
857 | Value = SDValue(Node, 0); |
858 | Chain = SDValue(Node, 1); |
859 | |
860 | if (isCustom) { |
861 | if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { |
862 | Value = Res; |
863 | Chain = Res.getValue(1); |
864 | } |
865 | } else { |
866 | |
867 | |
868 | EVT MemVT = LD->getMemoryVT(); |
869 | const DataLayout &DL = DAG.getDataLayout(); |
870 | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, |
871 | *LD->getMemOperand())) { |
872 | std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG); |
873 | } |
874 | } |
875 | break; |
876 | |
877 | case TargetLowering::Expand: { |
878 | EVT DestVT = Node->getValueType(0); |
879 | if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) { |
880 | |
881 | |
882 | EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT()); |
883 | if (TLI.isTypeLegal(SrcVT) || |
884 | TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) { |
885 | |
886 | |
887 | ISD::LoadExtType MidExtType = |
888 | (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType; |
889 | |
890 | SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr, |
891 | SrcVT, LD->getMemOperand()); |
892 | unsigned ExtendOp = |
893 | ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType); |
894 | Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); |
895 | Chain = Load.getValue(1); |
896 | break; |
897 | } |
898 | |
899 | |
900 | |
901 | |
902 | |
903 | if (SrcVT.getScalarType() == MVT::f16) { |
904 | EVT ISrcVT = SrcVT.changeTypeToInteger(); |
905 | EVT IDestVT = DestVT.changeTypeToInteger(); |
906 | EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT()); |
907 | |
908 | SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain, |
909 | Ptr, ISrcVT, LD->getMemOperand()); |
910 | Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result); |
911 | Chain = Result.getValue(1); |
912 | break; |
913 | } |
914 | } |
915 | |
916 | assert(!SrcVT.isVector() && |
917 | "Vector Loads are handled in LegalizeVectorOps"); |
918 | |
919 | |
920 | |
921 | |
922 | |
923 | assert(ExtType != ISD::EXTLOAD && |
924 | "EXTLOAD should always be supported!"); |
925 | |
926 | |
927 | SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, |
928 | Node->getValueType(0), |
929 | Chain, Ptr, SrcVT, |
930 | LD->getMemOperand()); |
931 | SDValue ValRes; |
932 | if (ExtType == ISD::SEXTLOAD) |
933 | ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, |
934 | Result.getValueType(), |
935 | Result, DAG.getValueType(SrcVT)); |
936 | else |
937 | ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT); |
938 | Value = ValRes; |
939 | Chain = Result.getValue(1); |
940 | break; |
941 | } |
942 | } |
943 | } |
944 | |
945 | |
946 | |
947 | if (Chain.getNode() != Node) { |
948 | assert(Value.getNode() != Node && "Load must be completely replaced"); |
949 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); |
950 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); |
951 | if (UpdatedNodes) { |
952 | UpdatedNodes->insert(Value.getNode()); |
953 | UpdatedNodes->insert(Chain.getNode()); |
954 | } |
955 | ReplacedNode(Node); |
956 | } |
957 | } |
958 | |
959 | |
960 | void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { |
961 | LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG)); |
962 | |
963 | |
964 | if (Node->getOpcode() == ISD::TargetConstant || |
965 | Node->getOpcode() == ISD::Register) |
966 | return; |
967 | |
968 | #ifndef NDEBUG |
969 | for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) |
970 | assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == |
971 | TargetLowering::TypeLegal && |
972 | "Unexpected illegal type!"); |
973 | |
974 | for (const SDValue &Op : Node->op_values()) |
975 | assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == |
976 | TargetLowering::TypeLegal || |
977 | Op.getOpcode() == ISD::TargetConstant || |
978 | Op.getOpcode() == ISD::Register) && |
979 | "Unexpected illegal type!"); |
980 | #endif |
981 | |
982 | |
983 | TargetLowering::LegalizeAction Action = TargetLowering::Legal; |
984 | bool SimpleFinishLegalizing = true; |
985 | switch (Node->getOpcode()) { |
986 | case ISD::INTRINSIC_W_CHAIN: |
987 | case ISD::INTRINSIC_WO_CHAIN: |
988 | case ISD::INTRINSIC_VOID: |
989 | case ISD::STACKSAVE: |
990 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); |
991 | break; |
992 | case ISD::GET_DYNAMIC_AREA_OFFSET: |
993 | Action = TLI.getOperationAction(Node->getOpcode(), |
994 | Node->getValueType(0)); |
995 | break; |
996 | case ISD::VAARG: |
997 | Action = TLI.getOperationAction(Node->getOpcode(), |
998 | Node->getValueType(0)); |
999 | if (Action != TargetLowering::Promote) |
1000 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); |
1001 | break; |
1002 | case ISD::FP_TO_FP16: |
1003 | case ISD::SINT_TO_FP: |
1004 | case ISD::UINT_TO_FP: |
1005 | case ISD::EXTRACT_VECTOR_ELT: |
1006 | case ISD::LROUND: |
1007 | case ISD::LLROUND: |
1008 | case ISD::LRINT: |
1009 | case ISD::LLRINT: |
1010 | Action = TLI.getOperationAction(Node->getOpcode(), |
1011 | Node->getOperand(0).getValueType()); |
1012 | break; |
1013 | case ISD::STRICT_FP_TO_FP16: |
1014 | case ISD::STRICT_SINT_TO_FP: |
1015 | case ISD::STRICT_UINT_TO_FP: |
1016 | case ISD::STRICT_LRINT: |
1017 | case ISD::STRICT_LLRINT: |
1018 | case ISD::STRICT_LROUND: |
1019 | case ISD::STRICT_LLROUND: |
1020 | |
1021 | |
1022 | |
1023 | Action = TLI.getOperationAction(Node->getOpcode(), |
1024 | Node->getOperand(1).getValueType()); |
1025 | break; |
1026 | case ISD::SIGN_EXTEND_INREG: { |
1027 | EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); |
1028 | Action = TLI.getOperationAction(Node->getOpcode(), InnerType); |
1029 | break; |
1030 | } |
1031 | case ISD::ATOMIC_STORE: |
1032 | Action = TLI.getOperationAction(Node->getOpcode(), |
1033 | Node->getOperand(2).getValueType()); |
1034 | break; |
1035 | case ISD::SELECT_CC: |
1036 | case ISD::STRICT_FSETCC: |
1037 | case ISD::STRICT_FSETCCS: |
1038 | case ISD::SETCC: |
1039 | case ISD::BR_CC: { |
1040 | unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : |
1041 | Node->getOpcode() == ISD::STRICT_FSETCC ? 3 : |
1042 | Node->getOpcode() == ISD::STRICT_FSETCCS ? 3 : |
1043 | Node->getOpcode() == ISD::SETCC ? 2 : 1; |
1044 | unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : |
1045 | Node->getOpcode() == ISD::STRICT_FSETCC ? 1 : |
1046 | Node->getOpcode() == ISD::STRICT_FSETCCS ? 1 : 0; |
1047 | MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); |
1048 | ISD::CondCode CCCode = |
1049 | cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); |
1050 | Action = TLI.getCondCodeAction(CCCode, OpVT); |
1051 | if (Action == TargetLowering::Legal) { |
1052 | if (Node->getOpcode() == ISD::SELECT_CC) |
1053 | Action = TLI.getOperationAction(Node->getOpcode(), |
1054 | Node->getValueType(0)); |
1055 | else |
1056 | Action = TLI.getOperationAction(Node->getOpcode(), OpVT); |
1057 | } |
1058 | break; |
1059 | } |
1060 | case ISD::LOAD: |
1061 | case ISD::STORE: |
1062 | |
1063 | |
1064 | SimpleFinishLegalizing = false; |
1065 | break; |
1066 | case ISD::CALLSEQ_START: |
1067 | case ISD::CALLSEQ_END: |
1068 | |
1069 | |
1070 | |
1071 | SimpleFinishLegalizing = false; |
1072 | break; |
1073 | case ISD::EXTRACT_ELEMENT: |
1074 | case ISD::FLT_ROUNDS_: |
1075 | case ISD::MERGE_VALUES: |
1076 | case ISD::EH_RETURN: |
1077 | case ISD::FRAME_TO_ARGS_OFFSET: |
1078 | case ISD::EH_DWARF_CFA: |
1079 | case ISD::EH_SJLJ_SETJMP: |
1080 | case ISD::EH_SJLJ_LONGJMP: |
1081 | case ISD::EH_SJLJ_SETUP_DISPATCH: |
1082 | |
1083 | |
1084 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1085 | if (Action == TargetLowering::Legal) |
1086 | Action = TargetLowering::Expand; |
1087 | break; |
1088 | case ISD::INIT_TRAMPOLINE: |
1089 | case ISD::ADJUST_TRAMPOLINE: |
1090 | case ISD::FRAMEADDR: |
1091 | case ISD::RETURNADDR: |
1092 | case ISD::ADDROFRETURNADDR: |
1093 | case ISD::SPONENTRY: |
1094 | |
1095 | |
1096 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1097 | if (Action == TargetLowering::Legal) |
1098 | Action = TargetLowering::Custom; |
1099 | break; |
1100 | case ISD::READCYCLECOUNTER: |
1101 | |
1102 | |
1103 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64); |
1104 | break; |
1105 | case ISD::READ_REGISTER: |
1106 | case ISD::WRITE_REGISTER: |
1107 | |
1108 | |
1109 | |
1110 | Action = TargetLowering::Legal; |
1111 | break; |
1112 | case ISD::UBSANTRAP: |
1113 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1114 | if (Action == TargetLowering::Expand) { |
1115 | |
1116 | SDValue NewVal; |
1117 | NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), |
1118 | Node->getOperand(0)); |
1119 | ReplaceNode(Node, NewVal.getNode()); |
1120 | LegalizeOp(NewVal.getNode()); |
1121 | return; |
1122 | } |
1123 | break; |
1124 | case ISD::DEBUGTRAP: |
1125 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1126 | if (Action == TargetLowering::Expand) { |
1127 | |
1128 | SDValue NewVal; |
1129 | NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), |
1130 | Node->getOperand(0)); |
1131 | ReplaceNode(Node, NewVal.getNode()); |
1132 | LegalizeOp(NewVal.getNode()); |
1133 | return; |
1134 | } |
1135 | break; |
1136 | case ISD::SADDSAT: |
1137 | case ISD::UADDSAT: |
1138 | case ISD::SSUBSAT: |
1139 | case ISD::USUBSAT: |
1140 | case ISD::SSHLSAT: |
1141 | case ISD::USHLSAT: |
1142 | case ISD::FP_TO_SINT_SAT: |
1143 | case ISD::FP_TO_UINT_SAT: |
1144 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1145 | break; |
1146 | case ISD::SMULFIX: |
1147 | case ISD::SMULFIXSAT: |
1148 | case ISD::UMULFIX: |
1149 | case ISD::UMULFIXSAT: |
1150 | case ISD::SDIVFIX: |
1151 | case ISD::SDIVFIXSAT: |
1152 | case ISD::UDIVFIX: |
1153 | case ISD::UDIVFIXSAT: { |
1154 | unsigned Scale = Node->getConstantOperandVal(2); |
1155 | Action = TLI.getFixedPointOperationAction(Node->getOpcode(), |
1156 | Node->getValueType(0), Scale); |
1157 | break; |
1158 | } |
1159 | case ISD::MSCATTER: |
1160 | Action = TLI.getOperationAction(Node->getOpcode(), |
1161 | cast<MaskedScatterSDNode>(Node)->getValue().getValueType()); |
1162 | break; |
1163 | case ISD::MSTORE: |
1164 | Action = TLI.getOperationAction(Node->getOpcode(), |
1165 | cast<MaskedStoreSDNode>(Node)->getValue().getValueType()); |
1166 | break; |
1167 | case ISD::VECREDUCE_FADD: |
1168 | case ISD::VECREDUCE_FMUL: |
1169 | case ISD::VECREDUCE_ADD: |
1170 | case ISD::VECREDUCE_MUL: |
1171 | case ISD::VECREDUCE_AND: |
1172 | case ISD::VECREDUCE_OR: |
1173 | case ISD::VECREDUCE_XOR: |
1174 | case ISD::VECREDUCE_SMAX: |
1175 | case ISD::VECREDUCE_SMIN: |
1176 | case ISD::VECREDUCE_UMAX: |
1177 | case ISD::VECREDUCE_UMIN: |
1178 | case ISD::VECREDUCE_FMAX: |
1179 | case ISD::VECREDUCE_FMIN: |
1180 | Action = TLI.getOperationAction( |
1181 | Node->getOpcode(), Node->getOperand(0).getValueType()); |
1182 | break; |
1183 | case ISD::VECREDUCE_SEQ_FADD: |
1184 | Action = TLI.getOperationAction( |
1185 | Node->getOpcode(), Node->getOperand(1).getValueType()); |
1186 | break; |
1187 | default: |
1188 | if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { |
1189 | Action = TargetLowering::Legal; |
1190 | } else { |
1191 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1192 | } |
1193 | break; |
1194 | } |
1195 | |
1196 | if (SimpleFinishLegalizing) { |
1197 | SDNode *NewNode = Node; |
1198 | switch (Node->getOpcode()) { |
1199 | default: break; |
1200 | case ISD::SHL: |
1201 | case ISD::SRL: |
1202 | case ISD::SRA: |
1203 | case ISD::ROTL: |
1204 | case ISD::ROTR: { |
1205 | |
1206 | |
1207 | SDValue Op0 = Node->getOperand(0); |
1208 | SDValue Op1 = Node->getOperand(1); |
1209 | if (!Op1.getValueType().isVector()) { |
1210 | SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1); |
1211 | |
1212 | |
1213 | |
1214 | |
1215 | |
1216 | if (SAO != Op1) |
1217 | NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO); |
1218 | } |
1219 | } |
1220 | break; |
1221 | case ISD::FSHL: |
1222 | case ISD::FSHR: |
1223 | case ISD::SRL_PARTS: |
1224 | case ISD::SRA_PARTS: |
1225 | case ISD::SHL_PARTS: { |
1226 | |
1227 | |
1228 | SDValue Op0 = Node->getOperand(0); |
1229 | SDValue Op1 = Node->getOperand(1); |
1230 | SDValue Op2 = Node->getOperand(2); |
1231 | if (!Op2.getValueType().isVector()) { |
1232 | SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2); |
1233 | |
1234 | |
1235 | |
1236 | if (SAO != Op2) |
1237 | NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO); |
1238 | } |
1239 | break; |
1240 | } |
1241 | } |
1242 | |
1243 | if (NewNode != Node) { |
1244 | ReplaceNode(Node, NewNode); |
1245 | Node = NewNode; |
1246 | } |
1247 | switch (Action) { |
1248 | case TargetLowering::Legal: |
1249 | LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n"); |
1250 | return; |
1251 | case TargetLowering::Custom: |
1252 | LLVM_DEBUG(dbgs() << "Trying custom legalization\n"); |
1253 | |
1254 | |
1255 | if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { |
1256 | if (!(Res.getNode() != Node || Res.getResNo() != 0)) |
1257 | return; |
1258 | |
1259 | if (Node->getNumValues() == 1) { |
1260 | |
1261 | |
1262 | assert((Res.getValueType() == Node->getValueType(0) || |
1263 | Node->getValueType(0) == MVT::Glue) && |
1264 | "Type mismatch for custom legalized operation"); |
1265 | LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n"); |
1266 | |
1267 | ReplaceNode(SDValue(Node, 0), Res); |
1268 | return; |
1269 | } |
1270 | |
1271 | SmallVector<SDValue, 8> ResultVals; |
1272 | for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { |
1273 | |
1274 | |
1275 | assert((Res->getValueType(i) == Node->getValueType(i) || |
1276 | Node->getValueType(i) == MVT::Glue) && |
1277 | "Type mismatch for custom legalized operation"); |
1278 | ResultVals.push_back(Res.getValue(i)); |
1279 | } |
1280 | LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n"); |
1281 | ReplaceNode(Node, ResultVals.data()); |
1282 | return; |
1283 | } |
1284 | LLVM_DEBUG(dbgs() << "Could not custom legalize node\n"); |
1285 | LLVM_FALLTHROUGH; |
1286 | case TargetLowering::Expand: |
1287 | if (ExpandNode(Node)) |
1288 | return; |
1289 | LLVM_FALLTHROUGH; |
1290 | case TargetLowering::LibCall: |
1291 | ConvertNodeToLibcall(Node); |
1292 | return; |
1293 | case TargetLowering::Promote: |
1294 | PromoteNode(Node); |
1295 | return; |
1296 | } |
1297 | } |
1298 | |
1299 | switch (Node->getOpcode()) { |
1300 | default: |
1301 | #ifndef NDEBUG |
1302 | dbgs() << "NODE: "; |
1303 | Node->dump( &DAG); |
1304 | dbgs() << "\n"; |
1305 | #endif |
1306 | llvm_unreachable("Do not know how to legalize this operator!"); |
1307 | |
1308 | case ISD::CALLSEQ_START: |
1309 | case ISD::CALLSEQ_END: |
1310 | break; |
1311 | case ISD::LOAD: |
1312 | return LegalizeLoadOps(Node); |
1313 | case ISD::STORE: |
1314 | return LegalizeStoreOps(Node); |
1315 | } |
1316 | } |
1317 | |
1318 | SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { |
1319 | SDValue Vec = Op.getOperand(0); |
1320 | SDValue Idx = Op.getOperand(1); |
1321 | SDLoc dl(Op); |
1322 | |
1323 | |
1324 | |
1325 | |
1326 | |
1327 | |
1328 | |
1329 | |
1330 | |
1331 | SmallPtrSet<const SDNode *, 32> Visited; |
1332 | SmallVector<const SDNode *, 16> Worklist; |
1333 | Visited.insert(Op.getNode()); |
1334 | Worklist.push_back(Idx.getNode()); |
1335 | SDValue StackPtr, Ch; |
1336 | for (SDNode::use_iterator UI = Vec.getNode()->use_begin(), |
1337 | UE = Vec.getNode()->use_end(); UI != UE; ++UI) { |
1338 | SDNode *User = *UI; |
1339 | if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) { |
1340 | if (ST->isIndexed() || ST->isTruncatingStore() || |
1341 | ST->getValue() != Vec) |
1342 | continue; |
1343 | |
1344 | |
1345 | |
1346 | if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode())) |
1347 | continue; |
1348 | |
1349 | |
1350 | |
1351 | |
1352 | |
1353 | |
1354 | if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) || |
1355 | ST->hasPredecessor(Op.getNode())) |
1356 | continue; |
1357 | |
1358 | StackPtr = ST->getBasePtr(); |
1359 | Ch = SDValue(ST, 0); |
1360 | break; |
1361 | } |
1362 | } |
1363 | |
1364 | EVT VecVT = Vec.getValueType(); |
1365 | |
1366 | if (!Ch.getNode()) { |
1367 | |
1368 | StackPtr = DAG.CreateStackTemporary(VecVT); |
1369 | Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, |
1370 | MachinePointerInfo()); |
1371 | } |
1372 | |
1373 | SDValue NewLoad; |
1374 | |
1375 | if (Op.getValueType().isVector()) { |
1376 | StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, |
1377 | Op.getValueType(), Idx); |
1378 | NewLoad = |
1379 | DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo()); |
1380 | } else { |
1381 | StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); |
1382 | NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, |
1383 | MachinePointerInfo(), |
1384 | VecVT.getVectorElementType()); |
1385 | } |
1386 | |
1387 | |
1388 | DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1)); |
1389 | |
1390 | |
1391 | |
1392 | SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(), |
1393 | NewLoad->op_end()); |
1394 | NewLoadOperands[0] = Ch; |
1395 | NewLoad = |
1396 | SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0); |
1397 | return NewLoad; |
1398 | } |
1399 | |
1400 | SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { |
1401 | assert(Op.getValueType().isVector() && "Non-vector insert subvector!"); |
1402 | |
1403 | SDValue Vec = Op.getOperand(0); |
1404 | SDValue Part = Op.getOperand(1); |
1405 | SDValue Idx = Op.getOperand(2); |
1406 | SDLoc dl(Op); |
1407 | |
1408 | |
1409 | EVT VecVT = Vec.getValueType(); |
1410 | EVT SubVecVT = Part.getValueType(); |
1411 | SDValue StackPtr = DAG.CreateStackTemporary(VecVT); |
1412 | int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); |
1413 | MachinePointerInfo PtrInfo = |
1414 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); |
1415 | |
1416 | |
1417 | SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo); |
1418 | |
1419 | |
1420 | SDValue SubStackPtr = |
1421 | TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx); |
1422 | |
1423 | |
1424 | Ch = DAG.getStore( |
1425 | Ch, dl, Part, SubStackPtr, |
1426 | MachinePointerInfo::getUnknownStack(DAG.getMachineFunction())); |
1427 | |
1428 | |
1429 | return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo); |
1430 | } |
1431 | |
1432 | SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { |
1433 | assert((Node->getOpcode() == ISD::BUILD_VECTOR || |
1434 | Node->getOpcode() == ISD::CONCAT_VECTORS) && |
1435 | "Unexpected opcode!"); |
1436 | |
1437 | |
1438 | |
1439 | |
1440 | |
1441 | EVT VT = Node->getValueType(0); |
1442 | EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType() |
1443 | : Node->getOperand(0).getValueType(); |
1444 | SDLoc dl(Node); |
1445 | SDValue FIPtr = DAG.CreateStackTemporary(VT); |
1446 | int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); |
1447 | MachinePointerInfo PtrInfo = |
1448 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); |
1449 | |
1450 | |
1451 | SmallVector<SDValue, 8> Stores; |
1452 | unsigned TypeByteSize = MemVT.getSizeInBits() / 8; |
1453 | assert(TypeByteSize > 0 && "Vector element type too small for stack store!"); |
1454 | |
1455 | |
1456 | |
1457 | bool Truncate = isa<BuildVectorSDNode>(Node) && |
1458 | MemVT.bitsLT(Node->getOperand(0).getValueType()); |
1459 | |
1460 | |
1461 | for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { |
1462 | |
1463 | if (Node->getOperand(i).isUndef()) continue; |
1464 | |
1465 | unsigned Offset = TypeByteSize*i; |
1466 | |
1467 | SDValue Idx = DAG.getMemBasePlusOffset(FIPtr, TypeSize::Fixed(Offset), dl); |
1468 | |
1469 | if (Truncate) |
1470 | Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, |
1471 | Node->getOperand(i), Idx, |
1472 | PtrInfo.getWithOffset(Offset), MemVT)); |
1473 | else |
1474 | Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), |
1475 | Idx, PtrInfo.getWithOffset(Offset))); |
1476 | } |
1477 | |
1478 | SDValue StoreChain; |
1479 | if (!Stores.empty()) |
1480 | StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); |
1481 | else |
1482 | StoreChain = DAG.getEntryNode(); |
1483 | |
1484 | |
1485 | return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo); |
1486 | } |
1487 | |
1488 | |
1489 | |
1490 | |
1491 | void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State, |
1492 | const SDLoc &DL, |
1493 | SDValue Value) const { |
1494 | EVT FloatVT = Value.getValueType(); |
1495 | unsigned NumBits = FloatVT.getScalarSizeInBits(); |
1496 | State.FloatVT = FloatVT; |
1497 | EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits); |
1498 | |
1499 | if (TLI.isTypeLegal(IVT)) { |
1500 | State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value); |
1501 | State.SignMask = APInt::getSignMask(NumBits); |
1502 | State.SignBit = NumBits - 1; |
1503 | return; |
1504 | } |
1505 | |
1506 | auto &DataLayout = DAG.getDataLayout(); |
1507 | |
1508 | MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8); |
1509 | |
1510 | SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); |
1511 | int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); |
1512 | |
1513 | State.FloatPtr = StackPtr; |
1514 | MachineFunction &MF = DAG.getMachineFunction(); |
1515 | State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI); |
1516 | State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr, |
1517 | State.FloatPointerInfo); |
1518 | |
1519 | SDValue IntPtr; |
1520 | if (DataLayout.isBigEndian()) { |
1521 | assert(FloatVT.isByteSized() && "Unsupported floating point type!"); |
1522 | |
1523 | IntPtr = StackPtr; |
1524 | State.IntPointerInfo = State.FloatPointerInfo; |
1525 | } else { |
1526 | |
1527 | unsigned ByteOffset = (NumBits / 8) - 1; |
1528 | IntPtr = |
1529 | DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(ByteOffset), DL); |
1530 | State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI, |
1531 | ByteOffset); |
1532 | } |
1533 | |
1534 | State.IntPtr = IntPtr; |
1535 | State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr, |
1536 | State.IntPointerInfo, MVT::i8); |
1537 | State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7); |
1538 | State.SignBit = 7; |
1539 | } |
1540 | |
1541 | |
1542 | |
1543 | SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State, |
1544 | const SDLoc &DL, |
1545 | SDValue NewIntValue) const { |
1546 | if (!State.Chain) |
1547 | return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue); |
1548 | |
1549 | |
1550 | SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr, |
1551 | State.IntPointerInfo, MVT::i8); |
1552 | return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr, |
1553 | State.FloatPointerInfo); |
1554 | } |
1555 | |
1556 | SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const { |
1557 | SDLoc DL(Node); |
1558 | SDValue Mag = Node->getOperand(0); |
1559 | SDValue Sign = Node->getOperand(1); |
1560 | |
1561 | |
1562 | FloatSignAsInt SignAsInt; |
1563 | getSignAsIntValue(SignAsInt, DL, Sign); |
1564 | |
1565 | EVT IntVT = SignAsInt.IntValue.getValueType(); |
1566 | SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT); |
1567 | SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue, |
1568 | SignMask); |
1569 | |
1570 | |
1571 | EVT FloatVT = Mag.getValueType(); |
1572 | if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) && |
1573 | TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) { |
1574 | SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag); |
1575 | SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue); |
1576 | SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit, |
1577 | DAG.getConstant(0, DL, IntVT), ISD::SETNE); |
1578 | return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue); |
1579 | } |
1580 | |
1581 | |
1582 | FloatSignAsInt MagAsInt; |
1583 | getSignAsIntValue(MagAsInt, DL, Mag); |
1584 | EVT MagVT = MagAsInt.IntValue.getValueType(); |
1585 | SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT); |
1586 | SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue, |
1587 | ClearSignMask); |
1588 | |
1589 | |
1590 | int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit; |
1591 | EVT ShiftVT = IntVT; |
1592 | if (SignBit.getScalarValueSizeInBits() < |
1593 | ClearedSign.getScalarValueSizeInBits()) { |
1594 | SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit); |
1595 | ShiftVT = MagVT; |
1596 | } |
1597 | if (ShiftAmount > 0) { |
1598 | SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT); |
1599 | SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst); |
1600 | } else if (ShiftAmount < 0) { |
1601 | SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT); |
1602 | SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst); |
1603 | } |
1604 | if (SignBit.getScalarValueSizeInBits() > |
1605 | ClearedSign.getScalarValueSizeInBits()) { |
1606 | SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit); |
1607 | } |
1608 | |
1609 | |
1610 | SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit); |
1611 | return modifySignAsInt(MagAsInt, DL, CopiedSign); |
1612 | } |
1613 | |
1614 | SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const { |
1615 | |
1616 | SDLoc DL(Node); |
1617 | FloatSignAsInt SignAsInt; |
1618 | getSignAsIntValue(SignAsInt, DL, Node->getOperand(0)); |
1619 | EVT IntVT = SignAsInt.IntValue.getValueType(); |
1620 | |
1621 | |
1622 | SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT); |
1623 | SDValue SignFlip = |
1624 | DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask); |
1625 | |
1626 | |
1627 | return modifySignAsInt(SignAsInt, DL, SignFlip); |
1628 | } |
1629 | |
1630 | SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const { |
1631 | SDLoc DL(Node); |
1632 | SDValue Value = Node->getOperand(0); |
1633 | |
1634 | |
1635 | EVT FloatVT = Value.getValueType(); |
1636 | if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) { |
1637 | SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT); |
1638 | return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero); |
1639 | } |
1640 | |
1641 | |
1642 | FloatSignAsInt ValueAsInt; |
1643 | getSignAsIntValue(ValueAsInt, DL, Value); |
1644 | EVT IntVT = ValueAsInt.IntValue.getValueType(); |
1645 | SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT); |
1646 | SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue, |
1647 | ClearSignMask); |
1648 | return modifySignAsInt(ValueAsInt, DL, ClearedSign); |
1649 | } |
1650 | |
1651 | void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, |
1652 | SmallVectorImpl<SDValue> &Results) { |
1653 | Register SPReg = TLI.getStackPointerRegisterToSaveRestore(); |
1654 | assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" |
1655 | " not tell us which reg is the stack pointer!"); |
1656 | SDLoc dl(Node); |
1657 | EVT VT = Node->getValueType(0); |
1658 | SDValue Tmp1 = SDValue(Node, 0); |
1659 | SDValue Tmp2 = SDValue(Node, 1); |
1660 | SDValue Tmp3 = Node->getOperand(2); |
1661 | SDValue Chain = Tmp1.getOperand(0); |
1662 | |
1663 | |
1664 | |
1665 | Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl); |
1666 | |
1667 | SDValue Size = Tmp2.getOperand(1); |
1668 | SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); |
1669 | Chain = SP.getValue(1); |
1670 | Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue(); |
1671 | const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering(); |
1672 | unsigned Opc = |
1673 | TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ? |
1674 | ISD::ADD : ISD::SUB; |
1675 | |
1676 | Align StackAlign = TFL->getStackAlign(); |
1677 | Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); |
1678 | if (Alignment > StackAlign) |
1679 | Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, |
1680 | DAG.getConstant(-Alignment.value(), dl, VT)); |
1681 | Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); |
1682 | |
1683 | Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true), |
1684 | DAG.getIntPtrConstant(0, dl, true), SDValue(), dl); |
1685 | |
1686 | Results.push_back(Tmp1); |
1687 | Results.push_back(Tmp2); |
1688 | } |
1689 | |
1690 | |
1691 | |
1692 | |
1693 | |
1694 | SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, |
1695 | EVT DestVT, const SDLoc &dl) { |
1696 | return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode()); |
1697 | } |
1698 | |
1699 | SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, |
1700 | EVT DestVT, const SDLoc &dl, |
1701 | SDValue Chain) { |
1702 | unsigned SrcSize = SrcOp.getValueSizeInBits(); |
1703 | unsigned SlotSize = SlotVT.getSizeInBits(); |
1704 | unsigned DestSize = DestVT.getSizeInBits(); |
1705 | Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); |
1706 | Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType); |
1707 | |
1708 | |
1709 | if ((SrcSize > SlotSize && |
1710 | !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) || |
1711 | (SlotSize < DestSize && |
1712 | !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT))) |
1713 | return SDValue(); |
1714 | |
1715 | |
1716 | Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign( |
1717 | SrcOp.getValueType().getTypeForEVT(*DAG.getContext())); |
1718 | SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign); |
1719 | |
1720 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); |
1721 | int SPFI = StackPtrFI->getIndex(); |
1722 | MachinePointerInfo PtrInfo = |
1723 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); |
1724 | |
1725 | |
1726 | |
1727 | SDValue Store; |
1728 | |
1729 | if (SrcSize > SlotSize) |
1730 | Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo, |
1731 | SlotVT, SrcAlign); |
1732 | else { |
1733 | assert(SrcSize == SlotSize && "Invalid store"); |
1734 | Store = |
1735 | DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign); |
1736 | } |
1737 | |
1738 | |
1739 | if (SlotSize == DestSize) |
1740 | return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign); |
1741 | |
1742 | assert(SlotSize < DestSize && "Unknown extension!"); |
1743 | return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT, |
1744 | DestAlign); |
1745 | } |
1746 | |
1747 | SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { |
1748 | SDLoc dl(Node); |
1749 | |
1750 | |
1751 | SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); |
1752 | |
1753 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); |
1754 | int SPFI = StackPtrFI->getIndex(); |
1755 | |
1756 | SDValue Ch = DAG.getTruncStore( |
1757 | DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr, |
1758 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI), |
1759 | Node->getValueType(0).getVectorElementType()); |
1760 | return DAG.getLoad( |
1761 | Node->getValueType(0), dl, Ch, StackPtr, |
1762 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); |
1763 | } |
1764 | |
1765 | static bool |
1766 | ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, |
1767 | const TargetLowering &TLI, SDValue &Res) { |
1768 | unsigned NumElems = Node->getNumOperands(); |
1769 | SDLoc dl(Node); |
1770 | EVT VT = Node->getValueType(0); |
1771 | |
1772 | |
1773 | |
1774 | |
1775 | |
1776 | |
1777 | |
1778 | |
1779 | for (int Phase = 0; Phase < 2; ++Phase) { |
1780 | SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals, |
1781 | NewIntermedVals; |
1782 | for (unsigned i = 0; i < NumElems; ++i) { |
1783 | SDValue V = Node->getOperand(i); |
1784 | if (V.isUndef()) |
1785 | continue; |
1786 | |
1787 | SDValue Vec; |
1788 | if (Phase) |
1789 | Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); |
1790 | IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i))); |
1791 | } |
1792 | |
1793 | while (IntermedVals.size() > 2) { |
1794 | NewIntermedVals.clear(); |
1795 | for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) { |
1796 | |
1797 | |
1798 | SmallVector<int, 16> ShuffleVec(NumElems, -1); |
1799 | |
1800 | SmallVector<int, 16> FinalIndices; |
1801 | FinalIndices.reserve(IntermedVals[i].second.size() + |
1802 | IntermedVals[i+1].second.size()); |
1803 | |
1804 | int k = 0; |
1805 | for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f; |
1806 | ++j, ++k) { |
1807 | ShuffleVec[k] = j; |
1808 | FinalIndices.push_back(IntermedVals[i].second[j]); |
1809 | } |
1810 | for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f; |
1811 | ++j, ++k) { |
1812 | ShuffleVec[k] = NumElems + j; |
1813 | FinalIndices.push_back(IntermedVals[i+1].second[j]); |
1814 | } |
1815 | |
1816 | SDValue Shuffle; |
1817 | if (Phase) |
1818 | Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first, |
1819 | IntermedVals[i+1].first, |
1820 | ShuffleVec); |
1821 | else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) |
1822 | return false; |
1823 | NewIntermedVals.push_back( |
1824 | std::make_pair(Shuffle, std::move(FinalIndices))); |
1825 | } |
1826 | |
1827 | |
1828 | |
1829 | if ((IntermedVals.size() & 1) != 0) |
1830 | NewIntermedVals.push_back(IntermedVals.back()); |
1831 | |
1832 | IntermedVals.swap(NewIntermedVals); |
1833 | } |
1834 | |
1835 | assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 && |
1836 | "Invalid number of intermediate vectors"); |
1837 | SDValue Vec1 = IntermedVals[0].first; |
1838 | SDValue Vec2; |
1839 | if (IntermedVals.size() > 1) |
1840 | Vec2 = IntermedVals[1].first; |
1841 | else if (Phase) |
1842 | Vec2 = DAG.getUNDEF(VT); |
1843 | |
1844 | SmallVector<int, 16> ShuffleVec(NumElems, -1); |
1845 | for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i) |
1846 | ShuffleVec[IntermedVals[0].second[i]] = i; |
1847 | for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i) |
1848 | ShuffleVec[IntermedVals[1].second[i]] = NumElems + i; |
1849 | |
1850 | if (Phase) |
1851 | Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); |
1852 | else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) |
1853 | return false; |
1854 | } |
1855 | |
1856 | return true; |
1857 | } |
1858 | |
1859 | |
1860 | |
1861 | SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { |
1862 | unsigned NumElems = Node->getNumOperands(); |
1863 | SDValue Value1, Value2; |
1864 | SDLoc dl(Node); |
1865 | EVT VT = Node->getValueType(0); |
1866 | EVT OpVT = Node->getOperand(0).getValueType(); |
1867 | EVT EltVT = VT.getVectorElementType(); |
1868 | |
1869 | |
1870 | |
1871 | bool isOnlyLowElement = true; |
1872 | bool MoreThanTwoValues = false; |
1873 | bool isConstant = true; |
1874 | for (unsigned i = 0; i < NumElems; ++i) { |
1875 | SDValue V = Node->getOperand(i); |
1876 | if (V.isUndef()) |
1877 | continue; |
1878 | if (i > 0) |
1879 | isOnlyLowElement = false; |
1880 | if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) |
1881 | isConstant = false; |
1882 | |
1883 | if (!Value1.getNode()) { |
1884 | Value1 = V; |
1885 | } else if (!Value2.getNode()) { |
1886 | if (V != Value1) |
1887 | Value2 = V; |
1888 | } else if (V != Value1 && V != Value2) { |
1889 | MoreThanTwoValues = true; |
1890 | } |
1891 | } |
1892 | |
1893 | if (!Value1.getNode()) |
1894 | return DAG.getUNDEF(VT); |
1895 | |
1896 | if (isOnlyLowElement) |
1897 | return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); |
1898 | |
1899 | |
1900 | if (isConstant) { |
1901 | SmallVector<Constant*, 16> CV; |
1902 | for (unsigned i = 0, e = NumElems; i != e; ++i) { |
1903 | if (ConstantFPSDNode *V = |
1904 | dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { |
1905 | CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); |
1906 | } else if (ConstantSDNode *V = |
1907 | dyn_cast<ConstantSDNode>(Node->getOperand(i))) { |
1908 | if (OpVT==EltVT) |
1909 | CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); |
1910 | else { |
1911 | |
1912 | |
1913 | |
1914 | const ConstantInt *CI = V->getConstantIntValue(); |
1915 | CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), |
1916 | CI->getZExtValue())); |
1917 | } |
1918 | } else { |
1919 | assert(Node->getOperand(i).isUndef()); |
1920 | Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); |
1921 | CV.push_back(UndefValue::get(OpNTy)); |
1922 | } |
1923 | } |
1924 | Constant *CP = ConstantVector::get(CV); |
1925 | SDValue CPIdx = |
1926 | DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout())); |
1927 | Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); |
1928 | return DAG.getLoad( |
1929 | VT, dl, DAG.getEntryNode(), CPIdx, |
1930 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), |
1931 | Alignment); |
1932 | } |
1933 | |
1934 | SmallSet<SDValue, 16> DefinedValues; |
1935 | for (unsigned i = 0; i < NumElems; ++i) { |
1936 | if (Node->getOperand(i).isUndef()) |
1937 | continue; |
1938 | DefinedValues.insert(Node->getOperand(i)); |
1939 | } |
1940 | |
1941 | if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) { |
1942 | if (!MoreThanTwoValues) { |
1943 | SmallVector<int, 8> ShuffleVec(NumElems, -1); |
1944 | for (unsigned i = 0; i < NumElems; ++i) { |
1945 | SDValue V = Node->getOperand(i); |
1946 | if (V.isUndef()) |
1947 | continue; |
1948 | ShuffleVec[i] = V == Value1 ? 0 : NumElems; |
1949 | } |
1950 | if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { |
1951 | |
1952 | SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); |
1953 | SDValue Vec2; |
1954 | if (Value2.getNode()) |
1955 | Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); |
1956 | else |
1957 | Vec2 = DAG.getUNDEF(VT); |
1958 | |
1959 | |
1960 | return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); |
1961 | } |
1962 | } else { |
1963 | SDValue Res; |
1964 | if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) |
1965 | return Res; |
1966 | } |
1967 | } |
1968 | |
1969 | |
1970 | return ExpandVectorBuildThroughStack(Node); |
1971 | } |
1972 | |
1973 | SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) { |
1974 | SDLoc DL(Node); |
1975 | EVT VT = Node->getValueType(0); |
1976 | SDValue SplatVal = Node->getOperand(0); |
1977 | |
1978 | return DAG.getSplatBuildVector(VT, DL, SplatVal); |
1979 | } |
1980 | |
1981 | |
1982 | |
1983 | |
1984 | |
1985 | SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, |
1986 | bool isSigned) { |
1987 | TargetLowering::ArgListTy Args; |
1988 | TargetLowering::ArgListEntry Entry; |
1989 | for (const SDValue &Op : Node->op_values()) { |
1990 | EVT ArgVT = Op.getValueType(); |
1991 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
1992 | Entry.Node = Op; |
1993 | Entry.Ty = ArgTy; |
1994 | Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned); |
1995 | Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned); |
1996 | Args.push_back(Entry); |
1997 | } |
1998 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
1999 | TLI.getPointerTy(DAG.getDataLayout())); |
2000 | |
2001 | EVT RetVT = Node->getValueType(0); |
2002 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); |
2003 | |
2004 | |
2005 | |
2006 | |
2007 | |
2008 | SDValue InChain = DAG.getEntryNode(); |
2009 | |
2010 | |
2011 | |
2012 | SDValue TCChain = InChain; |
2013 | const Function &F = DAG.getMachineFunction().getFunction(); |
2014 | bool isTailCall = |
2015 | TLI.isInTailCallPosition(DAG, Node, TCChain) && |
2016 | (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy()); |
2017 | if (isTailCall) |
2018 | InChain = TCChain; |
2019 | |
2020 | TargetLowering::CallLoweringInfo CLI(DAG); |
2021 | bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned); |
2022 | CLI.setDebugLoc(SDLoc(Node)) |
2023 | .setChain(InChain) |
2024 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, |
2025 | std::move(Args)) |
2026 | .setTailCall(isTailCall) |
2027 | .setSExtResult(signExtend) |
2028 | .setZExtResult(!signExtend) |
2029 | .setIsPostTypeLegalization(true); |
2030 | |
2031 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2032 | |
2033 | if (!CallInfo.second.getNode()) { |
2034 | LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG)); |
2035 | |
2036 | return DAG.getRoot(); |
2037 | } |
2038 | |
2039 | LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG)); |
2040 | return CallInfo.first; |
2041 | } |
2042 | |
2043 | void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, |
2044 | RTLIB::Libcall LC, |
2045 | SmallVectorImpl<SDValue> &Results) { |
2046 | if (LC == RTLIB::UNKNOWN_LIBCALL) |
2047 | llvm_unreachable("Can't create an unknown libcall!"); |
2048 | |
2049 | if (Node->isStrictFPOpcode()) { |
2050 | EVT RetVT = Node->getValueType(0); |
2051 | SmallVector<SDValue, 4> Ops(drop_begin(Node->ops())); |
2052 | TargetLowering::MakeLibCallOptions CallOptions; |
2053 | |
2054 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, |
2055 | Ops, CallOptions, |
2056 | SDLoc(Node), |
2057 | Node->getOperand(0)); |
2058 | Results.push_back(Tmp.first); |
2059 | Results.push_back(Tmp.second); |
2060 | } else { |
2061 | SDValue Tmp = ExpandLibCall(LC, Node, false); |
2062 | Results.push_back(Tmp); |
2063 | } |
2064 | } |
2065 | |
2066 | |
2067 | void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, |
2068 | RTLIB::Libcall Call_F32, |
2069 | RTLIB::Libcall Call_F64, |
2070 | RTLIB::Libcall Call_F80, |
2071 | RTLIB::Libcall Call_F128, |
2072 | RTLIB::Libcall Call_PPCF128, |
2073 | SmallVectorImpl<SDValue> &Results) { |
2074 | RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0), |
2075 | Call_F32, Call_F64, Call_F80, |
2076 | Call_F128, Call_PPCF128); |
2077 | ExpandFPLibCall(Node, LC, Results); |
2078 | } |
2079 | |
2080 | SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, |
2081 | RTLIB::Libcall Call_I8, |
2082 | RTLIB::Libcall Call_I16, |
2083 | RTLIB::Libcall Call_I32, |
2084 | RTLIB::Libcall Call_I64, |
2085 | RTLIB::Libcall Call_I128) { |
2086 | RTLIB::Libcall LC; |
2087 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2088 | default: llvm_unreachable("Unexpected request for libcall!"); |
2089 | case MVT::i8: LC = Call_I8; break; |
2090 | case MVT::i16: LC = Call_I16; break; |
2091 | case MVT::i32: LC = Call_I32; break; |
2092 | case MVT::i64: LC = Call_I64; break; |
2093 | case MVT::i128: LC = Call_I128; break; |
2094 | } |
2095 | return ExpandLibCall(LC, Node, isSigned); |
2096 | } |
2097 | |
2098 | |
2099 | |
2100 | void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node, |
2101 | RTLIB::Libcall Call_F32, |
2102 | RTLIB::Libcall Call_F64, |
2103 | RTLIB::Libcall Call_F80, |
2104 | RTLIB::Libcall Call_F128, |
2105 | RTLIB::Libcall Call_PPCF128, |
2106 | SmallVectorImpl<SDValue> &Results) { |
2107 | EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType(); |
2108 | RTLIB::Libcall LC = RTLIB::getFPLibCall(InVT.getSimpleVT(), |
2109 | Call_F32, Call_F64, Call_F80, |
2110 | Call_F128, Call_PPCF128); |
2111 | ExpandFPLibCall(Node, LC, Results); |
2112 | } |
2113 | |
2114 | |
2115 | void |
2116 | SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, |
2117 | SmallVectorImpl<SDValue> &Results) { |
2118 | unsigned Opcode = Node->getOpcode(); |
2119 | bool isSigned = Opcode == ISD::SDIVREM; |
2120 | |
2121 | RTLIB::Libcall LC; |
2122 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2123 | default: llvm_unreachable("Unexpected request for libcall!"); |
2124 | case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; |
2125 | case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; |
2126 | case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; |
2127 | case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; |
2128 | case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; |
2129 | } |
2130 | |
2131 | |
2132 | |
2133 | |
2134 | SDValue InChain = DAG.getEntryNode(); |
2135 | |
2136 | EVT RetVT = Node->getValueType(0); |
2137 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); |
2138 | |
2139 | TargetLowering::ArgListTy Args; |
2140 | TargetLowering::ArgListEntry Entry; |
2141 | for (const SDValue &Op : Node->op_values()) { |
2142 | EVT ArgVT = Op.getValueType(); |
2143 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
2144 | Entry.Node = Op; |
2145 | Entry.Ty = ArgTy; |
2146 | Entry.IsSExt = isSigned; |
2147 | Entry.IsZExt = !isSigned; |
2148 | Args.push_back(Entry); |
2149 | } |
2150 | |
2151 | |
2152 | SDValue FIPtr = DAG.CreateStackTemporary(RetVT); |
2153 | Entry.Node = FIPtr; |
2154 | Entry.Ty = RetTy->getPointerTo(); |
2155 | Entry.IsSExt = isSigned; |
2156 | Entry.IsZExt = !isSigned; |
2157 | Args.push_back(Entry); |
2158 | |
2159 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2160 | TLI.getPointerTy(DAG.getDataLayout())); |
2161 | |
2162 | SDLoc dl(Node); |
2163 | TargetLowering::CallLoweringInfo CLI(DAG); |
2164 | CLI.setDebugLoc(dl) |
2165 | .setChain(InChain) |
2166 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, |
2167 | std::move(Args)) |
2168 | .setSExtResult(isSigned) |
2169 | .setZExtResult(!isSigned); |
2170 | |
2171 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2172 | |
2173 | |
2174 | SDValue Rem = |
2175 | DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo()); |
2176 | Results.push_back(CallInfo.first); |
2177 | Results.push_back(Rem); |
2178 | } |
2179 | |
2180 | |
2181 | static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { |
2182 | RTLIB::Libcall LC; |
2183 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2184 | default: llvm_unreachable("Unexpected request for libcall!"); |
2185 | case MVT::f32: LC = RTLIB::SINCOS_F32; break; |
2186 | case MVT::f64: LC = RTLIB::SINCOS_F64; break; |
2187 | case MVT::f80: LC = RTLIB::SINCOS_F80; break; |
2188 | case MVT::f128: LC = RTLIB::SINCOS_F128; break; |
2189 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; |
2190 | } |
2191 | return TLI.getLibcallName(LC) != nullptr; |
2192 | } |
2193 | |
2194 | |
2195 | static bool useSinCos(SDNode *Node) { |
2196 | unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN |
2197 | ? ISD::FCOS : ISD::FSIN; |
2198 | |
2199 | SDValue Op0 = Node->getOperand(0); |
2200 | for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), |
2201 | UE = Op0.getNode()->use_end(); UI != UE; ++UI) { |
2202 | SDNode *User = *UI; |
2203 | if (User == Node) |
2204 | continue; |
2205 | |
2206 | if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) |
2207 | return true; |
2208 | } |
2209 | return false; |
2210 | } |
2211 | |
2212 | |
2213 | void |
2214 | SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, |
2215 | SmallVectorImpl<SDValue> &Results) { |
2216 | RTLIB::Libcall LC; |
2217 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2218 | default: llvm_unreachable("Unexpected request for libcall!"); |
2219 | case MVT::f32: LC = RTLIB::SINCOS_F32; break; |
2220 | case MVT::f64: LC = RTLIB::SINCOS_F64; break; |
2221 | case MVT::f80: LC = RTLIB::SINCOS_F80; break; |
2222 | case MVT::f128: LC = RTLIB::SINCOS_F128; break; |
2223 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; |
2224 | } |
2225 | |
2226 | |
2227 | |
2228 | |
2229 | SDValue InChain = DAG.getEntryNode(); |
2230 | |
2231 | EVT RetVT = Node->getValueType(0); |
2232 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); |
2233 | |
2234 | TargetLowering::ArgListTy Args; |
2235 | TargetLowering::ArgListEntry Entry; |
2236 | |
2237 | |
2238 | Entry.Node = Node->getOperand(0); |
2239 | Entry.Ty = RetTy; |
2240 | Entry.IsSExt = false; |
2241 | Entry.IsZExt = false; |
2242 | Args.push_back(Entry); |
2243 | |
2244 | |
2245 | SDValue SinPtr = DAG.CreateStackTemporary(RetVT); |
2246 | Entry.Node = SinPtr; |
2247 | Entry.Ty = RetTy->getPointerTo(); |
2248 | Entry.IsSExt = false; |
2249 | Entry.IsZExt = false; |
2250 | Args.push_back(Entry); |
2251 | |
2252 | |
2253 | SDValue CosPtr = DAG.CreateStackTemporary(RetVT); |
2254 | Entry.Node = CosPtr; |
2255 | Entry.Ty = RetTy->getPointerTo(); |
2256 | Entry.IsSExt = false; |
2257 | Entry.IsZExt = false; |
2258 | Args.push_back(Entry); |
2259 | |
2260 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2261 | TLI.getPointerTy(DAG.getDataLayout())); |
2262 | |
2263 | SDLoc dl(Node); |
2264 | TargetLowering::CallLoweringInfo CLI(DAG); |
2265 | CLI.setDebugLoc(dl).setChain(InChain).setLibCallee( |
2266 | TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee, |
2267 | std::move(Args)); |
2268 | |
2269 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2270 | |
2271 | Results.push_back( |
2272 | DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo())); |
2273 | Results.push_back( |
2274 | DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo())); |
2275 | } |
2276 | |
2277 | |
2278 | |
2279 | |
2280 | |
2281 | SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node, |
2282 | SDValue &Chain) { |
2283 | bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP || |
2284 | Node->getOpcode() == ISD::SINT_TO_FP); |
2285 | EVT DestVT = Node->getValueType(0); |
2286 | SDLoc dl(Node); |
2287 | unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0; |
2288 | SDValue Op0 = Node->getOperand(OpNo); |
2289 | EVT SrcVT = Op0.getValueType(); |
2290 | |
2291 | |
2292 | LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n"); |
2293 | if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) && |
2294 | (DestVT.bitsLE(MVT::f64) || |
2295 | TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND |
2296 | : ISD::FP_EXTEND, |
2297 | DestVT))) { |
2298 | LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double " |
2299 | "expansion\n"); |
2300 | |
2301 | |
2302 | SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); |
2303 | |
2304 | SDValue Lo = Op0; |
2305 | |
2306 | if (isSigned) { |
2307 | |
2308 | Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo, |
2309 | DAG.getConstant(0x80000000u, dl, MVT::i32)); |
2310 | } |
2311 | |
2312 | SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32); |
2313 | |
2314 | |
2315 | if (DAG.getDataLayout().isBigEndian()) |
2316 | std::swap(Lo, Hi); |
2317 | |
2318 | SDValue MemChain = DAG.getEntryNode(); |
2319 | |
2320 | |
2321 | SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot, |
2322 | MachinePointerInfo()); |
2323 | |
2324 | SDValue HiPtr = DAG.getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), dl); |
2325 | SDValue Store2 = |
2326 | DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo()); |
2327 | MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); |
2328 | |
2329 | |
2330 | SDValue Load = |
2331 | DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo()); |
2332 | |
2333 | SDValue Bias = DAG.getConstantFP(isSigned ? |
2334 | BitsToDouble(0x4330000080000000ULL) : |
2335 | BitsToDouble(0x4330000000000000ULL), |
2336 | dl, MVT::f64); |
2337 | |
2338 | SDValue Sub; |
2339 | SDValue Result; |
2340 | if (Node->isStrictFPOpcode()) { |
2341 | Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other}, |
2342 | {Node->getOperand(0), Load, Bias}); |
2343 | Chain = Sub.getValue(1); |
2344 | if (DestVT != Sub.getValueType()) { |
2345 | std::pair<SDValue, SDValue> ResultPair; |
2346 | ResultPair = |
2347 | DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT); |
2348 | Result = ResultPair.first; |
2349 | Chain = ResultPair.second; |
2350 | } |
2351 | else |
2352 | Result = Sub; |
2353 | } else { |
2354 | Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); |
2355 | Result = DAG.getFPExtendOrRound(Sub, dl, DestVT); |
2356 | } |
2357 | return Result; |
2358 | } |
2359 | |
2360 | if (isSigned) |
2361 | return SDValue(); |
2362 | |
2363 | |
2364 | if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) || |
2365 | (SrcVT == MVT::i64 && DestVT == MVT::f64)) { |
2366 | LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n"); |
2367 | |
2368 | |
2369 | |
2370 | |
2371 | |
2372 | |
2373 | |
2374 | |
2375 | |
2376 | |
2377 | |
2378 | |
2379 | |
2380 | |
2381 | EVT SetCCVT = getSetCCResultType(SrcVT); |
2382 | |
2383 | SDValue SignBitTest = DAG.getSetCC( |
2384 | dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT); |
2385 | |
2386 | EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout()); |
2387 | SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT); |
2388 | SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst); |
2389 | SDValue AndConst = DAG.getConstant(1, dl, SrcVT); |
2390 | SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst); |
2391 | SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr); |
2392 | |
2393 | SDValue Slow, Fast; |
2394 | if (Node->isStrictFPOpcode()) { |
2395 | |
2396 | |
2397 | SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0); |
2398 | Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, |
2399 | { Node->getOperand(0), InCvt }); |
2400 | Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, |
2401 | { Fast.getValue(1), Fast, Fast }); |
2402 | Chain = Slow.getValue(1); |
2403 | |
2404 | |
2405 | |
2406 | SDNodeFlags Flags; |
2407 | Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept()); |
2408 | Fast->setFlags(Flags); |
2409 | Flags.setNoFPExcept(true); |
2410 | Slow->setFlags(Flags); |
2411 | } else { |
2412 | SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or); |
2413 | Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt); |
2414 | Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); |
2415 | } |
2416 | |
2417 | return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast); |
2418 | } |
2419 | |
2420 | |
2421 | if (!TLI.isOperationLegalOrCustom( |
2422 | Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT)) |
2423 | return SDValue(); |
2424 | |
2425 | |
2426 | |
2427 | |
2428 | assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >= |
2429 | SrcVT.getSizeInBits() - 1 && |
2430 | "Cannot perform lossless SINT_TO_FP!"); |
2431 | |
2432 | SDValue Tmp1; |
2433 | if (Node->isStrictFPOpcode()) { |
2434 | Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, |
2435 | { Node->getOperand(0), Op0 }); |
2436 | } else |
2437 | Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); |
2438 | |
2439 | SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0, |
2440 | DAG.getConstant(0, dl, SrcVT), ISD::SETLT); |
2441 | SDValue Zero = DAG.getIntPtrConstant(0, dl), |
2442 | Four = DAG.getIntPtrConstant(4, dl); |
2443 | SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(), |
2444 | SignSet, Four, Zero); |
2445 | |
2446 | |
2447 | |
2448 | |
2449 | uint64_t FF; |
2450 | switch (SrcVT.getSimpleVT().SimpleTy) { |
2451 | default: |
2452 | return SDValue(); |
2453 | case MVT::i8 : FF = 0x43800000ULL; break; |
2454 | case MVT::i16: FF = 0x47800000ULL; break; |
2455 | case MVT::i32: FF = 0x4F800000ULL; break; |
2456 | case MVT::i64: FF = 0x5F800000ULL; break; |
2457 | } |
2458 | if (DAG.getDataLayout().isLittleEndian()) |
2459 | FF <<= 32; |
2460 | Constant *FudgeFactor = ConstantInt::get( |
2461 | Type::getInt64Ty(*DAG.getContext()), FF); |
2462 | |
2463 | SDValue CPIdx = |
2464 | DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout())); |
2465 | Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); |
2466 | CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); |
2467 | Alignment = commonAlignment(Alignment, 4); |
2468 | SDValue FudgeInReg; |
2469 | if (DestVT == MVT::f32) |
2470 | FudgeInReg = DAG.getLoad( |
2471 | MVT::f32, dl, DAG.getEntryNode(), CPIdx, |
2472 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), |
2473 | Alignment); |
2474 | else { |
2475 | SDValue Load = DAG.getExtLoad( |
2476 | ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx, |
2477 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, |
2478 | Alignment); |
2479 | HandleSDNode Handle(Load); |
2480 | LegalizeOp(Load.getNode()); |
2481 | FudgeInReg = Handle.getValue(); |
2482 | } |
2483 | |
2484 | if (Node->isStrictFPOpcode()) { |
2485 | SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, |
2486 | { Tmp1.getValue(1), Tmp1, FudgeInReg }); |
2487 | Chain = Result.getValue(1); |
2488 | return Result; |
2489 | } |
2490 | |
2491 | return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); |
2492 | } |
2493 | |
2494 | |
2495 | |
2496 | |
2497 | |
2498 | |
2499 | void SelectionDAGLegalize::PromoteLegalINT_TO_FP( |
2500 | SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) { |
2501 | bool IsStrict = N->isStrictFPOpcode(); |
2502 | bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP || |
2503 | N->getOpcode() == ISD::STRICT_SINT_TO_FP; |
2504 | EVT DestVT = N->getValueType(0); |
2505 | SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0); |
2506 | unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP; |
2507 | unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP; |
2508 | |
2509 | |
2510 | EVT NewInTy = LegalOp.getValueType(); |
2511 | |
2512 | unsigned OpToUse = 0; |
2513 | |
2514 | |
2515 | while (true) { |
2516 | NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); |
2517 | assert(NewInTy.isInteger() && "Ran out of possibilities!"); |
2518 | |
2519 | |
2520 | if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) { |
2521 | OpToUse = SIntOp; |
2522 | break; |
2523 | } |
2524 | if (IsSigned) |
2525 | continue; |
2526 | |
2527 | |
2528 | if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) { |
2529 | OpToUse = UIntOp; |
2530 | break; |
2531 | } |
2532 | |
2533 | |
2534 | } |
2535 | |
2536 | |
2537 | |
2538 | if (IsStrict) { |
2539 | SDValue Res = |
2540 | DAG.getNode(OpToUse, dl, {DestVT, MVT::Other}, |
2541 | {N->getOperand(0), |
2542 | DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, |
2543 | dl, NewInTy, LegalOp)}); |
2544 | Results.push_back(Res); |
2545 | Results.push_back(Res.getValue(1)); |
2546 | return; |
2547 | } |
2548 | |
2549 | Results.push_back( |
2550 | DAG.getNode(OpToUse, dl, DestVT, |
2551 | DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, |
2552 | dl, NewInTy, LegalOp))); |
2553 | } |
2554 | |
2555 | |
2556 | |
2557 | |
2558 | |
2559 | |
2560 | void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl, |
2561 | SmallVectorImpl<SDValue> &Results) { |
2562 | bool IsStrict = N->isStrictFPOpcode(); |
2563 | bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || |
2564 | N->getOpcode() == ISD::STRICT_FP_TO_SINT; |
2565 | EVT DestVT = N->getValueType(0); |
2566 | SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0); |
2567 | |
2568 | EVT NewOutTy = DestVT; |
2569 | |
2570 | unsigned OpToUse = 0; |
2571 | |
2572 | |
2573 | while (true) { |
2574 | NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); |
2575 | assert(NewOutTy.isInteger() && "Ran out of possibilities!"); |
2576 | |
2577 | |
2578 | |
2579 | OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT; |
2580 | if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy)) |
2581 | break; |
2582 | |
2583 | |
2584 | OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT; |
2585 | if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy)) |
2586 | break; |
2587 | |
2588 | |
2589 | } |
2590 | |
2591 | |
2592 | SDValue Operation; |
2593 | if (IsStrict) { |
2594 | SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other); |
2595 | Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp); |
2596 | } else |
2597 | Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); |
2598 | |
2599 | |
2600 | |
2601 | SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); |
2602 | Results.push_back(Trunc); |
2603 | if (IsStrict) |
2604 | Results.push_back(Operation.getValue(1)); |
2605 | } |
2606 | |
2607 | |
2608 | |
2609 | |
2610 | SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node, |
2611 | const SDLoc &dl) { |
2612 | unsigned Opcode = Node->getOpcode(); |
2613 | |
2614 | |
2615 | EVT NewOutTy = Node->getValueType(0); |
2616 | while (true) { |
2617 | NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1); |
2618 | assert(NewOutTy.isInteger() && "Ran out of possibilities!"); |
2619 | |
2620 | if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy)) |
2621 | break; |
2622 | } |
2623 | |
2624 | |
2625 | |
2626 | SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0), |
2627 | Node->getOperand(1)); |
2628 | return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); |
2629 | } |
2630 | |
2631 | |
2632 | SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) { |
2633 | EVT VT = Op.getValueType(); |
2634 | EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
2635 | unsigned Sz = VT.getScalarSizeInBits(); |
2636 | |
2637 | |
2638 | SDValue Result; |
2639 | if (TLI.isOperationLegal(ISD::CTPOP, VT)) { |
2640 | Result = DAG.getNode(ISD::CTPOP, dl, VT, Op); |
2641 | } else { |
2642 | Result = Op; |
2643 | for (unsigned i = Log2_32_Ceil(Sz); i != 0;) { |
2644 | SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result, |
2645 | DAG.getConstant(1ULL << (--i), dl, ShVT)); |
2646 | Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift); |
2647 | } |
2648 | } |
2649 | |
2650 | return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT)); |
2651 | } |
2652 | |
2653 | bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { |
2654 | LLVM_DEBUG(dbgs() << "Trying to expand node\n"); |
2655 | SmallVector<SDValue, 8> Results; |
2656 | SDLoc dl(Node); |
2657 | SDValue Tmp1, Tmp2, Tmp3, Tmp4; |
2658 | bool NeedInvert; |
2659 | switch (Node->getOpcode()) { |
2660 | case ISD::ABS: |
2661 | if (TLI.expandABS(Node, Tmp1, DAG)) |
2662 | Results.push_back(Tmp1); |
2663 | break; |
2664 | case ISD::CTPOP: |
2665 | if (TLI.expandCTPOP(Node, Tmp1, DAG)) |
2666 | Results.push_back(Tmp1); |
2667 | break; |
2668 | case ISD::CTLZ: |
2669 | case ISD::CTLZ_ZERO_UNDEF: |
2670 | if (TLI.expandCTLZ(Node, Tmp1, DAG)) |
2671 | Results.push_back(Tmp1); |
2672 | break; |
2673 | case ISD::CTTZ: |
2674 | case ISD::CTTZ_ZERO_UNDEF: |
2675 | if (TLI.expandCTTZ(Node, Tmp1, DAG)) |
2676 | Results.push_back(Tmp1); |
2677 | break; |
2678 | case ISD::BITREVERSE: |
2679 | if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG))) |
2680 | Results.push_back(Tmp1); |
2681 | break; |
2682 | case ISD::BSWAP: |
2683 | if ((Tmp1 = TLI.expandBSWAP(Node, DAG))) |
2684 | Results.push_back(Tmp1); |
2685 | break; |
2686 | case ISD::PARITY: |
2687 | Results.push_back(ExpandPARITY(Node->getOperand(0), dl)); |
2688 | break; |
2689 | case ISD::FRAMEADDR: |
2690 | case ISD::RETURNADDR: |
2691 | case ISD::FRAME_TO_ARGS_OFFSET: |
2692 | Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); |
2693 | break; |
2694 | case ISD::EH_DWARF_CFA: { |
2695 | SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl, |
2696 | TLI.getPointerTy(DAG.getDataLayout())); |
2697 | SDValue Offset = DAG.getNode(ISD::ADD, dl, |
2698 | CfaArg.getValueType(), |
2699 | DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl, |
2700 | CfaArg.getValueType()), |
2701 | CfaArg); |
2702 | SDValue FA = DAG.getNode( |
2703 | ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()), |
2704 | DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()))); |
2705 | Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(), |
2706 | FA, Offset)); |
2707 | break; |
2708 | } |
2709 | case ISD::FLT_ROUNDS_: |
2710 | Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0))); |
2711 | Results.push_back(Node->getOperand(0)); |
2712 | break; |
2713 | case ISD::EH_RETURN: |
2714 | case ISD::EH_LABEL: |
2715 | case ISD::PREFETCH: |
2716 | case ISD::VAEND: |
2717 | case ISD::EH_SJLJ_LONGJMP: |
2718 | |
2719 | |
2720 | Results.push_back(Node->getOperand(0)); |
2721 | break; |
2722 | case ISD::READCYCLECOUNTER: |
2723 | |
2724 | |
2725 | Results.append(Node->getNumValues() - 1, |
2726 | DAG.getConstant(0, dl, Node->getValueType(0))); |
2727 | Results.push_back(Node->getOperand(0)); |
2728 | break; |
2729 | case ISD::EH_SJLJ_SETJMP: |
2730 | |
2731 | |
2732 | Results.push_back(DAG.getConstant(0, dl, MVT::i32)); |
2733 | Results.push_back(Node->getOperand(0)); |
2734 | break; |
2735 | case ISD::ATOMIC_LOAD: { |
2736 | |
2737 | SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0)); |
2738 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
2739 | SDValue Swap = DAG.getAtomicCmpSwap( |
2740 | ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, |
2741 | Node->getOperand(0), Node->getOperand(1), Zero, Zero, |
2742 | cast<AtomicSDNode>(Node)->getMemOperand()); |
2743 | Results.push_back(Swap.getValue(0)); |
2744 | Results.push_back(Swap.getValue(1)); |
2745 | break; |
2746 | } |
2747 | case ISD::ATOMIC_STORE: { |
2748 | |
2749 | SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, |
2750 | cast<AtomicSDNode>(Node)->getMemoryVT(), |
2751 | Node->getOperand(0), |
2752 | Node->getOperand(1), Node->getOperand(2), |
2753 | cast<AtomicSDNode>(Node)->getMemOperand()); |
2754 | Results.push_back(Swap.getValue(1)); |
2755 | break; |
2756 | } |
2757 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { |
2758 | |
2759 | |
2760 | |
2761 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
2762 | SDValue Res = DAG.getAtomicCmpSwap( |
2763 | ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, |
2764 | Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), |
2765 | Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand()); |
2766 | |
2767 | SDValue ExtRes = Res; |
2768 | SDValue LHS = Res; |
2769 | SDValue RHS = Node->getOperand(1); |
2770 | |
2771 | EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT(); |
2772 | EVT OuterType = Node->getValueType(0); |
2773 | switch (TLI.getExtendForAtomicOps()) { |
2774 | case ISD::SIGN_EXTEND: |
2775 | LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res, |
2776 | DAG.getValueType(AtomicType)); |
2777 | RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType, |
2778 | Node->getOperand(2), DAG.getValueType(AtomicType)); |
2779 | ExtRes = LHS; |
2780 | break; |
2781 | case ISD::ZERO_EXTEND: |
2782 | LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res, |
2783 | DAG.getValueType(AtomicType)); |
2784 | RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); |
2785 | ExtRes = LHS; |
2786 | break; |
2787 | case ISD::ANY_EXTEND: |
2788 | LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType); |
2789 | RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); |
2790 | break; |
2791 | default: |
2792 | llvm_unreachable("Invalid atomic op extension"); |
2793 | } |
2794 | |
2795 | SDValue Success = |
2796 | DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); |
2797 | |
2798 | Results.push_back(ExtRes.getValue(0)); |
2799 | Results.push_back(Success); |
2800 | Results.push_back(Res.getValue(1)); |
2801 | break; |
2802 | } |
2803 | case ISD::DYNAMIC_STACKALLOC: |
2804 | ExpandDYNAMIC_STACKALLOC(Node, Results); |
2805 | break; |
2806 | case ISD::MERGE_VALUES: |
2807 | for (unsigned i = 0; i < Node->getNumValues(); i++) |
2808 | Results.push_back(Node->getOperand(i)); |
2809 | break; |
2810 | case ISD::UNDEF: { |
2811 | EVT VT = Node->getValueType(0); |
2812 | if (VT.isInteger()) |
2813 | Results.push_back(DAG.getConstant(0, dl, VT)); |
2814 | else { |
2815 | assert(VT.isFloatingPoint() && "Unknown value type!"); |
2816 | Results.push_back(DAG.getConstantFP(0, dl, VT)); |
2817 | } |
2818 | break; |
2819 | } |
2820 | case ISD::STRICT_FP_ROUND: |
2821 | |
2822 | |
2823 | if (TLI.isStrictFPEnabled()) |
2824 | break; |
2825 | |
2826 | |
2827 | if (TLI.getStrictFPOperationAction(Node->getOpcode(), |
2828 | Node->getValueType(0)) |
2829 | == TargetLowering::Legal) |
2830 | break; |
2831 | |
2832 | |
2833 | if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0), |
2834 | Node->getValueType(0), dl, |
2835 | Node->getOperand(0)))) { |
2836 | ReplaceNode(Node, Tmp1.getNode()); |
2837 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n"); |
2838 | return true; |
2839 | } |
2840 | break; |
2841 | case ISD::FP_ROUND: |
2842 | case ISD::BITCAST: |
2843 | if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), |
2844 | Node->getValueType(0), dl))) |
2845 | Results.push_back(Tmp1); |
2846 | break; |
2847 | case ISD::STRICT_FP_EXTEND: |
2848 | |
2849 | |
2850 | if (TLI.isStrictFPEnabled()) |
2851 | break; |
2852 | |
2853 | |
2854 | if (TLI.getStrictFPOperationAction(Node->getOpcode(), |
2855 | Node->getValueType(0)) |
2856 | == TargetLowering::Legal) |
2857 | break; |
2858 | |
2859 | |
2860 | if ((Tmp1 = EmitStackConvert( |
2861 | Node->getOperand(1), Node->getOperand(1).getValueType(), |
2862 | Node->getValueType(0), dl, Node->getOperand(0)))) { |
2863 | ReplaceNode(Node, Tmp1.getNode()); |
2864 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n"); |
2865 | return true; |
2866 | } |
2867 | break; |
2868 | case ISD::FP_EXTEND: |
2869 | if ((Tmp1 = EmitStackConvert(Node->getOperand(0), |
2870 | Node->getOperand(0).getValueType(), |
2871 | Node->getValueType(0), dl))) |
2872 | Results.push_back(Tmp1); |
2873 | break; |
2874 | case ISD::SIGN_EXTEND_INREG: { |
2875 | EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); |
2876 | EVT VT = Node->getValueType(0); |
2877 | |
2878 | |
2879 | |
2880 | |
2881 | |
2882 | |
2883 | |
2884 | |
2885 | if (ExtraVT.getSizeInBits() == 1) { |
2886 | SDValue One = DAG.getConstant(1, dl, VT); |
2887 | SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); |
2888 | SDValue Zero = DAG.getConstant(0, dl, VT); |
2889 | SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And); |
2890 | Results.push_back(Neg); |
2891 | break; |
2892 | } |
2893 | |
2894 | |
2895 | |
2896 | EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
2897 | unsigned BitsDiff = VT.getScalarSizeInBits() - |
2898 | ExtraVT.getScalarSizeInBits(); |
2899 | SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy); |
2900 | Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), |
2901 | Node->getOperand(0), ShiftCst); |
2902 | Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); |
2903 | Results.push_back(Tmp1); |
2904 | break; |
2905 | } |
2906 | case ISD::UINT_TO_FP: |
2907 | case ISD::STRICT_UINT_TO_FP: |
2908 | if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) { |
2909 | Results.push_back(Tmp1); |
2910 | if (Node->isStrictFPOpcode()) |
2911 | Results.push_back(Tmp2); |
2912 | break; |
2913 | } |
2914 | LLVM_FALLTHROUGH; |
2915 | case ISD::SINT_TO_FP: |
2916 | case ISD::STRICT_SINT_TO_FP: |
2917 | if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) { |
2918 | Results.push_back(Tmp1); |
2919 | if (Node->isStrictFPOpcode()) |
2920 | Results.push_back(Tmp2); |
2921 | } |
2922 | break; |
2923 | case ISD::FP_TO_SINT: |
2924 | if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) |
2925 | Results.push_back(Tmp1); |
2926 | break; |
2927 | case ISD::STRICT_FP_TO_SINT: |
2928 | if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) { |
2929 | ReplaceNode(Node, Tmp1.getNode()); |
2930 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n"); |
2931 | return true; |
2932 | } |
2933 | break; |
2934 | case ISD::FP_TO_UINT: |
2935 | if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) |
2936 | Results.push_back(Tmp1); |
2937 | break; |
2938 | case ISD::STRICT_FP_TO_UINT: |
2939 | if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) { |
2940 | |
2941 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2); |
2942 | |
2943 | ReplaceNodeWithValue(SDValue(Node, 0), Tmp1); |
2944 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n"); |
2945 | return true; |
2946 | } |
2947 | break; |
2948 | case ISD::FP_TO_SINT_SAT: |
2949 | case ISD::FP_TO_UINT_SAT: |
2950 | Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG)); |
2951 | break; |
2952 | case ISD::VAARG: |
2953 | Results.push_back(DAG.expandVAArg(Node)); |
2954 | Results.push_back(Results[0].getValue(1)); |
2955 | break; |
2956 | case ISD::VACOPY: |
2957 | Results.push_back(DAG.expandVACopy(Node)); |
2958 | break; |
2959 | case ISD::EXTRACT_VECTOR_ELT: |
2960 | if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) |
2961 | |
2962 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), |
2963 | Node->getOperand(0)); |
2964 | else |
2965 | Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); |
2966 | Results.push_back(Tmp1); |
2967 | break; |
2968 | case ISD::EXTRACT_SUBVECTOR: |
2969 | Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); |
2970 | break; |
2971 | case ISD::INSERT_SUBVECTOR: |
2972 | Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); |
2973 | break; |
2974 | case ISD::CONCAT_VECTORS: |
2975 | Results.push_back(ExpandVectorBuildThroughStack(Node)); |
2976 | break; |
2977 | case ISD::SCALAR_TO_VECTOR: |
2978 | Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); |
2979 | break; |
2980 | case ISD::INSERT_VECTOR_ELT: |
2981 | Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), |
2982 | Node->getOperand(1), |
2983 | Node->getOperand(2), dl)); |
2984 | break; |
2985 | case ISD::VECTOR_SHUFFLE: { |
2986 | SmallVector<int, 32> NewMask; |
2987 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); |
2988 | |
2989 | EVT VT = Node->getValueType(0); |
2990 | EVT EltVT = VT.getVectorElementType(); |
2991 | SDValue Op0 = Node->getOperand(0); |
2992 | SDValue Op1 = Node->getOperand(1); |
2993 | if (!TLI.isTypeLegal(EltVT)) { |
2994 | EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); |
2995 | |
2996 | |
2997 | |
2998 | |
2999 | if (NewEltVT.bitsLT(EltVT)) { |
3000 | |
3001 | |
3002 | |
3003 | |
3004 | |
3005 | EVT NewVT = |
3006 | EVT::getVectorVT(*DAG.getContext(), NewEltVT, |
3007 | VT.getSizeInBits() / NewEltVT.getSizeInBits()); |
3008 | assert(NewVT.bitsEq(VT)); |
3009 | |
3010 | |
3011 | Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); |
3012 | Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); |
3013 | |
3014 | |
3015 | unsigned int factor = |
3016 | NewVT.getVectorNumElements()/VT.getVectorNumElements(); |
3017 | |
3018 | |
3019 | assert(factor > 0); |
3020 | |
3021 | for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { |
3022 | if (Mask[i] < 0) { |
3023 | for (unsigned fi = 0; fi < factor; ++fi) |
3024 | NewMask.push_back(Mask[i]); |
3025 | } |
3026 | else { |
3027 | for (unsigned fi = 0; fi < factor; ++fi) |
3028 | NewMask.push_back(Mask[i]*factor+fi); |
3029 | } |
3030 | } |
3031 | Mask = NewMask; |
3032 | VT = NewVT; |
3033 | } |
3034 | EltVT = NewEltVT; |
3035 | } |
3036 | unsigned NumElems = VT.getVectorNumElements(); |
3037 | SmallVector<SDValue, 16> Ops; |
3038 | for (unsigned i = 0; i != NumElems; ++i) { |
3039 | if (Mask[i] < 0) { |
3040 | Ops.push_back(DAG.getUNDEF(EltVT)); |
3041 | continue; |
3042 | } |
3043 | unsigned Idx = Mask[i]; |
3044 | if (Idx < NumElems) |
3045 | Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, |
3046 | DAG.getVectorIdxConstant(Idx, dl))); |
3047 | else |
3048 | Ops.push_back( |
3049 | DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1, |
3050 | DAG.getVectorIdxConstant(Idx - NumElems, dl))); |
3051 | } |
3052 | |
3053 | Tmp1 = DAG.getBuildVector(VT, dl, Ops); |
3054 | |
3055 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); |
3056 | Results.push_back(Tmp1); |
3057 | break; |
3058 | } |
3059 | case ISD::VECTOR_SPLICE: { |
3060 | Results.push_back(TLI.expandVectorSplice(Node, DAG)); |
3061 | break; |
3062 | } |
3063 | case ISD::EXTRACT_ELEMENT: { |
3064 | EVT OpTy = Node->getOperand(0).getValueType(); |
3065 | if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { |
3066 | |
3067 | Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), |
3068 | DAG.getConstant(OpTy.getSizeInBits() / 2, dl, |
3069 | TLI.getShiftAmountTy( |
3070 | Node->getOperand(0).getValueType(), |
3071 | DAG.getDataLayout()))); |
3072 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); |
3073 | } else { |
3074 | |
3075 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), |
3076 | Node->getOperand(0)); |
3077 | } |
3078 | Results.push_back(Tmp1); |
3079 | break; |
3080 | } |
3081 | case ISD::STACKSAVE: |
3082 | |
3083 | |
3084 | if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) { |
3085 | Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, |
3086 | Node->getValueType(0))); |
3087 | Results.push_back(Results[0].getValue(1)); |
3088 | } else { |
3089 | Results.push_back(DAG.getUNDEF(Node->getValueType(0))); |
3090 | Results.push_back(Node->getOperand(0)); |
3091 | } |
3092 | break; |
3093 | case ISD::STACKRESTORE: |
3094 | |
3095 | |
3096 | if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) { |
3097 | Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, |
3098 | Node->getOperand(1))); |
3099 | } else { |
3100 | Results.push_back(Node->getOperand(0)); |
3101 | } |
3102 | break; |
3103 | case ISD::GET_DYNAMIC_AREA_OFFSET: |
3104 | Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); |
3105 | Results.push_back(Results[0].getValue(0)); |
3106 | break; |
3107 | case ISD::FCOPYSIGN: |
3108 | Results.push_back(ExpandFCOPYSIGN(Node)); |
3109 | break; |
3110 | case ISD::FNEG: |
3111 | Results.push_back(ExpandFNEG(Node)); |
3112 | break; |
3113 | case ISD::FABS: |
3114 | Results.push_back(ExpandFABS(Node)); |
3115 | break; |
3116 | case ISD::SMIN: |
3117 | case ISD::SMAX: |
3118 | case ISD::UMIN: |
3119 | case ISD::UMAX: { |
3120 | |
3121 | ISD::CondCode Pred; |
3122 | switch (Node->getOpcode()) { |
3123 | default: llvm_unreachable("How did we get here?"); |
3124 | case ISD::SMAX: Pred = ISD::SETGT; break; |
3125 | case ISD::SMIN: Pred = ISD::SETLT; break; |
3126 | case ISD::UMAX: Pred = ISD::SETUGT; break; |
3127 | case ISD::UMIN: Pred = ISD::SETULT; break; |
3128 | } |
3129 | Tmp1 = Node->getOperand(0); |
3130 | Tmp2 = Node->getOperand(1); |
3131 | Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred); |
3132 | Results.push_back(Tmp1); |
3133 | break; |
3134 | } |
3135 | case ISD::FMINNUM: |
3136 | case ISD::FMAXNUM: { |
3137 | if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) |
3138 | Results.push_back(Expanded); |
3139 | break; |
3140 | } |
3141 | case ISD::FSIN: |
3142 | case ISD::FCOS: { |
3143 | EVT VT = Node->getValueType(0); |
3144 | |
3145 | |
3146 | if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || |
3147 | isSinCosLibcallAvailable(Node, TLI)) |
3148 | && useSinCos(Node)) { |
3149 | SDVTList VTs = DAG.getVTList(VT, VT); |
3150 | Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); |
3151 | if (Node->getOpcode() == ISD::FCOS) |
3152 | Tmp1 = Tmp1.getValue(1); |
3153 | Results.push_back(Tmp1); |
3154 | } |
3155 | break; |
3156 | } |
3157 | case ISD::FMAD: |
3158 | llvm_unreachable("Illegal fmad should never be formed"); |
3159 | |
3160 | case ISD::FP16_TO_FP: |
3161 | if (Node->getValueType(0) != MVT::f32) { |
3162 | |
3163 | |
3164 | |
3165 | SDValue Res = |
3166 | DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); |
3167 | Results.push_back( |
3168 | DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); |
3169 | } |
3170 | break; |
3171 | case ISD::STRICT_FP16_TO_FP: |
3172 | if (Node->getValueType(0) != MVT::f32) { |
3173 | |
3174 | |
3175 | |
3176 | SDValue Res = |
3177 | DAG.getNode(ISD::STRICT_FP16_TO_FP, dl, {MVT::f32, MVT::Other}, |
3178 | {Node->getOperand(0), Node->getOperand(1)}); |
3179 | Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, |
3180 | {Node->getValueType(0), MVT::Other}, |
3181 | {Res.getValue(1), Res}); |
3182 | Results.push_back(Res); |
3183 | Results.push_back(Res.getValue(1)); |
3184 | } |
3185 | break; |
3186 | case ISD::FP_TO_FP16: |
3187 | LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n"); |
3188 | if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) { |
3189 | SDValue Op = Node->getOperand(0); |
3190 | MVT SVT = Op.getSimpleValueType(); |
3191 | if ((SVT == MVT::f64 || SVT == MVT::f80) && |
3192 | TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) { |
3193 | |
3194 | |
3195 | SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, |
3196 | DAG.getIntPtrConstant(0, dl)); |
3197 | Results.push_back( |
3198 | DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); |
3199 | } |
3200 | } |
3201 | break; |
3202 | case ISD::ConstantFP: { |
3203 | ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); |
3204 | |
3205 | |
3206 | if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0), |
3207 | DAG.shouldOptForSize())) |
3208 | Results.push_back(ExpandConstantFP(CFP, true)); |
3209 | break; |
3210 | } |
3211 | case ISD::Constant: { |
3212 | ConstantSDNode *CP = cast<ConstantSDNode>(Node); |
3213 | Results.push_back(ExpandConstant(CP)); |
3214 | break; |
3215 | } |
3216 | case ISD::FSUB: { |
3217 | EVT VT = Node->getValueType(0); |
3218 | if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) && |
3219 | TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) { |
3220 | const SDNodeFlags Flags = Node->getFlags(); |
3221 | Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); |
3222 | Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); |
3223 | Results.push_back(Tmp1); |
3224 | } |
3225 | break; |
3226 | } |
3227 | case ISD::SUB: { |
3228 | EVT VT = Node->getValueType(0); |
3229 | assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && |
3230 | TLI.isOperationLegalOrCustom(ISD::XOR, VT) && |
3231 | "Don't know how to expand this subtraction!"); |
3232 | Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), |
3233 | DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl, |
3234 | VT)); |
3235 | Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT)); |
3236 | Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); |
3237 | break; |
3238 | } |
3239 | case ISD::UREM: |
3240 | case ISD::SREM: |
3241 | if (TLI.expandREM(Node, Tmp1, DAG)) |
3242 | Results.push_back(Tmp1); |
3243 | break; |
3244 | case ISD::UDIV: |
3245 | case ISD::SDIV: { |
3246 | bool isSigned = Node->getOpcode() == ISD::SDIV; |
3247 | unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; |
3248 | EVT VT = Node->getValueType(0); |
3249 | if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) { |
3250 | SDVTList VTs = DAG.getVTList(VT, VT); |
3251 | Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), |
3252 | Node->getOperand(1)); |
3253 | Results.push_back(Tmp1); |
3254 | } |
3255 | break; |
3256 | } |
3257 | case ISD::MULHU: |
3258 | case ISD::MULHS: { |
3259 | unsigned ExpandOpcode = |
3260 | Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; |
3261 | EVT VT = Node->getValueType(0); |
3262 | SDVTList VTs = DAG.getVTList(VT, VT); |
3263 | |
3264 | Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), |
3265 | Node->getOperand(1)); |
3266 | Results.push_back(Tmp1.getValue(1)); |
3267 | break; |
3268 | } |
3269 | case ISD::UMUL_LOHI: |
3270 | case ISD::SMUL_LOHI: { |
3271 | SDValue LHS = Node->getOperand(0); |
3272 | SDValue RHS = Node->getOperand(1); |
3273 | MVT VT = LHS.getSimpleValueType(); |
3274 | unsigned MULHOpcode = |
3275 | Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; |
3276 | |
3277 | if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) { |
3278 | Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS)); |
3279 | Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS)); |
3280 | break; |
3281 | } |
3282 | |
3283 | SmallVector<SDValue, 4> Halves; |
3284 | EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext()); |
3285 | assert(TLI.isTypeLegal(HalfType)); |
3286 | if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves, |
3287 | HalfType, DAG, |
3288 | TargetLowering::MulExpansionKind::Always)) { |
3289 | for (unsigned i = 0; i < 2; ++i) { |
3290 | SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]); |
3291 | SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]); |
3292 | SDValue Shift = DAG.getConstant( |
3293 | HalfType.getScalarSizeInBits(), dl, |
3294 | TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); |
3295 | Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); |
3296 | Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); |
3297 | } |
3298 | break; |
3299 | } |
3300 | break; |
3301 | } |
3302 | case ISD::MUL: { |
3303 | EVT VT = Node->getValueType(0); |
3304 | SDVTList VTs = DAG.getVTList(VT, VT); |
3305 | |
3306 | |
3307 | |
3308 | |
3309 | |
3310 | bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); |
3311 | bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); |
3312 | bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); |
3313 | bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); |
3314 | unsigned OpToUse = 0; |
3315 | if (HasSMUL_LOHI && !HasMULHS) { |
3316 | OpToUse = ISD::SMUL_LOHI; |
3317 | } else if (HasUMUL_LOHI && !HasMULHU) { |
3318 | OpToUse = ISD::UMUL_LOHI; |
3319 | } else if (HasSMUL_LOHI) { |
3320 | OpToUse = ISD::SMUL_LOHI; |
3321 | } else if (HasUMUL_LOHI) { |
3322 | OpToUse = ISD::UMUL_LOHI; |
3323 | } |
3324 | if (OpToUse) { |
3325 | Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), |
3326 | Node->getOperand(1))); |
3327 | break; |
3328 | } |
3329 | |
3330 | SDValue Lo, Hi; |
3331 | EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext()); |
3332 | if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) && |
3333 | TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && |
3334 | TLI.isOperationLegalOrCustom(ISD::SHL, VT) && |
3335 | TLI.isOperationLegalOrCustom(ISD::OR, VT) && |
3336 | TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, |
3337 | TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) { |
3338 | Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); |
3339 | Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); |
3340 | SDValue Shift = |
3341 | DAG.getConstant(HalfType.getSizeInBits(), dl, |
3342 | TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); |
3343 | Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); |
3344 | Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); |
3345 | } |
3346 | break; |
3347 | } |
3348 | case ISD::FSHL: |
3349 | case ISD::FSHR: |
3350 | if (TLI.expandFunnelShift(Node, Tmp1, DAG)) |
3351 | Results.push_back(Tmp1); |
3352 | break; |
3353 | case ISD::ROTL: |
3354 | case ISD::ROTR: |
3355 | if (TLI.expandROT(Node, true , Tmp1, DAG)) |
3356 | Results.push_back(Tmp1); |
3357 | break; |
3358 | case ISD::SADDSAT: |
3359 | case ISD::UADDSAT: |
3360 | case ISD::SSUBSAT: |
3361 | case ISD::USUBSAT: |
3362 | Results.push_back(TLI.expandAddSubSat(Node, DAG)); |
3363 | break; |
3364 | case ISD::SSHLSAT: |
3365 | case ISD::USHLSAT: |
3366 | Results.push_back(TLI.expandShlSat(Node, DAG)); |
3367 | break; |
3368 | case ISD::SMULFIX: |
3369 | case ISD::SMULFIXSAT: |
3370 | case ISD::UMULFIX: |
3371 | case ISD::UMULFIXSAT: |
3372 | Results.push_back(TLI.expandFixedPointMul(Node, DAG)); |
3373 | break; |
3374 | case ISD::SDIVFIX: |
3375 | case ISD::SDIVFIXSAT: |
3376 | case ISD::UDIVFIX: |
3377 | case ISD::UDIVFIXSAT: |
3378 | if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node), |
3379 | Node->getOperand(0), |
3380 | Node->getOperand(1), |
3381 | Node->getConstantOperandVal(2), |
3382 | DAG)) { |
3383 | Results.push_back(V); |
3384 | break; |
3385 | } |
3386 | |
3387 | |
3388 | |
3389 | |
3390 | |
3391 | |
3392 | |
3393 | llvm_unreachable("Cannot expand DIVFIX!"); |
3394 | case ISD::ADDCARRY: |
3395 | case ISD::SUBCARRY: { |
3396 | SDValue LHS = Node->getOperand(0); |
3397 | SDValue RHS = Node->getOperand(1); |
3398 | SDValue Carry = Node->getOperand(2); |
3399 | |
3400 | bool IsAdd = Node->getOpcode() == ISD::ADDCARRY; |
3401 | |
3402 | |
3403 | unsigned Op = IsAdd ? ISD::ADD : ISD::SUB; |
3404 | EVT VT = LHS.getValueType(); |
3405 | SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS); |
3406 | |
3407 | |
3408 | EVT CarryType = Node->getValueType(1); |
3409 | EVT SetCCType = getSetCCResultType(Node->getValueType(0)); |
3410 | ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; |
3411 | SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC); |
3412 | |
3413 | |
3414 | SDValue One = DAG.getConstant(1, dl, VT); |
3415 | SDValue CarryExt = |
3416 | DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One); |
3417 | SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt); |
3418 | |
3419 | |
3420 | |
3421 | |
3422 | |
3423 | SDValue Zero = DAG.getConstant(0, dl, VT); |
3424 | SDValue Overflow2 = |
3425 | IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ) |
3426 | : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ); |
3427 | Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2, |
3428 | DAG.getZExtOrTrunc(Carry, dl, SetCCType)); |
3429 | |
3430 | SDValue ResultCarry = |
3431 | DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2); |
3432 | |
3433 | Results.push_back(Sum2); |
3434 | Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT)); |
3435 | break; |
3436 | } |
3437 | case ISD::SADDO: |
3438 | case ISD::SSUBO: { |
3439 | SDValue Result, Overflow; |
3440 | TLI.expandSADDSUBO(Node, Result, Overflow, DAG); |
3441 | Results.push_back(Result); |
3442 | Results.push_back(Overflow); |
3443 | break; |
3444 | } |
3445 | case ISD::UADDO: |
3446 | case ISD::USUBO: { |
3447 | SDValue Result, Overflow; |
3448 | TLI.expandUADDSUBO(Node, Result, Overflow, DAG); |
3449 | Results.push_back(Result); |
3450 | Results.push_back(Overflow); |
3451 | break; |
3452 | } |
3453 | case ISD::UMULO: |
3454 | case ISD::SMULO: { |
3455 | SDValue Result, Overflow; |
3456 | if (TLI.expandMULO(Node, Result, Overflow, DAG)) { |
3457 | Results.push_back(Result); |
3458 | Results.push_back(Overflow); |
3459 | } |
3460 | break; |
3461 | } |
3462 | case ISD::BUILD_PAIR: { |
3463 | EVT PairTy = Node->getValueType(0); |
3464 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); |
3465 | Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); |
3466 | Tmp2 = DAG.getNode( |
3467 | ISD::SHL, dl, PairTy, Tmp2, |
3468 | DAG.getConstant(PairTy.getSizeInBits() / 2, dl, |
3469 | TLI.getShiftAmountTy(PairTy, DAG.getDataLayout()))); |
3470 | Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); |
3471 | break; |
3472 | } |
3473 | case ISD::SELECT: |
3474 | Tmp1 = Node->getOperand(0); |
3475 | Tmp2 = Node->getOperand(1); |
3476 | Tmp3 = Node->getOperand(2); |
3477 | if (Tmp1.getOpcode() == ISD::SETCC) { |
3478 | Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), |
3479 | Tmp2, Tmp3, |
3480 | cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); |
3481 | } else { |
3482 | Tmp1 = DAG.getSelectCC(dl, Tmp1, |
3483 | DAG.getConstant(0, dl, Tmp1.getValueType()), |
3484 | Tmp2, Tmp3, ISD::SETNE); |
3485 | } |
3486 | Tmp1->setFlags(Node->getFlags()); |
3487 | Results.push_back(Tmp1); |
3488 | break; |
3489 | case ISD::BR_JT: { |
3490 | SDValue Chain = Node->getOperand(0); |
3491 | SDValue Table = Node->getOperand(1); |
3492 | SDValue Index = Node->getOperand(2); |
3493 | |
3494 | const DataLayout &TD = DAG.getDataLayout(); |
3495 | EVT PTy = TLI.getPointerTy(TD); |
3496 | |
3497 | unsigned EntrySize = |
3498 | DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); |
3499 | |
3500 | |
3501 | |
3502 | |
3503 | |
3504 | if (llvm::isPowerOf2_32(EntrySize)) |
3505 | Index = DAG.getNode( |
3506 | ISD::SHL, dl, Index.getValueType(), Index, |
3507 | DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType())); |
3508 | else |
3509 | Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index, |
3510 | DAG.getConstant(EntrySize, dl, Index.getValueType())); |
3511 | SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), |
3512 | Index, Table); |
3513 | |
3514 | EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); |
3515 | SDValue LD = DAG.getExtLoad( |
3516 | ISD::SEXTLOAD, dl, PTy, Chain, Addr, |
3517 | MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT); |
3518 | Addr = LD; |
3519 | if (TLI.isJumpTableRelative()) { |
3520 | |
3521 | |
3522 | |
3523 | Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, |
3524 | TLI.getPICJumpTableRelocBase(Table, DAG)); |
3525 | } |
3526 | |
3527 | Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG); |
3528 | Results.push_back(Tmp1); |
3529 | break; |
3530 | } |
3531 | case ISD::BRCOND: |
3532 | |
3533 | |
3534 | Tmp1 = Node->getOperand(0); |
3535 | Tmp2 = Node->getOperand(1); |
3536 | if (Tmp2.getOpcode() == ISD::SETCC) { |
3537 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, |
3538 | Tmp1, Tmp2.getOperand(2), |
3539 | Tmp2.getOperand(0), Tmp2.getOperand(1), |
3540 | Node->getOperand(2)); |
3541 | } else { |
3542 | |
3543 | if (Tmp2.isUndef() || |
3544 | (Tmp2.getOpcode() == ISD::AND && |
3545 | isa<ConstantSDNode>(Tmp2.getOperand(1)) && |
3546 | cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1)) |
3547 | Tmp3 = Tmp2; |
3548 | else |
3549 | Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, |
3550 | DAG.getConstant(1, dl, Tmp2.getValueType())); |
3551 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, |
3552 | DAG.getCondCode(ISD::SETNE), Tmp3, |
3553 | DAG.getConstant(0, dl, Tmp3.getValueType()), |
3554 | Node->getOperand(2)); |
3555 | } |
3556 | Results.push_back(Tmp1); |
3557 | break; |
3558 | case ISD::SETCC: |
3559 | case ISD::STRICT_FSETCC: |
3560 | case ISD::STRICT_FSETCCS: { |
3561 | bool IsStrict = Node->getOpcode() != ISD::SETCC; |
3562 | bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; |
3563 | SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); |
3564 | unsigned Offset = IsStrict ? 1 : 0; |
3565 | Tmp1 = Node->getOperand(0 + Offset); |
3566 | Tmp2 = Node->getOperand(1 + Offset); |
3567 | Tmp3 = Node->getOperand(2 + Offset); |
3568 | bool Legalized = |
3569 | TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, |
3570 | NeedInvert, dl, Chain, IsSignaling); |
3571 | |
3572 | if (Legalized) { |
3573 | |
3574 | |
3575 | if (Tmp3.getNode()) |
3576 | Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), |
3577 | Tmp1, Tmp2, Tmp3, Node->getFlags()); |
3578 | |
3579 | |
3580 | |
3581 | if (NeedInvert) |
3582 | Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0)); |
3583 | |
3584 | Results.push_back(Tmp1); |
3585 | if (IsStrict) |
3586 | Results.push_back(Chain); |
3587 | |
3588 | break; |
3589 | } |
3590 | |
3591 | |
3592 | |
3593 | assert(!IsStrict && "Don't know how to expand for strict nodes."); |
3594 | |
3595 | |
3596 | |
3597 | EVT VT = Node->getValueType(0); |
3598 | int TrueValue; |
3599 | switch (TLI.getBooleanContents(Tmp1.getValueType())) { |
3600 | case TargetLowering::ZeroOrOneBooleanContent: |
3601 | case TargetLowering::UndefinedBooleanContent: |
3602 | TrueValue = 1; |
3603 | break; |
3604 | case TargetLowering::ZeroOrNegativeOneBooleanContent: |
3605 | TrueValue = -1; |
3606 | break; |
3607 | } |
3608 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, |
3609 | DAG.getConstant(TrueValue, dl, VT), |
3610 | DAG.getConstant(0, dl, VT), |
3611 | Tmp3); |
3612 | Tmp1->setFlags(Node->getFlags()); |
3613 | Results.push_back(Tmp1); |
3614 | break; |
3615 | } |
3616 | case ISD::SELECT_CC: { |
3617 | |
3618 | Tmp1 = Node->getOperand(0); |
3619 | Tmp2 = Node->getOperand(1); |
3620 | Tmp3 = Node->getOperand(2); |
3621 | Tmp4 = Node->getOperand(3); |
3622 | EVT VT = Node->getValueType(0); |
3623 | SDValue Chain; |
3624 | SDValue CC = Node->getOperand(4); |
3625 | ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); |
3626 | |
3627 | if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) { |
3628 | |
3629 | |
3630 | EVT CmpVT = Tmp1.getValueType(); |
3631 | assert(!TLI.isOperationExpand(ISD::SELECT, VT) && |
3632 | "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be " |
3633 | "expanded."); |
3634 | EVT CCVT = getSetCCResultType(CmpVT); |
3635 | SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); |
3636 | Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4)); |
3637 | break; |
3638 | } |
3639 | |
3640 | |
3641 | bool Legalized = false; |
3642 | |
3643 | |
3644 | |
3645 | ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType()); |
3646 | if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) { |
3647 | |
3648 | Legalized = true; |
3649 | Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); |
3650 | Tmp1->setFlags(Node->getFlags()); |
3651 | } else { |
3652 | |
3653 | |
3654 | ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); |
3655 | if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) { |
3656 | |
3657 | |
3658 | Legalized = true; |
3659 | Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); |
3660 | Tmp1->setFlags(Node->getFlags()); |
3661 | } |
3662 | } |
3663 | |
3664 | if (!Legalized) { |
3665 | Legalized = TLI.LegalizeSetCCCondCode( |
3666 | DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, |
3667 | NeedInvert, dl, Chain); |
3668 | |
3669 | assert(Legalized && "Can't legalize SELECT_CC with legal condition!"); |
3670 | |
3671 | |
3672 | |
3673 | if (NeedInvert) |
3674 | std::swap(Tmp3, Tmp4); |
3675 | |
3676 | |
3677 | |
3678 | if (CC.getNode()) { |
3679 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), |
3680 | Tmp1, Tmp2, Tmp3, Tmp4, CC); |
3681 | } else { |
3682 | Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType()); |
3683 | CC = DAG.getCondCode(ISD::SETNE); |
3684 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, |
3685 | Tmp2, Tmp3, Tmp4, CC); |
3686 | } |
3687 | Tmp1->setFlags(Node->getFlags()); |
3688 | } |
3689 | Results.push_back(Tmp1); |
3690 | break; |
3691 | } |
3692 | case ISD::BR_CC: { |
3693 | |
3694 | SDValue Chain; |
3695 | Tmp1 = Node->getOperand(0); |
3696 | Tmp2 = Node->getOperand(2); |
3697 | Tmp3 = Node->getOperand(3); |
3698 | Tmp4 = Node->getOperand(1); |
3699 | |
3700 | bool Legalized = |
3701 | TLI.LegalizeSetCCCondCode(DAG, getSetCCResultType(Tmp2.getValueType()), |
3702 | Tmp2, Tmp3, Tmp4, NeedInvert, dl, Chain); |
3703 | (void)Legalized; |
3704 | assert(Legalized && "Can't legalize BR_CC with legal condition!"); |
3705 | |
3706 | |
3707 | |
3708 | if (Tmp4.getNode()) { |
3709 | assert(!NeedInvert && "Don't know how to invert BR_CC!"); |
3710 | |
3711 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, |
3712 | Tmp4, Tmp2, Tmp3, Node->getOperand(4)); |
3713 | } else { |
3714 | Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType()); |
3715 | Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE); |
3716 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, |
3717 | Tmp2, Tmp3, Node->getOperand(4)); |
3718 | } |
3719 | Results.push_back(Tmp1); |
3720 | break; |
3721 | } |
3722 | case ISD::BUILD_VECTOR: |
3723 | Results.push_back(ExpandBUILD_VECTOR(Node)); |
3724 | break; |
3725 | case ISD::SPLAT_VECTOR: |
3726 | Results.push_back(ExpandSPLAT_VECTOR(Node)); |
3727 | break; |
3728 | case ISD::SRA: |
3729 | case ISD::SRL: |
3730 | case ISD::SHL: { |
3731 | |
3732 | EVT VT = Node->getValueType(0); |
3733 | assert(VT.isVector() && "Unable to legalize non-vector shift"); |
3734 | assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"); |
3735 | unsigned NumElem = VT.getVectorNumElements(); |
3736 | |
3737 | SmallVector<SDValue, 8> Scalars; |
3738 | for (unsigned Idx = 0; Idx < NumElem; Idx++) { |
3739 | SDValue Ex = |
3740 | DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), |
3741 | Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl)); |
3742 | SDValue Sh = |
3743 | DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), |
3744 | Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl)); |
3745 | Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, |
3746 | VT.getScalarType(), Ex, Sh)); |
3747 | } |
3748 | |
3749 | SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars); |
3750 | Results.push_back(Result); |
3751 | break; |
3752 | } |
3753 | case ISD::VECREDUCE_FADD: |
3754 | case ISD::VECREDUCE_FMUL: |
3755 | case ISD::VECREDUCE_ADD: |
3756 | case ISD::VECREDUCE_MUL: |
3757 | case ISD::VECREDUCE_AND: |
3758 | case ISD::VECREDUCE_OR: |
3759 | case ISD::VECREDUCE_XOR: |
3760 | case ISD::VECREDUCE_SMAX: |
3761 | case ISD::VECREDUCE_SMIN: |
3762 | case ISD::VECREDUCE_UMAX: |
3763 | case ISD::VECREDUCE_UMIN: |
3764 | case ISD::VECREDUCE_FMAX: |
3765 | case ISD::VECREDUCE_FMIN: |
3766 | Results.push_back(TLI.expandVecReduce(Node, DAG)); |
3767 | break; |
3768 | case ISD::GLOBAL_OFFSET_TABLE: |
3769 | case ISD::GlobalAddress: |
3770 | case ISD::GlobalTLSAddress: |
3771 | case ISD::ExternalSymbol: |
3772 | case ISD::ConstantPool: |
3773 | case ISD::JumpTable: |
3774 | case ISD::INTRINSIC_W_CHAIN: |
3775 | case ISD::INTRINSIC_WO_CHAIN: |
3776 | case ISD::INTRINSIC_VOID: |
3777 | |
3778 | |
3779 | |
3780 | return true; |
3781 | } |
3782 | |
3783 | if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) { |
3784 | |
3785 | |
3786 | |
3787 | |
3788 | |
3789 | switch (Node->getOpcode()) { |
3790 | default: |
3791 | if (TLI.getStrictFPOperationAction(Node->getOpcode(), |
3792 | Node->getValueType(0)) |
3793 | == TargetLowering::Legal) |
3794 | return true; |
3795 | break; |
3796 | case ISD::STRICT_FSUB: { |
3797 | if (TLI.getStrictFPOperationAction( |
3798 | ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal) |
3799 | return true; |
3800 | if (TLI.getStrictFPOperationAction( |
3801 | ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal) |
3802 | break; |
3803 | |
3804 | EVT VT = Node->getValueType(0); |
3805 | const SDNodeFlags Flags = Node->getFlags(); |
3806 | SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags); |
3807 | SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(), |
3808 | {Node->getOperand(0), Node->getOperand(1), Neg}, |
3809 | Flags); |
3810 | |
3811 | Results.push_back(Fadd); |
3812 | Results.push_back(Fadd.getValue(1)); |
3813 | break; |
3814 | } |
3815 | case ISD::STRICT_SINT_TO_FP: |
3816 | case ISD::STRICT_UINT_TO_FP: |
3817 | case ISD::STRICT_LRINT: |
3818 | case ISD::STRICT_LLRINT: |
3819 | case ISD::STRICT_LROUND: |
3820 | case ISD::STRICT_LLROUND: |
3821 | |
3822 | |
3823 | if (TLI.getStrictFPOperationAction(Node->getOpcode(), |
3824 | Node->getOperand(1).getValueType()) |
3825 | == TargetLowering::Legal) |
3826 | return true; |
3827 | break; |
3828 | } |
3829 | } |
3830 | |
3831 | |
3832 | if (Results.empty()) { |
3833 | LLVM_DEBUG(dbgs() << "Cannot expand node\n"); |
3834 | return false; |
3835 | } |
3836 | |
3837 | LLVM_DEBUG(dbgs() << "Successfully expanded node\n"); |
3838 | ReplaceNode(Node, Results.data()); |
3839 | return true; |
3840 | } |
3841 | |
3842 | void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { |
3843 | LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n"); |
3844 | SmallVector<SDValue, 8> Results; |
3845 | SDLoc dl(Node); |
3846 | |
3847 | unsigned Opc = Node->getOpcode(); |
3848 | switch (Opc) { |
3849 | case ISD::ATOMIC_FENCE: { |
3850 | |
3851 | |
3852 | TargetLowering::ArgListTy Args; |
3853 | |
3854 | TargetLowering::CallLoweringInfo CLI(DAG); |
3855 | CLI.setDebugLoc(dl) |
3856 | .setChain(Node->getOperand(0)) |
3857 | .setLibCallee( |
3858 | CallingConv::C, Type::getVoidTy(*DAG.getContext()), |
3859 | DAG.getExternalSymbol("__sync_synchronize", |
3860 | TLI.getPointerTy(DAG.getDataLayout())), |
3861 | std::move(Args)); |
3862 | |
3863 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); |
3864 | |
3865 | Results.push_back(CallResult.second); |
3866 | break; |
3867 | } |
3868 | |
3869 | |
3870 | |
3871 | case ISD::ATOMIC_SWAP: |
3872 | case ISD::ATOMIC_LOAD_ADD: |
3873 | case ISD::ATOMIC_LOAD_SUB: |
3874 | case ISD::ATOMIC_LOAD_AND: |
3875 | case ISD::ATOMIC_LOAD_CLR: |
3876 | case ISD::ATOMIC_LOAD_OR: |
3877 | case ISD::ATOMIC_LOAD_XOR: |
3878 | case ISD::ATOMIC_LOAD_NAND: |
3879 | case ISD::ATOMIC_LOAD_MIN: |
3880 | case ISD::ATOMIC_LOAD_MAX: |
3881 | case ISD::ATOMIC_LOAD_UMIN: |
3882 | case ISD::ATOMIC_LOAD_UMAX: |
3883 | case ISD::ATOMIC_CMP_SWAP: { |
3884 | MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); |
3885 | AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering(); |
3886 | RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT); |
3887 | EVT RetVT = Node->getValueType(0); |
3888 | TargetLowering::MakeLibCallOptions CallOptions; |
3889 | SmallVector<SDValue, 4> Ops; |
3890 | if (TLI.getLibcallName(LC)) { |
3891 | |
3892 | Ops.append(Node->op_begin() + 2, Node->op_end()); |
3893 | Ops.push_back(Node->getOperand(1)); |
3894 | |
3895 | } else { |
3896 | LC = RTLIB::getSYNC(Opc, VT); |
3897 | assert(LC != RTLIB::UNKNOWN_LIBCALL && |
3898 | "Unexpected atomic op or value type!"); |
3899 | |
3900 | Ops.append(Node->op_begin() + 1, Node->op_end()); |
3901 | } |
3902 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, |
3903 | Ops, CallOptions, |
3904 | SDLoc(Node), |
3905 | Node->getOperand(0)); |
3906 | Results.push_back(Tmp.first); |
3907 | Results.push_back(Tmp.second); |
3908 | break; |
3909 | } |
3910 | case ISD::TRAP: { |
3911 | |
3912 | TargetLowering::ArgListTy Args; |
3913 | TargetLowering::CallLoweringInfo CLI(DAG); |
3914 | CLI.setDebugLoc(dl) |
3915 | .setChain(Node->getOperand(0)) |
3916 | .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), |
3917 | DAG.getExternalSymbol( |
3918 | "abort", TLI.getPointerTy(DAG.getDataLayout())), |
3919 | std::move(Args)); |
3920 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); |
3921 | |
3922 | Results.push_back(CallResult.second); |
3923 | break; |
3924 | } |
3925 | case ISD::FMINNUM: |
3926 | case ISD::STRICT_FMINNUM: |
3927 | ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64, |
3928 | RTLIB::FMIN_F80, RTLIB::FMIN_F128, |
3929 | RTLIB::FMIN_PPCF128, Results); |
3930 | break; |
3931 | case ISD::FMAXNUM: |
3932 | case ISD::STRICT_FMAXNUM: |
3933 | ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64, |
3934 | RTLIB::FMAX_F80, RTLIB::FMAX_F128, |
3935 | RTLIB::FMAX_PPCF128, Results); |
3936 | break; |
3937 | case ISD::FSQRT: |
3938 | case ISD::STRICT_FSQRT: |
3939 | ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, |
3940 | RTLIB::SQRT_F80, RTLIB::SQRT_F128, |
3941 | RTLIB::SQRT_PPCF128, Results); |
3942 | break; |
3943 | case ISD::FCBRT: |
3944 | ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64, |
3945 | RTLIB::CBRT_F80, RTLIB::CBRT_F128, |
3946 | RTLIB::CBRT_PPCF128, Results); |
3947 | break; |
3948 | case ISD::FSIN: |
3949 | case ISD::STRICT_FSIN: |
3950 | ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, |
3951 | RTLIB::SIN_F80, RTLIB::SIN_F128, |
3952 | RTLIB::SIN_PPCF128, Results); |
3953 | break; |
3954 | case ISD::FCOS: |
3955 | case ISD::STRICT_FCOS: |
3956 | ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, |
3957 | RTLIB::COS_F80, RTLIB::COS_F128, |
3958 | RTLIB::COS_PPCF128, Results); |
3959 | break; |
3960 | case ISD::FSINCOS: |
3961 | |
3962 | ExpandSinCosLibCall(Node, Results); |
3963 | break; |
3964 | case ISD::FLOG: |
3965 | case ISD::STRICT_FLOG: |
3966 | ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80, |
3967 | RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results); |
3968 | break; |
3969 | case ISD::FLOG2: |
3970 | case ISD::STRICT_FLOG2: |
3971 | ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80, |
3972 | RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results); |
3973 | break; |
3974 | case ISD::FLOG10: |
3975 | case ISD::STRICT_FLOG10: |
3976 | ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80, |
3977 | RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results); |
3978 | break; |
3979 | case ISD::FEXP: |
3980 | case ISD::STRICT_FEXP: |
3981 | ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80, |
3982 | RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results); |
3983 | break; |
3984 | case ISD::FEXP2: |
3985 | case ISD::STRICT_FEXP2: |
3986 | ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80, |
3987 | RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results); |
3988 | break; |
3989 | case ISD::FTRUNC: |
3990 | case ISD::STRICT_FTRUNC: |
3991 | ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, |
3992 | RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, |
3993 | RTLIB::TRUNC_PPCF128, Results); |
3994 | break; |
3995 | case ISD::FFLOOR: |
3996 | case ISD::STRICT_FFLOOR: |
3997 | ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, |
3998 | RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, |
3999 | RTLIB::FLOOR_PPCF128, Results); |
4000 | break; |
4001 | case ISD::FCEIL: |
4002 | case ISD::STRICT_FCEIL: |
4003 | ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, |
4004 | RTLIB::CEIL_F80, RTLIB::CEIL_F128, |
4005 | RTLIB::CEIL_PPCF128, Results); |
4006 | break; |
4007 | case ISD::FRINT: |
4008 | case ISD::STRICT_FRINT: |
4009 | ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, |
4010 | RTLIB::RINT_F80, RTLIB::RINT_F128, |
4011 | RTLIB::RINT_PPCF128, Results); |
4012 | break; |
4013 | case ISD::FNEARBYINT: |
4014 | case ISD::STRICT_FNEARBYINT: |
4015 | ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, |
4016 | RTLIB::NEARBYINT_F64, |
4017 | RTLIB::NEARBYINT_F80, |
4018 | RTLIB::NEARBYINT_F128, |
4019 | RTLIB::NEARBYINT_PPCF128, Results); |
4020 | break; |
4021 | case ISD::FROUND: |
4022 | case ISD::STRICT_FROUND: |
4023 | ExpandFPLibCall(Node, RTLIB::ROUND_F32, |
4024 | RTLIB::ROUND_F64, |
4025 | RTLIB::ROUND_F80, |
4026 | RTLIB::ROUND_F128, |
4027 | RTLIB::ROUND_PPCF128, Results); |
4028 | break; |
4029 | case ISD::FROUNDEVEN: |
4030 | case ISD::STRICT_FROUNDEVEN: |
4031 | ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32, |
4032 | RTLIB::ROUNDEVEN_F64, |
4033 | RTLIB::ROUNDEVEN_F80, |
4034 | RTLIB::ROUNDEVEN_F128, |
4035 | RTLIB::ROUNDEVEN_PPCF128, Results); |
4036 | break; |
4037 | case ISD::FPOWI: |
4038 | case ISD::STRICT_FPOWI: { |
4039 | RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0)); |
4040 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi."); |
4041 | if (!TLI.getLibcallName(LC)) { |
4042 | |
4043 | SDValue Exponent = DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), |
4044 | Node->getValueType(0), |
4045 | Node->getOperand(1)); |
4046 | Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node), |
4047 | Node->getValueType(0), Node->getOperand(0), |
4048 | Exponent)); |
4049 | break; |
4050 | } |
4051 | unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0; |
4052 | bool ExponentHasSizeOfInt = |
4053 | DAG.getLibInfo().getIntSize() == |
4054 | Node->getOperand(1 + Offset).getValueType().getSizeInBits(); |
4055 | if (!ExponentHasSizeOfInt) { |
4056 | |
4057 | |
4058 | DAG.getContext()->emitError("POWI exponent does not match sizeof(int)"); |
4059 | Results.push_back(DAG.getUNDEF(Node->getValueType(0))); |
4060 | break; |
4061 | } |
4062 | ExpandFPLibCall(Node, LC, Results); |
4063 | break; |
4064 | } |
4065 | case ISD::FPOW: |
4066 | case ISD::STRICT_FPOW: |
4067 | ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, |
4068 | RTLIB::POW_F128, RTLIB::POW_PPCF128, Results); |
4069 | break; |
4070 | case ISD::LROUND: |
4071 | case ISD::STRICT_LROUND: |
4072 | ExpandArgFPLibCall(Node, RTLIB::LROUND_F32, |
4073 | RTLIB::LROUND_F64, RTLIB::LROUND_F80, |
4074 | RTLIB::LROUND_F128, |
4075 | RTLIB::LROUND_PPCF128, Results); |
4076 | break; |
4077 | case ISD::LLROUND: |
4078 | case ISD::STRICT_LLROUND: |
4079 | ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32, |
4080 | RTLIB::LLROUND_F64, RTLIB::LLROUND_F80, |
4081 | RTLIB::LLROUND_F128, |
4082 | RTLIB::LLROUND_PPCF128, Results); |
4083 | break; |
4084 | case ISD::LRINT: |
4085 | case ISD::STRICT_LRINT: |
4086 | ExpandArgFPLibCall(Node, RTLIB::LRINT_F32, |
4087 | RTLIB::LRINT_F64, RTLIB::LRINT_F80, |
4088 | RTLIB::LRINT_F128, |
4089 | RTLIB::LRINT_PPCF128, Results); |
4090 | break; |
4091 | case ISD::LLRINT: |
4092 | case ISD::STRICT_LLRINT: |
4093 | ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32, |
4094 | RTLIB::LLRINT_F64, RTLIB::LLRINT_F80, |
4095 | RTLIB::LLRINT_F128, |
4096 | RTLIB::LLRINT_PPCF128, Results); |
4097 | break; |
4098 | case ISD::FDIV: |
4099 | case ISD::STRICT_FDIV: |
4100 | ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, |
4101 | RTLIB::DIV_F80, RTLIB::DIV_F128, |
4102 | RTLIB::DIV_PPCF128, Results); |
4103 | break; |
4104 | case ISD::FREM: |
4105 | case ISD::STRICT_FREM: |
4106 | ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, |
4107 | RTLIB::REM_F80, RTLIB::REM_F128, |
4108 | RTLIB::REM_PPCF128, Results); |
4109 | break; |
4110 | case ISD::FMA: |
4111 | case ISD::STRICT_FMA: |
4112 | ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, |
4113 | RTLIB::FMA_F80, RTLIB::FMA_F128, |
4114 | RTLIB::FMA_PPCF128, Results); |
4115 | break; |
4116 | case ISD::FADD: |
4117 | case ISD::STRICT_FADD: |
4118 | ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64, |
4119 | RTLIB::ADD_F80, RTLIB::ADD_F128, |
4120 | RTLIB::ADD_PPCF128, Results); |
4121 | break; |
4122 | case ISD::FMUL: |
4123 | case ISD::STRICT_FMUL: |
4124 | ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64, |
4125 | RTLIB::MUL_F80, RTLIB::MUL_F128, |
4126 | RTLIB::MUL_PPCF128, Results); |
4127 | break; |
4128 | case ISD::FP16_TO_FP: |
4129 | if (Node->getValueType(0) == MVT::f32) { |
4130 | Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); |
4131 | } |
4132 | break; |
4133 | case ISD::STRICT_FP16_TO_FP: { |
4134 | if (Node->getValueType(0) == MVT::f32) { |
4135 | TargetLowering::MakeLibCallOptions CallOptions; |
4136 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall( |
4137 | DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions, |
4138 | SDLoc(Node), Node->getOperand(0)); |
4139 | Results.push_back(Tmp.first); |
4140 | Results.push_back(Tmp.second); |
4141 | } |
4142 | break; |
4143 | } |
4144 | case ISD::FP_TO_FP16: { |
4145 | RTLIB::Libcall LC = |
4146 | RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); |
4147 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16"); |
4148 | Results.push_back(ExpandLibCall(LC, Node, false)); |
4149 | break; |
4150 | } |
4151 | case ISD::STRICT_SINT_TO_FP: |
4152 | case ISD::STRICT_UINT_TO_FP: |
4153 | case ISD::SINT_TO_FP: |
4154 | case ISD::UINT_TO_FP: { |
4155 | |
4156 | bool IsStrict = Node->isStrictFPOpcode(); |
4157 | bool Signed = Node->getOpcode() == ISD::SINT_TO_FP || |
4158 | Node->getOpcode() == ISD::STRICT_SINT_TO_FP; |
4159 | EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType(); |
4160 | EVT RVT = Node->getValueType(0); |
4161 | EVT NVT = EVT(); |
4162 | SDLoc dl(Node); |
4163 | |
4164 | |
4165 | |
4166 | |
4167 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4168 | for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE; |
4169 | t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; |
4170 | ++t) { |
4171 | NVT = (MVT::SimpleValueType)t; |
4172 | |
4173 | if (NVT.bitsGE(SVT)) |
4174 | LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT) |
4175 | : RTLIB::getUINTTOFP(NVT, RVT); |
4176 | } |
4177 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall"); |
4178 | |
4179 | SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); |
4180 | |
4181 | SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, |
4182 | NVT, Node->getOperand(IsStrict ? 1 : 0)); |
4183 | TargetLowering::MakeLibCallOptions CallOptions; |
4184 | CallOptions.setSExt(Signed); |
4185 | std::pair<SDValue, SDValue> Tmp = |
4186 | TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain); |
4187 | Results.push_back(Tmp.first); |
4188 | if (IsStrict) |
4189 | Results.push_back(Tmp.second); |
4190 | break; |
4191 | } |
4192 | case ISD::FP_TO_SINT: |
4193 | case ISD::FP_TO_UINT: |
4194 | case ISD::STRICT_FP_TO_SINT: |
4195 | case ISD::STRICT_FP_TO_UINT: { |
4196 | |
4197 | bool IsStrict = Node->isStrictFPOpcode(); |
4198 | bool Signed = Node->getOpcode() == ISD::FP_TO_SINT || |
4199 | Node->getOpcode() == ISD::STRICT_FP_TO_SINT; |
4200 | |
4201 | SDValue Op = Node->getOperand(IsStrict ? 1 : 0); |
4202 | EVT SVT = Op.getValueType(); |
4203 | EVT RVT = Node->getValueType(0); |
4204 | EVT NVT = EVT(); |
4205 | SDLoc dl(Node); |
4206 | |
4207 | |
4208 | |
4209 | |
4210 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4211 | for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE; |
4212 | IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; |
4213 | ++IntVT) { |
4214 | NVT = (MVT::SimpleValueType)IntVT; |
4215 | |
4216 | if (NVT.bitsGE(RVT)) |
4217 | LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT) |
4218 | : RTLIB::getFPTOUINT(SVT, NVT); |
4219 | } |
4220 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall"); |
4221 | |
4222 | SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); |
4223 | TargetLowering::MakeLibCallOptions CallOptions; |
4224 | std::pair<SDValue, SDValue> Tmp = |
4225 | TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain); |
4226 | |
4227 | |
4228 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first)); |
4229 | if (IsStrict) |
4230 | Results.push_back(Tmp.second); |
4231 | break; |
4232 | } |
4233 | |
4234 | case ISD::FP_ROUND: |
4235 | case ISD::STRICT_FP_ROUND: { |
4236 | |
4237 | |
4238 | |
4239 | |
4240 | |
4241 | bool IsStrict = Node->isStrictFPOpcode(); |
4242 | SDValue Op = Node->getOperand(IsStrict ? 1 : 0); |
4243 | SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); |
4244 | EVT VT = Node->getValueType(0); |
4245 | assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1)) |
4246 | ->isNullValue() && |
4247 | "Unable to expand as libcall if it is not normal rounding"); |
4248 | |
4249 | RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), VT); |
4250 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall"); |
4251 | |
4252 | TargetLowering::MakeLibCallOptions CallOptions; |
4253 | std::pair<SDValue, SDValue> Tmp = |
4254 | TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain); |
4255 | Results.push_back(Tmp.first); |
4256 | if (IsStrict) |
4257 | Results.push_back(Tmp.second); |
4258 | break; |
4259 | } |
4260 | case ISD::FP_EXTEND: { |
4261 | Results.push_back( |
4262 | ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(), |
4263 | Node->getValueType(0)), |
4264 | Node, false)); |
4265 | break; |
4266 | } |
4267 | case ISD::STRICT_FP_EXTEND: |
4268 | case ISD::STRICT_FP_TO_FP16: { |
4269 | RTLIB::Libcall LC = |
4270 | Node->getOpcode() == ISD::STRICT_FP_TO_FP16 |
4271 | ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16) |
4272 | : RTLIB::getFPEXT(Node->getOperand(1).getValueType(), |
4273 | Node->getValueType(0)); |
4274 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall"); |
4275 | |
4276 | TargetLowering::MakeLibCallOptions CallOptions; |
4277 | std::pair<SDValue, SDValue> Tmp = |
4278 | TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1), |
4279 | CallOptions, SDLoc(Node), Node->getOperand(0)); |
4280 | Results.push_back(Tmp.first); |
4281 | Results.push_back(Tmp.second); |
4282 | break; |
4283 | } |
4284 | case ISD::FSUB: |
4285 | case ISD::STRICT_FSUB: |
4286 | ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64, |
4287 | RTLIB::SUB_F80, RTLIB::SUB_F128, |
4288 | RTLIB::SUB_PPCF128, Results); |
4289 | break; |
4290 | case ISD::SREM: |
4291 | Results.push_back(ExpandIntLibCall(Node, true, |
4292 | RTLIB::SREM_I8, |
4293 | RTLIB::SREM_I16, RTLIB::SREM_I32, |
4294 | RTLIB::SREM_I64, RTLIB::SREM_I128)); |
4295 | break; |
4296 | case ISD::UREM: |
4297 | Results.push_back(ExpandIntLibCall(Node, false, |
4298 | RTLIB::UREM_I8, |
4299 | RTLIB::UREM_I16, RTLIB::UREM_I32, |
4300 | RTLIB::UREM_I64, RTLIB::UREM_I128)); |
4301 | break; |
4302 | case ISD::SDIV: |
4303 | Results.push_back(ExpandIntLibCall(Node, true, |
4304 | RTLIB::SDIV_I8, |
4305 | RTLIB::SDIV_I16, RTLIB::SDIV_I32, |
4306 | RTLIB::SDIV_I64, RTLIB::SDIV_I128)); |
4307 | break; |
4308 | case ISD::UDIV: |
4309 | Results.push_back(ExpandIntLibCall(Node, false, |
4310 | RTLIB::UDIV_I8, |
4311 | RTLIB::UDIV_I16, RTLIB::UDIV_I32, |
4312 | RTLIB::UDIV_I64, RTLIB::UDIV_I128)); |
4313 | break; |
4314 | case ISD::SDIVREM: |
4315 | case ISD::UDIVREM: |
4316 | |
4317 | ExpandDivRemLibCall(Node, Results); |
4318 | break; |
4319 | case ISD::MUL: |
4320 | Results.push_back(ExpandIntLibCall(Node, false, |
4321 | RTLIB::MUL_I8, |
4322 | RTLIB::MUL_I16, RTLIB::MUL_I32, |
4323 | RTLIB::MUL_I64, RTLIB::MUL_I128)); |
4324 | break; |
4325 | case ISD::CTLZ_ZERO_UNDEF: |
4326 | switch (Node->getSimpleValueType(0).SimpleTy) { |
4327 | default: |
4328 | llvm_unreachable("LibCall explicitly requested, but not available"); |
4329 | case MVT::i32: |
4330 | Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false)); |
4331 | break; |
4332 | case MVT::i64: |
4333 | Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false)); |
4334 | break; |
4335 | case MVT::i128: |
4336 | Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false)); |
4337 | break; |
4338 | } |
4339 | break; |
4340 | } |
4341 | |
4342 | |
4343 | if (!Results.empty()) { |
4344 | LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n"); |
4345 | ReplaceNode(Node, Results.data()); |
4346 | } else |
4347 | LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n"); |
4348 | } |
4349 | |
4350 | |
4351 | |
4352 | static MVT getPromotedVectorElementType(const TargetLowering &TLI, |
4353 | MVT EltVT, MVT NewEltVT) { |
4354 | unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits(); |
4355 | MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt); |
4356 | assert(TLI.isTypeLegal(MidVT) && "unexpected"); |
4357 | return MidVT; |
4358 | } |
4359 | |
4360 | void SelectionDAGLegalize::PromoteNode(SDNode *Node) { |
4361 | LLVM_DEBUG(dbgs() << "Trying to promote node\n"); |
4362 | SmallVector<SDValue, 8> Results; |
4363 | MVT OVT = Node->getSimpleValueType(0); |
4364 | if (Node->getOpcode() == ISD::UINT_TO_FP || |
4365 | Node->getOpcode() == ISD::SINT_TO_FP || |
4366 | Node->getOpcode() == ISD::SETCC || |
4367 | Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || |
4368 | Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { |
4369 | OVT = Node->getOperand(0).getSimpleValueType(); |
4370 | } |
4371 | if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP || |
4372 | Node->getOpcode() == ISD::STRICT_SINT_TO_FP || |
4373 | Node->getOpcode() == ISD::STRICT_FSETCC || |
4374 | Node->getOpcode() == ISD::STRICT_FSETCCS) |
4375 | OVT = Node->getOperand(1).getSimpleValueType(); |
4376 | if (Node->getOpcode() == ISD::BR_CC || |
4377 | Node->getOpcode() == ISD::SELECT_CC) |
4378 | OVT = Node->getOperand(2).getSimpleValueType(); |
4379 | MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); |
4380 | SDLoc dl(Node); |
4381 | SDValue Tmp1, Tmp2, Tmp3, Tmp4; |
4382 | switch (Node->getOpcode()) { |
4383 | case ISD::CTTZ: |
4384 | case ISD::CTTZ_ZERO_UNDEF: |
4385 | case ISD::CTLZ: |
4386 | case ISD::CTLZ_ZERO_UNDEF: |
4387 | case ISD::CTPOP: |
4388 | |
4389 | if (Node->getOpcode() == ISD::CTTZ || |
4390 | Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) |
4391 | Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); |
4392 | else |
4393 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); |
4394 | |
4395 | if (Node->getOpcode() == ISD::CTTZ) { |
4396 | |
4397 | |
4398 | |
4399 | auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(), |
4400 | OVT.getSizeInBits()); |
4401 | Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1, |
4402 | DAG.getConstant(TopBit, dl, NVT)); |
4403 | } |
4404 | |
4405 | |
4406 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); |
4407 | if (Node->getOpcode() == ISD::CTLZ || |
4408 | Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { |
4409 | |
4410 | Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, |
4411 | DAG.getConstant(NVT.getSizeInBits() - |
4412 | OVT.getSizeInBits(), dl, NVT)); |
4413 | } |
4414 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); |
4415 | break; |
4416 | case ISD::BITREVERSE: |
4417 | case ISD::BSWAP: { |
4418 | unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); |
4419 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); |
4420 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); |
4421 | Tmp1 = DAG.getNode( |
4422 | ISD::SRL, dl, NVT, Tmp1, |
4423 | DAG.getConstant(DiffBits, dl, |
4424 | TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); |
4425 | |
4426 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); |
4427 | break; |
4428 | } |
4429 | case ISD::FP_TO_UINT: |
4430 | case ISD::STRICT_FP_TO_UINT: |
4431 | case ISD::FP_TO_SINT: |
4432 | case ISD::STRICT_FP_TO_SINT: |
4433 | PromoteLegalFP_TO_INT(Node, dl, Results); |
4434 | break; |
4435 | case ISD::FP_TO_UINT_SAT: |
4436 | case ISD::FP_TO_SINT_SAT: |
4437 | Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl)); |
4438 | break; |
4439 | case ISD::UINT_TO_FP: |
4440 | case ISD::STRICT_UINT_TO_FP: |
4441 | case ISD::SINT_TO_FP: |
4442 | case ISD::STRICT_SINT_TO_FP: |
4443 | PromoteLegalINT_TO_FP(Node, dl, Results); |
4444 | break; |
4445 | case ISD::VAARG: { |
4446 | SDValue Chain = Node->getOperand(0); |
4447 | SDValue Ptr = Node->getOperand(1); |
4448 | |
4449 | unsigned TruncOp; |
4450 | if (OVT.isVector()) { |
4451 | TruncOp = ISD::BITCAST; |
4452 | } else { |
4453 | assert(OVT.isInteger() |
4454 | && "VAARG promotion is supported only for vectors or integer types"); |
4455 | TruncOp = ISD::TRUNCATE; |
4456 | } |
4457 | |
4458 | |
4459 | Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), |
4460 | Node->getConstantOperandVal(3)); |
4461 | Chain = Tmp1.getValue(1); |
4462 | |
4463 | Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); |
4464 | |
4465 | |
4466 | |
4467 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); |
4468 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); |
4469 | if (UpdatedNodes) { |
4470 | UpdatedNodes->insert(Tmp2.getNode()); |
4471 | UpdatedNodes->insert(Chain.getNode()); |
4472 | } |
4473 | ReplacedNode(Node); |
4474 | break; |
4475 | } |
4476 | case ISD::MUL: |
4477 | case ISD::SDIV: |
4478 | case ISD::SREM: |
4479 | case ISD::UDIV: |
4480 | case ISD::UREM: |
4481 | case ISD::AND: |
4482 | case ISD::OR: |
4483 | case ISD::XOR: { |
4484 | unsigned ExtOp, TruncOp; |
4485 | if (OVT.isVector()) { |
4486 | ExtOp = ISD::BITCAST; |
4487 | TruncOp = ISD::BITCAST; |
4488 | } else { |
4489 | assert(OVT.isInteger() && "Cannot promote logic operation"); |
4490 | |
4491 | switch (Node->getOpcode()) { |
4492 | default: |
4493 | ExtOp = ISD::ANY_EXTEND; |
4494 | break; |
4495 | case ISD::SDIV: |
4496 | case ISD::SREM: |
4497 | ExtOp = ISD::SIGN_EXTEND; |
4498 | break; |
4499 | case ISD::UDIV: |
4500 | case ISD::UREM: |
4501 | ExtOp = ISD::ZERO_EXTEND; |
4502 | break; |
4503 | } |
4504 | TruncOp = ISD::TRUNCATE; |
4505 | } |
4506 | |
4507 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4508 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4509 | |
4510 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); |
4511 | Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); |
4512 | break; |
4513 | } |
4514 | case ISD::UMUL_LOHI: |
4515 | case ISD::SMUL_LOHI: { |
4516 | |
4517 | unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND |
4518 | : ISD::SIGN_EXTEND; |
4519 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4520 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4521 | Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2); |
4522 | |
4523 | auto &DL = DAG.getDataLayout(); |
4524 | unsigned OriginalSize = OVT.getScalarSizeInBits(); |
4525 | Tmp2 = DAG.getNode( |
4526 | ISD::SRL, dl, NVT, Tmp1, |
4527 | DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT))); |
4528 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); |
4529 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); |
4530 | break; |
4531 | } |
4532 | case ISD::SELECT: { |
4533 | unsigned ExtOp, TruncOp; |
4534 | if (Node->getValueType(0).isVector() || |
4535 | Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) { |
4536 | ExtOp = ISD::BITCAST; |
4537 | TruncOp = ISD::BITCAST; |
4538 | } else if (Node->getValueType(0).isInteger()) { |
4539 | ExtOp = ISD::ANY_EXTEND; |
4540 | TruncOp = ISD::TRUNCATE; |
4541 | } else { |
4542 | ExtOp = ISD::FP_EXTEND; |
4543 | TruncOp = ISD::FP_ROUND; |
4544 | } |
4545 | Tmp1 = Node->getOperand(0); |
4546 | |
4547 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4548 | Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); |
4549 | |
4550 | Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3); |
4551 | Tmp1->setFlags(Node->getFlags()); |
4552 | if (TruncOp != ISD::FP_ROUND) |
4553 | Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); |
4554 | else |
4555 | Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, |
4556 | DAG.getIntPtrConstant(0, dl)); |
4557 | Results.push_back(Tmp1); |
4558 | break; |
4559 | } |
4560 | case ISD::VECTOR_SHUFFLE: { |
4561 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); |
4562 | |
4563 | |
4564 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); |
4565 | Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); |
4566 | |
4567 | |
4568 | Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); |
4569 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); |
4570 | Results.push_back(Tmp1); |
4571 | break; |
4572 | } |
4573 | case ISD::VECTOR_SPLICE: { |
4574 | Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); |
4575 | Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1)); |
4576 | Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2, |
4577 | Node->getOperand(2)); |
4578 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3)); |
4579 | break; |
4580 | } |
4581 | case ISD::SELECT_CC: { |
4582 | SDValue Cond = Node->getOperand(4); |
4583 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get(); |
4584 | |
4585 | MVT CVT = Node->getSimpleValueType(0); |
4586 | assert(CVT == OVT && "not handled"); |
4587 | |
4588 | unsigned ExtOp = ISD::FP_EXTEND; |
4589 | if (NVT.isInteger()) { |
4590 | ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
4591 | } |
4592 | |
4593 | |
4594 | if (TLI.isCondCodeLegal(CCCode, CVT)) { |
4595 | Tmp1 = Node->getOperand(0); |
4596 | Tmp2 = Node->getOperand(1); |
4597 | } else { |
4598 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4599 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4600 | } |
4601 | |
4602 | Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); |
4603 | Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); |
4604 | |
4605 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond}, |
4606 | Node->getFlags()); |
4607 | |
4608 | |
4609 | if (ExtOp != ISD::FP_EXTEND) |
4610 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1); |
4611 | else |
4612 | Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1, |
4613 | DAG.getIntPtrConstant(0, dl)); |
4614 | |
4615 | Results.push_back(Tmp1); |
4616 | break; |
4617 | } |
4618 | case ISD::SETCC: |
4619 | case ISD::STRICT_FSETCC: |
4620 | case ISD::STRICT_FSETCCS: { |
4621 | unsigned ExtOp = ISD::FP_EXTEND; |
4622 | if (NVT.isInteger()) { |
4623 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); |
4624 | ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
4625 | } |
4626 | if (Node->isStrictFPOpcode()) { |
4627 | SDValue InChain = Node->getOperand(0); |
4628 | std::tie(Tmp1, std::ignore) = |
4629 | DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT); |
4630 | std::tie(Tmp2, std::ignore) = |
4631 | DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT); |
4632 | SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(1), Tmp2.getValue(1)}; |
4633 | SDValue OutChain = DAG.getTokenFactor(dl, TmpChains); |
4634 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
4635 | Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs, |
4636 | {OutChain, Tmp1, Tmp2, Node->getOperand(3)}, |
4637 | Node->getFlags())); |
4638 | Results.push_back(Results.back().getValue(1)); |
4639 | break; |
4640 | } |
4641 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4642 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4643 | Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, |
4644 | Tmp2, Node->getOperand(2), Node->getFlags())); |
4645 | break; |
4646 | } |
4647 | case ISD::BR_CC: { |
4648 | unsigned ExtOp = ISD::FP_EXTEND; |
4649 | if (NVT.isInteger()) { |
4650 | ISD::CondCode CCCode = |
4651 | cast<CondCodeSDNode>(Node->getOperand(1))->get(); |
4652 | ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
4653 | } |
4654 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); |
4655 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); |
4656 | Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), |
4657 | Node->getOperand(0), Node->getOperand(1), |
4658 | Tmp1, Tmp2, Node->getOperand(4))); |
4659 | break; |
4660 | } |
4661 | case ISD::FADD: |
4662 | case ISD::FSUB: |
4663 | case ISD::FMUL: |
4664 | case ISD::FDIV: |
4665 | case ISD::FREM: |
4666 | case ISD::FMINNUM: |
4667 | case ISD::FMAXNUM: |
4668 | case ISD::FPOW: |
4669 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4670 | Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); |
4671 | Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, |
4672 | Node->getFlags()); |
4673 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4674 | Tmp3, DAG.getIntPtrConstant(0, dl))); |
4675 | break; |
4676 | case ISD::STRICT_FREM: |
4677 | case ISD::STRICT_FPOW: |
4678 | Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, |
4679 | {Node->getOperand(0), Node->getOperand(1)}); |
4680 | Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, |
4681 | {Node->getOperand(0), Node->getOperand(2)}); |
4682 | Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1), |
4683 | Tmp2.getValue(1)); |
4684 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, |
4685 | {Tmp3, Tmp1, Tmp2}); |
4686 | Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, |
4687 | {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)}); |
4688 | Results.push_back(Tmp1); |
4689 | Results.push_back(Tmp1.getValue(1)); |
4690 | break; |
4691 | case ISD::FMA: |
4692 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4693 | Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); |
4694 | Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); |
4695 | Results.push_back( |
4696 | DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4697 | DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), |
4698 | DAG.getIntPtrConstant(0, dl))); |
4699 | break; |
4700 | case ISD::FCOPYSIGN: |
4701 | case ISD::FPOWI: { |
4702 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4703 | Tmp2 = Node->getOperand(1); |
4704 | Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); |
4705 | |
4706 | |
4707 | |
4708 | |
4709 | |
4710 | |
4711 | const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); |
4712 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4713 | Tmp3, DAG.getIntPtrConstant(isTrunc, dl))); |
4714 | break; |
4715 | } |
4716 | case ISD::FFLOOR: |
4717 | case ISD::FCEIL: |
4718 | case ISD::FRINT: |
4719 | case ISD::FNEARBYINT: |
4720 | case ISD::FROUND: |
4721 | case ISD::FROUNDEVEN: |
4722 | case ISD::FTRUNC: |
4723 | case ISD::FNEG: |
4724 | case ISD::FSQRT: |
4725 | case ISD::FSIN: |
4726 | case ISD::FCOS: |
4727 | case ISD::FLOG: |
4728 | case ISD::FLOG2: |
4729 | case ISD::FLOG10: |
4730 | case ISD::FABS: |
4731 | case ISD::FEXP: |
4732 | case ISD::FEXP2: |
4733 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4734 | Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); |
4735 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4736 | Tmp2, DAG.getIntPtrConstant(0, dl))); |
4737 | break; |
4738 | case ISD::STRICT_FFLOOR: |
4739 | case ISD::STRICT_FCEIL: |
4740 | case ISD::STRICT_FSIN: |
4741 | case ISD::STRICT_FCOS: |
4742 | case ISD::STRICT_FLOG: |
4743 | case ISD::STRICT_FLOG10: |
4744 | case ISD::STRICT_FEXP: |
4745 | Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, |
4746 | {Node->getOperand(0), Node->getOperand(1)}); |
4747 | Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, |
4748 | {Tmp1.getValue(1), Tmp1}); |
4749 | Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, |
4750 | {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)}); |
4751 | Results.push_back(Tmp3); |
4752 | Results.push_back(Tmp3.getValue(1)); |
4753 | break; |
4754 | case ISD::BUILD_VECTOR: { |
4755 | MVT EltVT = OVT.getVectorElementType(); |
4756 | MVT NewEltVT = NVT.getVectorElementType(); |
4757 | |
4758 | |
4759 | |
4760 | |
4761 | |
4762 | |
4763 | |
4764 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
4765 | "Invalid promote type for build_vector"); |
4766 | assert(NewEltVT.bitsLT(EltVT) && "not handled"); |
4767 | |
4768 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4769 | |
4770 | SmallVector<SDValue, 8> NewOps; |
4771 | for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) { |
4772 | SDValue Op = Node->getOperand(I); |
4773 | NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op)); |
4774 | } |
4775 | |
4776 | SDLoc SL(Node); |
4777 | SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps); |
4778 | SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); |
4779 | Results.push_back(CvtVec); |
4780 | break; |
4781 | } |
4782 | case ISD::EXTRACT_VECTOR_ELT: { |
4783 | MVT EltVT = OVT.getVectorElementType(); |
4784 | MVT NewEltVT = NVT.getVectorElementType(); |
4785 | |
4786 | |
4787 | |
4788 | |
4789 | |
4790 | |
4791 | |
4792 | |
4793 | |
4794 | |
4795 | |
4796 | |
4797 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
4798 | "Invalid promote type for extract_vector_elt"); |
4799 | assert(NewEltVT.bitsLT(EltVT) && "not handled"); |
4800 | |
4801 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4802 | unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); |
4803 | |
4804 | SDValue Idx = Node->getOperand(1); |
4805 | EVT IdxVT = Idx.getValueType(); |
4806 | SDLoc SL(Node); |
4807 | SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT); |
4808 | SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); |
4809 | |
4810 | SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); |
4811 | |
4812 | SmallVector<SDValue, 8> NewOps; |
4813 | for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { |
4814 | SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); |
4815 | SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); |
4816 | |
4817 | SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, |
4818 | CastVec, TmpIdx); |
4819 | NewOps.push_back(Elt); |
4820 | } |
4821 | |
4822 | SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps); |
4823 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec)); |
4824 | break; |
4825 | } |
4826 | case ISD::INSERT_VECTOR_ELT: { |
4827 | MVT EltVT = OVT.getVectorElementType(); |
4828 | MVT NewEltVT = NVT.getVectorElementType(); |
4829 | |
4830 | |
4831 | |
4832 | |
4833 | |
4834 | |
4835 | |
4836 | |
4837 | |
4838 | |
4839 | |
4840 | |
4841 | |
4842 | |
4843 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
4844 | "Invalid promote type for insert_vector_elt"); |
4845 | assert(NewEltVT.bitsLT(EltVT) && "not handled"); |
4846 | |
4847 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4848 | unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); |
4849 | |
4850 | SDValue Val = Node->getOperand(1); |
4851 | SDValue Idx = Node->getOperand(2); |
4852 | EVT IdxVT = Idx.getValueType(); |
4853 | SDLoc SL(Node); |
4854 | |
4855 | SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT); |
4856 | SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); |
4857 | |
4858 | SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); |
4859 | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); |
4860 | |
4861 | SDValue NewVec = CastVec; |
4862 | for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { |
4863 | SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); |
4864 | SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); |
4865 | |
4866 | SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, |
4867 | CastVal, IdxOffset); |
4868 | |
4869 | NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT, |
4870 | NewVec, Elt, InEltIdx); |
4871 | } |
4872 | |
4873 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec)); |
4874 | break; |
4875 | } |
4876 | case ISD::SCALAR_TO_VECTOR: { |
4877 | MVT EltVT = OVT.getVectorElementType(); |
4878 | MVT NewEltVT = NVT.getVectorElementType(); |
4879 | |
4880 | |
4881 | |
4882 | |
4883 | |
4884 | |
4885 | |
4886 | |
4887 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4888 | SDValue Val = Node->getOperand(0); |
4889 | SDLoc SL(Node); |
4890 | |
4891 | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); |
4892 | SDValue Undef = DAG.getUNDEF(MidVT); |
4893 | |
4894 | SmallVector<SDValue, 8> NewElts; |
4895 | NewElts.push_back(CastVal); |
4896 | for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I) |
4897 | NewElts.push_back(Undef); |
4898 | |
4899 | SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts); |
4900 | SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); |
4901 | Results.push_back(CvtVec); |
4902 | break; |
4903 | } |
4904 | case ISD::ATOMIC_SWAP: { |
4905 | AtomicSDNode *AM = cast<AtomicSDNode>(Node); |
4906 | SDLoc SL(Node); |
4907 | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal()); |
4908 | assert(NVT.getSizeInBits() == OVT.getSizeInBits() && |
4909 | "unexpected promotion type"); |
4910 | assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && |
4911 | "unexpected atomic_swap with illegal type"); |
4912 | |
4913 | SDValue NewAtomic |
4914 | = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT, |
4915 | DAG.getVTList(NVT, MVT::Other), |
4916 | { AM->getChain(), AM->getBasePtr(), CastVal }, |
4917 | AM->getMemOperand()); |
4918 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic)); |
4919 | Results.push_back(NewAtomic.getValue(1)); |
4920 | break; |
4921 | } |
4922 | } |
4923 | |
4924 | |
4925 | if (!Results.empty()) { |
4926 | LLVM_DEBUG(dbgs() << "Successfully promoted node\n"); |
4927 | ReplaceNode(Node, Results.data()); |
4928 | } else |
4929 | LLVM_DEBUG(dbgs() << "Could not promote node\n"); |
4930 | } |
4931 | |
4932 | |
4933 | void SelectionDAG::Legalize() { |
4934 | AssignTopologicalOrder(); |
4935 | |
4936 | SmallPtrSet<SDNode *, 16> LegalizedNodes; |
4937 | |
4938 | |
4939 | |
4940 | |
4941 | DAGNodeDeletedListener DeleteListener( |
4942 | *this, |
4943 | [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); }); |
4944 | |
4945 | SelectionDAGLegalize Legalizer(*this, LegalizedNodes); |
4946 | |
4947 | |
4948 | |
4949 | |
4950 | |
4951 | while (true) { |
4952 | bool AnyLegalized = false; |
4953 | for (auto NI = allnodes_end(); NI != allnodes_begin();) { |
4954 | --NI; |
4955 | |
4956 | SDNode *N = &*NI; |
4957 | if (N->use_empty() && N != getRoot().getNode()) { |
4958 | ++NI; |
4959 | DeleteNode(N); |
4960 | continue; |
4961 | } |
4962 | |
4963 | if (LegalizedNodes.insert(N).second) { |
4964 | AnyLegalized = true; |
4965 | Legalizer.LegalizeOp(N); |
4966 | |
4967 | if (N->use_empty() && N != getRoot().getNode()) { |
4968 | ++NI; |
4969 | DeleteNode(N); |
4970 | } |
4971 | } |
4972 | } |
4973 | if (!AnyLegalized) |
4974 | break; |
4975 | |
4976 | } |
4977 | |
4978 | |
4979 | RemoveDeadNodes(); |
4980 | } |
4981 | |
4982 | bool SelectionDAG::LegalizeOp(SDNode *N, |
4983 | SmallSetVector<SDNode *, 16> &UpdatedNodes) { |
4984 | SmallPtrSet<SDNode *, 16> LegalizedNodes; |
4985 | SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes); |
4986 | |
4987 | |
4988 | |
4989 | LegalizedNodes.insert(N); |
4990 | Legalizer.LegalizeOp(N); |
4991 | |
4992 | return LegalizedNodes.count(N); |
4993 | } |