Bug Summary

File:src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Support/Alignment.h
Warning:line 85, column 47
The result of the left shift is undefined due to shifting by '255', which is greater or equal to the width of type 'uint64_t'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name MemorySanitizer.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Analysis -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ASMParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/BinaryFormat -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitstream -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /include/llvm/CodeGen -I /include/llvm/CodeGen/PBQP -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Coroutines -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData/Coverage -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/CodeView -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/DWARF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/MSF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/PDB -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Demangle -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/JITLink -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/Orc -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenACC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenMP -I /include/llvm/CodeGen/GlobalISel -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IRReader -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/LTO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Linker -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC/MCParser -I /include/llvm/CodeGen/MIRParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Object -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Option -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Passes -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Scalar -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ADT -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Support -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/Symbolize -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Target -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Utils -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Vectorize -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/IPO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include -I /usr/src/gnu/usr.bin/clang/libLLVM/../include -I /usr/src/gnu/usr.bin/clang/libLLVM/obj -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include -D NDEBUG -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D LLVM_PREFIX="/usr" -internal-isystem /usr/include/c++/v1 -internal-isystem /usr/local/lib/clang/13.0.0/include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -ferror-limit 19 -fvisibility-inlines-hidden -fwrapv -stack-protector 2 -fno-rtti -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/ben/Projects/vmm/scan-build/2022-01-12-194120-40624-1 -x c++ /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp

/usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp

1//===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of MemorySanitizer, a detector of uninitialized
11/// reads.
12///
13/// The algorithm of the tool is similar to Memcheck
14/// (http://goo.gl/QKbem). We associate a few shadow bits with every
15/// byte of the application memory, poison the shadow of the malloc-ed
16/// or alloca-ed memory, load the shadow bits on every memory read,
17/// propagate the shadow bits through some of the arithmetic
18/// instruction (including MOV), store the shadow bits on every memory
19/// write, report a bug on some other instructions (e.g. JMP) if the
20/// associated shadow is poisoned.
21///
22/// But there are differences too. The first and the major one:
23/// compiler instrumentation instead of binary instrumentation. This
24/// gives us much better register allocation, possible compiler
25/// optimizations and a fast start-up. But this brings the major issue
26/// as well: msan needs to see all program events, including system
27/// calls and reads/writes in system libraries, so we either need to
28/// compile *everything* with msan or use a binary translation
29/// component (e.g. DynamoRIO) to instrument pre-built libraries.
30/// Another difference from Memcheck is that we use 8 shadow bits per
31/// byte of application memory and use a direct shadow mapping. This
32/// greatly simplifies the instrumentation code and avoids races on
33/// shadow updates (Memcheck is single-threaded so races are not a
34/// concern there. Memcheck uses 2 shadow bits per byte with a slow
35/// path storage that uses 8 bits per byte).
36///
37/// The default value of shadow is 0, which means "clean" (not poisoned).
38///
39/// Every module initializer should call __msan_init to ensure that the
40/// shadow memory is ready. On error, __msan_warning is called. Since
41/// parameters and return values may be passed via registers, we have a
42/// specialized thread-local shadow for return values
43/// (__msan_retval_tls) and parameters (__msan_param_tls).
44///
45/// Origin tracking.
46///
47/// MemorySanitizer can track origins (allocation points) of all uninitialized
48/// values. This behavior is controlled with a flag (msan-track-origins) and is
49/// disabled by default.
50///
51/// Origins are 4-byte values created and interpreted by the runtime library.
52/// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53/// of application memory. Propagation of origins is basically a bunch of
54/// "select" instructions that pick the origin of a dirty argument, if an
55/// instruction has one.
56///
57/// Every 4 aligned, consecutive bytes of application memory have one origin
58/// value associated with them. If these bytes contain uninitialized data
59/// coming from 2 different allocations, the last store wins. Because of this,
60/// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61/// practice.
62///
63/// Origins are meaningless for fully initialized values, so MemorySanitizer
64/// avoids storing origin to memory when a fully initialized value is stored.
65/// This way it avoids needless overwriting origin of the 4-byte region on
66/// a short (i.e. 1 byte) clean store, and it is also good for performance.
67///
68/// Atomic handling.
69///
70/// Ideally, every atomic store of application value should update the
71/// corresponding shadow location in an atomic way. Unfortunately, atomic store
72/// of two disjoint locations can not be done without severe slowdown.
73///
74/// Therefore, we implement an approximation that may err on the safe side.
75/// In this implementation, every atomically accessed location in the program
76/// may only change from (partially) uninitialized to fully initialized, but
77/// not the other way around. We load the shadow _after_ the application load,
78/// and we store the shadow _before_ the app store. Also, we always store clean
79/// shadow (if the application store is atomic). This way, if the store-load
80/// pair constitutes a happens-before arc, shadow store and load are correctly
81/// ordered such that the load will get either the value that was stored, or
82/// some later value (which is always clean).
83///
84/// This does not work very well with Compare-And-Swap (CAS) and
85/// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86/// must store the new shadow before the app operation, and load the shadow
87/// after the app operation. Computers don't work this way. Current
88/// implementation ignores the load aspect of CAS/RMW, always returning a clean
89/// value. It implements the store part as a simple atomic store by storing a
90/// clean shadow.
91///
92/// Instrumenting inline assembly.
93///
94/// For inline assembly code LLVM has little idea about which memory locations
95/// become initialized depending on the arguments. It can be possible to figure
96/// out which arguments are meant to point to inputs and outputs, but the
97/// actual semantics can be only visible at runtime. In the Linux kernel it's
98/// also possible that the arguments only indicate the offset for a base taken
99/// from a segment register, so it's dangerous to treat any asm() arguments as
100/// pointers. We take a conservative approach generating calls to
101/// __msan_instrument_asm_store(ptr, size)
102/// , which defer the memory unpoisoning to the runtime library.
103/// The latter can perform more complex address checks to figure out whether
104/// it's safe to touch the shadow memory.
105/// Like with atomic operations, we call __msan_instrument_asm_store() before
106/// the assembly call, so that changes to the shadow memory will be seen by
107/// other threads together with main memory initialization.
108///
109/// KernelMemorySanitizer (KMSAN) implementation.
110///
111/// The major differences between KMSAN and MSan instrumentation are:
112/// - KMSAN always tracks the origins and implies msan-keep-going=true;
113/// - KMSAN allocates shadow and origin memory for each page separately, so
114/// there are no explicit accesses to shadow and origin in the
115/// instrumentation.
116/// Shadow and origin values for a particular X-byte memory location
117/// (X=1,2,4,8) are accessed through pointers obtained via the
118/// __msan_metadata_ptr_for_load_X(ptr)
119/// __msan_metadata_ptr_for_store_X(ptr)
120/// functions. The corresponding functions check that the X-byte accesses
121/// are possible and returns the pointers to shadow and origin memory.
122/// Arbitrary sized accesses are handled with:
123/// __msan_metadata_ptr_for_load_n(ptr, size)
124/// __msan_metadata_ptr_for_store_n(ptr, size);
125/// - TLS variables are stored in a single per-task struct. A call to a
126/// function __msan_get_context_state() returning a pointer to that struct
127/// is inserted into every instrumented function before the entry block;
128/// - __msan_warning() takes a 32-bit origin parameter;
129/// - local variables are poisoned with __msan_poison_alloca() upon function
130/// entry and unpoisoned with __msan_unpoison_alloca() before leaving the
131/// function;
132/// - the pass doesn't declare any global variables or add global constructors
133/// to the translation unit.
134///
135/// Also, KMSAN currently ignores uninitialized memory passed into inline asm
136/// calls, making sure we're on the safe side wrt. possible false positives.
137///
138/// KernelMemorySanitizer only supports X86_64 at the moment.
139///
140//
141// FIXME: This sanitizer does not yet handle scalable vectors
142//
143//===----------------------------------------------------------------------===//
144
145#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
146#include "llvm/ADT/APInt.h"
147#include "llvm/ADT/ArrayRef.h"
148#include "llvm/ADT/DepthFirstIterator.h"
149#include "llvm/ADT/SmallSet.h"
150#include "llvm/ADT/SmallString.h"
151#include "llvm/ADT/SmallVector.h"
152#include "llvm/ADT/StringExtras.h"
153#include "llvm/ADT/StringRef.h"
154#include "llvm/ADT/Triple.h"
155#include "llvm/Analysis/TargetLibraryInfo.h"
156#include "llvm/Analysis/ValueTracking.h"
157#include "llvm/IR/Argument.h"
158#include "llvm/IR/Attributes.h"
159#include "llvm/IR/BasicBlock.h"
160#include "llvm/IR/CallingConv.h"
161#include "llvm/IR/Constant.h"
162#include "llvm/IR/Constants.h"
163#include "llvm/IR/DataLayout.h"
164#include "llvm/IR/DerivedTypes.h"
165#include "llvm/IR/Function.h"
166#include "llvm/IR/GlobalValue.h"
167#include "llvm/IR/GlobalVariable.h"
168#include "llvm/IR/IRBuilder.h"
169#include "llvm/IR/InlineAsm.h"
170#include "llvm/IR/InstVisitor.h"
171#include "llvm/IR/InstrTypes.h"
172#include "llvm/IR/Instruction.h"
173#include "llvm/IR/Instructions.h"
174#include "llvm/IR/IntrinsicInst.h"
175#include "llvm/IR/Intrinsics.h"
176#include "llvm/IR/IntrinsicsX86.h"
177#include "llvm/IR/LLVMContext.h"
178#include "llvm/IR/MDBuilder.h"
179#include "llvm/IR/Module.h"
180#include "llvm/IR/Type.h"
181#include "llvm/IR/Value.h"
182#include "llvm/IR/ValueMap.h"
183#include "llvm/InitializePasses.h"
184#include "llvm/Pass.h"
185#include "llvm/Support/AtomicOrdering.h"
186#include "llvm/Support/Casting.h"
187#include "llvm/Support/CommandLine.h"
188#include "llvm/Support/Compiler.h"
189#include "llvm/Support/Debug.h"
190#include "llvm/Support/ErrorHandling.h"
191#include "llvm/Support/MathExtras.h"
192#include "llvm/Support/raw_ostream.h"
193#include "llvm/Transforms/Instrumentation.h"
194#include "llvm/Transforms/Utils/BasicBlockUtils.h"
195#include "llvm/Transforms/Utils/Local.h"
196#include "llvm/Transforms/Utils/ModuleUtils.h"
197#include <algorithm>
198#include <cassert>
199#include <cstddef>
200#include <cstdint>
201#include <memory>
202#include <string>
203#include <tuple>
204
205using namespace llvm;
206
207#define DEBUG_TYPE"msan" "msan"
208
209static const unsigned kOriginSize = 4;
210static const Align kMinOriginAlignment = Align(4);
211static const Align kShadowTLSAlignment = Align(8);
212
213// These constants must be kept in sync with the ones in msan.h.
214static const unsigned kParamTLSSize = 800;
215static const unsigned kRetvalTLSSize = 800;
216
217// Accesses sizes are powers of two: 1, 2, 4, 8.
218static const size_t kNumberOfAccessSizes = 4;
219
220/// Track origins of uninitialized values.
221///
222/// Adds a section to MemorySanitizer report that points to the allocation
223/// (stack or heap) the uninitialized bits came from originally.
224static cl::opt<int> ClTrackOrigins("msan-track-origins",
225 cl::desc("Track origins (allocation sites) of poisoned memory"),
226 cl::Hidden, cl::init(0));
227
228static cl::opt<bool> ClKeepGoing("msan-keep-going",
229 cl::desc("keep going after reporting a UMR"),
230 cl::Hidden, cl::init(false));
231
232static cl::opt<bool> ClPoisonStack("msan-poison-stack",
233 cl::desc("poison uninitialized stack variables"),
234 cl::Hidden, cl::init(true));
235
236static cl::opt<bool> ClPoisonStackWithCall("msan-poison-stack-with-call",
237 cl::desc("poison uninitialized stack variables with a call"),
238 cl::Hidden, cl::init(false));
239
240static cl::opt<int> ClPoisonStackPattern("msan-poison-stack-pattern",
241 cl::desc("poison uninitialized stack variables with the given pattern"),
242 cl::Hidden, cl::init(0xff));
243
244static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
245 cl::desc("poison undef temps"),
246 cl::Hidden, cl::init(true));
247
248static cl::opt<bool> ClHandleICmp("msan-handle-icmp",
249 cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
250 cl::Hidden, cl::init(true));
251
252static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact",
253 cl::desc("exact handling of relational integer ICmp"),
254 cl::Hidden, cl::init(false));
255
256static cl::opt<bool> ClHandleLifetimeIntrinsics(
257 "msan-handle-lifetime-intrinsics",
258 cl::desc(
259 "when possible, poison scoped variables at the beginning of the scope "
260 "(slower, but more precise)"),
261 cl::Hidden, cl::init(true));
262
263// When compiling the Linux kernel, we sometimes see false positives related to
264// MSan being unable to understand that inline assembly calls may initialize
265// local variables.
266// This flag makes the compiler conservatively unpoison every memory location
267// passed into an assembly call. Note that this may cause false positives.
268// Because it's impossible to figure out the array sizes, we can only unpoison
269// the first sizeof(type) bytes for each type* pointer.
270// The instrumentation is only enabled in KMSAN builds, and only if
271// -msan-handle-asm-conservative is on. This is done because we may want to
272// quickly disable assembly instrumentation when it breaks.
273static cl::opt<bool> ClHandleAsmConservative(
274 "msan-handle-asm-conservative",
275 cl::desc("conservative handling of inline assembly"), cl::Hidden,
276 cl::init(true));
277
278// This flag controls whether we check the shadow of the address
279// operand of load or store. Such bugs are very rare, since load from
280// a garbage address typically results in SEGV, but still happen
281// (e.g. only lower bits of address are garbage, or the access happens
282// early at program startup where malloc-ed memory is more likely to
283// be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
284static cl::opt<bool> ClCheckAccessAddress("msan-check-access-address",
285 cl::desc("report accesses through a pointer which has poisoned shadow"),
286 cl::Hidden, cl::init(true));
287
288static cl::opt<bool> ClEagerChecks(
289 "msan-eager-checks",
290 cl::desc("check arguments and return values at function call boundaries"),
291 cl::Hidden, cl::init(false));
292
293static cl::opt<bool> ClDumpStrictInstructions("msan-dump-strict-instructions",
294 cl::desc("print out instructions with default strict semantics"),
295 cl::Hidden, cl::init(false));
296
297static cl::opt<int> ClInstrumentationWithCallThreshold(
298 "msan-instrumentation-with-call-threshold",
299 cl::desc(
300 "If the function being instrumented requires more than "
301 "this number of checks and origin stores, use callbacks instead of "
302 "inline checks (-1 means never use callbacks)."),
303 cl::Hidden, cl::init(3500));
304
305static cl::opt<bool>
306 ClEnableKmsan("msan-kernel",
307 cl::desc("Enable KernelMemorySanitizer instrumentation"),
308 cl::Hidden, cl::init(false));
309
310// This is an experiment to enable handling of cases where shadow is a non-zero
311// compile-time constant. For some unexplainable reason they were silently
312// ignored in the instrumentation.
313static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow",
314 cl::desc("Insert checks for constant shadow values"),
315 cl::Hidden, cl::init(false));
316
317// This is off by default because of a bug in gold:
318// https://sourceware.org/bugzilla/show_bug.cgi?id=19002
319static cl::opt<bool> ClWithComdat("msan-with-comdat",
320 cl::desc("Place MSan constructors in comdat sections"),
321 cl::Hidden, cl::init(false));
322
323// These options allow to specify custom memory map parameters
324// See MemoryMapParams for details.
325static cl::opt<uint64_t> ClAndMask("msan-and-mask",
326 cl::desc("Define custom MSan AndMask"),
327 cl::Hidden, cl::init(0));
328
329static cl::opt<uint64_t> ClXorMask("msan-xor-mask",
330 cl::desc("Define custom MSan XorMask"),
331 cl::Hidden, cl::init(0));
332
333static cl::opt<uint64_t> ClShadowBase("msan-shadow-base",
334 cl::desc("Define custom MSan ShadowBase"),
335 cl::Hidden, cl::init(0));
336
337static cl::opt<uint64_t> ClOriginBase("msan-origin-base",
338 cl::desc("Define custom MSan OriginBase"),
339 cl::Hidden, cl::init(0));
340
341const char kMsanModuleCtorName[] = "msan.module_ctor";
342const char kMsanInitName[] = "__msan_init";
343
344namespace {
345
346// Memory map parameters used in application-to-shadow address calculation.
347// Offset = (Addr & ~AndMask) ^ XorMask
348// Shadow = ShadowBase + Offset
349// Origin = OriginBase + Offset
350struct MemoryMapParams {
351 uint64_t AndMask;
352 uint64_t XorMask;
353 uint64_t ShadowBase;
354 uint64_t OriginBase;
355};
356
357struct PlatformMemoryMapParams {
358 const MemoryMapParams *bits32;
359 const MemoryMapParams *bits64;
360};
361
362} // end anonymous namespace
363
364// i386 Linux
365static const MemoryMapParams Linux_I386_MemoryMapParams = {
366 0x000080000000, // AndMask
367 0, // XorMask (not used)
368 0, // ShadowBase (not used)
369 0x000040000000, // OriginBase
370};
371
372// x86_64 Linux
373static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
374#ifdef MSAN_LINUX_X86_64_OLD_MAPPING
375 0x400000000000, // AndMask
376 0, // XorMask (not used)
377 0, // ShadowBase (not used)
378 0x200000000000, // OriginBase
379#else
380 0, // AndMask (not used)
381 0x500000000000, // XorMask
382 0, // ShadowBase (not used)
383 0x100000000000, // OriginBase
384#endif
385};
386
387// mips64 Linux
388static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
389 0, // AndMask (not used)
390 0x008000000000, // XorMask
391 0, // ShadowBase (not used)
392 0x002000000000, // OriginBase
393};
394
395// ppc64 Linux
396static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
397 0xE00000000000, // AndMask
398 0x100000000000, // XorMask
399 0x080000000000, // ShadowBase
400 0x1C0000000000, // OriginBase
401};
402
403// s390x Linux
404static const MemoryMapParams Linux_S390X_MemoryMapParams = {
405 0xC00000000000, // AndMask
406 0, // XorMask (not used)
407 0x080000000000, // ShadowBase
408 0x1C0000000000, // OriginBase
409};
410
411// aarch64 Linux
412static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
413 0, // AndMask (not used)
414 0x06000000000, // XorMask
415 0, // ShadowBase (not used)
416 0x01000000000, // OriginBase
417};
418
419// i386 FreeBSD
420static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
421 0x000180000000, // AndMask
422 0x000040000000, // XorMask
423 0x000020000000, // ShadowBase
424 0x000700000000, // OriginBase
425};
426
427// x86_64 FreeBSD
428static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
429 0xc00000000000, // AndMask
430 0x200000000000, // XorMask
431 0x100000000000, // ShadowBase
432 0x380000000000, // OriginBase
433};
434
435// x86_64 NetBSD
436static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
437 0, // AndMask
438 0x500000000000, // XorMask
439 0, // ShadowBase
440 0x100000000000, // OriginBase
441};
442
443static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
444 &Linux_I386_MemoryMapParams,
445 &Linux_X86_64_MemoryMapParams,
446};
447
448static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
449 nullptr,
450 &Linux_MIPS64_MemoryMapParams,
451};
452
453static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
454 nullptr,
455 &Linux_PowerPC64_MemoryMapParams,
456};
457
458static const PlatformMemoryMapParams Linux_S390_MemoryMapParams = {
459 nullptr,
460 &Linux_S390X_MemoryMapParams,
461};
462
463static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
464 nullptr,
465 &Linux_AArch64_MemoryMapParams,
466};
467
468static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
469 &FreeBSD_I386_MemoryMapParams,
470 &FreeBSD_X86_64_MemoryMapParams,
471};
472
473static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
474 nullptr,
475 &NetBSD_X86_64_MemoryMapParams,
476};
477
478namespace {
479
480/// Instrument functions of a module to detect uninitialized reads.
481///
482/// Instantiating MemorySanitizer inserts the msan runtime library API function
483/// declarations into the module if they don't exist already. Instantiating
484/// ensures the __msan_init function is in the list of global constructors for
485/// the module.
486class MemorySanitizer {
487public:
488 MemorySanitizer(Module &M, MemorySanitizerOptions Options)
489 : CompileKernel(Options.Kernel), TrackOrigins(Options.TrackOrigins),
490 Recover(Options.Recover) {
491 initializeModule(M);
492 }
493
494 // MSan cannot be moved or copied because of MapParams.
495 MemorySanitizer(MemorySanitizer &&) = delete;
496 MemorySanitizer &operator=(MemorySanitizer &&) = delete;
497 MemorySanitizer(const MemorySanitizer &) = delete;
498 MemorySanitizer &operator=(const MemorySanitizer &) = delete;
499
500 bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
501
502private:
503 friend struct MemorySanitizerVisitor;
504 friend struct VarArgAMD64Helper;
505 friend struct VarArgMIPS64Helper;
506 friend struct VarArgAArch64Helper;
507 friend struct VarArgPowerPC64Helper;
508 friend struct VarArgSystemZHelper;
509
510 void initializeModule(Module &M);
511 void initializeCallbacks(Module &M);
512 void createKernelApi(Module &M);
513 void createUserspaceApi(Module &M);
514
515 /// True if we're compiling the Linux kernel.
516 bool CompileKernel;
517 /// Track origins (allocation points) of uninitialized values.
518 int TrackOrigins;
519 bool Recover;
520
521 LLVMContext *C;
522 Type *IntptrTy;
523 Type *OriginTy;
524
525 // XxxTLS variables represent the per-thread state in MSan and per-task state
526 // in KMSAN.
527 // For the userspace these point to thread-local globals. In the kernel land
528 // they point to the members of a per-task struct obtained via a call to
529 // __msan_get_context_state().
530
531 /// Thread-local shadow storage for function parameters.
532 Value *ParamTLS;
533
534 /// Thread-local origin storage for function parameters.
535 Value *ParamOriginTLS;
536
537 /// Thread-local shadow storage for function return value.
538 Value *RetvalTLS;
539
540 /// Thread-local origin storage for function return value.
541 Value *RetvalOriginTLS;
542
543 /// Thread-local shadow storage for in-register va_arg function
544 /// parameters (x86_64-specific).
545 Value *VAArgTLS;
546
547 /// Thread-local shadow storage for in-register va_arg function
548 /// parameters (x86_64-specific).
549 Value *VAArgOriginTLS;
550
551 /// Thread-local shadow storage for va_arg overflow area
552 /// (x86_64-specific).
553 Value *VAArgOverflowSizeTLS;
554
555 /// Are the instrumentation callbacks set up?
556 bool CallbacksInitialized = false;
557
558 /// The run-time callback to print a warning.
559 FunctionCallee WarningFn;
560
561 // These arrays are indexed by log2(AccessSize).
562 FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
563 FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
564
565 /// Run-time helper that generates a new origin value for a stack
566 /// allocation.
567 FunctionCallee MsanSetAllocaOrigin4Fn;
568
569 /// Run-time helper that poisons stack on function entry.
570 FunctionCallee MsanPoisonStackFn;
571
572 /// Run-time helper that records a store (or any event) of an
573 /// uninitialized value and returns an updated origin id encoding this info.
574 FunctionCallee MsanChainOriginFn;
575
576 /// Run-time helper that paints an origin over a region.
577 FunctionCallee MsanSetOriginFn;
578
579 /// MSan runtime replacements for memmove, memcpy and memset.
580 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
581
582 /// KMSAN callback for task-local function argument shadow.
583 StructType *MsanContextStateTy;
584 FunctionCallee MsanGetContextStateFn;
585
586 /// Functions for poisoning/unpoisoning local variables
587 FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
588
589 /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin
590 /// pointers.
591 FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
592 FunctionCallee MsanMetadataPtrForLoad_1_8[4];
593 FunctionCallee MsanMetadataPtrForStore_1_8[4];
594 FunctionCallee MsanInstrumentAsmStoreFn;
595
596 /// Helper to choose between different MsanMetadataPtrXxx().
597 FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
598
599 /// Memory map parameters used in application-to-shadow calculation.
600 const MemoryMapParams *MapParams;
601
602 /// Custom memory map parameters used when -msan-shadow-base or
603 // -msan-origin-base is provided.
604 MemoryMapParams CustomMapParams;
605
606 MDNode *ColdCallWeights;
607
608 /// Branch weights for origin store.
609 MDNode *OriginStoreWeights;
610};
611
612void insertModuleCtor(Module &M) {
613 getOrCreateSanitizerCtorAndInitFunctions(
614 M, kMsanModuleCtorName, kMsanInitName,
615 /*InitArgTypes=*/{},
616 /*InitArgs=*/{},
617 // This callback is invoked when the functions are created the first
618 // time. Hook them into the global ctors list in that case:
619 [&](Function *Ctor, FunctionCallee) {
620 if (!ClWithComdat) {
621 appendToGlobalCtors(M, Ctor, 0);
622 return;
623 }
624 Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
625 Ctor->setComdat(MsanCtorComdat);
626 appendToGlobalCtors(M, Ctor, 0, Ctor);
627 });
628}
629
630/// A legacy function pass for msan instrumentation.
631///
632/// Instruments functions to detect uninitialized reads.
633struct MemorySanitizerLegacyPass : public FunctionPass {
634 // Pass identification, replacement for typeid.
635 static char ID;
636
637 MemorySanitizerLegacyPass(MemorySanitizerOptions Options = {})
638 : FunctionPass(ID), Options(Options) {
639 initializeMemorySanitizerLegacyPassPass(*PassRegistry::getPassRegistry());
640 }
641 StringRef getPassName() const override { return "MemorySanitizerLegacyPass"; }
642
643 void getAnalysisUsage(AnalysisUsage &AU) const override {
644 AU.addRequired<TargetLibraryInfoWrapperPass>();
645 }
646
647 bool runOnFunction(Function &F) override {
648 return MSan->sanitizeFunction(
649 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F));
650 }
651 bool doInitialization(Module &M) override;
652
653 Optional<MemorySanitizer> MSan;
654 MemorySanitizerOptions Options;
655};
656
657template <class T> T getOptOrDefault(const cl::opt<T> &Opt, T Default) {
658 return (Opt.getNumOccurrences() > 0) ? Opt : Default;
659}
660
661} // end anonymous namespace
662
663MemorySanitizerOptions::MemorySanitizerOptions(int TO, bool R, bool K)
664 : Kernel(getOptOrDefault(ClEnableKmsan, K)),
665 TrackOrigins(getOptOrDefault(ClTrackOrigins, Kernel ? 2 : TO)),
666 Recover(getOptOrDefault(ClKeepGoing, Kernel || R)) {}
667
668PreservedAnalyses MemorySanitizerPass::run(Function &F,
669 FunctionAnalysisManager &FAM) {
670 MemorySanitizer Msan(*F.getParent(), Options);
671 if (Msan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F)))
672 return PreservedAnalyses::none();
673 return PreservedAnalyses::all();
674}
675
676PreservedAnalyses MemorySanitizerPass::run(Module &M,
677 ModuleAnalysisManager &AM) {
678 if (Options.Kernel)
679 return PreservedAnalyses::all();
680 insertModuleCtor(M);
681 return PreservedAnalyses::none();
682}
683
684char MemorySanitizerLegacyPass::ID = 0;
685
686INITIALIZE_PASS_BEGIN(MemorySanitizerLegacyPass, "msan",static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry
&Registry) {
687 "MemorySanitizer: detects uninitialized reads.", false,static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry
&Registry) {
688 false)static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry
&Registry) {
689INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
690INITIALIZE_PASS_END(MemorySanitizerLegacyPass, "msan",PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads."
, "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<MemorySanitizerLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void
llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag
, initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry
)); }
691 "MemorySanitizer: detects uninitialized reads.", false,PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads."
, "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<MemorySanitizerLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void
llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag
, initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry
)); }
692 false)PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads."
, "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<MemorySanitizerLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void
llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag
, initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry
)); }
693
694FunctionPass *
695llvm::createMemorySanitizerLegacyPassPass(MemorySanitizerOptions Options) {
696 return new MemorySanitizerLegacyPass(Options);
697}
698
699/// Create a non-const global initialized with the given string.
700///
701/// Creates a writable global for Str so that we can pass it to the
702/// run-time lib. Runtime uses first 4 bytes of the string to store the
703/// frame ID, so the string needs to be mutable.
704static GlobalVariable *createPrivateNonConstGlobalForString(Module &M,
705 StringRef Str) {
706 Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
707 return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/false,
708 GlobalValue::PrivateLinkage, StrConst, "");
709}
710
711/// Create KMSAN API callbacks.
712void MemorySanitizer::createKernelApi(Module &M) {
713 IRBuilder<> IRB(*C);
714
715 // These will be initialized in insertKmsanPrologue().
716 RetvalTLS = nullptr;
717 RetvalOriginTLS = nullptr;
718 ParamTLS = nullptr;
719 ParamOriginTLS = nullptr;
720 VAArgTLS = nullptr;
721 VAArgOriginTLS = nullptr;
722 VAArgOverflowSizeTLS = nullptr;
723
724 WarningFn = M.getOrInsertFunction("__msan_warning", IRB.getVoidTy(),
725 IRB.getInt32Ty());
726 // Requests the per-task context state (kmsan_context_state*) from the
727 // runtime library.
728 MsanContextStateTy = StructType::get(
729 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
730 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8),
731 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
732 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */
733 IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy,
734 OriginTy);
735 MsanGetContextStateFn = M.getOrInsertFunction(
736 "__msan_get_context_state", PointerType::get(MsanContextStateTy, 0));
737
738 Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0),
739 PointerType::get(IRB.getInt32Ty(), 0));
740
741 for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
742 std::string name_load =
743 "__msan_metadata_ptr_for_load_" + std::to_string(size);
744 std::string name_store =
745 "__msan_metadata_ptr_for_store_" + std::to_string(size);
746 MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction(
747 name_load, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
748 MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction(
749 name_store, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
750 }
751
752 MsanMetadataPtrForLoadN = M.getOrInsertFunction(
753 "__msan_metadata_ptr_for_load_n", RetTy,
754 PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
755 MsanMetadataPtrForStoreN = M.getOrInsertFunction(
756 "__msan_metadata_ptr_for_store_n", RetTy,
757 PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
758
759 // Functions for poisoning and unpoisoning memory.
760 MsanPoisonAllocaFn =
761 M.getOrInsertFunction("__msan_poison_alloca", IRB.getVoidTy(),
762 IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy());
763 MsanUnpoisonAllocaFn = M.getOrInsertFunction(
764 "__msan_unpoison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
765}
766
767static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
768 return M.getOrInsertGlobal(Name, Ty, [&] {
769 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
770 nullptr, Name, nullptr,
771 GlobalVariable::InitialExecTLSModel);
772 });
773}
774
775/// Insert declarations for userspace-specific functions and globals.
776void MemorySanitizer::createUserspaceApi(Module &M) {
777 IRBuilder<> IRB(*C);
778
779 // Create the callback.
780 // FIXME: this function should have "Cold" calling conv,
781 // which is not yet implemented.
782 StringRef WarningFnName = Recover ? "__msan_warning_with_origin"
783 : "__msan_warning_with_origin_noreturn";
784 WarningFn =
785 M.getOrInsertFunction(WarningFnName, IRB.getVoidTy(), IRB.getInt32Ty());
786
787 // Create the global TLS variables.
788 RetvalTLS =
789 getOrInsertGlobal(M, "__msan_retval_tls",
790 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8));
791
792 RetvalOriginTLS = getOrInsertGlobal(M, "__msan_retval_origin_tls", OriginTy);
793
794 ParamTLS =
795 getOrInsertGlobal(M, "__msan_param_tls",
796 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
797
798 ParamOriginTLS =
799 getOrInsertGlobal(M, "__msan_param_origin_tls",
800 ArrayType::get(OriginTy, kParamTLSSize / 4));
801
802 VAArgTLS =
803 getOrInsertGlobal(M, "__msan_va_arg_tls",
804 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
805
806 VAArgOriginTLS =
807 getOrInsertGlobal(M, "__msan_va_arg_origin_tls",
808 ArrayType::get(OriginTy, kParamTLSSize / 4));
809
810 VAArgOverflowSizeTLS =
811 getOrInsertGlobal(M, "__msan_va_arg_overflow_size_tls", IRB.getInt64Ty());
812
813 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
814 AccessSizeIndex++) {
815 unsigned AccessSize = 1 << AccessSizeIndex;
816 std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize);
817 SmallVector<std::pair<unsigned, Attribute>, 2> MaybeWarningFnAttrs;
818 MaybeWarningFnAttrs.push_back(std::make_pair(
819 AttributeList::FirstArgIndex, Attribute::get(*C, Attribute::ZExt)));
820 MaybeWarningFnAttrs.push_back(std::make_pair(
821 AttributeList::FirstArgIndex + 1, Attribute::get(*C, Attribute::ZExt)));
822 MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
823 FunctionName, AttributeList::get(*C, MaybeWarningFnAttrs),
824 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
825
826 FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
827 SmallVector<std::pair<unsigned, Attribute>, 2> MaybeStoreOriginFnAttrs;
828 MaybeStoreOriginFnAttrs.push_back(std::make_pair(
829 AttributeList::FirstArgIndex, Attribute::get(*C, Attribute::ZExt)));
830 MaybeStoreOriginFnAttrs.push_back(std::make_pair(
831 AttributeList::FirstArgIndex + 2, Attribute::get(*C, Attribute::ZExt)));
832 MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
833 FunctionName, AttributeList::get(*C, MaybeStoreOriginFnAttrs),
834 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt8PtrTy(),
835 IRB.getInt32Ty());
836 }
837
838 MsanSetAllocaOrigin4Fn = M.getOrInsertFunction(
839 "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy,
840 IRB.getInt8PtrTy(), IntptrTy);
841 MsanPoisonStackFn =
842 M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(),
843 IRB.getInt8PtrTy(), IntptrTy);
844}
845
846/// Insert extern declaration of runtime-provided functions and globals.
847void MemorySanitizer::initializeCallbacks(Module &M) {
848 // Only do this once.
849 if (CallbacksInitialized)
850 return;
851
852 IRBuilder<> IRB(*C);
853 // Initialize callbacks that are common for kernel and userspace
854 // instrumentation.
855 MsanChainOriginFn = M.getOrInsertFunction(
856 "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty());
857 MsanSetOriginFn =
858 M.getOrInsertFunction("__msan_set_origin", IRB.getVoidTy(),
859 IRB.getInt8PtrTy(), IntptrTy, IRB.getInt32Ty());
860 MemmoveFn = M.getOrInsertFunction(
861 "__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
862 IRB.getInt8PtrTy(), IntptrTy);
863 MemcpyFn = M.getOrInsertFunction(
864 "__msan_memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
865 IntptrTy);
866 MemsetFn = M.getOrInsertFunction(
867 "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
868 IntptrTy);
869
870 MsanInstrumentAsmStoreFn =
871 M.getOrInsertFunction("__msan_instrument_asm_store", IRB.getVoidTy(),
872 PointerType::get(IRB.getInt8Ty(), 0), IntptrTy);
873
874 if (CompileKernel) {
875 createKernelApi(M);
876 } else {
877 createUserspaceApi(M);
878 }
879 CallbacksInitialized = true;
880}
881
882FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
883 int size) {
884 FunctionCallee *Fns =
885 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
886 switch (size) {
887 case 1:
888 return Fns[0];
889 case 2:
890 return Fns[1];
891 case 4:
892 return Fns[2];
893 case 8:
894 return Fns[3];
895 default:
896 return nullptr;
897 }
898}
899
900/// Module-level initialization.
901///
902/// inserts a call to __msan_init to the module's constructor list.
903void MemorySanitizer::initializeModule(Module &M) {
904 auto &DL = M.getDataLayout();
905
906 bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
907 bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
908 // Check the overrides first
909 if (ShadowPassed || OriginPassed) {
910 CustomMapParams.AndMask = ClAndMask;
911 CustomMapParams.XorMask = ClXorMask;
912 CustomMapParams.ShadowBase = ClShadowBase;
913 CustomMapParams.OriginBase = ClOriginBase;
914 MapParams = &CustomMapParams;
915 } else {
916 Triple TargetTriple(M.getTargetTriple());
917 switch (TargetTriple.getOS()) {
918 case Triple::FreeBSD:
919 switch (TargetTriple.getArch()) {
920 case Triple::x86_64:
921 MapParams = FreeBSD_X86_MemoryMapParams.bits64;
922 break;
923 case Triple::x86:
924 MapParams = FreeBSD_X86_MemoryMapParams.bits32;
925 break;
926 default:
927 report_fatal_error("unsupported architecture");
928 }
929 break;
930 case Triple::NetBSD:
931 switch (TargetTriple.getArch()) {
932 case Triple::x86_64:
933 MapParams = NetBSD_X86_MemoryMapParams.bits64;
934 break;
935 default:
936 report_fatal_error("unsupported architecture");
937 }
938 break;
939 case Triple::Linux:
940 switch (TargetTriple.getArch()) {
941 case Triple::x86_64:
942 MapParams = Linux_X86_MemoryMapParams.bits64;
943 break;
944 case Triple::x86:
945 MapParams = Linux_X86_MemoryMapParams.bits32;
946 break;
947 case Triple::mips64:
948 case Triple::mips64el:
949 MapParams = Linux_MIPS_MemoryMapParams.bits64;
950 break;
951 case Triple::ppc64:
952 case Triple::ppc64le:
953 MapParams = Linux_PowerPC_MemoryMapParams.bits64;
954 break;
955 case Triple::systemz:
956 MapParams = Linux_S390_MemoryMapParams.bits64;
957 break;
958 case Triple::aarch64:
959 case Triple::aarch64_be:
960 MapParams = Linux_ARM_MemoryMapParams.bits64;
961 break;
962 default:
963 report_fatal_error("unsupported architecture");
964 }
965 break;
966 default:
967 report_fatal_error("unsupported operating system");
968 }
969 }
970
971 C = &(M.getContext());
972 IRBuilder<> IRB(*C);
973 IntptrTy = IRB.getIntPtrTy(DL);
974 OriginTy = IRB.getInt32Ty();
975
976 ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000);
977 OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000);
978
979 if (!CompileKernel) {
980 if (TrackOrigins)
981 M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] {
982 return new GlobalVariable(
983 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
984 IRB.getInt32(TrackOrigins), "__msan_track_origins");
985 });
986
987 if (Recover)
988 M.getOrInsertGlobal("__msan_keep_going", IRB.getInt32Ty(), [&] {
989 return new GlobalVariable(M, IRB.getInt32Ty(), true,
990 GlobalValue::WeakODRLinkage,
991 IRB.getInt32(Recover), "__msan_keep_going");
992 });
993}
994}
995
996bool MemorySanitizerLegacyPass::doInitialization(Module &M) {
997 if (!Options.Kernel)
998 insertModuleCtor(M);
999 MSan.emplace(M, Options);
1000 return true;
1001}
1002
1003namespace {
1004
1005/// A helper class that handles instrumentation of VarArg
1006/// functions on a particular platform.
1007///
1008/// Implementations are expected to insert the instrumentation
1009/// necessary to propagate argument shadow through VarArg function
1010/// calls. Visit* methods are called during an InstVisitor pass over
1011/// the function, and should avoid creating new basic blocks. A new
1012/// instance of this class is created for each instrumented function.
1013struct VarArgHelper {
1014 virtual ~VarArgHelper() = default;
1015
1016 /// Visit a CallBase.
1017 virtual void visitCallBase(CallBase &CB, IRBuilder<> &IRB) = 0;
1018
1019 /// Visit a va_start call.
1020 virtual void visitVAStartInst(VAStartInst &I) = 0;
1021
1022 /// Visit a va_copy call.
1023 virtual void visitVACopyInst(VACopyInst &I) = 0;
1024
1025 /// Finalize function instrumentation.
1026 ///
1027 /// This method is called after visiting all interesting (see above)
1028 /// instructions in a function.
1029 virtual void finalizeInstrumentation() = 0;
1030};
1031
1032struct MemorySanitizerVisitor;
1033
1034} // end anonymous namespace
1035
1036static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
1037 MemorySanitizerVisitor &Visitor);
1038
1039static unsigned TypeSizeToSizeIndex(unsigned TypeSize) {
1040 if (TypeSize <= 8) return 0;
1041 return Log2_32_Ceil((TypeSize + 7) / 8);
1042}
1043
1044namespace {
1045
1046/// This class does all the work for a given function. Store and Load
1047/// instructions store and load corresponding shadow and origin
1048/// values. Most instructions propagate shadow from arguments to their
1049/// return values. Certain instructions (most importantly, BranchInst)
1050/// test their argument shadow and print reports (with a runtime call) if it's
1051/// non-zero.
1052struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
1053 Function &F;
1054 MemorySanitizer &MS;
1055 SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
1056 ValueMap<Value*, Value*> ShadowMap, OriginMap;
1057 std::unique_ptr<VarArgHelper> VAHelper;
1058 const TargetLibraryInfo *TLI;
1059 Instruction *FnPrologueEnd;
1060
1061 // The following flags disable parts of MSan instrumentation based on
1062 // exclusion list contents and command-line options.
1063 bool InsertChecks;
1064 bool PropagateShadow;
1065 bool PoisonStack;
1066 bool PoisonUndef;
1067
1068 struct ShadowOriginAndInsertPoint {
1069 Value *Shadow;
1070 Value *Origin;
1071 Instruction *OrigIns;
1072
1073 ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
1074 : Shadow(S), Origin(O), OrigIns(I) {}
1075 };
1076 SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
1077 bool InstrumentLifetimeStart = ClHandleLifetimeIntrinsics;
1078 SmallSet<AllocaInst *, 16> AllocaSet;
1079 SmallVector<std::pair<IntrinsicInst *, AllocaInst *>, 16> LifetimeStartList;
1080 SmallVector<StoreInst *, 16> StoreList;
1081
1082 MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
1083 const TargetLibraryInfo &TLI)
1084 : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)), TLI(&TLI) {
1085 bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeMemory);
1086 InsertChecks = SanitizeFunction;
1087 PropagateShadow = SanitizeFunction;
1088 PoisonStack = SanitizeFunction && ClPoisonStack;
1089 PoisonUndef = SanitizeFunction && ClPoisonUndef;
1090
1091 // In the presence of unreachable blocks, we may see Phi nodes with
1092 // incoming nodes from such blocks. Since InstVisitor skips unreachable
1093 // blocks, such nodes will not have any shadow value associated with them.
1094 // It's easier to remove unreachable blocks than deal with missing shadow.
1095 removeUnreachableBlocks(F);
1096
1097 MS.initializeCallbacks(*F.getParent());
1098 FnPrologueEnd = IRBuilder<>(F.getEntryBlock().getFirstNonPHI())
1099 .CreateIntrinsic(Intrinsic::donothing, {}, {});
1100
1101 if (MS.CompileKernel) {
1102 IRBuilder<> IRB(FnPrologueEnd);
1103 insertKmsanPrologue(IRB);
1104 }
1105
1106 LLVM_DEBUG(if (!InsertChecks) dbgs()do { } while (false)
1107 << "MemorySanitizer is not inserting checks into '"do { } while (false)
1108 << F.getName() << "'\n")do { } while (false);
1109 }
1110
1111 bool isInPrologue(Instruction &I) {
1112 return I.getParent() == FnPrologueEnd->getParent() &&
1113 (&I == FnPrologueEnd || I.comesBefore(FnPrologueEnd));
1114 }
1115
1116 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1117 if (MS.TrackOrigins <= 1) return V;
1118 return IRB.CreateCall(MS.MsanChainOriginFn, V);
1119 }
1120
1121 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1122 const DataLayout &DL = F.getParent()->getDataLayout();
1123 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1124 if (IntptrSize == kOriginSize) return Origin;
1125 assert(IntptrSize == kOriginSize * 2)((void)0);
1126 Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
1127 return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
1128 }
1129
1130 /// Fill memory range with the given origin value.
1131 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1132 unsigned Size, Align Alignment) {
1133 const DataLayout &DL = F.getParent()->getDataLayout();
1134 const Align IntptrAlignment = DL.getABITypeAlign(MS.IntptrTy);
1135 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1136 assert(IntptrAlignment >= kMinOriginAlignment)((void)0);
1137 assert(IntptrSize >= kOriginSize)((void)0);
1138
1139 unsigned Ofs = 0;
1140 Align CurrentAlignment = Alignment;
1141 if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
1142 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1143 Value *IntptrOriginPtr =
1144 IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0));
1145 for (unsigned i = 0; i < Size / IntptrSize; ++i) {
1146 Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
1147 : IntptrOriginPtr;
1148 IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
1149 Ofs += IntptrSize / kOriginSize;
1150 CurrentAlignment = IntptrAlignment;
1151 }
1152 }
1153
1154 for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
1155 Value *GEP =
1156 i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr;
1157 IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
1158 CurrentAlignment = kMinOriginAlignment;
1159 }
1160 }
1161
1162 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1163 Value *OriginPtr, Align Alignment, bool AsCall) {
1164 const DataLayout &DL = F.getParent()->getDataLayout();
1165 const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1166 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
1167 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1168 if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1169 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
1170 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1171 OriginAlignment);
1172 return;
1173 }
1174
1175 unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
1176 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1177 if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1178 FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
1179 Value *ConvertedShadow2 =
1180 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1181 CallBase *CB = IRB.CreateCall(
1182 Fn, {ConvertedShadow2,
1183 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), Origin});
1184 CB->addParamAttr(0, Attribute::ZExt);
1185 CB->addParamAttr(2, Attribute::ZExt);
1186 } else {
1187 Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
1188 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1189 Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
1190 IRBuilder<> IRBNew(CheckTerm);
1191 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1192 OriginAlignment);
1193 }
1194 }
1195
1196 void materializeStores(bool InstrumentWithCalls) {
1197 for (StoreInst *SI : StoreList) {
1198 IRBuilder<> IRB(SI);
1199 Value *Val = SI->getValueOperand();
1200 Value *Addr = SI->getPointerOperand();
1201 Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1202 Value *ShadowPtr, *OriginPtr;
1203 Type *ShadowTy = Shadow->getType();
1204 const Align Alignment = assumeAligned(SI->getAlignment());
1205 const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1206 std::tie(ShadowPtr, OriginPtr) =
1207 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1208
1209 StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
1210 LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n")do { } while (false);
1211 (void)NewSI;
1212
1213 if (SI->isAtomic())
1214 SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
1215
1216 if (MS.TrackOrigins && !SI->isAtomic())
1217 storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr,
1218 OriginAlignment, InstrumentWithCalls);
1219 }
1220 }
1221
1222 /// Helper function to insert a warning at IRB's current insert point.
1223 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1224 if (!Origin)
1225 Origin = (Value *)IRB.getInt32(0);
1226 assert(Origin->getType()->isIntegerTy())((void)0);
1227 IRB.CreateCall(MS.WarningFn, Origin)->setCannotMerge();
1228 // FIXME: Insert UnreachableInst if !MS.Recover?
1229 // This may invalidate some of the following checks and needs to be done
1230 // at the very end.
1231 }
1232
1233 void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin,
1234 bool AsCall) {
1235 IRBuilder<> IRB(OrigIns);
1236 LLVM_DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n")do { } while (false);
1237 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1238 LLVM_DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n")do { } while (false);
1239
1240 if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1241 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) {
1242 insertWarningFn(IRB, Origin);
1243 }
1244 return;
1245 }
1246
1247 const DataLayout &DL = OrigIns->getModule()->getDataLayout();
1248
1249 unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
1250 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1251 if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1252 FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
1253 Value *ConvertedShadow2 =
1254 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1255 CallBase *CB = IRB.CreateCall(
1256 Fn, {ConvertedShadow2,
1257 MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(0)});
1258 CB->addParamAttr(0, Attribute::ZExt);
1259 CB->addParamAttr(1, Attribute::ZExt);
1260 } else {
1261 Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
1262 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1263 Cmp, OrigIns,
1264 /* Unreachable */ !MS.Recover, MS.ColdCallWeights);
1265
1266 IRB.SetInsertPoint(CheckTerm);
1267 insertWarningFn(IRB, Origin);
1268 LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n")do { } while (false);
1269 }
1270 }
1271
1272 void materializeChecks(bool InstrumentWithCalls) {
1273 for (const auto &ShadowData : InstrumentationList) {
1274 Instruction *OrigIns = ShadowData.OrigIns;
1275 Value *Shadow = ShadowData.Shadow;
1276 Value *Origin = ShadowData.Origin;
1277 materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls);
1278 }
1279 LLVM_DEBUG(dbgs() << "DONE:\n" << F)do { } while (false);
1280 }
1281
1282 // Returns the last instruction in the new prologue
1283 void insertKmsanPrologue(IRBuilder<> &IRB) {
1284 Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {});
1285 Constant *Zero = IRB.getInt32(0);
1286 MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1287 {Zero, IRB.getInt32(0)}, "param_shadow");
1288 MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1289 {Zero, IRB.getInt32(1)}, "retval_shadow");
1290 MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1291 {Zero, IRB.getInt32(2)}, "va_arg_shadow");
1292 MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1293 {Zero, IRB.getInt32(3)}, "va_arg_origin");
1294 MS.VAArgOverflowSizeTLS =
1295 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1296 {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
1297 MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1298 {Zero, IRB.getInt32(5)}, "param_origin");
1299 MS.RetvalOriginTLS =
1300 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1301 {Zero, IRB.getInt32(6)}, "retval_origin");
1302 }
1303
1304 /// Add MemorySanitizer instrumentation to a function.
1305 bool runOnFunction() {
1306 // Iterate all BBs in depth-first order and create shadow instructions
1307 // for all instructions (where applicable).
1308 // For PHI nodes we create dummy shadow PHIs which will be finalized later.
1309 for (BasicBlock *BB : depth_first(FnPrologueEnd->getParent()))
1310 visit(*BB);
1311
1312 // Finalize PHI nodes.
1313 for (PHINode *PN : ShadowPHINodes) {
1314 PHINode *PNS = cast<PHINode>(getShadow(PN));
1315 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1316 size_t NumValues = PN->getNumIncomingValues();
1317 for (size_t v = 0; v < NumValues; v++) {
1318 PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1319 if (PNO) PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1320 }
1321 }
1322
1323 VAHelper->finalizeInstrumentation();
1324
1325 // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to
1326 // instrumenting only allocas.
1327 if (InstrumentLifetimeStart) {
1328 for (auto Item : LifetimeStartList) {
1329 instrumentAlloca(*Item.second, Item.first);
1330 AllocaSet.erase(Item.second);
1331 }
1332 }
1333 // Poison the allocas for which we didn't instrument the corresponding
1334 // lifetime intrinsics.
1335 for (AllocaInst *AI : AllocaSet)
1336 instrumentAlloca(*AI);
1337
1338 bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 &&
1339 InstrumentationList.size() + StoreList.size() >
1340 (unsigned)ClInstrumentationWithCallThreshold;
1341
1342 // Insert shadow value checks.
1343 materializeChecks(InstrumentWithCalls);
1344
1345 // Delayed instrumentation of StoreInst.
1346 // This may not add new address checks.
1347 materializeStores(InstrumentWithCalls);
1348
1349 return true;
1350 }
1351
1352 /// Compute the shadow type that corresponds to a given Value.
1353 Type *getShadowTy(Value *V) {
1354 return getShadowTy(V->getType());
1355 }
1356
1357 /// Compute the shadow type that corresponds to a given Type.
1358 Type *getShadowTy(Type *OrigTy) {
1359 if (!OrigTy->isSized()) {
1360 return nullptr;
1361 }
1362 // For integer type, shadow is the same as the original type.
1363 // This may return weird-sized types like i1.
1364 if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy))
1365 return IT;
1366 const DataLayout &DL = F.getParent()->getDataLayout();
1367 if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1368 uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType());
1369 return FixedVectorType::get(IntegerType::get(*MS.C, EltSize),
1370 cast<FixedVectorType>(VT)->getNumElements());
1371 }
1372 if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1373 return ArrayType::get(getShadowTy(AT->getElementType()),
1374 AT->getNumElements());
1375 }
1376 if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
1377 SmallVector<Type*, 4> Elements;
1378 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1379 Elements.push_back(getShadowTy(ST->getElementType(i)));
1380 StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked());
1381 LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n")do { } while (false);
1382 return Res;
1383 }
1384 uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy);
1385 return IntegerType::get(*MS.C, TypeSize);
1386 }
1387
1388 /// Flatten a vector type.
1389 Type *getShadowTyNoVec(Type *ty) {
1390 if (VectorType *vt = dyn_cast<VectorType>(ty))
1391 return IntegerType::get(*MS.C,
1392 vt->getPrimitiveSizeInBits().getFixedSize());
1393 return ty;
1394 }
1395
1396 /// Extract combined shadow of struct elements as a bool
1397 Value *collapseStructShadow(StructType *Struct, Value *Shadow,
1398 IRBuilder<> &IRB) {
1399 Value *FalseVal = IRB.getIntN(/* width */ 1, /* value */ 0);
1400 Value *Aggregator = FalseVal;
1401
1402 for (unsigned Idx = 0; Idx < Struct->getNumElements(); Idx++) {
1403 // Combine by ORing together each element's bool shadow
1404 Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1405 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1406 Value *ShadowBool = convertToBool(ShadowInner, IRB);
1407
1408 if (Aggregator != FalseVal)
1409 Aggregator = IRB.CreateOr(Aggregator, ShadowBool);
1410 else
1411 Aggregator = ShadowBool;
1412 }
1413
1414 return Aggregator;
1415 }
1416
1417 // Extract combined shadow of array elements
1418 Value *collapseArrayShadow(ArrayType *Array, Value *Shadow,
1419 IRBuilder<> &IRB) {
1420 if (!Array->getNumElements())
1421 return IRB.getIntN(/* width */ 1, /* value */ 0);
1422
1423 Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
1424 Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1425
1426 for (unsigned Idx = 1; Idx < Array->getNumElements(); Idx++) {
1427 Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1428 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1429 Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
1430 }
1431 return Aggregator;
1432 }
1433
1434 /// Convert a shadow value to it's flattened variant. The resulting
1435 /// shadow may not necessarily have the same bit width as the input
1436 /// value, but it will always be comparable to zero.
1437 Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
1438 if (StructType *Struct = dyn_cast<StructType>(V->getType()))
1439 return collapseStructShadow(Struct, V, IRB);
1440 if (ArrayType *Array = dyn_cast<ArrayType>(V->getType()))
1441 return collapseArrayShadow(Array, V, IRB);
1442 Type *Ty = V->getType();
1443 Type *NoVecTy = getShadowTyNoVec(Ty);
1444 if (Ty == NoVecTy) return V;
1445 return IRB.CreateBitCast(V, NoVecTy);
1446 }
1447
1448 // Convert a scalar value to an i1 by comparing with 0
1449 Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
1450 Type *VTy = V->getType();
1451 assert(VTy->isIntegerTy())((void)0);
1452 if (VTy->getIntegerBitWidth() == 1)
1453 // Just converting a bool to a bool, so do nothing.
1454 return V;
1455 return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), name);
1456 }
1457
1458 /// Compute the integer shadow offset that corresponds to a given
1459 /// application address.
1460 ///
1461 /// Offset = (Addr & ~AndMask) ^ XorMask
1462 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1463 Value *OffsetLong = IRB.CreatePointerCast(Addr, MS.IntptrTy);
1464
1465 uint64_t AndMask = MS.MapParams->AndMask;
1466 if (AndMask)
1467 OffsetLong =
1468 IRB.CreateAnd(OffsetLong, ConstantInt::get(MS.IntptrTy, ~AndMask));
1469
1470 uint64_t XorMask = MS.MapParams->XorMask;
1471 if (XorMask)
1472 OffsetLong =
1473 IRB.CreateXor(OffsetLong, ConstantInt::get(MS.IntptrTy, XorMask));
1474 return OffsetLong;
1475 }
1476
1477 /// Compute the shadow and origin addresses corresponding to a given
1478 /// application address.
1479 ///
1480 /// Shadow = ShadowBase + Offset
1481 /// Origin = (OriginBase + Offset) & ~3ULL
1482 std::pair<Value *, Value *>
1483 getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1484 MaybeAlign Alignment) {
1485 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1486 Value *ShadowLong = ShadowOffset;
1487 uint64_t ShadowBase = MS.MapParams->ShadowBase;
1488 if (ShadowBase != 0) {
1489 ShadowLong =
1490 IRB.CreateAdd(ShadowLong,
1491 ConstantInt::get(MS.IntptrTy, ShadowBase));
1492 }
1493 Value *ShadowPtr =
1494 IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
1495 Value *OriginPtr = nullptr;
1496 if (MS.TrackOrigins) {
1497 Value *OriginLong = ShadowOffset;
1498 uint64_t OriginBase = MS.MapParams->OriginBase;
1499 if (OriginBase != 0)
1500 OriginLong = IRB.CreateAdd(OriginLong,
1501 ConstantInt::get(MS.IntptrTy, OriginBase));
1502 if (!Alignment || *Alignment < kMinOriginAlignment) {
1503 uint64_t Mask = kMinOriginAlignment.value() - 1;
1504 OriginLong =
1505 IRB.CreateAnd(OriginLong, ConstantInt::get(MS.IntptrTy, ~Mask));
1506 }
1507 OriginPtr =
1508 IRB.CreateIntToPtr(OriginLong, PointerType::get(MS.OriginTy, 0));
1509 }
1510 return std::make_pair(ShadowPtr, OriginPtr);
1511 }
1512
1513 std::pair<Value *, Value *> getShadowOriginPtrKernel(Value *Addr,
1514 IRBuilder<> &IRB,
1515 Type *ShadowTy,
1516 bool isStore) {
1517 Value *ShadowOriginPtrs;
1518 const DataLayout &DL = F.getParent()->getDataLayout();
1519 int Size = DL.getTypeStoreSize(ShadowTy);
1520
1521 FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size);
1522 Value *AddrCast =
1523 IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0));
1524 if (Getter) {
1525 ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast);
1526 } else {
1527 Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
1528 ShadowOriginPtrs = IRB.CreateCall(isStore ? MS.MsanMetadataPtrForStoreN
1529 : MS.MsanMetadataPtrForLoadN,
1530 {AddrCast, SizeVal});
1531 }
1532 Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0);
1533 ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0));
1534 Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1);
1535
1536 return std::make_pair(ShadowPtr, OriginPtr);
1537 }
1538
1539 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1540 Type *ShadowTy,
1541 MaybeAlign Alignment,
1542 bool isStore) {
1543 if (MS.CompileKernel)
1544 return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore);
1545 return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1546 }
1547
1548 /// Compute the shadow address for a given function argument.
1549 ///
1550 /// Shadow = ParamTLS+ArgOffset.
1551 Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB,
1552 int ArgOffset) {
1553 Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
1554 if (ArgOffset)
1555 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1556 return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
1557 "_msarg");
1558 }
1559
1560 /// Compute the origin address for a given function argument.
1561 Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB,
1562 int ArgOffset) {
1563 if (!MS.TrackOrigins)
1564 return nullptr;
1565 Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
1566 if (ArgOffset)
1567 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1568 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
1569 "_msarg_o");
1570 }
1571
1572 /// Compute the shadow address for a retval.
1573 Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
1574 return IRB.CreatePointerCast(MS.RetvalTLS,
1575 PointerType::get(getShadowTy(A), 0),
1576 "_msret");
1577 }
1578
1579 /// Compute the origin address for a retval.
1580 Value *getOriginPtrForRetval(IRBuilder<> &IRB) {
1581 // We keep a single origin for the entire retval. Might be too optimistic.
1582 return MS.RetvalOriginTLS;
1583 }
1584
1585 /// Set SV to be the shadow value for V.
1586 void setShadow(Value *V, Value *SV) {
1587 assert(!ShadowMap.count(V) && "Values may only have one shadow")((void)0);
1588 ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
1589 }
1590
1591 /// Set Origin to be the origin value for V.
1592 void setOrigin(Value *V, Value *Origin) {
1593 if (!MS.TrackOrigins) return;
1594 assert(!OriginMap.count(V) && "Values may only have one origin")((void)0);
1595 LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n")do { } while (false);
1596 OriginMap[V] = Origin;
1597 }
1598
1599 Constant *getCleanShadow(Type *OrigTy) {
1600 Type *ShadowTy = getShadowTy(OrigTy);
1601 if (!ShadowTy)
1602 return nullptr;
1603 return Constant::getNullValue(ShadowTy);
1604 }
1605
1606 /// Create a clean shadow value for a given value.
1607 ///
1608 /// Clean shadow (all zeroes) means all bits of the value are defined
1609 /// (initialized).
1610 Constant *getCleanShadow(Value *V) {
1611 return getCleanShadow(V->getType());
1612 }
1613
1614 /// Create a dirty shadow of a given shadow type.
1615 Constant *getPoisonedShadow(Type *ShadowTy) {
1616 assert(ShadowTy)((void)0);
1617 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1618 return Constant::getAllOnesValue(ShadowTy);
1619 if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1620 SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1621 getPoisonedShadow(AT->getElementType()));
1622 return ConstantArray::get(AT, Vals);
1623 }
1624 if (StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1625 SmallVector<Constant *, 4> Vals;
1626 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1627 Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
1628 return ConstantStruct::get(ST, Vals);
1629 }
1630 llvm_unreachable("Unexpected shadow type")__builtin_unreachable();
1631 }
1632
1633 /// Create a dirty shadow for a given value.
1634 Constant *getPoisonedShadow(Value *V) {
1635 Type *ShadowTy = getShadowTy(V);
1636 if (!ShadowTy)
1637 return nullptr;
1638 return getPoisonedShadow(ShadowTy);
1639 }
1640
1641 /// Create a clean (zero) origin.
1642 Value *getCleanOrigin() {
1643 return Constant::getNullValue(MS.OriginTy);
1644 }
1645
1646 /// Get the shadow value for a given Value.
1647 ///
1648 /// This function either returns the value set earlier with setShadow,
1649 /// or extracts if from ParamTLS (for function arguments).
1650 Value *getShadow(Value *V) {
1651 if (!PropagateShadow) return getCleanShadow(V);
1652 if (Instruction *I = dyn_cast<Instruction>(V)) {
1653 if (I->getMetadata("nosanitize"))
1654 return getCleanShadow(V);
1655 // For instructions the shadow is already stored in the map.
1656 Value *Shadow = ShadowMap[V];
1657 if (!Shadow) {
1658 LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()))do { } while (false);
1659 (void)I;
1660 assert(Shadow && "No shadow for a value")((void)0);
1661 }
1662 return Shadow;
1663 }
1664 if (UndefValue *U = dyn_cast<UndefValue>(V)) {
1665 Value *AllOnes = PoisonUndef ? getPoisonedShadow(V) : getCleanShadow(V);
1666 LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n")do { } while (false);
1667 (void)U;
1668 return AllOnes;
1669 }
1670 if (Argument *A = dyn_cast<Argument>(V)) {
1671 // For arguments we compute the shadow on demand and store it in the map.
1672 Value **ShadowPtr = &ShadowMap[V];
1673 if (*ShadowPtr)
1674 return *ShadowPtr;
1675 Function *F = A->getParent();
1676 IRBuilder<> EntryIRB(FnPrologueEnd);
1677 unsigned ArgOffset = 0;
1678 const DataLayout &DL = F->getParent()->getDataLayout();
1679 for (auto &FArg : F->args()) {
1680 if (!FArg.getType()->isSized()) {
1681 LLVM_DEBUG(dbgs() << "Arg is not sized\n")do { } while (false);
1682 continue;
1683 }
1684
1685 bool FArgByVal = FArg.hasByValAttr();
1686 bool FArgNoUndef = FArg.hasAttribute(Attribute::NoUndef);
1687 bool FArgEagerCheck = ClEagerChecks && !FArgByVal && FArgNoUndef;
1688 unsigned Size =
1689 FArg.hasByValAttr()
1690 ? DL.getTypeAllocSize(FArg.getParamByValType())
1691 : DL.getTypeAllocSize(FArg.getType());
1692
1693 if (A == &FArg) {
1694 bool Overflow = ArgOffset + Size > kParamTLSSize;
1695 if (FArgEagerCheck) {
1696 *ShadowPtr = getCleanShadow(V);
1697 setOrigin(A, getCleanOrigin());
1698 continue;
1699 } else if (FArgByVal) {
1700 Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1701 // ByVal pointer itself has clean shadow. We copy the actual
1702 // argument shadow to the underlying memory.
1703 // Figure out maximal valid memcpy alignment.
1704 const Align ArgAlign = DL.getValueOrABITypeAlignment(
1705 MaybeAlign(FArg.getParamAlignment()), FArg.getParamByValType());
1706 Value *CpShadowPtr =
1707 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1708 /*isStore*/ true)
1709 .first;
1710 // TODO(glider): need to copy origins.
1711 if (Overflow) {
1712 // ParamTLS overflow.
1713 EntryIRB.CreateMemSet(
1714 CpShadowPtr, Constant::getNullValue(EntryIRB.getInt8Ty()),
1715 Size, ArgAlign);
1716 } else {
1717 const Align CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
1718 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base,
1719 CopyAlign, Size);
1720 LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n")do { } while (false);
1721 (void)Cpy;
1722 }
1723 *ShadowPtr = getCleanShadow(V);
1724 } else {
1725 // Shadow over TLS
1726 Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1727 if (Overflow) {
1728 // ParamTLS overflow.
1729 *ShadowPtr = getCleanShadow(V);
1730 } else {
1731 *ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg), Base,
1732 kShadowTLSAlignment);
1733 }
1734 }
1735 LLVM_DEBUG(dbgs()do { } while (false)
1736 << " ARG: " << FArg << " ==> " << **ShadowPtr << "\n")do { } while (false);
1737 if (MS.TrackOrigins && !Overflow) {
1738 Value *OriginPtr =
1739 getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1740 setOrigin(A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
1741 } else {
1742 setOrigin(A, getCleanOrigin());
1743 }
1744
1745 break;
1746 }
1747
1748 if (!FArgEagerCheck)
1749 ArgOffset += alignTo(Size, kShadowTLSAlignment);
1750 }
1751 assert(*ShadowPtr && "Could not find shadow for an argument")((void)0);
1752 return *ShadowPtr;
1753 }
1754 // For everything else the shadow is zero.
1755 return getCleanShadow(V);
1756 }
1757
1758 /// Get the shadow for i-th argument of the instruction I.
1759 Value *getShadow(Instruction *I, int i) {
1760 return getShadow(I->getOperand(i));
1761 }
1762
1763 /// Get the origin for a value.
1764 Value *getOrigin(Value *V) {
1765 if (!MS.TrackOrigins) return nullptr;
1766 if (!PropagateShadow) return getCleanOrigin();
1767 if (isa<Constant>(V)) return getCleanOrigin();
1768 assert((isa<Instruction>(V) || isa<Argument>(V)) &&((void)0)
1769 "Unexpected value type in getOrigin()")((void)0);
1770 if (Instruction *I = dyn_cast<Instruction>(V)) {
1771 if (I->getMetadata("nosanitize"))
1772 return getCleanOrigin();
1773 }
1774 Value *Origin = OriginMap[V];
1775 assert(Origin && "Missing origin")((void)0);
1776 return Origin;
1777 }
1778
1779 /// Get the origin for i-th argument of the instruction I.
1780 Value *getOrigin(Instruction *I, int i) {
1781 return getOrigin(I->getOperand(i));
1782 }
1783
1784 /// Remember the place where a shadow check should be inserted.
1785 ///
1786 /// This location will be later instrumented with a check that will print a
1787 /// UMR warning in runtime if the shadow value is not 0.
1788 void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
1789 assert(Shadow)((void)0);
1790 if (!InsertChecks) return;
1791#ifndef NDEBUG1
1792 Type *ShadowTy = Shadow->getType();
1793 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||((void)0)
1794 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&((void)0)
1795 "Can only insert checks for integer, vector, and aggregate shadow "((void)0)
1796 "types")((void)0);
1797#endif
1798 InstrumentationList.push_back(
1799 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
1800 }
1801
1802 /// Remember the place where a shadow check should be inserted.
1803 ///
1804 /// This location will be later instrumented with a check that will print a
1805 /// UMR warning in runtime if the value is not fully defined.
1806 void insertShadowCheck(Value *Val, Instruction *OrigIns) {
1807 assert(Val)((void)0);
1808 Value *Shadow, *Origin;
1809 if (ClCheckConstantShadow) {
1810 Shadow = getShadow(Val);
1811 if (!Shadow) return;
1812 Origin = getOrigin(Val);
1813 } else {
1814 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
1815 if (!Shadow) return;
1816 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
1817 }
1818 insertShadowCheck(Shadow, Origin, OrigIns);
1819 }
1820
1821 AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
1822 switch (a) {
1823 case AtomicOrdering::NotAtomic:
1824 return AtomicOrdering::NotAtomic;
1825 case AtomicOrdering::Unordered:
1826 case AtomicOrdering::Monotonic:
1827 case AtomicOrdering::Release:
1828 return AtomicOrdering::Release;
1829 case AtomicOrdering::Acquire:
1830 case AtomicOrdering::AcquireRelease:
1831 return AtomicOrdering::AcquireRelease;
1832 case AtomicOrdering::SequentiallyConsistent:
1833 return AtomicOrdering::SequentiallyConsistent;
1834 }
1835 llvm_unreachable("Unknown ordering")__builtin_unreachable();
1836 }
1837
1838 Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
1839 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
1840 uint32_t OrderingTable[NumOrderings] = {};
1841
1842 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
1843 OrderingTable[(int)AtomicOrderingCABI::release] =
1844 (int)AtomicOrderingCABI::release;
1845 OrderingTable[(int)AtomicOrderingCABI::consume] =
1846 OrderingTable[(int)AtomicOrderingCABI::acquire] =
1847 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
1848 (int)AtomicOrderingCABI::acq_rel;
1849 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
1850 (int)AtomicOrderingCABI::seq_cst;
1851
1852 return ConstantDataVector::get(IRB.getContext(),
1853 makeArrayRef(OrderingTable, NumOrderings));
1854 }
1855
1856 AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
1857 switch (a) {
1858 case AtomicOrdering::NotAtomic:
1859 return AtomicOrdering::NotAtomic;
1860 case AtomicOrdering::Unordered:
1861 case AtomicOrdering::Monotonic:
1862 case AtomicOrdering::Acquire:
1863 return AtomicOrdering::Acquire;
1864 case AtomicOrdering::Release:
1865 case AtomicOrdering::AcquireRelease:
1866 return AtomicOrdering::AcquireRelease;
1867 case AtomicOrdering::SequentiallyConsistent:
1868 return AtomicOrdering::SequentiallyConsistent;
1869 }
1870 llvm_unreachable("Unknown ordering")__builtin_unreachable();
1871 }
1872
1873 Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
1874 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
1875 uint32_t OrderingTable[NumOrderings] = {};
1876
1877 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
1878 OrderingTable[(int)AtomicOrderingCABI::acquire] =
1879 OrderingTable[(int)AtomicOrderingCABI::consume] =
1880 (int)AtomicOrderingCABI::acquire;
1881 OrderingTable[(int)AtomicOrderingCABI::release] =
1882 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
1883 (int)AtomicOrderingCABI::acq_rel;
1884 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
1885 (int)AtomicOrderingCABI::seq_cst;
1886
1887 return ConstantDataVector::get(IRB.getContext(),
1888 makeArrayRef(OrderingTable, NumOrderings));
1889 }
1890
1891 // ------------------- Visitors.
1892 using InstVisitor<MemorySanitizerVisitor>::visit;
1893 void visit(Instruction &I) {
1894 if (I.getMetadata("nosanitize"))
1895 return;
1896 // Don't want to visit if we're in the prologue
1897 if (isInPrologue(I))
1898 return;
1899 InstVisitor<MemorySanitizerVisitor>::visit(I);
1900 }
1901
1902 /// Instrument LoadInst
1903 ///
1904 /// Loads the corresponding shadow and (optionally) origin.
1905 /// Optionally, checks that the load address is fully defined.
1906 void visitLoadInst(LoadInst &I) {
1907 assert(I.getType()->isSized() && "Load type must have size")((void)0);
1908 assert(!I.getMetadata("nosanitize"))((void)0);
1909 IRBuilder<> IRB(I.getNextNode());
1910 Type *ShadowTy = getShadowTy(&I);
1911 Value *Addr = I.getPointerOperand();
1912 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
1913 const Align Alignment = assumeAligned(I.getAlignment());
1
Calling 'LoadInst::getAlignment'
1914 if (PropagateShadow) {
1915 std::tie(ShadowPtr, OriginPtr) =
1916 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
1917 setShadow(&I,
1918 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
1919 } else {
1920 setShadow(&I, getCleanShadow(&I));
1921 }
1922
1923 if (ClCheckAccessAddress)
1924 insertShadowCheck(I.getPointerOperand(), &I);
1925
1926 if (I.isAtomic())
1927 I.setOrdering(addAcquireOrdering(I.getOrdering()));
1928
1929 if (MS.TrackOrigins) {
1930 if (PropagateShadow) {
1931 const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1932 setOrigin(
1933 &I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, OriginAlignment));
1934 } else {
1935 setOrigin(&I, getCleanOrigin());
1936 }
1937 }
1938 }
1939
1940 /// Instrument StoreInst
1941 ///
1942 /// Stores the corresponding shadow and (optionally) origin.
1943 /// Optionally, checks that the store address is fully defined.
1944 void visitStoreInst(StoreInst &I) {
1945 StoreList.push_back(&I);
1946 if (ClCheckAccessAddress)
1947 insertShadowCheck(I.getPointerOperand(), &I);
1948 }
1949
1950 void handleCASOrRMW(Instruction &I) {
1951 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I))((void)0);
1952
1953 IRBuilder<> IRB(&I);
1954 Value *Addr = I.getOperand(0);
1955 Value *Val = I.getOperand(1);
1956 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, Val->getType(), Align(1),
1957 /*isStore*/ true)
1958 .first;
1959
1960 if (ClCheckAccessAddress)
1961 insertShadowCheck(Addr, &I);
1962
1963 // Only test the conditional argument of cmpxchg instruction.
1964 // The other argument can potentially be uninitialized, but we can not
1965 // detect this situation reliably without possible false positives.
1966 if (isa<AtomicCmpXchgInst>(I))
1967 insertShadowCheck(Val, &I);
1968
1969 IRB.CreateStore(getCleanShadow(Val), ShadowPtr);
1970
1971 setShadow(&I, getCleanShadow(&I));
1972 setOrigin(&I, getCleanOrigin());
1973 }
1974
1975 void visitAtomicRMWInst(AtomicRMWInst &I) {
1976 handleCASOrRMW(I);
1977 I.setOrdering(addReleaseOrdering(I.getOrdering()));
1978 }
1979
1980 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
1981 handleCASOrRMW(I);
1982 I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
1983 }
1984
1985 // Vector manipulation.
1986 void visitExtractElementInst(ExtractElementInst &I) {
1987 insertShadowCheck(I.getOperand(1), &I);
1988 IRBuilder<> IRB(&I);
1989 setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
1990 "_msprop"));
1991 setOrigin(&I, getOrigin(&I, 0));
1992 }
1993
1994 void visitInsertElementInst(InsertElementInst &I) {
1995 insertShadowCheck(I.getOperand(2), &I);
1996 IRBuilder<> IRB(&I);
1997 setShadow(&I, IRB.CreateInsertElement(getShadow(&I, 0), getShadow(&I, 1),
1998 I.getOperand(2), "_msprop"));
1999 setOriginForNaryOp(I);
2000 }
2001
2002 void visitShuffleVectorInst(ShuffleVectorInst &I) {
2003 IRBuilder<> IRB(&I);
2004 setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
2005 I.getShuffleMask(), "_msprop"));
2006 setOriginForNaryOp(I);
2007 }
2008
2009 // Casts.
2010 void visitSExtInst(SExtInst &I) {
2011 IRBuilder<> IRB(&I);
2012 setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
2013 setOrigin(&I, getOrigin(&I, 0));
2014 }
2015
2016 void visitZExtInst(ZExtInst &I) {
2017 IRBuilder<> IRB(&I);
2018 setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
2019 setOrigin(&I, getOrigin(&I, 0));
2020 }
2021
2022 void visitTruncInst(TruncInst &I) {
2023 IRBuilder<> IRB(&I);
2024 setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
2025 setOrigin(&I, getOrigin(&I, 0));
2026 }
2027
2028 void visitBitCastInst(BitCastInst &I) {
2029 // Special case: if this is the bitcast (there is exactly 1 allowed) between
2030 // a musttail call and a ret, don't instrument. New instructions are not
2031 // allowed after a musttail call.
2032 if (auto *CI = dyn_cast<CallInst>(I.getOperand(0)))
2033 if (CI->isMustTailCall())
2034 return;
2035 IRBuilder<> IRB(&I);
2036 setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
2037 setOrigin(&I, getOrigin(&I, 0));
2038 }
2039
2040 void visitPtrToIntInst(PtrToIntInst &I) {
2041 IRBuilder<> IRB(&I);
2042 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
2043 "_msprop_ptrtoint"));
2044 setOrigin(&I, getOrigin(&I, 0));
2045 }
2046
2047 void visitIntToPtrInst(IntToPtrInst &I) {
2048 IRBuilder<> IRB(&I);
2049 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
2050 "_msprop_inttoptr"));
2051 setOrigin(&I, getOrigin(&I, 0));
2052 }
2053
2054 void visitFPToSIInst(CastInst& I) { handleShadowOr(I); }
2055 void visitFPToUIInst(CastInst& I) { handleShadowOr(I); }
2056 void visitSIToFPInst(CastInst& I) { handleShadowOr(I); }
2057 void visitUIToFPInst(CastInst& I) { handleShadowOr(I); }
2058 void visitFPExtInst(CastInst& I) { handleShadowOr(I); }
2059 void visitFPTruncInst(CastInst& I) { handleShadowOr(I); }
2060
2061 /// Propagate shadow for bitwise AND.
2062 ///
2063 /// This code is exact, i.e. if, for example, a bit in the left argument
2064 /// is defined and 0, then neither the value not definedness of the
2065 /// corresponding bit in B don't affect the resulting shadow.
2066 void visitAnd(BinaryOperator &I) {
2067 IRBuilder<> IRB(&I);
2068 // "And" of 0 and a poisoned value results in unpoisoned value.
2069 // 1&1 => 1; 0&1 => 0; p&1 => p;
2070 // 1&0 => 0; 0&0 => 0; p&0 => 0;
2071 // 1&p => p; 0&p => 0; p&p => p;
2072 // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
2073 Value *S1 = getShadow(&I, 0);
2074 Value *S2 = getShadow(&I, 1);
2075 Value *V1 = I.getOperand(0);
2076 Value *V2 = I.getOperand(1);
2077 if (V1->getType() != S1->getType()) {
2078 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
2079 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
2080 }
2081 Value *S1S2 = IRB.CreateAnd(S1, S2);
2082 Value *V1S2 = IRB.CreateAnd(V1, S2);
2083 Value *S1V2 = IRB.CreateAnd(S1, V2);
2084 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
2085 setOriginForNaryOp(I);
2086 }
2087
2088 void visitOr(BinaryOperator &I) {
2089 IRBuilder<> IRB(&I);
2090 // "Or" of 1 and a poisoned value results in unpoisoned value.
2091 // 1|1 => 1; 0|1 => 1; p|1 => 1;
2092 // 1|0 => 1; 0|0 => 0; p|0 => p;
2093 // 1|p => 1; 0|p => p; p|p => p;
2094 // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
2095 Value *S1 = getShadow(&I, 0);
2096 Value *S2 = getShadow(&I, 1);
2097 Value *V1 = IRB.CreateNot(I.getOperand(0));
2098 Value *V2 = IRB.CreateNot(I.getOperand(1));
2099 if (V1->getType() != S1->getType()) {
2100 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
2101 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
2102 }
2103 Value *S1S2 = IRB.CreateAnd(S1, S2);
2104 Value *V1S2 = IRB.CreateAnd(V1, S2);
2105 Value *S1V2 = IRB.CreateAnd(S1, V2);
2106 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
2107 setOriginForNaryOp(I);
2108 }
2109
2110 /// Default propagation of shadow and/or origin.
2111 ///
2112 /// This class implements the general case of shadow propagation, used in all
2113 /// cases where we don't know and/or don't care about what the operation
2114 /// actually does. It converts all input shadow values to a common type
2115 /// (extending or truncating as necessary), and bitwise OR's them.
2116 ///
2117 /// This is much cheaper than inserting checks (i.e. requiring inputs to be
2118 /// fully initialized), and less prone to false positives.
2119 ///
2120 /// This class also implements the general case of origin propagation. For a
2121 /// Nary operation, result origin is set to the origin of an argument that is
2122 /// not entirely initialized. If there is more than one such arguments, the
2123 /// rightmost of them is picked. It does not matter which one is picked if all
2124 /// arguments are initialized.
2125 template <bool CombineShadow>
2126 class Combiner {
2127 Value *Shadow = nullptr;
2128 Value *Origin = nullptr;
2129 IRBuilder<> &IRB;
2130 MemorySanitizerVisitor *MSV;
2131
2132 public:
2133 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
2134 : IRB(IRB), MSV(MSV) {}
2135
2136 /// Add a pair of shadow and origin values to the mix.
2137 Combiner &Add(Value *OpShadow, Value *OpOrigin) {
2138 if (CombineShadow) {
2139 assert(OpShadow)((void)0);
2140 if (!Shadow)
2141 Shadow = OpShadow;
2142 else {
2143 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2144 Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
2145 }
2146 }
2147
2148 if (MSV->MS.TrackOrigins) {
2149 assert(OpOrigin)((void)0);
2150 if (!Origin) {
2151 Origin = OpOrigin;
2152 } else {
2153 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2154 // No point in adding something that might result in 0 origin value.
2155 if (!ConstOrigin || !ConstOrigin->isNullValue()) {
2156 Value *FlatShadow = MSV->convertShadowToScalar(OpShadow, IRB);
2157 Value *Cond =
2158 IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
2159 Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
2160 }
2161 }
2162 }
2163 return *this;
2164 }
2165
2166 /// Add an application value to the mix.
2167 Combiner &Add(Value *V) {
2168 Value *OpShadow = MSV->getShadow(V);
2169 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
2170 return Add(OpShadow, OpOrigin);
2171 }
2172
2173 /// Set the current combined values as the given instruction's shadow
2174 /// and origin.
2175 void Done(Instruction *I) {
2176 if (CombineShadow) {
2177 assert(Shadow)((void)0);
2178 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
2179 MSV->setShadow(I, Shadow);
2180 }
2181 if (MSV->MS.TrackOrigins) {
2182 assert(Origin)((void)0);
2183 MSV->setOrigin(I, Origin);
2184 }
2185 }
2186 };
2187
2188 using ShadowAndOriginCombiner = Combiner<true>;
2189 using OriginCombiner = Combiner<false>;
2190
2191 /// Propagate origin for arbitrary operation.
2192 void setOriginForNaryOp(Instruction &I) {
2193 if (!MS.TrackOrigins) return;
2194 IRBuilder<> IRB(&I);
2195 OriginCombiner OC(this, IRB);
2196 for (Use &Op : I.operands())
2197 OC.Add(Op.get());
2198 OC.Done(&I);
2199 }
2200
2201 size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
2202 assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&((void)0)
2203 "Vector of pointers is not a valid shadow type")((void)0);
2204 return Ty->isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2205 Ty->getScalarSizeInBits()
2206 : Ty->getPrimitiveSizeInBits();
2207 }
2208
2209 /// Cast between two shadow types, extending or truncating as
2210 /// necessary.
2211 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2212 bool Signed = false) {
2213 Type *srcTy = V->getType();
2214 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2215 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2216 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2217 return IRB.CreateICmpNE(V, getCleanShadow(V));
2218
2219 if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
2220 return IRB.CreateIntCast(V, dstTy, Signed);
2221 if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
2222 cast<FixedVectorType>(dstTy)->getNumElements() ==
2223 cast<FixedVectorType>(srcTy)->getNumElements())
2224 return IRB.CreateIntCast(V, dstTy, Signed);
2225 Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
2226 Value *V2 =
2227 IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
2228 return IRB.CreateBitCast(V2, dstTy);
2229 // TODO: handle struct types.
2230 }
2231
2232 /// Cast an application value to the type of its own shadow.
2233 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2234 Type *ShadowTy = getShadowTy(V);
2235 if (V->getType() == ShadowTy)
2236 return V;
2237 if (V->getType()->isPtrOrPtrVectorTy())
2238 return IRB.CreatePtrToInt(V, ShadowTy);
2239 else
2240 return IRB.CreateBitCast(V, ShadowTy);
2241 }
2242
2243 /// Propagate shadow for arbitrary operation.
2244 void handleShadowOr(Instruction &I) {
2245 IRBuilder<> IRB(&I);
2246 ShadowAndOriginCombiner SC(this, IRB);
2247 for (Use &Op : I.operands())
2248 SC.Add(Op.get());
2249 SC.Done(&I);
2250 }
2251
2252 void visitFNeg(UnaryOperator &I) { handleShadowOr(I); }
2253
2254 // Handle multiplication by constant.
2255 //
2256 // Handle a special case of multiplication by constant that may have one or
2257 // more zeros in the lower bits. This makes corresponding number of lower bits
2258 // of the result zero as well. We model it by shifting the other operand
2259 // shadow left by the required number of bits. Effectively, we transform
2260 // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
2261 // We use multiplication by 2**N instead of shift to cover the case of
2262 // multiplication by 0, which may occur in some elements of a vector operand.
2263 void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
2264 Value *OtherArg) {
2265 Constant *ShadowMul;
2266 Type *Ty = ConstArg->getType();
2267 if (auto *VTy = dyn_cast<VectorType>(Ty)) {
2268 unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2269 Type *EltTy = VTy->getElementType();
2270 SmallVector<Constant *, 16> Elements;
2271 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
2272 if (ConstantInt *Elt =
2273 dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx))) {
2274 const APInt &V = Elt->getValue();
2275 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2276 Elements.push_back(ConstantInt::get(EltTy, V2));
2277 } else {
2278 Elements.push_back(ConstantInt::get(EltTy, 1));
2279 }
2280 }
2281 ShadowMul = ConstantVector::get(Elements);
2282 } else {
2283 if (ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2284 const APInt &V = Elt->getValue();
2285 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2286 ShadowMul = ConstantInt::get(Ty, V2);
2287 } else {
2288 ShadowMul = ConstantInt::get(Ty, 1);
2289 }
2290 }
2291
2292 IRBuilder<> IRB(&I);
2293 setShadow(&I,
2294 IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
2295 setOrigin(&I, getOrigin(OtherArg));
2296 }
2297
2298 void visitMul(BinaryOperator &I) {
2299 Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0));
2300 Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1));
2301 if (constOp0 && !constOp1)
2302 handleMulByConstant(I, constOp0, I.getOperand(1));
2303 else if (constOp1 && !constOp0)
2304 handleMulByConstant(I, constOp1, I.getOperand(0));
2305 else
2306 handleShadowOr(I);
2307 }
2308
2309 void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
2310 void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
2311 void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
2312 void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
2313 void visitSub(BinaryOperator &I) { handleShadowOr(I); }
2314 void visitXor(BinaryOperator &I) { handleShadowOr(I); }
2315
2316 void handleIntegerDiv(Instruction &I) {
2317 IRBuilder<> IRB(&I);
2318 // Strict on the second argument.
2319 insertShadowCheck(I.getOperand(1), &I);
2320 setShadow(&I, getShadow(&I, 0));
2321 setOrigin(&I, getOrigin(&I, 0));
2322 }
2323
2324 void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2325 void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2326 void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
2327 void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
2328
2329 // Floating point division is side-effect free. We can not require that the
2330 // divisor is fully initialized and must propagate shadow. See PR37523.
2331 void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
2332 void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
2333
2334 /// Instrument == and != comparisons.
2335 ///
2336 /// Sometimes the comparison result is known even if some of the bits of the
2337 /// arguments are not.
2338 void handleEqualityComparison(ICmpInst &I) {
2339 IRBuilder<> IRB(&I);
2340 Value *A = I.getOperand(0);
2341 Value *B = I.getOperand(1);
2342 Value *Sa = getShadow(A);
2343 Value *Sb = getShadow(B);
2344
2345 // Get rid of pointers and vectors of pointers.
2346 // For ints (and vectors of ints), types of A and Sa match,
2347 // and this is a no-op.
2348 A = IRB.CreatePointerCast(A, Sa->getType());
2349 B = IRB.CreatePointerCast(B, Sb->getType());
2350
2351 // A == B <==> (C = A^B) == 0
2352 // A != B <==> (C = A^B) != 0
2353 // Sc = Sa | Sb
2354 Value *C = IRB.CreateXor(A, B);
2355 Value *Sc = IRB.CreateOr(Sa, Sb);
2356 // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
2357 // Result is defined if one of the following is true
2358 // * there is a defined 1 bit in C
2359 // * C is fully defined
2360 // Si = !(C & ~Sc) && Sc
2361 Value *Zero = Constant::getNullValue(Sc->getType());
2362 Value *MinusOne = Constant::getAllOnesValue(Sc->getType());
2363 Value *Si =
2364 IRB.CreateAnd(IRB.CreateICmpNE(Sc, Zero),
2365 IRB.CreateICmpEQ(
2366 IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero));
2367 Si->setName("_msprop_icmp");
2368 setShadow(&I, Si);
2369 setOriginForNaryOp(I);
2370 }
2371
2372 /// Build the lowest possible value of V, taking into account V's
2373 /// uninitialized bits.
2374 Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2375 bool isSigned) {
2376 if (isSigned) {
2377 // Split shadow into sign bit and other bits.
2378 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2379 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2380 // Maximise the undefined shadow bit, minimize other undefined bits.
2381 return
2382 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), SaSignBit);
2383 } else {
2384 // Minimize undefined bits.
2385 return IRB.CreateAnd(A, IRB.CreateNot(Sa));
2386 }
2387 }
2388
2389 /// Build the highest possible value of V, taking into account V's
2390 /// uninitialized bits.
2391 Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2392 bool isSigned) {
2393 if (isSigned) {
2394 // Split shadow into sign bit and other bits.
2395 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2396 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2397 // Minimise the undefined shadow bit, maximise other undefined bits.
2398 return
2399 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), SaOtherBits);
2400 } else {
2401 // Maximize undefined bits.
2402 return IRB.CreateOr(A, Sa);
2403 }
2404 }
2405
2406 /// Instrument relational comparisons.
2407 ///
2408 /// This function does exact shadow propagation for all relational
2409 /// comparisons of integers, pointers and vectors of those.
2410 /// FIXME: output seems suboptimal when one of the operands is a constant
2411 void handleRelationalComparisonExact(ICmpInst &I) {
2412 IRBuilder<> IRB(&I);
2413 Value *A = I.getOperand(0);
2414 Value *B = I.getOperand(1);
2415 Value *Sa = getShadow(A);
2416 Value *Sb = getShadow(B);
2417
2418 // Get rid of pointers and vectors of pointers.
2419 // For ints (and vectors of ints), types of A and Sa match,
2420 // and this is a no-op.
2421 A = IRB.CreatePointerCast(A, Sa->getType());
2422 B = IRB.CreatePointerCast(B, Sb->getType());
2423
2424 // Let [a0, a1] be the interval of possible values of A, taking into account
2425 // its undefined bits. Let [b0, b1] be the interval of possible values of B.
2426 // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
2427 bool IsSigned = I.isSigned();
2428 Value *S1 = IRB.CreateICmp(I.getPredicate(),
2429 getLowestPossibleValue(IRB, A, Sa, IsSigned),
2430 getHighestPossibleValue(IRB, B, Sb, IsSigned));
2431 Value *S2 = IRB.CreateICmp(I.getPredicate(),
2432 getHighestPossibleValue(IRB, A, Sa, IsSigned),
2433 getLowestPossibleValue(IRB, B, Sb, IsSigned));
2434 Value *Si = IRB.CreateXor(S1, S2);
2435 setShadow(&I, Si);
2436 setOriginForNaryOp(I);
2437 }
2438
2439 /// Instrument signed relational comparisons.
2440 ///
2441 /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
2442 /// bit of the shadow. Everything else is delegated to handleShadowOr().
2443 void handleSignedRelationalComparison(ICmpInst &I) {
2444 Constant *constOp;
2445 Value *op = nullptr;
2446 CmpInst::Predicate pre;
2447 if ((constOp = dyn_cast<Constant>(I.getOperand(1)))) {
2448 op = I.getOperand(0);
2449 pre = I.getPredicate();
2450 } else if ((constOp = dyn_cast<Constant>(I.getOperand(0)))) {
2451 op = I.getOperand(1);
2452 pre = I.getSwappedPredicate();
2453 } else {
2454 handleShadowOr(I);
2455 return;
2456 }
2457
2458 if ((constOp->isNullValue() &&
2459 (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
2460 (constOp->isAllOnesValue() &&
2461 (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
2462 IRBuilder<> IRB(&I);
2463 Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
2464 "_msprop_icmp_s");
2465 setShadow(&I, Shadow);
2466 setOrigin(&I, getOrigin(op));
2467 } else {
2468 handleShadowOr(I);
2469 }
2470 }
2471
2472 void visitICmpInst(ICmpInst &I) {
2473 if (!ClHandleICmp) {
2474 handleShadowOr(I);
2475 return;
2476 }
2477 if (I.isEquality()) {
2478 handleEqualityComparison(I);
2479 return;
2480 }
2481
2482 assert(I.isRelational())((void)0);
2483 if (ClHandleICmpExact) {
2484 handleRelationalComparisonExact(I);
2485 return;
2486 }
2487 if (I.isSigned()) {
2488 handleSignedRelationalComparison(I);
2489 return;
2490 }
2491
2492 assert(I.isUnsigned())((void)0);
2493 if ((isa<Constant>(I.getOperand(0)) || isa<Constant>(I.getOperand(1)))) {
2494 handleRelationalComparisonExact(I);
2495 return;
2496 }
2497
2498 handleShadowOr(I);
2499 }
2500
2501 void visitFCmpInst(FCmpInst &I) {
2502 handleShadowOr(I);
2503 }
2504
2505 void handleShift(BinaryOperator &I) {
2506 IRBuilder<> IRB(&I);
2507 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2508 // Otherwise perform the same shift on S1.
2509 Value *S1 = getShadow(&I, 0);
2510 Value *S2 = getShadow(&I, 1);
2511 Value *S2Conv = IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)),
2512 S2->getType());
2513 Value *V2 = I.getOperand(1);
2514 Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
2515 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2516 setOriginForNaryOp(I);
2517 }
2518
2519 void visitShl(BinaryOperator &I) { handleShift(I); }
2520 void visitAShr(BinaryOperator &I) { handleShift(I); }
2521 void visitLShr(BinaryOperator &I) { handleShift(I); }
2522
2523 void handleFunnelShift(IntrinsicInst &I) {
2524 IRBuilder<> IRB(&I);
2525 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2526 // Otherwise perform the same shift on S0 and S1.
2527 Value *S0 = getShadow(&I, 0);
2528 Value *S1 = getShadow(&I, 1);
2529 Value *S2 = getShadow(&I, 2);
2530 Value *S2Conv =
2531 IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType());
2532 Value *V2 = I.getOperand(2);
2533 Function *Intrin = Intrinsic::getDeclaration(
2534 I.getModule(), I.getIntrinsicID(), S2Conv->getType());
2535 Value *Shift = IRB.CreateCall(Intrin, {S0, S1, V2});
2536 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2537 setOriginForNaryOp(I);
2538 }
2539
2540 /// Instrument llvm.memmove
2541 ///
2542 /// At this point we don't know if llvm.memmove will be inlined or not.
2543 /// If we don't instrument it and it gets inlined,
2544 /// our interceptor will not kick in and we will lose the memmove.
2545 /// If we instrument the call here, but it does not get inlined,
2546 /// we will memove the shadow twice: which is bad in case
2547 /// of overlapping regions. So, we simply lower the intrinsic to a call.
2548 ///
2549 /// Similar situation exists for memcpy and memset.
2550 void visitMemMoveInst(MemMoveInst &I) {
2551 IRBuilder<> IRB(&I);
2552 IRB.CreateCall(
2553 MS.MemmoveFn,
2554 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2555 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2556 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2557 I.eraseFromParent();
2558 }
2559
2560 // Similar to memmove: avoid copying shadow twice.
2561 // This is somewhat unfortunate as it may slowdown small constant memcpys.
2562 // FIXME: consider doing manual inline for small constant sizes and proper
2563 // alignment.
2564 void visitMemCpyInst(MemCpyInst &I) {
2565 IRBuilder<> IRB(&I);
2566 IRB.CreateCall(
2567 MS.MemcpyFn,
2568 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2569 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2570 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2571 I.eraseFromParent();
2572 }
2573
2574 // Same as memcpy.
2575 void visitMemSetInst(MemSetInst &I) {
2576 IRBuilder<> IRB(&I);
2577 IRB.CreateCall(
2578 MS.MemsetFn,
2579 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2580 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2581 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2582 I.eraseFromParent();
2583 }
2584
2585 void visitVAStartInst(VAStartInst &I) {
2586 VAHelper->visitVAStartInst(I);
2587 }
2588
2589 void visitVACopyInst(VACopyInst &I) {
2590 VAHelper->visitVACopyInst(I);
2591 }
2592
2593 /// Handle vector store-like intrinsics.
2594 ///
2595 /// Instrument intrinsics that look like a simple SIMD store: writes memory,
2596 /// has 1 pointer argument and 1 vector argument, returns void.
2597 bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
2598 IRBuilder<> IRB(&I);
2599 Value* Addr = I.getArgOperand(0);
2600 Value *Shadow = getShadow(&I, 1);
2601 Value *ShadowPtr, *OriginPtr;
2602
2603 // We don't know the pointer alignment (could be unaligned SSE store!).
2604 // Have to assume to worst case.
2605 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2606 Addr, IRB, Shadow->getType(), Align(1), /*isStore*/ true);
2607 IRB.CreateAlignedStore(Shadow, ShadowPtr, Align(1));
2608
2609 if (ClCheckAccessAddress)
2610 insertShadowCheck(Addr, &I);
2611
2612 // FIXME: factor out common code from materializeStores
2613 if (MS.TrackOrigins) IRB.CreateStore(getOrigin(&I, 1), OriginPtr);
2614 return true;
2615 }
2616
2617 /// Handle vector load-like intrinsics.
2618 ///
2619 /// Instrument intrinsics that look like a simple SIMD load: reads memory,
2620 /// has 1 pointer argument, returns a vector.
2621 bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
2622 IRBuilder<> IRB(&I);
2623 Value *Addr = I.getArgOperand(0);
2624
2625 Type *ShadowTy = getShadowTy(&I);
2626 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
2627 if (PropagateShadow) {
2628 // We don't know the pointer alignment (could be unaligned SSE load!).
2629 // Have to assume to worst case.
2630 const Align Alignment = Align(1);
2631 std::tie(ShadowPtr, OriginPtr) =
2632 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2633 setShadow(&I,
2634 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
2635 } else {
2636 setShadow(&I, getCleanShadow(&I));
2637 }
2638
2639 if (ClCheckAccessAddress)
2640 insertShadowCheck(Addr, &I);
2641
2642 if (MS.TrackOrigins) {
2643 if (PropagateShadow)
2644 setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr));
2645 else
2646 setOrigin(&I, getCleanOrigin());
2647 }
2648 return true;
2649 }
2650
2651 /// Handle (SIMD arithmetic)-like intrinsics.
2652 ///
2653 /// Instrument intrinsics with any number of arguments of the same type,
2654 /// equal to the return type. The type should be simple (no aggregates or
2655 /// pointers; vectors are fine).
2656 /// Caller guarantees that this intrinsic does not access memory.
2657 bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
2658 Type *RetTy = I.getType();
2659 if (!(RetTy->isIntOrIntVectorTy() ||
2660 RetTy->isFPOrFPVectorTy() ||
2661 RetTy->isX86_MMXTy()))
2662 return false;
2663
2664 unsigned NumArgOperands = I.getNumArgOperands();
2665 for (unsigned i = 0; i < NumArgOperands; ++i) {
2666 Type *Ty = I.getArgOperand(i)->getType();
2667 if (Ty != RetTy)
2668 return false;
2669 }
2670
2671 IRBuilder<> IRB(&I);
2672 ShadowAndOriginCombiner SC(this, IRB);
2673 for (unsigned i = 0; i < NumArgOperands; ++i)
2674 SC.Add(I.getArgOperand(i));
2675 SC.Done(&I);
2676
2677 return true;
2678 }
2679
2680 /// Heuristically instrument unknown intrinsics.
2681 ///
2682 /// The main purpose of this code is to do something reasonable with all
2683 /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
2684 /// We recognize several classes of intrinsics by their argument types and
2685 /// ModRefBehaviour and apply special instrumentation when we are reasonably
2686 /// sure that we know what the intrinsic does.
2687 ///
2688 /// We special-case intrinsics where this approach fails. See llvm.bswap
2689 /// handling as an example of that.
2690 bool handleUnknownIntrinsic(IntrinsicInst &I) {
2691 unsigned NumArgOperands = I.getNumArgOperands();
2692 if (NumArgOperands == 0)
2693 return false;
2694
2695 if (NumArgOperands == 2 &&
2696 I.getArgOperand(0)->getType()->isPointerTy() &&
2697 I.getArgOperand(1)->getType()->isVectorTy() &&
2698 I.getType()->isVoidTy() &&
2699 !I.onlyReadsMemory()) {
2700 // This looks like a vector store.
2701 return handleVectorStoreIntrinsic(I);
2702 }
2703
2704 if (NumArgOperands == 1 &&
2705 I.getArgOperand(0)->getType()->isPointerTy() &&
2706 I.getType()->isVectorTy() &&
2707 I.onlyReadsMemory()) {
2708 // This looks like a vector load.
2709 return handleVectorLoadIntrinsic(I);
2710 }
2711
2712 if (I.doesNotAccessMemory())
2713 if (maybeHandleSimpleNomemIntrinsic(I))
2714 return true;
2715
2716 // FIXME: detect and handle SSE maskstore/maskload
2717 return false;
2718 }
2719
2720 void handleInvariantGroup(IntrinsicInst &I) {
2721 setShadow(&I, getShadow(&I, 0));
2722 setOrigin(&I, getOrigin(&I, 0));
2723 }
2724
2725 void handleLifetimeStart(IntrinsicInst &I) {
2726 if (!PoisonStack)
2727 return;
2728 AllocaInst *AI = llvm::findAllocaForValue(I.getArgOperand(1));
2729 if (!AI)
2730 InstrumentLifetimeStart = false;
2731 LifetimeStartList.push_back(std::make_pair(&I, AI));
2732 }
2733
2734 void handleBswap(IntrinsicInst &I) {
2735 IRBuilder<> IRB(&I);
2736 Value *Op = I.getArgOperand(0);
2737 Type *OpType = Op->getType();
2738 Function *BswapFunc = Intrinsic::getDeclaration(
2739 F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1));
2740 setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
2741 setOrigin(&I, getOrigin(Op));
2742 }
2743
2744 // Instrument vector convert intrinsic.
2745 //
2746 // This function instruments intrinsics like cvtsi2ss:
2747 // %Out = int_xxx_cvtyyy(%ConvertOp)
2748 // or
2749 // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
2750 // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
2751 // number \p Out elements, and (if has 2 arguments) copies the rest of the
2752 // elements from \p CopyOp.
2753 // In most cases conversion involves floating-point value which may trigger a
2754 // hardware exception when not fully initialized. For this reason we require
2755 // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
2756 // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
2757 // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
2758 // return a fully initialized value.
2759 void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements,
2760 bool HasRoundingMode = false) {
2761 IRBuilder<> IRB(&I);
2762 Value *CopyOp, *ConvertOp;
2763
2764 assert((!HasRoundingMode ||((void)0)
2765 isa<ConstantInt>(I.getArgOperand(I.getNumArgOperands() - 1))) &&((void)0)
2766 "Invalid rounding mode")((void)0);
2767
2768 switch (I.getNumArgOperands() - HasRoundingMode) {
2769 case 2:
2770 CopyOp = I.getArgOperand(0);
2771 ConvertOp = I.getArgOperand(1);
2772 break;
2773 case 1:
2774 ConvertOp = I.getArgOperand(0);
2775 CopyOp = nullptr;
2776 break;
2777 default:
2778 llvm_unreachable("Cvt intrinsic with unsupported number of arguments.")__builtin_unreachable();
2779 }
2780
2781 // The first *NumUsedElements* elements of ConvertOp are converted to the
2782 // same number of output elements. The rest of the output is copied from
2783 // CopyOp, or (if not available) filled with zeroes.
2784 // Combine shadow for elements of ConvertOp that are used in this operation,
2785 // and insert a check.
2786 // FIXME: consider propagating shadow of ConvertOp, at least in the case of
2787 // int->any conversion.
2788 Value *ConvertShadow = getShadow(ConvertOp);
2789 Value *AggShadow = nullptr;
2790 if (ConvertOp->getType()->isVectorTy()) {
2791 AggShadow = IRB.CreateExtractElement(
2792 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2793 for (int i = 1; i < NumUsedElements; ++i) {
2794 Value *MoreShadow = IRB.CreateExtractElement(
2795 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2796 AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
2797 }
2798 } else {
2799 AggShadow = ConvertShadow;
2800 }
2801 assert(AggShadow->getType()->isIntegerTy())((void)0);
2802 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I);
2803
2804 // Build result shadow by zero-filling parts of CopyOp shadow that come from
2805 // ConvertOp.
2806 if (CopyOp) {
2807 assert(CopyOp->getType() == I.getType())((void)0);
2808 assert(CopyOp->getType()->isVectorTy())((void)0);
2809 Value *ResultShadow = getShadow(CopyOp);
2810 Type *EltTy = cast<VectorType>(ResultShadow->getType())->getElementType();
2811 for (int i = 0; i < NumUsedElements; ++i) {
2812 ResultShadow = IRB.CreateInsertElement(
2813 ResultShadow, ConstantInt::getNullValue(EltTy),
2814 ConstantInt::get(IRB.getInt32Ty(), i));
2815 }
2816 setShadow(&I, ResultShadow);
2817 setOrigin(&I, getOrigin(CopyOp));
2818 } else {
2819 setShadow(&I, getCleanShadow(&I));
2820 setOrigin(&I, getCleanOrigin());
2821 }
2822 }
2823
2824 // Given a scalar or vector, extract lower 64 bits (or less), and return all
2825 // zeroes if it is zero, and all ones otherwise.
2826 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2827 if (S->getType()->isVectorTy())
2828 S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
2829 assert(S->getType()->getPrimitiveSizeInBits() <= 64)((void)0);
2830 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2831 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2832 }
2833
2834 // Given a vector, extract its first element, and return all
2835 // zeroes if it is zero, and all ones otherwise.
2836 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2837 Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
2838 Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
2839 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2840 }
2841
2842 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
2843 Type *T = S->getType();
2844 assert(T->isVectorTy())((void)0);
2845 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2846 return IRB.CreateSExt(S2, T);
2847 }
2848
2849 // Instrument vector shift intrinsic.
2850 //
2851 // This function instruments intrinsics like int_x86_avx2_psll_w.
2852 // Intrinsic shifts %In by %ShiftSize bits.
2853 // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
2854 // size, and the rest is ignored. Behavior is defined even if shift size is
2855 // greater than register (or field) width.
2856 void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
2857 assert(I.getNumArgOperands() == 2)((void)0);
2858 IRBuilder<> IRB(&I);
2859 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2860 // Otherwise perform the same shift on S1.
2861 Value *S1 = getShadow(&I, 0);
2862 Value *S2 = getShadow(&I, 1);
2863 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
2864 : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
2865 Value *V1 = I.getOperand(0);
2866 Value *V2 = I.getOperand(1);
2867 Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledOperand(),
2868 {IRB.CreateBitCast(S1, V1->getType()), V2});
2869 Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
2870 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2871 setOriginForNaryOp(I);
2872 }
2873
2874 // Get an X86_MMX-sized vector type.
2875 Type *getMMXVectorTy(unsigned EltSizeInBits) {
2876 const unsigned X86_MMXSizeInBits = 64;
2877 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&((void)0)
2878 "Illegal MMX vector element size")((void)0);
2879 return FixedVectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
2880 X86_MMXSizeInBits / EltSizeInBits);
2881 }
2882
2883 // Returns a signed counterpart for an (un)signed-saturate-and-pack
2884 // intrinsic.
2885 Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
2886 switch (id) {
2887 case Intrinsic::x86_sse2_packsswb_128:
2888 case Intrinsic::x86_sse2_packuswb_128:
2889 return Intrinsic::x86_sse2_packsswb_128;
2890
2891 case Intrinsic::x86_sse2_packssdw_128:
2892 case Intrinsic::x86_sse41_packusdw:
2893 return Intrinsic::x86_sse2_packssdw_128;
2894
2895 case Intrinsic::x86_avx2_packsswb:
2896 case Intrinsic::x86_avx2_packuswb:
2897 return Intrinsic::x86_avx2_packsswb;
2898
2899 case Intrinsic::x86_avx2_packssdw:
2900 case Intrinsic::x86_avx2_packusdw:
2901 return Intrinsic::x86_avx2_packssdw;
2902
2903 case Intrinsic::x86_mmx_packsswb:
2904 case Intrinsic::x86_mmx_packuswb:
2905 return Intrinsic::x86_mmx_packsswb;
2906
2907 case Intrinsic::x86_mmx_packssdw:
2908 return Intrinsic::x86_mmx_packssdw;
2909 default:
2910 llvm_unreachable("unexpected intrinsic id")__builtin_unreachable();
2911 }
2912 }
2913
2914 // Instrument vector pack intrinsic.
2915 //
2916 // This function instruments intrinsics like x86_mmx_packsswb, that
2917 // packs elements of 2 input vectors into half as many bits with saturation.
2918 // Shadow is propagated with the signed variant of the same intrinsic applied
2919 // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
2920 // EltSizeInBits is used only for x86mmx arguments.
2921 void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) {
2922 assert(I.getNumArgOperands() == 2)((void)0);
2923 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2924 IRBuilder<> IRB(&I);
2925 Value *S1 = getShadow(&I, 0);
2926 Value *S2 = getShadow(&I, 1);
2927 assert(isX86_MMX || S1->getType()->isVectorTy())((void)0);
2928
2929 // SExt and ICmpNE below must apply to individual elements of input vectors.
2930 // In case of x86mmx arguments, cast them to appropriate vector types and
2931 // back.
2932 Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType();
2933 if (isX86_MMX) {
2934 S1 = IRB.CreateBitCast(S1, T);
2935 S2 = IRB.CreateBitCast(S2, T);
2936 }
2937 Value *S1_ext = IRB.CreateSExt(
2938 IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T);
2939 Value *S2_ext = IRB.CreateSExt(
2940 IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T);
2941 if (isX86_MMX) {
2942 Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C);
2943 S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy);
2944 S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy);
2945 }
2946
2947 Function *ShadowFn = Intrinsic::getDeclaration(
2948 F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
2949
2950 Value *S =
2951 IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
2952 if (isX86_MMX) S = IRB.CreateBitCast(S, getShadowTy(&I));
2953 setShadow(&I, S);
2954 setOriginForNaryOp(I);
2955 }
2956
2957 // Instrument sum-of-absolute-differences intrinsic.
2958 void handleVectorSadIntrinsic(IntrinsicInst &I) {
2959 const unsigned SignificantBitsPerResultElement = 16;
2960 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2961 Type *ResTy = isX86_MMX ? IntegerType::get(*MS.C, 64) : I.getType();
2962 unsigned ZeroBitsPerResultElement =
2963 ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
2964
2965 IRBuilder<> IRB(&I);
2966 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2967 S = IRB.CreateBitCast(S, ResTy);
2968 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2969 ResTy);
2970 S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
2971 S = IRB.CreateBitCast(S, getShadowTy(&I));
2972 setShadow(&I, S);
2973 setOriginForNaryOp(I);
2974 }
2975
2976 // Instrument multiply-add intrinsic.
2977 void handleVectorPmaddIntrinsic(IntrinsicInst &I,
2978 unsigned EltSizeInBits = 0) {
2979 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2980 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) : I.getType();
2981 IRBuilder<> IRB(&I);
2982 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2983 S = IRB.CreateBitCast(S, ResTy);
2984 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2985 ResTy);
2986 S = IRB.CreateBitCast(S, getShadowTy(&I));
2987 setShadow(&I, S);
2988 setOriginForNaryOp(I);
2989 }
2990
2991 // Instrument compare-packed intrinsic.
2992 // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
2993 // all-ones shadow.
2994 void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
2995 IRBuilder<> IRB(&I);
2996 Type *ResTy = getShadowTy(&I);
2997 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2998 Value *S = IRB.CreateSExt(
2999 IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
3000 setShadow(&I, S);
3001 setOriginForNaryOp(I);
3002 }
3003
3004 // Instrument compare-scalar intrinsic.
3005 // This handles both cmp* intrinsics which return the result in the first
3006 // element of a vector, and comi* which return the result as i32.
3007 void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
3008 IRBuilder<> IRB(&I);
3009 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
3010 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
3011 setShadow(&I, S);
3012 setOriginForNaryOp(I);
3013 }
3014
3015 // Instrument generic vector reduction intrinsics
3016 // by ORing together all their fields.
3017 void handleVectorReduceIntrinsic(IntrinsicInst &I) {
3018 IRBuilder<> IRB(&I);
3019 Value *S = IRB.CreateOrReduce(getShadow(&I, 0));
3020 setShadow(&I, S);
3021 setOrigin(&I, getOrigin(&I, 0));
3022 }
3023
3024 // Instrument vector.reduce.or intrinsic.
3025 // Valid (non-poisoned) set bits in the operand pull low the
3026 // corresponding shadow bits.
3027 void handleVectorReduceOrIntrinsic(IntrinsicInst &I) {
3028 IRBuilder<> IRB(&I);
3029 Value *OperandShadow = getShadow(&I, 0);
3030 Value *OperandUnsetBits = IRB.CreateNot(I.getOperand(0));
3031 Value *OperandUnsetOrPoison = IRB.CreateOr(OperandUnsetBits, OperandShadow);
3032 // Bit N is clean if any field's bit N is 1 and unpoison
3033 Value *OutShadowMask = IRB.CreateAndReduce(OperandUnsetOrPoison);
3034 // Otherwise, it is clean if every field's bit N is unpoison
3035 Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3036 Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3037
3038 setShadow(&I, S);
3039 setOrigin(&I, getOrigin(&I, 0));
3040 }
3041
3042 // Instrument vector.reduce.and intrinsic.
3043 // Valid (non-poisoned) unset bits in the operand pull down the
3044 // corresponding shadow bits.
3045 void handleVectorReduceAndIntrinsic(IntrinsicInst &I) {
3046 IRBuilder<> IRB(&I);
3047 Value *OperandShadow = getShadow(&I, 0);
3048 Value *OperandSetOrPoison = IRB.CreateOr(I.getOperand(0), OperandShadow);
3049 // Bit N is clean if any field's bit N is 0 and unpoison
3050 Value *OutShadowMask = IRB.CreateAndReduce(OperandSetOrPoison);
3051 // Otherwise, it is clean if every field's bit N is unpoison
3052 Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3053 Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3054
3055 setShadow(&I, S);
3056 setOrigin(&I, getOrigin(&I, 0));
3057 }
3058
3059 void handleStmxcsr(IntrinsicInst &I) {
3060 IRBuilder<> IRB(&I);
3061 Value* Addr = I.getArgOperand(0);
3062 Type *Ty = IRB.getInt32Ty();
3063 Value *ShadowPtr =
3064 getShadowOriginPtr(Addr, IRB, Ty, Align(1), /*isStore*/ true).first;
3065
3066 IRB.CreateStore(getCleanShadow(Ty),
3067 IRB.CreatePointerCast(ShadowPtr, Ty->getPointerTo()));
3068
3069 if (ClCheckAccessAddress)
3070 insertShadowCheck(Addr, &I);
3071 }
3072
3073 void handleLdmxcsr(IntrinsicInst &I) {
3074 if (!InsertChecks) return;
3075
3076 IRBuilder<> IRB(&I);
3077 Value *Addr = I.getArgOperand(0);
3078 Type *Ty = IRB.getInt32Ty();
3079 const Align Alignment = Align(1);
3080 Value *ShadowPtr, *OriginPtr;
3081 std::tie(ShadowPtr, OriginPtr) =
3082 getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
3083
3084 if (ClCheckAccessAddress)
3085 insertShadowCheck(Addr, &I);
3086
3087 Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr");
3088 Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr)
3089 : getCleanOrigin();
3090 insertShadowCheck(Shadow, Origin, &I);
3091 }
3092
3093 void handleMaskedStore(IntrinsicInst &I) {
3094 IRBuilder<> IRB(&I);
3095 Value *V = I.getArgOperand(0);
3096 Value *Addr = I.getArgOperand(1);
3097 const Align Alignment(
3098 cast<ConstantInt>(I.getArgOperand(2))->getZExtValue());
3099 Value *Mask = I.getArgOperand(3);
3100 Value *Shadow = getShadow(V);
3101
3102 Value *ShadowPtr;
3103 Value *OriginPtr;
3104 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3105 Addr, IRB, Shadow->getType(), Alignment, /*isStore*/ true);
3106
3107 if (ClCheckAccessAddress) {
3108 insertShadowCheck(Addr, &I);
3109 // Uninitialized mask is kind of like uninitialized address, but not as
3110 // scary.
3111 insertShadowCheck(Mask, &I);
3112 }
3113
3114 IRB.CreateMaskedStore(Shadow, ShadowPtr, Alignment, Mask);
3115
3116 if (MS.TrackOrigins) {
3117 auto &DL = F.getParent()->getDataLayout();
3118 paintOrigin(IRB, getOrigin(V), OriginPtr,
3119 DL.getTypeStoreSize(Shadow->getType()),
3120 std::max(Alignment, kMinOriginAlignment));
3121 }
3122 }
3123
3124 bool handleMaskedLoad(IntrinsicInst &I) {
3125 IRBuilder<> IRB(&I);
3126 Value *Addr = I.getArgOperand(0);
3127 const Align Alignment(
3128 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
3129 Value *Mask = I.getArgOperand(2);
3130 Value *PassThru = I.getArgOperand(3);
3131
3132 Type *ShadowTy = getShadowTy(&I);
3133 Value *ShadowPtr, *OriginPtr;
3134 if (PropagateShadow) {
3135 std::tie(ShadowPtr, OriginPtr) =
3136 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
3137 setShadow(&I, IRB.CreateMaskedLoad(ShadowTy, ShadowPtr, Alignment, Mask,
3138 getShadow(PassThru), "_msmaskedld"));
3139 } else {
3140 setShadow(&I, getCleanShadow(&I));
3141 }
3142
3143 if (ClCheckAccessAddress) {
3144 insertShadowCheck(Addr, &I);
3145 insertShadowCheck(Mask, &I);
3146 }
3147
3148 if (MS.TrackOrigins) {
3149 if (PropagateShadow) {
3150 // Choose between PassThru's and the loaded value's origins.
3151 Value *MaskedPassThruShadow = IRB.CreateAnd(
3152 getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy));
3153
3154 Value *Acc = IRB.CreateExtractElement(
3155 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
3156 for (int i = 1, N = cast<FixedVectorType>(PassThru->getType())
3157 ->getNumElements();
3158 i < N; ++i) {
3159 Value *More = IRB.CreateExtractElement(
3160 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), i));
3161 Acc = IRB.CreateOr(Acc, More);
3162 }
3163
3164 Value *Origin = IRB.CreateSelect(
3165 IRB.CreateICmpNE(Acc, Constant::getNullValue(Acc->getType())),
3166 getOrigin(PassThru), IRB.CreateLoad(MS.OriginTy, OriginPtr));
3167
3168 setOrigin(&I, Origin);
3169 } else {
3170 setOrigin(&I, getCleanOrigin());
3171 }
3172 }
3173 return true;
3174 }
3175
3176 // Instrument BMI / BMI2 intrinsics.
3177 // All of these intrinsics are Z = I(X, Y)
3178 // where the types of all operands and the result match, and are either i32 or i64.
3179 // The following instrumentation happens to work for all of them:
3180 // Sz = I(Sx, Y) | (sext (Sy != 0))
3181 void handleBmiIntrinsic(IntrinsicInst &I) {
3182 IRBuilder<> IRB(&I);
3183 Type *ShadowTy = getShadowTy(&I);
3184
3185 // If any bit of the mask operand is poisoned, then the whole thing is.
3186 Value *SMask = getShadow(&I, 1);
3187 SMask = IRB.CreateSExt(IRB.CreateICmpNE(SMask, getCleanShadow(ShadowTy)),
3188 ShadowTy);
3189 // Apply the same intrinsic to the shadow of the first operand.
3190 Value *S = IRB.CreateCall(I.getCalledFunction(),
3191 {getShadow(&I, 0), I.getOperand(1)});
3192 S = IRB.CreateOr(SMask, S);
3193 setShadow(&I, S);
3194 setOriginForNaryOp(I);
3195 }
3196
3197 SmallVector<int, 8> getPclmulMask(unsigned Width, bool OddElements) {
3198 SmallVector<int, 8> Mask;
3199 for (unsigned X = OddElements ? 1 : 0; X < Width; X += 2) {
3200 Mask.append(2, X);
3201 }
3202 return Mask;
3203 }
3204
3205 // Instrument pclmul intrinsics.
3206 // These intrinsics operate either on odd or on even elements of the input
3207 // vectors, depending on the constant in the 3rd argument, ignoring the rest.
3208 // Replace the unused elements with copies of the used ones, ex:
3209 // (0, 1, 2, 3) -> (0, 0, 2, 2) (even case)
3210 // or
3211 // (0, 1, 2, 3) -> (1, 1, 3, 3) (odd case)
3212 // and then apply the usual shadow combining logic.
3213 void handlePclmulIntrinsic(IntrinsicInst &I) {
3214 IRBuilder<> IRB(&I);
3215 unsigned Width =
3216 cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements();
3217 assert(isa<ConstantInt>(I.getArgOperand(2)) &&((void)0)
3218 "pclmul 3rd operand must be a constant")((void)0);
3219 unsigned Imm = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3220 Value *Shuf0 = IRB.CreateShuffleVector(getShadow(&I, 0),
3221 getPclmulMask(Width, Imm & 0x01));
3222 Value *Shuf1 = IRB.CreateShuffleVector(getShadow(&I, 1),
3223 getPclmulMask(Width, Imm & 0x10));
3224 ShadowAndOriginCombiner SOC(this, IRB);
3225 SOC.Add(Shuf0, getOrigin(&I, 0));
3226 SOC.Add(Shuf1, getOrigin(&I, 1));
3227 SOC.Done(&I);
3228 }
3229
3230 // Instrument _mm_*_sd intrinsics
3231 void handleUnarySdIntrinsic(IntrinsicInst &I) {
3232 IRBuilder<> IRB(&I);
3233 Value *First = getShadow(&I, 0);
3234 Value *Second = getShadow(&I, 1);
3235 // High word of first operand, low word of second
3236 Value *Shadow =
3237 IRB.CreateShuffleVector(First, Second, llvm::makeArrayRef<int>({2, 1}));
3238
3239 setShadow(&I, Shadow);
3240 setOriginForNaryOp(I);
3241 }
3242
3243 void handleBinarySdIntrinsic(IntrinsicInst &I) {
3244 IRBuilder<> IRB(&I);
3245 Value *First = getShadow(&I, 0);
3246 Value *Second = getShadow(&I, 1);
3247 Value *OrShadow = IRB.CreateOr(First, Second);
3248 // High word of first operand, low word of both OR'd together
3249 Value *Shadow = IRB.CreateShuffleVector(First, OrShadow,
3250 llvm::makeArrayRef<int>({2, 1}));
3251
3252 setShadow(&I, Shadow);
3253 setOriginForNaryOp(I);
3254 }
3255
3256 // Instrument abs intrinsic.
3257 // handleUnknownIntrinsic can't handle it because of the last
3258 // is_int_min_poison argument which does not match the result type.
3259 void handleAbsIntrinsic(IntrinsicInst &I) {
3260 assert(I.getType()->isIntOrIntVectorTy())((void)0);
3261 assert(I.getArgOperand(0)->getType() == I.getType())((void)0);
3262
3263 // FIXME: Handle is_int_min_poison.
3264 IRBuilder<> IRB(&I);
3265 setShadow(&I, getShadow(&I, 0));
3266 setOrigin(&I, getOrigin(&I, 0));
3267 }
3268
3269 void visitIntrinsicInst(IntrinsicInst &I) {
3270 switch (I.getIntrinsicID()) {
3271 case Intrinsic::abs:
3272 handleAbsIntrinsic(I);
3273 break;
3274 case Intrinsic::lifetime_start:
3275 handleLifetimeStart(I);
3276 break;
3277 case Intrinsic::launder_invariant_group:
3278 case Intrinsic::strip_invariant_group:
3279 handleInvariantGroup(I);
3280 break;
3281 case Intrinsic::bswap:
3282 handleBswap(I);
3283 break;
3284 case Intrinsic::masked_store:
3285 handleMaskedStore(I);
3286 break;
3287 case Intrinsic::masked_load:
3288 handleMaskedLoad(I);
3289 break;
3290 case Intrinsic::vector_reduce_and:
3291 handleVectorReduceAndIntrinsic(I);
3292 break;
3293 case Intrinsic::vector_reduce_or:
3294 handleVectorReduceOrIntrinsic(I);
3295 break;
3296 case Intrinsic::vector_reduce_add:
3297 case Intrinsic::vector_reduce_xor:
3298 case Intrinsic::vector_reduce_mul:
3299 handleVectorReduceIntrinsic(I);
3300 break;
3301 case Intrinsic::x86_sse_stmxcsr:
3302 handleStmxcsr(I);
3303 break;
3304 case Intrinsic::x86_sse_ldmxcsr:
3305 handleLdmxcsr(I);
3306 break;
3307 case Intrinsic::x86_avx512_vcvtsd2usi64:
3308 case Intrinsic::x86_avx512_vcvtsd2usi32:
3309 case Intrinsic::x86_avx512_vcvtss2usi64:
3310 case Intrinsic::x86_avx512_vcvtss2usi32:
3311 case Intrinsic::x86_avx512_cvttss2usi64:
3312 case Intrinsic::x86_avx512_cvttss2usi:
3313 case Intrinsic::x86_avx512_cvttsd2usi64:
3314 case Intrinsic::x86_avx512_cvttsd2usi:
3315 case Intrinsic::x86_avx512_cvtusi2ss:
3316 case Intrinsic::x86_avx512_cvtusi642sd:
3317 case Intrinsic::x86_avx512_cvtusi642ss:
3318 handleVectorConvertIntrinsic(I, 1, true);
3319 break;
3320 case Intrinsic::x86_sse2_cvtsd2si64:
3321 case Intrinsic::x86_sse2_cvtsd2si:
3322 case Intrinsic::x86_sse2_cvtsd2ss:
3323 case Intrinsic::x86_sse2_cvttsd2si64:
3324 case Intrinsic::x86_sse2_cvttsd2si:
3325 case Intrinsic::x86_sse_cvtss2si64:
3326 case Intrinsic::x86_sse_cvtss2si:
3327 case Intrinsic::x86_sse_cvttss2si64:
3328 case Intrinsic::x86_sse_cvttss2si:
3329 handleVectorConvertIntrinsic(I, 1);
3330 break;
3331 case Intrinsic::x86_sse_cvtps2pi:
3332 case Intrinsic::x86_sse_cvttps2pi:
3333 handleVectorConvertIntrinsic(I, 2);
3334 break;
3335
3336 case Intrinsic::x86_avx512_psll_w_512:
3337 case Intrinsic::x86_avx512_psll_d_512:
3338 case Intrinsic::x86_avx512_psll_q_512:
3339 case Intrinsic::x86_avx512_pslli_w_512:
3340 case Intrinsic::x86_avx512_pslli_d_512:
3341 case Intrinsic::x86_avx512_pslli_q_512:
3342 case Intrinsic::x86_avx512_psrl_w_512:
3343 case Intrinsic::x86_avx512_psrl_d_512:
3344 case Intrinsic::x86_avx512_psrl_q_512:
3345 case Intrinsic::x86_avx512_psra_w_512:
3346 case Intrinsic::x86_avx512_psra_d_512:
3347 case Intrinsic::x86_avx512_psra_q_512:
3348 case Intrinsic::x86_avx512_psrli_w_512:
3349 case Intrinsic::x86_avx512_psrli_d_512:
3350 case Intrinsic::x86_avx512_psrli_q_512:
3351 case Intrinsic::x86_avx512_psrai_w_512:
3352 case Intrinsic::x86_avx512_psrai_d_512:
3353 case Intrinsic::x86_avx512_psrai_q_512:
3354 case Intrinsic::x86_avx512_psra_q_256:
3355 case Intrinsic::x86_avx512_psra_q_128:
3356 case Intrinsic::x86_avx512_psrai_q_256:
3357 case Intrinsic::x86_avx512_psrai_q_128:
3358 case Intrinsic::x86_avx2_psll_w:
3359 case Intrinsic::x86_avx2_psll_d:
3360 case Intrinsic::x86_avx2_psll_q:
3361 case Intrinsic::x86_avx2_pslli_w:
3362 case Intrinsic::x86_avx2_pslli_d:
3363 case Intrinsic::x86_avx2_pslli_q:
3364 case Intrinsic::x86_avx2_psrl_w:
3365 case Intrinsic::x86_avx2_psrl_d:
3366 case Intrinsic::x86_avx2_psrl_q:
3367 case Intrinsic::x86_avx2_psra_w:
3368 case Intrinsic::x86_avx2_psra_d:
3369 case Intrinsic::x86_avx2_psrli_w:
3370 case Intrinsic::x86_avx2_psrli_d:
3371 case Intrinsic::x86_avx2_psrli_q:
3372 case Intrinsic::x86_avx2_psrai_w:
3373 case Intrinsic::x86_avx2_psrai_d:
3374 case Intrinsic::x86_sse2_psll_w:
3375 case Intrinsic::x86_sse2_psll_d:
3376 case Intrinsic::x86_sse2_psll_q:
3377 case Intrinsic::x86_sse2_pslli_w:
3378 case Intrinsic::x86_sse2_pslli_d:
3379 case Intrinsic::x86_sse2_pslli_q:
3380 case Intrinsic::x86_sse2_psrl_w:
3381 case Intrinsic::x86_sse2_psrl_d:
3382 case Intrinsic::x86_sse2_psrl_q:
3383 case Intrinsic::x86_sse2_psra_w:
3384 case Intrinsic::x86_sse2_psra_d:
3385 case Intrinsic::x86_sse2_psrli_w:
3386 case Intrinsic::x86_sse2_psrli_d:
3387 case Intrinsic::x86_sse2_psrli_q:
3388 case Intrinsic::x86_sse2_psrai_w:
3389 case Intrinsic::x86_sse2_psrai_d:
3390 case Intrinsic::x86_mmx_psll_w:
3391 case Intrinsic::x86_mmx_psll_d:
3392 case Intrinsic::x86_mmx_psll_q:
3393 case Intrinsic::x86_mmx_pslli_w:
3394 case Intrinsic::x86_mmx_pslli_d:
3395 case Intrinsic::x86_mmx_pslli_q:
3396 case Intrinsic::x86_mmx_psrl_w:
3397 case Intrinsic::x86_mmx_psrl_d:
3398 case Intrinsic::x86_mmx_psrl_q:
3399 case Intrinsic::x86_mmx_psra_w:
3400 case Intrinsic::x86_mmx_psra_d:
3401 case Intrinsic::x86_mmx_psrli_w:
3402 case Intrinsic::x86_mmx_psrli_d:
3403 case Intrinsic::x86_mmx_psrli_q:
3404 case Intrinsic::x86_mmx_psrai_w:
3405 case Intrinsic::x86_mmx_psrai_d:
3406 handleVectorShiftIntrinsic(I, /* Variable */ false);
3407 break;
3408 case Intrinsic::x86_avx2_psllv_d:
3409 case Intrinsic::x86_avx2_psllv_d_256:
3410 case Intrinsic::x86_avx512_psllv_d_512:
3411 case Intrinsic::x86_avx2_psllv_q:
3412 case Intrinsic::x86_avx2_psllv_q_256:
3413 case Intrinsic::x86_avx512_psllv_q_512:
3414 case Intrinsic::x86_avx2_psrlv_d:
3415 case Intrinsic::x86_avx2_psrlv_d_256:
3416 case Intrinsic::x86_avx512_psrlv_d_512:
3417 case Intrinsic::x86_avx2_psrlv_q:
3418 case Intrinsic::x86_avx2_psrlv_q_256:
3419 case Intrinsic::x86_avx512_psrlv_q_512:
3420 case Intrinsic::x86_avx2_psrav_d:
3421 case Intrinsic::x86_avx2_psrav_d_256:
3422 case Intrinsic::x86_avx512_psrav_d_512:
3423 case Intrinsic::x86_avx512_psrav_q_128:
3424 case Intrinsic::x86_avx512_psrav_q_256:
3425 case Intrinsic::x86_avx512_psrav_q_512:
3426 handleVectorShiftIntrinsic(I, /* Variable */ true);
3427 break;
3428
3429 case Intrinsic::x86_sse2_packsswb_128:
3430 case Intrinsic::x86_sse2_packssdw_128:
3431 case Intrinsic::x86_sse2_packuswb_128:
3432 case Intrinsic::x86_sse41_packusdw:
3433 case Intrinsic::x86_avx2_packsswb:
3434 case Intrinsic::x86_avx2_packssdw:
3435 case Intrinsic::x86_avx2_packuswb:
3436 case Intrinsic::x86_avx2_packusdw:
3437 handleVectorPackIntrinsic(I);
3438 break;
3439
3440 case Intrinsic::x86_mmx_packsswb:
3441 case Intrinsic::x86_mmx_packuswb:
3442 handleVectorPackIntrinsic(I, 16);
3443 break;
3444
3445 case Intrinsic::x86_mmx_packssdw:
3446 handleVectorPackIntrinsic(I, 32);
3447 break;
3448
3449 case Intrinsic::x86_mmx_psad_bw:
3450 case Intrinsic::x86_sse2_psad_bw:
3451 case Intrinsic::x86_avx2_psad_bw:
3452 handleVectorSadIntrinsic(I);
3453 break;
3454
3455 case Intrinsic::x86_sse2_pmadd_wd:
3456 case Intrinsic::x86_avx2_pmadd_wd:
3457 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3458 case Intrinsic::x86_avx2_pmadd_ub_sw:
3459 handleVectorPmaddIntrinsic(I);
3460 break;
3461
3462 case Intrinsic::x86_ssse3_pmadd_ub_sw:
3463 handleVectorPmaddIntrinsic(I, 8);
3464 break;
3465
3466 case Intrinsic::x86_mmx_pmadd_wd:
3467 handleVectorPmaddIntrinsic(I, 16);
3468 break;
3469
3470 case Intrinsic::x86_sse_cmp_ss:
3471 case Intrinsic::x86_sse2_cmp_sd:
3472 case Intrinsic::x86_sse_comieq_ss:
3473 case Intrinsic::x86_sse_comilt_ss:
3474 case Intrinsic::x86_sse_comile_ss:
3475 case Intrinsic::x86_sse_comigt_ss:
3476 case Intrinsic::x86_sse_comige_ss:
3477 case Intrinsic::x86_sse_comineq_ss:
3478 case Intrinsic::x86_sse_ucomieq_ss:
3479 case Intrinsic::x86_sse_ucomilt_ss:
3480 case Intrinsic::x86_sse_ucomile_ss:
3481 case Intrinsic::x86_sse_ucomigt_ss:
3482 case Intrinsic::x86_sse_ucomige_ss:
3483 case Intrinsic::x86_sse_ucomineq_ss:
3484 case Intrinsic::x86_sse2_comieq_sd:
3485 case Intrinsic::x86_sse2_comilt_sd:
3486 case Intrinsic::x86_sse2_comile_sd:
3487 case Intrinsic::x86_sse2_comigt_sd:
3488 case Intrinsic::x86_sse2_comige_sd:
3489 case Intrinsic::x86_sse2_comineq_sd:
3490 case Intrinsic::x86_sse2_ucomieq_sd:
3491 case Intrinsic::x86_sse2_ucomilt_sd:
3492 case Intrinsic::x86_sse2_ucomile_sd:
3493 case Intrinsic::x86_sse2_ucomigt_sd:
3494 case Intrinsic::x86_sse2_ucomige_sd:
3495 case Intrinsic::x86_sse2_ucomineq_sd:
3496 handleVectorCompareScalarIntrinsic(I);
3497 break;
3498
3499 case Intrinsic::x86_sse_cmp_ps:
3500 case Intrinsic::x86_sse2_cmp_pd:
3501 // FIXME: For x86_avx_cmp_pd_256 and x86_avx_cmp_ps_256 this function
3502 // generates reasonably looking IR that fails in the backend with "Do not
3503 // know how to split the result of this operator!".
3504 handleVectorComparePackedIntrinsic(I);
3505 break;
3506
3507 case Intrinsic::x86_bmi_bextr_32:
3508 case Intrinsic::x86_bmi_bextr_64:
3509 case Intrinsic::x86_bmi_bzhi_32:
3510 case Intrinsic::x86_bmi_bzhi_64:
3511 case Intrinsic::x86_bmi_pdep_32:
3512 case Intrinsic::x86_bmi_pdep_64:
3513 case Intrinsic::x86_bmi_pext_32:
3514 case Intrinsic::x86_bmi_pext_64:
3515 handleBmiIntrinsic(I);
3516 break;
3517
3518 case Intrinsic::x86_pclmulqdq:
3519 case Intrinsic::x86_pclmulqdq_256:
3520 case Intrinsic::x86_pclmulqdq_512:
3521 handlePclmulIntrinsic(I);
3522 break;
3523
3524 case Intrinsic::x86_sse41_round_sd:
3525 handleUnarySdIntrinsic(I);
3526 break;
3527 case Intrinsic::x86_sse2_max_sd:
3528 case Intrinsic::x86_sse2_min_sd:
3529 handleBinarySdIntrinsic(I);
3530 break;
3531
3532 case Intrinsic::fshl:
3533 case Intrinsic::fshr:
3534 handleFunnelShift(I);
3535 break;
3536
3537 case Intrinsic::is_constant:
3538 // The result of llvm.is.constant() is always defined.
3539 setShadow(&I, getCleanShadow(&I));
3540 setOrigin(&I, getCleanOrigin());
3541 break;
3542
3543 default:
3544 if (!handleUnknownIntrinsic(I))
3545 visitInstruction(I);
3546 break;
3547 }
3548 }
3549
3550 void visitLibAtomicLoad(CallBase &CB) {
3551 // Since we use getNextNode here, we can't have CB terminate the BB.
3552 assert(isa<CallInst>(CB))((void)0);
3553
3554 IRBuilder<> IRB(&CB);
3555 Value *Size = CB.getArgOperand(0);
3556 Value *SrcPtr = CB.getArgOperand(1);
3557 Value *DstPtr = CB.getArgOperand(2);
3558 Value *Ordering = CB.getArgOperand(3);
3559 // Convert the call to have at least Acquire ordering to make sure
3560 // the shadow operations aren't reordered before it.
3561 Value *NewOrdering =
3562 IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering);
3563 CB.setArgOperand(3, NewOrdering);
3564
3565 IRBuilder<> NextIRB(CB.getNextNode());
3566 NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
3567
3568 Value *SrcShadowPtr, *SrcOriginPtr;
3569 std::tie(SrcShadowPtr, SrcOriginPtr) =
3570 getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
3571 /*isStore*/ false);
3572 Value *DstShadowPtr =
3573 getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
3574 /*isStore*/ true)
3575 .first;
3576
3577 NextIRB.CreateMemCpy(DstShadowPtr, Align(1), SrcShadowPtr, Align(1), Size);
3578 if (MS.TrackOrigins) {
3579 Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
3580 kMinOriginAlignment);
3581 Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
3582 NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
3583 }
3584 }
3585
3586 void visitLibAtomicStore(CallBase &CB) {
3587 IRBuilder<> IRB(&CB);
3588 Value *Size = CB.getArgOperand(0);
3589 Value *DstPtr = CB.getArgOperand(2);
3590 Value *Ordering = CB.getArgOperand(3);
3591 // Convert the call to have at least Release ordering to make sure
3592 // the shadow operations aren't reordered after it.
3593 Value *NewOrdering =
3594 IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering);
3595 CB.setArgOperand(3, NewOrdering);
3596
3597 Value *DstShadowPtr =
3598 getShadowOriginPtr(DstPtr, IRB, IRB.getInt8Ty(), Align(1),
3599 /*isStore*/ true)
3600 .first;
3601
3602 // Atomic store always paints clean shadow/origin. See file header.
3603 IRB.CreateMemSet(DstShadowPtr, getCleanShadow(IRB.getInt8Ty()), Size,
3604 Align(1));
3605 }
3606
3607 void visitCallBase(CallBase &CB) {
3608 assert(!CB.getMetadata("nosanitize"))((void)0);
3609 if (CB.isInlineAsm()) {
3610 // For inline asm (either a call to asm function, or callbr instruction),
3611 // do the usual thing: check argument shadow and mark all outputs as
3612 // clean. Note that any side effects of the inline asm that are not
3613 // immediately visible in its constraints are not handled.
3614 if (ClHandleAsmConservative && MS.CompileKernel)
3615 visitAsmInstruction(CB);
3616 else
3617 visitInstruction(CB);
3618 return;
3619 }
3620 LibFunc LF;
3621 if (TLI->getLibFunc(CB, LF)) {
3622 // libatomic.a functions need to have special handling because there isn't
3623 // a good way to intercept them or compile the library with
3624 // instrumentation.
3625 switch (LF) {
3626 case LibFunc_atomic_load:
3627 if (!isa<CallInst>(CB)) {
3628 llvm::errs() << "MSAN -- cannot instrument invoke of libatomic load."
3629 "Ignoring!\n";
3630 break;
3631 }
3632 visitLibAtomicLoad(CB);
3633 return;
3634 case LibFunc_atomic_store:
3635 visitLibAtomicStore(CB);
3636 return;
3637 default:
3638 break;
3639 }
3640 }
3641
3642 if (auto *Call = dyn_cast<CallInst>(&CB)) {
3643 assert(!isa<IntrinsicInst>(Call) && "intrinsics are handled elsewhere")((void)0);
3644
3645 // We are going to insert code that relies on the fact that the callee
3646 // will become a non-readonly function after it is instrumented by us. To
3647 // prevent this code from being optimized out, mark that function
3648 // non-readonly in advance.
3649 AttrBuilder B;
3650 B.addAttribute(Attribute::ReadOnly)
3651 .addAttribute(Attribute::ReadNone)
3652 .addAttribute(Attribute::WriteOnly)
3653 .addAttribute(Attribute::ArgMemOnly)
3654 .addAttribute(Attribute::Speculatable);
3655
3656 Call->removeAttributes(AttributeList::FunctionIndex, B);
3657 if (Function *Func = Call->getCalledFunction()) {
3658 Func->removeAttributes(AttributeList::FunctionIndex, B);
3659 }
3660
3661 maybeMarkSanitizerLibraryCallNoBuiltin(Call, TLI);
3662 }
3663 IRBuilder<> IRB(&CB);
3664 bool MayCheckCall = ClEagerChecks;
3665 if (Function *Func = CB.getCalledFunction()) {
3666 // __sanitizer_unaligned_{load,store} functions may be called by users
3667 // and always expects shadows in the TLS. So don't check them.
3668 MayCheckCall &= !Func->getName().startswith("__sanitizer_unaligned_");
3669 }
3670
3671 unsigned ArgOffset = 0;
3672 LLVM_DEBUG(dbgs() << " CallSite: " << CB << "\n")do { } while (false);
3673 for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
3674 ++ArgIt) {
3675 Value *A = *ArgIt;
3676 unsigned i = ArgIt - CB.arg_begin();
3677 if (!A->getType()->isSized()) {
3678 LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << CB << "\n")do { } while (false);
3679 continue;
3680 }
3681 unsigned Size = 0;
3682 Value *Store = nullptr;
3683 // Compute the Shadow for arg even if it is ByVal, because
3684 // in that case getShadow() will copy the actual arg shadow to
3685 // __msan_param_tls.
3686 Value *ArgShadow = getShadow(A);
3687 Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset);
3688 LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *Ado { } while (false)
3689 << " Shadow: " << *ArgShadow << "\n")do { } while (false);
3690 bool ArgIsInitialized = false;
3691 const DataLayout &DL = F.getParent()->getDataLayout();
3692
3693 bool ByVal = CB.paramHasAttr(i, Attribute::ByVal);
3694 bool NoUndef = CB.paramHasAttr(i, Attribute::NoUndef);
3695 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
3696
3697 if (EagerCheck) {
3698 insertShadowCheck(A, &CB);
3699 continue;
3700 }
3701 if (ByVal) {
3702 // ByVal requires some special handling as it's too big for a single
3703 // load
3704 assert(A->getType()->isPointerTy() &&((void)0)
3705 "ByVal argument is not a pointer!")((void)0);
3706 Size = DL.getTypeAllocSize(CB.getParamByValType(i));
3707 if (ArgOffset + Size > kParamTLSSize) break;
3708 const MaybeAlign ParamAlignment(CB.getParamAlign(i));
3709 MaybeAlign Alignment = llvm::None;
3710 if (ParamAlignment)
3711 Alignment = std::min(*ParamAlignment, kShadowTLSAlignment);
3712 Value *AShadowPtr =
3713 getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment,
3714 /*isStore*/ false)
3715 .first;
3716
3717 Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr,
3718 Alignment, Size);
3719 // TODO(glider): need to copy origins.
3720 } else {
3721 // Any other parameters mean we need bit-grained tracking of uninit data
3722 Size = DL.getTypeAllocSize(A->getType());
3723 if (ArgOffset + Size > kParamTLSSize) break;
3724 Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
3725 kShadowTLSAlignment);
3726 Constant *Cst = dyn_cast<Constant>(ArgShadow);
3727 if (Cst && Cst->isNullValue()) ArgIsInitialized = true;
3728 }
3729 if (MS.TrackOrigins && !ArgIsInitialized)
3730 IRB.CreateStore(getOrigin(A),
3731 getOriginPtrForArgument(A, IRB, ArgOffset));
3732 (void)Store;
3733 assert(Size != 0 && Store != nullptr)((void)0);
3734 LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n")do { } while (false);
3735 ArgOffset += alignTo(Size, kShadowTLSAlignment);
3736 }
3737 LLVM_DEBUG(dbgs() << " done with call args\n")do { } while (false);
3738
3739 FunctionType *FT = CB.getFunctionType();
3740 if (FT->isVarArg()) {
3741 VAHelper->visitCallBase(CB, IRB);
3742 }
3743
3744 // Now, get the shadow for the RetVal.
3745 if (!CB.getType()->isSized())
3746 return;
3747 // Don't emit the epilogue for musttail call returns.
3748 if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
3749 return;
3750
3751 if (MayCheckCall && CB.hasRetAttr(Attribute::NoUndef)) {
3752 setShadow(&CB, getCleanShadow(&CB));
3753 setOrigin(&CB, getCleanOrigin());
3754 return;
3755 }
3756
3757 IRBuilder<> IRBBefore(&CB);
3758 // Until we have full dynamic coverage, make sure the retval shadow is 0.
3759 Value *Base = getShadowPtrForRetval(&CB, IRBBefore);
3760 IRBBefore.CreateAlignedStore(getCleanShadow(&CB), Base,
3761 kShadowTLSAlignment);
3762 BasicBlock::iterator NextInsn;
3763 if (isa<CallInst>(CB)) {
3764 NextInsn = ++CB.getIterator();
3765 assert(NextInsn != CB.getParent()->end())((void)0);
3766 } else {
3767 BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
3768 if (!NormalDest->getSinglePredecessor()) {
3769 // FIXME: this case is tricky, so we are just conservative here.
3770 // Perhaps we need to split the edge between this BB and NormalDest,
3771 // but a naive attempt to use SplitEdge leads to a crash.
3772 setShadow(&CB, getCleanShadow(&CB));
3773 setOrigin(&CB, getCleanOrigin());
3774 return;
3775 }
3776 // FIXME: NextInsn is likely in a basic block that has not been visited yet.
3777 // Anything inserted there will be instrumented by MSan later!
3778 NextInsn = NormalDest->getFirstInsertionPt();
3779 assert(NextInsn != NormalDest->end() &&((void)0)
3780 "Could not find insertion point for retval shadow load")((void)0);
3781 }
3782 IRBuilder<> IRBAfter(&*NextInsn);
3783 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
3784 getShadowTy(&CB), getShadowPtrForRetval(&CB, IRBAfter),
3785 kShadowTLSAlignment, "_msret");
3786 setShadow(&CB, RetvalShadow);
3787 if (MS.TrackOrigins)
3788 setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy,
3789 getOriginPtrForRetval(IRBAfter)));
3790 }
3791
3792 bool isAMustTailRetVal(Value *RetVal) {
3793 if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
3794 RetVal = I->getOperand(0);
3795 }
3796 if (auto *I = dyn_cast<CallInst>(RetVal)) {
3797 return I->isMustTailCall();
3798 }
3799 return false;
3800 }
3801
3802 void visitReturnInst(ReturnInst &I) {
3803 IRBuilder<> IRB(&I);
3804 Value *RetVal = I.getReturnValue();
3805 if (!RetVal) return;
3806 // Don't emit the epilogue for musttail call returns.
3807 if (isAMustTailRetVal(RetVal)) return;
3808 Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB);
3809 bool HasNoUndef =
3810 F.hasAttribute(AttributeList::ReturnIndex, Attribute::NoUndef);
3811 bool StoreShadow = !(ClEagerChecks && HasNoUndef);
3812 // FIXME: Consider using SpecialCaseList to specify a list of functions that
3813 // must always return fully initialized values. For now, we hardcode "main".
3814 bool EagerCheck = (ClEagerChecks && HasNoUndef) || (F.getName() == "main");
3815
3816 Value *Shadow = getShadow(RetVal);
3817 bool StoreOrigin = true;
3818 if (EagerCheck) {
3819 insertShadowCheck(RetVal, &I);
3820 Shadow = getCleanShadow(RetVal);
3821 StoreOrigin = false;
3822 }
3823
3824 // The caller may still expect information passed over TLS if we pass our
3825 // check
3826 if (StoreShadow) {
3827 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
3828 if (MS.TrackOrigins && StoreOrigin)
3829 IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB));
3830 }
3831 }
3832
3833 void visitPHINode(PHINode &I) {
3834 IRBuilder<> IRB(&I);
3835 if (!PropagateShadow) {
3836 setShadow(&I, getCleanShadow(&I));
3837 setOrigin(&I, getCleanOrigin());
3838 return;
3839 }
3840
3841 ShadowPHINodes.push_back(&I);
3842 setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
3843 "_msphi_s"));
3844 if (MS.TrackOrigins)
3845 setOrigin(&I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(),
3846 "_msphi_o"));
3847 }
3848
3849 Value *getLocalVarDescription(AllocaInst &I) {
3850 SmallString<2048> StackDescriptionStorage;
3851 raw_svector_ostream StackDescription(StackDescriptionStorage);
3852 // We create a string with a description of the stack allocation and
3853 // pass it into __msan_set_alloca_origin.
3854 // It will be printed by the run-time if stack-originated UMR is found.
3855 // The first 4 bytes of the string are set to '----' and will be replaced
3856 // by __msan_va_arg_overflow_size_tls at the first call.
3857 StackDescription << "----" << I.getName() << "@" << F.getName();
3858 return createPrivateNonConstGlobalForString(*F.getParent(),
3859 StackDescription.str());
3860 }
3861
3862 void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
3863 if (PoisonStack && ClPoisonStackWithCall) {
3864 IRB.CreateCall(MS.MsanPoisonStackFn,
3865 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3866 } else {
3867 Value *ShadowBase, *OriginBase;
3868 std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
3869 &I, IRB, IRB.getInt8Ty(), Align(1), /*isStore*/ true);
3870
3871 Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
3872 IRB.CreateMemSet(ShadowBase, PoisonValue, Len,
3873 MaybeAlign(I.getAlignment()));
3874 }
3875
3876 if (PoisonStack && MS.TrackOrigins) {
3877 Value *Descr = getLocalVarDescription(I);
3878 IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn,
3879 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3880 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()),
3881 IRB.CreatePointerCast(&F, MS.IntptrTy)});
3882 }
3883 }
3884
3885 void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
3886 Value *Descr = getLocalVarDescription(I);
3887 if (PoisonStack) {
3888 IRB.CreateCall(MS.MsanPoisonAllocaFn,
3889 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3890 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())});
3891 } else {
3892 IRB.CreateCall(MS.MsanUnpoisonAllocaFn,
3893 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3894 }
3895 }
3896
3897 void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) {
3898 if (!InsPoint)
3899 InsPoint = &I;
3900 IRBuilder<> IRB(InsPoint->getNextNode());
3901 const DataLayout &DL = F.getParent()->getDataLayout();
3902 uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType());
3903 Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize);
3904 if (I.isArrayAllocation())
3905 Len = IRB.CreateMul(Len, I.getArraySize());
3906
3907 if (MS.CompileKernel)
3908 poisonAllocaKmsan(I, IRB, Len);
3909 else
3910 poisonAllocaUserspace(I, IRB, Len);
3911 }
3912
3913 void visitAllocaInst(AllocaInst &I) {
3914 setShadow(&I, getCleanShadow(&I));
3915 setOrigin(&I, getCleanOrigin());
3916 // We'll get to this alloca later unless it's poisoned at the corresponding
3917 // llvm.lifetime.start.
3918 AllocaSet.insert(&I);
3919 }
3920
3921 void visitSelectInst(SelectInst& I) {
3922 IRBuilder<> IRB(&I);
3923 // a = select b, c, d
3924 Value *B = I.getCondition();
3925 Value *C = I.getTrueValue();
3926 Value *D = I.getFalseValue();
3927 Value *Sb = getShadow(B);
3928 Value *Sc = getShadow(C);
3929 Value *Sd = getShadow(D);
3930
3931 // Result shadow if condition shadow is 0.
3932 Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
3933 Value *Sa1;
3934 if (I.getType()->isAggregateType()) {
3935 // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
3936 // an extra "select". This results in much more compact IR.
3937 // Sa = select Sb, poisoned, (select b, Sc, Sd)
3938 Sa1 = getPoisonedShadow(getShadowTy(I.getType()));
3939 } else {
3940 // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
3941 // If Sb (condition is poisoned), look for bits in c and d that are equal
3942 // and both unpoisoned.
3943 // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
3944
3945 // Cast arguments to shadow-compatible type.
3946 C = CreateAppToShadowCast(IRB, C);
3947 D = CreateAppToShadowCast(IRB, D);
3948
3949 // Result shadow if condition shadow is 1.
3950 Sa1 = IRB.CreateOr({IRB.CreateXor(C, D), Sc, Sd});
3951 }
3952 Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
3953 setShadow(&I, Sa);
3954 if (MS.TrackOrigins) {
3955 // Origins are always i32, so any vector conditions must be flattened.
3956 // FIXME: consider tracking vector origins for app vectors?
3957 if (B->getType()->isVectorTy()) {
3958 Type *FlatTy = getShadowTyNoVec(B->getType());
3959 B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy),
3960 ConstantInt::getNullValue(FlatTy));
3961 Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy),
3962 ConstantInt::getNullValue(FlatTy));
3963 }
3964 // a = select b, c, d
3965 // Oa = Sb ? Ob : (b ? Oc : Od)
3966 setOrigin(
3967 &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()),
3968 IRB.CreateSelect(B, getOrigin(I.getTrueValue()),
3969 getOrigin(I.getFalseValue()))));
3970 }
3971 }
3972
3973 void visitLandingPadInst(LandingPadInst &I) {
3974 // Do nothing.
3975 // See https://github.com/google/sanitizers/issues/504
3976 setShadow(&I, getCleanShadow(&I));
3977 setOrigin(&I, getCleanOrigin());
3978 }
3979
3980 void visitCatchSwitchInst(CatchSwitchInst &I) {
3981 setShadow(&I, getCleanShadow(&I));
3982 setOrigin(&I, getCleanOrigin());
3983 }
3984
3985 void visitFuncletPadInst(FuncletPadInst &I) {
3986 setShadow(&I, getCleanShadow(&I));
3987 setOrigin(&I, getCleanOrigin());
3988 }
3989
3990 void visitGetElementPtrInst(GetElementPtrInst &I) {
3991 handleShadowOr(I);
3992 }
3993
3994 void visitExtractValueInst(ExtractValueInst &I) {
3995 IRBuilder<> IRB(&I);
3996 Value *Agg = I.getAggregateOperand();
3997 LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n")do { } while (false);
3998 Value *AggShadow = getShadow(Agg);
3999 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n")do { } while (false);
4000 Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
4001 LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n")do { } while (false);
4002 setShadow(&I, ResShadow);
4003 setOriginForNaryOp(I);
4004 }
4005
4006 void visitInsertValueInst(InsertValueInst &I) {
4007 IRBuilder<> IRB(&I);
4008 LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n")do { } while (false);
4009 Value *AggShadow = getShadow(I.getAggregateOperand());
4010 Value *InsShadow = getShadow(I.getInsertedValueOperand());
4011 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n")do { } while (false);
4012 LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n")do { } while (false);
4013 Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
4014 LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n")do { } while (false);
4015 setShadow(&I, Res);
4016 setOriginForNaryOp(I);
4017 }
4018
4019 void dumpInst(Instruction &I) {
4020 if (CallInst *CI = dyn_cast<CallInst>(&I)) {
4021 errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n";
4022 } else {
4023 errs() << "ZZZ " << I.getOpcodeName() << "\n";
4024 }
4025 errs() << "QQQ " << I << "\n";
4026 }
4027
4028 void visitResumeInst(ResumeInst &I) {
4029 LLVM_DEBUG(dbgs() << "Resume: " << I << "\n")do { } while (false);
4030 // Nothing to do here.
4031 }
4032
4033 void visitCleanupReturnInst(CleanupReturnInst &CRI) {
4034 LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n")do { } while (false);
4035 // Nothing to do here.
4036 }
4037
4038 void visitCatchReturnInst(CatchReturnInst &CRI) {
4039 LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n")do { } while (false);
4040 // Nothing to do here.
4041 }
4042
4043 void instrumentAsmArgument(Value *Operand, Instruction &I, IRBuilder<> &IRB,
4044 const DataLayout &DL, bool isOutput) {
4045 // For each assembly argument, we check its value for being initialized.
4046 // If the argument is a pointer, we assume it points to a single element
4047 // of the corresponding type (or to a 8-byte word, if the type is unsized).
4048 // Each such pointer is instrumented with a call to the runtime library.
4049 Type *OpType = Operand->getType();
4050 // Check the operand value itself.
4051 insertShadowCheck(Operand, &I);
4052 if (!OpType->isPointerTy() || !isOutput) {
4053 assert(!isOutput)((void)0);
4054 return;
4055 }
4056 Type *ElType = OpType->getPointerElementType();
4057 if (!ElType->isSized())
4058 return;
4059 int Size = DL.getTypeStoreSize(ElType);
4060 Value *Ptr = IRB.CreatePointerCast(Operand, IRB.getInt8PtrTy());
4061 Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
4062 IRB.CreateCall(MS.MsanInstrumentAsmStoreFn, {Ptr, SizeVal});
4063 }
4064
4065 /// Get the number of output arguments returned by pointers.
4066 int getNumOutputArgs(InlineAsm *IA, CallBase *CB) {
4067 int NumRetOutputs = 0;
4068 int NumOutputs = 0;
4069 Type *RetTy = cast<Value>(CB)->getType();
4070 if (!RetTy->isVoidTy()) {
4071 // Register outputs are returned via the CallInst return value.
4072 auto *ST = dyn_cast<StructType>(RetTy);
4073 if (ST)
4074 NumRetOutputs = ST->getNumElements();
4075 else
4076 NumRetOutputs = 1;
4077 }
4078 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
4079 for (const InlineAsm::ConstraintInfo &Info : Constraints) {
4080 switch (Info.Type) {
4081 case InlineAsm::isOutput:
4082 NumOutputs++;
4083 break;
4084 default:
4085 break;
4086 }
4087 }
4088 return NumOutputs - NumRetOutputs;
4089 }
4090
4091 void visitAsmInstruction(Instruction &I) {
4092 // Conservative inline assembly handling: check for poisoned shadow of
4093 // asm() arguments, then unpoison the result and all the memory locations
4094 // pointed to by those arguments.
4095 // An inline asm() statement in C++ contains lists of input and output
4096 // arguments used by the assembly code. These are mapped to operands of the
4097 // CallInst as follows:
4098 // - nR register outputs ("=r) are returned by value in a single structure
4099 // (SSA value of the CallInst);
4100 // - nO other outputs ("=m" and others) are returned by pointer as first
4101 // nO operands of the CallInst;
4102 // - nI inputs ("r", "m" and others) are passed to CallInst as the
4103 // remaining nI operands.
4104 // The total number of asm() arguments in the source is nR+nO+nI, and the
4105 // corresponding CallInst has nO+nI+1 operands (the last operand is the
4106 // function to be called).
4107 const DataLayout &DL = F.getParent()->getDataLayout();
4108 CallBase *CB = cast<CallBase>(&I);
4109 IRBuilder<> IRB(&I);
4110 InlineAsm *IA = cast<InlineAsm>(CB->getCalledOperand());
4111 int OutputArgs = getNumOutputArgs(IA, CB);
4112 // The last operand of a CallInst is the function itself.
4113 int NumOperands = CB->getNumOperands() - 1;
4114
4115 // Check input arguments. Doing so before unpoisoning output arguments, so
4116 // that we won't overwrite uninit values before checking them.
4117 for (int i = OutputArgs; i < NumOperands; i++) {
4118 Value *Operand = CB->getOperand(i);
4119 instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ false);
4120 }
4121 // Unpoison output arguments. This must happen before the actual InlineAsm
4122 // call, so that the shadow for memory published in the asm() statement
4123 // remains valid.
4124 for (int i = 0; i < OutputArgs; i++) {
4125 Value *Operand = CB->getOperand(i);
4126 instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ true);
4127 }
4128
4129 setShadow(&I, getCleanShadow(&I));
4130 setOrigin(&I, getCleanOrigin());
4131 }
4132
4133 void visitFreezeInst(FreezeInst &I) {
4134 // Freeze always returns a fully defined value.
4135 setShadow(&I, getCleanShadow(&I));
4136 setOrigin(&I, getCleanOrigin());
4137 }
4138
4139 void visitInstruction(Instruction &I) {
4140 // Everything else: stop propagating and check for poisoned shadow.
4141 if (ClDumpStrictInstructions)
4142 dumpInst(I);
4143 LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n")do { } while (false);
4144 for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
4145 Value *Operand = I.getOperand(i);
4146 if (Operand->getType()->isSized())
4147 insertShadowCheck(Operand, &I);
4148 }
4149 setShadow(&I, getCleanShadow(&I));
4150 setOrigin(&I, getCleanOrigin());
4151 }
4152};
4153
4154/// AMD64-specific implementation of VarArgHelper.
4155struct VarArgAMD64Helper : public VarArgHelper {
4156 // An unfortunate workaround for asymmetric lowering of va_arg stuff.
4157 // See a comment in visitCallBase for more details.
4158 static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
4159 static const unsigned AMD64FpEndOffsetSSE = 176;
4160 // If SSE is disabled, fp_offset in va_list is zero.
4161 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
4162
4163 unsigned AMD64FpEndOffset;
4164 Function &F;
4165 MemorySanitizer &MS;
4166 MemorySanitizerVisitor &MSV;
4167 Value *VAArgTLSCopy = nullptr;
4168 Value *VAArgTLSOriginCopy = nullptr;
4169 Value *VAArgOverflowSize = nullptr;
4170
4171 SmallVector<CallInst*, 16> VAStartInstrumentationList;
4172
4173 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4174
4175 VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
4176 MemorySanitizerVisitor &MSV)
4177 : F(F), MS(MS), MSV(MSV) {
4178 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
4179 for (const auto &Attr : F.getAttributes().getFnAttributes()) {
4180 if (Attr.isStringAttribute() &&
4181 (Attr.getKindAsString() == "target-features")) {
4182 if (Attr.getValueAsString().contains("-sse"))
4183 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
4184 break;
4185 }
4186 }
4187 }
4188
4189 ArgKind classifyArgument(Value* arg) {
4190 // A very rough approximation of X86_64 argument classification rules.
4191 Type *T = arg->getType();
4192 if (T->isFPOrFPVectorTy() || T->isX86_MMXTy())
4193 return AK_FloatingPoint;
4194 if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
4195 return AK_GeneralPurpose;
4196 if (T->isPointerTy())
4197 return AK_GeneralPurpose;
4198 return AK_Memory;
4199 }
4200
4201 // For VarArg functions, store the argument shadow in an ABI-specific format
4202 // that corresponds to va_list layout.
4203 // We do this because Clang lowers va_arg in the frontend, and this pass
4204 // only sees the low level code that deals with va_list internals.
4205 // A much easier alternative (provided that Clang emits va_arg instructions)
4206 // would have been to associate each live instance of va_list with a copy of
4207 // MSanParamTLS, and extract shadow on va_arg() call in the argument list
4208 // order.
4209 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4210 unsigned GpOffset = 0;
4211 unsigned FpOffset = AMD64GpEndOffset;
4212 unsigned OverflowOffset = AMD64FpEndOffset;
4213 const DataLayout &DL = F.getParent()->getDataLayout();
4214 for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
4215 ++ArgIt) {
4216 Value *A = *ArgIt;
4217 unsigned ArgNo = CB.getArgOperandNo(ArgIt);
4218 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
4219 bool IsByVal = CB.paramHasAttr(ArgNo, Attribute::ByVal);
4220 if (IsByVal) {
4221 // ByVal arguments always go to the overflow area.
4222 // Fixed arguments passed through the overflow area will be stepped
4223 // over by va_start, so don't count them towards the offset.
4224 if (IsFixed)
4225 continue;
4226 assert(A->getType()->isPointerTy())((void)0);
4227 Type *RealTy = CB.getParamByValType(ArgNo);
4228 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
4229 Value *ShadowBase = getShadowPtrForVAArgument(
4230 RealTy, IRB, OverflowOffset, alignTo(ArgSize, 8));
4231 Value *OriginBase = nullptr;
4232 if (MS.TrackOrigins)
4233 OriginBase = getOriginPtrForVAArgument(RealTy, IRB, OverflowOffset);
4234 OverflowOffset += alignTo(ArgSize, 8);
4235 if (!ShadowBase)
4236 continue;
4237 Value *ShadowPtr, *OriginPtr;
4238 std::tie(ShadowPtr, OriginPtr) =
4239 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment,
4240 /*isStore*/ false);
4241
4242 IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr,
4243 kShadowTLSAlignment, ArgSize);
4244 if (MS.TrackOrigins)
4245 IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr,
4246 kShadowTLSAlignment, ArgSize);
4247 } else {
4248 ArgKind AK = classifyArgument(A);
4249 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
4250 AK = AK_Memory;
4251 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
4252 AK = AK_Memory;
4253 Value *ShadowBase, *OriginBase = nullptr;
4254 switch (AK) {
4255 case AK_GeneralPurpose:
4256 ShadowBase =
4257 getShadowPtrForVAArgument(A->getType(), IRB, GpOffset, 8);
4258 if (MS.TrackOrigins)
4259 OriginBase =
4260 getOriginPtrForVAArgument(A->getType(), IRB, GpOffset);
4261 GpOffset += 8;
4262 break;
4263 case AK_FloatingPoint:
4264 ShadowBase =
4265 getShadowPtrForVAArgument(A->getType(), IRB, FpOffset, 16);
4266 if (MS.TrackOrigins)
4267 OriginBase =
4268 getOriginPtrForVAArgument(A->getType(), IRB, FpOffset);
4269 FpOffset += 16;
4270 break;
4271 case AK_Memory:
4272 if (IsFixed)
4273 continue;
4274 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4275 ShadowBase =
4276 getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset, 8);
4277 if (MS.TrackOrigins)
4278 OriginBase =
4279 getOriginPtrForVAArgument(A->getType(), IRB, OverflowOffset);
4280 OverflowOffset += alignTo(ArgSize, 8);
4281 }
4282 // Take fixed arguments into account for GpOffset and FpOffset,
4283 // but don't actually store shadows for them.
4284 // TODO(glider): don't call get*PtrForVAArgument() for them.
4285 if (IsFixed)
4286 continue;
4287 if (!ShadowBase)
4288 continue;
4289 Value *Shadow = MSV.getShadow(A);
4290 IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment);
4291 if (MS.TrackOrigins) {
4292 Value *Origin = MSV.getOrigin(A);
4293 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
4294 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
4295 std::max(kShadowTLSAlignment, kMinOriginAlignment));
4296 }
4297 }
4298 }
4299 Constant *OverflowSize =
4300 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
4301 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4302 }
4303
4304 /// Compute the shadow address for a given va_arg.
4305 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4306 unsigned ArgOffset, unsigned ArgSize) {
4307 // Make sure we don't overflow __msan_va_arg_tls.
4308 if (ArgOffset + ArgSize > kParamTLSSize)
4309 return nullptr;
4310 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4311 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4312 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4313 "_msarg_va_s");
4314 }
4315
4316 /// Compute the origin address for a given va_arg.
4317 Value *getOriginPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) {
4318 Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy);
4319 // getOriginPtrForVAArgument() is always called after
4320 // getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never
4321 // overflow.
4322 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4323 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
4324 "_msarg_va_o");
4325 }
4326
4327 void unpoisonVAListTagForInst(IntrinsicInst &I) {
4328 IRBuilder<> IRB(&I);
4329 Value *VAListTag = I.getArgOperand(0);
4330 Value *ShadowPtr, *OriginPtr;
4331 const Align Alignment = Align(8);
4332 std::tie(ShadowPtr, OriginPtr) =
4333 MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
4334 /*isStore*/ true);
4335
4336 // Unpoison the whole __va_list_tag.
4337 // FIXME: magic ABI constants.
4338 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4339 /* size */ 24, Alignment, false);
4340 // We shouldn't need to zero out the origins, as they're only checked for
4341 // nonzero shadow.
4342 }
4343
4344 void visitVAStartInst(VAStartInst &I) override {
4345 if (F.getCallingConv() == CallingConv::Win64)
4346 return;
4347 VAStartInstrumentationList.push_back(&I);
4348 unpoisonVAListTagForInst(I);
4349 }
4350
4351 void visitVACopyInst(VACopyInst &I) override {
4352 if (F.getCallingConv() == CallingConv::Win64) return;
4353 unpoisonVAListTagForInst(I);
4354 }
4355
4356 void finalizeInstrumentation() override {
4357 assert(!VAArgOverflowSize && !VAArgTLSCopy &&((void)0)
4358 "finalizeInstrumentation called twice")((void)0);
4359 if (!VAStartInstrumentationList.empty()) {
4360 // If there is a va_start in this function, make a backup copy of
4361 // va_arg_tls somewhere in the function entry block.
4362 IRBuilder<> IRB(MSV.FnPrologueEnd);
4363 VAArgOverflowSize =
4364 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4365 Value *CopySize =
4366 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
4367 VAArgOverflowSize);
4368 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4369 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4370 if (MS.TrackOrigins) {
4371 VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4372 IRB.CreateMemCpy(VAArgTLSOriginCopy, Align(8), MS.VAArgOriginTLS,
4373 Align(8), CopySize);
4374 }
4375 }
4376
4377 // Instrument va_start.
4378 // Copy va_list shadow from the backup copy of the TLS contents.
4379 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4380 CallInst *OrigInst = VAStartInstrumentationList[i];
4381 IRBuilder<> IRB(OrigInst->getNextNode());
4382 Value *VAListTag = OrigInst->getArgOperand(0);
4383
4384 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4385 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
4386 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4387 ConstantInt::get(MS.IntptrTy, 16)),
4388 PointerType::get(RegSaveAreaPtrTy, 0));
4389 Value *RegSaveAreaPtr =
4390 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4391 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4392 const Align Alignment = Align(16);
4393 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4394 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4395 Alignment, /*isStore*/ true);
4396 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4397 AMD64FpEndOffset);
4398 if (MS.TrackOrigins)
4399 IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
4400 Alignment, AMD64FpEndOffset);
4401 Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4402 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
4403 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4404 ConstantInt::get(MS.IntptrTy, 8)),
4405 PointerType::get(OverflowArgAreaPtrTy, 0));
4406 Value *OverflowArgAreaPtr =
4407 IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
4408 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
4409 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
4410 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
4411 Alignment, /*isStore*/ true);
4412 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
4413 AMD64FpEndOffset);
4414 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
4415 VAArgOverflowSize);
4416 if (MS.TrackOrigins) {
4417 SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
4418 AMD64FpEndOffset);
4419 IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
4420 VAArgOverflowSize);
4421 }
4422 }
4423 }
4424};
4425
4426/// MIPS64-specific implementation of VarArgHelper.
4427struct VarArgMIPS64Helper : public VarArgHelper {
4428 Function &F;
4429 MemorySanitizer &MS;
4430 MemorySanitizerVisitor &MSV;
4431 Value *VAArgTLSCopy = nullptr;
4432 Value *VAArgSize = nullptr;
4433
4434 SmallVector<CallInst*, 16> VAStartInstrumentationList;
4435
4436 VarArgMIPS64Helper(Function &F, MemorySanitizer &MS,
4437 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4438
4439 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4440 unsigned VAArgOffset = 0;
4441 const DataLayout &DL = F.getParent()->getDataLayout();
4442 for (auto ArgIt = CB.arg_begin() + CB.getFunctionType()->getNumParams(),
4443 End = CB.arg_end();
4444 ArgIt != End; ++ArgIt) {
4445 Triple TargetTriple(F.getParent()->getTargetTriple());
4446 Value *A = *ArgIt;
4447 Value *Base;
4448 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4449 if (TargetTriple.getArch() == Triple::mips64) {
4450 // Adjusting the shadow for argument with size < 8 to match the placement
4451 // of bits in big endian system
4452 if (ArgSize < 8)
4453 VAArgOffset += (8 - ArgSize);
4454 }
4455 Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset, ArgSize);
4456 VAArgOffset += ArgSize;
4457 VAArgOffset = alignTo(VAArgOffset, 8);
4458 if (!Base)
4459 continue;
4460 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4461 }
4462
4463 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset);
4464 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
4465 // a new class member i.e. it is the total size of all VarArgs.
4466 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4467 }
4468
4469 /// Compute the shadow address for a given va_arg.
4470 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4471 unsigned ArgOffset, unsigned ArgSize) {
4472 // Make sure we don't overflow __msan_va_arg_tls.
4473 if (ArgOffset + ArgSize > kParamTLSSize)
4474 return nullptr;
4475 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4476 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4477 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4478 "_msarg");
4479 }
4480
4481 void visitVAStartInst(VAStartInst &I) override {
4482 IRBuilder<> IRB(&I);
4483 VAStartInstrumentationList.push_back(&I);
4484 Value *VAListTag = I.getArgOperand(0);
4485 Value *ShadowPtr, *OriginPtr;
4486 const Align Alignment = Align(8);
4487 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4488 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4489 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4490 /* size */ 8, Alignment, false);
4491 }
4492
4493 void visitVACopyInst(VACopyInst &I) override {
4494 IRBuilder<> IRB(&I);
4495 VAStartInstrumentationList.push_back(&I);
4496 Value *VAListTag = I.getArgOperand(0);
4497 Value *ShadowPtr, *OriginPtr;
4498 const Align Alignment = Align(8);
4499 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4500 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4501 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4502 /* size */ 8, Alignment, false);
4503 }
4504
4505 void finalizeInstrumentation() override {
4506 assert(!VAArgSize && !VAArgTLSCopy &&((void)0)
4507 "finalizeInstrumentation called twice")((void)0);
4508 IRBuilder<> IRB(MSV.FnPrologueEnd);
4509 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4510 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
4511 VAArgSize);
4512
4513 if (!VAStartInstrumentationList.empty()) {
4514 // If there is a va_start in this function, make a backup copy of
4515 // va_arg_tls somewhere in the function entry block.
4516 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4517 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4518 }
4519
4520 // Instrument va_start.
4521 // Copy va_list shadow from the backup copy of the TLS contents.
4522 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4523 CallInst *OrigInst = VAStartInstrumentationList[i];
4524 IRBuilder<> IRB(OrigInst->getNextNode());
4525 Value *VAListTag = OrigInst->getArgOperand(0);
4526 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4527 Value *RegSaveAreaPtrPtr =
4528 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4529 PointerType::get(RegSaveAreaPtrTy, 0));
4530 Value *RegSaveAreaPtr =
4531 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4532 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4533 const Align Alignment = Align(8);
4534 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4535 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4536 Alignment, /*isStore*/ true);
4537 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4538 CopySize);
4539 }
4540 }
4541};
4542
4543/// AArch64-specific implementation of VarArgHelper.
4544struct VarArgAArch64Helper : public VarArgHelper {
4545 static const unsigned kAArch64GrArgSize = 64;
4546 static const unsigned kAArch64VrArgSize = 128;
4547
4548 static const unsigned AArch64GrBegOffset = 0;
4549 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
4550 // Make VR space aligned to 16 bytes.
4551 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
4552 static const unsigned AArch64VrEndOffset = AArch64VrBegOffset
4553 + kAArch64VrArgSize;
4554 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
4555
4556 Function &F;
4557 MemorySanitizer &MS;
4558 MemorySanitizerVisitor &MSV;
4559 Value *VAArgTLSCopy = nullptr;
4560 Value *VAArgOverflowSize = nullptr;
4561
4562 SmallVector<CallInst*, 16> VAStartInstrumentationList;
4563
4564 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4565
4566 VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
4567 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4568
4569 ArgKind classifyArgument(Value* arg) {
4570 Type *T = arg->getType();
4571 if (T->isFPOrFPVectorTy())
4572 return AK_FloatingPoint;
4573 if ((T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
4574 || (T->isPointerTy()))
4575 return AK_GeneralPurpose;
4576 return AK_Memory;
4577 }
4578
4579 // The instrumentation stores the argument shadow in a non ABI-specific
4580 // format because it does not know which argument is named (since Clang,
4581 // like x86_64 case, lowers the va_args in the frontend and this pass only
4582 // sees the low level code that deals with va_list internals).
4583 // The first seven GR registers are saved in the first 56 bytes of the
4584 // va_arg tls arra, followers by the first 8 FP/SIMD registers, and then
4585 // the remaining arguments.
4586 // Using constant offset within the va_arg TLS array allows fast copy
4587 // in the finalize instrumentation.
4588 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4589 unsigned GrOffset = AArch64GrBegOffset;
4590 unsigned VrOffset = AArch64VrBegOffset;
4591 unsigned OverflowOffset = AArch64VAEndOffset;
4592
4593 const DataLayout &DL = F.getParent()->getDataLayout();
4594 for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
4595 ++ArgIt) {
4596 Value *A = *ArgIt;
4597 unsigned ArgNo = CB.getArgOperandNo(ArgIt);
4598 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
4599 ArgKind AK = classifyArgument(A);
4600 if (AK == AK_GeneralPurpose && GrOffset >= AArch64GrEndOffset)
4601 AK = AK_Memory;
4602 if (AK == AK_FloatingPoint && VrOffset >= AArch64VrEndOffset)
4603 AK = AK_Memory;
4604 Value *Base;
4605 switch (AK) {
4606 case AK_GeneralPurpose:
4607 Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset, 8);
4608 GrOffset += 8;
4609 break;
4610 case AK_FloatingPoint:
4611 Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset, 8);
4612 VrOffset += 16;
4613 break;
4614 case AK_Memory:
4615 // Don't count fixed arguments in the overflow area - va_start will
4616 // skip right over them.
4617 if (IsFixed)
4618 continue;
4619 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4620 Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset,
4621 alignTo(ArgSize, 8));
4622 OverflowOffset += alignTo(ArgSize, 8);
4623 break;
4624 }
4625 // Count Gp/Vr fixed arguments to their respective offsets, but don't
4626 // bother to actually store a shadow.
4627 if (IsFixed)
4628 continue;
4629 if (!Base)
4630 continue;
4631 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4632 }
4633 Constant *OverflowSize =
4634 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
4635 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4636 }
4637
4638 /// Compute the shadow address for a given va_arg.
4639 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4640 unsigned ArgOffset, unsigned ArgSize) {
4641 // Make sure we don't overflow __msan_va_arg_tls.
4642 if (ArgOffset + ArgSize > kParamTLSSize)
4643 return nullptr;
4644 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4645 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4646 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4647 "_msarg");
4648 }
4649
4650 void visitVAStartInst(VAStartInst &I) override {
4651 IRBuilder<> IRB(&I);
4652 VAStartInstrumentationList.push_back(&I);
4653 Value *VAListTag = I.getArgOperand(0);
4654 Value *ShadowPtr, *OriginPtr;
4655 const Align Alignment = Align(8);
4656 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4657 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4658 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4659 /* size */ 32, Alignment, false);
4660 }
4661
4662 void visitVACopyInst(VACopyInst &I) override {
4663 IRBuilder<> IRB(&I);
4664 VAStartInstrumentationList.push_back(&I);
4665 Value *VAListTag = I.getArgOperand(0);
4666 Value *ShadowPtr, *OriginPtr;
4667 const Align Alignment = Align(8);
4668 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4669 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4670 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4671 /* size */ 32, Alignment, false);
4672 }
4673
4674 // Retrieve a va_list field of 'void*' size.
4675 Value* getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4676 Value *SaveAreaPtrPtr =
4677 IRB.CreateIntToPtr(
4678 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4679 ConstantInt::get(MS.IntptrTy, offset)),
4680 Type::getInt64PtrTy(*MS.C));
4681 return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr);
4682 }
4683
4684 // Retrieve a va_list field of 'int' size.
4685 Value* getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4686 Value *SaveAreaPtr =
4687 IRB.CreateIntToPtr(
4688 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4689 ConstantInt::get(MS.IntptrTy, offset)),
4690 Type::getInt32PtrTy(*MS.C));
4691 Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr);
4692 return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
4693 }
4694
4695 void finalizeInstrumentation() override {
4696 assert(!VAArgOverflowSize && !VAArgTLSCopy &&((void)0)
4697 "finalizeInstrumentation called twice")((void)0);
4698 if (!VAStartInstrumentationList.empty()) {
4699 // If there is a va_start in this function, make a backup copy of
4700 // va_arg_tls somewhere in the function entry block.
4701 IRBuilder<> IRB(MSV.FnPrologueEnd);
4702 VAArgOverflowSize =
4703 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4704 Value *CopySize =
4705 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
4706 VAArgOverflowSize);
4707 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4708 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4709 }
4710
4711 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
4712 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
4713
4714 // Instrument va_start, copy va_list shadow from the backup copy of
4715 // the TLS contents.
4716 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4717 CallInst *OrigInst = VAStartInstrumentationList[i];
4718 IRBuilder<> IRB(OrigInst->getNextNode());
4719
4720 Value *VAListTag = OrigInst->getArgOperand(0);
4721
4722 // The variadic ABI for AArch64 creates two areas to save the incoming
4723 // argument registers (one for 64-bit general register xn-x7 and another
4724 // for 128-bit FP/SIMD vn-v7).
4725 // We need then to propagate the shadow arguments on both regions
4726 // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
4727 // The remaining arguments are saved on shadow for 'va::stack'.
4728 // One caveat is it requires only to propagate the non-named arguments,
4729 // however on the call site instrumentation 'all' the arguments are
4730 // saved. So to copy the shadow values from the va_arg TLS array
4731 // we need to adjust the offset for both GR and VR fields based on
4732 // the __{gr,vr}_offs value (since they are stores based on incoming
4733 // named arguments).
4734
4735 // Read the stack pointer from the va_list.
4736 Value *StackSaveAreaPtr = getVAField64(IRB, VAListTag, 0);
4737
4738 // Read both the __gr_top and __gr_off and add them up.
4739 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
4740 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
4741
4742 Value *GrRegSaveAreaPtr = IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea);
4743
4744 // Read both the __vr_top and __vr_off and add them up.
4745 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
4746 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
4747
4748 Value *VrRegSaveAreaPtr = IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea);
4749
4750 // It does not know how many named arguments is being used and, on the
4751 // callsite all the arguments were saved. Since __gr_off is defined as
4752 // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
4753 // argument by ignoring the bytes of shadow from named arguments.
4754 Value *GrRegSaveAreaShadowPtrOff =
4755 IRB.CreateAdd(GrArgSize, GrOffSaveArea);
4756
4757 Value *GrRegSaveAreaShadowPtr =
4758 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4759 Align(8), /*isStore*/ true)
4760 .first;
4761
4762 Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4763 GrRegSaveAreaShadowPtrOff);
4764 Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
4765
4766 IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, Align(8), GrSrcPtr, Align(8),
4767 GrCopySize);
4768
4769 // Again, but for FP/SIMD values.
4770 Value *VrRegSaveAreaShadowPtrOff =
4771 IRB.CreateAdd(VrArgSize, VrOffSaveArea);
4772
4773 Value *VrRegSaveAreaShadowPtr =
4774 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4775 Align(8), /*isStore*/ true)
4776 .first;
4777
4778 Value *VrSrcPtr = IRB.CreateInBoundsGEP(
4779 IRB.getInt8Ty(),
4780 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4781 IRB.getInt32(AArch64VrBegOffset)),
4782 VrRegSaveAreaShadowPtrOff);
4783 Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
4784
4785 IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, Align(8), VrSrcPtr, Align(8),
4786 VrCopySize);
4787
4788 // And finally for remaining arguments.
4789 Value *StackSaveAreaShadowPtr =
4790 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(),
4791 Align(16), /*isStore*/ true)
4792 .first;
4793
4794 Value *StackSrcPtr =
4795 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4796 IRB.getInt32(AArch64VAEndOffset));
4797
4798 IRB.CreateMemCpy(StackSaveAreaShadowPtr, Align(16), StackSrcPtr,
4799 Align(16), VAArgOverflowSize);
4800 }
4801 }
4802};
4803
4804/// PowerPC64-specific implementation of VarArgHelper.
4805struct VarArgPowerPC64Helper : public VarArgHelper {
4806 Function &F;
4807 MemorySanitizer &MS;
4808 MemorySanitizerVisitor &MSV;
4809 Value *VAArgTLSCopy = nullptr;
4810 Value *VAArgSize = nullptr;
4811
4812 SmallVector<CallInst*, 16> VAStartInstrumentationList;
4813
4814 VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
4815 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4816
4817 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4818 // For PowerPC, we need to deal with alignment of stack arguments -
4819 // they are mostly aligned to 8 bytes, but vectors and i128 arrays
4820 // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
4821 // For that reason, we compute current offset from stack pointer (which is
4822 // always properly aligned), and offset for the first vararg, then subtract
4823 // them.
4824 unsigned VAArgBase;
4825 Triple TargetTriple(F.getParent()->getTargetTriple());
4826 // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
4827 // and 32 bytes for ABIv2. This is usually determined by target
4828 // endianness, but in theory could be overridden by function attribute.
4829 if (TargetTriple.getArch() == Triple::ppc64)
4830 VAArgBase = 48;
4831 else
4832 VAArgBase = 32;
4833 unsigned VAArgOffset = VAArgBase;
4834 const DataLayout &DL = F.getParent()->getDataLayout();
4835 for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
4836 ++ArgIt) {
4837 Value *A = *ArgIt;
4838 unsigned ArgNo = CB.getArgOperandNo(ArgIt);
4839 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
4840 bool IsByVal = CB.paramHasAttr(ArgNo, Attribute::ByVal);
4841 if (IsByVal) {
4842 assert(A->getType()->isPointerTy())((void)0);
4843 Type *RealTy = CB.getParamByValType(ArgNo);
4844 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
4845 MaybeAlign ArgAlign = CB.getParamAlign(ArgNo);
4846 if (!ArgAlign || *ArgAlign < Align(8))
4847 ArgAlign = Align(8);
4848 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4849 if (!IsFixed) {
4850 Value *Base = getShadowPtrForVAArgument(
4851 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
4852 if (Base) {
4853 Value *AShadowPtr, *AOriginPtr;
4854 std::tie(AShadowPtr, AOriginPtr) =
4855 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
4856 kShadowTLSAlignment, /*isStore*/ false);
4857
4858 IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
4859 kShadowTLSAlignment, ArgSize);
4860 }
4861 }
4862 VAArgOffset += alignTo(ArgSize, 8);
4863 } else {
4864 Value *Base;
4865 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4866 uint64_t ArgAlign = 8;
4867 if (A->getType()->isArrayTy()) {
4868 // Arrays are aligned to element size, except for long double
4869 // arrays, which are aligned to 8 bytes.
4870 Type *ElementTy = A->getType()->getArrayElementType();
4871 if (!ElementTy->isPPC_FP128Ty())
4872 ArgAlign = DL.getTypeAllocSize(ElementTy);
4873 } else if (A->getType()->isVectorTy()) {
4874 // Vectors are naturally aligned.
4875 ArgAlign = DL.getTypeAllocSize(A->getType());
4876 }
4877 if (ArgAlign < 8)
4878 ArgAlign = 8;
4879 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4880 if (DL.isBigEndian()) {
4881 // Adjusting the shadow for argument with size < 8 to match the placement
4882 // of bits in big endian system
4883 if (ArgSize < 8)
4884 VAArgOffset += (8 - ArgSize);
4885 }
4886 if (!IsFixed) {
4887 Base = getShadowPtrForVAArgument(A->getType(), IRB,
4888 VAArgOffset - VAArgBase, ArgSize);
4889 if (Base)
4890 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4891 }
4892 VAArgOffset += ArgSize;
4893 VAArgOffset = alignTo(VAArgOffset, 8);
4894 }
4895 if (IsFixed)
4896 VAArgBase = VAArgOffset;
4897 }
4898
4899 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(),
4900 VAArgOffset - VAArgBase);
4901 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
4902 // a new class member i.e. it is the total size of all VarArgs.
4903 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4904 }
4905
4906 /// Compute the shadow address for a given va_arg.
4907 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4908 unsigned ArgOffset, unsigned ArgSize) {
4909 // Make sure we don't overflow __msan_va_arg_tls.
4910 if (ArgOffset + ArgSize > kParamTLSSize)
4911 return nullptr;
4912 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4913 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4914 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4915 "_msarg");
4916 }
4917
4918 void visitVAStartInst(VAStartInst &I) override {
4919 IRBuilder<> IRB(&I);
4920 VAStartInstrumentationList.push_back(&I);
4921 Value *VAListTag = I.getArgOperand(0);
4922 Value *ShadowPtr, *OriginPtr;
4923 const Align Alignment = Align(8);
4924 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4925 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4926 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4927 /* size */ 8, Alignment, false);
4928 }
4929
4930 void visitVACopyInst(VACopyInst &I) override {
4931 IRBuilder<> IRB(&I);
4932 Value *VAListTag = I.getArgOperand(0);
4933 Value *ShadowPtr, *OriginPtr;
4934 const Align Alignment = Align(8);
4935 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4936 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4937 // Unpoison the whole __va_list_tag.
4938 // FIXME: magic ABI constants.
4939 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4940 /* size */ 8, Alignment, false);
4941 }
4942
4943 void finalizeInstrumentation() override {
4944 assert(!VAArgSize && !VAArgTLSCopy &&((void)0)
4945 "finalizeInstrumentation called twice")((void)0);
4946 IRBuilder<> IRB(MSV.FnPrologueEnd);
4947 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4948 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
4949 VAArgSize);
4950
4951 if (!VAStartInstrumentationList.empty()) {
4952 // If there is a va_start in this function, make a backup copy of
4953 // va_arg_tls somewhere in the function entry block.
4954 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4955 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4956 }
4957
4958 // Instrument va_start.
4959 // Copy va_list shadow from the backup copy of the TLS contents.
4960 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4961 CallInst *OrigInst = VAStartInstrumentationList[i];
4962 IRBuilder<> IRB(OrigInst->getNextNode());
4963 Value *VAListTag = OrigInst->getArgOperand(0);
4964 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4965 Value *RegSaveAreaPtrPtr =
4966 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4967 PointerType::get(RegSaveAreaPtrTy, 0));
4968 Value *RegSaveAreaPtr =
4969 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4970 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4971 const Align Alignment = Align(8);
4972 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4973 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4974 Alignment, /*isStore*/ true);
4975 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4976 CopySize);
4977 }
4978 }
4979};
4980
4981/// SystemZ-specific implementation of VarArgHelper.
4982struct VarArgSystemZHelper : public VarArgHelper {
4983 static const unsigned SystemZGpOffset = 16;
4984 static const unsigned SystemZGpEndOffset = 56;
4985 static const unsigned SystemZFpOffset = 128;
4986 static const unsigned SystemZFpEndOffset = 160;
4987 static const unsigned SystemZMaxVrArgs = 8;
4988 static const unsigned SystemZRegSaveAreaSize = 160;
4989 static const unsigned SystemZOverflowOffset = 160;
4990 static const unsigned SystemZVAListTagSize = 32;
4991 static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
4992 static const unsigned SystemZRegSaveAreaPtrOffset = 24;
4993
4994 Function &F;
4995 MemorySanitizer &MS;
4996 MemorySanitizerVisitor &MSV;
4997 Value *VAArgTLSCopy = nullptr;
4998 Value *VAArgTLSOriginCopy = nullptr;
4999 Value *VAArgOverflowSize = nullptr;
5000
5001 SmallVector<CallInst *, 16> VAStartInstrumentationList;
5002
5003 enum class ArgKind {
5004 GeneralPurpose,
5005 FloatingPoint,
5006 Vector,
5007 Memory,
5008 Indirect,
5009 };
5010
5011 enum class ShadowExtension { None, Zero, Sign };
5012
5013 VarArgSystemZHelper(Function &F, MemorySanitizer &MS,
5014 MemorySanitizerVisitor &MSV)
5015 : F(F), MS(MS), MSV(MSV) {}
5016
5017 ArgKind classifyArgument(Type *T, bool IsSoftFloatABI) {
5018 // T is a SystemZABIInfo::classifyArgumentType() output, and there are
5019 // only a few possibilities of what it can be. In particular, enums, single
5020 // element structs and large types have already been taken care of.
5021
5022 // Some i128 and fp128 arguments are converted to pointers only in the
5023 // back end.
5024 if (T->isIntegerTy(128) || T->isFP128Ty())
5025 return ArgKind::Indirect;
5026 if (T->isFloatingPointTy())
5027 return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
5028 if (T->isIntegerTy() || T->isPointerTy())
5029 return ArgKind::GeneralPurpose;
5030 if (T->isVectorTy())
5031 return ArgKind::Vector;
5032 return ArgKind::Memory;
5033 }
5034
5035 ShadowExtension getShadowExtension(const CallBase &CB, unsigned ArgNo) {
5036 // ABI says: "One of the simple integer types no more than 64 bits wide.
5037 // ... If such an argument is shorter than 64 bits, replace it by a full
5038 // 64-bit integer representing the same number, using sign or zero
5039 // extension". Shadow for an integer argument has the same type as the
5040 // argument itself, so it can be sign or zero extended as well.
5041 bool ZExt = CB.paramHasAttr(ArgNo, Attribute::ZExt);
5042 bool SExt = CB.paramHasAttr(ArgNo, Attribute::SExt);
5043 if (ZExt) {
5044 assert(!SExt)((void)0);
5045 return ShadowExtension::Zero;
5046 }
5047 if (SExt) {
5048 assert(!ZExt)((void)0);
5049 return ShadowExtension::Sign;
5050 }
5051 return ShadowExtension::None;
5052 }
5053
5054 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5055 bool IsSoftFloatABI = CB.getCalledFunction()
5056 ->getFnAttribute("use-soft-float")
5057 .getValueAsBool();
5058 unsigned GpOffset = SystemZGpOffset;
5059 unsigned FpOffset = SystemZFpOffset;
5060 unsigned VrIndex = 0;
5061 unsigned OverflowOffset = SystemZOverflowOffset;
5062 const DataLayout &DL = F.getParent()->getDataLayout();
5063 for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
5064 ++ArgIt) {
5065 Value *A = *ArgIt;
5066 unsigned ArgNo = CB.getArgOperandNo(ArgIt);
5067 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
5068 // SystemZABIInfo does not produce ByVal parameters.
5069 assert(!CB.paramHasAttr(ArgNo, Attribute::ByVal))((void)0);
5070 Type *T = A->getType();
5071 ArgKind AK = classifyArgument(T, IsSoftFloatABI);
5072 if (AK == ArgKind::Indirect) {
5073 T = PointerType::get(T, 0);
5074 AK = ArgKind::GeneralPurpose;
5075 }
5076 if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
5077 AK = ArgKind::Memory;
5078 if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
5079 AK = ArgKind::Memory;
5080 if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
5081 AK = ArgKind::Memory;
5082 Value *ShadowBase = nullptr;
5083 Value *OriginBase = nullptr;
5084 ShadowExtension SE = ShadowExtension::None;
5085 switch (AK) {
5086 case ArgKind::GeneralPurpose: {
5087 // Always keep track of GpOffset, but store shadow only for varargs.
5088 uint64_t ArgSize = 8;
5089 if (GpOffset + ArgSize <= kParamTLSSize) {
5090 if (!IsFixed) {
5091 SE = getShadowExtension(CB, ArgNo);
5092 uint64_t GapSize = 0;
5093 if (SE == ShadowExtension::None) {
5094 uint64_t ArgAllocSize = DL.getTypeAllocSize(T);
5095 assert(ArgAllocSize <= ArgSize)((void)0);
5096 GapSize = ArgSize - ArgAllocSize;
5097 }
5098 ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
5099 if (MS.TrackOrigins)
5100 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
5101 }
5102 GpOffset += ArgSize;
5103 } else {
5104 GpOffset = kParamTLSSize;
5105 }
5106 break;
5107 }
5108 case ArgKind::FloatingPoint: {
5109 // Always keep track of FpOffset, but store shadow only for varargs.
5110 uint64_t ArgSize = 8;
5111 if (FpOffset + ArgSize <= kParamTLSSize) {
5112 if (!IsFixed) {
5113 // PoP says: "A short floating-point datum requires only the
5114 // left-most 32 bit positions of a floating-point register".
5115 // Therefore, in contrast to AK_GeneralPurpose and AK_Memory,
5116 // don't extend shadow and don't mind the gap.
5117 ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
5118 if (MS.TrackOrigins)
5119 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5120 }
5121 FpOffset += ArgSize;
5122 } else {
5123 FpOffset = kParamTLSSize;
5124 }
5125 break;
5126 }
5127 case ArgKind::Vector: {
5128 // Keep track of VrIndex. No need to store shadow, since vector varargs
5129 // go through AK_Memory.
5130 assert(IsFixed)((void)0);
5131 VrIndex++;
5132 break;
5133 }
5134 case ArgKind::Memory: {
5135 // Keep track of OverflowOffset and store shadow only for varargs.
5136 // Ignore fixed args, since we need to copy only the vararg portion of
5137 // the overflow area shadow.
5138 if (!IsFixed) {
5139 uint64_t ArgAllocSize = DL.getTypeAllocSize(T);
5140 uint64_t ArgSize = alignTo(ArgAllocSize, 8);
5141 if (OverflowOffset + ArgSize <= kParamTLSSize) {
5142 SE = getShadowExtension(CB, ArgNo);
5143 uint64_t GapSize =
5144 SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
5145 ShadowBase =
5146 getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
5147 if (MS.TrackOrigins)
5148 OriginBase =
5149 getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);
5150 OverflowOffset += ArgSize;
5151 } else {
5152 OverflowOffset = kParamTLSSize;
5153 }
5154 }
5155 break;
5156 }
5157 case ArgKind::Indirect:
5158 llvm_unreachable("Indirect must be converted to GeneralPurpose")__builtin_unreachable();
5159 }
5160 if (ShadowBase == nullptr)
5161 continue;
5162 Value *Shadow = MSV.getShadow(A);
5163 if (SE != ShadowExtension::None)
5164 Shadow = MSV.CreateShadowCast(IRB, Shadow, IRB.getInt64Ty(),
5165 /*Signed*/ SE == ShadowExtension::Sign);
5166 ShadowBase = IRB.CreateIntToPtr(
5167 ShadowBase, PointerType::get(Shadow->getType(), 0), "_msarg_va_s");
5168 IRB.CreateStore(Shadow, ShadowBase);
5169 if (MS.TrackOrigins) {
5170 Value *Origin = MSV.getOrigin(A);
5171 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
5172 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5173 kMinOriginAlignment);
5174 }
5175 }
5176 Constant *OverflowSize = ConstantInt::get(
5177 IRB.getInt64Ty(), OverflowOffset - SystemZOverflowOffset);
5178 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5179 }
5180
5181 Value *getShadowAddrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
5182 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
5183 return IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
5184 }
5185
5186 Value *getOriginPtrForVAArgument(IRBuilder<> &IRB, int ArgOffset) {
5187 Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy);
5188 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
5189 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
5190 "_msarg_va_o");
5191 }
5192
5193 void unpoisonVAListTagForInst(IntrinsicInst &I) {
5194 IRBuilder<> IRB(&I);
5195 Value *VAListTag = I.getArgOperand(0);
5196 Value *ShadowPtr, *OriginPtr;
5197 const Align Alignment = Align(8);
5198 std::tie(ShadowPtr, OriginPtr) =
5199 MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
5200 /*isStore*/ true);
5201 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
5202 SystemZVAListTagSize, Alignment, false);
5203 }
5204
5205 void visitVAStartInst(VAStartInst &I) override {
5206 VAStartInstrumentationList.push_back(&I);
5207 unpoisonVAListTagForInst(I);
5208 }
5209
5210 void visitVACopyInst(VACopyInst &I) override { unpoisonVAListTagForInst(I); }
5211
5212 void copyRegSaveArea(IRBuilder<> &IRB, Value *VAListTag) {
5213 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
5214 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
5215 IRB.CreateAdd(
5216 IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5217 ConstantInt::get(MS.IntptrTy, SystemZRegSaveAreaPtrOffset)),
5218 PointerType::get(RegSaveAreaPtrTy, 0));
5219 Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5220 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5221 const Align Alignment = Align(8);
5222 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5223 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), Alignment,
5224 /*isStore*/ true);
5225 // TODO(iii): copy only fragments filled by visitCallBase()
5226 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5227 SystemZRegSaveAreaSize);
5228 if (MS.TrackOrigins)
5229 IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5230 Alignment, SystemZRegSaveAreaSize);
5231 }
5232
5233 void copyOverflowArea(IRBuilder<> &IRB, Value *VAListTag) {
5234 Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C);
5235 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
5236 IRB.CreateAdd(
5237 IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5238 ConstantInt::get(MS.IntptrTy, SystemZOverflowArgAreaPtrOffset)),
5239 PointerType::get(OverflowArgAreaPtrTy, 0));
5240 Value *OverflowArgAreaPtr =
5241 IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5242 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5243 const Align Alignment = Align(8);
5244 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5245 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
5246 Alignment, /*isStore*/ true);
5247 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
5248 SystemZOverflowOffset);
5249 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5250 VAArgOverflowSize);
5251 if (MS.TrackOrigins) {
5252 SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
5253 SystemZOverflowOffset);
5254 IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5255 VAArgOverflowSize);
5256 }
5257 }
5258
5259 void finalizeInstrumentation() override {
5260 assert(!VAArgOverflowSize && !VAArgTLSCopy &&((void)0)
5261 "finalizeInstrumentation called twice")((void)0);
5262 if (!VAStartInstrumentationList.empty()) {
5263 // If there is a va_start in this function, make a backup copy of
5264 // va_arg_tls somewhere in the function entry block.
5265 IRBuilder<> IRB(MSV.FnPrologueEnd);
5266 VAArgOverflowSize =
5267 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
5268 Value *CopySize =
5269 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset),
5270 VAArgOverflowSize);
5271 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5272 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
5273 if (MS.TrackOrigins) {
5274 VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5275 IRB.CreateMemCpy(VAArgTLSOriginCopy, Align(8), MS.VAArgOriginTLS,
5276 Align(8), CopySize);
5277 }
5278 }
5279
5280 // Instrument va_start.
5281 // Copy va_list shadow from the backup copy of the TLS contents.
5282 for (size_t VaStartNo = 0, VaStartNum = VAStartInstrumentationList.size();
5283 VaStartNo < VaStartNum; VaStartNo++) {
5284 CallInst *OrigInst = VAStartInstrumentationList[VaStartNo];
5285 IRBuilder<> IRB(OrigInst->getNextNode());
5286 Value *VAListTag = OrigInst->getArgOperand(0);
5287 copyRegSaveArea(IRB, VAListTag);
5288 copyOverflowArea(IRB, VAListTag);
5289 }
5290 }
5291};
5292
5293/// A no-op implementation of VarArgHelper.
5294struct VarArgNoOpHelper : public VarArgHelper {
5295 VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
5296 MemorySanitizerVisitor &MSV) {}
5297
5298 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {}
5299
5300 void visitVAStartInst(VAStartInst &I) override {}
5301
5302 void visitVACopyInst(VACopyInst &I) override {}
5303
5304 void finalizeInstrumentation() override {}
5305};
5306
5307} // end anonymous namespace
5308
5309static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
5310 MemorySanitizerVisitor &Visitor) {
5311 // VarArg handling is only implemented on AMD64. False positives are possible
5312 // on other platforms.
5313 Triple TargetTriple(Func.getParent()->getTargetTriple());
5314 if (TargetTriple.getArch() == Triple::x86_64)
5315 return new VarArgAMD64Helper(Func, Msan, Visitor);
5316 else if (TargetTriple.isMIPS64())
5317 return new VarArgMIPS64Helper(Func, Msan, Visitor);
5318 else if (TargetTriple.getArch() == Triple::aarch64)
5319 return new VarArgAArch64Helper(Func, Msan, Visitor);
5320 else if (TargetTriple.getArch() == Triple::ppc64 ||
5321 TargetTriple.getArch() == Triple::ppc64le)
5322 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
5323 else if (TargetTriple.getArch() == Triple::systemz)
5324 return new VarArgSystemZHelper(Func, Msan, Visitor);
5325 else
5326 return new VarArgNoOpHelper(Func, Msan, Visitor);
5327}
5328
5329bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) {
5330 if (!CompileKernel && F.getName() == kMsanModuleCtorName)
5331 return false;
5332
5333 MemorySanitizerVisitor Visitor(F, *this, TLI);
5334
5335 // Clear out readonly/readnone attributes.
5336 AttrBuilder B;
5337 B.addAttribute(Attribute::ReadOnly)
5338 .addAttribute(Attribute::ReadNone)
5339 .addAttribute(Attribute::WriteOnly)
5340 .addAttribute(Attribute::ArgMemOnly)
5341 .addAttribute(Attribute::Speculatable);
5342 F.removeAttributes(AttributeList::FunctionIndex, B);
5343
5344 return Visitor.runOnFunction();
5345}

/usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IR/Instructions.h

1//===- llvm/Instructions.h - Instruction subclass definitions ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file exposes the class definitions of all of the subclasses of the
10// Instruction class. This is meant to be an easy way to get access to all
11// instruction subclasses.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_INSTRUCTIONS_H
16#define LLVM_IR_INSTRUCTIONS_H
17
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/Bitfields.h"
20#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/None.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/ADT/iterator.h"
27#include "llvm/ADT/iterator_range.h"
28#include "llvm/IR/Attributes.h"
29#include "llvm/IR/BasicBlock.h"
30#include "llvm/IR/CallingConv.h"
31#include "llvm/IR/CFG.h"
32#include "llvm/IR/Constant.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/Function.h"
35#include "llvm/IR/InstrTypes.h"
36#include "llvm/IR/Instruction.h"
37#include "llvm/IR/OperandTraits.h"
38#include "llvm/IR/Type.h"
39#include "llvm/IR/Use.h"
40#include "llvm/IR/User.h"
41#include "llvm/IR/Value.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include <cassert>
46#include <cstddef>
47#include <cstdint>
48#include <iterator>
49
50namespace llvm {
51
52class APInt;
53class ConstantInt;
54class DataLayout;
55class LLVMContext;
56
57//===----------------------------------------------------------------------===//
58// AllocaInst Class
59//===----------------------------------------------------------------------===//
60
61/// an instruction to allocate memory on the stack
62class AllocaInst : public UnaryInstruction {
63 Type *AllocatedType;
64
65 using AlignmentField = AlignmentBitfieldElementT<0>;
66 using UsedWithInAllocaField = BoolBitfieldElementT<AlignmentField::NextBit>;
67 using SwiftErrorField = BoolBitfieldElementT<UsedWithInAllocaField::NextBit>;
68 static_assert(Bitfield::areContiguous<AlignmentField, UsedWithInAllocaField,
69 SwiftErrorField>(),
70 "Bitfields must be contiguous");
71
72protected:
73 // Note: Instruction needs to be a friend here to call cloneImpl.
74 friend class Instruction;
75
76 AllocaInst *cloneImpl() const;
77
78public:
79 explicit AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
80 const Twine &Name, Instruction *InsertBefore);
81 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
82 const Twine &Name, BasicBlock *InsertAtEnd);
83
84 AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name,
85 Instruction *InsertBefore);
86 AllocaInst(Type *Ty, unsigned AddrSpace,
87 const Twine &Name, BasicBlock *InsertAtEnd);
88
89 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, Align Align,
90 const Twine &Name = "", Instruction *InsertBefore = nullptr);
91 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, Align Align,
92 const Twine &Name, BasicBlock *InsertAtEnd);
93
94 /// Return true if there is an allocation size parameter to the allocation
95 /// instruction that is not 1.
96 bool isArrayAllocation() const;
97
98 /// Get the number of elements allocated. For a simple allocation of a single
99 /// element, this will return a constant 1 value.
100 const Value *getArraySize() const { return getOperand(0); }
101 Value *getArraySize() { return getOperand(0); }
102
103 /// Overload to return most specific pointer type.
104 PointerType *getType() const {
105 return cast<PointerType>(Instruction::getType());
106 }
107
108 /// Get allocation size in bits. Returns None if size can't be determined,
109 /// e.g. in case of a VLA.
110 Optional<TypeSize> getAllocationSizeInBits(const DataLayout &DL) const;
111
112 /// Return the type that is being allocated by the instruction.
113 Type *getAllocatedType() const { return AllocatedType; }
114 /// for use only in special circumstances that need to generically
115 /// transform a whole instruction (eg: IR linking and vectorization).
116 void setAllocatedType(Type *Ty) { AllocatedType = Ty; }
117
118 /// Return the alignment of the memory that is being allocated by the
119 /// instruction.
120 Align getAlign() const {
121 return Align(1ULL << getSubclassData<AlignmentField>());
122 }
123
124 void setAlignment(Align Align) {
125 setSubclassData<AlignmentField>(Log2(Align));
126 }
127
128 // FIXME: Remove this one transition to Align is over.
129 unsigned getAlignment() const { return getAlign().value(); }
130
131 /// Return true if this alloca is in the entry block of the function and is a
132 /// constant size. If so, the code generator will fold it into the
133 /// prolog/epilog code, so it is basically free.
134 bool isStaticAlloca() const;
135
136 /// Return true if this alloca is used as an inalloca argument to a call. Such
137 /// allocas are never considered static even if they are in the entry block.
138 bool isUsedWithInAlloca() const {
139 return getSubclassData<UsedWithInAllocaField>();
140 }
141
142 /// Specify whether this alloca is used to represent the arguments to a call.
143 void setUsedWithInAlloca(bool V) {
144 setSubclassData<UsedWithInAllocaField>(V);
145 }
146
147 /// Return true if this alloca is used as a swifterror argument to a call.
148 bool isSwiftError() const { return getSubclassData<SwiftErrorField>(); }
149 /// Specify whether this alloca is used to represent a swifterror.
150 void setSwiftError(bool V) { setSubclassData<SwiftErrorField>(V); }
151
152 // Methods for support type inquiry through isa, cast, and dyn_cast:
153 static bool classof(const Instruction *I) {
154 return (I->getOpcode() == Instruction::Alloca);
155 }
156 static bool classof(const Value *V) {
157 return isa<Instruction>(V) && classof(cast<Instruction>(V));
158 }
159
160private:
161 // Shadow Instruction::setInstructionSubclassData with a private forwarding
162 // method so that subclasses cannot accidentally use it.
163 template <typename Bitfield>
164 void setSubclassData(typename Bitfield::Type Value) {
165 Instruction::setSubclassData<Bitfield>(Value);
166 }
167};
168
169//===----------------------------------------------------------------------===//
170// LoadInst Class
171//===----------------------------------------------------------------------===//
172
173/// An instruction for reading from memory. This uses the SubclassData field in
174/// Value to store whether or not the load is volatile.
175class LoadInst : public UnaryInstruction {
176 using VolatileField = BoolBitfieldElementT<0>;
177 using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
178 using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
179 static_assert(
180 Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
181 "Bitfields must be contiguous");
182
183 void AssertOK();
184
185protected:
186 // Note: Instruction needs to be a friend here to call cloneImpl.
187 friend class Instruction;
188
189 LoadInst *cloneImpl() const;
190
191public:
192 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr,
193 Instruction *InsertBefore);
194 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, BasicBlock *InsertAtEnd);
195 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
196 Instruction *InsertBefore);
197 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
198 BasicBlock *InsertAtEnd);
199 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
200 Align Align, Instruction *InsertBefore = nullptr);
201 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
202 Align Align, BasicBlock *InsertAtEnd);
203 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
204 Align Align, AtomicOrdering Order,
205 SyncScope::ID SSID = SyncScope::System,
206 Instruction *InsertBefore = nullptr);
207 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
208 Align Align, AtomicOrdering Order, SyncScope::ID SSID,
209 BasicBlock *InsertAtEnd);
210
211 /// Return true if this is a load from a volatile memory location.
212 bool isVolatile() const { return getSubclassData<VolatileField>(); }
213
214 /// Specify whether this is a volatile load or not.
215 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
216
217 /// Return the alignment of the access that is being performed.
218 /// FIXME: Remove this function once transition to Align is over.
219 /// Use getAlign() instead.
220 unsigned getAlignment() const { return getAlign().value(); }
2
Calling 'LoadInst::getAlign'
9
Returning from 'LoadInst::getAlign'
10
Calling 'Align::value'
221
222 /// Return the alignment of the access that is being performed.
223 Align getAlign() const {
224 return Align(1ULL << (getSubclassData<AlignmentField>()));
3
Calling constructor for 'Align'
8
Returning from constructor for 'Align'
225 }
226
227 void setAlignment(Align Align) {
228 setSubclassData<AlignmentField>(Log2(Align));
229 }
230
231 /// Returns the ordering constraint of this load instruction.
232 AtomicOrdering getOrdering() const {
233 return getSubclassData<OrderingField>();
234 }
235 /// Sets the ordering constraint of this load instruction. May not be Release
236 /// or AcquireRelease.
237 void setOrdering(AtomicOrdering Ordering) {
238 setSubclassData<OrderingField>(Ordering);
239 }
240
241 /// Returns the synchronization scope ID of this load instruction.
242 SyncScope::ID getSyncScopeID() const {
243 return SSID;
244 }
245
246 /// Sets the synchronization scope ID of this load instruction.
247 void setSyncScopeID(SyncScope::ID SSID) {
248 this->SSID = SSID;
249 }
250
251 /// Sets the ordering constraint and the synchronization scope ID of this load
252 /// instruction.
253 void setAtomic(AtomicOrdering Ordering,
254 SyncScope::ID SSID = SyncScope::System) {
255 setOrdering(Ordering);
256 setSyncScopeID(SSID);
257 }
258
259 bool isSimple() const { return !isAtomic() && !isVolatile(); }
260
261 bool isUnordered() const {
262 return (getOrdering() == AtomicOrdering::NotAtomic ||
263 getOrdering() == AtomicOrdering::Unordered) &&
264 !isVolatile();
265 }
266
267 Value *getPointerOperand() { return getOperand(0); }
268 const Value *getPointerOperand() const { return getOperand(0); }
269 static unsigned getPointerOperandIndex() { return 0U; }
270 Type *getPointerOperandType() const { return getPointerOperand()->getType(); }
271
272 /// Returns the address space of the pointer operand.
273 unsigned getPointerAddressSpace() const {
274 return getPointerOperandType()->getPointerAddressSpace();
275 }
276
277 // Methods for support type inquiry through isa, cast, and dyn_cast:
278 static bool classof(const Instruction *I) {
279 return I->getOpcode() == Instruction::Load;
280 }
281 static bool classof(const Value *V) {
282 return isa<Instruction>(V) && classof(cast<Instruction>(V));
283 }
284
285private:
286 // Shadow Instruction::setInstructionSubclassData with a private forwarding
287 // method so that subclasses cannot accidentally use it.
288 template <typename Bitfield>
289 void setSubclassData(typename Bitfield::Type Value) {
290 Instruction::setSubclassData<Bitfield>(Value);
291 }
292
293 /// The synchronization scope ID of this load instruction. Not quite enough
294 /// room in SubClassData for everything, so synchronization scope ID gets its
295 /// own field.
296 SyncScope::ID SSID;
297};
298
299//===----------------------------------------------------------------------===//
300// StoreInst Class
301//===----------------------------------------------------------------------===//
302
303/// An instruction for storing to memory.
304class StoreInst : public Instruction {
305 using VolatileField = BoolBitfieldElementT<0>;
306 using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
307 using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
308 static_assert(
309 Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
310 "Bitfields must be contiguous");
311
312 void AssertOK();
313
314protected:
315 // Note: Instruction needs to be a friend here to call cloneImpl.
316 friend class Instruction;
317
318 StoreInst *cloneImpl() const;
319
320public:
321 StoreInst(Value *Val, Value *Ptr, Instruction *InsertBefore);
322 StoreInst(Value *Val, Value *Ptr, BasicBlock *InsertAtEnd);
323 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Instruction *InsertBefore);
324 StoreInst(Value *Val, Value *Ptr, bool isVolatile, BasicBlock *InsertAtEnd);
325 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
326 Instruction *InsertBefore = nullptr);
327 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
328 BasicBlock *InsertAtEnd);
329 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
330 AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System,
331 Instruction *InsertBefore = nullptr);
332 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
333 AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd);
334
335 // allocate space for exactly two operands
336 void *operator new(size_t S) { return User::operator new(S, 2); }
337 void operator delete(void *Ptr) { User::operator delete(Ptr); }
338
339 /// Return true if this is a store to a volatile memory location.
340 bool isVolatile() const { return getSubclassData<VolatileField>(); }
341
342 /// Specify whether this is a volatile store or not.
343 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
344
345 /// Transparently provide more efficient getOperand methods.
346 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
347
348 /// Return the alignment of the access that is being performed
349 /// FIXME: Remove this function once transition to Align is over.
350 /// Use getAlign() instead.
351 unsigned getAlignment() const { return getAlign().value(); }
352
353 Align getAlign() const {
354 return Align(1ULL << (getSubclassData<AlignmentField>()));
355 }
356
357 void setAlignment(Align Align) {
358 setSubclassData<AlignmentField>(Log2(Align));
359 }
360
361 /// Returns the ordering constraint of this store instruction.
362 AtomicOrdering getOrdering() const {
363 return getSubclassData<OrderingField>();
364 }
365
366 /// Sets the ordering constraint of this store instruction. May not be
367 /// Acquire or AcquireRelease.
368 void setOrdering(AtomicOrdering Ordering) {
369 setSubclassData<OrderingField>(Ordering);
370 }
371
372 /// Returns the synchronization scope ID of this store instruction.
373 SyncScope::ID getSyncScopeID() const {
374 return SSID;
375 }
376
377 /// Sets the synchronization scope ID of this store instruction.
378 void setSyncScopeID(SyncScope::ID SSID) {
379 this->SSID = SSID;
380 }
381
382 /// Sets the ordering constraint and the synchronization scope ID of this
383 /// store instruction.
384 void setAtomic(AtomicOrdering Ordering,
385 SyncScope::ID SSID = SyncScope::System) {
386 setOrdering(Ordering);
387 setSyncScopeID(SSID);
388 }
389
390 bool isSimple() const { return !isAtomic() && !isVolatile(); }
391
392 bool isUnordered() const {
393 return (getOrdering() == AtomicOrdering::NotAtomic ||
394 getOrdering() == AtomicOrdering::Unordered) &&
395 !isVolatile();
396 }
397
398 Value *getValueOperand() { return getOperand(0); }
399 const Value *getValueOperand() const { return getOperand(0); }
400
401 Value *getPointerOperand() { return getOperand(1); }
402 const Value *getPointerOperand() const { return getOperand(1); }
403 static unsigned getPointerOperandIndex() { return 1U; }
404 Type *getPointerOperandType() const { return getPointerOperand()->getType(); }
405
406 /// Returns the address space of the pointer operand.
407 unsigned getPointerAddressSpace() const {
408 return getPointerOperandType()->getPointerAddressSpace();
409 }
410
411 // Methods for support type inquiry through isa, cast, and dyn_cast:
412 static bool classof(const Instruction *I) {
413 return I->getOpcode() == Instruction::Store;
414 }
415 static bool classof(const Value *V) {
416 return isa<Instruction>(V) && classof(cast<Instruction>(V));
417 }
418
419private:
420 // Shadow Instruction::setInstructionSubclassData with a private forwarding
421 // method so that subclasses cannot accidentally use it.
422 template <typename Bitfield>
423 void setSubclassData(typename Bitfield::Type Value) {
424 Instruction::setSubclassData<Bitfield>(Value);
425 }
426
427 /// The synchronization scope ID of this store instruction. Not quite enough
428 /// room in SubClassData for everything, so synchronization scope ID gets its
429 /// own field.
430 SyncScope::ID SSID;
431};
432
433template <>
434struct OperandTraits<StoreInst> : public FixedNumOperandTraits<StoreInst, 2> {
435};
436
437DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value)StoreInst::op_iterator StoreInst::op_begin() { return OperandTraits
<StoreInst>::op_begin(this); } StoreInst::const_op_iterator
StoreInst::op_begin() const { return OperandTraits<StoreInst
>::op_begin(const_cast<StoreInst*>(this)); } StoreInst
::op_iterator StoreInst::op_end() { return OperandTraits<StoreInst
>::op_end(this); } StoreInst::const_op_iterator StoreInst::
op_end() const { return OperandTraits<StoreInst>::op_end
(const_cast<StoreInst*>(this)); } Value *StoreInst::getOperand
(unsigned i_nocapture) const { ((void)0); return cast_or_null
<Value>( OperandTraits<StoreInst>::op_begin(const_cast
<StoreInst*>(this))[i_nocapture].get()); } void StoreInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((
void)0); OperandTraits<StoreInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned StoreInst::getNumOperands() const
{ return OperandTraits<StoreInst>::operands(this); } template
<int Idx_nocapture> Use &StoreInst::Op() { return this
->OpFrom<Idx_nocapture>(this); } template <int Idx_nocapture
> const Use &StoreInst::Op() const { return this->OpFrom
<Idx_nocapture>(this); }
438
439//===----------------------------------------------------------------------===//
440// FenceInst Class
441//===----------------------------------------------------------------------===//
442
443/// An instruction for ordering other memory operations.
444class FenceInst : public Instruction {
445 using OrderingField = AtomicOrderingBitfieldElementT<0>;
446
447 void Init(AtomicOrdering Ordering, SyncScope::ID SSID);
448
449protected:
450 // Note: Instruction needs to be a friend here to call cloneImpl.
451 friend class Instruction;
452
453 FenceInst *cloneImpl() const;
454
455public:
456 // Ordering may only be Acquire, Release, AcquireRelease, or
457 // SequentiallyConsistent.
458 FenceInst(LLVMContext &C, AtomicOrdering Ordering,
459 SyncScope::ID SSID = SyncScope::System,
460 Instruction *InsertBefore = nullptr);
461 FenceInst(LLVMContext &C, AtomicOrdering Ordering, SyncScope::ID SSID,
462 BasicBlock *InsertAtEnd);
463
464 // allocate space for exactly zero operands
465 void *operator new(size_t S) { return User::operator new(S, 0); }
466 void operator delete(void *Ptr) { User::operator delete(Ptr); }
467
468 /// Returns the ordering constraint of this fence instruction.
469 AtomicOrdering getOrdering() const {
470 return getSubclassData<OrderingField>();
471 }
472
473 /// Sets the ordering constraint of this fence instruction. May only be
474 /// Acquire, Release, AcquireRelease, or SequentiallyConsistent.
475 void setOrdering(AtomicOrdering Ordering) {
476 setSubclassData<OrderingField>(Ordering);
477 }
478
479 /// Returns the synchronization scope ID of this fence instruction.
480 SyncScope::ID getSyncScopeID() const {
481 return SSID;
482 }
483
484 /// Sets the synchronization scope ID of this fence instruction.
485 void setSyncScopeID(SyncScope::ID SSID) {
486 this->SSID = SSID;
487 }
488
489 // Methods for support type inquiry through isa, cast, and dyn_cast:
490 static bool classof(const Instruction *I) {
491 return I->getOpcode() == Instruction::Fence;
492 }
493 static bool classof(const Value *V) {
494 return isa<Instruction>(V) && classof(cast<Instruction>(V));
495 }
496
497private:
498 // Shadow Instruction::setInstructionSubclassData with a private forwarding
499 // method so that subclasses cannot accidentally use it.
500 template <typename Bitfield>
501 void setSubclassData(typename Bitfield::Type Value) {
502 Instruction::setSubclassData<Bitfield>(Value);
503 }
504
505 /// The synchronization scope ID of this fence instruction. Not quite enough
506 /// room in SubClassData for everything, so synchronization scope ID gets its
507 /// own field.
508 SyncScope::ID SSID;
509};
510
511//===----------------------------------------------------------------------===//
512// AtomicCmpXchgInst Class
513//===----------------------------------------------------------------------===//
514
515/// An instruction that atomically checks whether a
516/// specified value is in a memory location, and, if it is, stores a new value
517/// there. The value returned by this instruction is a pair containing the
518/// original value as first element, and an i1 indicating success (true) or
519/// failure (false) as second element.
520///
521class AtomicCmpXchgInst : public Instruction {
522 void Init(Value *Ptr, Value *Cmp, Value *NewVal, Align Align,
523 AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
524 SyncScope::ID SSID);
525
526 template <unsigned Offset>
527 using AtomicOrderingBitfieldElement =
528 typename Bitfield::Element<AtomicOrdering, Offset, 3,
529 AtomicOrdering::LAST>;
530
531protected:
532 // Note: Instruction needs to be a friend here to call cloneImpl.
533 friend class Instruction;
534
535 AtomicCmpXchgInst *cloneImpl() const;
536
537public:
538 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, Align Alignment,
539 AtomicOrdering SuccessOrdering,
540 AtomicOrdering FailureOrdering, SyncScope::ID SSID,
541 Instruction *InsertBefore = nullptr);
542 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, Align Alignment,
543 AtomicOrdering SuccessOrdering,
544 AtomicOrdering FailureOrdering, SyncScope::ID SSID,
545 BasicBlock *InsertAtEnd);
546
547 // allocate space for exactly three operands
548 void *operator new(size_t S) { return User::operator new(S, 3); }
549 void operator delete(void *Ptr) { User::operator delete(Ptr); }
550
551 using VolatileField = BoolBitfieldElementT<0>;
552 using WeakField = BoolBitfieldElementT<VolatileField::NextBit>;
553 using SuccessOrderingField =
554 AtomicOrderingBitfieldElementT<WeakField::NextBit>;
555 using FailureOrderingField =
556 AtomicOrderingBitfieldElementT<SuccessOrderingField::NextBit>;
557 using AlignmentField =
558 AlignmentBitfieldElementT<FailureOrderingField::NextBit>;
559 static_assert(
560 Bitfield::areContiguous<VolatileField, WeakField, SuccessOrderingField,
561 FailureOrderingField, AlignmentField>(),
562 "Bitfields must be contiguous");
563
564 /// Return the alignment of the memory that is being allocated by the
565 /// instruction.
566 Align getAlign() const {
567 return Align(1ULL << getSubclassData<AlignmentField>());
568 }
569
570 void setAlignment(Align Align) {
571 setSubclassData<AlignmentField>(Log2(Align));
572 }
573
574 /// Return true if this is a cmpxchg from a volatile memory
575 /// location.
576 ///
577 bool isVolatile() const { return getSubclassData<VolatileField>(); }
578
579 /// Specify whether this is a volatile cmpxchg.
580 ///
581 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
582
583 /// Return true if this cmpxchg may spuriously fail.
584 bool isWeak() const { return getSubclassData<WeakField>(); }
585
586 void setWeak(bool IsWeak) { setSubclassData<WeakField>(IsWeak); }
587
588 /// Transparently provide more efficient getOperand methods.
589 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
590
591 static bool isValidSuccessOrdering(AtomicOrdering Ordering) {
592 return Ordering != AtomicOrdering::NotAtomic &&
593 Ordering != AtomicOrdering::Unordered;
594 }
595
596 static bool isValidFailureOrdering(AtomicOrdering Ordering) {
597 return Ordering != AtomicOrdering::NotAtomic &&
598 Ordering != AtomicOrdering::Unordered &&
599 Ordering != AtomicOrdering::AcquireRelease &&
600 Ordering != AtomicOrdering::Release;
601 }
602
603 /// Returns the success ordering constraint of this cmpxchg instruction.
604 AtomicOrdering getSuccessOrdering() const {
605 return getSubclassData<SuccessOrderingField>();
606 }
607
608 /// Sets the success ordering constraint of this cmpxchg instruction.
609 void setSuccessOrdering(AtomicOrdering Ordering) {
610 assert(isValidSuccessOrdering(Ordering) &&((void)0)
611 "invalid CmpXchg success ordering")((void)0);
612 setSubclassData<SuccessOrderingField>(Ordering);
613 }
614
615 /// Returns the failure ordering constraint of this cmpxchg instruction.
616 AtomicOrdering getFailureOrdering() const {
617 return getSubclassData<FailureOrderingField>();
618 }
619
620 /// Sets the failure ordering constraint of this cmpxchg instruction.
621 void setFailureOrdering(AtomicOrdering Ordering) {
622 assert(isValidFailureOrdering(Ordering) &&((void)0)
623 "invalid CmpXchg failure ordering")((void)0);
624 setSubclassData<FailureOrderingField>(Ordering);
625 }
626
627 /// Returns a single ordering which is at least as strong as both the
628 /// success and failure orderings for this cmpxchg.
629 AtomicOrdering getMergedOrdering() const {
630 if (getFailureOrdering() == AtomicOrdering::SequentiallyConsistent)
631 return AtomicOrdering::SequentiallyConsistent;
632 if (getFailureOrdering() == AtomicOrdering::Acquire) {
633 if (getSuccessOrdering() == AtomicOrdering::Monotonic)
634 return AtomicOrdering::Acquire;
635 if (getSuccessOrdering() == AtomicOrdering::Release)
636 return AtomicOrdering::AcquireRelease;
637 }
638 return getSuccessOrdering();
639 }
640
641 /// Returns the synchronization scope ID of this cmpxchg instruction.
642 SyncScope::ID getSyncScopeID() const {
643 return SSID;
644 }
645
646 /// Sets the synchronization scope ID of this cmpxchg instruction.
647 void setSyncScopeID(SyncScope::ID SSID) {
648 this->SSID = SSID;
649 }
650
651 Value *getPointerOperand() { return getOperand(0); }
652 const Value *getPointerOperand() const { return getOperand(0); }
653 static unsigned getPointerOperandIndex() { return 0U; }
654
655 Value *getCompareOperand() { return getOperand(1); }
656 const Value *getCompareOperand() const { return getOperand(1); }
657
658 Value *getNewValOperand() { return getOperand(2); }
659 const Value *getNewValOperand() const { return getOperand(2); }
660
661 /// Returns the address space of the pointer operand.
662 unsigned getPointerAddressSpace() const {
663 return getPointerOperand()->getType()->getPointerAddressSpace();
664 }
665
666 /// Returns the strongest permitted ordering on failure, given the
667 /// desired ordering on success.
668 ///
669 /// If the comparison in a cmpxchg operation fails, there is no atomic store
670 /// so release semantics cannot be provided. So this function drops explicit
671 /// Release requests from the AtomicOrdering. A SequentiallyConsistent
672 /// operation would remain SequentiallyConsistent.
673 static AtomicOrdering
674 getStrongestFailureOrdering(AtomicOrdering SuccessOrdering) {
675 switch (SuccessOrdering) {
676 default:
677 llvm_unreachable("invalid cmpxchg success ordering")__builtin_unreachable();
678 case AtomicOrdering::Release:
679 case AtomicOrdering::Monotonic:
680 return AtomicOrdering::Monotonic;
681 case AtomicOrdering::AcquireRelease:
682 case AtomicOrdering::Acquire:
683 return AtomicOrdering::Acquire;
684 case AtomicOrdering::SequentiallyConsistent:
685 return AtomicOrdering::SequentiallyConsistent;
686 }
687 }
688
689 // Methods for support type inquiry through isa, cast, and dyn_cast:
690 static bool classof(const Instruction *I) {
691 return I->getOpcode() == Instruction::AtomicCmpXchg;
692 }
693 static bool classof(const Value *V) {
694 return isa<Instruction>(V) && classof(cast<Instruction>(V));
695 }
696
697private:
698 // Shadow Instruction::setInstructionSubclassData with a private forwarding
699 // method so that subclasses cannot accidentally use it.
700 template <typename Bitfield>
701 void setSubclassData(typename Bitfield::Type Value) {
702 Instruction::setSubclassData<Bitfield>(Value);
703 }
704
705 /// The synchronization scope ID of this cmpxchg instruction. Not quite
706 /// enough room in SubClassData for everything, so synchronization scope ID
707 /// gets its own field.
708 SyncScope::ID SSID;
709};
710
711template <>
712struct OperandTraits<AtomicCmpXchgInst> :
713 public FixedNumOperandTraits<AtomicCmpXchgInst, 3> {
714};
715
716DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicCmpXchgInst, Value)AtomicCmpXchgInst::op_iterator AtomicCmpXchgInst::op_begin() {
return OperandTraits<AtomicCmpXchgInst>::op_begin(this
); } AtomicCmpXchgInst::const_op_iterator AtomicCmpXchgInst::
op_begin() const { return OperandTraits<AtomicCmpXchgInst>
::op_begin(const_cast<AtomicCmpXchgInst*>(this)); } AtomicCmpXchgInst
::op_iterator AtomicCmpXchgInst::op_end() { return OperandTraits
<AtomicCmpXchgInst>::op_end(this); } AtomicCmpXchgInst::
const_op_iterator AtomicCmpXchgInst::op_end() const { return OperandTraits
<AtomicCmpXchgInst>::op_end(const_cast<AtomicCmpXchgInst
*>(this)); } Value *AtomicCmpXchgInst::getOperand(unsigned
i_nocapture) const { ((void)0); return cast_or_null<Value
>( OperandTraits<AtomicCmpXchgInst>::op_begin(const_cast
<AtomicCmpXchgInst*>(this))[i_nocapture].get()); } void
AtomicCmpXchgInst::setOperand(unsigned i_nocapture, Value *Val_nocapture
) { ((void)0); OperandTraits<AtomicCmpXchgInst>::op_begin
(this)[i_nocapture] = Val_nocapture; } unsigned AtomicCmpXchgInst
::getNumOperands() const { return OperandTraits<AtomicCmpXchgInst
>::operands(this); } template <int Idx_nocapture> Use
&AtomicCmpXchgInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
AtomicCmpXchgInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
717
718//===----------------------------------------------------------------------===//
719// AtomicRMWInst Class
720//===----------------------------------------------------------------------===//
721
722/// an instruction that atomically reads a memory location,
723/// combines it with another value, and then stores the result back. Returns
724/// the old value.
725///
726class AtomicRMWInst : public Instruction {
727protected:
728 // Note: Instruction needs to be a friend here to call cloneImpl.
729 friend class Instruction;
730
731 AtomicRMWInst *cloneImpl() const;
732
733public:
734 /// This enumeration lists the possible modifications atomicrmw can make. In
735 /// the descriptions, 'p' is the pointer to the instruction's memory location,
736 /// 'old' is the initial value of *p, and 'v' is the other value passed to the
737 /// instruction. These instructions always return 'old'.
738 enum BinOp : unsigned {
739 /// *p = v
740 Xchg,
741 /// *p = old + v
742 Add,
743 /// *p = old - v
744 Sub,
745 /// *p = old & v
746 And,
747 /// *p = ~(old & v)
748 Nand,
749 /// *p = old | v
750 Or,
751 /// *p = old ^ v
752 Xor,
753 /// *p = old >signed v ? old : v
754 Max,
755 /// *p = old <signed v ? old : v
756 Min,
757 /// *p = old >unsigned v ? old : v
758 UMax,
759 /// *p = old <unsigned v ? old : v
760 UMin,
761
762 /// *p = old + v
763 FAdd,
764
765 /// *p = old - v
766 FSub,
767
768 FIRST_BINOP = Xchg,
769 LAST_BINOP = FSub,
770 BAD_BINOP
771 };
772
773private:
774 template <unsigned Offset>
775 using AtomicOrderingBitfieldElement =
776 typename Bitfield::Element<AtomicOrdering, Offset, 3,
777 AtomicOrdering::LAST>;
778
779 template <unsigned Offset>
780 using BinOpBitfieldElement =
781 typename Bitfield::Element<BinOp, Offset, 4, BinOp::LAST_BINOP>;
782
783public:
784 AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, Align Alignment,
785 AtomicOrdering Ordering, SyncScope::ID SSID,
786 Instruction *InsertBefore = nullptr);
787 AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, Align Alignment,
788 AtomicOrdering Ordering, SyncScope::ID SSID,
789 BasicBlock *InsertAtEnd);
790
791 // allocate space for exactly two operands
792 void *operator new(size_t S) { return User::operator new(S, 2); }
793 void operator delete(void *Ptr) { User::operator delete(Ptr); }
794
795 using VolatileField = BoolBitfieldElementT<0>;
796 using AtomicOrderingField =
797 AtomicOrderingBitfieldElementT<VolatileField::NextBit>;
798 using OperationField = BinOpBitfieldElement<AtomicOrderingField::NextBit>;
799 using AlignmentField = AlignmentBitfieldElementT<OperationField::NextBit>;
800 static_assert(Bitfield::areContiguous<VolatileField, AtomicOrderingField,
801 OperationField, AlignmentField>(),
802 "Bitfields must be contiguous");
803
804 BinOp getOperation() const { return getSubclassData<OperationField>(); }
805
806 static StringRef getOperationName(BinOp Op);
807
808 static bool isFPOperation(BinOp Op) {
809 switch (Op) {
810 case AtomicRMWInst::FAdd:
811 case AtomicRMWInst::FSub:
812 return true;
813 default:
814 return false;
815 }
816 }
817
818 void setOperation(BinOp Operation) {
819 setSubclassData<OperationField>(Operation);
820 }
821
822 /// Return the alignment of the memory that is being allocated by the
823 /// instruction.
824 Align getAlign() const {
825 return Align(1ULL << getSubclassData<AlignmentField>());
826 }
827
828 void setAlignment(Align Align) {
829 setSubclassData<AlignmentField>(Log2(Align));
830 }
831
832 /// Return true if this is a RMW on a volatile memory location.
833 ///
834 bool isVolatile() const { return getSubclassData<VolatileField>(); }
835
836 /// Specify whether this is a volatile RMW or not.
837 ///
838 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
839
840 /// Transparently provide more efficient getOperand methods.
841 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
842
843 /// Returns the ordering constraint of this rmw instruction.
844 AtomicOrdering getOrdering() const {
845 return getSubclassData<AtomicOrderingField>();
846 }
847
848 /// Sets the ordering constraint of this rmw instruction.
849 void setOrdering(AtomicOrdering Ordering) {
850 assert(Ordering != AtomicOrdering::NotAtomic &&((void)0)
851 "atomicrmw instructions can only be atomic.")((void)0);
852 setSubclassData<AtomicOrderingField>(Ordering);
853 }
854
855 /// Returns the synchronization scope ID of this rmw instruction.
856 SyncScope::ID getSyncScopeID() const {
857 return SSID;
858 }
859
860 /// Sets the synchronization scope ID of this rmw instruction.
861 void setSyncScopeID(SyncScope::ID SSID) {
862 this->SSID = SSID;
863 }
864
865 Value *getPointerOperand() { return getOperand(0); }
866 const Value *getPointerOperand() const { return getOperand(0); }
867 static unsigned getPointerOperandIndex() { return 0U; }
868
869 Value *getValOperand() { return getOperand(1); }
870 const Value *getValOperand() const { return getOperand(1); }
871
872 /// Returns the address space of the pointer operand.
873 unsigned getPointerAddressSpace() const {
874 return getPointerOperand()->getType()->getPointerAddressSpace();
875 }
876
877 bool isFloatingPointOperation() const {
878 return isFPOperation(getOperation());
879 }
880
881 // Methods for support type inquiry through isa, cast, and dyn_cast:
882 static bool classof(const Instruction *I) {
883 return I->getOpcode() == Instruction::AtomicRMW;
884 }
885 static bool classof(const Value *V) {
886 return isa<Instruction>(V) && classof(cast<Instruction>(V));
887 }
888
889private:
890 void Init(BinOp Operation, Value *Ptr, Value *Val, Align Align,
891 AtomicOrdering Ordering, SyncScope::ID SSID);
892
893 // Shadow Instruction::setInstructionSubclassData with a private forwarding
894 // method so that subclasses cannot accidentally use it.
895 template <typename Bitfield>
896 void setSubclassData(typename Bitfield::Type Value) {
897 Instruction::setSubclassData<Bitfield>(Value);
898 }
899
900 /// The synchronization scope ID of this rmw instruction. Not quite enough
901 /// room in SubClassData for everything, so synchronization scope ID gets its
902 /// own field.
903 SyncScope::ID SSID;
904};
905
906template <>
907struct OperandTraits<AtomicRMWInst>
908 : public FixedNumOperandTraits<AtomicRMWInst,2> {
909};
910
911DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicRMWInst, Value)AtomicRMWInst::op_iterator AtomicRMWInst::op_begin() { return
OperandTraits<AtomicRMWInst>::op_begin(this); } AtomicRMWInst
::const_op_iterator AtomicRMWInst::op_begin() const { return OperandTraits
<AtomicRMWInst>::op_begin(const_cast<AtomicRMWInst*>
(this)); } AtomicRMWInst::op_iterator AtomicRMWInst::op_end()
{ return OperandTraits<AtomicRMWInst>::op_end(this); }
AtomicRMWInst::const_op_iterator AtomicRMWInst::op_end() const
{ return OperandTraits<AtomicRMWInst>::op_end(const_cast
<AtomicRMWInst*>(this)); } Value *AtomicRMWInst::getOperand
(unsigned i_nocapture) const { ((void)0); return cast_or_null
<Value>( OperandTraits<AtomicRMWInst>::op_begin(const_cast
<AtomicRMWInst*>(this))[i_nocapture].get()); } void AtomicRMWInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((
void)0); OperandTraits<AtomicRMWInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned AtomicRMWInst::getNumOperands()
const { return OperandTraits<AtomicRMWInst>::operands(
this); } template <int Idx_nocapture> Use &AtomicRMWInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &AtomicRMWInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
912
913//===----------------------------------------------------------------------===//
914// GetElementPtrInst Class
915//===----------------------------------------------------------------------===//
916
917// checkGEPType - Simple wrapper function to give a better assertion failure
918// message on bad indexes for a gep instruction.
919//
920inline Type *checkGEPType(Type *Ty) {
921 assert(Ty && "Invalid GetElementPtrInst indices for type!")((void)0);
922 return Ty;
923}
924
925/// an instruction for type-safe pointer arithmetic to
926/// access elements of arrays and structs
927///
928class GetElementPtrInst : public Instruction {
929 Type *SourceElementType;
930 Type *ResultElementType;
931
932 GetElementPtrInst(const GetElementPtrInst &GEPI);
933
934 /// Constructors - Create a getelementptr instruction with a base pointer an
935 /// list of indices. The first ctor can optionally insert before an existing
936 /// instruction, the second appends the new instruction to the specified
937 /// BasicBlock.
938 inline GetElementPtrInst(Type *PointeeType, Value *Ptr,
939 ArrayRef<Value *> IdxList, unsigned Values,
940 const Twine &NameStr, Instruction *InsertBefore);
941 inline GetElementPtrInst(Type *PointeeType, Value *Ptr,
942 ArrayRef<Value *> IdxList, unsigned Values,
943 const Twine &NameStr, BasicBlock *InsertAtEnd);
944
945 void init(Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr);
946
947protected:
948 // Note: Instruction needs to be a friend here to call cloneImpl.
949 friend class Instruction;
950
951 GetElementPtrInst *cloneImpl() const;
952
953public:
954 static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
955 ArrayRef<Value *> IdxList,
956 const Twine &NameStr = "",
957 Instruction *InsertBefore = nullptr) {
958 unsigned Values = 1 + unsigned(IdxList.size());
959 assert(PointeeType && "Must specify element type")((void)0);
960 assert(cast<PointerType>(Ptr->getType()->getScalarType())((void)0)
961 ->isOpaqueOrPointeeTypeMatches(PointeeType))((void)0);
962 return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
963 NameStr, InsertBefore);
964 }
965
966 static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
967 ArrayRef<Value *> IdxList,
968 const Twine &NameStr,
969 BasicBlock *InsertAtEnd) {
970 unsigned Values = 1 + unsigned(IdxList.size());
971 assert(PointeeType && "Must specify element type")((void)0);
972 assert(cast<PointerType>(Ptr->getType()->getScalarType())((void)0)
973 ->isOpaqueOrPointeeTypeMatches(PointeeType))((void)0);
974 return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
975 NameStr, InsertAtEnd);
976 }
977
978 LLVM_ATTRIBUTE_DEPRECATED(static GetElementPtrInst *CreateInBounds([[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr = "", Instruction
*InsertBefore = nullptr)
979 Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr = "",[[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr = "", Instruction
*InsertBefore = nullptr)
980 Instruction *InsertBefore = nullptr),[[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr = "", Instruction
*InsertBefore = nullptr)
981 "Use the version with explicit element type instead")[[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr = "", Instruction
*InsertBefore = nullptr)
{
982 return CreateInBounds(
983 Ptr->getType()->getScalarType()->getPointerElementType(), Ptr, IdxList,
984 NameStr, InsertBefore);
985 }
986
987 /// Create an "inbounds" getelementptr. See the documentation for the
988 /// "inbounds" flag in LangRef.html for details.
989 static GetElementPtrInst *
990 CreateInBounds(Type *PointeeType, Value *Ptr, ArrayRef<Value *> IdxList,
991 const Twine &NameStr = "",
992 Instruction *InsertBefore = nullptr) {
993 GetElementPtrInst *GEP =
994 Create(PointeeType, Ptr, IdxList, NameStr, InsertBefore);
995 GEP->setIsInBounds(true);
996 return GEP;
997 }
998
999 LLVM_ATTRIBUTE_DEPRECATED(static GetElementPtrInst *CreateInBounds([[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr, BasicBlock
*InsertAtEnd)
1000 Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr,[[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr, BasicBlock
*InsertAtEnd)
1001 BasicBlock *InsertAtEnd),[[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr, BasicBlock
*InsertAtEnd)
1002 "Use the version with explicit element type instead")[[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr, BasicBlock
*InsertAtEnd)
{
1003 return CreateInBounds(
1004 Ptr->getType()->getScalarType()->getPointerElementType(), Ptr, IdxList,
1005 NameStr, InsertAtEnd);
1006 }
1007
1008 static GetElementPtrInst *CreateInBounds(Type *PointeeType, Value *Ptr,
1009 ArrayRef<Value *> IdxList,
1010 const Twine &NameStr,
1011 BasicBlock *InsertAtEnd) {
1012 GetElementPtrInst *GEP =
1013 Create(PointeeType, Ptr, IdxList, NameStr, InsertAtEnd);
1014 GEP->setIsInBounds(true);
1015 return GEP;
1016 }
1017
1018 /// Transparently provide more efficient getOperand methods.
1019 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1020
1021 Type *getSourceElementType() const { return SourceElementType; }
1022
1023 void setSourceElementType(Type *Ty) { SourceElementType = Ty; }
1024 void setResultElementType(Type *Ty) { ResultElementType = Ty; }
1025
1026 Type *getResultElementType() const {
1027 assert(cast<PointerType>(getType()->getScalarType())((void)0)
1028 ->isOpaqueOrPointeeTypeMatches(ResultElementType))((void)0);
1029 return ResultElementType;
1030 }
1031
1032 /// Returns the address space of this instruction's pointer type.
1033 unsigned getAddressSpace() const {
1034 // Note that this is always the same as the pointer operand's address space
1035 // and that is cheaper to compute, so cheat here.
1036 return getPointerAddressSpace();
1037 }
1038
1039 /// Returns the result type of a getelementptr with the given source
1040 /// element type and indexes.
1041 ///
1042 /// Null is returned if the indices are invalid for the specified
1043 /// source element type.
1044 static Type *getIndexedType(Type *Ty, ArrayRef<Value *> IdxList);
1045 static Type *getIndexedType(Type *Ty, ArrayRef<Constant *> IdxList);
1046 static Type *getIndexedType(Type *Ty, ArrayRef<uint64_t> IdxList);
1047
1048 /// Return the type of the element at the given index of an indexable
1049 /// type. This is equivalent to "getIndexedType(Agg, {Zero, Idx})".
1050 ///
1051 /// Returns null if the type can't be indexed, or the given index is not
1052 /// legal for the given type.
1053 static Type *getTypeAtIndex(Type *Ty, Value *Idx);
1054 static Type *getTypeAtIndex(Type *Ty, uint64_t Idx);
1055
1056 inline op_iterator idx_begin() { return op_begin()+1; }
1057 inline const_op_iterator idx_begin() const { return op_begin()+1; }
1058 inline op_iterator idx_end() { return op_end(); }
1059 inline const_op_iterator idx_end() const { return op_end(); }
1060
1061 inline iterator_range<op_iterator> indices() {
1062 return make_range(idx_begin(), idx_end());
1063 }
1064
1065 inline iterator_range<const_op_iterator> indices() const {
1066 return make_range(idx_begin(), idx_end());
1067 }
1068
1069 Value *getPointerOperand() {
1070 return getOperand(0);
1071 }
1072 const Value *getPointerOperand() const {
1073 return getOperand(0);
1074 }
1075 static unsigned getPointerOperandIndex() {
1076 return 0U; // get index for modifying correct operand.
1077 }
1078
1079 /// Method to return the pointer operand as a
1080 /// PointerType.
1081 Type *getPointerOperandType() const {
1082 return getPointerOperand()->getType();
1083 }
1084
1085 /// Returns the address space of the pointer operand.
1086 unsigned getPointerAddressSpace() const {
1087 return getPointerOperandType()->getPointerAddressSpace();
1088 }
1089
1090 /// Returns the pointer type returned by the GEP
1091 /// instruction, which may be a vector of pointers.
1092 static Type *getGEPReturnType(Type *ElTy, Value *Ptr,
1093 ArrayRef<Value *> IdxList) {
1094 PointerType *OrigPtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
1095 unsigned AddrSpace = OrigPtrTy->getAddressSpace();
1096 Type *ResultElemTy = checkGEPType(getIndexedType(ElTy, IdxList));
1097 Type *PtrTy = OrigPtrTy->isOpaque()
1098 ? PointerType::get(OrigPtrTy->getContext(), AddrSpace)
1099 : PointerType::get(ResultElemTy, AddrSpace);
1100 // Vector GEP
1101 if (auto *PtrVTy = dyn_cast<VectorType>(Ptr->getType())) {
1102 ElementCount EltCount = PtrVTy->getElementCount();
1103 return VectorType::get(PtrTy, EltCount);
1104 }
1105 for (Value *Index : IdxList)
1106 if (auto *IndexVTy = dyn_cast<VectorType>(Index->getType())) {
1107 ElementCount EltCount = IndexVTy->getElementCount();
1108 return VectorType::get(PtrTy, EltCount);
1109 }
1110 // Scalar GEP
1111 return PtrTy;
1112 }
1113
1114 unsigned getNumIndices() const { // Note: always non-negative
1115 return getNumOperands() - 1;
1116 }
1117
1118 bool hasIndices() const {
1119 return getNumOperands() > 1;
1120 }
1121
1122 /// Return true if all of the indices of this GEP are
1123 /// zeros. If so, the result pointer and the first operand have the same
1124 /// value, just potentially different types.
1125 bool hasAllZeroIndices() const;
1126
1127 /// Return true if all of the indices of this GEP are
1128 /// constant integers. If so, the result pointer and the first operand have
1129 /// a constant offset between them.
1130 bool hasAllConstantIndices() const;
1131
1132 /// Set or clear the inbounds flag on this GEP instruction.
1133 /// See LangRef.html for the meaning of inbounds on a getelementptr.
1134 void setIsInBounds(bool b = true);
1135
1136 /// Determine whether the GEP has the inbounds flag.
1137 bool isInBounds() const;
1138
1139 /// Accumulate the constant address offset of this GEP if possible.
1140 ///
1141 /// This routine accepts an APInt into which it will accumulate the constant
1142 /// offset of this GEP if the GEP is in fact constant. If the GEP is not
1143 /// all-constant, it returns false and the value of the offset APInt is
1144 /// undefined (it is *not* preserved!). The APInt passed into this routine
1145 /// must be at least as wide as the IntPtr type for the address space of
1146 /// the base GEP pointer.
1147 bool accumulateConstantOffset(const DataLayout &DL, APInt &Offset) const;
1148 bool collectOffset(const DataLayout &DL, unsigned BitWidth,
1149 MapVector<Value *, APInt> &VariableOffsets,
1150 APInt &ConstantOffset) const;
1151 // Methods for support type inquiry through isa, cast, and dyn_cast:
1152 static bool classof(const Instruction *I) {
1153 return (I->getOpcode() == Instruction::GetElementPtr);
1154 }
1155 static bool classof(const Value *V) {
1156 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1157 }
1158};
1159
1160template <>
1161struct OperandTraits<GetElementPtrInst> :
1162 public VariadicOperandTraits<GetElementPtrInst, 1> {
1163};
1164
1165GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
1166 ArrayRef<Value *> IdxList, unsigned Values,
1167 const Twine &NameStr,
1168 Instruction *InsertBefore)
1169 : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
1170 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
1171 Values, InsertBefore),
1172 SourceElementType(PointeeType),
1173 ResultElementType(getIndexedType(PointeeType, IdxList)) {
1174 assert(cast<PointerType>(getType()->getScalarType())((void)0)
1175 ->isOpaqueOrPointeeTypeMatches(ResultElementType))((void)0);
1176 init(Ptr, IdxList, NameStr);
1177}
1178
1179GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
1180 ArrayRef<Value *> IdxList, unsigned Values,
1181 const Twine &NameStr,
1182 BasicBlock *InsertAtEnd)
1183 : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
1184 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
1185 Values, InsertAtEnd),
1186 SourceElementType(PointeeType),
1187 ResultElementType(getIndexedType(PointeeType, IdxList)) {
1188 assert(cast<PointerType>(getType()->getScalarType())((void)0)
1189 ->isOpaqueOrPointeeTypeMatches(ResultElementType))((void)0);
1190 init(Ptr, IdxList, NameStr);
1191}
1192
1193DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GetElementPtrInst, Value)GetElementPtrInst::op_iterator GetElementPtrInst::op_begin() {
return OperandTraits<GetElementPtrInst>::op_begin(this
); } GetElementPtrInst::const_op_iterator GetElementPtrInst::
op_begin() const { return OperandTraits<GetElementPtrInst>
::op_begin(const_cast<GetElementPtrInst*>(this)); } GetElementPtrInst
::op_iterator GetElementPtrInst::op_end() { return OperandTraits
<GetElementPtrInst>::op_end(this); } GetElementPtrInst::
const_op_iterator GetElementPtrInst::op_end() const { return OperandTraits
<GetElementPtrInst>::op_end(const_cast<GetElementPtrInst
*>(this)); } Value *GetElementPtrInst::getOperand(unsigned
i_nocapture) const { ((void)0); return cast_or_null<Value
>( OperandTraits<GetElementPtrInst>::op_begin(const_cast
<GetElementPtrInst*>(this))[i_nocapture].get()); } void
GetElementPtrInst::setOperand(unsigned i_nocapture, Value *Val_nocapture
) { ((void)0); OperandTraits<GetElementPtrInst>::op_begin
(this)[i_nocapture] = Val_nocapture; } unsigned GetElementPtrInst
::getNumOperands() const { return OperandTraits<GetElementPtrInst
>::operands(this); } template <int Idx_nocapture> Use
&GetElementPtrInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
GetElementPtrInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
1194
1195//===----------------------------------------------------------------------===//
1196// ICmpInst Class
1197//===----------------------------------------------------------------------===//
1198
1199/// This instruction compares its operands according to the predicate given
1200/// to the constructor. It only operates on integers or pointers. The operands
1201/// must be identical types.
1202/// Represent an integer comparison operator.
1203class ICmpInst: public CmpInst {
1204 void AssertOK() {
1205 assert(isIntPredicate() &&((void)0)
1206 "Invalid ICmp predicate value")((void)0);
1207 assert(getOperand(0)->getType() == getOperand(1)->getType() &&((void)0)
1208 "Both operands to ICmp instruction are not of the same type!")((void)0);
1209 // Check that the operands are the right type
1210 assert((getOperand(0)->getType()->isIntOrIntVectorTy() ||((void)0)
1211 getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&((void)0)
1212 "Invalid operand types for ICmp instruction")((void)0);
1213 }
1214
1215protected:
1216 // Note: Instruction needs to be a friend here to call cloneImpl.
1217 friend class Instruction;
1218
1219 /// Clone an identical ICmpInst
1220 ICmpInst *cloneImpl() const;
1221
1222public:
1223 /// Constructor with insert-before-instruction semantics.
1224 ICmpInst(
1225 Instruction *InsertBefore, ///< Where to insert
1226 Predicate pred, ///< The predicate to use for the comparison
1227 Value *LHS, ///< The left-hand-side of the expression
1228 Value *RHS, ///< The right-hand-side of the expression
1229 const Twine &NameStr = "" ///< Name of the instruction
1230 ) : CmpInst(makeCmpResultType(LHS->getType()),
1231 Instruction::ICmp, pred, LHS, RHS, NameStr,
1232 InsertBefore) {
1233#ifndef NDEBUG1
1234 AssertOK();
1235#endif
1236 }
1237
1238 /// Constructor with insert-at-end semantics.
1239 ICmpInst(
1240 BasicBlock &InsertAtEnd, ///< Block to insert into.
1241 Predicate pred, ///< The predicate to use for the comparison
1242 Value *LHS, ///< The left-hand-side of the expression
1243 Value *RHS, ///< The right-hand-side of the expression
1244 const Twine &NameStr = "" ///< Name of the instruction
1245 ) : CmpInst(makeCmpResultType(LHS->getType()),
1246 Instruction::ICmp, pred, LHS, RHS, NameStr,
1247 &InsertAtEnd) {
1248#ifndef NDEBUG1
1249 AssertOK();
1250#endif
1251 }
1252
1253 /// Constructor with no-insertion semantics
1254 ICmpInst(
1255 Predicate pred, ///< The predicate to use for the comparison
1256 Value *LHS, ///< The left-hand-side of the expression
1257 Value *RHS, ///< The right-hand-side of the expression
1258 const Twine &NameStr = "" ///< Name of the instruction
1259 ) : CmpInst(makeCmpResultType(LHS->getType()),
1260 Instruction::ICmp, pred, LHS, RHS, NameStr) {
1261#ifndef NDEBUG1
1262 AssertOK();
1263#endif
1264 }
1265
1266 /// For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
1267 /// @returns the predicate that would be the result if the operand were
1268 /// regarded as signed.
1269 /// Return the signed version of the predicate
1270 Predicate getSignedPredicate() const {
1271 return getSignedPredicate(getPredicate());
1272 }
1273
1274 /// This is a static version that you can use without an instruction.
1275 /// Return the signed version of the predicate.
1276 static Predicate getSignedPredicate(Predicate pred);
1277
1278 /// For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
1279 /// @returns the predicate that would be the result if the operand were
1280 /// regarded as unsigned.
1281 /// Return the unsigned version of the predicate
1282 Predicate getUnsignedPredicate() const {
1283 return getUnsignedPredicate(getPredicate());
1284 }
1285
1286 /// This is a static version that you can use without an instruction.
1287 /// Return the unsigned version of the predicate.
1288 static Predicate getUnsignedPredicate(Predicate pred);
1289
1290 /// Return true if this predicate is either EQ or NE. This also
1291 /// tests for commutativity.
1292 static bool isEquality(Predicate P) {
1293 return P == ICMP_EQ || P == ICMP_NE;
1294 }
1295
1296 /// Return true if this predicate is either EQ or NE. This also
1297 /// tests for commutativity.
1298 bool isEquality() const {
1299 return isEquality(getPredicate());
1300 }
1301
1302 /// @returns true if the predicate of this ICmpInst is commutative
1303 /// Determine if this relation is commutative.
1304 bool isCommutative() const { return isEquality(); }
1305
1306 /// Return true if the predicate is relational (not EQ or NE).
1307 ///
1308 bool isRelational() const {
1309 return !isEquality();
1310 }
1311
1312 /// Return true if the predicate is relational (not EQ or NE).
1313 ///
1314 static bool isRelational(Predicate P) {
1315 return !isEquality(P);
1316 }
1317
1318 /// Return true if the predicate is SGT or UGT.
1319 ///
1320 static bool isGT(Predicate P) {
1321 return P == ICMP_SGT || P == ICMP_UGT;
1322 }
1323
1324 /// Return true if the predicate is SLT or ULT.
1325 ///
1326 static bool isLT(Predicate P) {
1327 return P == ICMP_SLT || P == ICMP_ULT;
1328 }
1329
1330 /// Return true if the predicate is SGE or UGE.
1331 ///
1332 static bool isGE(Predicate P) {
1333 return P == ICMP_SGE || P == ICMP_UGE;
1334 }
1335
1336 /// Return true if the predicate is SLE or ULE.
1337 ///
1338 static bool isLE(Predicate P) {
1339 return P == ICMP_SLE || P == ICMP_ULE;
1340 }
1341
1342 /// Exchange the two operands to this instruction in such a way that it does
1343 /// not modify the semantics of the instruction. The predicate value may be
1344 /// changed to retain the same result if the predicate is order dependent
1345 /// (e.g. ult).
1346 /// Swap operands and adjust predicate.
1347 void swapOperands() {
1348 setPredicate(getSwappedPredicate());
1349 Op<0>().swap(Op<1>());
1350 }
1351
1352 // Methods for support type inquiry through isa, cast, and dyn_cast:
1353 static bool classof(const Instruction *I) {
1354 return I->getOpcode() == Instruction::ICmp;
1355 }
1356 static bool classof(const Value *V) {
1357 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1358 }
1359};
1360
1361//===----------------------------------------------------------------------===//
1362// FCmpInst Class
1363//===----------------------------------------------------------------------===//
1364
1365/// This instruction compares its operands according to the predicate given
1366/// to the constructor. It only operates on floating point values or packed
1367/// vectors of floating point values. The operands must be identical types.
1368/// Represents a floating point comparison operator.
1369class FCmpInst: public CmpInst {
1370 void AssertOK() {
1371 assert(isFPPredicate() && "Invalid FCmp predicate value")((void)0);
1372 assert(getOperand(0)->getType() == getOperand(1)->getType() &&((void)0)
1373 "Both operands to FCmp instruction are not of the same type!")((void)0);
1374 // Check that the operands are the right type
1375 assert(getOperand(0)->getType()->isFPOrFPVectorTy() &&((void)0)
1376 "Invalid operand types for FCmp instruction")((void)0);
1377 }
1378
1379protected:
1380 // Note: Instruction needs to be a friend here to call cloneImpl.
1381 friend class Instruction;
1382
1383 /// Clone an identical FCmpInst
1384 FCmpInst *cloneImpl() const;
1385
1386public:
1387 /// Constructor with insert-before-instruction semantics.
1388 FCmpInst(
1389 Instruction *InsertBefore, ///< Where to insert
1390 Predicate pred, ///< The predicate to use for the comparison
1391 Value *LHS, ///< The left-hand-side of the expression
1392 Value *RHS, ///< The right-hand-side of the expression
1393 const Twine &NameStr = "" ///< Name of the instruction
1394 ) : CmpInst(makeCmpResultType(LHS->getType()),
1395 Instruction::FCmp, pred, LHS, RHS, NameStr,
1396 InsertBefore) {
1397 AssertOK();
1398 }
1399
1400 /// Constructor with insert-at-end semantics.
1401 FCmpInst(
1402 BasicBlock &InsertAtEnd, ///< Block to insert into.
1403 Predicate pred, ///< The predicate to use for the comparison
1404 Value *LHS, ///< The left-hand-side of the expression
1405 Value *RHS, ///< The right-hand-side of the expression
1406 const Twine &NameStr = "" ///< Name of the instruction
1407 ) : CmpInst(makeCmpResultType(LHS->getType()),
1408 Instruction::FCmp, pred, LHS, RHS, NameStr,
1409 &InsertAtEnd) {
1410 AssertOK();
1411 }
1412
1413 /// Constructor with no-insertion semantics
1414 FCmpInst(
1415 Predicate Pred, ///< The predicate to use for the comparison
1416 Value *LHS, ///< The left-hand-side of the expression
1417 Value *RHS, ///< The right-hand-side of the expression
1418 const Twine &NameStr = "", ///< Name of the instruction
1419 Instruction *FlagsSource = nullptr
1420 ) : CmpInst(makeCmpResultType(LHS->getType()), Instruction::FCmp, Pred, LHS,
1421 RHS, NameStr, nullptr, FlagsSource) {
1422 AssertOK();
1423 }
1424
1425 /// @returns true if the predicate of this instruction is EQ or NE.
1426 /// Determine if this is an equality predicate.
1427 static bool isEquality(Predicate Pred) {
1428 return Pred == FCMP_OEQ || Pred == FCMP_ONE || Pred == FCMP_UEQ ||
1429 Pred == FCMP_UNE;
1430 }
1431
1432 /// @returns true if the predicate of this instruction is EQ or NE.
1433 /// Determine if this is an equality predicate.
1434 bool isEquality() const { return isEquality(getPredicate()); }
1435
1436 /// @returns true if the predicate of this instruction is commutative.
1437 /// Determine if this is a commutative predicate.
1438 bool isCommutative() const {
1439 return isEquality() ||
1440 getPredicate() == FCMP_FALSE ||
1441 getPredicate() == FCMP_TRUE ||
1442 getPredicate() == FCMP_ORD ||
1443 getPredicate() == FCMP_UNO;
1444 }
1445
1446 /// @returns true if the predicate is relational (not EQ or NE).
1447 /// Determine if this a relational predicate.
1448 bool isRelational() const { return !isEquality(); }
1449
1450 /// Exchange the two operands to this instruction in such a way that it does
1451 /// not modify the semantics of the instruction. The predicate value may be
1452 /// changed to retain the same result if the predicate is order dependent
1453 /// (e.g. ult).
1454 /// Swap operands and adjust predicate.
1455 void swapOperands() {
1456 setPredicate(getSwappedPredicate());
1457 Op<0>().swap(Op<1>());
1458 }
1459
1460 /// Methods for support type inquiry through isa, cast, and dyn_cast:
1461 static bool classof(const Instruction *I) {
1462 return I->getOpcode() == Instruction::FCmp;
1463 }
1464 static bool classof(const Value *V) {
1465 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1466 }
1467};
1468
1469//===----------------------------------------------------------------------===//
1470/// This class represents a function call, abstracting a target
1471/// machine's calling convention. This class uses low bit of the SubClassData
1472/// field to indicate whether or not this is a tail call. The rest of the bits
1473/// hold the calling convention of the call.
1474///
1475class CallInst : public CallBase {
1476 CallInst(const CallInst &CI);
1477
1478 /// Construct a CallInst given a range of arguments.
1479 /// Construct a CallInst from a range of arguments
1480 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1481 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1482 Instruction *InsertBefore);
1483
1484 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1485 const Twine &NameStr, Instruction *InsertBefore)
1486 : CallInst(Ty, Func, Args, None, NameStr, InsertBefore) {}
1487
1488 /// Construct a CallInst given a range of arguments.
1489 /// Construct a CallInst from a range of arguments
1490 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1491 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1492 BasicBlock *InsertAtEnd);
1493
1494 explicit CallInst(FunctionType *Ty, Value *F, const Twine &NameStr,
1495 Instruction *InsertBefore);
1496
1497 CallInst(FunctionType *ty, Value *F, const Twine &NameStr,
1498 BasicBlock *InsertAtEnd);
1499
1500 void init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
1501 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
1502 void init(FunctionType *FTy, Value *Func, const Twine &NameStr);
1503
1504 /// Compute the number of operands to allocate.
1505 static int ComputeNumOperands(int NumArgs, int NumBundleInputs = 0) {
1506 // We need one operand for the called function, plus the input operand
1507 // counts provided.
1508 return 1 + NumArgs + NumBundleInputs;
1509 }
1510
1511protected:
1512 // Note: Instruction needs to be a friend here to call cloneImpl.
1513 friend class Instruction;
1514
1515 CallInst *cloneImpl() const;
1516
1517public:
1518 static CallInst *Create(FunctionType *Ty, Value *F, const Twine &NameStr = "",
1519 Instruction *InsertBefore = nullptr) {
1520 return new (ComputeNumOperands(0)) CallInst(Ty, F, NameStr, InsertBefore);
1521 }
1522
1523 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1524 const Twine &NameStr,
1525 Instruction *InsertBefore = nullptr) {
1526 return new (ComputeNumOperands(Args.size()))
1527 CallInst(Ty, Func, Args, None, NameStr, InsertBefore);
1528 }
1529
1530 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1531 ArrayRef<OperandBundleDef> Bundles = None,
1532 const Twine &NameStr = "",
1533 Instruction *InsertBefore = nullptr) {
1534 const int NumOperands =
1535 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
1536 const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
1537
1538 return new (NumOperands, DescriptorBytes)
1539 CallInst(Ty, Func, Args, Bundles, NameStr, InsertBefore);
1540 }
1541
1542 static CallInst *Create(FunctionType *Ty, Value *F, const Twine &NameStr,
1543 BasicBlock *InsertAtEnd) {
1544 return new (ComputeNumOperands(0)) CallInst(Ty, F, NameStr, InsertAtEnd);
1545 }
1546
1547 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1548 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1549 return new (ComputeNumOperands(Args.size()))
1550 CallInst(Ty, Func, Args, None, NameStr, InsertAtEnd);
1551 }
1552
1553 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1554 ArrayRef<OperandBundleDef> Bundles,
1555 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1556 const int NumOperands =
1557 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
1558 const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
1559
1560 return new (NumOperands, DescriptorBytes)
1561 CallInst(Ty, Func, Args, Bundles, NameStr, InsertAtEnd);
1562 }
1563
1564 static CallInst *Create(FunctionCallee Func, const Twine &NameStr = "",
1565 Instruction *InsertBefore = nullptr) {
1566 return Create(Func.getFunctionType(), Func.getCallee(), NameStr,
1567 InsertBefore);
1568 }
1569
1570 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1571 ArrayRef<OperandBundleDef> Bundles = None,
1572 const Twine &NameStr = "",
1573 Instruction *InsertBefore = nullptr) {
1574 return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles,
1575 NameStr, InsertBefore);
1576 }
1577
1578 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1579 const Twine &NameStr,
1580 Instruction *InsertBefore = nullptr) {
1581 return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr,
1582 InsertBefore);
1583 }
1584
1585 static CallInst *Create(FunctionCallee Func, const Twine &NameStr,
1586 BasicBlock *InsertAtEnd) {
1587 return Create(Func.getFunctionType(), Func.getCallee(), NameStr,
1588 InsertAtEnd);
1589 }
1590
1591 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1592 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1593 return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr,
1594 InsertAtEnd);
1595 }
1596
1597 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1598 ArrayRef<OperandBundleDef> Bundles,
1599 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1600 return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles,
1601 NameStr, InsertAtEnd);
1602 }
1603
1604 /// Create a clone of \p CI with a different set of operand bundles and
1605 /// insert it before \p InsertPt.
1606 ///
1607 /// The returned call instruction is identical \p CI in every way except that
1608 /// the operand bundles for the new instruction are set to the operand bundles
1609 /// in \p Bundles.
1610 static CallInst *Create(CallInst *CI, ArrayRef<OperandBundleDef> Bundles,
1611 Instruction *InsertPt = nullptr);
1612
1613 /// Generate the IR for a call to malloc:
1614 /// 1. Compute the malloc call's argument as the specified type's size,
1615 /// possibly multiplied by the array size if the array size is not
1616 /// constant 1.
1617 /// 2. Call malloc with that argument.
1618 /// 3. Bitcast the result of the malloc call to the specified type.
1619 static Instruction *CreateMalloc(Instruction *InsertBefore, Type *IntPtrTy,
1620 Type *AllocTy, Value *AllocSize,
1621 Value *ArraySize = nullptr,
1622 Function *MallocF = nullptr,
1623 const Twine &Name = "");
1624 static Instruction *CreateMalloc(BasicBlock *InsertAtEnd, Type *IntPtrTy,
1625 Type *AllocTy, Value *AllocSize,
1626 Value *ArraySize = nullptr,
1627 Function *MallocF = nullptr,
1628 const Twine &Name = "");
1629 static Instruction *CreateMalloc(Instruction *InsertBefore, Type *IntPtrTy,
1630 Type *AllocTy, Value *AllocSize,
1631 Value *ArraySize = nullptr,
1632 ArrayRef<OperandBundleDef> Bundles = None,
1633 Function *MallocF = nullptr,
1634 const Twine &Name = "");
1635 static Instruction *CreateMalloc(BasicBlock *InsertAtEnd, Type *IntPtrTy,
1636 Type *AllocTy, Value *AllocSize,
1637 Value *ArraySize = nullptr,
1638 ArrayRef<OperandBundleDef> Bundles = None,
1639 Function *MallocF = nullptr,
1640 const Twine &Name = "");
1641 /// Generate the IR for a call to the builtin free function.
1642 static Instruction *CreateFree(Value *Source, Instruction *InsertBefore);
1643 static Instruction *CreateFree(Value *Source, BasicBlock *InsertAtEnd);
1644 static Instruction *CreateFree(Value *Source,
1645 ArrayRef<OperandBundleDef> Bundles,
1646 Instruction *InsertBefore);
1647 static Instruction *CreateFree(Value *Source,
1648 ArrayRef<OperandBundleDef> Bundles,
1649 BasicBlock *InsertAtEnd);
1650
1651 // Note that 'musttail' implies 'tail'.
1652 enum TailCallKind : unsigned {
1653 TCK_None = 0,
1654 TCK_Tail = 1,
1655 TCK_MustTail = 2,
1656 TCK_NoTail = 3,
1657 TCK_LAST = TCK_NoTail
1658 };
1659
1660 using TailCallKindField = Bitfield::Element<TailCallKind, 0, 2, TCK_LAST>;
1661 static_assert(
1662 Bitfield::areContiguous<TailCallKindField, CallBase::CallingConvField>(),
1663 "Bitfields must be contiguous");
1664
1665 TailCallKind getTailCallKind() const {
1666 return getSubclassData<TailCallKindField>();
1667 }
1668
1669 bool isTailCall() const {
1670 TailCallKind Kind = getTailCallKind();
1671 return Kind == TCK_Tail || Kind == TCK_MustTail;
1672 }
1673
1674 bool isMustTailCall() const { return getTailCallKind() == TCK_MustTail; }
1675
1676 bool isNoTailCall() const { return getTailCallKind() == TCK_NoTail; }
1677
1678 void setTailCallKind(TailCallKind TCK) {
1679 setSubclassData<TailCallKindField>(TCK);
1680 }
1681
1682 void setTailCall(bool IsTc = true) {
1683 setTailCallKind(IsTc ? TCK_Tail : TCK_None);
1684 }
1685
1686 /// Return true if the call can return twice
1687 bool canReturnTwice() const { return hasFnAttr(Attribute::ReturnsTwice); }
1688 void setCanReturnTwice() {
1689 addAttribute(AttributeList::FunctionIndex, Attribute::ReturnsTwice);
1690 }
1691
1692 // Methods for support type inquiry through isa, cast, and dyn_cast:
1693 static bool classof(const Instruction *I) {
1694 return I->getOpcode() == Instruction::Call;
1695 }
1696 static bool classof(const Value *V) {
1697 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1698 }
1699
1700 /// Updates profile metadata by scaling it by \p S / \p T.
1701 void updateProfWeight(uint64_t S, uint64_t T);
1702
1703private:
1704 // Shadow Instruction::setInstructionSubclassData with a private forwarding
1705 // method so that subclasses cannot accidentally use it.
1706 template <typename Bitfield>
1707 void setSubclassData(typename Bitfield::Type Value) {
1708 Instruction::setSubclassData<Bitfield>(Value);
1709 }
1710};
1711
1712CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1713 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1714 BasicBlock *InsertAtEnd)
1715 : CallBase(Ty->getReturnType(), Instruction::Call,
1716 OperandTraits<CallBase>::op_end(this) -
1717 (Args.size() + CountBundleInputs(Bundles) + 1),
1718 unsigned(Args.size() + CountBundleInputs(Bundles) + 1),
1719 InsertAtEnd) {
1720 init(Ty, Func, Args, Bundles, NameStr);
1721}
1722
1723CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1724 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1725 Instruction *InsertBefore)
1726 : CallBase(Ty->getReturnType(), Instruction::Call,
1727 OperandTraits<CallBase>::op_end(this) -
1728 (Args.size() + CountBundleInputs(Bundles) + 1),
1729 unsigned(Args.size() + CountBundleInputs(Bundles) + 1),
1730 InsertBefore) {
1731 init(Ty, Func, Args, Bundles, NameStr);
1732}
1733
1734//===----------------------------------------------------------------------===//
1735// SelectInst Class
1736//===----------------------------------------------------------------------===//
1737
1738/// This class represents the LLVM 'select' instruction.
1739///
1740class SelectInst : public Instruction {
1741 SelectInst(Value *C, Value *S1, Value *S2, const Twine &NameStr,
1742 Instruction *InsertBefore)
1743 : Instruction(S1->getType(), Instruction::Select,
1744 &Op<0>(), 3, InsertBefore) {
1745 init(C, S1, S2);
1746 setName(NameStr);
1747 }
1748
1749 SelectInst(Value *C, Value *S1, Value *S2, const Twine &NameStr,
1750 BasicBlock *InsertAtEnd)
1751 : Instruction(S1->getType(), Instruction::Select,
1752 &Op<0>(), 3, InsertAtEnd) {
1753 init(C, S1, S2);
1754 setName(NameStr);
1755 }
1756
1757 void init(Value *C, Value *S1, Value *S2) {
1758 assert(!areInvalidOperands(C, S1, S2) && "Invalid operands for select")((void)0);
1759 Op<0>() = C;
1760 Op<1>() = S1;
1761 Op<2>() = S2;
1762 }
1763
1764protected:
1765 // Note: Instruction needs to be a friend here to call cloneImpl.
1766 friend class Instruction;
1767
1768 SelectInst *cloneImpl() const;
1769
1770public:
1771 static SelectInst *Create(Value *C, Value *S1, Value *S2,
1772 const Twine &NameStr = "",
1773 Instruction *InsertBefore = nullptr,
1774 Instruction *MDFrom = nullptr) {
1775 SelectInst *Sel = new(3) SelectInst(C, S1, S2, NameStr, InsertBefore);
1776 if (MDFrom)
1777 Sel->copyMetadata(*MDFrom);
1778 return Sel;
1779 }
1780
1781 static SelectInst *Create(Value *C, Value *S1, Value *S2,
1782 const Twine &NameStr,
1783 BasicBlock *InsertAtEnd) {
1784 return new(3) SelectInst(C, S1, S2, NameStr, InsertAtEnd);
1785 }
1786
1787 const Value *getCondition() const { return Op<0>(); }
1788 const Value *getTrueValue() const { return Op<1>(); }
1789 const Value *getFalseValue() const { return Op<2>(); }
1790 Value *getCondition() { return Op<0>(); }
1791 Value *getTrueValue() { return Op<1>(); }
1792 Value *getFalseValue() { return Op<2>(); }
1793
1794 void setCondition(Value *V) { Op<0>() = V; }
1795 void setTrueValue(Value *V) { Op<1>() = V; }
1796 void setFalseValue(Value *V) { Op<2>() = V; }
1797
1798 /// Swap the true and false values of the select instruction.
1799 /// This doesn't swap prof metadata.
1800 void swapValues() { Op<1>().swap(Op<2>()); }
1801
1802 /// Return a string if the specified operands are invalid
1803 /// for a select operation, otherwise return null.
1804 static const char *areInvalidOperands(Value *Cond, Value *True, Value *False);
1805
1806 /// Transparently provide more efficient getOperand methods.
1807 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1808
1809 OtherOps getOpcode() const {
1810 return static_cast<OtherOps>(Instruction::getOpcode());
1811 }
1812
1813 // Methods for support type inquiry through isa, cast, and dyn_cast:
1814 static bool classof(const Instruction *I) {
1815 return I->getOpcode() == Instruction::Select;
1816 }
1817 static bool classof(const Value *V) {
1818 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1819 }
1820};
1821
1822template <>
1823struct OperandTraits<SelectInst> : public FixedNumOperandTraits<SelectInst, 3> {
1824};
1825
1826DEFINE_TRANSPARENT_OPERAND_ACCESSORS(SelectInst, Value)SelectInst::op_iterator SelectInst::op_begin() { return OperandTraits
<SelectInst>::op_begin(this); } SelectInst::const_op_iterator
SelectInst::op_begin() const { return OperandTraits<SelectInst
>::op_begin(const_cast<SelectInst*>(this)); } SelectInst
::op_iterator SelectInst::op_end() { return OperandTraits<
SelectInst>::op_end(this); } SelectInst::const_op_iterator
SelectInst::op_end() const { return OperandTraits<SelectInst
>::op_end(const_cast<SelectInst*>(this)); } Value *SelectInst
::getOperand(unsigned i_nocapture) const { ((void)0); return cast_or_null
<Value>( OperandTraits<SelectInst>::op_begin(const_cast
<SelectInst*>(this))[i_nocapture].get()); } void SelectInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((
void)0); OperandTraits<SelectInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned SelectInst::getNumOperands() const
{ return OperandTraits<SelectInst>::operands(this); } template
<int Idx_nocapture> Use &SelectInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &SelectInst::Op() const { return
this->OpFrom<Idx_nocapture>(this); }
1827
1828//===----------------------------------------------------------------------===//
1829// VAArgInst Class
1830//===----------------------------------------------------------------------===//
1831
1832/// This class represents the va_arg llvm instruction, which returns
1833/// an argument of the specified type given a va_list and increments that list
1834///
1835class VAArgInst : public UnaryInstruction {
1836protected:
1837 // Note: Instruction needs to be a friend here to call cloneImpl.
1838 friend class Instruction;
1839
1840 VAArgInst *cloneImpl() const;
1841
1842public:
1843 VAArgInst(Value *List, Type *Ty, const Twine &NameStr = "",
1844 Instruction *InsertBefore = nullptr)
1845 : UnaryInstruction(Ty, VAArg, List, InsertBefore) {
1846 setName(NameStr);
1847 }
1848
1849 VAArgInst(Value *List, Type *Ty, const Twine &NameStr,
1850 BasicBlock *InsertAtEnd)
1851 : UnaryInstruction(Ty, VAArg, List, InsertAtEnd) {
1852 setName(NameStr);
1853 }
1854
1855 Value *getPointerOperand() { return getOperand(0); }
1856 const Value *getPointerOperand() const { return getOperand(0); }
1857 static unsigned getPointerOperandIndex() { return 0U; }
1858
1859 // Methods for support type inquiry through isa, cast, and dyn_cast:
1860 static bool classof(const Instruction *I) {
1861 return I->getOpcode() == VAArg;
1862 }
1863 static bool classof(const Value *V) {
1864 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1865 }
1866};
1867
1868//===----------------------------------------------------------------------===//
1869// ExtractElementInst Class
1870//===----------------------------------------------------------------------===//
1871
1872/// This instruction extracts a single (scalar)
1873/// element from a VectorType value
1874///
1875class ExtractElementInst : public Instruction {
1876 ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr = "",
1877 Instruction *InsertBefore = nullptr);
1878 ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr,
1879 BasicBlock *InsertAtEnd);
1880
1881protected:
1882 // Note: Instruction needs to be a friend here to call cloneImpl.
1883 friend class Instruction;
1884
1885 ExtractElementInst *cloneImpl() const;
1886
1887public:
1888 static ExtractElementInst *Create(Value *Vec, Value *Idx,
1889 const Twine &NameStr = "",
1890 Instruction *InsertBefore = nullptr) {
1891 return new(2) ExtractElementInst(Vec, Idx, NameStr, InsertBefore);
1892 }
1893
1894 static ExtractElementInst *Create(Value *Vec, Value *Idx,
1895 const Twine &NameStr,
1896 BasicBlock *InsertAtEnd) {
1897 return new(2) ExtractElementInst(Vec, Idx, NameStr, InsertAtEnd);
1898 }
1899
1900 /// Return true if an extractelement instruction can be
1901 /// formed with the specified operands.
1902 static bool isValidOperands(const Value *Vec, const Value *Idx);
1903
1904 Value *getVectorOperand() { return Op<0>(); }
1905 Value *getIndexOperand() { return Op<1>(); }
1906 const Value *getVectorOperand() const { return Op<0>(); }
1907 const Value *getIndexOperand() const { return Op<1>(); }
1908
1909 VectorType *getVectorOperandType() const {
1910 return cast<VectorType>(getVectorOperand()->getType());
1911 }
1912
1913 /// Transparently provide more efficient getOperand methods.
1914 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1915
1916 // Methods for support type inquiry through isa, cast, and dyn_cast:
1917 static bool classof(const Instruction *I) {
1918 return I->getOpcode() == Instruction::ExtractElement;
1919 }
1920 static bool classof(const Value *V) {
1921 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1922 }
1923};
1924
1925template <>
1926struct OperandTraits<ExtractElementInst> :
1927 public FixedNumOperandTraits<ExtractElementInst, 2> {
1928};
1929
1930DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ExtractElementInst, Value)ExtractElementInst::op_iterator ExtractElementInst::op_begin(
) { return OperandTraits<ExtractElementInst>::op_begin(
this); } ExtractElementInst::const_op_iterator ExtractElementInst
::op_begin() const { return OperandTraits<ExtractElementInst
>::op_begin(const_cast<ExtractElementInst*>(this)); }
ExtractElementInst::op_iterator ExtractElementInst::op_end()
{ return OperandTraits<ExtractElementInst>::op_end(this
); } ExtractElementInst::const_op_iterator ExtractElementInst
::op_end() const { return OperandTraits<ExtractElementInst
>::op_end(const_cast<ExtractElementInst*>(this)); } Value
*ExtractElementInst::getOperand(unsigned i_nocapture) const {
((void)0); return cast_or_null<Value>( OperandTraits<
ExtractElementInst>::op_begin(const_cast<ExtractElementInst
*>(this))[i_nocapture].get()); } void ExtractElementInst::
setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((void
)0); OperandTraits<ExtractElementInst>::op_begin(this)[
i_nocapture] = Val_nocapture; } unsigned ExtractElementInst::
getNumOperands() const { return OperandTraits<ExtractElementInst
>::operands(this); } template <int Idx_nocapture> Use
&ExtractElementInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
ExtractElementInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
1931
1932//===----------------------------------------------------------------------===//
1933// InsertElementInst Class
1934//===----------------------------------------------------------------------===//
1935
1936/// This instruction inserts a single (scalar)
1937/// element into a VectorType value
1938///
1939class InsertElementInst : public Instruction {
1940 InsertElementInst(Value *Vec, Value *NewElt, Value *Idx,
1941 const Twine &NameStr = "",
1942 Instruction *InsertBefore = nullptr);
1943 InsertElementInst(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr,
1944 BasicBlock *InsertAtEnd);
1945
1946protected:
1947 // Note: Instruction needs to be a friend here to call cloneImpl.
1948 friend class Instruction;
1949
1950 InsertElementInst *cloneImpl() const;
1951
1952public:
1953 static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
1954 const Twine &NameStr = "",
1955 Instruction *InsertBefore = nullptr) {
1956 return new(3) InsertElementInst(Vec, NewElt, Idx, NameStr, InsertBefore);
1957 }
1958
1959 static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
1960 const Twine &NameStr,
1961 BasicBlock *InsertAtEnd) {
1962 return new(3) InsertElementInst(Vec, NewElt, Idx, NameStr, InsertAtEnd);
1963 }
1964
1965 /// Return true if an insertelement instruction can be
1966 /// formed with the specified operands.
1967 static bool isValidOperands(const Value *Vec, const Value *NewElt,
1968 const Value *Idx);
1969
1970 /// Overload to return most specific vector type.
1971 ///
1972 VectorType *getType() const {
1973 return cast<VectorType>(Instruction::getType());
1974 }
1975
1976 /// Transparently provide more efficient getOperand methods.
1977 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1978
1979 // Methods for support type inquiry through isa, cast, and dyn_cast:
1980 static bool classof(const Instruction *I) {
1981 return I->getOpcode() == Instruction::InsertElement;
1982 }
1983 static bool classof(const Value *V) {
1984 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1985 }
1986};
1987
1988template <>
1989struct OperandTraits<InsertElementInst> :
1990 public FixedNumOperandTraits<InsertElementInst, 3> {
1991};
1992
1993DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementInst, Value)InsertElementInst::op_iterator InsertElementInst::op_begin() {
return OperandTraits<InsertElementInst>::op_begin(this
); } InsertElementInst::const_op_iterator InsertElementInst::
op_begin() const { return OperandTraits<InsertElementInst>
::op_begin(const_cast<InsertElementInst*>(this)); } InsertElementInst
::op_iterator InsertElementInst::op_end() { return OperandTraits
<InsertElementInst>::op_end(this); } InsertElementInst::
const_op_iterator InsertElementInst::op_end() const { return OperandTraits
<InsertElementInst>::op_end(const_cast<InsertElementInst
*>(this)); } Value *InsertElementInst::getOperand(unsigned
i_nocapture) const { ((void)0); return cast_or_null<Value
>( OperandTraits<InsertElementInst>::op_begin(const_cast
<InsertElementInst*>(this))[i_nocapture].get()); } void
InsertElementInst::setOperand(unsigned i_nocapture, Value *Val_nocapture
) { ((void)0); OperandTraits<InsertElementInst>::op_begin
(this)[i_nocapture] = Val_nocapture; } unsigned InsertElementInst
::getNumOperands() const { return OperandTraits<InsertElementInst
>::operands(this); } template <int Idx_nocapture> Use
&InsertElementInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
InsertElementInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
1994
1995//===----------------------------------------------------------------------===//
1996// ShuffleVectorInst Class
1997//===----------------------------------------------------------------------===//
1998
1999constexpr int UndefMaskElem = -1;
2000
2001/// This instruction constructs a fixed permutation of two
2002/// input vectors.
2003///
2004/// For each element of the result vector, the shuffle mask selects an element
2005/// from one of the input vectors to copy to the result. Non-negative elements
2006/// in the mask represent an index into the concatenated pair of input vectors.
2007/// UndefMaskElem (-1) specifies that the result element is undefined.
2008///
2009/// For scalable vectors, all the elements of the mask must be 0 or -1. This
2010/// requirement may be relaxed in the future.
2011class ShuffleVectorInst : public Instruction {
2012 SmallVector<int, 4> ShuffleMask;
2013 Constant *ShuffleMaskForBitcode;
2014
2015protected:
2016 // Note: Instruction needs to be a friend here to call cloneImpl.
2017 friend class Instruction;
2018
2019 ShuffleVectorInst *cloneImpl() const;
2020
2021public:
2022 ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
2023 const Twine &NameStr = "",
2024 Instruction *InsertBefor = nullptr);
2025 ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
2026 const Twine &NameStr, BasicBlock *InsertAtEnd);
2027 ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
2028 const Twine &NameStr = "",
2029 Instruction *InsertBefor = nullptr);
2030 ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
2031 const Twine &NameStr, BasicBlock *InsertAtEnd);
2032
2033 void *operator new(size_t S) { return User::operator new(S, 2); }
2034 void operator delete(void *Ptr) { return User::operator delete(Ptr); }
2035
2036 /// Swap the operands and adjust the mask to preserve the semantics
2037 /// of the instruction.
2038 void commute();
2039
2040 /// Return true if a shufflevector instruction can be
2041 /// formed with the specified operands.
2042 static bool isValidOperands(const Value *V1, const Value *V2,
2043 const Value *Mask);
2044 static bool isValidOperands(const Value *V1, const Value *V2,
2045 ArrayRef<int> Mask);
2046
2047 /// Overload to return most specific vector type.
2048 ///
2049 VectorType *getType() const {
2050 return cast<VectorType>(Instruction::getType());
2051 }
2052
2053 /// Transparently provide more efficient getOperand methods.
2054 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2055
2056 /// Return the shuffle mask value of this instruction for the given element
2057 /// index. Return UndefMaskElem if the element is undef.
2058 int getMaskValue(unsigned Elt) const { return ShuffleMask[Elt]; }
2059
2060 /// Convert the input shuffle mask operand to a vector of integers. Undefined
2061 /// elements of the mask are returned as UndefMaskElem.
2062 static void getShuffleMask(const Constant *Mask,
2063 SmallVectorImpl<int> &Result);
2064
2065 /// Return the mask for this instruction as a vector of integers. Undefined
2066 /// elements of the mask are returned as UndefMaskElem.
2067 void getShuffleMask(SmallVectorImpl<int> &Result) const {
2068 Result.assign(ShuffleMask.begin(), ShuffleMask.end());
2069 }
2070
2071 /// Return the mask for this instruction, for use in bitcode.
2072 ///
2073 /// TODO: This is temporary until we decide a new bitcode encoding for
2074 /// shufflevector.
2075 Constant *getShuffleMaskForBitcode() const { return ShuffleMaskForBitcode; }
2076
2077 static Constant *convertShuffleMaskForBitcode(ArrayRef<int> Mask,
2078 Type *ResultTy);
2079
2080 void setShuffleMask(ArrayRef<int> Mask);
2081
2082 ArrayRef<int> getShuffleMask() const { return ShuffleMask; }
2083
2084 /// Return true if this shuffle returns a vector with a different number of
2085 /// elements than its source vectors.
2086 /// Examples: shufflevector <4 x n> A, <4 x n> B, <1,2,3>
2087 /// shufflevector <4 x n> A, <4 x n> B, <1,2,3,4,5>
2088 bool changesLength() const {
2089 unsigned NumSourceElts = cast<VectorType>(Op<0>()->getType())
2090 ->getElementCount()
2091 .getKnownMinValue();
2092 unsigned NumMaskElts = ShuffleMask.size();
2093 return NumSourceElts != NumMaskElts;
2094 }
2095
2096 /// Return true if this shuffle returns a vector with a greater number of
2097 /// elements than its source vectors.
2098 /// Example: shufflevector <2 x n> A, <2 x n> B, <1,2,3>
2099 bool increasesLength() const {
2100 unsigned NumSourceElts = cast<VectorType>(Op<0>()->getType())
2101 ->getElementCount()
2102 .getKnownMinValue();
2103 unsigned NumMaskElts = ShuffleMask.size();
2104 return NumSourceElts < NumMaskElts;
2105 }
2106
2107 /// Return true if this shuffle mask chooses elements from exactly one source
2108 /// vector.
2109 /// Example: <7,5,undef,7>
2110 /// This assumes that vector operands are the same length as the mask.
2111 static bool isSingleSourceMask(ArrayRef<int> Mask);
2112 static bool isSingleSourceMask(const Constant *Mask) {
2113 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")((void)0);
2114 SmallVector<int, 16> MaskAsInts;
2115 getShuffleMask(Mask, MaskAsInts);
2116 return isSingleSourceMask(MaskAsInts);
2117 }
2118
2119 /// Return true if this shuffle chooses elements from exactly one source
2120 /// vector without changing the length of that vector.
2121 /// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
2122 /// TODO: Optionally allow length-changing shuffles.
2123 bool isSingleSource() const {
2124 return !changesLength() && isSingleSourceMask(ShuffleMask);
2125 }
2126
2127 /// Return true if this shuffle mask chooses elements from exactly one source
2128 /// vector without lane crossings. A shuffle using this mask is not
2129 /// necessarily a no-op because it may change the number of elements from its
2130 /// input vectors or it may provide demanded bits knowledge via undef lanes.
2131 /// Example: <undef,undef,2,3>
2132 static bool isIdentityMask(ArrayRef<int> Mask);
2133 static bool isIdentityMask(const Constant *Mask) {
2134 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")((void)0);
2135 SmallVector<int, 16> MaskAsInts;
2136 getShuffleMask(Mask, MaskAsInts);
2137 return isIdentityMask(MaskAsInts);
2138 }
2139
2140 /// Return true if this shuffle chooses elements from exactly one source
2141 /// vector without lane crossings and does not change the number of elements
2142 /// from its input vectors.
2143 /// Example: shufflevector <4 x n> A, <4 x n> B, <4,undef,6,undef>
2144 bool isIdentity() const {
2145 return !changesLength() && isIdentityMask(ShuffleMask);
2146 }
2147
2148 /// Return true if this shuffle lengthens exactly one source vector with
2149 /// undefs in the high elements.
2150 bool isIdentityWithPadding() const;
2151
2152 /// Return true if this shuffle extracts the first N elements of exactly one
2153 /// source vector.
2154 bool isIdentityWithExtract() const;
2155
2156 /// Return true if this shuffle concatenates its 2 source vectors. This
2157 /// returns false if either input is undefined. In that case, the shuffle is
2158 /// is better classified as an identity with padding operation.
2159 bool isConcat() const;
2160
2161 /// Return true if this shuffle mask chooses elements from its source vectors
2162 /// without lane crossings. A shuffle using this mask would be
2163 /// equivalent to a vector select with a constant condition operand.
2164 /// Example: <4,1,6,undef>
2165 /// This returns false if the mask does not choose from both input vectors.
2166 /// In that case, the shuffle is better classified as an identity shuffle.
2167 /// This assumes that vector operands are the same length as the mask
2168 /// (a length-changing shuffle can never be equivalent to a vector select).
2169 static bool isSelectMask(ArrayRef<int> Mask);
2170 static bool isSelectMask(const Constant *Mask) {
2171 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")((void)0);
2172 SmallVector<int, 16> MaskAsInts;
2173 getShuffleMask(Mask, MaskAsInts);
2174 return isSelectMask(MaskAsInts);
2175 }
2176
2177 /// Return true if this shuffle chooses elements from its source vectors
2178 /// without lane crossings and all operands have the same number of elements.
2179 /// In other words, this shuffle is equivalent to a vector select with a
2180 /// constant condition operand.
2181 /// Example: shufflevector <4 x n> A, <4 x n> B, <undef,1,6,3>
2182 /// This returns false if the mask does not choose from both input vectors.
2183 /// In that case, the shuffle is better classified as an identity shuffle.
2184 /// TODO: Optionally allow length-changing shuffles.
2185 bool isSelect() const {
2186 return !changesLength() && isSelectMask(ShuffleMask);
2187 }
2188
2189 /// Return true if this shuffle mask swaps the order of elements from exactly
2190 /// one source vector.
2191 /// Example: <7,6,undef,4>
2192 /// This assumes that vector operands are the same length as the mask.
2193 static bool isReverseMask(ArrayRef<int> Mask);
2194 static bool isReverseMask(const Constant *Mask) {
2195 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")((void)0);
2196 SmallVector<int, 16> MaskAsInts;
2197 getShuffleMask(Mask, MaskAsInts);
2198 return isReverseMask(MaskAsInts);
2199 }
2200
2201 /// Return true if this shuffle swaps the order of elements from exactly
2202 /// one source vector.
2203 /// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
2204 /// TODO: Optionally allow length-changing shuffles.
2205 bool isReverse() const {
2206 return !changesLength() && isReverseMask(ShuffleMask);
2207 }
2208
2209 /// Return true if this shuffle mask chooses all elements with the same value
2210 /// as the first element of exactly one source vector.
2211 /// Example: <4,undef,undef,4>
2212 /// This assumes that vector operands are the same length as the mask.
2213 static bool isZeroEltSplatMask(ArrayRef<int> Mask);
2214 static bool isZeroEltSplatMask(const Constant *Mask) {
2215 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")((void)0);
2216 SmallVector<int, 16> MaskAsInts;
2217 getShuffleMask(Mask, MaskAsInts);
2218 return isZeroEltSplatMask(MaskAsInts);
2219 }
2220
2221 /// Return true if all elements of this shuffle are the same value as the
2222 /// first element of exactly one source vector without changing the length
2223 /// of that vector.
2224 /// Example: shufflevector <4 x n> A, <4 x n> B, <undef,0,undef,0>
2225 /// TODO: Optionally allow length-changing shuffles.
2226 /// TODO: Optionally allow splats from other elements.
2227 bool isZeroEltSplat() const {
2228 return !changesLength() && isZeroEltSplatMask(ShuffleMask);
2229 }
2230
2231 /// Return true if this shuffle mask is a transpose mask.
2232 /// Transpose vector masks transpose a 2xn matrix. They read corresponding
2233 /// even- or odd-numbered vector elements from two n-dimensional source
2234 /// vectors and write each result into consecutive elements of an
2235 /// n-dimensional destination vector. Two shuffles are necessary to complete
2236 /// the transpose, one for the even elements and another for the odd elements.
2237 /// This description closely follows how the TRN1 and TRN2 AArch64
2238 /// instructions operate.
2239 ///
2240 /// For example, a simple 2x2 matrix can be transposed with:
2241 ///
2242 /// ; Original matrix
2243 /// m0 = < a, b >
2244 /// m1 = < c, d >
2245 ///
2246 /// ; Transposed matrix
2247 /// t0 = < a, c > = shufflevector m0, m1, < 0, 2 >
2248 /// t1 = < b, d > = shufflevector m0, m1, < 1, 3 >
2249 ///
2250 /// For matrices having greater than n columns, the resulting nx2 transposed
2251 /// matrix is stored in two result vectors such that one vector contains
2252 /// interleaved elements from all the even-numbered rows and the other vector
2253 /// contains interleaved elements from all the odd-numbered rows. For example,
2254 /// a 2x4 matrix can be transposed with:
2255 ///
2256 /// ; Original matrix
2257 /// m0 = < a, b, c, d >
2258 /// m1 = < e, f, g, h >
2259 ///
2260 /// ; Transposed matrix
2261 /// t0 = < a, e, c, g > = shufflevector m0, m1 < 0, 4, 2, 6 >
2262 /// t1 = < b, f, d, h > = shufflevector m0, m1 < 1, 5, 3, 7 >
2263 static bool isTransposeMask(ArrayRef<int> Mask);
2264 static bool isTransposeMask(const Constant *Mask) {
2265 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")((void)0);
2266 SmallVector<int, 16> MaskAsInts;
2267 getShuffleMask(Mask, MaskAsInts);
2268 return isTransposeMask(MaskAsInts);
2269 }
2270
2271 /// Return true if this shuffle transposes the elements of its inputs without
2272 /// changing the length of the vectors. This operation may also be known as a
2273 /// merge or interleave. See the description for isTransposeMask() for the
2274 /// exact specification.
2275 /// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
2276 bool isTranspose() const {
2277 return !changesLength() && isTransposeMask(ShuffleMask);
2278 }
2279
2280 /// Return true if this shuffle mask is an extract subvector mask.
2281 /// A valid extract subvector mask returns a smaller vector from a single
2282 /// source operand. The base extraction index is returned as well.
2283 static bool isExtractSubvectorMask(ArrayRef<int> Mask, int NumSrcElts,
2284 int &Index);
2285 static bool isExtractSubvectorMask(const Constant *Mask, int NumSrcElts,
2286 int &Index) {
2287 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")((void)0);
2288 // Not possible to express a shuffle mask for a scalable vector for this
2289 // case.
2290 if (isa<ScalableVectorType>(Mask->getType()))
2291 return false;
2292 SmallVector<int, 16> MaskAsInts;
2293 getShuffleMask(Mask, MaskAsInts);
2294 return isExtractSubvectorMask(MaskAsInts, NumSrcElts, Index);
2295 }
2296
2297 /// Return true if this shuffle mask is an extract subvector mask.
2298 bool isExtractSubvectorMask(int &Index) const {
2299 // Not possible to express a shuffle mask for a scalable vector for this
2300 // case.
2301 if (isa<ScalableVectorType>(getType()))
2302 return false;
2303
2304 int NumSrcElts =
2305 cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
2306 return isExtractSubvectorMask(ShuffleMask, NumSrcElts, Index);
2307 }
2308
2309 /// Change values in a shuffle permute mask assuming the two vector operands
2310 /// of length InVecNumElts have swapped position.
2311 static void commuteShuffleMask(MutableArrayRef<int> Mask,
2312 unsigned InVecNumElts) {
2313 for (int &Idx : Mask) {
2314 if (Idx == -1)
2315 continue;
2316 Idx = Idx < (int)InVecNumElts ? Idx + InVecNumElts : Idx - InVecNumElts;
2317 assert(Idx >= 0 && Idx < (int)InVecNumElts * 2 &&((void)0)
2318 "shufflevector mask index out of range")((void)0);
2319 }
2320 }
2321
2322 // Methods for support type inquiry through isa, cast, and dyn_cast:
2323 static bool classof(const Instruction *I) {
2324 return I->getOpcode() == Instruction::ShuffleVector;
2325 }
2326 static bool classof(const Value *V) {
2327 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2328 }
2329};
2330
2331template <>
2332struct OperandTraits<ShuffleVectorInst>
2333 : public FixedNumOperandTraits<ShuffleVectorInst, 2> {};
2334
2335DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorInst, Value)ShuffleVectorInst::op_iterator ShuffleVectorInst::op_begin() {
return OperandTraits<ShuffleVectorInst>::op_begin(this
); } ShuffleVectorInst::const_op_iterator ShuffleVectorInst::
op_begin() const { return OperandTraits<ShuffleVectorInst>
::op_begin(const_cast<ShuffleVectorInst*>(this)); } ShuffleVectorInst
::op_iterator ShuffleVectorInst::op_end() { return OperandTraits
<ShuffleVectorInst>::op_end(this); } ShuffleVectorInst::
const_op_iterator ShuffleVectorInst::op_end() const { return OperandTraits
<ShuffleVectorInst>::op_end(const_cast<ShuffleVectorInst
*>(this)); } Value *ShuffleVectorInst::getOperand(unsigned
i_nocapture) const { ((void)0); return cast_or_null<Value
>( OperandTraits<ShuffleVectorInst>::op_begin(const_cast
<ShuffleVectorInst*>(this))[i_nocapture].get()); } void
ShuffleVectorInst::setOperand(unsigned i_nocapture, Value *Val_nocapture
) { ((void)0); OperandTraits<ShuffleVectorInst>::op_begin
(this)[i_nocapture] = Val_nocapture; } unsigned ShuffleVectorInst
::getNumOperands() const { return OperandTraits<ShuffleVectorInst
>::operands(this); } template <int Idx_nocapture> Use
&ShuffleVectorInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
ShuffleVectorInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
2336
2337//===----------------------------------------------------------------------===//
2338// ExtractValueInst Class
2339//===----------------------------------------------------------------------===//
2340
2341/// This instruction extracts a struct member or array
2342/// element value from an aggregate value.
2343///
2344class ExtractValueInst : public UnaryInstruction {
2345 SmallVector<unsigned, 4> Indices;
2346
2347 ExtractValueInst(const ExtractValueInst &EVI);
2348
2349 /// Constructors - Create a extractvalue instruction with a base aggregate
2350 /// value and a list of indices. The first ctor can optionally insert before
2351 /// an existing instruction, the second appends the new instruction to the
2352 /// specified BasicBlock.
2353 inline ExtractValueInst(Value *Agg,
2354 ArrayRef<unsigned> Idxs,
2355 const Twine &NameStr,
2356 Instruction *InsertBefore);
2357 inline ExtractValueInst(Value *Agg,
2358 ArrayRef<unsigned> Idxs,
2359 const Twine &NameStr, BasicBlock *InsertAtEnd);
2360
2361 void init(ArrayRef<unsigned> Idxs, const Twine &NameStr);
2362
2363protected:
2364 // Note: Instruction needs to be a friend here to call cloneImpl.
2365 friend class Instruction;
2366
2367 ExtractValueInst *cloneImpl() const;
2368
2369public:
2370 static ExtractValueInst *Create(Value *Agg,
2371 ArrayRef<unsigned> Idxs,
2372 const Twine &NameStr = "",
2373 Instruction *InsertBefore = nullptr) {
2374 return new
2375 ExtractValueInst(Agg, Idxs, NameStr, InsertBefore);
2376 }
2377
2378 static ExtractValueInst *Create(Value *Agg,
2379 ArrayRef<unsigned> Idxs,
2380 const Twine &NameStr,
2381 BasicBlock *InsertAtEnd) {
2382 return new ExtractValueInst(Agg, Idxs, NameStr, InsertAtEnd);
2383 }
2384
2385 /// Returns the type of the element that would be extracted
2386 /// with an extractvalue instruction with the specified parameters.
2387 ///
2388 /// Null is returned if the indices are invalid for the specified type.
2389 static Type *getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs);
2390
2391 using idx_iterator = const unsigned*;
2392
2393 inline idx_iterator idx_begin() const { return Indices.begin(); }
2394 inline idx_iterator idx_end() const { return Indices.end(); }
2395 inline iterator_range<idx_iterator> indices() const {
2396 return make_range(idx_begin(), idx_end());
2397 }
2398
2399 Value *getAggregateOperand() {
2400 return getOperand(0);
2401 }
2402 const Value *getAggregateOperand() const {
2403 return getOperand(0);
2404 }
2405 static unsigned getAggregateOperandIndex() {
2406 return 0U; // get index for modifying correct operand
2407 }
2408
2409 ArrayRef<unsigned> getIndices() const {
2410 return Indices;
2411 }
2412
2413 unsigned getNumIndices() const {
2414 return (unsigned)Indices.size();
2415 }
2416
2417 bool hasIndices() const {
2418 return true;
2419 }
2420
2421 // Methods for support type inquiry through isa, cast, and dyn_cast:
2422 static bool classof(const Instruction *I) {
2423 return I->getOpcode() == Instruction::ExtractValue;
2424 }
2425 static bool classof(const Value *V) {
2426 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2427 }
2428};
2429
2430ExtractValueInst::ExtractValueInst(Value *Agg,
2431 ArrayRef<unsigned> Idxs,
2432 const Twine &NameStr,
2433 Instruction *InsertBefore)
2434 : UnaryInstruction(checkGEPType(getIndexedType(Agg->getType(), Idxs)),
2435 ExtractValue, Agg, InsertBefore) {
2436 init(Idxs, NameStr);
2437}
2438
2439ExtractValueInst::ExtractValueInst(Value *Agg,
2440 ArrayRef<unsigned> Idxs,
2441 const Twine &NameStr,
2442 BasicBlock *InsertAtEnd)
2443 : UnaryInstruction(checkGEPType(getIndexedType(Agg->getType(), Idxs)),
2444 ExtractValue, Agg, InsertAtEnd) {
2445 init(Idxs, NameStr);
2446}
2447
2448//===----------------------------------------------------------------------===//
2449// InsertValueInst Class
2450//===----------------------------------------------------------------------===//
2451
2452/// This instruction inserts a struct field of array element
2453/// value into an aggregate value.
2454///
2455class InsertValueInst : public Instruction {
2456 SmallVector<unsigned, 4> Indices;
2457
2458 InsertValueInst(const InsertValueInst &IVI);
2459
2460 /// Constructors - Create a insertvalue instruction with a base aggregate
2461 /// value, a value to insert, and a list of indices. The first ctor can
2462 /// optionally insert before an existing instruction, the second appends
2463 /// the new instruction to the specified BasicBlock.
2464 inline InsertValueInst(Value *Agg, Value *Val,
2465 ArrayRef<unsigned> Idxs,
2466 const Twine &NameStr,
2467 Instruction *InsertBefore);
2468 inline InsertValueInst(Value *Agg, Value *Val,
2469 ArrayRef<unsigned> Idxs,
2470 const Twine &NameStr, BasicBlock *InsertAtEnd);
2471
2472 /// Constructors - These two constructors are convenience methods because one
2473 /// and two index insertvalue instructions are so common.
2474 InsertValueInst(Value *Agg, Value *Val, unsigned Idx,
2475 const Twine &NameStr = "",
2476 Instruction *InsertBefore = nullptr);
2477 InsertValueInst(Value *Agg, Value *Val, unsigned Idx, const Twine &NameStr,
2478 BasicBlock *InsertAtEnd);
2479
2480 void init(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
2481 const Twine &NameStr);
2482
2483protected:
2484 // Note: Instruction needs to be a friend here to call cloneImpl.
2485 friend class Instruction;
2486
2487 InsertValueInst *cloneImpl() const;
2488
2489public:
2490 // allocate space for exactly two operands
2491 void *operator new(size_t S) { return User::operator new(S, 2); }
2492 void operator delete(void *Ptr) { User::operator delete(Ptr); }
2493
2494 static InsertValueInst *Create(Value *Agg, Value *Val,
2495 ArrayRef<unsigned> Idxs,
2496 const Twine &NameStr = "",
2497 Instruction *InsertBefore = nullptr) {
2498 return new InsertValueInst(Agg, Val, Idxs, NameStr, InsertBefore);
2499 }
2500
2501 static InsertValueInst *Create(Value *Agg, Value *Val,
2502 ArrayRef<unsigned> Idxs,
2503 const Twine &NameStr,
2504 BasicBlock *InsertAtEnd) {
2505 return new InsertValueInst(Agg, Val, Idxs, NameStr, InsertAtEnd);
2506 }
2507
2508 /// Transparently provide more efficient getOperand methods.
2509 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2510
2511 using idx_iterator = const unsigned*;
2512
2513 inline idx_iterator idx_begin() const { return Indices.begin(); }
2514 inline idx_iterator idx_end() const { return Indices.end(); }
2515 inline iterator_range<idx_iterator> indices() const {
2516 return make_range(idx_begin(), idx_end());
2517 }
2518
2519 Value *getAggregateOperand() {
2520 return getOperand(0);
2521 }
2522 const Value *getAggregateOperand() const {
2523 return getOperand(0);
2524 }
2525 static unsigned getAggregateOperandIndex() {
2526 return 0U; // get index for modifying correct operand
2527 }
2528
2529 Value *getInsertedValueOperand() {
2530 return getOperand(1);
2531 }
2532 const Value *getInsertedValueOperand() const {
2533 return getOperand(1);
2534 }
2535 static unsigned getInsertedValueOperandIndex() {
2536 return 1U; // get index for modifying correct operand
2537 }
2538
2539 ArrayRef<unsigned> getIndices() const {
2540 return Indices;
2541 }
2542
2543 unsigned getNumIndices() const {
2544 return (unsigned)Indices.size();
2545 }
2546
2547 bool hasIndices() const {
2548 return true;
2549 }
2550
2551 // Methods for support type inquiry through isa, cast, and dyn_cast:
2552 static bool classof(const Instruction *I) {
2553 return I->getOpcode() == Instruction::InsertValue;
2554 }
2555 static bool classof(const Value *V) {
2556 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2557 }
2558};
2559
2560template <>
2561struct OperandTraits<InsertValueInst> :
2562 public FixedNumOperandTraits<InsertValueInst, 2> {
2563};
2564
2565InsertValueInst::InsertValueInst(Value *Agg,
2566 Value *Val,
2567 ArrayRef<unsigned> Idxs,
2568 const Twine &NameStr,
2569 Instruction *InsertBefore)
2570 : Instruction(Agg->getType(), InsertValue,
2571 OperandTraits<InsertValueInst>::op_begin(this),
2572 2, InsertBefore) {
2573 init(Agg, Val, Idxs, NameStr);
2574}
2575
2576InsertValueInst::InsertValueInst(Value *Agg,
2577 Value *Val,
2578 ArrayRef<unsigned> Idxs,
2579 const Twine &NameStr,
2580 BasicBlock *InsertAtEnd)
2581 : Instruction(Agg->getType(), InsertValue,
2582 OperandTraits<InsertValueInst>::op_begin(this),
2583 2, InsertAtEnd) {
2584 init(Agg, Val, Idxs, NameStr);
2585}
2586
2587DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertValueInst, Value)InsertValueInst::op_iterator InsertValueInst::op_begin() { return
OperandTraits<InsertValueInst>::op_begin(this); } InsertValueInst
::const_op_iterator InsertValueInst::op_begin() const { return
OperandTraits<InsertValueInst>::op_begin(const_cast<
InsertValueInst*>(this)); } InsertValueInst::op_iterator InsertValueInst
::op_end() { return OperandTraits<InsertValueInst>::op_end
(this); } InsertValueInst::const_op_iterator InsertValueInst::
op_end() const { return OperandTraits<InsertValueInst>::
op_end(const_cast<InsertValueInst*>(this)); } Value *InsertValueInst
::getOperand(unsigned i_nocapture) const { ((void)0); return cast_or_null
<Value>( OperandTraits<InsertValueInst>::op_begin
(const_cast<InsertValueInst*>(this))[i_nocapture].get()
); } void InsertValueInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { ((void)0); OperandTraits<InsertValueInst
>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
InsertValueInst::getNumOperands() const { return OperandTraits
<InsertValueInst>::operands(this); } template <int Idx_nocapture
> Use &InsertValueInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
const Use &InsertValueInst::Op() const { return this->
OpFrom<Idx_nocapture>(this); }
2588
2589//===----------------------------------------------------------------------===//
2590// PHINode Class
2591//===----------------------------------------------------------------------===//
2592
2593// PHINode - The PHINode class is used to represent the magical mystical PHI
2594// node, that can not exist in nature, but can be synthesized in a computer
2595// scientist's overactive imagination.
2596//
2597class PHINode : public Instruction {
2598 /// The number of operands actually allocated. NumOperands is
2599 /// the number actually in use.
2600 unsigned ReservedSpace;
2601
2602 PHINode(const PHINode &PN);
2603
2604 explicit PHINode(Type *Ty, unsigned NumReservedValues,
2605 const Twine &NameStr = "",
2606 Instruction *InsertBefore = nullptr)
2607 : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertBefore),
2608 ReservedSpace(NumReservedValues) {
2609 assert(!Ty->isTokenTy() && "PHI nodes cannot have token type!")((void)0);
2610 setName(NameStr);
2611 allocHungoffUses(ReservedSpace);
2612 }
2613
2614 PHINode(Type *Ty, unsigned NumReservedValues, const Twine &NameStr,
2615 BasicBlock *InsertAtEnd)
2616 : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertAtEnd),
2617 ReservedSpace(NumReservedValues) {
2618 assert(!Ty->isTokenTy() && "PHI nodes cannot have token type!")((void)0);
2619 setName(NameStr);
2620 allocHungoffUses(ReservedSpace);
2621 }
2622
2623protected:
2624 // Note: Instruction needs to be a friend here to call cloneImpl.
2625 friend class Instruction;
2626
2627 PHINode *cloneImpl() const;
2628
2629 // allocHungoffUses - this is more complicated than the generic
2630 // User::allocHungoffUses, because we have to allocate Uses for the incoming
2631 // values and pointers to the incoming blocks, all in one allocation.
2632 void allocHungoffUses(unsigned N) {
2633 User::allocHungoffUses(N, /* IsPhi */ true);
2634 }
2635
2636public:
2637 /// Constructors - NumReservedValues is a hint for the number of incoming
2638 /// edges that this phi node will have (use 0 if you really have no idea).
2639 static PHINode *Create(Type *Ty, unsigned NumReservedValues,
2640 const Twine &NameStr = "",
2641 Instruction *InsertBefore = nullptr) {
2642 return new PHINode(Ty, NumReservedValues, NameStr, InsertBefore);
2643 }
2644
2645 static PHINode *Create(Type *Ty, unsigned NumReservedValues,
2646 const Twine &NameStr, BasicBlock *InsertAtEnd) {
2647 return new PHINode(Ty, NumReservedValues, NameStr, InsertAtEnd);
2648 }
2649
2650 /// Provide fast operand accessors
2651 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2652
2653 // Block iterator interface. This provides access to the list of incoming
2654 // basic blocks, which parallels the list of incoming values.
2655
2656 using block_iterator = BasicBlock **;
2657 using const_block_iterator = BasicBlock * const *;
2658
2659 block_iterator block_begin() {
2660 return reinterpret_cast<block_iterator>(op_begin() + ReservedSpace);
2661 }
2662
2663 const_block_iterator block_begin() const {
2664 return reinterpret_cast<const_block_iterator>(op_begin() + ReservedSpace);
2665 }
2666
2667 block_iterator block_end() {
2668 return block_begin() + getNumOperands();
2669 }
2670
2671 const_block_iterator block_end() const {
2672 return block_begin() + getNumOperands();
2673 }
2674
2675 iterator_range<block_iterator> blocks() {
2676 return make_range(block_begin(), block_end());
2677 }
2678
2679 iterator_range<const_block_iterator> blocks() const {
2680 return make_range(block_begin(), block_end());
2681 }
2682
2683 op_range incoming_values() { return operands(); }
2684
2685 const_op_range incoming_values() const { return operands(); }
2686
2687 /// Return the number of incoming edges
2688 ///
2689 unsigned getNumIncomingValues() const { return getNumOperands(); }
2690
2691 /// Return incoming value number x
2692 ///
2693 Value *getIncomingValue(unsigned i) const {
2694 return getOperand(i);
2695 }
2696 void setIncomingValue(unsigned i, Value *V) {
2697 assert(V && "PHI node got a null value!")((void)0);
2698 assert(getType() == V->getType() &&((void)0)
2699 "All operands to PHI node must be the same type as the PHI node!")((void)0);
2700 setOperand(i, V);
2701 }
2702
2703 static unsigned getOperandNumForIncomingValue(unsigned i) {
2704 return i;
2705 }
2706
2707 static unsigned getIncomingValueNumForOperand(unsigned i) {
2708 return i;
2709 }
2710
2711 /// Return incoming basic block number @p i.
2712 ///
2713 BasicBlock *getIncomingBlock(unsigned i) const {
2714 return block_begin()[i];
2715 }
2716
2717 /// Return incoming basic block corresponding
2718 /// to an operand of the PHI.
2719 ///
2720 BasicBlock *getIncomingBlock(const Use &U) const {
2721 assert(this == U.getUser() && "Iterator doesn't point to PHI's Uses?")((void)0);
2722 return getIncomingBlock(unsigned(&U - op_begin()));
2723 }
2724
2725 /// Return incoming basic block corresponding
2726 /// to value use iterator.
2727 ///
2728 BasicBlock *getIncomingBlock(Value::const_user_iterator I) const {
2729 return getIncomingBlock(I.getUse());
2730 }
2731
2732 void setIncomingBlock(unsigned i, BasicBlock *BB) {
2733 assert(BB && "PHI node got a null basic block!")((void)0);
2734 block_begin()[i] = BB;
2735 }
2736
2737 /// Replace every incoming basic block \p Old to basic block \p New.
2738 void replaceIncomingBlockWith(const BasicBlock *Old, BasicBlock *New) {
2739 assert(New && Old && "PHI node got a null basic block!")((void)0);
2740 for (unsigned Op = 0, NumOps = getNumOperands(); Op != NumOps; ++Op)
2741 if (getIncomingBlock(Op) == Old)
2742 setIncomingBlock(Op, New);
2743 }
2744
2745 /// Add an incoming value to the end of the PHI list
2746 ///
2747 void addIncoming(Value *V, BasicBlock *BB) {
2748 if (getNumOperands() == ReservedSpace)
2749 growOperands(); // Get more space!
2750 // Initialize some new operands.
2751 setNumHungOffUseOperands(getNumOperands() + 1);
2752 setIncomingValue(getNumOperands() - 1, V);
2753 setIncomingBlock(getNumOperands() - 1, BB);
2754 }
2755
2756 /// Remove an incoming value. This is useful if a
2757 /// predecessor basic block is deleted. The value removed is returned.
2758 ///
2759 /// If the last incoming value for a PHI node is removed (and DeletePHIIfEmpty
2760 /// is true), the PHI node is destroyed and any uses of it are replaced with
2761 /// dummy values. The only time there should be zero incoming values to a PHI
2762 /// node is when the block is dead, so this strategy is sound.
2763 ///
2764 Value *removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty = true);
2765
2766 Value *removeIncomingValue(const BasicBlock *BB, bool DeletePHIIfEmpty=true) {
2767 int Idx = getBasicBlockIndex(BB);
2768 assert(Idx >= 0 && "Invalid basic block argument to remove!")((void)0);
2769 return removeIncomingValue(Idx, DeletePHIIfEmpty);
2770 }
2771
2772 /// Return the first index of the specified basic
2773 /// block in the value list for this PHI. Returns -1 if no instance.
2774 ///
2775 int getBasicBlockIndex(const BasicBlock *BB) const {
2776 for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
2777 if (block_begin()[i] == BB)
2778 return i;
2779 return -1;
2780 }
2781
2782 Value *getIncomingValueForBlock(const BasicBlock *BB) const {
2783 int Idx = getBasicBlockIndex(BB);
2784 assert(Idx >= 0 && "Invalid basic block argument!")((void)0);
2785 return getIncomingValue(Idx);
2786 }
2787
2788 /// Set every incoming value(s) for block \p BB to \p V.
2789 void setIncomingValueForBlock(const BasicBlock *BB, Value *V) {
2790 assert(BB && "PHI node got a null basic block!")((void)0);
2791 bool Found = false;
2792 for (unsigned Op = 0, NumOps = getNumOperands(); Op != NumOps; ++Op)
2793 if (getIncomingBlock(Op) == BB) {
2794 Found = true;
2795 setIncomingValue(Op, V);
2796 }
2797 (void)Found;
2798 assert(Found && "Invalid basic block argument to set!")((void)0);
2799 }
2800
2801 /// If the specified PHI node always merges together the
2802 /// same value, return the value, otherwise return null.
2803 Value *hasConstantValue() const;
2804
2805 /// Whether the specified PHI node always merges
2806 /// together the same value, assuming undefs are equal to a unique
2807 /// non-undef value.
2808 bool hasConstantOrUndefValue() const;
2809
2810 /// If the PHI node is complete which means all of its parent's predecessors
2811 /// have incoming value in this PHI, return true, otherwise return false.
2812 bool isComplete() const {
2813 return llvm::all_of(predecessors(getParent()),
2814 [this](const BasicBlock *Pred) {
2815 return getBasicBlockIndex(Pred) >= 0;
2816 });
2817 }
2818
2819 /// Methods for support type inquiry through isa, cast, and dyn_cast:
2820 static bool classof(const Instruction *I) {
2821 return I->getOpcode() == Instruction::PHI;
2822 }
2823 static bool classof(const Value *V) {
2824 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2825 }
2826
2827private:
2828 void growOperands();
2829};
2830
2831template <>
2832struct OperandTraits<PHINode> : public HungoffOperandTraits<2> {
2833};
2834
2835DEFINE_TRANSPARENT_OPERAND_ACCESSORS(PHINode, Value)PHINode::op_iterator PHINode::op_begin() { return OperandTraits
<PHINode>::op_begin(this); } PHINode::const_op_iterator
PHINode::op_begin() const { return OperandTraits<PHINode>
::op_begin(const_cast<PHINode*>(this)); } PHINode::op_iterator
PHINode::op_end() { return OperandTraits<PHINode>::op_end
(this); } PHINode::const_op_iterator PHINode::op_end() const {
return OperandTraits<PHINode>::op_end(const_cast<PHINode
*>(this)); } Value *PHINode::getOperand(unsigned i_nocapture
) const { ((void)0); return cast_or_null<Value>( OperandTraits
<PHINode>::op_begin(const_cast<PHINode*>(this))[i_nocapture
].get()); } void PHINode::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { ((void)0); OperandTraits<PHINode>::op_begin
(this)[i_nocapture] = Val_nocapture; } unsigned PHINode::getNumOperands
() const { return OperandTraits<PHINode>::operands(this
); } template <int Idx_nocapture> Use &PHINode::Op(
) { return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &PHINode::Op() const
{ return this->OpFrom<Idx_nocapture>(this); }
2836
2837//===----------------------------------------------------------------------===//
2838// LandingPadInst Class
2839//===----------------------------------------------------------------------===//
2840
2841//===---------------------------------------------------------------------------
2842/// The landingpad instruction holds all of the information
2843/// necessary to generate correct exception handling. The landingpad instruction
2844/// cannot be moved from the top of a landing pad block, which itself is
2845/// accessible only from the 'unwind' edge of an invoke. This uses the
2846/// SubclassData field in Value to store whether or not the landingpad is a
2847/// cleanup.
2848///
2849class LandingPadInst : public Instruction {
2850 using CleanupField = BoolBitfieldElementT<0>;
2851
2852 /// The number of operands actually allocated. NumOperands is
2853 /// the number actually in use.
2854 unsigned ReservedSpace;
2855
2856 LandingPadInst(const LandingPadInst &LP);
2857
2858public:
2859 enum ClauseType { Catch, Filter };
2860
2861private:
2862 explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
2863 const Twine &NameStr, Instruction *InsertBefore);
2864 explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
2865 const Twine &NameStr, BasicBlock *InsertAtEnd);
2866
2867 // Allocate space for exactly zero operands.
2868 void *operator new(size_t S) { return User::operator new(S); }
2869
2870 void growOperands(unsigned Size);
2871 void init(unsigned NumReservedValues, const Twine &NameStr);
2872
2873protected:
2874 // Note: Instruction needs to be a friend here to call cloneImpl.
2875 friend class Instruction;
2876
2877 LandingPadInst *cloneImpl() const;
2878
2879public:
2880 void operator delete(void *Ptr) { User::operator delete(Ptr); }
2881
2882 /// Constructors - NumReservedClauses is a hint for the number of incoming
2883 /// clauses that this landingpad will have (use 0 if you really have no idea).
2884 static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
2885 const Twine &NameStr = "",
2886 Instruction *InsertBefore = nullptr);
2887 static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
2888 const Twine &NameStr, BasicBlock *InsertAtEnd);
2889
2890 /// Provide fast operand accessors
2891 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2892
2893 /// Return 'true' if this landingpad instruction is a
2894 /// cleanup. I.e., it should be run when unwinding even if its landing pad
2895 /// doesn't catch the exception.
2896 bool isCleanup() const { return getSubclassData<CleanupField>(); }
2897
2898 /// Indicate that this landingpad instruction is a cleanup.
2899 void setCleanup(bool V) { setSubclassData<CleanupField>(V); }
2900
2901 /// Add a catch or filter clause to the landing pad.
2902 void addClause(Constant *ClauseVal);
2903
2904 /// Get the value of the clause at index Idx. Use isCatch/isFilter to
2905 /// determine what type of clause this is.
2906 Constant *getClause(unsigned Idx) const {
2907 return cast<Constant>(getOperandList()[Idx]);
2908 }
2909
2910 /// Return 'true' if the clause and index Idx is a catch clause.
2911 bool isCatch(unsigned Idx) const {
2912 return !isa<ArrayType>(getOperandList()[Idx]->getType());
2913 }
2914
2915 /// Return 'true' if the clause and index Idx is a filter clause.
2916 bool isFilter(unsigned Idx) const {
2917 return isa<ArrayType>(getOperandList()[Idx]->getType());
2918 }
2919
2920 /// Get the number of clauses for this landing pad.
2921 unsigned getNumClauses() const { return getNumOperands(); }
2922
2923 /// Grow the size of the operand list to accommodate the new
2924 /// number of clauses.
2925 void reserveClauses(unsigned Size) { growOperands(Size); }
2926
2927 // Methods for support type inquiry through isa, cast, and dyn_cast:
2928 static bool classof(const Instruction *I) {
2929 return I->getOpcode() == Instruction::LandingPad;
2930 }
2931 static bool classof(const Value *V) {
2932 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2933 }
2934};
2935
2936template <>
2937struct OperandTraits<LandingPadInst> : public HungoffOperandTraits<1> {
2938};
2939
2940DEFINE_TRANSPARENT_OPERAND_ACCESSORS(LandingPadInst, Value)LandingPadInst::op_iterator LandingPadInst::op_begin() { return
OperandTraits<LandingPadInst>::op_begin(this); } LandingPadInst
::const_op_iterator LandingPadInst::op_begin() const { return
OperandTraits<LandingPadInst>::op_begin(const_cast<
LandingPadInst*>(this)); } LandingPadInst::op_iterator LandingPadInst
::op_end() { return OperandTraits<LandingPadInst>::op_end
(this); } LandingPadInst::const_op_iterator LandingPadInst::op_end
() const { return OperandTraits<LandingPadInst>::op_end
(const_cast<LandingPadInst*>(this)); } Value *LandingPadInst
::getOperand(unsigned i_nocapture) const { ((void)0); return cast_or_null
<Value>( OperandTraits<LandingPadInst>::op_begin(
const_cast<LandingPadInst*>(this))[i_nocapture].get());
} void LandingPadInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { ((void)0); OperandTraits<LandingPadInst
>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
LandingPadInst::getNumOperands() const { return OperandTraits
<LandingPadInst>::operands(this); } template <int Idx_nocapture
> Use &LandingPadInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
const Use &LandingPadInst::Op() const { return this->
OpFrom<Idx_nocapture>(this); }
2941
2942//===----------------------------------------------------------------------===//
2943// ReturnInst Class
2944//===----------------------------------------------------------------------===//
2945
2946//===---------------------------------------------------------------------------
2947/// Return a value (possibly void), from a function. Execution
2948/// does not continue in this function any longer.
2949///
2950class ReturnInst : public Instruction {
2951 ReturnInst(const ReturnInst &RI);
2952
2953private:
2954 // ReturnInst constructors:
2955 // ReturnInst() - 'ret void' instruction
2956 // ReturnInst( null) - 'ret void' instruction
2957 // ReturnInst(Value* X) - 'ret X' instruction
2958 // ReturnInst( null, Inst *I) - 'ret void' instruction, insert before I
2959 // ReturnInst(Value* X, Inst *I) - 'ret X' instruction, insert before I
2960 // ReturnInst( null, BB *B) - 'ret void' instruction, insert @ end of B
2961 // ReturnInst(Value* X, BB *B) - 'ret X' instruction, insert @ end of B
2962 //
2963 // NOTE: If the Value* passed is of type void then the constructor behaves as
2964 // if it was passed NULL.
2965 explicit ReturnInst(LLVMContext &C, Value *retVal = nullptr,
2966 Instruction *InsertBefore = nullptr);
2967 ReturnInst(LLVMContext &C, Value *retVal, BasicBlock *InsertAtEnd);
2968 explicit ReturnInst(LLVMContext &C, BasicBlock *InsertAtEnd);
2969
2970protected:
2971 // Note: Instruction needs to be a friend here to call cloneImpl.
2972 friend class Instruction;
2973
2974 ReturnInst *cloneImpl() const;
2975
2976public:
2977 static ReturnInst* Create(LLVMContext &C, Value *retVal = nullptr,
2978 Instruction *InsertBefore = nullptr) {
2979 return new(!!retVal) ReturnInst(C, retVal, InsertBefore);
2980 }
2981
2982 static ReturnInst* Create(LLVMContext &C, Value *retVal,
2983 BasicBlock *InsertAtEnd) {
2984 return new(!!retVal) ReturnInst(C, retVal, InsertAtEnd);
2985 }
2986
2987 static ReturnInst* Create(LLVMContext &C, BasicBlock *InsertAtEnd) {
2988 return new(0) ReturnInst(C, InsertAtEnd);
2989 }
2990
2991 /// Provide fast operand accessors
2992 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2993
2994 /// Convenience accessor. Returns null if there is no return value.
2995 Value *getReturnValue() const {
2996 return getNumOperands() != 0 ? getOperand(0) : nullptr;
2997 }
2998
2999 unsigned getNumSuccessors() const { return 0; }
3000
3001 // Methods for support type inquiry through isa, cast, and dyn_cast:
3002 static bool classof(const Instruction *I) {
3003 return (I->getOpcode() == Instruction::Ret);
3004 }
3005 static bool classof(const Value *V) {
3006 return isa<Instruction>(V) && classof(cast<Instruction>(V));
3007 }
3008
3009private:
3010 BasicBlock *getSuccessor(unsigned idx) const {
3011 llvm_unreachable("ReturnInst has no successors!")__builtin_unreachable();
3012 }
3013
3014 void setSuccessor(unsigned idx, BasicBlock *B) {
3015 llvm_unreachable("ReturnInst has no successors!")__builtin_unreachable();
3016 }
3017};
3018
3019template <>
3020struct OperandTraits<ReturnInst> : public VariadicOperandTraits<ReturnInst> {
3021};
3022
3023DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ReturnInst, Value)ReturnInst::op_iterator ReturnInst::op_begin() { return OperandTraits
<ReturnInst>::op_begin(this); } ReturnInst::const_op_iterator
ReturnInst::op_begin() const { return OperandTraits<ReturnInst
>::op_begin(const_cast<ReturnInst*>(this)); } ReturnInst
::op_iterator ReturnInst::op_end() { return OperandTraits<
ReturnInst>::op_end(this); } ReturnInst::const_op_iterator
ReturnInst::op_end() const { return OperandTraits<ReturnInst
>::op_end(const_cast<ReturnInst*>(this)); } Value *ReturnInst
::getOperand(unsigned i_nocapture) const { ((void)0); return cast_or_null
<Value>( OperandTraits<ReturnInst>::op_begin(const_cast
<ReturnInst*>(this))[i_nocapture].get()); } void ReturnInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((
void)0); OperandTraits<ReturnInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned ReturnInst::getNumOperands() const
{ return OperandTraits<ReturnInst>::operands(this); } template
<int Idx_nocapture> Use &ReturnInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &ReturnInst::Op() const { return
this->OpFrom<Idx_nocapture>(this); }
3024
3025//===----------------------------------------------------------------------===//
3026// BranchInst Class
3027//===----------------------------------------------------------------------===//
3028
3029//===---------------------------------------------------------------------------
3030/// Conditional or Unconditional Branch instruction.
3031///
3032class BranchInst : public Instruction {
3033 /// Ops list - Branches are strange. The operands are ordered:
3034 /// [Cond, FalseDest,] TrueDest. This makes some accessors faster because
3035 /// they don't have to check for cond/uncond branchness. These are mostly
3036 /// accessed relative from op_end().
3037 BranchInst(const BranchInst &BI);
3038 // BranchInst constructors (where {B, T, F} are blocks, and C is a condition):
3039 // BranchInst(BB *B) - 'br B'
3040 // BranchInst(BB* T, BB *F, Value *C) - 'br C, T, F'
3041 // BranchInst(BB* B, Inst *I) - 'br B' insert before I
3042 // BranchInst(BB* T, BB *F, Value *C, Inst *I) - 'br C, T, F', insert before I
3043 // BranchInst(BB* B, BB *I) - 'br B' insert at end
3044 // BranchInst(BB* T, BB *F, Value *C, BB *I) - 'br C, T, F', insert at end
3045 explicit BranchInst(BasicBlock *IfTrue, Instruction *InsertBefore = nullptr);
3046 BranchInst(BasicBlock *IfTrue, BasicBlock *IfFalse, Value *Cond,
3047 Instruction *InsertBefore = nullptr);
3048 BranchInst(BasicBlock *IfTrue, BasicBlock *InsertAtEnd);
3049 BranchInst(BasicBlock *IfTrue, BasicBlock *IfFalse, Value *Cond,
3050 BasicBlock *InsertAtEnd);
3051
3052 void AssertOK();
3053
3054protected:
3055 // Note: Instruction needs to be a friend here to call cloneImpl.
3056 friend class Instruction;
3057
3058 BranchInst *cloneImpl() const;
3059
3060public:
3061 /// Iterator type that casts an operand to a basic block.
3062 ///
3063 /// This only makes sense because the successors are stored as adjacent
3064 /// operands for branch instructions.
3065 struct succ_op_iterator
3066 : iterator_adaptor_base<succ_op_iterator, value_op_iterator,
3067 std::random_access_iterator_tag, BasicBlock *,
3068 ptrdiff_t, BasicBlock *, BasicBlock *> {
3069 explicit succ_op_iterator(value_op_iterator I) : iterator_adaptor_base(I) {}
3070
3071 BasicBlock *operator*() const { return cast<BasicBlock>(*I); }
3072 BasicBlock *operator->() const { return operator*(); }
3073 };
3074
3075 /// The const version of `succ_op_iterator`.
3076 struct const_succ_op_iterator
3077 : iterator_adaptor_base<const_succ_op_iterator, const_value_op_iterator,
3078 std::random_access_iterator_tag,
3079 const BasicBlock *, ptrdiff_t, const BasicBlock *,
3080 const BasicBlock *> {
3081 explicit const_succ_op_iterator(const_value_op_iterator I)
3082 : iterator_adaptor_base(I) {}
3083
3084 const BasicBlock *operator*() const { return cast<BasicBlock>(*I); }
3085 const BasicBlock *operator->() const { return operator*(); }
3086 };
3087
3088 static BranchInst *Create(BasicBlock *IfTrue,
3089 Instruction *InsertBefore = nullptr) {
3090 return new(1) BranchInst(IfTrue, InsertBefore);
3091 }
3092
3093 static BranchInst *Create(BasicBlock *IfTrue, BasicBlock *IfFalse,
3094 Value *Cond, Instruction *InsertBefore = nullptr) {
3095 return new(3) BranchInst(IfTrue, IfFalse, Cond, InsertBefore);
3096 }
3097
3098 static BranchInst *Create(BasicBlock *IfTrue, BasicBlock *InsertAtEnd) {
3099 return new(1) BranchInst(IfTrue, InsertAtEnd);
3100 }
3101
3102 static BranchInst *Create(BasicBlock *IfTrue, BasicBlock *IfFalse,
3103 Value *Cond, BasicBlock *InsertAtEnd) {
3104 return new(3) BranchInst(IfTrue, IfFalse, Cond, InsertAtEnd);
3105 }
3106
3107 /// Transparently provide more efficient getOperand methods.
3108 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
3109
3110 bool isUnconditional() const { return getNumOperands() == 1; }
3111 bool isConditional() const { return getNumOperands() == 3; }
3112
3113 Value *getCondition() const {
3114 assert(isConditional() && "Cannot get condition of an uncond branch!")((void)0);
3115 return Op<-3>();
3116 }
3117
3118 void setCondition(Value *V) {
3119 assert(isConditional() && "Cannot set condition of unconditional branch!")((void)0);
3120 Op<-3>() = V;
3121 }
3122
3123 unsigned getNumSuccessors() const { return 1+isConditional(); }
3124
3125 BasicBlock *getSuccessor(unsigned i) const {
3126 assert(i < getNumSuccessors() && "Successor # out of range for Branch!")((void)0);
3127 return cast_or_null<BasicBlock>((&Op<-1>() - i)->get());
3128 }
3129
3130 void setSuccessor(unsigned idx, BasicBlock *NewSucc) {
3131 assert(idx < getNumSuccessors() && "Successor # out of range for Branch!")((void)0);
3132 *(&Op<-1>() - idx) = NewSucc;
3133 }
3134
3135 /// Swap the successors of this branch instruction.
3136 ///
3137 /// Swaps the successors of the branch instruction. This also swaps any
3138 /// branch weight metadata associated with the instruction so that it
3139 /// continues to map correctly to each operand.
3140 void swapSuccessors();
3141
3142 iterator_range<succ_op_iterator> successors() {
3143 return make_range(
3144 succ_op_iterator(std::next(value_op_begin(), isConditional() ? 1 : 0)),
3145 succ_op_iterator(value_op_end()));
3146 }
3147
3148 iterator_range<const_succ_op_iterator> successors() const {
3149 return make_range(const_succ_op_iterator(
3150 std::next(value_op_begin(), isConditional() ? 1 : 0)),
3151 const_succ_op_iterator(value_op_end()));
3152 }
3153
3154 // Methods for support type inquiry through isa, cast, and dyn_cast:
3155 static bool classof(const Instruction *I) {
3156 return (I->getOpcode() == Instruction::Br);
3157 }
3158 static bool classof(const Value *V) {
3159 return isa<Instruction>(V) && classof(cast<Instruction>(V));
3160 }
3161};
3162
3163template <>
3164struct OperandTraits<BranchInst> : public VariadicOperandTraits<BranchInst, 1> {
3165};
3166
3167DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BranchInst, Value)BranchInst::op_iterator BranchInst::op_begin() { return OperandTraits
<BranchInst>::op_begin(this); } BranchInst::const_op_iterator
BranchInst::op_begin() const { return OperandTraits<BranchInst
>::op_begin(const_cast<BranchInst*>(this)); } BranchInst
::op_iterator BranchInst::op_end() { return OperandTraits<
BranchInst>::op_end(this); } BranchInst::const_op_iterator
BranchInst::op_end() const { return OperandTraits<BranchInst
>::op_end(const_cast<BranchInst*>(this)); } Value *BranchInst
::getOperand(unsigned i_nocapture) const { ((void)0); return cast_or_null
<Value>( OperandTraits<BranchInst>::op_begin(const_cast
<BranchInst*>(this))[i_nocapture].get()); } void BranchInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((
void)0); OperandTraits<BranchInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned BranchInst::getNumOperands() const
{ return OperandTraits<BranchInst>::operands(this); } template
<int Idx_nocapture> Use &BranchInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &BranchInst::Op() const { return
this->OpFrom<Idx_nocapture>(this); }
3168
3169//===----------------------------------------------------------------------===//
3170// SwitchInst Class
3171//===----------------------------------------------------------------------===//
3172
3173//===---------------------------------------------------------------------------
3174/// Multiway switch
3175///
3176class SwitchInst : public Instruction {
3177 unsigned ReservedSpace;
3178
3179 // Operand[0] = Value to switch on
3180 // Operand[1] = Default basic block destination
3181 // Operand[2n ] = Value to match
3182 // Operand[2n+1] = BasicBlock to go to on match
3183 SwitchInst(const SwitchInst &SI);
3184
3185 /// Create a new switch instruction, specifying a value to switch on and a
3186 /// default destination. The number of additional cases can be specified here
3187 /// to make memory allocation more efficient. This constructor can also
3188 /// auto-insert before another instruction.
3189 SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases,
3190 Instruction *InsertBefore);
3191
3192 /// Create a new switch instruction, specifying a value to switch on and a
3193 /// default destination. The number of additional cases can be specified here
3194 /// to make memory allocation more efficient. This constructor also
3195 /// auto-inserts at the end of the specified BasicBlock.
3196 SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases,
3197 BasicBlock *InsertAtEnd);
3198
3199 // allocate space for exactly zero operands
3200 void *operator new(size_t S) { return User::operator new(S); }
3201
3202 void init(Value *Value, BasicBlock *Default, unsigned NumReserved);
3203 void growOperands();
3204
3205protected:
3206 // Note: Instruction needs to be a friend here to call cloneImpl.
3207 friend class Instruction;
3208
3209 SwitchInst *cloneImpl() const;
3210
3211public:
3212 void operator delete(void *Ptr) { User::operator delete(Ptr); }
3213
3214 // -2
3215 static const unsigned DefaultPseudoIndex = static_cast<unsigned>(~0L-1);
3216
3217 template <typename CaseHandleT> class CaseIteratorImpl;
3218
3219 /// A handle to a particular switch case. It exposes a convenient interface
3220 /// to both the case value and the successor block.
3221 ///
3222 /// We define this as a template and instantiate it to form both a const and
3223 /// non-const handle.
3224 template <typename SwitchInstT, typename ConstantIntT, typename BasicBlockT>
3225 class CaseHandleImpl {
3226 // Directly befriend both const and non-const iterators.
3227 friend class SwitchInst::CaseIteratorImpl<
3228 CaseHandleImpl<SwitchInstT, ConstantIntT, BasicBlockT>>;
3229
3230 protected:
3231 // Expose the switch type we're parameterized with to the iterator.
3232 using SwitchInstType = SwitchInstT;
3233
3234 SwitchInstT *SI;
3235 ptrdiff_t Index;
3236
3237 CaseHandleImpl() = default;
3238 CaseHandleImpl(SwitchInstT *SI, ptrdiff_t Index) : SI(SI), Index(Index) {}
3239
3240 public:
3241 /// Resolves case value for current case.
3242 ConstantIntT *getCaseValue() const {
3243 assert((unsigned)Index < SI->getNumCases() &&((void)0)
3244 "Index out the number of cases.")((void)0);
3245 return reinterpret_cast<ConstantIntT *>(SI->getOperand(2 + Index * 2));
3246 }
3247
3248 /// Resolves successor for current case.
3249 BasicBlockT *getCaseSuccessor() const {
3250 assert(((unsigned)Index < SI->getNumCases() ||((void)0)
3251 (unsigned)Index == DefaultPseudoIndex) &&((void)0)
3252 "Index out the number of cases.")((void)0);
3253 return SI->getSuccessor(getSuccessorIndex());
3254 }
3255
3256 /// Returns number of current case.
3257 unsigned getCaseIndex() const { return Index; }
3258
3259 /// Returns successor index for current case successor.
3260 unsigned getSuccessorIndex() const {
3261 assert(((unsigned)Index == DefaultPseudoIndex ||((void)0)
3262 (unsigned)Index < SI->getNumCases()) &&((void)0)
3263 "Index out the number of cases.")((void)0);
3264 return (unsigned)Index != DefaultPseudoIndex ? Index + 1 : 0;
3265 }
3266
3267 bool operator==(const CaseHandleImpl &RHS) const {
3268 assert(SI == RHS.SI && "Incompatible operators.")((void)0);
3269 return Index == RHS.Index;
3270 }
3271 };
3272
3273 using ConstCaseHandle =
3274 CaseHandleImpl<const SwitchInst, const ConstantInt, const BasicBlock>;
3275
3276 class CaseHandle
3277 : public CaseHandleImpl<SwitchInst, ConstantInt, BasicBlock> {
3278 friend class SwitchInst::CaseIteratorImpl<CaseHandle>;
3279
3280 public:
3281 CaseHandle(SwitchInst *SI, ptrdiff_t Index) : CaseHandleImpl(SI, Index) {}
3282
3283 /// Sets the new value for current case.
3284 void setValue(ConstantInt *V) {
3285 assert((unsigned)Index < SI->getNumCases() &&((void)0)
3286 "Index out the number of cases.")((void)0);
3287 SI->setOperand(2 + Index*2, reinterpret_cast<Value*>(V));
3288 }
3289
3290 /// Sets the new successor for current case.
3291 void setSuccessor(BasicBlock *S) {
3292 SI->setSuccessor(getSuccessorIndex(), S);
3293 }
3294 };
3295
3296 template <typename CaseHandleT>
3297 class CaseIteratorImpl
3298 : public iterator_facade_base<CaseIteratorImpl<CaseHandleT>,
3299 std::random_access_iterator_tag,
3300 CaseHandleT> {
3301 using SwitchInstT = typename CaseHandleT::SwitchInstType;
3302
3303 CaseHandleT Case;
3304
3305 public:
3306 /// Default constructed iterator is in an invalid state until assigned to
3307 /// a case for a particular switch.
3308 CaseIteratorImpl() = default;
3309
3310 /// Initializes case iterator for given SwitchInst and for given
3311 /// case number.
3312 CaseIteratorImpl(SwitchInstT *SI, unsigned CaseNum) : Case(SI, CaseNum) {}
3313
3314 /// Initializes case iterator for given SwitchInst and for given
3315 /// successor index.
3316 static CaseIteratorImpl fromSuccessorIndex(SwitchInstT *SI,
3317 unsigned SuccessorIndex) {
3318 assert(SuccessorIndex < SI->getNumSuccessors() &&((void)0)
3319 "Successor index # out of range!")((void)0);
3320 return SuccessorIndex != 0 ? CaseIteratorImpl(SI, SuccessorIndex - 1)
3321 : CaseIteratorImpl(SI, DefaultPseudoIndex);
3322 }
3323
3324 /// Support converting to the const variant. This will be a no-op for const
3325 /// variant.
3326 operator CaseIteratorImpl<ConstCaseHandle>() const {
3327 return CaseIteratorImpl<ConstCaseHandle>(Case.SI, Case.Index);
3328 }
3329
3330 CaseIteratorImpl &operator+=(ptrdiff_t N) {
3331 // Check index correctness after addition.
3332 // Note: Index == getNumCases() means end().
3333 assert(Case.Index + N >= 0 &&((void)0)
3334 (unsigned)(Case.Index + N) <= Case.SI->getNumCases() &&((void)0)
3335 "Case.Index out the number of cases.")((void)0);
3336 Case.Index += N;
3337 return *this;
3338 }
3339 CaseIteratorImpl &operator-=(ptrdiff_t N) {
3340 // Check index correctness after subtraction.
3341 // Note: Case.Index == getNumCases() means end().
3342 assert(Case.Index - N >= 0 &&((void)0)
3343 (unsigned)(Case.Index - N) <= Case.SI->getNumCases() &&((void)0)
3344 "Case.Index out the number of cases.")((void)0);
3345 Case.Index -= N;
3346 return *this;
3347 }
3348 ptrdiff_t operator-(const CaseIteratorImpl &RHS) const {
3349 assert(Case.SI == RHS.Case.SI && "Incompatible operators.")((void)0);
3350 return Case.Index - RHS.Case.Index;
3351 }
3352 bool operator==(const CaseIteratorImpl &RHS) const {
3353 return Case == RHS.Case;
3354 }
3355 bool operator<(const CaseIteratorImpl &RHS) const {
3356 assert(Case.SI == RHS.Case.SI && "Incompatible operators.")((void)0);
3357 return Case.Index < RHS.Case.Index;
3358 }
3359 CaseHandleT &operator*() { return Case; }
3360 const CaseHandleT &operator*() const { return Case; }
3361 };
3362
3363 using CaseIt = CaseIteratorImpl<CaseHandle>;
3364 using ConstCaseIt = CaseIteratorImpl<ConstCaseHandle>;
3365
3366 static SwitchInst *Create(Value *Value, BasicBlock *Default,
3367 unsigned NumCases,
3368 Instruction *InsertBefore = nullptr) {
3369 return new SwitchInst(Value, Default, NumCases, InsertBefore);
3370 }
3371
3372 static SwitchInst *Create(Value *Value, BasicBlock *Default,
3373 unsigned NumCases, BasicBlock *InsertAtEnd) {
3374 return new SwitchInst(Value, Default, NumCases, InsertAtEnd);
3375 }
3376
3377 /// Provide fast operand accessors
3378 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
3379
3380 // Accessor Methods for Switch stmt
3381 Value *getCondition() const { return getOperand(0); }
3382 void setCondition(Value *V) { setOperand(0, V); }
3383
3384 BasicBlock *getDefaultDest() const {
3385 return cast<BasicBlock>(getOperand(1));
3386 }
3387
3388 void setDefaultDest(BasicBlock *DefaultCase) {
3389 setOperand(1, reinterpret_cast<Value*>(DefaultCase));
3390 }
3391
3392 /// Return the number of 'cases' in this switch instruction, excluding the
3393 /// default case.
3394 unsigned getNumCases() const {
3395 return getNumOperands()/2 - 1;
3396 }
3397
3398 /// Returns a read/write iterator that points to the first case in the
3399 /// SwitchInst.
3400 CaseIt case_begin() {
3401 return CaseIt(this, 0);
3402 }
3403
3404 /// Returns a read-only iterator that points to the first case in the
3405 /// SwitchInst.
3406 ConstCaseIt case_begin() const {
3407 return ConstCaseIt(this, 0);
3408 }
3409
3410 /// Returns a read/write iterator that points one past the last in the
3411 /// SwitchInst.
3412 CaseIt case_end() {
3413 return CaseIt(this, getNumCases());
3414 }
3415
3416 /// Returns a read-only iterator that points one past the last in the
3417 /// SwitchInst.
3418 ConstCaseIt case_end() const {
3419 return ConstCaseIt(this, getNumCases());
3420 }
3421
3422 /// Iteration adapter for range-for loops.
3423 iterator_range<CaseIt> cases() {
3424 return make_range(case_begin(), case_end());
3425 }
3426
3427 /// Constant iteration adapter for range-for loops.
3428 iterator_range<ConstCaseIt> cases() const {
3429 return make_range(case_begin(), case_end());
3430 }
3431
3432 /// Returns an iterator that points to the default case.
3433 /// Note: this iterator allows to resolve successor only. Attempt
3434 /// to resolve case value causes an assertion.
3435 /// Also note, that increment and decrement also causes an assertion and
3436 /// makes iterator invalid.
3437 CaseIt case_default() {
3438 return CaseIt(this, DefaultPseudoIndex);
3439 }
3440 ConstCaseIt case_default() const {
3441 return ConstCaseIt(this, DefaultPseudoIndex);
3442 }
3443
3444 /// Search all of the case values for the specified constant. If it is
3445 /// explicitly handled, return the case iterator of it, otherwise return
3446 /// default case iterator to indicate that it is handled by the default
3447 /// handler.
3448 CaseIt findCaseValue(const ConstantInt *C) {
3449 CaseIt I = llvm::find_if(
3450 cases(), [C](CaseHandle &Case) { return Case.getCaseValue() == C; });
3451 if (I != case_end())
3452 return I;
3453
3454 return case_default();
3455 }
3456 ConstCaseIt findCaseValue(const ConstantInt *C) const {
3457 ConstCaseIt I = llvm::find_if(cases(), [C](ConstCaseHandle &Case) {
3458 return Case.getCaseValue() == C;
3459 });
3460 if (I != case_end())
3461 return I;
3462
3463 return case_default();
3464 }
3465
3466 /// Finds the unique case value for a given successor. Returns null if the
3467 /// successor is not found, not unique, or is the default case.
3468 ConstantInt *findCaseDest(BasicBlock *BB) {
3469 if (BB == getDefaultDest())
3470 return nullptr;
3471
3472 ConstantInt *CI = nullptr;
3473 for (auto Case : cases()) {
3474 if (Case.getCaseSuccessor() != BB)
3475 continue;
3476
3477 if (CI)
3478 return nullptr; // Multiple cases lead to BB.
3479
3480 CI = Case.getCaseValue();
3481 }
3482
3483 return CI;
3484 }
3485
3486 /// Add an entry to the switch instruction.
3487 /// Note:
3488 /// This action invalidates case_end(). Old case_end() iterator will
3489 /// point to the added case.
3490 void addCase(ConstantInt *OnVal, BasicBlock *Dest);
3491
3492 /// This method removes the specified case and its successor from the switch
3493 /// instruction. Note that this operation may reorder the remaining cases at
3494 /// index idx and above.
3495 /// Note:
3496 /// This action invalidates iterators for all cases following the one removed,
3497 /// including the case_end() iterator. It returns an iterator for the next
3498 /// case.
3499 CaseIt removeCase(CaseIt I);
3500
3501 unsigned getNumSuccessors() const { return getNumOperands()/2; }
3502 BasicBlock *getSuccessor(unsigned idx) const {
3503 assert(idx < getNumSuccessors() &&"Successor idx out of range for switch!")((void)0);
3504 return cast<BasicBlock>(getOperand(idx*2+1));
3505 }
3506 void setSuccessor(unsigned idx, BasicBlock *NewSucc) {
3507 assert(idx < getNumSuccessors() && "Successor # out of range for switch!")((void)0);
3508 setOperand(idx * 2 + 1, NewSucc);
3509 }
3510
3511 // Methods for support type inquiry through isa, cast, and dyn_cast:
3512 static bool classof(const Instruction *I) {
3513 return I->getOpcode() == Instruction::Switch;
3514 }
3515 static bool classof(const Value *V) {
3516 return isa<Instruction>(V) && classof(cast<Instruction>(V));
3517 }
3518};
3519
3520/// A wrapper class to simplify modification of SwitchInst cases along with
3521/// their prof branch_weights metadata.
3522class SwitchInstProfUpdateWrapper {
3523 SwitchInst &SI;
3524 Optional<SmallVector<uint32_t, 8> > Weights = None;
3525 bool Changed = false;
3526
3527protected:
3528 static MDNode *getProfBranchWeightsMD(const SwitchInst &SI);
3529
3530 MDNode *buildProfBranchWeightsMD();
3531
3532 void init();
3533
3534public:
3535 using CaseWeightOpt = Optional<uint32_t>;
3536 SwitchInst *operator->() { return &SI; }
3537 SwitchInst &operator*() { return SI; }
3538 operator SwitchInst *() { return &SI; }
3539
3540 SwitchInstProfUpdateWrapper(SwitchInst &SI) : SI(SI) { init(); }
3541
3542 ~SwitchInstProfUpdateWrapper() {
3543 if (Changed)
3544 SI.setMetadata(LLVMContext::MD_prof, buildProfBranchWeightsMD());
3545 }
3546
3547 /// Delegate the call to the underlying SwitchInst::removeCase() and remove
3548 /// correspondent branch weight.
3549 SwitchInst::CaseIt removeCase(SwitchInst::CaseIt I);
3550
3551 /// Delegate the call to the underlying SwitchInst::addCase() and set the
3552 /// specified branch weight for the added case.
3553 void addCase(ConstantInt *OnVal, BasicBlock *Dest, CaseWeightOpt W);
3554
3555 /// Delegate the call to the underlying SwitchInst::eraseFromParent() and mark
3556 /// this object to not touch the underlying SwitchInst in destructor.
3557 SymbolTableList<Instruction>::iterator eraseFromParent();
3558
3559 void setSuccessorWeight(unsigned idx, CaseWeightOpt W);
3560 CaseWeightOpt getSuccessorWeight(unsigned idx);
3561
3562 static CaseWeightOpt getSuccessorWeight(const SwitchInst &SI, unsigned idx);
3563};
3564
3565template <>
3566struct OperandTraits<SwitchInst> : public HungoffOperandTraits<2> {
3567};
3568
3569DEFINE_TRANSPARENT_OPERAND_ACCESSORS(SwitchInst, Value)SwitchInst::op_iterator SwitchInst::op_begin() { return OperandTraits
<SwitchInst>::op_begin(this); } SwitchInst::const_op_iterator
SwitchInst::op_begin() const { return OperandTraits<SwitchInst
>::op_begin(const_cast<SwitchInst*>(this)); } SwitchInst
::op_iterator SwitchInst::op_end() { return OperandTraits<
SwitchInst>::op_end(this); } SwitchInst::const_op_iterator
SwitchInst::op_end() const { return OperandTraits<SwitchInst
>::op_end(const_cast<SwitchInst*>(this)); } Value *SwitchInst
::getOperand(unsigned i_nocapture) const { ((void)0); return cast_or_null
<Value>( OperandTraits<SwitchInst>::op_begin(const_cast
<SwitchInst*>(this))[i_nocapture].get()); } void SwitchInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((
void)0); OperandTraits<SwitchInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned SwitchInst::getNumOperands() const
{ return OperandTraits<SwitchInst>::operands(this); } template
<int Idx_nocapture> Use &SwitchInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &SwitchInst::Op() const { return
this->OpFrom<Idx_nocapture>(this); }
3570
3571//===----------------------------------------------------------------------===//
3572// IndirectBrInst Class
3573//===----------------------------------------------------------------------===//
3574
3575//===---------------------------------------------------------------------------
3576/// Indirect Branch Instruction.
3577///
3578class IndirectBrInst : public Instruction {
3579 unsigned ReservedSpace;
3580
3581 // Operand[0] = Address to jump to
3582 // Operand[n+1] = n-th destination
3583 IndirectBrInst(const IndirectBrInst &IBI);
3584
3585 /// Create a new indirectbr instruction, specifying an
3586 /// Address to jump to. The number of expected destinations can be specified
3587 /// here to make memory allocation more efficient. This constructor can also
3588 /// autoinsert before another instruction.
3589 IndirectBrInst(Value *Address, unsigned NumDests, Instruction *InsertBefore);
3590
3591 /// Create a new indirectbr instruction, specifying an
3592 /// Address to jump to. The number of expected destinations can be specified
3593 /// here to make memory allocation more efficient. This constructor also
3594 /// autoinserts at the end of the specified BasicBlock.
3595 IndirectBrInst(Value *Address, unsigned NumDests, BasicBlock *InsertAtEnd);
3596
3597 // allocate space for exactly zero operands
3598 void *operator new(size_t S) { return User::operator new(S); }
3599
3600 void init(Value *Address, unsigned NumDests);
3601 void growOperands();
3602
3603protected:
3604 // Note: Instruction needs to be a friend here to call cloneImpl.
3605 friend class Instruction;
3606
3607 IndirectBrInst *cloneImpl() const;
3608
3609public:
3610 void operator delete(void *Ptr) { User::operator delete(Ptr); }
3611
3612 /// Iterator type that casts an operand to a basic block.
3613 ///
3614 /// This only makes sense because the successors are stored as adjacent
3615 /// operands for indirectbr instructions.
3616 struct succ_op_iterator
3617 : iterator_adaptor_base<succ_op_iterator, value_op_iterator,
3618 std::random_access_iterator_tag, BasicBlock *,
3619 ptrdiff_t, BasicBlock *, BasicBlock *> {
3620 explicit succ_op_iterator(value_op_iterator I) : iterator_adaptor_base(I) {}
3621
3622 BasicBlock *operator*() const { return cast<BasicBlock>(*I); }
3623 BasicBlock *operator->() const { return operator*(); }
3624 };
3625
3626 /// The const version of `succ_op_iterator`.
3627 struct const_succ_op_iterator
3628 : iterator_adaptor_base<const_succ_op_iterator, const_value_op_iterator,
3629 std::random_access_iterator_tag,
3630 const BasicBlock *, ptrdiff_t, const BasicBlock *,
3631 const BasicBlock *> {
3632 explicit const_succ_op_iterator(const_value_op_iterator I)
3633 : iterator_adaptor_base(I) {}
3634
3635 const BasicBlock *operator*() const { return cast<BasicBlock>(*I); }
3636 const BasicBlock *operator->() const { return operator*(); }
3637 };
3638
3639 static IndirectBrInst *Create(Value *Address, unsigned NumDests,
3640 Instruction *InsertBefore = nullptr) {
3641 return new IndirectBrInst(Address, NumDests, InsertBefore);
3642 }
3643
3644 static IndirectBrInst *Create(Value *Address, unsigned NumDests,
3645 BasicBlock *InsertAtEnd) {
3646 return new IndirectBrInst(Address, NumDests, InsertAtEnd);
3647 }
3648
3649 /// Provide fast operand accessors.
3650 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
3651
3652 // Accessor Methods for IndirectBrInst instruction.
3653 Value *getAddress() { return getOperand(0); }
3654 const Value *getAddress() const { return getOperand(0); }
3655 void setAddress(Value *V) { setOperand(0, V); }
3656
3657 /// return the number of possible destinations in this
3658 /// indirectbr instruction.
3659 unsigned getNumDestinations() const { return getNumOperands()-1; }
3660
3661 /// Return the specified destination.
3662 BasicBlock *getDestination(unsigned i) { return getSuccessor(i); }
3663 const BasicBlock *getDestination(unsigned i) const { return getSuccessor(i); }
3664
3665 /// Add a destination.
3666 ///
3667 void addDestination(BasicBlock *Dest);
3668
3669 /// This method removes the specified successor from the
3670 /// indirectbr instruction.
3671 void removeDestination(unsigned i);
3672
3673 unsigned getNumSuccessors() const { return getNumOperands()-1; }
3674 BasicBlock *getSuccessor(unsigned i) const {
3675 return cast<BasicBlock>(getOperand(i+1));
3676 }
3677 void setSuccessor(unsigned i, BasicBlock *NewSucc) {
3678 setOperand(i + 1, NewSucc);
3679 }
3680
3681 iterator_range<succ_op_iterator> successors() {
3682 return make_range(succ_op_iterator(std::next(value_op_begin())),
3683 succ_op_iterator(value_op_end()));
3684 }
3685
3686 iterator_range<const_succ_op_iterator> successors() const {
3687 return make_range(const_succ_op_iterator(std::next(value_op_begin())),
3688 const_succ_op_iterator(value_op_end()));
3689 }
3690
3691 // Methods for support type inquiry through isa, cast, and dyn_cast:
3692 static bool classof(const Instruction *I) {
3693 return I->getOpcode() == Instruction::IndirectBr;
3694 }
3695 static bool classof(const Value *V) {
3696 return isa<Instruction>(V) && classof(cast<Instruction>(V));
3697 }
3698};
3699
3700template <>
3701struct OperandTraits<IndirectBrInst> : public HungoffOperandTraits<1> {
3702};
3703
3704DEFINE_TRANSPARENT_OPERAND_ACCESSORS(IndirectBrInst, Value)IndirectBrInst::op_iterator IndirectBrInst::op_begin() { return
OperandTraits<IndirectBrInst>::op_begin(this); } IndirectBrInst
::const_op_iterator IndirectBrInst::op_begin() const { return
OperandTraits<IndirectBrInst>::op_begin(const_cast<
IndirectBrInst*>(this)); } IndirectBrInst::op_iterator IndirectBrInst
::op_end() { return OperandTraits<IndirectBrInst>::op_end
(this); } IndirectBrInst::const_op_iterator IndirectBrInst::op_end
() const { return OperandTraits<IndirectBrInst>::op_end
(const_cast<IndirectBrInst*>(this)); } Value *IndirectBrInst
::getOperand(unsigned i_nocapture) const { ((void)0); return cast_or_null
<Value>( OperandTraits<IndirectBrInst>::op_begin(
const_cast<IndirectBrInst*>(this))[i_nocapture].get());
} void IndirectBrInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { ((void)0); OperandTraits<IndirectBrInst
>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
IndirectBrInst::getNumOperands() const { return OperandTraits
<IndirectBrInst>::operands(this); } template <int Idx_nocapture
> Use &IndirectBrInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
const Use &IndirectBrInst::Op() const { return this->
OpFrom<Idx_nocapture>(this); }
3705
3706//===----------------------------------------------------------------------===//
3707// InvokeInst Class
3708//===----------------------------------------------------------------------===//
3709
3710/// Invoke instruction. The SubclassData field is used to hold the
3711/// calling convention of the call.
3712///
3713class InvokeInst : public CallBase {
3714 /// The number of operands for this call beyond the called function,
3715 /// arguments, and operand bundles.
3716 static constexpr int NumExtraOperands = 2;
3717
3718 /// The index from the end of the operand array to the normal destination.
3719 static constexpr int NormalDestOpEndIdx = -3;
3720
3721 /// The index from the end of the operand array to the unwind destination.
3722 static constexpr int UnwindDestOpEndIdx = -2;
3723
3724 InvokeInst(const InvokeInst &BI);
3725
3726 /// Construct an InvokeInst given a range of arguments.
3727 ///
3728 /// Construct an InvokeInst from a range of arguments
3729 inline InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3730 BasicBlock *IfException, ArrayRef<Value *> Args,
3731 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
3732 const Twine &NameStr, Instruction *InsertBefore);
3733
3734 inline InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3735 BasicBlock *IfException, ArrayRef<Value *> Args,
3736 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
3737 const Twine &NameStr, BasicBlock *InsertAtEnd);
3738
3739 void init(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3740 BasicBlock *IfException, ArrayRef<Value *> Args,
3741 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
3742
3743 /// Compute the number of operands to allocate.
3744 static int ComputeNumOperands(int NumArgs, int NumBundleInputs = 0) {
3745 // We need one operand for the called function, plus our extra operands and
3746 // the input operand counts provided.
3747 return 1 + NumExtraOperands + NumArgs + NumBundleInputs;
3748 }
3749
3750protected:
3751 // Note: Instruction needs to be a friend here to call cloneImpl.
3752 friend class Instruction;
3753
3754 InvokeInst *cloneImpl() const;
3755
3756public:
3757 static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3758 BasicBlock *IfException, ArrayRef<Value *> Args,
3759 const Twine &NameStr,
3760 Instruction *InsertBefore = nullptr) {
3761 int NumOperands = ComputeNumOperands(Args.size());
3762 return new (NumOperands)
3763 InvokeInst(Ty, Func, IfNormal, IfException, Args, None, NumOperands,
3764 NameStr, InsertBefore);
3765 }
3766
3767 static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3768 BasicBlock *IfException, ArrayRef<Value *> Args,
3769 ArrayRef<OperandBundleDef> Bundles = None,
3770 const Twine &NameStr = "",
3771 Instruction *InsertBefore = nullptr) {
3772 int NumOperands =
3773 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
3774 unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
3775
3776 return new (NumOperands, DescriptorBytes)
3777 InvokeInst(Ty, Func, IfNormal, IfException, Args, Bundles, NumOperands,
3778 NameStr, InsertBefore);
3779 }
3780
3781 static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3782 BasicBlock *IfException, ArrayRef<Value *> Args,
3783 const Twine &NameStr, BasicBlock *InsertAtEnd) {
3784 int NumOperands = ComputeNumOperands(Args.size());
3785 return new (NumOperands)
3786 InvokeInst(Ty, Func, IfNormal, IfException, Args, None, NumOperands,
3787 NameStr, InsertAtEnd);
3788 }
3789
3790 static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3791 BasicBlock *IfException, ArrayRef<Value *> Args,
3792 ArrayRef<OperandBundleDef> Bundles,
3793 const Twine &NameStr, BasicBlock *InsertAtEnd) {
3794 int NumOperands =
3795 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
3796 unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
3797
3798 return new (NumOperands, DescriptorBytes)
3799 InvokeInst(Ty, Func, IfNormal, IfException, Args, Bundles, NumOperands,
3800 NameStr, InsertAtEnd);
3801 }
3802
3803 static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal,
3804 BasicBlock *IfException, ArrayRef<Value *> Args,
3805 const Twine &NameStr,
3806 Instruction *InsertBefore = nullptr) {
3807 return Create(Func.getFunctionType(), Func.getCallee(), IfNormal,
3808 IfException, Args, None, NameStr, InsertBefore);
3809 }
3810
3811 static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal,
3812 BasicBlock *IfException, ArrayRef<Value *> Args,
3813 ArrayRef<OperandBundleDef> Bundles = None,
3814 const Twine &NameStr = "",
3815 Instruction *InsertBefore = nullptr) {
3816 return Create(Func.getFunctionType(), Func.getCallee(), IfNormal,
3817 IfException, Args, Bundles, NameStr, InsertBefore);
3818 }
3819
3820 static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal,
3821 BasicBlock *IfException, ArrayRef<Value *> Args,
3822 const Twine &NameStr, BasicBlock *InsertAtEnd) {
3823 return Create(Func.getFunctionType(), Func.getCallee(), IfNormal,
3824 IfException, Args, NameStr, InsertAtEnd);
3825 }
3826
3827 static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal,
3828 BasicBlock *IfException, ArrayRef<Value *> Args,
3829 ArrayRef<OperandBundleDef> Bundles,
3830 const Twine &NameStr, BasicBlock *InsertAtEnd) {
3831 return Create(Func.getFunctionType(), Func.getCallee(), IfNormal,
3832 IfException, Args, Bundles, NameStr, InsertAtEnd);
3833 }
3834
3835 /// Create a clone of \p II with a different set of operand bundles and
3836 /// insert it before \p InsertPt.
3837 ///
3838 /// The returned invoke instruction is identical to \p II in every way except
3839 /// that the operand bundles for the new instruction are set to the operand
3840 /// bundles in \p Bundles.
3841 static InvokeInst *Create(InvokeInst *II, ArrayRef<OperandBundleDef> Bundles,
3842 Instruction *InsertPt = nullptr);
3843
3844 // get*Dest - Return the destination basic blocks...
3845 BasicBlock *getNormalDest() const {
3846 return cast<BasicBlock>(Op<NormalDestOpEndIdx>());
3847 }
3848 BasicBlock *getUnwindDest() const {
3849 return cast<BasicBlock>(Op<UnwindDestOpEndIdx>());
3850 }
3851 void setNormalDest(BasicBlock *B) {
3852 Op<NormalDestOpEndIdx>() = reinterpret_cast<Value *>(B);
3853 }
3854 void setUnwindDest(BasicBlock *B) {
3855 Op<UnwindDestOpEndIdx>() = reinterpret_cast<Value *>(B);
3856 }
3857
3858 /// Get the landingpad instruction from the landing pad
3859 /// block (the unwind destination).
3860 LandingPadInst *getLandingPadInst() const;
3861
3862 BasicBlock *getSuccessor(unsigned i) const {
3863 assert(i < 2 && "Successor # out of range for invoke!")((void)0);
3864 return i == 0 ? getNormalDest() : getUnwindDest();
3865 }
3866
3867 void setSuccessor(unsigned i, BasicBlock *NewSucc) {
3868 assert(i < 2 && "Successor # out of range for invoke!")((void)0);
3869 if (i == 0)
3870 setNormalDest(NewSucc);
3871 else
3872 setUnwindDest(NewSucc);
3873 }
3874
3875 unsigned getNumSuccessors() const { return 2; }
3876
3877 // Methods for support type inquiry through isa, cast, and dyn_cast:
3878 static bool classof(const Instruction *I) {
3879 return (I->getOpcode() == Instruction::Invoke);
3880 }
3881 static bool classof(const Value *V) {
3882 return isa<Instruction>(V) && classof(cast<Instruction>(V));
3883 }
3884
3885private:
3886 // Shadow Instruction::setInstructionSubclassData with a private forwarding
3887 // method so that subclasses cannot accidentally use it.
3888 template <typename Bitfield>
3889 void setSubclassData(typename Bitfield::Type Value) {
3890 Instruction::setSubclassData<Bitfield>(Value);
3891 }
3892};
3893
3894InvokeInst::InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3895 BasicBlock *IfException, ArrayRef<Value *> Args,
3896 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
3897 const Twine &NameStr, Instruction *InsertBefore)
3898 : CallBase(Ty->getReturnType(), Instruction::Invoke,
3899 OperandTraits<CallBase>::op_end(this) - NumOperands, NumOperands,
3900 InsertBefore) {
3901 init(Ty, Func, IfNormal, IfException, Args, Bundles, NameStr);
3902}
3903
3904InvokeInst::InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3905 BasicBlock *IfException, ArrayRef<Value *> Args,
3906 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
3907 const Twine &NameStr, BasicBlock *InsertAtEnd)
3908 : CallBase(Ty->getReturnType(), Instruction::Invoke,
3909 OperandTraits<CallBase>::op_end(this) - NumOperands, NumOperands,
3910 InsertAtEnd) {
3911 init(Ty, Func, IfNormal, IfException, Args, Bundles, NameStr);
3912}
3913
3914//===----------------------------------------------------------------------===//
3915// CallBrInst Class
3916//===----------------------------------------------------------------------===//
3917
3918/// CallBr instruction, tracking function calls that may not return control but
3919/// instead transfer it to a third location. The SubclassData field is used to
3920/// hold the calling convention of the call.
3921///
3922class CallBrInst : public CallBase {
3923
3924 unsigned NumIndirectDests;
3925
3926 CallBrInst(const CallBrInst &BI);
3927
3928 /// Construct a CallBrInst given a range of arguments.
3929 ///
3930 /// Construct a CallBrInst from a range of arguments
3931 inline CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest,
3932 ArrayRef<BasicBlock *> IndirectDests,
3933 ArrayRef<Value *> Args,
3934 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
3935 const Twine &NameStr, Instruction *InsertBefore);
3936
3937 inline CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest,
3938 ArrayRef<BasicBlock *> IndirectDests,
3939 ArrayRef<Value *> Args,
3940 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
3941 const Twine &NameStr, BasicBlock *InsertAtEnd);
3942
3943 void init(FunctionType *FTy, Value *Func, BasicBlock *DefaultDest,
3944 ArrayRef<BasicBlock *> IndirectDests, ArrayRef<Value *> Args,
3945 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
3946
3947 /// Should the Indirect Destinations change, scan + update the Arg list.
3948 void updateArgBlockAddresses(unsigned i, BasicBlock *B);
3949
3950 /// Compute the number of operands to allocate.
3951 static int ComputeNumOperands(int NumArgs, int NumIndirectDests,
3952 int NumBundleInputs = 0) {
3953 // We need one operand for the called function, plus our extra operands and
3954 // the input operand counts provided.
3955 return 2 + NumIndirectDests + NumArgs + NumBundleInputs;
3956 }
3957
3958protected:
3959 // Note: Instruction needs to be a friend here to call cloneImpl.
3960 friend class Instruction;
3961
3962 CallBrInst *cloneImpl() const;
3963
3964public:
3965 static CallBrInst *Create(FunctionType *Ty, Value *Func,
3966 BasicBlock *DefaultDest,
3967 ArrayRef<BasicBlock *> IndirectDests,
3968 ArrayRef<Value *> Args, const Twine &NameStr,
3969 Instruction *InsertBefore = nullptr) {
3970 int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size());
3971 return new (NumOperands)
3972 CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, None,
3973 NumOperands, NameStr, InsertBefore);
3974 }
3975
3976 static CallBrInst *Create(FunctionType *Ty, Value *Func,
3977 BasicBlock *DefaultDest,
3978 ArrayRef<BasicBlock *> IndirectDests,
3979 ArrayRef<Value *> Args,
3980 ArrayRef<OperandBundleDef> Bundles = None,
3981 const Twine &NameStr = "",
3982 Instruction *InsertBefore = nullptr) {
3983 int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size(),
3984 CountBundleInputs(Bundles));
3985 unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
3986
3987 return new (NumOperands, DescriptorBytes)
3988 CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, Bundles,
3989 NumOperands, NameStr, InsertBefore);
3990 }
3991
3992 static CallBrInst *Create(FunctionType *Ty, Value *Func,
3993 BasicBlock *DefaultDest,
3994 ArrayRef<BasicBlock *> IndirectDests,
3995 ArrayRef<Value *> Args, const Twine &NameStr,
3996 BasicBlock *InsertAtEnd) {
3997 int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size());
3998 return new (NumOperands)
3999 CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, None,
4000 NumOperands, NameStr, InsertAtEnd);
4001 }
4002
4003 static CallBrInst *Create(FunctionType *Ty, Value *Func,
4004 BasicBlock *DefaultDest,
4005 ArrayRef<BasicBlock *> IndirectDests,
4006 ArrayRef<Value *> Args,
4007 ArrayRef<OperandBundleDef> Bundles,
4008 const Twine &NameStr, BasicBlock *InsertAtEnd) {
4009 int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size(),
4010 CountBundleInputs(Bundles));
4011 unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
4012
4013 return new (NumOperands, DescriptorBytes)
4014 CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, Bundles,
4015 NumOperands, NameStr, InsertAtEnd);
4016 }
4017
4018 static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest,
4019 ArrayRef<BasicBlock *> IndirectDests,
4020 ArrayRef<Value *> Args, const Twine &NameStr,
4021 Instruction *InsertBefore = nullptr) {
4022 return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest,
4023 IndirectDests, Args, NameStr, InsertBefore);
4024 }
4025
4026 static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest,
4027 ArrayRef<BasicBlock *> IndirectDests,
4028 ArrayRef<Value *> Args,
4029 ArrayRef<OperandBundleDef> Bundles = None,
4030 const Twine &NameStr = "",
4031 Instruction *InsertBefore = nullptr) {
4032 return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest,
4033 IndirectDests, Args, Bundles, NameStr, InsertBefore);
4034 }
4035
4036 static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest,
4037 ArrayRef<BasicBlock *> IndirectDests,
4038 ArrayRef<Value *> Args, const Twine &NameStr,
4039 BasicBlock *InsertAtEnd) {
4040 return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest,
4041 IndirectDests, Args, NameStr, InsertAtEnd);
4042 }
4043
4044 static CallBrInst *Create(FunctionCallee Func,
4045 BasicBlock *DefaultDest,
4046 ArrayRef<BasicBlock *> IndirectDests,
4047 ArrayRef<Value *> Args,
4048 ArrayRef<OperandBundleDef> Bundles,
4049 const Twine &NameStr, BasicBlock *InsertAtEnd) {
4050 return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest,
4051 IndirectDests, Args, Bundles, NameStr, InsertAtEnd);
4052 }
4053
4054 /// Create a clone of \p CBI with a different set of operand bundles and
4055 /// insert it before \p InsertPt.
4056 ///
4057 /// The returned callbr instruction is identical to \p CBI in every way
4058 /// except that the operand bundles for the new instruction are set to the
4059 /// operand bundles in \p Bundles.
4060 static CallBrInst *Create(CallBrInst *CBI,
4061 ArrayRef<OperandBundleDef> Bundles,
4062 Instruction *InsertPt = nullptr);
4063
4064 /// Return the number of callbr indirect dest labels.
4065 ///
4066 unsigned getNumIndirectDests() const { return NumIndirectDests; }
4067
4068 /// getIndirectDestLabel - Return the i-th indirect dest label.
4069 ///
4070 Value *getIndirectDestLabel(unsigned i) const {
4071 assert(i < getNumIndirectDests() && "Out of bounds!")((void)0);
4072 return getOperand(i + getNumArgOperands() + getNumTotalBundleOperands() +
4073 1);
4074 }
4075
4076 Value *getIndirectDestLabelUse(unsigned i) const {
4077 assert(i < getNumIndirectDests() && "Out of bounds!")((void)0);
4078 return getOperandUse(i + getNumArgOperands() + getNumTotalBundleOperands() +
4079 1);
4080 }
4081
4082 // Return the destination basic blocks...
4083 BasicBlock *getDefaultDest() const {
4084 return cast<BasicBlock>(*(&Op<-1>() - getNumIndirectDests() - 1));
4085 }
4086 BasicBlock *getIndirectDest(unsigned i) const {
4087 return cast_or_null<BasicBlock>(*(&Op<-1>() - getNumIndirectDests() + i));
4088 }
4089 SmallVector<BasicBlock *, 16> getIndirectDests() const {
4090 SmallVector<BasicBlock *, 16> IndirectDests;
4091 for (unsigned i = 0, e = getNumIndirectDests(); i < e; ++i)
4092 IndirectDests.push_back(getIndirectDest(i));
4093 return IndirectDests;
4094 }
4095 void setDefaultDest(BasicBlock *B) {
4096 *(&Op<-1>() - getNumIndirectDests() - 1) = reinterpret_cast<Value *>(B);
4097 }
4098 void setIndirectDest(unsigned i, BasicBlock *B) {
4099 updateArgBlockAddresses(i, B);
4100 *(&Op<-1>() - getNumIndirectDests() + i) = reinterpret_cast<Value *>(B);
4101 }
4102
4103 BasicBlock *getSuccessor(unsigned i) const {
4104 assert(i < getNumSuccessors() + 1 &&((void)0)
4105 "Successor # out of range for callbr!")((void)0);
4106 return i == 0 ? getDefaultDest() : getIndirectDest(i - 1);
4107 }
4108
4109 void setSuccessor(unsigned i, BasicBlock *NewSucc) {
4110 assert(i < getNumIndirectDests() + 1 &&((void)0)
4111 "Successor # out of range for callbr!")((void)0);
4112 return i == 0 ? setDefaultDest(NewSucc) : setIndirectDest(i - 1, NewSucc);
4113 }
4114
4115 unsigned getNumSuccessors() const { return getNumIndirectDests() + 1; }
4116
4117 // Methods for support type inquiry through isa, cast, and dyn_cast:
4118 static bool classof(const Instruction *I) {
4119 return (I->getOpcode() == Instruction::CallBr);
4120 }
4121 static bool classof(const Value *V) {
4122 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4123 }
4124
4125private:
4126 // Shadow Instruction::setInstructionSubclassData with a private forwarding
4127 // method so that subclasses cannot accidentally use it.
4128 template <typename Bitfield>
4129 void setSubclassData(typename Bitfield::Type Value) {
4130 Instruction::setSubclassData<Bitfield>(Value);
4131 }
4132};
4133
4134CallBrInst::CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest,
4135 ArrayRef<BasicBlock *> IndirectDests,
4136 ArrayRef<Value *> Args,
4137 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
4138 const Twine &NameStr, Instruction *InsertBefore)
4139 : CallBase(Ty->getReturnType(), Instruction::CallBr,
4140 OperandTraits<CallBase>::op_end(this) - NumOperands, NumOperands,
4141 InsertBefore) {
4142 init(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, NameStr);
4143}
4144
4145CallBrInst::CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest,
4146 ArrayRef<BasicBlock *> IndirectDests,
4147 ArrayRef<Value *> Args,
4148 ArrayRef<OperandBundleDef> Bundles, int NumOperands,
4149 const Twine &NameStr, BasicBlock *InsertAtEnd)
4150 : CallBase(Ty->getReturnType(), Instruction::CallBr,
4151 OperandTraits<CallBase>::op_end(this) - NumOperands, NumOperands,
4152 InsertAtEnd) {
4153 init(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, NameStr);
4154}
4155
4156//===----------------------------------------------------------------------===//
4157// ResumeInst Class
4158//===----------------------------------------------------------------------===//
4159
4160//===---------------------------------------------------------------------------
4161/// Resume the propagation of an exception.
4162///
4163class ResumeInst : public Instruction {
4164 ResumeInst(const ResumeInst &RI);
4165
4166 explicit ResumeInst(Value *Exn, Instruction *InsertBefore=nullptr);
4167 ResumeInst(Value *Exn, BasicBlock *InsertAtEnd);
4168
4169protected:
4170 // Note: Instruction needs to be a friend here to call cloneImpl.
4171 friend class Instruction;
4172
4173 ResumeInst *cloneImpl() const;
4174
4175public:
4176 static ResumeInst *Create(Value *Exn, Instruction *InsertBefore = nullptr) {
4177 return new(1) ResumeInst(Exn, InsertBefore);
4178 }
4179
4180 static ResumeInst *Create(Value *Exn, BasicBlock *InsertAtEnd) {
4181 return new(1) ResumeInst(Exn, InsertAtEnd);
4182 }
4183
4184 /// Provide fast operand accessors
4185 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
4186
4187 /// Convenience accessor.
4188 Value *getValue() const { return Op<0>(); }
4189
4190 unsigned getNumSuccessors() const { return 0; }
4191
4192 // Methods for support type inquiry through isa, cast, and dyn_cast:
4193 static bool classof(const Instruction *I) {
4194 return I->getOpcode() == Instruction::Resume;
4195 }
4196 static bool classof(const Value *V) {
4197 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4198 }
4199
4200private:
4201 BasicBlock *getSuccessor(unsigned idx) const {
4202 llvm_unreachable("ResumeInst has no successors!")__builtin_unreachable();
4203 }
4204
4205 void setSuccessor(unsigned idx, BasicBlock *NewSucc) {
4206 llvm_unreachable("ResumeInst has no successors!")__builtin_unreachable();
4207 }
4208};
4209
4210template <>
4211struct OperandTraits<ResumeInst> :
4212 public FixedNumOperandTraits<ResumeInst, 1> {
4213};
4214
4215DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ResumeInst, Value)ResumeInst::op_iterator ResumeInst::op_begin() { return OperandTraits
<ResumeInst>::op_begin(this); } ResumeInst::const_op_iterator
ResumeInst::op_begin() const { return OperandTraits<ResumeInst
>::op_begin(const_cast<ResumeInst*>(this)); } ResumeInst
::op_iterator ResumeInst::op_end() { return OperandTraits<
ResumeInst>::op_end(this); } ResumeInst::const_op_iterator
ResumeInst::op_end() const { return OperandTraits<ResumeInst
>::op_end(const_cast<ResumeInst*>(this)); } Value *ResumeInst
::getOperand(unsigned i_nocapture) const { ((void)0); return cast_or_null
<Value>( OperandTraits<ResumeInst>::op_begin(const_cast
<ResumeInst*>(this))[i_nocapture].get()); } void ResumeInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { ((
void)0); OperandTraits<ResumeInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned ResumeInst::getNumOperands() const
{ return OperandTraits<ResumeInst>::operands(this); } template
<int Idx_nocapture> Use &ResumeInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &ResumeInst::Op() const { return
this->OpFrom<Idx_nocapture>(this); }
4216
4217//===----------------------------------------------------------------------===//
4218// CatchSwitchInst Class
4219//===----------------------------------------------------------------------===//
4220class CatchSwitchInst : public Instruction {
4221 using UnwindDestField = BoolBitfieldElementT<0>;
4222
4223 /// The number of operands actually allocated. NumOperands is
4224 /// the number actually in use.
4225 unsigned ReservedSpace;
4226
4227 // Operand[0] = Outer scope
4228 // Operand[1] = Unwind block destination
4229 // Operand[n] = BasicBlock to go to on match
4230 CatchSwitchInst(const CatchSwitchInst &CSI);
4231
4232 /// Create a new switch instruction, specifying a
4233 /// default destination. The number of additional handlers can be specified
4234 /// here to make memory allocation more efficient.
4235 /// This constructor can also autoinsert before another instruction.
4236 CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest,
4237 unsigned NumHandlers, const Twine &NameStr,
4238 Instruction *InsertBefore);
4239
4240 /// Create a new switch instruction, specifying a
4241 /// default destination. The number of additional handlers can be specified
4242 /// here to make memory allocation more efficient.
4243 /// This constructor also autoinserts at the end of the specified BasicBlock.
4244 CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest,
4245 unsigned NumHandlers, const Twine &NameStr,
4246 BasicBlock *InsertAtEnd);
4247
4248 // allocate space for exactly zero operands
4249 void *operator new(size_t S) { return User::operator new(S); }
4250
4251 void init(Value *ParentPad, BasicBlock *UnwindDest, unsigned NumReserved);
4252 void growOperands(unsigned Size);
4253
4254protected:
4255 // Note: Instruction needs to be a friend here to call cloneImpl.
4256 friend class Instruction;
4257
4258 CatchSwitchInst *cloneImpl() const;
4259
4260public:
4261 void operator delete(void *Ptr) { return User::operator delete(Ptr); }
4262
4263 static CatchSwitchInst *Create(Value *ParentPad, BasicBlock *UnwindDest,
4264 unsigned NumHandlers,
4265 const Twine &NameStr = "",
4266 Instruction *InsertBefore = nullptr) {
4267 return new CatchSwitchInst(ParentPad, UnwindDest, NumHandlers, NameStr,
4268 InsertBefore);
4269 }
4270
4271 static CatchSwitchInst *Create(Value *ParentPad, BasicBlock *UnwindDest,
4272 unsigned NumHandlers, const Twine &NameStr,
4273 BasicBlock *InsertAtEnd) {
4274 return new CatchSwitchInst(ParentPad, UnwindDest, NumHandlers, NameStr,
4275 InsertAtEnd);
4276 }
4277
4278 /// Provide fast operand accessors
4279 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
4280
4281 // Accessor Methods for CatchSwitch stmt
4282 Value *getParentPad() const { return getOperand(0); }
4283 void setParentPad(Value *ParentPad) { setOperand(0, ParentPad); }
4284
4285 // Accessor Methods for CatchSwitch stmt
4286 bool hasUnwindDest() const { return getSubclassData<UnwindDestField>(); }
4287 bool unwindsToCaller() const { return !hasUnwindDest(); }
4288 BasicBlock *getUnwindDest() const {
4289 if (hasUnwindDest())
4290 return cast<BasicBlock>(getOperand(1));
4291 return nullptr;
4292 }
4293 void setUnwindDest(BasicBlock *UnwindDest) {
4294 assert(UnwindDest)((void)0);
4295 assert(hasUnwindDest())((void)0);
4296 setOperand(1, UnwindDest);
4297 }
4298
4299 /// return the number of 'handlers' in this catchswitch
4300 /// instruction, except the default handler
4301 unsigned getNumHandlers() const {
4302 if (hasUnwindDest())
4303 return getNumOperands() - 2;
4304 return getNumOperands() - 1;
4305 }
4306
4307private:
4308 static BasicBlock *handler_helper(Value *V) { return cast<BasicBlock>(V); }
4309 static const BasicBlock *handler_helper(const Value *V) {
4310 return cast<BasicBlock>(V);
4311 }
4312
4313public:
4314 using DerefFnTy = BasicBlock *(*)(Value *);
4315 using handler_iterator = mapped_iterator<op_iterator, DerefFnTy>;
4316 using handler_range = iterator_range<handler_iterator>;
4317 using ConstDerefFnTy = const BasicBlock *(*)(const Value *);
4318 using const_handler_iterator =
4319 mapped_iterator<const_op_iterator, ConstDerefFnTy>;
4320 using const_handler_range = iterator_range<const_handler_iterator>;
4321
4322 /// Returns an iterator that points to the first handler in CatchSwitchInst.
4323 handler_iterator handler_begin() {
4324 op_iterator It = op_begin() + 1;
4325 if (hasUnwindDest())
4326 ++It;
4327 return handler_iterator(It, DerefFnTy(handler_helper));
4328 }
4329
4330 /// Returns an iterator that points to the first handler in the
4331 /// CatchSwitchInst.
4332 const_handler_iterator handler_begin() const {
4333 const_op_iterator It = op_begin() + 1;
4334 if (hasUnwindDest())
4335 ++It;
4336 return const_handler_iterator(It, ConstDerefFnTy(handler_helper));
4337 }
4338
4339 /// Returns a read-only iterator that points one past the last
4340 /// handler in the CatchSwitchInst.
4341 handler_iterator handler_end() {
4342 return handler_iterator(op_end(), DerefFnTy(handler_helper));
4343 }
4344
4345 /// Returns an iterator that points one past the last handler in the
4346 /// CatchSwitchInst.
4347 const_handler_iterator handler_end() const {
4348 return const_handler_iterator(op_end(), ConstDerefFnTy(handler_helper));
4349 }
4350
4351 /// iteration adapter for range-for loops.
4352 handler_range handlers() {
4353 return make_range(handler_begin(), handler_end());
4354 }
4355
4356 /// iteration adapter for range-for loops.
4357 const_handler_range handlers() const {
4358 return make_range(handler_begin(), handler_end());
4359 }
4360
4361 /// Add an entry to the switch instruction...
4362 /// Note:
4363 /// This action invalidates handler_end(). Old handler_end() iterator will
4364 /// point to the added handler.
4365 void addHandler(BasicBlock *Dest);
4366
4367 void removeHandler(handler_iterator HI);
4368
4369 unsigned getNumSuccessors() const { return getNumOperands() - 1; }
4370 BasicBlock *getSuccessor(unsigned Idx) const {
4371 assert(Idx < getNumSuccessors() &&((void)0)
4372 "Successor # out of range for catchswitch!")((void)0);
4373 return cast<BasicBlock>(getOperand(Idx + 1));
4374 }
4375 void setSuccessor(unsigned Idx, BasicBlock *NewSucc) {
4376 assert(Idx < getNumSuccessors() &&((void)0)
4377 "Successor # out of range for catchswitch!")((void)0);
4378 setOperand(Idx + 1, NewSucc);
4379 }
4380
4381 // Methods for support type inquiry through isa, cast, and dyn_cast:
4382 static bool classof(const Instruction *I) {
4383 return I->getOpcode() == Instruction::CatchSwitch;
4384 }
4385 static bool classof(const Value *V) {
4386 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4387 }
4388};
4389
4390template <>
4391struct OperandTraits<CatchSwitchInst> : public HungoffOperandTraits<2> {};
4392
4393DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CatchSwitchInst, Value)CatchSwitchInst::op_iterator CatchSwitchInst::op_begin() { return
OperandTraits<CatchSwitchInst>::op_begin(this); } CatchSwitchInst
::const_op_iterator CatchSwitchInst::op_begin() const { return
OperandTraits<CatchSwitchInst>::op_begin(const_cast<
CatchSwitchInst*>(this)); } CatchSwitchInst::op_iterator CatchSwitchInst
::op_end() { return OperandTraits<CatchSwitchInst>::op_end
(this); } CatchSwitchInst::const_op_iterator CatchSwitchInst::
op_end() const { return OperandTraits<CatchSwitchInst>::
op_end(const_cast<CatchSwitchInst*>(this)); } Value *CatchSwitchInst
::getOperand(unsigned i_nocapture) const { ((void)0); return cast_or_null
<Value>( OperandTraits<CatchSwitchInst>::op_begin
(const_cast<CatchSwitchInst*>(this))[i_nocapture].get()
); } void CatchSwitchInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { ((void)0); OperandTraits<CatchSwitchInst
>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
CatchSwitchInst::getNumOperands() const { return OperandTraits
<CatchSwitchInst>::operands(this); } template <int Idx_nocapture
> Use &CatchSwitchInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
const Use &CatchSwitchInst::Op() const { return this->
OpFrom<Idx_nocapture>(this); }
4394
4395//===----------------------------------------------------------------------===//
4396// CleanupPadInst Class
4397//===----------------------------------------------------------------------===//
4398class CleanupPadInst : public FuncletPadInst {
4399private:
4400 explicit CleanupPadInst(Value *ParentPad, ArrayRef<Value *> Args,
4401 unsigned Values, const Twine &NameStr,
4402 Instruction *InsertBefore)
4403 : FuncletPadInst(Instruction::CleanupPad, ParentPad, Args, Values,
4404 NameStr, InsertBefore) {}
4405 explicit CleanupPadInst(Value *ParentPad, ArrayRef<Value *> Args,
4406 unsigned Values, const Twine &NameStr,
4407 BasicBlock *InsertAtEnd)
4408 : FuncletPadInst(Instruction::CleanupPad, ParentPad, Args, Values,
4409 NameStr, InsertAtEnd) {}
4410
4411public:
4412 static CleanupPadInst *Create(Value *ParentPad, ArrayRef<Value *> Args = None,
4413 const Twine &NameStr = "",
4414 Instruction *InsertBefore = nullptr) {
4415 unsigned Values = 1 + Args.size();
4416 return new (Values)
4417 CleanupPadInst(ParentPad, Args, Values, NameStr, InsertBefore);
4418 }
4419
4420 static CleanupPadInst *Create(Value *ParentPad, ArrayRef<Value *> Args,
4421 const Twine &NameStr, BasicBlock *InsertAtEnd) {
4422 unsigned Values = 1 + Args.size();
4423 return new (Values)
4424 CleanupPadInst(ParentPad, Args, Values, NameStr, InsertAtEnd);
4425 }
4426
4427 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4428 static bool classof(const Instruction *I) {
4429 return I->getOpcode() == Instruction::CleanupPad;
4430 }
4431 static bool classof(const Value *V) {
4432 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4433 }
4434};
4435
4436//===----------------------------------------------------------------------===//
4437// CatchPadInst Class
4438//===----------------------------------------------------------------------===//
4439class CatchPadInst : public FuncletPadInst {
4440private:
4441 explicit CatchPadInst(Value *CatchSwitch, ArrayRef<Value *> Args,
4442 unsigned Values, const Twine &NameStr,
4443 Instruction *InsertBefore)
4444 : FuncletPadInst(Instruction::CatchPad, CatchSwitch, Args, Values,
4445 NameStr, InsertBefore) {}
4446 explicit CatchPadInst(Value *CatchSwitch, ArrayRef<Value *> Args,
4447 unsigned Values, const Twine &NameStr,
4448 BasicBlock *InsertAtEnd)
4449 : FuncletPadInst(Instruction::CatchPad, CatchSwitch, Args, Values,
4450 NameStr, InsertAtEnd) {}
4451
4452public:
4453 static CatchPadInst *Create(Value *CatchSwitch, ArrayRef<Value *> Args,
4454 const Twine &NameStr = "",
4455 Instruction *InsertBefore = nullptr) {
4456 unsigned Values = 1 + Args.size();
4457 return new (Values)
4458 CatchPadInst(CatchSwitch, Args, Values, NameStr, InsertBefore);
4459 }
4460
4461 static CatchPadInst *Create(Value *CatchSwitch, ArrayRef<Value *> Args,
4462 const Twine &NameStr, BasicBlock *InsertAtEnd) {
4463 unsigned Values = 1 + Args.size();
4464 return new (Values)
4465 CatchPadInst(CatchSwitch, Args, Values, NameStr, InsertAtEnd);
4466 }
4467
4468 /// Convenience accessors
4469 CatchSwitchInst *getCatchSwitch() const {
4470 return cast<CatchSwitchInst>(Op<-1>());
4471 }
4472 void setCatchSwitch(Value *CatchSwitch) {
4473 assert(CatchSwitch)((void)0);
4474 Op<-1>() = CatchSwitch;
4475 }
4476
4477 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4478 static bool classof(const Instruction *I) {
4479 return I->getOpcode() == Instruction::CatchPad;
4480 }
4481 static bool classof(const Value *V) {
4482 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4483 }
4484};
4485
4486//===----------------------------------------------------------------------===//
4487// CatchReturnInst Class
4488//===----------------------------------------------------------------------===//
4489
4490class CatchReturnInst : public Instruction {
4491 CatchReturnInst(const CatchReturnInst &RI);
4492 CatchReturnInst(Value *CatchPad, BasicBlock *BB, Instruction *InsertBefore);
4493 CatchReturnInst(Value *CatchPad, BasicBlock *BB, BasicBlock *InsertAtEnd);
4494
4495 void init(Value *CatchPad, BasicBlock *BB);
4496
4497protected:
4498 // Note: Instruction needs to be a friend here to call cloneImpl.
4499 friend class Instruction;
4500
4501 CatchReturnInst *cloneImpl() const;
4502
4503public:
4504 static CatchReturnInst *Create(Value *CatchPad, BasicBlock *BB,
4505 Instruction *InsertBefore = nullptr) {
4506 assert(CatchPad)((void)0);
4507 assert(BB)((void)0);
4508 return new (2) CatchReturnInst(CatchPad, BB, InsertBefore);
4509 }
4510
4511 static CatchReturnInst *Create(Value *CatchPad, BasicBlock *BB,
4512 BasicBlock *InsertAtEnd) {
4513 assert(CatchPad)((void)0);
4514 assert(BB)((void)0);
4515 return new (2) CatchReturnInst(CatchPad, BB, InsertAtEnd);
4516 }
4517
4518 /// Provide fast operand accessors
4519 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
4520
4521 /// Convenience accessors.
4522 CatchPadInst *getCatchPad() const { return cast<CatchPadInst>(Op<0>()); }
4523 void setCatchPad(CatchPadInst *CatchPad) {
4524 assert(CatchPad)((void)0);
4525 Op<0>() = CatchPad;
4526 }
4527
4528 BasicBlock *getSuccessor() const { return cast<BasicBlock>(Op<1>()); }
4529 void setSuccessor(BasicBlock *NewSucc) {
4530 assert(NewSucc)((void)0);
4531 Op<1>() = NewSucc;
4532 }
4533 unsigned getNumSuccessors() const { return 1; }
4534
4535 /// Get the parentPad of this catchret's catchpad's catchswitch.
4536 /// The successor block is implicitly a member of this funclet.
4537 Value *getCatchSwitchParentPad() const {
4538 return getCatchPad()->getCatchSwitch()->getParentPad();
4539 }
4540
4541 // Methods for support type inquiry through isa, cast, and dyn_cast:
4542 static bool classof(const Instruction *I) {
4543 return (I->getOpcode() == Instruction::CatchRet);
4544 }
4545 static bool classof(const Value *V) {
4546 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4547 }
4548
4549private:
4550 BasicBlock *getSuccessor(unsigned Idx) const {
4551 assert(Idx < getNumSuccessors() && "Successor # out of range for catchret!")((void)0);
4552 return getSuccessor();
4553 }
4554
4555 void setSuccessor(unsigned Idx, BasicBlock *B) {
4556 assert(Idx < getNumSuccessors() && "Successor # out of range for catchret!")((void)0);
4557 setSuccessor(B);
4558 }
4559};
4560
4561template <>
4562struct OperandTraits<CatchReturnInst>
4563 : public FixedNumOperandTraits<CatchReturnInst, 2> {};
4564
4565DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CatchReturnInst, Value)CatchReturnInst::op_iterator CatchReturnInst::op_begin() { return
OperandTraits<CatchReturnInst>::op_begin(this); } CatchReturnInst
::const_op_iterator CatchReturnInst::op_begin() const { return
OperandTraits<CatchReturnInst>::op_begin(const_cast<
CatchReturnInst*>(this)); } CatchReturnInst::op_iterator CatchReturnInst
::op_end() { return OperandTraits<CatchReturnInst>::op_end
(this); } CatchReturnInst::const_op_iterator CatchReturnInst::
op_end() const { return OperandTraits<CatchReturnInst>::
op_end(const_cast<CatchReturnInst*>(this)); } Value *CatchReturnInst
::getOperand(unsigned i_nocapture) const { ((void)0); return cast_or_null
<Value>( OperandTraits<CatchReturnInst>::op_begin
(const_cast<CatchReturnInst*>(this))[i_nocapture].get()
); } void CatchReturnInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { ((void)0); OperandTraits<CatchReturnInst
>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
CatchReturnInst::getNumOperands() const { return OperandTraits
<CatchReturnInst>::operands(this); } template <int Idx_nocapture
> Use &CatchReturnInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
const Use &CatchReturnInst::Op() const { return this->
OpFrom<Idx_nocapture>(this); }
4566
4567//===----------------------------------------------------------------------===//
4568// CleanupReturnInst Class
4569//===----------------------------------------------------------------------===//
4570
4571class CleanupReturnInst : public Instruction {
4572 using UnwindDestField = BoolBitfieldElementT<0>;
4573
4574private:
4575 CleanupReturnInst(const CleanupReturnInst &RI);
4576 CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB, unsigned Values,
4577 Instruction *InsertBefore = nullptr);
4578 CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB, unsigned Values,
4579 BasicBlock *InsertAtEnd);
4580
4581 void init(Value *CleanupPad, BasicBlock *UnwindBB);
4582
4583protected:
4584 // Note: Instruction needs to be a friend here to call cloneImpl.
4585 friend class Instruction;
4586
4587 CleanupReturnInst *cloneImpl() const;
4588
4589public:
4590 static CleanupReturnInst *Create(Value *CleanupPad,
4591 BasicBlock *UnwindBB = nullptr,
4592 Instruction *InsertBefore = nullptr) {
4593 assert(CleanupPad)((void)0);
4594 unsigned Values = 1;
4595 if (UnwindBB)
4596 ++Values;
4597 return new (Values)
4598 CleanupReturnInst(CleanupPad, UnwindBB, Values, InsertBefore);
4599 }
4600
4601 static CleanupReturnInst *Create(Value *CleanupPad, BasicBlock *UnwindBB,
4602 BasicBlock *InsertAtEnd) {
4603 assert(CleanupPad)((void)0);
4604 unsigned Values = 1;
4605 if (UnwindBB)
4606 ++Values;
4607 return new (Values)
4608 CleanupReturnInst(CleanupPad, UnwindBB, Values, InsertAtEnd);
4609 }
4610
4611 /// Provide fast operand accessors
4612 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
4613
4614 bool hasUnwindDest() const { return getSubclassData<UnwindDestField>(); }
4615 bool unwindsToCaller() const { return !hasUnwindDest(); }
4616
4617 /// Convenience accessor.
4618 CleanupPadInst *getCleanupPad() const {
4619 return cast<CleanupPadInst>(Op<0>());
4620 }
4621 void setCleanupPad(CleanupPadInst *CleanupPad) {
4622 assert(CleanupPad)((void)0);
4623 Op<0>() = CleanupPad;
4624 }
4625
4626 unsigned getNumSuccessors() const { return hasUnwindDest() ? 1 : 0; }
4627
4628 BasicBlock *getUnwindDest() const {
4629 return hasUnwindDest() ? cast<BasicBlock>(Op<1>()) : nullptr;
4630 }
4631 void setUnwindDest(BasicBlock *NewDest) {
4632 assert(NewDest)((void)0);
4633 assert(hasUnwindDest())((void)0);
4634 Op<1>() = NewDest;
4635 }
4636
4637 // Methods for support type inquiry through isa, cast, and dyn_cast:
4638 static bool classof(const Instruction *I) {
4639 return (I->getOpcode() == Instruction::CleanupRet);
4640 }
4641 static bool classof(const Value *V) {
4642 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4643 }
4644
4645private:
4646 BasicBlock *getSuccessor(unsigned Idx) const {
4647 assert(Idx == 0)((void)0);
4648 return getUnwindDest();
4649 }
4650
4651 void setSuccessor(unsigned Idx, BasicBlock *B) {
4652 assert(Idx == 0)((void)0);
4653 setUnwindDest(B);
4654 }
4655
4656 // Shadow Instruction::setInstructionSubclassData with a private forwarding
4657 // method so that subclasses cannot accidentally use it.
4658 template <typename Bitfield>
4659 void setSubclassData(typename Bitfield::Type Value) {
4660 Instruction::setSubclassData<Bitfield>(Value);
4661 }
4662};
4663
4664template <>
4665struct OperandTraits<CleanupReturnInst>
4666 : public VariadicOperandTraits<CleanupReturnInst, /*MINARITY=*/1> {};
4667
4668DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CleanupReturnInst, Value)CleanupReturnInst::op_iterator CleanupReturnInst::op_begin() {
return OperandTraits<CleanupReturnInst>::op_begin(this
); } CleanupReturnInst::const_op_iterator CleanupReturnInst::
op_begin() const { return OperandTraits<CleanupReturnInst>
::op_begin(const_cast<CleanupReturnInst*>(this)); } CleanupReturnInst
::op_iterator CleanupReturnInst::op_end() { return OperandTraits
<CleanupReturnInst>::op_end(this); } CleanupReturnInst::
const_op_iterator CleanupReturnInst::op_end() const { return OperandTraits
<CleanupReturnInst>::op_end(const_cast<CleanupReturnInst
*>(this)); } Value *CleanupReturnInst::getOperand(unsigned
i_nocapture) const { ((void)0); return cast_or_null<Value
>( OperandTraits<CleanupReturnInst>::op_begin(const_cast
<CleanupReturnInst*>(this))[i_nocapture].get()); } void
CleanupReturnInst::setOperand(unsigned i_nocapture, Value *Val_nocapture
) { ((void)0); OperandTraits<CleanupReturnInst>::op_begin
(this)[i_nocapture] = Val_nocapture; } unsigned CleanupReturnInst
::getNumOperands() const { return OperandTraits<CleanupReturnInst
>::operands(this); } template <int Idx_nocapture> Use
&CleanupReturnInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
CleanupReturnInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
4669
4670//===----------------------------------------------------------------------===//
4671// UnreachableInst Class
4672//===----------------------------------------------------------------------===//
4673
4674//===---------------------------------------------------------------------------
4675/// This function has undefined behavior. In particular, the
4676/// presence of this instruction indicates some higher level knowledge that the
4677/// end of the block cannot be reached.
4678///
4679class UnreachableInst : public Instruction {
4680protected:
4681 // Note: Instruction needs to be a friend here to call cloneImpl.
4682 friend class Instruction;
4683
4684 UnreachableInst *cloneImpl() const;
4685
4686public:
4687 explicit UnreachableInst(LLVMContext &C, Instruction *InsertBefore = nullptr);
4688 explicit UnreachableInst(LLVMContext &C, BasicBlock *InsertAtEnd);
4689
4690 // allocate space for exactly zero operands
4691 void *operator new(size_t S) { return User::operator new(S, 0); }
4692 void operator delete(void *Ptr) { User::operator delete(Ptr); }
4693
4694 unsigned getNumSuccessors() const { return 0; }
4695
4696 // Methods for support type inquiry through isa, cast, and dyn_cast:
4697 static bool classof(const Instruction *I) {
4698 return I->getOpcode() == Instruction::Unreachable;
4699 }
4700 static bool classof(const Value *V) {
4701 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4702 }
4703
4704private:
4705 BasicBlock *getSuccessor(unsigned idx) const {
4706 llvm_unreachable("UnreachableInst has no successors!")__builtin_unreachable();
4707 }
4708
4709 void setSuccessor(unsigned idx, BasicBlock *B) {
4710 llvm_unreachable("UnreachableInst has no successors!")__builtin_unreachable();
4711 }
4712};
4713
4714//===----------------------------------------------------------------------===//
4715// TruncInst Class
4716//===----------------------------------------------------------------------===//
4717
4718/// This class represents a truncation of integer types.
4719class TruncInst : public CastInst {
4720protected:
4721 // Note: Instruction needs to be a friend here to call cloneImpl.
4722 friend class Instruction;
4723
4724 /// Clone an identical TruncInst
4725 TruncInst *cloneImpl() const;
4726
4727public:
4728 /// Constructor with insert-before-instruction semantics
4729 TruncInst(
4730 Value *S, ///< The value to be truncated
4731 Type *Ty, ///< The (smaller) type to truncate to
4732 const Twine &NameStr = "", ///< A name for the new instruction
4733 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4734 );
4735
4736 /// Constructor with insert-at-end-of-block semantics
4737 TruncInst(
4738 Value *S, ///< The value to be truncated
4739 Type *Ty, ///< The (smaller) type to truncate to
4740 const Twine &NameStr, ///< A name for the new instruction
4741 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4742 );
4743
4744 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4745 static bool classof(const Instruction *I) {
4746 return I->getOpcode() == Trunc;
4747 }
4748 static bool classof(const Value *V) {
4749 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4750 }
4751};
4752
4753//===----------------------------------------------------------------------===//
4754// ZExtInst Class
4755//===----------------------------------------------------------------------===//
4756
4757/// This class represents zero extension of integer types.
4758class ZExtInst : public CastInst {
4759protected:
4760 // Note: Instruction needs to be a friend here to call cloneImpl.
4761 friend class Instruction;
4762
4763 /// Clone an identical ZExtInst
4764 ZExtInst *cloneImpl() const;
4765
4766public:
4767 /// Constructor with insert-before-instruction semantics
4768 ZExtInst(
4769 Value *S, ///< The value to be zero extended
4770 Type *Ty, ///< The type to zero extend to
4771 const Twine &NameStr = "", ///< A name for the new instruction
4772 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4773 );
4774
4775 /// Constructor with insert-at-end semantics.
4776 ZExtInst(
4777 Value *S, ///< The value to be zero extended
4778 Type *Ty, ///< The type to zero extend to
4779 const Twine &NameStr, ///< A name for the new instruction
4780 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4781 );
4782
4783 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4784 static bool classof(const Instruction *I) {
4785 return I->getOpcode() == ZExt;
4786 }
4787 static bool classof(const Value *V) {
4788 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4789 }
4790};
4791
4792//===----------------------------------------------------------------------===//
4793// SExtInst Class
4794//===----------------------------------------------------------------------===//
4795
4796/// This class represents a sign extension of integer types.
4797class SExtInst : public CastInst {
4798protected:
4799 // Note: Instruction needs to be a friend here to call cloneImpl.
4800 friend class Instruction;
4801
4802 /// Clone an identical SExtInst
4803 SExtInst *cloneImpl() const;
4804
4805public:
4806 /// Constructor with insert-before-instruction semantics
4807 SExtInst(
4808 Value *S, ///< The value to be sign extended
4809 Type *Ty, ///< The type to sign extend to
4810 const Twine &NameStr = "", ///< A name for the new instruction
4811 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4812 );
4813
4814 /// Constructor with insert-at-end-of-block semantics
4815 SExtInst(
4816 Value *S, ///< The value to be sign extended
4817 Type *Ty, ///< The type to sign extend to
4818 const Twine &NameStr, ///< A name for the new instruction
4819 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4820 );
4821
4822 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4823 static bool classof(const Instruction *I) {
4824 return I->getOpcode() == SExt;
4825 }
4826 static bool classof(const Value *V) {
4827 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4828 }
4829};
4830
4831//===----------------------------------------------------------------------===//
4832// FPTruncInst Class
4833//===----------------------------------------------------------------------===//
4834
4835/// This class represents a truncation of floating point types.
4836class FPTruncInst : public CastInst {
4837protected:
4838 // Note: Instruction needs to be a friend here to call cloneImpl.
4839 friend class Instruction;
4840
4841 /// Clone an identical FPTruncInst
4842 FPTruncInst *cloneImpl() const;
4843
4844public:
4845 /// Constructor with insert-before-instruction semantics
4846 FPTruncInst(
4847 Value *S, ///< The value to be truncated
4848 Type *Ty, ///< The type to truncate to
4849 const Twine &NameStr = "", ///< A name for the new instruction
4850 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4851 );
4852
4853 /// Constructor with insert-before-instruction semantics
4854 FPTruncInst(
4855 Value *S, ///< The value to be truncated
4856 Type *Ty, ///< The type to truncate to
4857 const Twine &NameStr, ///< A name for the new instruction
4858 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4859 );
4860
4861 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4862 static bool classof(const Instruction *I) {
4863 return I->getOpcode() == FPTrunc;
4864 }
4865 static bool classof(const Value *V) {
4866 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4867 }
4868};
4869
4870//===----------------------------------------------------------------------===//
4871// FPExtInst Class
4872//===----------------------------------------------------------------------===//
4873
4874/// This class represents an extension of floating point types.
4875class FPExtInst : public CastInst {
4876protected:
4877 // Note: Instruction needs to be a friend here to call cloneImpl.
4878 friend class Instruction;
4879
4880 /// Clone an identical FPExtInst
4881 FPExtInst *cloneImpl() const;
4882
4883public:
4884 /// Constructor with insert-before-instruction semantics
4885 FPExtInst(
4886 Value *S, ///< The value to be extended
4887 Type *Ty, ///< The type to extend to
4888 const Twine &NameStr = "", ///< A name for the new instruction
4889 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4890 );
4891
4892 /// Constructor with insert-at-end-of-block semantics
4893 FPExtInst(
4894 Value *S, ///< The value to be extended
4895 Type *Ty, ///< The type to extend to
4896 const Twine &NameStr, ///< A name for the new instruction
4897 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4898 );
4899
4900 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4901 static bool classof(const Instruction *I) {
4902 return I->getOpcode() == FPExt;
4903 }
4904 static bool classof(const Value *V) {
4905 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4906 }
4907};
4908
4909//===----------------------------------------------------------------------===//
4910// UIToFPInst Class
4911//===----------------------------------------------------------------------===//
4912
4913/// This class represents a cast unsigned integer to floating point.
4914class UIToFPInst : public CastInst {
4915protected:
4916 // Note: Instruction needs to be a friend here to call cloneImpl.
4917 friend class Instruction;
4918
4919 /// Clone an identical UIToFPInst
4920 UIToFPInst *cloneImpl() const;
4921
4922public:
4923 /// Constructor with insert-before-instruction semantics
4924 UIToFPInst(
4925 Value *S, ///< The value to be converted
4926 Type *Ty, ///< The type to convert to
4927 const Twine &NameStr = "", ///< A name for the new instruction
4928 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4929 );
4930
4931 /// Constructor with insert-at-end-of-block semantics
4932 UIToFPInst(
4933 Value *S, ///< The value to be converted
4934 Type *Ty, ///< The type to convert to
4935 const Twine &NameStr, ///< A name for the new instruction
4936 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4937 );
4938
4939 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4940 static bool classof(const Instruction *I) {
4941 return I->getOpcode() == UIToFP;
4942 }
4943 static bool classof(const Value *V) {
4944 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4945 }
4946};
4947
4948//===----------------------------------------------------------------------===//
4949// SIToFPInst Class
4950//===----------------------------------------------------------------------===//
4951
4952/// This class represents a cast from signed integer to floating point.
4953class SIToFPInst : public CastInst {
4954protected:
4955 // Note: Instruction needs to be a friend here to call cloneImpl.
4956 friend class Instruction;
4957
4958 /// Clone an identical SIToFPInst
4959 SIToFPInst *cloneImpl() const;
4960
4961public:
4962 /// Constructor with insert-before-instruction semantics
4963 SIToFPInst(
4964 Value *S, ///< The value to be converted
4965 Type *Ty, ///< The type to convert to
4966 const Twine &NameStr = "", ///< A name for the new instruction
4967 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4968 );
4969
4970 /// Constructor with insert-at-end-of-block semantics
4971 SIToFPInst(
4972 Value *S, ///< The value to be converted
4973 Type *Ty, ///< The type to convert to
4974 const Twine &NameStr, ///< A name for the new instruction
4975 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4976 );
4977
4978 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4979 static bool classof(const Instruction *I) {
4980 return I->getOpcode() == SIToFP;
4981 }
4982 static bool classof(const Value *V) {
4983 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4984 }
4985};
4986
4987//===----------------------------------------------------------------------===//
4988// FPToUIInst Class
4989//===----------------------------------------------------------------------===//
4990
4991/// This class represents a cast from floating point to unsigned integer
4992class FPToUIInst : public CastInst {
4993protected:
4994 // Note: Instruction needs to be a friend here to call cloneImpl.
4995 friend class Instruction;
4996
4997 /// Clone an identical FPToUIInst
4998 FPToUIInst *cloneImpl() const;
4999
5000public:
5001 /// Constructor with insert-before-instruction semantics
5002 FPToUIInst(
5003 Value *S, ///< The value to be converted
5004 Type *Ty, ///< The type to convert to
5005 const Twine &NameStr = "", ///< A name for the new instruction
5006 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5007 );
5008
5009 /// Constructor with insert-at-end-of-block semantics
5010 FPToUIInst(
5011 Value *S, ///< The value to be converted
5012 Type *Ty, ///< The type to convert to
5013 const Twine &NameStr, ///< A name for the new instruction
5014 BasicBlock *InsertAtEnd ///< Where to insert the new instruction
5015 );
5016
5017 /// Methods for support type inquiry through isa, cast, and dyn_cast:
5018 static bool classof(const Instruction *I) {
5019 return I->getOpcode() == FPToUI;
5020 }
5021 static bool classof(const Value *V) {
5022 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5023 }
5024};
5025
5026//===----------------------------------------------------------------------===//
5027// FPToSIInst Class
5028//===----------------------------------------------------------------------===//
5029
5030/// This class represents a cast from floating point to signed integer.
5031class FPToSIInst : public CastInst {
5032protected:
5033 // Note: Instruction needs to be a friend here to call cloneImpl.
5034 friend class Instruction;
5035
5036 /// Clone an identical FPToSIInst
5037 FPToSIInst *cloneImpl() const;
5038
5039public:
5040 /// Constructor with insert-before-instruction semantics
5041 FPToSIInst(
5042 Value *S, ///< The value to be converted
5043 Type *Ty, ///< The type to convert to
5044 const Twine &NameStr = "", ///< A name for the new instruction
5045 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5046 );
5047
5048 /// Constructor with insert-at-end-of-block semantics
5049 FPToSIInst(
5050 Value *S, ///< The value to be converted
5051 Type *Ty, ///< The type to convert to
5052 const Twine &NameStr, ///< A name for the new instruction
5053 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5054 );
5055
5056 /// Methods for support type inquiry through isa, cast, and dyn_cast:
5057 static bool classof(const Instruction *I) {
5058 return I->getOpcode() == FPToSI;
5059 }
5060 static bool classof(const Value *V) {
5061 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5062 }
5063};
5064
5065//===----------------------------------------------------------------------===//
5066// IntToPtrInst Class
5067//===----------------------------------------------------------------------===//
5068
5069/// This class represents a cast from an integer to a pointer.
5070class IntToPtrInst : public CastInst {
5071public:
5072 // Note: Instruction needs to be a friend here to call cloneImpl.
5073 friend class Instruction;
5074
5075 /// Constructor with insert-before-instruction semantics
5076 IntToPtrInst(
5077 Value *S, ///< The value to be converted
5078 Type *Ty, ///< The type to convert to
5079 const Twine &NameStr = "", ///< A name for the new instruction
5080 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5081 );
5082
5083 /// Constructor with insert-at-end-of-block semantics
5084 IntToPtrInst(
5085 Value *S, ///< The value to be converted
5086 Type *Ty, ///< The type to convert to
5087 const Twine &NameStr, ///< A name for the new instruction
5088 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5089 );
5090
5091 /// Clone an identical IntToPtrInst.
5092 IntToPtrInst *cloneImpl() const;
5093
5094 /// Returns the address space of this instruction's pointer type.
5095 unsigned getAddressSpace() const {
5096 return getType()->getPointerAddressSpace();
5097 }
5098
5099 // Methods for support type inquiry through isa, cast, and dyn_cast:
5100 static bool classof(const Instruction *I) {
5101 return I->getOpcode() == IntToPtr;
5102 }
5103 static bool classof(const Value *V) {
5104 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5105 }
5106};
5107
5108//===----------------------------------------------------------------------===//
5109// PtrToIntInst Class
5110//===----------------------------------------------------------------------===//
5111
5112/// This class represents a cast from a pointer to an integer.
5113class PtrToIntInst : public CastInst {
5114protected:
5115 // Note: Instruction needs to be a friend here to call cloneImpl.
5116 friend class Instruction;
5117
5118 /// Clone an identical PtrToIntInst.
5119 PtrToIntInst *cloneImpl() const;
5120
5121public:
5122 /// Constructor with insert-before-instruction semantics
5123 PtrToIntInst(
5124 Value *S, ///< The value to be converted
5125 Type *Ty, ///< The type to convert to
5126 const Twine &NameStr = "", ///< A name for the new instruction
5127 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5128 );
5129
5130 /// Constructor with insert-at-end-of-block semantics
5131 PtrToIntInst(
5132 Value *S, ///< The value to be converted
5133 Type *Ty, ///< The type to convert to
5134 const Twine &NameStr, ///< A name for the new instruction
5135 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5136 );
5137
5138 /// Gets the pointer operand.
5139 Value *getPointerOperand() { return getOperand(0); }
5140 /// Gets the pointer operand.
5141 const Value *getPointerOperand() const { return getOperand(0); }
5142 /// Gets the operand index of the pointer operand.
5143 static unsigned getPointerOperandIndex() { return 0U; }
5144
5145 /// Returns the address space of the pointer operand.
5146 unsigned getPointerAddressSpace() const {
5147 return getPointerOperand()->getType()->getPointerAddressSpace();
5148 }
5149
5150 // Methods for support type inquiry through isa, cast, and dyn_cast:
5151 static bool classof(const Instruction *I) {
5152 return I->getOpcode() == PtrToInt;
5153 }
5154 static bool classof(const Value *V) {
5155 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5156 }
5157};
5158
5159//===----------------------------------------------------------------------===//
5160// BitCastInst Class
5161//===----------------------------------------------------------------------===//
5162
5163/// This class represents a no-op cast from one type to another.
5164class BitCastInst : public CastInst {
5165protected:
5166 // Note: Instruction needs to be a friend here to call cloneImpl.
5167 friend class Instruction;
5168
5169 /// Clone an identical BitCastInst.
5170 BitCastInst *cloneImpl() const;
5171
5172public:
5173 /// Constructor with insert-before-instruction semantics
5174 BitCastInst(
5175 Value *S, ///< The value to be casted
5176 Type *Ty, ///< The type to casted to
5177 const Twine &NameStr = "", ///< A name for the new instruction
5178 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5179 );
5180
5181 /// Constructor with insert-at-end-of-block semantics
5182 BitCastInst(
5183 Value *S, ///< The value to be casted
5184 Type *Ty, ///< The type to casted to
5185 const Twine &NameStr, ///< A name for the new instruction
5186 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5187 );
5188
5189 // Methods for support type inquiry through isa, cast, and dyn_cast:
5190 static bool classof(const Instruction *I) {
5191 return I->getOpcode() == BitCast;
5192 }
5193 static bool classof(const Value *V) {
5194 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5195 }
5196};
5197
5198//===----------------------------------------------------------------------===//
5199// AddrSpaceCastInst Class
5200//===----------------------------------------------------------------------===//
5201
5202/// This class represents a conversion between pointers from one address space
5203/// to another.
5204class AddrSpaceCastInst : public CastInst {
5205protected:
5206 // Note: Instruction needs to be a friend here to call cloneImpl.
5207 friend class Instruction;
5208
5209 /// Clone an identical AddrSpaceCastInst.
5210 AddrSpaceCastInst *cloneImpl() const;
5211
5212public:
5213 /// Constructor with insert-before-instruction semantics
5214 AddrSpaceCastInst(
5215 Value *S, ///< The value to be casted
5216 Type *Ty, ///< The type to casted to
5217 const Twine &NameStr = "", ///< A name for the new instruction
5218 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5219 );
5220
5221 /// Constructor with insert-at-end-of-block semantics
5222 AddrSpaceCastInst(
5223 Value *S, ///< The value to be casted
5224 Type *Ty, ///< The type to casted to
5225 const Twine &NameStr, ///< A name for the new instruction
5226 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5227 );
5228
5229 // Methods for support type inquiry through isa, cast, and dyn_cast:
5230 static bool classof(const Instruction *I) {
5231 return I->getOpcode() == AddrSpaceCast;
5232 }
5233 static bool classof(const Value *V) {
5234 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5235 }
5236
5237 /// Gets the pointer operand.
5238 Value *getPointerOperand() {
5239 return getOperand(0);
5240 }
5241
5242 /// Gets the pointer operand.
5243 const Value *getPointerOperand() const {
5244 return getOperand(0);
5245 }
5246
5247 /// Gets the operand index of the pointer operand.
5248 static unsigned getPointerOperandIndex() {
5249 return 0U;
5250 }
5251
5252 /// Returns the address space of the pointer operand.
5253 unsigned getSrcAddressSpace() const {
5254 return getPointerOperand()->getType()->getPointerAddressSpace();
5255 }
5256
5257 /// Returns the address space of the result.
5258 unsigned getDestAddressSpace() const {
5259 return getType()->getPointerAddressSpace();
5260 }
5261};
5262
5263/// A helper function that returns the pointer operand of a load or store
5264/// instruction. Returns nullptr if not load or store.
5265inline const Value *getLoadStorePointerOperand(const Value *V) {
5266 if (auto *Load = dyn_cast<LoadInst>(V))
5267 return Load->getPointerOperand();
5268 if (auto *Store = dyn_cast<StoreInst>(V))
5269 return Store->getPointerOperand();
5270 return nullptr;
5271}
5272inline Value *getLoadStorePointerOperand(Value *V) {
5273 return const_cast<Value *>(
5274 getLoadStorePointerOperand(static_cast<const Value *>(V)));
5275}
5276
5277/// A helper function that returns the pointer operand of a load, store
5278/// or GEP instruction. Returns nullptr if not load, store, or GEP.
5279inline const Value *getPointerOperand(const Value *V) {
5280 if (auto *Ptr = getLoadStorePointerOperand(V))
5281 return Ptr;
5282 if (auto *Gep = dyn_cast<GetElementPtrInst>(V))
5283 return Gep->getPointerOperand();
5284 return nullptr;
5285}
5286inline Value *getPointerOperand(Value *V) {
5287 return const_cast<Value *>(getPointerOperand(static_cast<const Value *>(V)));
5288}
5289
5290/// A helper function that returns the alignment of load or store instruction.
5291inline Align getLoadStoreAlignment(Value *I) {
5292 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&((void)0)
5293 "Expected Load or Store instruction")((void)0);
5294 if (auto *LI = dyn_cast<LoadInst>(I))
5295 return LI->getAlign();
5296 return cast<StoreInst>(I)->getAlign();
5297}
5298
5299/// A helper function that returns the address space of the pointer operand of
5300/// load or store instruction.
5301inline unsigned getLoadStoreAddressSpace(Value *I) {
5302 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&((void)0)
5303 "Expected Load or Store instruction")((void)0);
5304 if (auto *LI = dyn_cast<LoadInst>(I))
5305 return LI->getPointerAddressSpace();
5306 return cast<StoreInst>(I)->getPointerAddressSpace();
5307}
5308
5309/// A helper function that returns the type of a load or store instruction.
5310inline Type *getLoadStoreType(Value *I) {
5311 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&((void)0)
5312 "Expected Load or Store instruction")((void)0);
5313 if (auto *LI = dyn_cast<LoadInst>(I))
5314 return LI->getType();
5315 return cast<StoreInst>(I)->getValueOperand()->getType();
5316}
5317
5318//===----------------------------------------------------------------------===//
5319// FreezeInst Class
5320//===----------------------------------------------------------------------===//
5321
5322/// This class represents a freeze function that returns random concrete
5323/// value if an operand is either a poison value or an undef value
5324class FreezeInst : public UnaryInstruction {
5325protected:
5326 // Note: Instruction needs to be a friend here to call cloneImpl.
5327 friend class Instruction;
5328
5329 /// Clone an identical FreezeInst
5330 FreezeInst *cloneImpl() const;
5331
5332public:
5333 explicit FreezeInst(Value *S,
5334 const Twine &NameStr = "",
5335 Instruction *InsertBefore = nullptr);
5336 FreezeInst(Value *S, const Twine &NameStr, BasicBlock *InsertAtEnd);
5337
5338 // Methods for support type inquiry through isa, cast, and dyn_cast:
5339 static inline bool classof(const Instruction *I) {
5340 return I->getOpcode() == Freeze;
5341 }
5342 static inline bool classof(const Value *V) {
5343 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5344 }
5345};
5346
5347} // end namespace llvm
5348
5349#endif // LLVM_IR_INSTRUCTIONS_H

/usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Support/Alignment.h

1//===-- llvm/Support/Alignment.h - Useful alignment functions ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains types to represent alignments.
10// They are instrumented to guarantee some invariants are preserved and prevent
11// invalid manipulations.
12//
13// - Align represents an alignment in bytes, it is always set and always a valid
14// power of two, its minimum value is 1 which means no alignment requirements.
15//
16// - MaybeAlign is an optional type, it may be undefined or set. When it's set
17// you can get the underlying Align type by using the getValue() method.
18//
19//===----------------------------------------------------------------------===//
20
21#ifndef LLVM_SUPPORT_ALIGNMENT_H_
22#define LLVM_SUPPORT_ALIGNMENT_H_
23
24#include "llvm/ADT/Optional.h"
25#include "llvm/Support/MathExtras.h"
26#include <cassert>
27#ifndef NDEBUG1
28#include <string>
29#endif // NDEBUG
30
31namespace llvm {
32
33#define ALIGN_CHECK_ISPOSITIVE(decl) \
34 assert(decl > 0 && (#decl " should be defined"))((void)0)
35
36/// This struct is a compact representation of a valid (non-zero power of two)
37/// alignment.
38/// It is suitable for use as static global constants.
39struct Align {
40private:
41 uint8_t ShiftValue = 0; /// The log2 of the required alignment.
42 /// ShiftValue is less than 64 by construction.
43
44 friend struct MaybeAlign;
45 friend unsigned Log2(Align);
46 friend bool operator==(Align Lhs, Align Rhs);
47 friend bool operator!=(Align Lhs, Align Rhs);
48 friend bool operator<=(Align Lhs, Align Rhs);
49 friend bool operator>=(Align Lhs, Align Rhs);
50 friend bool operator<(Align Lhs, Align Rhs);
51 friend bool operator>(Align Lhs, Align Rhs);
52 friend unsigned encode(struct MaybeAlign A);
53 friend struct MaybeAlign decodeMaybeAlign(unsigned Value);
54
55 /// A trivial type to allow construction of constexpr Align.
56 /// This is currently needed to workaround a bug in GCC 5.3 which prevents
57 /// definition of constexpr assign operators.
58 /// https://stackoverflow.com/questions/46756288/explicitly-defaulted-function-cannot-be-declared-as-constexpr-because-the-implic
59 /// FIXME: Remove this, make all assign operators constexpr and introduce user
60 /// defined literals when we don't have to support GCC 5.3 anymore.
61 /// https://llvm.org/docs/GettingStarted.html#getting-a-modern-host-c-toolchain
62 struct LogValue {
63 uint8_t Log;
64 };
65
66public:
67 /// Default is byte-aligned.
68 constexpr Align() = default;
69 /// Do not perform checks in case of copy/move construct/assign, because the
70 /// checks have been performed when building `Other`.
71 constexpr Align(const Align &Other) = default;
72 constexpr Align(Align &&Other) = default;
73 Align &operator=(const Align &Other) = default;
74 Align &operator=(Align &&Other) = default;
75
76 explicit Align(uint64_t Value) {
77 assert(Value > 0 && "Value must not be 0")((void)0);
78 assert(llvm::isPowerOf2_64(Value) && "Alignment is not a power of 2")((void)0);
79 ShiftValue = Log2_64(Value);
4
Calling 'Log2_64'
6
Returning from 'Log2_64'
7
The value 255 is assigned to field 'ShiftValue'
80 assert(ShiftValue < 64 && "Broken invariant")((void)0);
81 }
82
83 /// This is a hole in the type system and should not be abused.
84 /// Needed to interact with C for instance.
85 uint64_t value() const { return uint64_t(1) << ShiftValue; }
11
The result of the left shift is undefined due to shifting by '255', which is greater or equal to the width of type 'uint64_t'
86
87 /// Allow constructions of constexpr Align.
88 template <size_t kValue> constexpr static LogValue Constant() {
89 return LogValue{static_cast<uint8_t>(CTLog2<kValue>())};
90 }
91
92 /// Allow constructions of constexpr Align from types.
93 /// Compile time equivalent to Align(alignof(T)).
94 template <typename T> constexpr static LogValue Of() {
95 return Constant<std::alignment_of<T>::value>();
96 }
97
98 /// Constexpr constructor from LogValue type.
99 constexpr Align(LogValue CA) : ShiftValue(CA.Log) {}
100};
101
102/// Treats the value 0 as a 1, so Align is always at least 1.
103inline Align assumeAligned(uint64_t Value) {
104 return Value ? Align(Value) : Align();
105}
106
107/// This struct is a compact representation of a valid (power of two) or
108/// undefined (0) alignment.
109struct MaybeAlign : public llvm::Optional<Align> {
110private:
111 using UP = llvm::Optional<Align>;
112
113public:
114 /// Default is undefined.
115 MaybeAlign() = default;
116 /// Do not perform checks in case of copy/move construct/assign, because the
117 /// checks have been performed when building `Other`.
118 MaybeAlign(const MaybeAlign &Other) = default;
119 MaybeAlign &operator=(const MaybeAlign &Other) = default;
120 MaybeAlign(MaybeAlign &&Other) = default;
121 MaybeAlign &operator=(MaybeAlign &&Other) = default;
122
123 /// Use llvm::Optional<Align> constructor.
124 using UP::UP;
125
126 explicit MaybeAlign(uint64_t Value) {
127 assert((Value == 0 || llvm::isPowerOf2_64(Value)) &&((void)0)
128 "Alignment is neither 0 nor a power of 2")((void)0);
129 if (Value)
130 emplace(Value);
131 }
132
133 /// For convenience, returns a valid alignment or 1 if undefined.
134 Align valueOrOne() const { return hasValue() ? getValue() : Align(); }
135};
136
137/// Checks that SizeInBytes is a multiple of the alignment.
138inline bool isAligned(Align Lhs, uint64_t SizeInBytes) {
139 return SizeInBytes % Lhs.value() == 0;
140}
141
142/// Checks that Addr is a multiple of the alignment.
143inline bool isAddrAligned(Align Lhs, const void *Addr) {
144 return isAligned(Lhs, reinterpret_cast<uintptr_t>(Addr));
145}
146
147/// Returns a multiple of A needed to store `Size` bytes.
148inline uint64_t alignTo(uint64_t Size, Align A) {
149 const uint64_t Value = A.value();
150 // The following line is equivalent to `(Size + Value - 1) / Value * Value`.
151
152 // The division followed by a multiplication can be thought of as a right
153 // shift followed by a left shift which zeros out the extra bits produced in
154 // the bump; `~(Value - 1)` is a mask where all those bits being zeroed out
155 // are just zero.
156
157 // Most compilers can generate this code but the pattern may be missed when
158 // multiple functions gets inlined.
159 return (Size + Value - 1) & ~(Value - 1U);
160}
161
162/// If non-zero \p Skew is specified, the return value will be a minimal integer
163/// that is greater than or equal to \p Size and equal to \p A * N + \p Skew for
164/// some integer N. If \p Skew is larger than \p A, its value is adjusted to '\p
165/// Skew mod \p A'.
166///
167/// Examples:
168/// \code
169/// alignTo(5, Align(8), 7) = 7
170/// alignTo(17, Align(8), 1) = 17
171/// alignTo(~0LL, Align(8), 3) = 3
172/// \endcode
173inline uint64_t alignTo(uint64_t Size, Align A, uint64_t Skew) {
174 const uint64_t Value = A.value();
175 Skew %= Value;
176 return ((Size + Value - 1 - Skew) & ~(Value - 1U)) + Skew;
177}
178
179/// Returns a multiple of A needed to store `Size` bytes.
180/// Returns `Size` if current alignment is undefined.
181inline uint64_t alignTo(uint64_t Size, MaybeAlign A) {
182 return A ? alignTo(Size, A.getValue()) : Size;
183}
184
185/// Aligns `Addr` to `Alignment` bytes, rounding up.
186inline uintptr_t alignAddr(const void *Addr, Align Alignment) {
187 uintptr_t ArithAddr = reinterpret_cast<uintptr_t>(Addr);
188 assert(static_cast<uintptr_t>(ArithAddr + Alignment.value() - 1) >=((void)0)
189 ArithAddr &&((void)0)
190 "Overflow")((void)0);
191 return alignTo(ArithAddr, Alignment);
192}
193
194/// Returns the offset to the next integer (mod 2**64) that is greater than
195/// or equal to \p Value and is a multiple of \p Align.
196inline uint64_t offsetToAlignment(uint64_t Value, Align Alignment) {
197 return alignTo(Value, Alignment) - Value;
198}
199
200/// Returns the necessary adjustment for aligning `Addr` to `Alignment`
201/// bytes, rounding up.
202inline uint64_t offsetToAlignedAddr(const void *Addr, Align Alignment) {
203 return offsetToAlignment(reinterpret_cast<uintptr_t>(Addr), Alignment);
204}
205
206/// Returns the log2 of the alignment.
207inline unsigned Log2(Align A) { return A.ShiftValue; }
208
209/// Returns the alignment that satisfies both alignments.
210/// Same semantic as MinAlign.
211inline Align commonAlignment(Align A, Align B) { return std::min(A, B); }
212
213/// Returns the alignment that satisfies both alignments.
214/// Same semantic as MinAlign.
215inline Align commonAlignment(Align A, uint64_t Offset) {
216 return Align(MinAlign(A.value(), Offset));
217}
218
219/// Returns the alignment that satisfies both alignments.
220/// Same semantic as MinAlign.
221inline MaybeAlign commonAlignment(MaybeAlign A, MaybeAlign B) {
222 return A && B ? commonAlignment(*A, *B) : A ? A : B;
223}
224
225/// Returns the alignment that satisfies both alignments.
226/// Same semantic as MinAlign.
227inline MaybeAlign commonAlignment(MaybeAlign A, uint64_t Offset) {
228 return MaybeAlign(MinAlign((*A).value(), Offset));
229}
230
231/// Returns a representation of the alignment that encodes undefined as 0.
232inline unsigned encode(MaybeAlign A) { return A ? A->ShiftValue + 1 : 0; }
233
234/// Dual operation of the encode function above.
235inline MaybeAlign decodeMaybeAlign(unsigned Value) {
236 if (Value == 0)
237 return MaybeAlign();
238 Align Out;
239 Out.ShiftValue = Value - 1;
240 return Out;
241}
242
243/// Returns a representation of the alignment, the encoded value is positive by
244/// definition.
245inline unsigned encode(Align A) { return encode(MaybeAlign(A)); }
246
247/// Comparisons between Align and scalars. Rhs must be positive.
248inline bool operator==(Align Lhs, uint64_t Rhs) {
249 ALIGN_CHECK_ISPOSITIVE(Rhs);
250 return Lhs.value() == Rhs;
251}
252inline bool operator!=(Align Lhs, uint64_t Rhs) {
253 ALIGN_CHECK_ISPOSITIVE(Rhs);
254 return Lhs.value() != Rhs;
255}
256inline bool operator<=(Align Lhs, uint64_t Rhs) {
257 ALIGN_CHECK_ISPOSITIVE(Rhs);
258 return Lhs.value() <= Rhs;
259}
260inline bool operator>=(Align Lhs, uint64_t Rhs) {
261 ALIGN_CHECK_ISPOSITIVE(Rhs);
262 return Lhs.value() >= Rhs;
263}
264inline bool operator<(Align Lhs, uint64_t Rhs) {
265 ALIGN_CHECK_ISPOSITIVE(Rhs);
266 return Lhs.value() < Rhs;
267}
268inline bool operator>(Align Lhs, uint64_t Rhs) {
269 ALIGN_CHECK_ISPOSITIVE(Rhs);
270 return Lhs.value() > Rhs;
271}
272
273/// Comparisons between MaybeAlign and scalars.
274inline bool operator==(MaybeAlign Lhs, uint64_t Rhs) {
275 return Lhs ? (*Lhs).value() == Rhs : Rhs == 0;
276}
277inline bool operator!=(MaybeAlign Lhs, uint64_t Rhs) {
278 return Lhs ? (*Lhs).value() != Rhs : Rhs != 0;
279}
280
281/// Comparisons operators between Align.
282inline bool operator==(Align Lhs, Align Rhs) {
283 return Lhs.ShiftValue == Rhs.ShiftValue;
284}
285inline bool operator!=(Align Lhs, Align Rhs) {
286 return Lhs.ShiftValue != Rhs.ShiftValue;
287}
288inline bool operator<=(Align Lhs, Align Rhs) {
289 return Lhs.ShiftValue <= Rhs.ShiftValue;
290}
291inline bool operator>=(Align Lhs, Align Rhs) {
292 return Lhs.ShiftValue >= Rhs.ShiftValue;
293}
294inline bool operator<(Align Lhs, Align Rhs) {
295 return Lhs.ShiftValue < Rhs.ShiftValue;
296}
297inline bool operator>(Align Lhs, Align Rhs) {
298 return Lhs.ShiftValue > Rhs.ShiftValue;
299}
300
301// Don't allow relational comparisons with MaybeAlign.
302bool operator<=(Align Lhs, MaybeAlign Rhs) = delete;
303bool operator>=(Align Lhs, MaybeAlign Rhs) = delete;
304bool operator<(Align Lhs, MaybeAlign Rhs) = delete;
305bool operator>(Align Lhs, MaybeAlign Rhs) = delete;
306
307bool operator<=(MaybeAlign Lhs, Align Rhs) = delete;
308bool operator>=(MaybeAlign Lhs, Align Rhs) = delete;
309bool operator<(MaybeAlign Lhs, Align Rhs) = delete;
310bool operator>(MaybeAlign Lhs, Align Rhs) = delete;
311
312bool operator<=(MaybeAlign Lhs, MaybeAlign Rhs) = delete;
313bool operator>=(MaybeAlign Lhs, MaybeAlign Rhs) = delete;
314bool operator<(MaybeAlign Lhs, MaybeAlign Rhs) = delete;
315bool operator>(MaybeAlign Lhs, MaybeAlign Rhs) = delete;
316
317inline Align operator*(Align Lhs, uint64_t Rhs) {
318 assert(Rhs > 0 && "Rhs must be positive")((void)0);
319 return Align(Lhs.value() * Rhs);
320}
321
322inline MaybeAlign operator*(MaybeAlign Lhs, uint64_t Rhs) {
323 assert(Rhs > 0 && "Rhs must be positive")((void)0);
324 return Lhs ? Lhs.getValue() * Rhs : MaybeAlign();
325}
326
327inline Align operator/(Align Lhs, uint64_t Divisor) {
328 assert(llvm::isPowerOf2_64(Divisor) &&((void)0)
329 "Divisor must be positive and a power of 2")((void)0);
330 assert(Lhs != 1 && "Can't halve byte alignment")((void)0);
331 return Align(Lhs.value() / Divisor);
332}
333
334inline MaybeAlign operator/(MaybeAlign Lhs, uint64_t Divisor) {
335 assert(llvm::isPowerOf2_64(Divisor) &&((void)0)
336 "Divisor must be positive and a power of 2")((void)0);
337 return Lhs ? Lhs.getValue() / Divisor : MaybeAlign();
338}
339
340inline Align max(MaybeAlign Lhs, Align Rhs) {
341 return Lhs && *Lhs > Rhs ? *Lhs : Rhs;
342}
343
344inline Align max(Align Lhs, MaybeAlign Rhs) {
345 return Rhs && *Rhs > Lhs ? *Rhs : Lhs;
346}
347
348#ifndef NDEBUG1
349// For usage in LLVM_DEBUG macros.
350inline std::string DebugStr(const Align &A) {
351 return std::to_string(A.value());
352}
353// For usage in LLVM_DEBUG macros.
354inline std::string DebugStr(const MaybeAlign &MA) {
355 if (MA)
356 return std::to_string(MA->value());
357 return "None";
358}
359#endif // NDEBUG
360
361#undef ALIGN_CHECK_ISPOSITIVE
362
363} // namespace llvm
364
365#endif // LLVM_SUPPORT_ALIGNMENT_H_

/usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Support/MathExtras.h

1//===-- llvm/Support/MathExtras.h - Useful math functions -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains some functions that are useful for math stuff.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_SUPPORT_MATHEXTRAS_H
14#define LLVM_SUPPORT_MATHEXTRAS_H
15
16#include "llvm/Support/Compiler.h"
17#include <cassert>
18#include <climits>
19#include <cmath>
20#include <cstdint>
21#include <cstring>
22#include <limits>
23#include <type_traits>
24
25#ifdef __ANDROID_NDK__
26#include <android/api-level.h>
27#endif
28
29#ifdef _MSC_VER
30// Declare these intrinsics manually rather including intrin.h. It's very
31// expensive, and MathExtras.h is popular.
32// #include <intrin.h>
33extern "C" {
34unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask);
35unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
36unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask);
37unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
38}
39#endif
40
41namespace llvm {
42
43/// The behavior an operation has on an input of 0.
44enum ZeroBehavior {
45 /// The returned value is undefined.
46 ZB_Undefined,
47 /// The returned value is numeric_limits<T>::max()
48 ZB_Max,
49 /// The returned value is numeric_limits<T>::digits
50 ZB_Width
51};
52
53/// Mathematical constants.
54namespace numbers {
55// TODO: Track C++20 std::numbers.
56// TODO: Favor using the hexadecimal FP constants (requires C++17).
57constexpr double e = 2.7182818284590452354, // (0x1.5bf0a8b145749P+1) https://oeis.org/A001113
58 egamma = .57721566490153286061, // (0x1.2788cfc6fb619P-1) https://oeis.org/A001620
59 ln2 = .69314718055994530942, // (0x1.62e42fefa39efP-1) https://oeis.org/A002162
60 ln10 = 2.3025850929940456840, // (0x1.24bb1bbb55516P+1) https://oeis.org/A002392
61 log2e = 1.4426950408889634074, // (0x1.71547652b82feP+0)
62 log10e = .43429448190325182765, // (0x1.bcb7b1526e50eP-2)
63 pi = 3.1415926535897932385, // (0x1.921fb54442d18P+1) https://oeis.org/A000796
64 inv_pi = .31830988618379067154, // (0x1.45f306bc9c883P-2) https://oeis.org/A049541
65 sqrtpi = 1.7724538509055160273, // (0x1.c5bf891b4ef6bP+0) https://oeis.org/A002161
66 inv_sqrtpi = .56418958354775628695, // (0x1.20dd750429b6dP-1) https://oeis.org/A087197
67 sqrt2 = 1.4142135623730950488, // (0x1.6a09e667f3bcdP+0) https://oeis.org/A00219
68 inv_sqrt2 = .70710678118654752440, // (0x1.6a09e667f3bcdP-1)
69 sqrt3 = 1.7320508075688772935, // (0x1.bb67ae8584caaP+0) https://oeis.org/A002194
70 inv_sqrt3 = .57735026918962576451, // (0x1.279a74590331cP-1)
71 phi = 1.6180339887498948482; // (0x1.9e3779b97f4a8P+0) https://oeis.org/A001622
72constexpr float ef = 2.71828183F, // (0x1.5bf0a8P+1) https://oeis.org/A001113
73 egammaf = .577215665F, // (0x1.2788d0P-1) https://oeis.org/A001620
74 ln2f = .693147181F, // (0x1.62e430P-1) https://oeis.org/A002162
75 ln10f = 2.30258509F, // (0x1.26bb1cP+1) https://oeis.org/A002392
76 log2ef = 1.44269504F, // (0x1.715476P+0)
77 log10ef = .434294482F, // (0x1.bcb7b2P-2)
78 pif = 3.14159265F, // (0x1.921fb6P+1) https://oeis.org/A000796
79 inv_pif = .318309886F, // (0x1.45f306P-2) https://oeis.org/A049541
80 sqrtpif = 1.77245385F, // (0x1.c5bf8aP+0) https://oeis.org/A002161
81 inv_sqrtpif = .564189584F, // (0x1.20dd76P-1) https://oeis.org/A087197
82 sqrt2f = 1.41421356F, // (0x1.6a09e6P+0) https://oeis.org/A002193
83 inv_sqrt2f = .707106781F, // (0x1.6a09e6P-1)
84 sqrt3f = 1.73205081F, // (0x1.bb67aeP+0) https://oeis.org/A002194
85 inv_sqrt3f = .577350269F, // (0x1.279a74P-1)
86 phif = 1.61803399F; // (0x1.9e377aP+0) https://oeis.org/A001622
87} // namespace numbers
88
89namespace detail {
90template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter {
91 static unsigned count(T Val, ZeroBehavior) {
92 if (!Val)
93 return std::numeric_limits<T>::digits;
94 if (Val & 0x1)
95 return 0;
96
97 // Bisection method.
98 unsigned ZeroBits = 0;
99 T Shift = std::numeric_limits<T>::digits >> 1;
100 T Mask = std::numeric_limits<T>::max() >> Shift;
101 while (Shift) {
102 if ((Val & Mask) == 0) {
103 Val >>= Shift;
104 ZeroBits |= Shift;
105 }
106 Shift >>= 1;
107 Mask >>= Shift;
108 }
109 return ZeroBits;
110 }
111};
112
113#if defined(__GNUC__4) || defined(_MSC_VER)
114template <typename T> struct TrailingZerosCounter<T, 4> {
115 static unsigned count(T Val, ZeroBehavior ZB) {
116 if (ZB != ZB_Undefined && Val == 0)
117 return 32;
118
119#if __has_builtin(__builtin_ctz)1 || defined(__GNUC__4)
120 return __builtin_ctz(Val);
121#elif defined(_MSC_VER)
122 unsigned long Index;
123 _BitScanForward(&Index, Val);
124 return Index;
125#endif
126 }
127};
128
129#if !defined(_MSC_VER) || defined(_M_X64)
130template <typename T> struct TrailingZerosCounter<T, 8> {
131 static unsigned count(T Val, ZeroBehavior ZB) {
132 if (ZB != ZB_Undefined && Val == 0)
133 return 64;
134
135#if __has_builtin(__builtin_ctzll)1 || defined(__GNUC__4)
136 return __builtin_ctzll(Val);
137#elif defined(_MSC_VER)
138 unsigned long Index;
139 _BitScanForward64(&Index, Val);
140 return Index;
141#endif
142 }
143};
144#endif
145#endif
146} // namespace detail
147
148/// Count number of 0's from the least significant bit to the most
149/// stopping at the first 1.
150///
151/// Only unsigned integral types are allowed.
152///
153/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
154/// valid arguments.
155template <typename T>
156unsigned countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
157 static_assert(std::numeric_limits<T>::is_integer &&
158 !std::numeric_limits<T>::is_signed,
159 "Only unsigned integral types are allowed.");
160 return llvm::detail::TrailingZerosCounter<T, sizeof(T)>::count(Val, ZB);
161}
162
163namespace detail {
164template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter {
165 static unsigned count(T Val, ZeroBehavior) {
166 if (!Val)
167 return std::numeric_limits<T>::digits;
168
169 // Bisection method.
170 unsigned ZeroBits = 0;
171 for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) {
172 T Tmp = Val >> Shift;
173 if (Tmp)
174 Val = Tmp;
175 else
176 ZeroBits |= Shift;
177 }
178 return ZeroBits;
179 }
180};
181
182#if defined(__GNUC__4) || defined(_MSC_VER)
183template <typename T> struct LeadingZerosCounter<T, 4> {
184 static unsigned count(T Val, ZeroBehavior ZB) {
185 if (ZB != ZB_Undefined && Val == 0)
186 return 32;
187
188#if __has_builtin(__builtin_clz)1 || defined(__GNUC__4)
189 return __builtin_clz(Val);
190#elif defined(_MSC_VER)
191 unsigned long Index;
192 _BitScanReverse(&Index, Val);
193 return Index ^ 31;
194#endif
195 }
196};
197
198#if !defined(_MSC_VER) || defined(_M_X64)
199template <typename T> struct LeadingZerosCounter<T, 8> {
200 static unsigned count(T Val, ZeroBehavior ZB) {
201 if (ZB != ZB_Undefined && Val == 0)
202 return 64;
203
204#if __has_builtin(__builtin_clzll)1 || defined(__GNUC__4)
205 return __builtin_clzll(Val);
206#elif defined(_MSC_VER)
207 unsigned long Index;
208 _BitScanReverse64(&Index, Val);
209 return Index ^ 63;
210#endif
211 }
212};
213#endif
214#endif
215} // namespace detail
216
217/// Count number of 0's from the most significant bit to the least
218/// stopping at the first 1.
219///
220/// Only unsigned integral types are allowed.
221///
222/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
223/// valid arguments.
224template <typename T>
225unsigned countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
226 static_assert(std::numeric_limits<T>::is_integer &&
227 !std::numeric_limits<T>::is_signed,
228 "Only unsigned integral types are allowed.");
229 return llvm::detail::LeadingZerosCounter<T, sizeof(T)>::count(Val, ZB);
230}
231
232/// Get the index of the first set bit starting from the least
233/// significant bit.
234///
235/// Only unsigned integral types are allowed.
236///
237/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
238/// valid arguments.
239template <typename T> T findFirstSet(T Val, ZeroBehavior ZB = ZB_Max) {
240 if (ZB == ZB_Max && Val == 0)
241 return std::numeric_limits<T>::max();
242
243 return countTrailingZeros(Val, ZB_Undefined);
244}
245
246/// Create a bitmask with the N right-most bits set to 1, and all other
247/// bits set to 0. Only unsigned types are allowed.
248template <typename T> T maskTrailingOnes(unsigned N) {
249 static_assert(std::is_unsigned<T>::value, "Invalid type!");
250 const unsigned Bits = CHAR_BIT8 * sizeof(T);
251 assert(N <= Bits && "Invalid bit index")((void)0);
252 return N == 0 ? 0 : (T(-1) >> (Bits - N));
253}
254
255/// Create a bitmask with the N left-most bits set to 1, and all other
256/// bits set to 0. Only unsigned types are allowed.
257template <typename T> T maskLeadingOnes(unsigned N) {
258 return ~maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
259}
260
261/// Create a bitmask with the N right-most bits set to 0, and all other
262/// bits set to 1. Only unsigned types are allowed.
263template <typename T> T maskTrailingZeros(unsigned N) {
264 return maskLeadingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
265}
266
267/// Create a bitmask with the N left-most bits set to 0, and all other
268/// bits set to 1. Only unsigned types are allowed.
269template <typename T> T maskLeadingZeros(unsigned N) {
270 return maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
271}
272
273/// Get the index of the last set bit starting from the least
274/// significant bit.
275///
276/// Only unsigned integral types are allowed.
277///
278/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
279/// valid arguments.
280template <typename T> T findLastSet(T Val, ZeroBehavior ZB = ZB_Max) {
281 if (ZB == ZB_Max && Val == 0)
282 return std::numeric_limits<T>::max();
283
284 // Use ^ instead of - because both gcc and llvm can remove the associated ^
285 // in the __builtin_clz intrinsic on x86.
286 return countLeadingZeros(Val, ZB_Undefined) ^
287 (std::numeric_limits<T>::digits - 1);
288}
289
290/// Macro compressed bit reversal table for 256 bits.
291///
292/// http://graphics.stanford.edu/~seander/bithacks.html#BitReverseTable
293static const unsigned char BitReverseTable256[256] = {
294#define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64
295#define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16)
296#define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4)
297 R6(0), R6(2), R6(1), R6(3)
298#undef R2
299#undef R4
300#undef R6
301};
302
303/// Reverse the bits in \p Val.
304template <typename T>
305T reverseBits(T Val) {
306 unsigned char in[sizeof(Val)];
307 unsigned char out[sizeof(Val)];
308 std::memcpy(in, &Val, sizeof(Val));
309 for (unsigned i = 0; i < sizeof(Val); ++i)
310 out[(sizeof(Val) - i) - 1] = BitReverseTable256[in[i]];
311 std::memcpy(&Val, out, sizeof(Val));
312 return Val;
313}
314
315#if __has_builtin(__builtin_bitreverse8)1
316template<>
317inline uint8_t reverseBits<uint8_t>(uint8_t Val) {
318 return __builtin_bitreverse8(Val);
319}
320#endif
321
322#if __has_builtin(__builtin_bitreverse16)1
323template<>
324inline uint16_t reverseBits<uint16_t>(uint16_t Val) {
325 return __builtin_bitreverse16(Val);
326}
327#endif
328
329#if __has_builtin(__builtin_bitreverse32)1
330template<>
331inline uint32_t reverseBits<uint32_t>(uint32_t Val) {
332 return __builtin_bitreverse32(Val);
333}
334#endif
335
336#if __has_builtin(__builtin_bitreverse64)1
337template<>
338inline uint64_t reverseBits<uint64_t>(uint64_t Val) {
339 return __builtin_bitreverse64(Val);
340}
341#endif
342
343// NOTE: The following support functions use the _32/_64 extensions instead of
344// type overloading so that signed and unsigned integers can be used without
345// ambiguity.
346
347/// Return the high 32 bits of a 64 bit value.
348constexpr inline uint32_t Hi_32(uint64_t Value) {
349 return static_cast<uint32_t>(Value >> 32);
350}
351
352/// Return the low 32 bits of a 64 bit value.
353constexpr inline uint32_t Lo_32(uint64_t Value) {
354 return static_cast<uint32_t>(Value);
355}
356
357/// Make a 64-bit integer from a high / low pair of 32-bit integers.
358constexpr inline uint64_t Make_64(uint32_t High, uint32_t Low) {
359 return ((uint64_t)High << 32) | (uint64_t)Low;
360}
361
362/// Checks if an integer fits into the given bit width.
363template <unsigned N> constexpr inline bool isInt(int64_t x) {
364 return N >= 64 || (-(INT64_C(1)1LL<<(N-1)) <= x && x < (INT64_C(1)1LL<<(N-1)));
365}
366// Template specializations to get better code for common cases.
367template <> constexpr inline bool isInt<8>(int64_t x) {
368 return static_cast<int8_t>(x) == x;
369}
370template <> constexpr inline bool isInt<16>(int64_t x) {
371 return static_cast<int16_t>(x) == x;
372}
373template <> constexpr inline bool isInt<32>(int64_t x) {
374 return static_cast<int32_t>(x) == x;
375}
376
377/// Checks if a signed integer is an N bit number shifted left by S.
378template <unsigned N, unsigned S>
379constexpr inline bool isShiftedInt(int64_t x) {
380 static_assert(
381 N > 0, "isShiftedInt<0> doesn't make sense (refers to a 0-bit number.");
382 static_assert(N + S <= 64, "isShiftedInt<N, S> with N + S > 64 is too wide.");
383 return isInt<N + S>(x) && (x % (UINT64_C(1)1ULL << S) == 0);
384}
385
386/// Checks if an unsigned integer fits into the given bit width.
387///
388/// This is written as two functions rather than as simply
389///
390/// return N >= 64 || X < (UINT64_C(1) << N);
391///
392/// to keep MSVC from (incorrectly) warning on isUInt<64> that we're shifting
393/// left too many places.
394template <unsigned N>
395constexpr inline std::enable_if_t<(N < 64), bool> isUInt(uint64_t X) {
396 static_assert(N > 0, "isUInt<0> doesn't make sense");
397 return X < (UINT64_C(1)1ULL << (N));
398}
399template <unsigned N>
400constexpr inline std::enable_if_t<N >= 64, bool> isUInt(uint64_t) {
401 return true;
402}
403
404// Template specializations to get better code for common cases.
405template <> constexpr inline bool isUInt<8>(uint64_t x) {
406 return static_cast<uint8_t>(x) == x;
407}
408template <> constexpr inline bool isUInt<16>(uint64_t x) {
409 return static_cast<uint16_t>(x) == x;
410}
411template <> constexpr inline bool isUInt<32>(uint64_t x) {
412 return static_cast<uint32_t>(x) == x;
413}
414
415/// Checks if a unsigned integer is an N bit number shifted left by S.
416template <unsigned N, unsigned S>
417constexpr inline bool isShiftedUInt(uint64_t x) {
418 static_assert(
419 N > 0, "isShiftedUInt<0> doesn't make sense (refers to a 0-bit number)");
420 static_assert(N + S <= 64,
421 "isShiftedUInt<N, S> with N + S > 64 is too wide.");
422 // Per the two static_asserts above, S must be strictly less than 64. So
423 // 1 << S is not undefined behavior.
424 return isUInt<N + S>(x) && (x % (UINT64_C(1)1ULL << S) == 0);
425}
426
427/// Gets the maximum value for a N-bit unsigned integer.
428inline uint64_t maxUIntN(uint64_t N) {
429 assert(N > 0 && N <= 64 && "integer width out of range")((void)0);
430
431 // uint64_t(1) << 64 is undefined behavior, so we can't do
432 // (uint64_t(1) << N) - 1
433 // without checking first that N != 64. But this works and doesn't have a
434 // branch.
435 return UINT64_MAX0xffffffffffffffffULL >> (64 - N);
436}
437
438/// Gets the minimum value for a N-bit signed integer.
439inline int64_t minIntN(int64_t N) {
440 assert(N > 0 && N <= 64 && "integer width out of range")((void)0);
441
442 return UINT64_C(1)1ULL + ~(UINT64_C(1)1ULL << (N - 1));
443}
444
445/// Gets the maximum value for a N-bit signed integer.
446inline int64_t maxIntN(int64_t N) {
447 assert(N > 0 && N <= 64 && "integer width out of range")((void)0);
448
449 // This relies on two's complement wraparound when N == 64, so we convert to
450 // int64_t only at the very end to avoid UB.
451 return (UINT64_C(1)1ULL << (N - 1)) - 1;
452}
453
454/// Checks if an unsigned integer fits into the given (dynamic) bit width.
455inline bool isUIntN(unsigned N, uint64_t x) {
456 return N >= 64 || x <= maxUIntN(N);
457}
458
459/// Checks if an signed integer fits into the given (dynamic) bit width.
460inline bool isIntN(unsigned N, int64_t x) {
461 return N >= 64 || (minIntN(N) <= x && x <= maxIntN(N));
462}
463
464/// Return true if the argument is a non-empty sequence of ones starting at the
465/// least significant bit with the remainder zero (32 bit version).
466/// Ex. isMask_32(0x0000FFFFU) == true.
467constexpr inline bool isMask_32(uint32_t Value) {
468 return Value && ((Value + 1) & Value) == 0;
469}
470
471/// Return true if the argument is a non-empty sequence of ones starting at the
472/// least significant bit with the remainder zero (64 bit version).
473constexpr inline bool isMask_64(uint64_t Value) {
474 return Value && ((Value + 1) & Value) == 0;
475}
476
477/// Return true if the argument contains a non-empty sequence of ones with the
478/// remainder zero (32 bit version.) Ex. isShiftedMask_32(0x0000FF00U) == true.
479constexpr inline bool isShiftedMask_32(uint32_t Value) {
480 return Value && isMask_32((Value - 1) | Value);
481}
482
483/// Return true if the argument contains a non-empty sequence of ones with the
484/// remainder zero (64 bit version.)
485constexpr inline bool isShiftedMask_64(uint64_t Value) {
486 return Value && isMask_64((Value - 1) | Value);
487}
488
489/// Return true if the argument is a power of two > 0.
490/// Ex. isPowerOf2_32(0x00100000U) == true (32 bit edition.)
491constexpr inline bool isPowerOf2_32(uint32_t Value) {
492 return Value && !(Value & (Value - 1));
493}
494
495/// Return true if the argument is a power of two > 0 (64 bit edition.)
496constexpr inline bool isPowerOf2_64(uint64_t Value) {
497 return Value && !(Value & (Value - 1));
498}
499
500/// Count the number of ones from the most significant bit to the first
501/// zero bit.
502///
503/// Ex. countLeadingOnes(0xFF0FFF00) == 8.
504/// Only unsigned integral types are allowed.
505///
506/// \param ZB the behavior on an input of all ones. Only ZB_Width and
507/// ZB_Undefined are valid arguments.
508template <typename T>
509unsigned countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
510 static_assert(std::numeric_limits<T>::is_integer &&
511 !std::numeric_limits<T>::is_signed,
512 "Only unsigned integral types are allowed.");
513 return countLeadingZeros<T>(~Value, ZB);
514}
515
516/// Count the number of ones from the least significant bit to the first
517/// zero bit.
518///
519/// Ex. countTrailingOnes(0x00FF00FF) == 8.
520/// Only unsigned integral types are allowed.
521///
522/// \param ZB the behavior on an input of all ones. Only ZB_Width and
523/// ZB_Undefined are valid arguments.
524template <typename T>
525unsigned countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
526 static_assert(std::numeric_limits<T>::is_integer &&
527 !std::numeric_limits<T>::is_signed,
528 "Only unsigned integral types are allowed.");
529 return countTrailingZeros<T>(~Value, ZB);
530}
531
532namespace detail {
533template <typename T, std::size_t SizeOfT> struct PopulationCounter {
534 static unsigned count(T Value) {
535 // Generic version, forward to 32 bits.
536 static_assert(SizeOfT <= 4, "Not implemented!");
537#if defined(__GNUC__4)
538 return __builtin_popcount(Value);
539#else
540 uint32_t v = Value;
541 v = v - ((v >> 1) & 0x55555555);
542 v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
543 return ((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24;
544#endif
545 }
546};
547
548template <typename T> struct PopulationCounter<T, 8> {
549 static unsigned count(T Value) {
550#if defined(__GNUC__4)
551 return __builtin_popcountll(Value);
552#else
553 uint64_t v = Value;
554 v = v - ((v >> 1) & 0x5555555555555555ULL);
555 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
556 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
557 return unsigned((uint64_t)(v * 0x0101010101010101ULL) >> 56);
558#endif
559 }
560};
561} // namespace detail
562
563/// Count the number of set bits in a value.
564/// Ex. countPopulation(0xF000F000) = 8
565/// Returns 0 if the word is zero.
566template <typename T>
567inline unsigned countPopulation(T Value) {
568 static_assert(std::numeric_limits<T>::is_integer &&
569 !std::numeric_limits<T>::is_signed,
570 "Only unsigned integral types are allowed.");
571 return detail::PopulationCounter<T, sizeof(T)>::count(Value);
572}
573
574/// Compile time Log2.
575/// Valid only for positive powers of two.
576template <size_t kValue> constexpr inline size_t CTLog2() {
577 static_assert(kValue > 0 && llvm::isPowerOf2_64(kValue),
578 "Value is not a valid power of 2");
579 return 1 + CTLog2<kValue / 2>();
580}
581
582template <> constexpr inline size_t CTLog2<1>() { return 0; }
583
584/// Return the log base 2 of the specified value.
585inline double Log2(double Value) {
586#if defined(__ANDROID_API__) && __ANDROID_API__ < 18
587 return __builtin_log(Value) / __builtin_log(2.0);
588#else
589 return log2(Value);
590#endif
591}
592
593/// Return the floor log base 2 of the specified value, -1 if the value is zero.
594/// (32 bit edition.)
595/// Ex. Log2_32(32) == 5, Log2_32(1) == 0, Log2_32(0) == -1, Log2_32(6) == 2
596inline unsigned Log2_32(uint32_t Value) {
597 return 31 - countLeadingZeros(Value);
598}
599
600/// Return the floor log base 2 of the specified value, -1 if the value is zero.
601/// (64 bit edition.)
602inline unsigned Log2_64(uint64_t Value) {
603 return 63 - countLeadingZeros(Value);
5
Returning the value 4294967295
604}
605
606/// Return the ceil log base 2 of the specified value, 32 if the value is zero.
607/// (32 bit edition).
608/// Ex. Log2_32_Ceil(32) == 5, Log2_32_Ceil(1) == 0, Log2_32_Ceil(6) == 3
609inline unsigned Log2_32_Ceil(uint32_t Value) {
610 return 32 - countLeadingZeros(Value - 1);
611}
612
613/// Return the ceil log base 2 of the specified value, 64 if the value is zero.
614/// (64 bit edition.)
615inline unsigned Log2_64_Ceil(uint64_t Value) {
616 return 64 - countLeadingZeros(Value - 1);
617}
618
619/// Return the greatest common divisor of the values using Euclid's algorithm.
620template <typename T>
621inline T greatestCommonDivisor(T A, T B) {
622 while (B) {
623 T Tmp = B;
624 B = A % B;
625 A = Tmp;
626 }
627 return A;
628}
629
630inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) {
631 return greatestCommonDivisor<uint64_t>(A, B);
632}
633
634/// This function takes a 64-bit integer and returns the bit equivalent double.
635inline double BitsToDouble(uint64_t Bits) {
636 double D;
637 static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
638 memcpy(&D, &Bits, sizeof(Bits));
639 return D;
640}
641
642/// This function takes a 32-bit integer and returns the bit equivalent float.
643inline float BitsToFloat(uint32_t Bits) {
644 float F;
645 static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
646 memcpy(&F, &Bits, sizeof(Bits));
647 return F;
648}
649
650/// This function takes a double and returns the bit equivalent 64-bit integer.
651/// Note that copying doubles around changes the bits of NaNs on some hosts,
652/// notably x86, so this routine cannot be used if these bits are needed.
653inline uint64_t DoubleToBits(double Double) {
654 uint64_t Bits;
655 static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
656 memcpy(&Bits, &Double, sizeof(Double));
657 return Bits;
658}
659
660/// This function takes a float and returns the bit equivalent 32-bit integer.
661/// Note that copying floats around changes the bits of NaNs on some hosts,
662/// notably x86, so this routine cannot be used if these bits are needed.
663inline uint32_t FloatToBits(float Float) {
664 uint32_t Bits;
665 static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
666 memcpy(&Bits, &Float, sizeof(Float));
667 return Bits;
668}
669
670/// A and B are either alignments or offsets. Return the minimum alignment that
671/// may be assumed after adding the two together.
672constexpr inline uint64_t MinAlign(uint64_t A, uint64_t B) {
673 // The largest power of 2 that divides both A and B.
674 //
675 // Replace "-Value" by "1+~Value" in the following commented code to avoid
676 // MSVC warning C4146
677 // return (A | B) & -(A | B);
678 return (A | B) & (1 + ~(A | B));
679}
680
681/// Returns the next power of two (in 64-bits) that is strictly greater than A.
682/// Returns zero on overflow.
683inline uint64_t NextPowerOf2(uint64_t A) {
684 A |= (A >> 1);
685 A |= (A >> 2);
686 A |= (A >> 4);
687 A |= (A >> 8);
688 A |= (A >> 16);
689 A |= (A >> 32);
690 return A + 1;
691}
692
693/// Returns the power of two which is less than or equal to the given value.
694/// Essentially, it is a floor operation across the domain of powers of two.
695inline uint64_t PowerOf2Floor(uint64_t A) {
696 if (!A) return 0;
697 return 1ull << (63 - countLeadingZeros(A, ZB_Undefined));
698}
699
700/// Returns the power of two which is greater than or equal to the given value.
701/// Essentially, it is a ceil operation across the domain of powers of two.
702inline uint64_t PowerOf2Ceil(uint64_t A) {
703 if (!A)
704 return 0;
705 return NextPowerOf2(A - 1);
706}
707
708/// Returns the next integer (mod 2**64) that is greater than or equal to
709/// \p Value and is a multiple of \p Align. \p Align must be non-zero.
710///
711/// If non-zero \p Skew is specified, the return value will be a minimal
712/// integer that is greater than or equal to \p Value and equal to
713/// \p Align * N + \p Skew for some integer N. If \p Skew is larger than
714/// \p Align, its value is adjusted to '\p Skew mod \p Align'.
715///
716/// Examples:
717/// \code
718/// alignTo(5, 8) = 8
719/// alignTo(17, 8) = 24
720/// alignTo(~0LL, 8) = 0
721/// alignTo(321, 255) = 510
722///
723/// alignTo(5, 8, 7) = 7
724/// alignTo(17, 8, 1) = 17
725/// alignTo(~0LL, 8, 3) = 3
726/// alignTo(321, 255, 42) = 552
727/// \endcode
728inline uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
729 assert(Align != 0u && "Align can't be 0.")((void)0);
730 Skew %= Align;
731 return (Value + Align - 1 - Skew) / Align * Align + Skew;
732}
733
734/// Returns the next integer (mod 2**64) that is greater than or equal to
735/// \p Value and is a multiple of \c Align. \c Align must be non-zero.
736template <uint64_t Align> constexpr inline uint64_t alignTo(uint64_t Value) {
737 static_assert(Align != 0u, "Align must be non-zero");
738 return (Value + Align - 1) / Align * Align;
739}
740
741/// Returns the integer ceil(Numerator / Denominator).
742inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) {
743 return alignTo(Numerator, Denominator) / Denominator;
744}
745
746/// Returns the integer nearest(Numerator / Denominator).
747inline uint64_t divideNearest(uint64_t Numerator, uint64_t Denominator) {
748 return (Numerator + (Denominator / 2)) / Denominator;
749}
750
751/// Returns the largest uint64_t less than or equal to \p Value and is
752/// \p Skew mod \p Align. \p Align must be non-zero
753inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
754 assert(Align != 0u && "Align can't be 0.")((void)0);
755 Skew %= Align;
756 return (Value - Skew) / Align * Align + Skew;
757}
758
759/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
760/// Requires 0 < B <= 32.
761template <unsigned B> constexpr inline int32_t SignExtend32(uint32_t X) {
762 static_assert(B > 0, "Bit width can't be 0.");
763 static_assert(B <= 32, "Bit width out of range.");
764 return int32_t(X << (32 - B)) >> (32 - B);
765}
766
767/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
768/// Requires 0 < B <= 32.
769inline int32_t SignExtend32(uint32_t X, unsigned B) {
770 assert(B > 0 && "Bit width can't be 0.")((void)0);
771 assert(B <= 32 && "Bit width out of range.")((void)0);
772 return int32_t(X << (32 - B)) >> (32 - B);
773}
774
775/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
776/// Requires 0 < B <= 64.
777template <unsigned B> constexpr inline int64_t SignExtend64(uint64_t x) {
778 static_assert(B > 0, "Bit width can't be 0.");
779 static_assert(B <= 64, "Bit width out of range.");
780 return int64_t(x << (64 - B)) >> (64 - B);
781}
782
783/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
784/// Requires 0 < B <= 64.
785inline int64_t SignExtend64(uint64_t X, unsigned B) {
786 assert(B > 0 && "Bit width can't be 0.")((void)0);
787 assert(B <= 64 && "Bit width out of range.")((void)0);
788 return int64_t(X << (64 - B)) >> (64 - B);
789}
790
791/// Subtract two unsigned integers, X and Y, of type T and return the absolute
792/// value of the result.
793template <typename T>
794std::enable_if_t<std::is_unsigned<T>::value, T> AbsoluteDifference(T X, T Y) {
795 return X > Y ? (X - Y) : (Y - X);
796}
797
798/// Add two unsigned integers, X and Y, of type T. Clamp the result to the
799/// maximum representable value of T on overflow. ResultOverflowed indicates if
800/// the result is larger than the maximum representable value of type T.
801template <typename T>
802std::enable_if_t<std::is_unsigned<T>::value, T>
803SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) {
804 bool Dummy;
805 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
806 // Hacker's Delight, p. 29
807 T Z = X + Y;
808 Overflowed = (Z < X || Z < Y);
809 if (Overflowed)
810 return std::numeric_limits<T>::max();
811 else
812 return Z;
813}
814
815/// Multiply two unsigned integers, X and Y, of type T. Clamp the result to the
816/// maximum representable value of T on overflow. ResultOverflowed indicates if
817/// the result is larger than the maximum representable value of type T.
818template <typename T>
819std::enable_if_t<std::is_unsigned<T>::value, T>
820SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) {
821 bool Dummy;
822 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
823
824 // Hacker's Delight, p. 30 has a different algorithm, but we don't use that
825 // because it fails for uint16_t (where multiplication can have undefined
826 // behavior due to promotion to int), and requires a division in addition
827 // to the multiplication.
828
829 Overflowed = false;
830
831 // Log2(Z) would be either Log2Z or Log2Z + 1.
832 // Special case: if X or Y is 0, Log2_64 gives -1, and Log2Z
833 // will necessarily be less than Log2Max as desired.
834 int Log2Z = Log2_64(X) + Log2_64(Y);
835 const T Max = std::numeric_limits<T>::max();
836 int Log2Max = Log2_64(Max);
837 if (Log2Z < Log2Max) {
838 return X * Y;
839 }
840 if (Log2Z > Log2Max) {
841 Overflowed = true;
842 return Max;
843 }
844
845 // We're going to use the top bit, and maybe overflow one
846 // bit past it. Multiply all but the bottom bit then add
847 // that on at the end.
848 T Z = (X >> 1) * Y;
849 if (Z & ~(Max >> 1)) {
850 Overflowed = true;
851 return Max;
852 }
853 Z <<= 1;
854 if (X & 1)
855 return SaturatingAdd(Z, Y, ResultOverflowed);
856
857 return Z;
858}
859
860/// Multiply two unsigned integers, X and Y, and add the unsigned integer, A to
861/// the product. Clamp the result to the maximum representable value of T on
862/// overflow. ResultOverflowed indicates if the result is larger than the
863/// maximum representable value of type T.
864template <typename T>
865std::enable_if_t<std::is_unsigned<T>::value, T>
866SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) {
867 bool Dummy;
868 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
869
870 T Product = SaturatingMultiply(X, Y, &Overflowed);
871 if (Overflowed)
872 return Product;
873
874 return SaturatingAdd(A, Product, &Overflowed);
875}
876
877/// Use this rather than HUGE_VALF; the latter causes warnings on MSVC.
878extern const float huge_valf;
879
880
881/// Add two signed integers, computing the two's complement truncated result,
882/// returning true if overflow occured.
883template <typename T>
884std::enable_if_t<std::is_signed<T>::value, T> AddOverflow(T X, T Y, T &Result) {
885#if __has_builtin(__builtin_add_overflow)1
886 return __builtin_add_overflow(X, Y, &Result);
887#else
888 // Perform the unsigned addition.
889 using U = std::make_unsigned_t<T>;
890 const U UX = static_cast<U>(X);
891 const U UY = static_cast<U>(Y);
892 const U UResult = UX + UY;
893
894 // Convert to signed.
895 Result = static_cast<T>(UResult);
896
897 // Adding two positive numbers should result in a positive number.
898 if (X > 0 && Y > 0)
899 return Result <= 0;
900 // Adding two negatives should result in a negative number.
901 if (X < 0 && Y < 0)
902 return Result >= 0;
903 return false;
904#endif
905}
906
907/// Subtract two signed integers, computing the two's complement truncated
908/// result, returning true if an overflow ocurred.
909template <typename T>
910std::enable_if_t<std::is_signed<T>::value, T> SubOverflow(T X, T Y, T &Result) {
911#if __has_builtin(__builtin_sub_overflow)1
912 return __builtin_sub_overflow(X, Y, &Result);
913#else
914 // Perform the unsigned addition.
915 using U = std::make_unsigned_t<T>;
916 const U UX = static_cast<U>(X);
917 const U UY = static_cast<U>(Y);
918 const U UResult = UX - UY;
919
920 // Convert to signed.
921 Result = static_cast<T>(UResult);
922
923 // Subtracting a positive number from a negative results in a negative number.
924 if (X <= 0 && Y > 0)
925 return Result >= 0;
926 // Subtracting a negative number from a positive results in a positive number.
927 if (X >= 0 && Y < 0)
928 return Result <= 0;
929 return false;
930#endif
931}
932
933/// Multiply two signed integers, computing the two's complement truncated
934/// result, returning true if an overflow ocurred.
935template <typename T>
936std::enable_if_t<std::is_signed<T>::value, T> MulOverflow(T X, T Y, T &Result) {
937 // Perform the unsigned multiplication on absolute values.
938 using U = std::make_unsigned_t<T>;
939 const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X);
940 const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y);
941 const U UResult = UX * UY;
942
943 // Convert to signed.
944 const bool IsNegative = (X < 0) ^ (Y < 0);
945 Result = IsNegative ? (0 - UResult) : UResult;
946
947 // If any of the args was 0, result is 0 and no overflow occurs.
948 if (UX == 0 || UY == 0)
949 return false;
950
951 // UX and UY are in [1, 2^n], where n is the number of digits.
952 // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
953 // positive) divided by an argument compares to the other.
954 if (IsNegative)
955 return UX > (static_cast<U>(std::numeric_limits<T>::max()) + U(1)) / UY;
956 else
957 return UX > (static_cast<U>(std::numeric_limits<T>::max())) / UY;
958}
959
960} // End llvm namespace
961
962#endif