clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name InstrProf.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Analysis -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ASMParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/BinaryFormat -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitstream -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /include/llvm/CodeGen -I /include/llvm/CodeGen/PBQP -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Coroutines -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData/Coverage -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/CodeView -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/DWARF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/MSF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/PDB -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Demangle -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/JITLink -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/Orc -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenACC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenMP -I /include/llvm/CodeGen/GlobalISel -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IRReader -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/LTO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Linker -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC/MCParser -I /include/llvm/CodeGen/MIRParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Object -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Option -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Passes -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Scalar -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ADT -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Support -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/Symbolize -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Target -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Utils -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Vectorize -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/IPO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include -I /usr/src/gnu/usr.bin/clang/libLLVM/../include -I /usr/src/gnu/usr.bin/clang/libLLVM/obj -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include -D NDEBUG -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D LLVM_PREFIX="/usr" -internal-isystem /usr/include/c++/v1 -internal-isystem /usr/local/lib/clang/13.0.0/include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -ferror-limit 19 -fvisibility-inlines-hidden -fwrapv -stack-protector 2 -fno-rtti -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/ben/Projects/vmm/scan-build/2022-01-12-194120-40624-1 -x c++ /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/ProfileData/InstrProf.cpp
1 | |
2 | |
3 | |
4 | |
5 | |
6 | |
7 | |
8 | |
9 | |
10 | |
11 | |
12 | |
13 | |
14 | #include "llvm/ProfileData/InstrProf.h" |
15 | #include "llvm/ADT/ArrayRef.h" |
16 | #include "llvm/ADT/SmallString.h" |
17 | #include "llvm/ADT/SmallVector.h" |
18 | #include "llvm/ADT/StringExtras.h" |
19 | #include "llvm/ADT/StringRef.h" |
20 | #include "llvm/ADT/Triple.h" |
21 | #include "llvm/Config/config.h" |
22 | #include "llvm/IR/Constant.h" |
23 | #include "llvm/IR/Constants.h" |
24 | #include "llvm/IR/Function.h" |
25 | #include "llvm/IR/GlobalValue.h" |
26 | #include "llvm/IR/GlobalVariable.h" |
27 | #include "llvm/IR/Instruction.h" |
28 | #include "llvm/IR/LLVMContext.h" |
29 | #include "llvm/IR/MDBuilder.h" |
30 | #include "llvm/IR/Metadata.h" |
31 | #include "llvm/IR/Module.h" |
32 | #include "llvm/IR/Type.h" |
33 | #include "llvm/ProfileData/InstrProfReader.h" |
34 | #include "llvm/Support/Casting.h" |
35 | #include "llvm/Support/CommandLine.h" |
36 | #include "llvm/Support/Compiler.h" |
37 | #include "llvm/Support/Compression.h" |
38 | #include "llvm/Support/Endian.h" |
39 | #include "llvm/Support/Error.h" |
40 | #include "llvm/Support/ErrorHandling.h" |
41 | #include "llvm/Support/LEB128.h" |
42 | #include "llvm/Support/ManagedStatic.h" |
43 | #include "llvm/Support/MathExtras.h" |
44 | #include "llvm/Support/Path.h" |
45 | #include "llvm/Support/SwapByteOrder.h" |
46 | #include <algorithm> |
47 | #include <cassert> |
48 | #include <cstddef> |
49 | #include <cstdint> |
50 | #include <cstring> |
51 | #include <memory> |
52 | #include <string> |
53 | #include <system_error> |
54 | #include <utility> |
55 | #include <vector> |
56 | |
57 | using namespace llvm; |
58 | |
59 | static cl::opt<bool> StaticFuncFullModulePrefix( |
60 | "static-func-full-module-prefix", cl::init(true), cl::Hidden, |
61 | cl::desc("Use full module build paths in the profile counter names for " |
62 | "static functions.")); |
63 | |
64 | |
65 | |
66 | |
67 | |
68 | |
69 | |
70 | |
71 | |
72 | static cl::opt<unsigned> StaticFuncStripDirNamePrefix( |
73 | "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, |
74 | cl::desc("Strip specified level of directory name from source path in " |
75 | "the profile counter name for static functions.")); |
76 | |
77 | static std::string getInstrProfErrString(instrprof_error Err) { |
78 | switch (Err) { |
79 | case instrprof_error::success: |
80 | return "success"; |
81 | case instrprof_error::eof: |
82 | return "end of File"; |
83 | case instrprof_error::unrecognized_format: |
84 | return "unrecognized instrumentation profile encoding format"; |
85 | case instrprof_error::bad_magic: |
86 | return "invalid instrumentation profile data (bad magic)"; |
87 | case instrprof_error::bad_header: |
88 | return "invalid instrumentation profile data (file header is corrupt)"; |
89 | case instrprof_error::unsupported_version: |
90 | return "unsupported instrumentation profile format version"; |
91 | case instrprof_error::unsupported_hash_type: |
92 | return "unsupported instrumentation profile hash type"; |
93 | case instrprof_error::too_large: |
94 | return "too much profile data"; |
95 | case instrprof_error::truncated: |
96 | return "truncated profile data"; |
97 | case instrprof_error::malformed: |
98 | return "malformed instrumentation profile data"; |
99 | case instrprof_error::invalid_prof: |
100 | return "invalid profile created. Please file a bug " |
101 | "at: " BUG_REPORT_URL |
102 | " and include the profraw files that caused this error."; |
103 | case instrprof_error::unknown_function: |
104 | return "no profile data available for function"; |
105 | case instrprof_error::hash_mismatch: |
106 | return "function control flow change detected (hash mismatch)"; |
107 | case instrprof_error::count_mismatch: |
108 | return "function basic block count change detected (counter mismatch)"; |
109 | case instrprof_error::counter_overflow: |
110 | return "counter overflow"; |
111 | case instrprof_error::value_site_count_mismatch: |
112 | return "function value site count change detected (counter mismatch)"; |
113 | case instrprof_error::compress_failed: |
114 | return "failed to compress data (zlib)"; |
115 | case instrprof_error::uncompress_failed: |
116 | return "failed to uncompress data (zlib)"; |
117 | case instrprof_error::empty_raw_profile: |
118 | return "empty raw profile file"; |
119 | case instrprof_error::zlib_unavailable: |
120 | return "profile uses zlib compression but the profile reader was built " |
121 | "without zlib support"; |
122 | } |
123 | llvm_unreachable("A value of instrprof_error has no message."); |
124 | } |
125 | |
126 | namespace { |
127 | |
128 | |
129 | |
130 | |
131 | class InstrProfErrorCategoryType : public std::error_category { |
132 | const char *name() const noexcept override { return "llvm.instrprof"; } |
133 | |
134 | std::string message(int IE) const override { |
135 | return getInstrProfErrString(static_cast<instrprof_error>(IE)); |
136 | } |
137 | }; |
138 | |
139 | } |
140 | |
141 | static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory; |
142 | |
143 | const std::error_category &llvm::instrprof_category() { |
144 | return *ErrorCategory; |
145 | } |
146 | |
147 | namespace { |
148 | |
149 | const char *InstrProfSectNameCommon[] = { |
150 | #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ |
151 | SectNameCommon, |
152 | #include "llvm/ProfileData/InstrProfData.inc" |
153 | }; |
154 | |
155 | const char *InstrProfSectNameCoff[] = { |
156 | #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ |
157 | SectNameCoff, |
158 | #include "llvm/ProfileData/InstrProfData.inc" |
159 | }; |
160 | |
161 | const char *InstrProfSectNamePrefix[] = { |
162 | #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ |
163 | Prefix, |
164 | #include "llvm/ProfileData/InstrProfData.inc" |
165 | }; |
166 | |
167 | } |
168 | |
169 | namespace llvm { |
170 | |
171 | cl::opt<bool> DoInstrProfNameCompression( |
172 | "enable-name-compression", |
173 | cl::desc("Enable name/filename string compression"), cl::init(true)); |
174 | |
175 | std::string getInstrProfSectionName(InstrProfSectKind IPSK, |
176 | Triple::ObjectFormatType OF, |
177 | bool AddSegmentInfo) { |
178 | std::string SectName; |
179 | |
180 | if (OF == Triple::MachO && AddSegmentInfo) |
181 | SectName = InstrProfSectNamePrefix[IPSK]; |
182 | |
183 | if (OF == Triple::COFF) |
184 | SectName += InstrProfSectNameCoff[IPSK]; |
185 | else |
186 | SectName += InstrProfSectNameCommon[IPSK]; |
187 | |
188 | if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo) |
189 | SectName += ",regular,live_support"; |
190 | |
191 | return SectName; |
192 | } |
193 | |
194 | void SoftInstrProfErrors::addError(instrprof_error IE) { |
195 | if (IE == instrprof_error::success) |
196 | return; |
197 | |
198 | if (FirstError == instrprof_error::success) |
199 | FirstError = IE; |
200 | |
201 | switch (IE) { |
202 | case instrprof_error::hash_mismatch: |
203 | ++NumHashMismatches; |
204 | break; |
205 | case instrprof_error::count_mismatch: |
206 | ++NumCountMismatches; |
207 | break; |
208 | case instrprof_error::counter_overflow: |
209 | ++NumCounterOverflows; |
210 | break; |
211 | case instrprof_error::value_site_count_mismatch: |
212 | ++NumValueSiteCountMismatches; |
213 | break; |
214 | default: |
215 | llvm_unreachable("Not a soft error"); |
216 | } |
217 | } |
218 | |
219 | std::string InstrProfError::message() const { |
220 | return getInstrProfErrString(Err); |
221 | } |
222 | |
223 | char InstrProfError::ID = 0; |
224 | |
225 | std::string getPGOFuncName(StringRef RawFuncName, |
226 | GlobalValue::LinkageTypes Linkage, |
227 | StringRef FileName, |
228 | uint64_t Version LLVM_ATTRIBUTE_UNUSED) { |
229 | return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName); |
230 | } |
231 | |
232 | |
233 | |
234 | |
235 | static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) { |
236 | uint32_t Count = NumPrefix; |
237 | uint32_t Pos = 0, LastPos = 0; |
238 | for (auto & CI : PathNameStr) { |
239 | ++Pos; |
240 | if (llvm::sys::path::is_separator(CI)) { |
241 | LastPos = Pos; |
242 | --Count; |
243 | } |
244 | if (Count == 0) |
245 | break; |
246 | } |
247 | return PathNameStr.substr(LastPos); |
248 | } |
249 | |
250 | |
251 | |
252 | |
253 | |
254 | |
255 | |
256 | |
257 | |
258 | |
259 | |
260 | |
261 | |
262 | |
263 | std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { |
264 | if (!InLTO) { |
265 | StringRef FileName(F.getParent()->getSourceFileName()); |
266 | uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1; |
267 | if (StripLevel < StaticFuncStripDirNamePrefix) |
268 | StripLevel = StaticFuncStripDirNamePrefix; |
269 | if (StripLevel) |
270 | FileName = stripDirPrefix(FileName, StripLevel); |
271 | return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version); |
272 | } |
273 | |
274 | |
275 | if (MDNode *MD = getPGOFuncNameMetadata(F)) { |
276 | StringRef S = cast<MDString>(MD->getOperand(0))->getString(); |
277 | return S.str(); |
278 | } |
279 | |
280 | |
281 | |
282 | |
283 | return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, ""); |
284 | } |
285 | |
286 | StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { |
287 | if (FileName.empty()) |
288 | return PGOFuncName; |
289 | |
290 | if (PGOFuncName.startswith(FileName)) |
291 | PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1); |
292 | return PGOFuncName; |
293 | } |
294 | |
295 | |
296 | |
297 | std::string getPGOFuncNameVarName(StringRef FuncName, |
298 | GlobalValue::LinkageTypes Linkage) { |
299 | std::string VarName = std::string(getInstrProfNameVarPrefix()); |
300 | VarName += FuncName; |
301 | |
302 | if (!GlobalValue::isLocalLinkage(Linkage)) |
303 | return VarName; |
304 | |
305 | |
306 | const char *InvalidChars = "-:<>/\"'"; |
307 | size_t found = VarName.find_first_of(InvalidChars); |
308 | while (found != std::string::npos) { |
309 | VarName[found] = '_'; |
310 | found = VarName.find_first_of(InvalidChars, found + 1); |
311 | } |
312 | return VarName; |
313 | } |
314 | |
315 | GlobalVariable *createPGOFuncNameVar(Module &M, |
316 | GlobalValue::LinkageTypes Linkage, |
317 | StringRef PGOFuncName) { |
318 | |
319 | |
320 | |
321 | if (Linkage == GlobalValue::ExternalWeakLinkage) |
322 | Linkage = GlobalValue::LinkOnceAnyLinkage; |
323 | else if (Linkage == GlobalValue::AvailableExternallyLinkage) |
324 | Linkage = GlobalValue::LinkOnceODRLinkage; |
325 | else if (Linkage == GlobalValue::InternalLinkage || |
326 | Linkage == GlobalValue::ExternalLinkage) |
327 | Linkage = GlobalValue::PrivateLinkage; |
328 | |
329 | auto *Value = |
330 | ConstantDataArray::getString(M.getContext(), PGOFuncName, false); |
331 | auto FuncNameVar = |
332 | new GlobalVariable(M, Value->getType(), true, Linkage, Value, |
333 | getPGOFuncNameVarName(PGOFuncName, Linkage)); |
334 | |
335 | |
336 | if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage())) |
337 | FuncNameVar->setVisibility(GlobalValue::HiddenVisibility); |
338 | |
339 | return FuncNameVar; |
340 | } |
341 | |
342 | GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) { |
343 | return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName); |
344 | } |
345 | |
346 | Error InstrProfSymtab::create(Module &M, bool InLTO) { |
347 | for (Function &F : M) { |
348 | |
349 | |
350 | if (!F.hasName()) |
351 | continue; |
352 | const std::string &PGOFuncName = getPGOFuncName(F, InLTO); |
353 | if (Error E = addFuncName(PGOFuncName)) |
354 | return E; |
355 | MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F); |
356 | |
357 | |
358 | |
359 | |
360 | |
361 | |
362 | |
363 | |
364 | |
365 | |
366 | const std::string UniqSuffix = ".__uniq."; |
367 | auto pos = PGOFuncName.find(UniqSuffix); |
368 | |
369 | |
370 | if (pos != std::string::npos) |
371 | pos += UniqSuffix.length(); |
372 | else |
373 | pos = 0; |
374 | pos = PGOFuncName.find('.', pos); |
375 | if (pos != std::string::npos && pos != 0) { |
376 | const std::string &OtherFuncName = PGOFuncName.substr(0, pos); |
377 | if (Error E = addFuncName(OtherFuncName)) |
378 | return E; |
379 | MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F); |
380 | } |
381 | } |
382 | Sorted = false; |
383 | finalizeSymtab(); |
384 | return Error::success(); |
385 | } |
386 | |
387 | uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) { |
388 | finalizeSymtab(); |
389 | auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) { |
390 | return A.first < Address; |
391 | }); |
392 | |
393 | |
394 | |
395 | |
396 | if (It != AddrToMD5Map.end() && It->first == Address) |
397 | return (uint64_t)It->second; |
398 | return 0; |
399 | } |
400 | |
401 | Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs, |
402 | bool doCompression, std::string &Result) { |
403 | assert(!NameStrs.empty() && "No name data to emit"); |
404 | |
405 | uint8_t Header[16], *P = Header; |
406 | std::string UncompressedNameStrings = |
407 | join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator()); |
408 | |
409 | assert(StringRef(UncompressedNameStrings) |
410 | .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && |
411 | "PGO name is invalid (contains separator token)"); |
412 | |
413 | unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P); |
414 | P += EncLen; |
415 | |
416 | auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) { |
417 | EncLen = encodeULEB128(CompressedLen, P); |
418 | P += EncLen; |
419 | char *HeaderStr = reinterpret_cast<char *>(&Header[0]); |
420 | unsigned HeaderLen = P - &Header[0]; |
421 | Result.append(HeaderStr, HeaderLen); |
422 | Result += InputStr; |
423 | return Error::success(); |
424 | }; |
425 | |
426 | if (!doCompression) { |
427 | return WriteStringToResult(0, UncompressedNameStrings); |
428 | } |
429 | |
430 | SmallString<128> CompressedNameStrings; |
431 | Error E = zlib::compress(StringRef(UncompressedNameStrings), |
432 | CompressedNameStrings, zlib::BestSizeCompression); |
433 | if (E) { |
434 | consumeError(std::move(E)); |
435 | return make_error<InstrProfError>(instrprof_error::compress_failed); |
436 | } |
437 | |
438 | return WriteStringToResult(CompressedNameStrings.size(), |
439 | CompressedNameStrings); |
440 | } |
441 | |
442 | StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) { |
443 | auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer()); |
444 | StringRef NameStr = |
445 | Arr->isCString() ? Arr->getAsCString() : Arr->getAsString(); |
446 | return NameStr; |
447 | } |
448 | |
449 | Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars, |
450 | std::string &Result, bool doCompression) { |
451 | std::vector<std::string> NameStrs; |
452 | for (auto *NameVar : NameVars) { |
453 | NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar))); |
454 | } |
455 | return collectPGOFuncNameStrings( |
456 | NameStrs, zlib::isAvailable() && doCompression, Result); |
457 | } |
458 | |
459 | Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { |
460 | const uint8_t *P = NameStrings.bytes_begin(); |
461 | const uint8_t *EndP = NameStrings.bytes_end(); |
462 | while (P < EndP) { |
| 1 | Assuming 'P' is < 'EndP' | |
|
| 2 | | Loop condition is true. Entering loop body | |
|
463 | uint32_t N; |
464 | uint64_t UncompressedSize = decodeULEB128(P, &N); |
| |
| 10 | | Returning from 'decodeULEB128' | |
|
| 11 | | 'UncompressedSize' initialized to 0 | |
|
465 | P += N; |
466 | uint64_t CompressedSize = decodeULEB128(P, &N); |
467 | P += N; |
468 | bool isCompressed = (CompressedSize != 0); |
469 | SmallString<128> UncompressedNameStrings; |
470 | StringRef NameStrings; |
471 | if (isCompressed) { |
| |
472 | if (!llvm::zlib::isAvailable()) |
473 | return make_error<InstrProfError>(instrprof_error::zlib_unavailable); |
474 | |
475 | StringRef CompressedNameStrings(reinterpret_cast<const char *>(P), |
476 | CompressedSize); |
477 | if (Error E = |
478 | zlib::uncompress(CompressedNameStrings, UncompressedNameStrings, |
479 | UncompressedSize)) { |
480 | consumeError(std::move(E)); |
481 | return make_error<InstrProfError>(instrprof_error::uncompress_failed); |
482 | } |
483 | P += CompressedSize; |
484 | NameStrings = StringRef(UncompressedNameStrings.data(), |
485 | UncompressedNameStrings.size()); |
486 | } else { |
487 | NameStrings = |
488 | StringRef(reinterpret_cast<const char *>(P), UncompressedSize); |
489 | P += UncompressedSize; |
| 13 | | Null pointer value stored to 'P' | |
|
490 | } |
491 | |
492 | SmallVector<StringRef, 0> Names; |
493 | NameStrings.split(Names, getInstrProfNameSeparator()); |
494 | for (StringRef &Name : Names) |
| 14 | | Assuming '__begin2' is equal to '__end2' | |
|
495 | if (Error E = Symtab.addFuncName(Name)) |
496 | return E; |
497 | |
498 | while (P < EndP && *P == 0) |
| 15 | | Dereference of null pointer (loaded from variable 'P') |
|
499 | P++; |
500 | } |
501 | return Error::success(); |
502 | } |
503 | |
504 | void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const { |
505 | uint64_t FuncSum = 0; |
506 | Sum.NumEntries += Counts.size(); |
507 | for (size_t F = 0, E = Counts.size(); F < E; ++F) |
508 | FuncSum += Counts[F]; |
509 | Sum.CountSum += FuncSum; |
510 | |
511 | for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) { |
512 | uint64_t KindSum = 0; |
513 | uint32_t NumValueSites = getNumValueSites(VK); |
514 | for (size_t I = 0; I < NumValueSites; ++I) { |
515 | uint32_t NV = getNumValueDataForSite(VK, I); |
516 | std::unique_ptr<InstrProfValueData[]> VD = getValueForSite(VK, I); |
517 | for (uint32_t V = 0; V < NV; V++) |
518 | KindSum += VD[V].Count; |
519 | } |
520 | Sum.ValueCounts[VK] += KindSum; |
521 | } |
522 | } |
523 | |
524 | void InstrProfValueSiteRecord::overlap(InstrProfValueSiteRecord &Input, |
525 | uint32_t ValueKind, |
526 | OverlapStats &Overlap, |
527 | OverlapStats &FuncLevelOverlap) { |
528 | this->sortByTargetValues(); |
529 | Input.sortByTargetValues(); |
530 | double Score = 0.0f, FuncLevelScore = 0.0f; |
531 | auto I = ValueData.begin(); |
532 | auto IE = ValueData.end(); |
533 | auto J = Input.ValueData.begin(); |
534 | auto JE = Input.ValueData.end(); |
535 | while (I != IE && J != JE) { |
536 | if (I->Value == J->Value) { |
537 | Score += OverlapStats::score(I->Count, J->Count, |
538 | Overlap.Base.ValueCounts[ValueKind], |
539 | Overlap.Test.ValueCounts[ValueKind]); |
540 | FuncLevelScore += OverlapStats::score( |
541 | I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind], |
542 | FuncLevelOverlap.Test.ValueCounts[ValueKind]); |
543 | ++I; |
544 | } else if (I->Value < J->Value) { |
545 | ++I; |
546 | continue; |
547 | } |
548 | ++J; |
549 | } |
550 | Overlap.Overlap.ValueCounts[ValueKind] += Score; |
551 | FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore; |
552 | } |
553 | |
554 | |
555 | void InstrProfRecord::overlapValueProfData(uint32_t ValueKind, |
556 | InstrProfRecord &Other, |
557 | OverlapStats &Overlap, |
558 | OverlapStats &FuncLevelOverlap) { |
559 | uint32_t ThisNumValueSites = getNumValueSites(ValueKind); |
560 | assert(ThisNumValueSites == Other.getNumValueSites(ValueKind)); |
561 | if (!ThisNumValueSites) |
562 | return; |
563 | |
564 | std::vector<InstrProfValueSiteRecord> &ThisSiteRecords = |
565 | getOrCreateValueSitesForKind(ValueKind); |
566 | MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords = |
567 | Other.getValueSitesForKind(ValueKind); |
568 | for (uint32_t I = 0; I < ThisNumValueSites; I++) |
569 | ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap, |
570 | FuncLevelOverlap); |
571 | } |
572 | |
573 | void InstrProfRecord::overlap(InstrProfRecord &Other, OverlapStats &Overlap, |
574 | OverlapStats &FuncLevelOverlap, |
575 | uint64_t ValueCutoff) { |
576 | |
577 | assert(FuncLevelOverlap.Test.CountSum >= 1.0f); |
578 | accumulateCounts(FuncLevelOverlap.Base); |
579 | bool Mismatch = (Counts.size() != Other.Counts.size()); |
580 | |
581 | |
582 | if (!Mismatch) { |
583 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) { |
584 | uint32_t ThisNumValueSites = getNumValueSites(Kind); |
585 | uint32_t OtherNumValueSites = Other.getNumValueSites(Kind); |
586 | if (ThisNumValueSites != OtherNumValueSites) { |
587 | Mismatch = true; |
588 | break; |
589 | } |
590 | } |
591 | } |
592 | if (Mismatch) { |
593 | Overlap.addOneMismatch(FuncLevelOverlap.Test); |
594 | return; |
595 | } |
596 | |
597 | |
598 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) |
599 | overlapValueProfData(Kind, Other, Overlap, FuncLevelOverlap); |
600 | |
601 | double Score = 0.0; |
602 | uint64_t MaxCount = 0; |
603 | |
604 | for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) { |
605 | Score += OverlapStats::score(Counts[I], Other.Counts[I], |
606 | Overlap.Base.CountSum, Overlap.Test.CountSum); |
607 | MaxCount = std::max(Other.Counts[I], MaxCount); |
608 | } |
609 | Overlap.Overlap.CountSum += Score; |
610 | Overlap.Overlap.NumEntries += 1; |
611 | |
612 | if (MaxCount >= ValueCutoff) { |
613 | double FuncScore = 0.0; |
614 | for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) |
615 | FuncScore += OverlapStats::score(Counts[I], Other.Counts[I], |
616 | FuncLevelOverlap.Base.CountSum, |
617 | FuncLevelOverlap.Test.CountSum); |
618 | FuncLevelOverlap.Overlap.CountSum = FuncScore; |
619 | FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size(); |
620 | FuncLevelOverlap.Valid = true; |
621 | } |
622 | } |
623 | |
624 | void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input, |
625 | uint64_t Weight, |
626 | function_ref<void(instrprof_error)> Warn) { |
627 | this->sortByTargetValues(); |
628 | Input.sortByTargetValues(); |
629 | auto I = ValueData.begin(); |
630 | auto IE = ValueData.end(); |
631 | for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE; |
632 | ++J) { |
633 | while (I != IE && I->Value < J->Value) |
634 | ++I; |
635 | if (I != IE && I->Value == J->Value) { |
636 | bool Overflowed; |
637 | I->Count = SaturatingMultiplyAdd(J->Count, Weight, I->Count, &Overflowed); |
638 | if (Overflowed) |
639 | Warn(instrprof_error::counter_overflow); |
640 | ++I; |
641 | continue; |
642 | } |
643 | ValueData.insert(I, *J); |
644 | } |
645 | } |
646 | |
647 | void InstrProfValueSiteRecord::scale(uint64_t N, uint64_t D, |
648 | function_ref<void(instrprof_error)> Warn) { |
649 | for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) { |
650 | bool Overflowed; |
651 | I->Count = SaturatingMultiply(I->Count, N, &Overflowed) / D; |
652 | if (Overflowed) |
653 | Warn(instrprof_error::counter_overflow); |
654 | } |
655 | } |
656 | |
657 | |
658 | |
659 | void InstrProfRecord::mergeValueProfData( |
660 | uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight, |
661 | function_ref<void(instrprof_error)> Warn) { |
662 | uint32_t ThisNumValueSites = getNumValueSites(ValueKind); |
663 | uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind); |
664 | if (ThisNumValueSites != OtherNumValueSites) { |
665 | Warn(instrprof_error::value_site_count_mismatch); |
666 | return; |
667 | } |
668 | if (!ThisNumValueSites) |
669 | return; |
670 | std::vector<InstrProfValueSiteRecord> &ThisSiteRecords = |
671 | getOrCreateValueSitesForKind(ValueKind); |
672 | MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords = |
673 | Src.getValueSitesForKind(ValueKind); |
674 | for (uint32_t I = 0; I < ThisNumValueSites; I++) |
675 | ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn); |
676 | } |
677 | |
678 | void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight, |
679 | function_ref<void(instrprof_error)> Warn) { |
680 | |
681 | |
682 | if (Counts.size() != Other.Counts.size()) { |
683 | Warn(instrprof_error::count_mismatch); |
684 | return; |
685 | } |
686 | |
687 | for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) { |
688 | bool Overflowed; |
689 | Counts[I] = |
690 | SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed); |
691 | if (Overflowed) |
692 | Warn(instrprof_error::counter_overflow); |
693 | } |
694 | |
695 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) |
696 | mergeValueProfData(Kind, Other, Weight, Warn); |
697 | } |
698 | |
699 | void InstrProfRecord::scaleValueProfData( |
700 | uint32_t ValueKind, uint64_t N, uint64_t D, |
701 | function_ref<void(instrprof_error)> Warn) { |
702 | for (auto &R : getValueSitesForKind(ValueKind)) |
703 | R.scale(N, D, Warn); |
704 | } |
705 | |
706 | void InstrProfRecord::scale(uint64_t N, uint64_t D, |
707 | function_ref<void(instrprof_error)> Warn) { |
708 | assert(D != 0 && "D cannot be 0"); |
709 | for (auto &Count : this->Counts) { |
710 | bool Overflowed; |
711 | Count = SaturatingMultiply(Count, N, &Overflowed) / D; |
712 | if (Overflowed) |
713 | Warn(instrprof_error::counter_overflow); |
714 | } |
715 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) |
716 | scaleValueProfData(Kind, N, D, Warn); |
717 | } |
718 | |
719 | |
720 | uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind, |
721 | InstrProfSymtab *SymTab) { |
722 | if (!SymTab) |
723 | return Value; |
724 | |
725 | if (ValueKind == IPVK_IndirectCallTarget) |
726 | return SymTab->getFunctionHashFromAddress(Value); |
727 | |
728 | return Value; |
729 | } |
730 | |
731 | void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site, |
732 | InstrProfValueData *VData, uint32_t N, |
733 | InstrProfSymtab *ValueMap) { |
734 | for (uint32_t I = 0; I < N; I++) { |
735 | VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap); |
736 | } |
737 | std::vector<InstrProfValueSiteRecord> &ValueSites = |
738 | getOrCreateValueSitesForKind(ValueKind); |
739 | if (N == 0) |
740 | ValueSites.emplace_back(); |
741 | else |
742 | ValueSites.emplace_back(VData, VData + N); |
743 | } |
744 | |
745 | #define INSTR_PROF_COMMON_API_IMPL |
746 | #include "llvm/ProfileData/InstrProfData.inc" |
747 | |
748 | |
749 | |
750 | |
751 | |
752 | |
753 | uint32_t getNumValueKindsInstrProf(const void *Record) { |
754 | return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds(); |
755 | } |
756 | |
757 | uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) { |
758 | return reinterpret_cast<const InstrProfRecord *>(Record) |
759 | ->getNumValueSites(VKind); |
760 | } |
761 | |
762 | uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) { |
763 | return reinterpret_cast<const InstrProfRecord *>(Record) |
764 | ->getNumValueData(VKind); |
765 | } |
766 | |
767 | uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, |
768 | uint32_t S) { |
769 | return reinterpret_cast<const InstrProfRecord *>(R) |
770 | ->getNumValueDataForSite(VK, S); |
771 | } |
772 | |
773 | void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, |
774 | uint32_t K, uint32_t S) { |
775 | reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S); |
776 | } |
777 | |
778 | ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) { |
779 | ValueProfData *VD = |
780 | (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData()); |
781 | memset(VD, 0, TotalSizeInBytes); |
782 | return VD; |
783 | } |
784 | |
785 | static ValueProfRecordClosure InstrProfRecordClosure = { |
786 | nullptr, |
787 | getNumValueKindsInstrProf, |
788 | getNumValueSitesInstrProf, |
789 | getNumValueDataInstrProf, |
790 | getNumValueDataForSiteInstrProf, |
791 | nullptr, |
792 | getValueForSiteInstrProf, |
793 | allocValueProfDataInstrProf}; |
794 | |
795 | |
796 | uint32_t ValueProfData::getSize(const InstrProfRecord &Record) { |
797 | auto Closure = InstrProfRecordClosure; |
798 | Closure.Record = &Record; |
799 | return getValueProfDataSize(&Closure); |
800 | } |
801 | |
802 | |
803 | std::unique_ptr<ValueProfData> |
804 | ValueProfData::serializeFrom(const InstrProfRecord &Record) { |
805 | InstrProfRecordClosure.Record = &Record; |
806 | |
807 | std::unique_ptr<ValueProfData> VPD( |
808 | serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr)); |
809 | return VPD; |
810 | } |
811 | |
812 | void ValueProfRecord::deserializeTo(InstrProfRecord &Record, |
813 | InstrProfSymtab *SymTab) { |
814 | Record.reserveSites(Kind, NumValueSites); |
815 | |
816 | InstrProfValueData *ValueData = getValueProfRecordValueData(this); |
817 | for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) { |
818 | uint8_t ValueDataCount = this->SiteCountArray[VSite]; |
819 | Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab); |
820 | ValueData += ValueDataCount; |
821 | } |
822 | } |
823 | |
824 | |
825 | |
826 | |
827 | void ValueProfRecord::swapBytes(support::endianness Old, |
828 | support::endianness New) { |
829 | using namespace support; |
830 | |
831 | if (Old == New) |
832 | return; |
833 | |
834 | if (getHostEndianness() != Old) { |
835 | sys::swapByteOrder<uint32_t>(NumValueSites); |
836 | sys::swapByteOrder<uint32_t>(Kind); |
837 | } |
838 | uint32_t ND = getValueProfRecordNumValueData(this); |
839 | InstrProfValueData *VD = getValueProfRecordValueData(this); |
840 | |
841 | |
842 | for (uint32_t I = 0; I < ND; I++) { |
843 | sys::swapByteOrder<uint64_t>(VD[I].Value); |
844 | sys::swapByteOrder<uint64_t>(VD[I].Count); |
845 | } |
846 | if (getHostEndianness() == Old) { |
847 | sys::swapByteOrder<uint32_t>(NumValueSites); |
848 | sys::swapByteOrder<uint32_t>(Kind); |
849 | } |
850 | } |
851 | |
852 | void ValueProfData::deserializeTo(InstrProfRecord &Record, |
853 | InstrProfSymtab *SymTab) { |
854 | if (NumValueKinds == 0) |
855 | return; |
856 | |
857 | ValueProfRecord *VR = getFirstValueProfRecord(this); |
858 | for (uint32_t K = 0; K < NumValueKinds; K++) { |
859 | VR->deserializeTo(Record, SymTab); |
860 | VR = getValueProfRecordNext(VR); |
861 | } |
862 | } |
863 | |
864 | template <class T> |
865 | static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) { |
866 | using namespace support; |
867 | |
868 | if (Orig == little) |
869 | return endian::readNext<T, little, unaligned>(D); |
870 | else |
871 | return endian::readNext<T, big, unaligned>(D); |
872 | } |
873 | |
874 | static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) { |
875 | return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize)) |
876 | ValueProfData()); |
877 | } |
878 | |
879 | Error ValueProfData::checkIntegrity() { |
880 | if (NumValueKinds > IPVK_Last + 1) |
881 | return make_error<InstrProfError>(instrprof_error::malformed); |
882 | |
883 | if (TotalSize % sizeof(uint64_t)) |
884 | return make_error<InstrProfError>(instrprof_error::malformed); |
885 | |
886 | ValueProfRecord *VR = getFirstValueProfRecord(this); |
887 | for (uint32_t K = 0; K < this->NumValueKinds; K++) { |
888 | if (VR->Kind > IPVK_Last) |
889 | return make_error<InstrProfError>(instrprof_error::malformed); |
890 | VR = getValueProfRecordNext(VR); |
891 | if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize) |
892 | return make_error<InstrProfError>(instrprof_error::malformed); |
893 | } |
894 | return Error::success(); |
895 | } |
896 | |
897 | Expected<std::unique_ptr<ValueProfData>> |
898 | ValueProfData::getValueProfData(const unsigned char *D, |
899 | const unsigned char *const BufferEnd, |
900 | support::endianness Endianness) { |
901 | using namespace support; |
902 | |
903 | if (D + sizeof(ValueProfData) > BufferEnd) |
904 | return make_error<InstrProfError>(instrprof_error::truncated); |
905 | |
906 | const unsigned char *Header = D; |
907 | uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness); |
908 | if (D + TotalSize > BufferEnd) |
909 | return make_error<InstrProfError>(instrprof_error::too_large); |
910 | |
911 | std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize); |
912 | memcpy(VPD.get(), D, TotalSize); |
913 | |
914 | VPD->swapBytesToHost(Endianness); |
915 | |
916 | Error E = VPD->checkIntegrity(); |
917 | if (E) |
918 | return std::move(E); |
919 | |
920 | return std::move(VPD); |
921 | } |
922 | |
923 | void ValueProfData::swapBytesToHost(support::endianness Endianness) { |
924 | using namespace support; |
925 | |
926 | if (Endianness == getHostEndianness()) |
927 | return; |
928 | |
929 | sys::swapByteOrder<uint32_t>(TotalSize); |
930 | sys::swapByteOrder<uint32_t>(NumValueKinds); |
931 | |
932 | ValueProfRecord *VR = getFirstValueProfRecord(this); |
933 | for (uint32_t K = 0; K < NumValueKinds; K++) { |
934 | VR->swapBytes(Endianness, getHostEndianness()); |
935 | VR = getValueProfRecordNext(VR); |
936 | } |
937 | } |
938 | |
939 | void ValueProfData::swapBytesFromHost(support::endianness Endianness) { |
940 | using namespace support; |
941 | |
942 | if (Endianness == getHostEndianness()) |
943 | return; |
944 | |
945 | ValueProfRecord *VR = getFirstValueProfRecord(this); |
946 | for (uint32_t K = 0; K < NumValueKinds; K++) { |
947 | ValueProfRecord *NVR = getValueProfRecordNext(VR); |
948 | VR->swapBytes(getHostEndianness(), Endianness); |
949 | VR = NVR; |
950 | } |
951 | sys::swapByteOrder<uint32_t>(TotalSize); |
952 | sys::swapByteOrder<uint32_t>(NumValueKinds); |
953 | } |
954 | |
955 | void annotateValueSite(Module &M, Instruction &Inst, |
956 | const InstrProfRecord &InstrProfR, |
957 | InstrProfValueKind ValueKind, uint32_t SiteIdx, |
958 | uint32_t MaxMDCount) { |
959 | uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx); |
960 | if (!NV) |
961 | return; |
962 | |
963 | uint64_t Sum = 0; |
964 | std::unique_ptr<InstrProfValueData[]> VD = |
965 | InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum); |
966 | |
967 | ArrayRef<InstrProfValueData> VDs(VD.get(), NV); |
968 | annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount); |
969 | } |
970 | |
971 | void annotateValueSite(Module &M, Instruction &Inst, |
972 | ArrayRef<InstrProfValueData> VDs, |
973 | uint64_t Sum, InstrProfValueKind ValueKind, |
974 | uint32_t MaxMDCount) { |
975 | LLVMContext &Ctx = M.getContext(); |
976 | MDBuilder MDHelper(Ctx); |
977 | SmallVector<Metadata *, 3> Vals; |
978 | |
979 | Vals.push_back(MDHelper.createString("VP")); |
980 | |
981 | Vals.push_back(MDHelper.createConstant( |
982 | ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind))); |
983 | |
984 | Vals.push_back( |
985 | MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum))); |
986 | |
987 | |
988 | uint32_t MDCount = MaxMDCount; |
989 | for (auto &VD : VDs) { |
990 | Vals.push_back(MDHelper.createConstant( |
991 | ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value))); |
992 | Vals.push_back(MDHelper.createConstant( |
993 | ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count))); |
994 | if (--MDCount == 0) |
995 | break; |
996 | } |
997 | Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals)); |
998 | } |
999 | |
1000 | bool getValueProfDataFromInst(const Instruction &Inst, |
1001 | InstrProfValueKind ValueKind, |
1002 | uint32_t MaxNumValueData, |
1003 | InstrProfValueData ValueData[], |
1004 | uint32_t &ActualNumValueData, uint64_t &TotalC, |
1005 | bool GetNoICPValue) { |
1006 | MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof); |
1007 | if (!MD) |
1008 | return false; |
1009 | |
1010 | unsigned NOps = MD->getNumOperands(); |
1011 | |
1012 | if (NOps < 5) |
1013 | return false; |
1014 | |
1015 | |
1016 | MDString *Tag = cast<MDString>(MD->getOperand(0)); |
1017 | if (!Tag) |
1018 | return false; |
1019 | |
1020 | if (!Tag->getString().equals("VP")) |
1021 | return false; |
1022 | |
1023 | |
1024 | ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1)); |
1025 | if (!KindInt) |
1026 | return false; |
1027 | if (KindInt->getZExtValue() != ValueKind) |
1028 | return false; |
1029 | |
1030 | |
1031 | ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2)); |
1032 | if (!TotalCInt) |
1033 | return false; |
1034 | TotalC = TotalCInt->getZExtValue(); |
1035 | |
1036 | ActualNumValueData = 0; |
1037 | |
1038 | for (unsigned I = 3; I < NOps; I += 2) { |
1039 | if (ActualNumValueData >= MaxNumValueData) |
1040 | break; |
1041 | ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I)); |
1042 | ConstantInt *Count = |
1043 | mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1)); |
1044 | if (!Value || !Count) |
1045 | return false; |
1046 | uint64_t CntValue = Count->getZExtValue(); |
1047 | if (!GetNoICPValue && (CntValue == NOMORE_ICP_MAGICNUM)) |
1048 | continue; |
1049 | ValueData[ActualNumValueData].Value = Value->getZExtValue(); |
1050 | ValueData[ActualNumValueData].Count = CntValue; |
1051 | ActualNumValueData++; |
1052 | } |
1053 | return true; |
1054 | } |
1055 | |
1056 | MDNode *getPGOFuncNameMetadata(const Function &F) { |
1057 | return F.getMetadata(getPGOFuncNameMetadataName()); |
1058 | } |
1059 | |
1060 | void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { |
1061 | |
1062 | if (PGOFuncName == F.getName()) |
1063 | return; |
1064 | |
1065 | if (getPGOFuncNameMetadata(F)) |
1066 | return; |
1067 | LLVMContext &C = F.getContext(); |
1068 | MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName)); |
1069 | F.setMetadata(getPGOFuncNameMetadataName(), N); |
1070 | } |
1071 | |
1072 | bool needsComdatForCounter(const Function &F, const Module &M) { |
1073 | if (F.hasComdat()) |
1074 | return true; |
1075 | |
1076 | if (!Triple(M.getTargetTriple()).supportsCOMDAT()) |
1077 | return false; |
1078 | |
1079 | |
1080 | |
1081 | |
1082 | |
1083 | |
1084 | |
1085 | |
1086 | |
1087 | |
1088 | |
1089 | GlobalValue::LinkageTypes Linkage = F.getLinkage(); |
1090 | if (Linkage != GlobalValue::ExternalWeakLinkage && |
1091 | Linkage != GlobalValue::AvailableExternallyLinkage) |
1092 | return false; |
1093 | |
1094 | return true; |
1095 | } |
1096 | |
1097 | |
1098 | bool isIRPGOFlagSet(const Module *M) { |
1099 | auto IRInstrVar = |
1100 | M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)); |
1101 | if (!IRInstrVar || IRInstrVar->isDeclaration() || |
1102 | IRInstrVar->hasLocalLinkage()) |
1103 | return false; |
1104 | |
1105 | |
1106 | if (!IRInstrVar->hasInitializer()) |
1107 | return false; |
1108 | |
1109 | auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer()); |
1110 | if (!InitVal) |
1111 | return false; |
1112 | return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0; |
1113 | } |
1114 | |
1115 | |
1116 | bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) { |
1117 | if (F.getName().empty()) |
1118 | return false; |
1119 | if (!needsComdatForCounter(F, *(F.getParent()))) |
1120 | return false; |
1121 | |
1122 | |
1123 | if (CheckAddressTaken && F.hasAddressTaken()) |
1124 | return false; |
1125 | |
1126 | |
1127 | if (!GlobalValue::isDiscardableIfUnused(F.getLinkage())) |
1128 | return false; |
1129 | |
1130 | |
1131 | if (!F.hasComdat()) { |
1132 | assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage); |
1133 | return true; |
1134 | } |
1135 | return true; |
1136 | } |
1137 | |
1138 | |
1139 | |
1140 | void createIRLevelProfileFlagVar(Module &M, bool IsCS, |
1141 | bool InstrEntryBBEnabled) { |
1142 | const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)); |
1143 | Type *IntTy64 = Type::getInt64Ty(M.getContext()); |
1144 | uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF); |
1145 | if (IsCS) |
1146 | ProfileVersion |= VARIANT_MASK_CSIR_PROF; |
1147 | if (InstrEntryBBEnabled) |
1148 | ProfileVersion |= VARIANT_MASK_INSTR_ENTRY; |
1149 | auto IRLevelVersionVariable = new GlobalVariable( |
1150 | M, IntTy64, true, GlobalValue::WeakAnyLinkage, |
1151 | Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName); |
1152 | IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility); |
1153 | Triple TT(M.getTargetTriple()); |
1154 | if (TT.supportsCOMDAT()) { |
1155 | IRLevelVersionVariable->setLinkage(GlobalValue::ExternalLinkage); |
1156 | IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName)); |
1157 | } |
1158 | } |
1159 | |
1160 | |
1161 | void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) { |
1162 | if (InstrProfileOutput.empty()) |
1163 | return; |
1164 | Constant *ProfileNameConst = |
1165 | ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true); |
1166 | GlobalVariable *ProfileNameVar = new GlobalVariable( |
1167 | M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage, |
1168 | ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)); |
1169 | Triple TT(M.getTargetTriple()); |
1170 | if (TT.supportsCOMDAT()) { |
1171 | ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage); |
1172 | ProfileNameVar->setComdat(M.getOrInsertComdat( |
1173 | StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)))); |
1174 | } |
1175 | } |
1176 | |
1177 | Error OverlapStats::accumulateCounts(const std::string &BaseFilename, |
1178 | const std::string &TestFilename, |
1179 | bool IsCS) { |
1180 | auto getProfileSum = [IsCS](const std::string &Filename, |
1181 | CountSumOrPercent &Sum) -> Error { |
1182 | auto ReaderOrErr = InstrProfReader::create(Filename); |
1183 | if (Error E = ReaderOrErr.takeError()) { |
1184 | return E; |
1185 | } |
1186 | auto Reader = std::move(ReaderOrErr.get()); |
1187 | Reader->accumulateCounts(Sum, IsCS); |
1188 | return Error::success(); |
1189 | }; |
1190 | auto Ret = getProfileSum(BaseFilename, Base); |
1191 | if (Ret) |
1192 | return Ret; |
1193 | Ret = getProfileSum(TestFilename, Test); |
1194 | if (Ret) |
1195 | return Ret; |
1196 | this->BaseFilename = &BaseFilename; |
1197 | this->TestFilename = &TestFilename; |
1198 | Valid = true; |
1199 | return Error::success(); |
1200 | } |
1201 | |
1202 | void OverlapStats::addOneMismatch(const CountSumOrPercent &MismatchFunc) { |
1203 | Mismatch.NumEntries += 1; |
1204 | Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum; |
1205 | for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { |
1206 | if (Test.ValueCounts[I] >= 1.0f) |
1207 | Mismatch.ValueCounts[I] += |
1208 | MismatchFunc.ValueCounts[I] / Test.ValueCounts[I]; |
1209 | } |
1210 | } |
1211 | |
1212 | void OverlapStats::addOneUnique(const CountSumOrPercent &UniqueFunc) { |
1213 | Unique.NumEntries += 1; |
1214 | Unique.CountSum += UniqueFunc.CountSum / Test.CountSum; |
1215 | for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { |
1216 | if (Test.ValueCounts[I] >= 1.0f) |
1217 | Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I]; |
1218 | } |
1219 | } |
1220 | |
1221 | void OverlapStats::dump(raw_fd_ostream &OS) const { |
1222 | if (!Valid) |
1223 | return; |
1224 | |
1225 | const char *EntryName = |
1226 | (Level == ProgramLevel ? "functions" : "edge counters"); |
1227 | if (Level == ProgramLevel) { |
1228 | OS << "Profile overlap infomation for base_profile: " << *BaseFilename |
1229 | << " and test_profile: " << *TestFilename << "\nProgram level:\n"; |
1230 | } else { |
1231 | OS << "Function level:\n" |
1232 | << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n"; |
1233 | } |
1234 | |
1235 | OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n"; |
1236 | if (Mismatch.NumEntries) |
1237 | OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries |
1238 | << "\n"; |
1239 | if (Unique.NumEntries) |
1240 | OS << " # of " << EntryName |
1241 | << " only in test_profile: " << Unique.NumEntries << "\n"; |
1242 | |
1243 | OS << " Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100) |
1244 | << "\n"; |
1245 | if (Mismatch.NumEntries) |
1246 | OS << " Mismatched count percentage (Edge): " |
1247 | << format("%.3f%%", Mismatch.CountSum * 100) << "\n"; |
1248 | if (Unique.NumEntries) |
1249 | OS << " Percentage of Edge profile only in test_profile: " |
1250 | << format("%.3f%%", Unique.CountSum * 100) << "\n"; |
1251 | OS << " Edge profile base count sum: " << format("%.0f", Base.CountSum) |
1252 | << "\n" |
1253 | << " Edge profile test count sum: " << format("%.0f", Test.CountSum) |
1254 | << "\n"; |
1255 | |
1256 | for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { |
1257 | if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f) |
1258 | continue; |
1259 | char ProfileKindName[20]; |
1260 | switch (I) { |
1261 | case IPVK_IndirectCallTarget: |
1262 | strncpy(ProfileKindName, "IndirectCall", 19); |
1263 | break; |
1264 | case IPVK_MemOPSize: |
1265 | strncpy(ProfileKindName, "MemOP", 19); |
1266 | break; |
1267 | default: |
1268 | snprintf(ProfileKindName, 19, "VP[%d]", I); |
1269 | break; |
1270 | } |
1271 | OS << " " << ProfileKindName |
1272 | << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100) |
1273 | << "\n"; |
1274 | if (Mismatch.NumEntries) |
1275 | OS << " Mismatched count percentage (" << ProfileKindName |
1276 | << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n"; |
1277 | if (Unique.NumEntries) |
1278 | OS << " Percentage of " << ProfileKindName |
1279 | << " profile only in test_profile: " |
1280 | << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n"; |
1281 | OS << " " << ProfileKindName |
1282 | << " profile base count sum: " << format("%.0f", Base.ValueCounts[I]) |
1283 | << "\n" |
1284 | << " " << ProfileKindName |
1285 | << " profile test count sum: " << format("%.0f", Test.ValueCounts[I]) |
1286 | << "\n"; |
1287 | } |
1288 | } |
1289 | |
1290 | } |