clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name BlockFrequencyInfoImpl.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Analysis -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ASMParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/BinaryFormat -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitstream -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /include/llvm/CodeGen -I /include/llvm/CodeGen/PBQP -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Coroutines -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData/Coverage -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/CodeView -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/DWARF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/MSF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/PDB -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Demangle -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/JITLink -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/Orc -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenACC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenMP -I /include/llvm/CodeGen/GlobalISel -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IRReader -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/LTO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Linker -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC/MCParser -I /include/llvm/CodeGen/MIRParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Object -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Option -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Passes -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Scalar -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ADT -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Support -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/Symbolize -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Target -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Utils -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Vectorize -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/IPO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include -I /usr/src/gnu/usr.bin/clang/libLLVM/../include -I /usr/src/gnu/usr.bin/clang/libLLVM/obj -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include -D NDEBUG -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D LLVM_PREFIX="/usr" -internal-isystem /usr/include/c++/v1 -internal-isystem /usr/local/lib/clang/13.0.0/include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -ferror-limit 19 -fvisibility-inlines-hidden -fwrapv -stack-protector 2 -fno-rtti -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/ben/Projects/vmm/scan-build/2022-01-12-194120-40624-1 -x c++ /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp
1 | |
2 | |
3 | |
4 | |
5 | |
6 | |
7 | |
8 | |
9 | |
10 | |
11 | |
12 | |
13 | #include "llvm/Analysis/BlockFrequencyInfoImpl.h" |
14 | #include "llvm/ADT/APInt.h" |
15 | #include "llvm/ADT/DenseMap.h" |
16 | #include "llvm/ADT/GraphTraits.h" |
17 | #include "llvm/ADT/None.h" |
18 | #include "llvm/ADT/SCCIterator.h" |
19 | #include "llvm/Config/llvm-config.h" |
20 | #include "llvm/IR/Function.h" |
21 | #include "llvm/Support/BlockFrequency.h" |
22 | #include "llvm/Support/BranchProbability.h" |
23 | #include "llvm/Support/Compiler.h" |
24 | #include "llvm/Support/Debug.h" |
25 | #include "llvm/Support/ScaledNumber.h" |
26 | #include "llvm/Support/MathExtras.h" |
27 | #include "llvm/Support/raw_ostream.h" |
28 | #include <algorithm> |
29 | #include <cassert> |
30 | #include <cstddef> |
31 | #include <cstdint> |
32 | #include <iterator> |
33 | #include <list> |
34 | #include <numeric> |
35 | #include <utility> |
36 | #include <vector> |
37 | |
38 | using namespace llvm; |
39 | using namespace llvm::bfi_detail; |
40 | |
41 | #define DEBUG_TYPE "block-freq" |
42 | |
43 | namespace llvm { |
44 | cl::opt<bool> CheckBFIUnknownBlockQueries( |
45 | "check-bfi-unknown-block-queries", |
46 | cl::init(false), cl::Hidden, |
47 | cl::desc("Check if block frequency is queried for an unknown block " |
48 | "for debugging missed BFI updates")); |
49 | |
50 | cl::opt<bool> UseIterativeBFIInference( |
51 | "use-iterative-bfi-inference", cl::init(false), cl::Hidden, cl::ZeroOrMore, |
52 | cl::desc("Apply an iterative post-processing to infer correct BFI counts")); |
53 | |
54 | cl::opt<unsigned> IterativeBFIMaxIterationsPerBlock( |
55 | "iterative-bfi-max-iterations-per-block", cl::init(1000), cl::Hidden, |
56 | cl::desc("Iterative inference: maximum number of update iterations " |
57 | "per block")); |
58 | |
59 | cl::opt<double> IterativeBFIPrecision( |
60 | "iterative-bfi-precision", cl::init(1e-12), cl::Hidden, |
61 | cl::desc("Iterative inference: delta convergence precision; smaller values " |
62 | "typically lead to better results at the cost of worsen runtime")); |
63 | } |
64 | |
65 | ScaledNumber<uint64_t> BlockMass::toScaled() const { |
66 | if (isFull()) |
67 | return ScaledNumber<uint64_t>(1, 0); |
68 | return ScaledNumber<uint64_t>(getMass() + 1, -64); |
69 | } |
70 | |
71 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) |
72 | LLVM_DUMP_METHOD void BlockMass::dump() const { print(dbgs()); } |
73 | #endif |
74 | |
75 | static char getHexDigit(int N) { |
76 | assert(N < 16); |
77 | if (N < 10) |
78 | return '0' + N; |
79 | return 'a' + N - 10; |
80 | } |
81 | |
82 | raw_ostream &BlockMass::print(raw_ostream &OS) const { |
83 | for (int Digits = 0; Digits < 16; ++Digits) |
84 | OS << getHexDigit(Mass >> (60 - Digits * 4) & 0xf); |
85 | return OS; |
86 | } |
87 | |
88 | namespace { |
89 | |
90 | using BlockNode = BlockFrequencyInfoImplBase::BlockNode; |
91 | using Distribution = BlockFrequencyInfoImplBase::Distribution; |
92 | using WeightList = BlockFrequencyInfoImplBase::Distribution::WeightList; |
93 | using Scaled64 = BlockFrequencyInfoImplBase::Scaled64; |
94 | using LoopData = BlockFrequencyInfoImplBase::LoopData; |
95 | using Weight = BlockFrequencyInfoImplBase::Weight; |
96 | using FrequencyData = BlockFrequencyInfoImplBase::FrequencyData; |
97 | |
98 | |
99 | |
100 | |
101 | |
102 | |
103 | |
104 | |
105 | |
106 | |
107 | |
108 | |
109 | |
110 | |
111 | |
112 | |
113 | |
114 | |
115 | |
116 | struct DitheringDistributer { |
117 | uint32_t RemWeight; |
118 | BlockMass RemMass; |
119 | |
120 | DitheringDistributer(Distribution &Dist, const BlockMass &Mass); |
121 | |
122 | BlockMass takeMass(uint32_t Weight); |
123 | }; |
124 | |
125 | } |
126 | |
127 | DitheringDistributer::DitheringDistributer(Distribution &Dist, |
128 | const BlockMass &Mass) { |
129 | Dist.normalize(); |
130 | RemWeight = Dist.Total; |
131 | RemMass = Mass; |
132 | } |
133 | |
134 | BlockMass DitheringDistributer::takeMass(uint32_t Weight) { |
135 | assert(Weight && "invalid weight"); |
136 | assert(Weight <= RemWeight); |
137 | BlockMass Mass = RemMass * BranchProbability(Weight, RemWeight); |
138 | |
139 | |
140 | RemWeight -= Weight; |
141 | RemMass -= Mass; |
142 | return Mass; |
143 | } |
144 | |
145 | void Distribution::add(const BlockNode &Node, uint64_t Amount, |
146 | Weight::DistType Type) { |
147 | assert(Amount && "invalid weight of 0"); |
148 | uint64_t NewTotal = Total + Amount; |
149 | |
150 | |
151 | bool IsOverflow = NewTotal < Total; |
152 | assert(!(DidOverflow && IsOverflow) && "unexpected repeated overflow"); |
153 | DidOverflow |= IsOverflow; |
154 | |
155 | |
156 | Total = NewTotal; |
157 | |
158 | |
159 | Weights.push_back(Weight(Type, Node, Amount)); |
160 | } |
161 | |
162 | static void combineWeight(Weight &W, const Weight &OtherW) { |
163 | assert(OtherW.TargetNode.isValid()); |
164 | if (!W.Amount) { |
165 | W = OtherW; |
166 | return; |
167 | } |
168 | assert(W.Type == OtherW.Type); |
169 | assert(W.TargetNode == OtherW.TargetNode); |
170 | assert(OtherW.Amount && "Expected non-zero weight"); |
171 | if (W.Amount > W.Amount + OtherW.Amount) |
172 | |
173 | W.Amount = UINT64_MAX; |
174 | else |
175 | W.Amount += OtherW.Amount; |
176 | } |
177 | |
178 | static void combineWeightsBySorting(WeightList &Weights) { |
179 | |
180 | llvm::sort(Weights, [](const Weight &L, const Weight &R) { |
181 | return L.TargetNode < R.TargetNode; |
182 | }); |
183 | |
184 | |
185 | WeightList::iterator O = Weights.begin(); |
186 | for (WeightList::const_iterator I = O, L = O, E = Weights.end(); I != E; |
187 | ++O, (I = L)) { |
188 | *O = *I; |
189 | |
190 | |
191 | for (++L; L != E && I->TargetNode == L->TargetNode; ++L) |
192 | combineWeight(*O, *L); |
193 | } |
194 | |
195 | |
196 | Weights.erase(O, Weights.end()); |
197 | } |
198 | |
199 | static void combineWeightsByHashing(WeightList &Weights) { |
200 | |
201 | using HashTable = DenseMap<BlockNode::IndexType, Weight>; |
202 | |
203 | HashTable Combined(NextPowerOf2(2 * Weights.size())); |
204 | for (const Weight &W : Weights) |
205 | combineWeight(Combined[W.TargetNode.Index], W); |
206 | |
207 | |
208 | if (Weights.size() == Combined.size()) |
209 | return; |
210 | |
211 | |
212 | Weights.clear(); |
213 | Weights.reserve(Combined.size()); |
214 | for (const auto &I : Combined) |
215 | Weights.push_back(I.second); |
216 | } |
217 | |
218 | static void combineWeights(WeightList &Weights) { |
219 | |
220 | if (Weights.size() > 128) { |
221 | combineWeightsByHashing(Weights); |
222 | return; |
223 | } |
224 | |
225 | combineWeightsBySorting(Weights); |
226 | } |
227 | |
228 | static uint64_t shiftRightAndRound(uint64_t N, int Shift) { |
229 | assert(Shift >= 0); |
230 | assert(Shift < 64); |
231 | if (!Shift) |
232 | return N; |
233 | return (N >> Shift) + (UINT64_C(1) & N >> (Shift - 1)); |
234 | } |
235 | |
236 | void Distribution::normalize() { |
237 | |
238 | if (Weights.empty()) |
239 | return; |
240 | |
241 | |
242 | if (Weights.size() > 1) |
243 | combineWeights(Weights); |
244 | |
245 | |
246 | if (Weights.size() == 1) { |
247 | Total = 1; |
248 | Weights.front().Amount = 1; |
249 | return; |
250 | } |
251 | |
252 | |
253 | |
254 | |
255 | |
256 | int Shift = 0; |
257 | if (DidOverflow) |
258 | Shift = 33; |
259 | else if (Total > UINT32_MAX) |
260 | Shift = 33 - countLeadingZeros(Total); |
261 | |
262 | |
263 | if (!Shift) { |
264 | |
265 | |
266 | assert(Total == std::accumulate(Weights.begin(), Weights.end(), UINT64_C(0), |
267 | [](uint64_t Sum, const Weight &W) { |
268 | return Sum + W.Amount; |
269 | }) && |
270 | "Expected total to be correct"); |
271 | return; |
272 | } |
273 | |
274 | |
275 | |
276 | Total = 0; |
277 | |
278 | |
279 | for (Weight &W : Weights) { |
280 | |
281 | |
282 | assert(W.TargetNode.isValid()); |
283 | W.Amount = std::max(UINT64_C(1), shiftRightAndRound(W.Amount, Shift)); |
284 | assert(W.Amount <= UINT32_MAX); |
285 | |
286 | |
287 | Total += W.Amount; |
288 | } |
289 | assert(Total <= UINT32_MAX); |
290 | } |
291 | |
292 | void BlockFrequencyInfoImplBase::clear() { |
293 | |
294 | |
295 | std::vector<FrequencyData>().swap(Freqs); |
296 | IsIrrLoopHeader.clear(); |
297 | std::vector<WorkingData>().swap(Working); |
298 | Loops.clear(); |
299 | } |
300 | |
301 | |
302 | |
303 | |
304 | static void cleanup(BlockFrequencyInfoImplBase &BFI) { |
305 | std::vector<FrequencyData> SavedFreqs(std::move(BFI.Freqs)); |
306 | SparseBitVector<> SavedIsIrrLoopHeader(std::move(BFI.IsIrrLoopHeader)); |
307 | BFI.clear(); |
308 | BFI.Freqs = std::move(SavedFreqs); |
309 | BFI.IsIrrLoopHeader = std::move(SavedIsIrrLoopHeader); |
310 | } |
311 | |
312 | bool BlockFrequencyInfoImplBase::addToDist(Distribution &Dist, |
313 | const LoopData *OuterLoop, |
314 | const BlockNode &Pred, |
315 | const BlockNode &Succ, |
316 | uint64_t Weight) { |
317 | if (!Weight) |
318 | Weight = 1; |
319 | |
320 | auto isLoopHeader = [&OuterLoop](const BlockNode &Node) { |
321 | return OuterLoop && OuterLoop->isHeader(Node); |
322 | }; |
323 | |
324 | BlockNode Resolved = Working[Succ.Index].getResolvedNode(); |
325 | |
326 | #ifndef NDEBUG |
327 | auto debugSuccessor = [&](const char *Type) { |
328 | dbgs() << " =>" |
329 | << " [" << Type << "] weight = " << Weight; |
330 | if (!isLoopHeader(Resolved)) |
331 | dbgs() << ", succ = " << getBlockName(Succ); |
332 | if (Resolved != Succ) |
333 | dbgs() << ", resolved = " << getBlockName(Resolved); |
334 | dbgs() << "\n"; |
335 | }; |
336 | (void)debugSuccessor; |
337 | #endif |
338 | |
339 | if (isLoopHeader(Resolved)) { |
340 | LLVM_DEBUG(debugSuccessor("backedge")); |
341 | Dist.addBackedge(Resolved, Weight); |
342 | return true; |
343 | } |
344 | |
345 | if (Working[Resolved.Index].getContainingLoop() != OuterLoop) { |
346 | LLVM_DEBUG(debugSuccessor(" exit ")); |
347 | Dist.addExit(Resolved, Weight); |
348 | return true; |
349 | } |
350 | |
351 | if (Resolved < Pred) { |
352 | if (!isLoopHeader(Pred)) { |
353 | |
354 | assert((!OuterLoop || !OuterLoop->isIrreducible()) && |
355 | "unhandled irreducible control flow"); |
356 | |
357 | |
358 | LLVM_DEBUG(debugSuccessor("abort!!!")); |
359 | return false; |
360 | } |
361 | |
362 | |
363 | |
364 | |
365 | assert(OuterLoop && OuterLoop->isIrreducible() && !isLoopHeader(Resolved) && |
366 | "unhandled irreducible control flow"); |
367 | } |
368 | |
369 | LLVM_DEBUG(debugSuccessor(" local ")); |
370 | Dist.addLocal(Resolved, Weight); |
371 | return true; |
372 | } |
373 | |
374 | bool BlockFrequencyInfoImplBase::addLoopSuccessorsToDist( |
375 | const LoopData *OuterLoop, LoopData &Loop, Distribution &Dist) { |
376 | |
377 | for (const auto &I : Loop.Exits) |
378 | if (!addToDist(Dist, OuterLoop, Loop.getHeader(), I.first, |
379 | I.second.getMass())) |
380 | |
381 | return false; |
382 | |
383 | return true; |
384 | } |
385 | |
386 | |
387 | void BlockFrequencyInfoImplBase::computeLoopScale(LoopData &Loop) { |
388 | |
389 | LLVM_DEBUG(dbgs() << "compute-loop-scale: " << getLoopName(Loop) << "\n"); |
390 | |
391 | |
392 | |
393 | |
394 | |
395 | |
396 | |
397 | |
398 | |
399 | |
400 | const Scaled64 InfiniteLoopScale(1, 12); |
401 | |
402 | |
403 | |
404 | BlockMass TotalBackedgeMass; |
405 | for (auto &Mass : Loop.BackedgeMass) |
406 | TotalBackedgeMass += Mass; |
407 | BlockMass ExitMass = BlockMass::getFull() - TotalBackedgeMass; |
408 | |
409 | |
410 | |
411 | |
412 | Loop.Scale = |
413 | ExitMass.isEmpty() ? InfiniteLoopScale : ExitMass.toScaled().inverse(); |
414 | |
415 | LLVM_DEBUG(dbgs() << " - exit-mass = " << ExitMass << " (" |
416 | << BlockMass::getFull() << " - " << TotalBackedgeMass |
417 | << ")\n" |
418 | << " - scale = " << Loop.Scale << "\n"); |
419 | } |
420 | |
421 | |
422 | void BlockFrequencyInfoImplBase::packageLoop(LoopData &Loop) { |
423 | LLVM_DEBUG(dbgs() << "packaging-loop: " << getLoopName(Loop) << "\n"); |
424 | |
425 | |
426 | for (const BlockNode &M : Loop.Nodes) { |
427 | if (auto *Loop = Working[M.Index].getPackagedLoop()) |
428 | Loop->Exits.clear(); |
429 | LLVM_DEBUG(dbgs() << " - node: " << getBlockName(M.Index) << "\n"); |
430 | } |
431 | Loop.IsPackaged = true; |
432 | } |
433 | |
434 | #ifndef NDEBUG |
435 | static void debugAssign(const BlockFrequencyInfoImplBase &BFI, |
436 | const DitheringDistributer &D, const BlockNode &T, |
437 | const BlockMass &M, const char *Desc) { |
438 | dbgs() << " => assign " << M << " (" << D.RemMass << ")"; |
439 | if (Desc) |
440 | dbgs() << " [" << Desc << "]"; |
441 | if (T.isValid()) |
442 | dbgs() << " to " << BFI.getBlockName(T); |
443 | dbgs() << "\n"; |
444 | } |
445 | #endif |
446 | |
447 | void BlockFrequencyInfoImplBase::distributeMass(const BlockNode &Source, |
448 | LoopData *OuterLoop, |
449 | Distribution &Dist) { |
450 | BlockMass Mass = Working[Source.Index].getMass(); |
451 | LLVM_DEBUG(dbgs() << " => mass: " << Mass << "\n"); |
452 | |
453 | |
454 | DitheringDistributer D(Dist, Mass); |
455 | |
456 | for (const Weight &W : Dist.Weights) { |
457 | |
458 | BlockMass Taken = D.takeMass(W.Amount); |
459 | if (W.Type == Weight::Local) { |
460 | Working[W.TargetNode.Index].getMass() += Taken; |
461 | LLVM_DEBUG(debugAssign(*this, D, W.TargetNode, Taken, nullptr)); |
462 | continue; |
463 | } |
464 | |
465 | |
466 | assert(OuterLoop && "backedge or exit outside of loop"); |
467 | |
468 | |
469 | if (W.Type == Weight::Backedge) { |
470 | OuterLoop->BackedgeMass[OuterLoop->getHeaderIndex(W.TargetNode)] += Taken; |
471 | LLVM_DEBUG(debugAssign(*this, D, W.TargetNode, Taken, "back")); |
472 | continue; |
473 | } |
474 | |
475 | |
476 | assert(W.Type == Weight::Exit); |
477 | OuterLoop->Exits.push_back(std::make_pair(W.TargetNode, Taken)); |
478 | LLVM_DEBUG(debugAssign(*this, D, W.TargetNode, Taken, "exit")); |
479 | } |
480 | } |
481 | |
482 | static void convertFloatingToInteger(BlockFrequencyInfoImplBase &BFI, |
483 | const Scaled64 &Min, const Scaled64 &Max) { |
484 | |
485 | |
486 | |
487 | |
488 | |
489 | |
490 | const unsigned MaxBits = 64; |
491 | const unsigned SpreadBits = (Max / Min).lg(); |
492 | Scaled64 ScalingFactor; |
493 | if (SpreadBits <= MaxBits - 3) { |
| |
494 | |
495 | |
496 | ScalingFactor = Min.inverse(); |
497 | ScalingFactor <<= 3; |
498 | } else { |
499 | |
500 | |
501 | |
502 | ScalingFactor = Scaled64(1, MaxBits) / Max; |
503 | } |
504 | |
505 | |
506 | LLVM_DEBUG(dbgs() << "float-to-int: min = " << Min << ", max = " << Max |
| 5 | | Loop condition is false. Exiting loop | |
|
507 | << ", factor = " << ScalingFactor << "\n"); |
508 | for (size_t Index = 0; Index < BFI.Freqs.size(); ++Index) { |
| 6 | | Assuming the condition is true | |
|
| 7 | | Loop condition is true. Entering loop body | |
|
509 | Scaled64 Scaled = BFI.Freqs[Index].Scaled * ScalingFactor; |
| 8 | | Calling 'operator*<unsigned long long>' | |
|
| 10 | | Returning from 'operator*<unsigned long long>' | |
|
510 | BFI.Freqs[Index].Integer = std::max(UINT64_C(1), Scaled.toInt<uint64_t>()); |
| 11 | | Calling 'ScaledNumber::toInt' | |
|
511 | LLVM_DEBUG(dbgs() << " - " << BFI.getBlockName(Index) << ": float = " |
512 | << BFI.Freqs[Index].Scaled << ", scaled = " << Scaled |
513 | << ", int = " << BFI.Freqs[Index].Integer << "\n"); |
514 | } |
515 | } |
516 | |
517 | |
518 | |
519 | |
520 | |
521 | static void unwrapLoop(BlockFrequencyInfoImplBase &BFI, LoopData &Loop) { |
522 | LLVM_DEBUG(dbgs() << "unwrap-loop-package: " << BFI.getLoopName(Loop) |
523 | << ": mass = " << Loop.Mass << ", scale = " << Loop.Scale |
524 | << "\n"); |
525 | Loop.Scale *= Loop.Mass.toScaled(); |
526 | Loop.IsPackaged = false; |
527 | LLVM_DEBUG(dbgs() << " => combined-scale = " << Loop.Scale << "\n"); |
528 | |
529 | |
530 | |
531 | |
532 | for (const BlockNode &N : Loop.Nodes) { |
533 | const auto &Working = BFI.Working[N.Index]; |
534 | Scaled64 &F = Working.isAPackage() ? Working.getPackagedLoop()->Scale |
535 | : BFI.Freqs[N.Index].Scaled; |
536 | Scaled64 New = Loop.Scale * F; |
537 | LLVM_DEBUG(dbgs() << " - " << BFI.getBlockName(N) << ": " << F << " => " |
538 | << New << "\n"); |
539 | F = New; |
540 | } |
541 | } |
542 | |
543 | void BlockFrequencyInfoImplBase::unwrapLoops() { |
544 | |
545 | for (size_t Index = 0; Index < Working.size(); ++Index) |
546 | Freqs[Index].Scaled = Working[Index].Mass.toScaled(); |
547 | |
548 | for (LoopData &Loop : Loops) |
549 | unwrapLoop(*this, Loop); |
550 | } |
551 | |
552 | void BlockFrequencyInfoImplBase::finalizeMetrics() { |
553 | |
554 | |
555 | auto Min = Scaled64::getLargest(); |
556 | auto Max = Scaled64::getZero(); |
557 | for (size_t Index = 0; Index < Working.size(); ++Index) { |
| 1 | Assuming the condition is false | |
|
| 2 | | Loop condition is false. Execution continues on line 564 | |
|
558 | |
559 | Min = std::min(Min, Freqs[Index].Scaled); |
560 | Max = std::max(Max, Freqs[Index].Scaled); |
561 | } |
562 | |
563 | |
564 | convertFloatingToInteger(*this, Min, Max); |
| 3 | | Calling 'convertFloatingToInteger' | |
|
565 | |
566 | |
567 | cleanup(*this); |
568 | |
569 | |
570 | LLVM_DEBUG(dump()); |
571 | } |
572 | |
573 | BlockFrequency |
574 | BlockFrequencyInfoImplBase::getBlockFreq(const BlockNode &Node) const { |
575 | if (!Node.isValid()) { |
576 | #ifndef NDEBUG |
577 | if (CheckBFIUnknownBlockQueries) { |
578 | SmallString<256> Msg; |
579 | raw_svector_ostream OS(Msg); |
580 | OS << "*** Detected BFI query for unknown block " << getBlockName(Node); |
581 | report_fatal_error(OS.str()); |
582 | } |
583 | #endif |
584 | return 0; |
585 | } |
586 | return Freqs[Node.Index].Integer; |
587 | } |
588 | |
589 | Optional<uint64_t> |
590 | BlockFrequencyInfoImplBase::getBlockProfileCount(const Function &F, |
591 | const BlockNode &Node, |
592 | bool AllowSynthetic) const { |
593 | return getProfileCountFromFreq(F, getBlockFreq(Node).getFrequency(), |
594 | AllowSynthetic); |
595 | } |
596 | |
597 | Optional<uint64_t> |
598 | BlockFrequencyInfoImplBase::getProfileCountFromFreq(const Function &F, |
599 | uint64_t Freq, |
600 | bool AllowSynthetic) const { |
601 | auto EntryCount = F.getEntryCount(AllowSynthetic); |
602 | if (!EntryCount) |
603 | return None; |
604 | |
605 | APInt BlockCount(128, EntryCount.getCount()); |
606 | APInt BlockFreq(128, Freq); |
607 | APInt EntryFreq(128, getEntryFreq()); |
608 | BlockCount *= BlockFreq; |
609 | |
610 | |
611 | BlockCount = (BlockCount + EntryFreq.lshr(1)).udiv(EntryFreq); |
612 | return BlockCount.getLimitedValue(); |
613 | } |
614 | |
615 | bool |
616 | BlockFrequencyInfoImplBase::isIrrLoopHeader(const BlockNode &Node) { |
617 | if (!Node.isValid()) |
618 | return false; |
619 | return IsIrrLoopHeader.test(Node.Index); |
620 | } |
621 | |
622 | Scaled64 |
623 | BlockFrequencyInfoImplBase::getFloatingBlockFreq(const BlockNode &Node) const { |
624 | if (!Node.isValid()) |
625 | return Scaled64::getZero(); |
626 | return Freqs[Node.Index].Scaled; |
627 | } |
628 | |
629 | void BlockFrequencyInfoImplBase::setBlockFreq(const BlockNode &Node, |
630 | uint64_t Freq) { |
631 | assert(Node.isValid() && "Expected valid node"); |
632 | assert(Node.Index < Freqs.size() && "Expected legal index"); |
633 | Freqs[Node.Index].Integer = Freq; |
634 | } |
635 | |
636 | std::string |
637 | BlockFrequencyInfoImplBase::getBlockName(const BlockNode &Node) const { |
638 | return {}; |
639 | } |
640 | |
641 | std::string |
642 | BlockFrequencyInfoImplBase::getLoopName(const LoopData &Loop) const { |
643 | return getBlockName(Loop.getHeader()) + (Loop.isIrreducible() ? "**" : "*"); |
644 | } |
645 | |
646 | raw_ostream & |
647 | BlockFrequencyInfoImplBase::printBlockFreq(raw_ostream &OS, |
648 | const BlockNode &Node) const { |
649 | return OS << getFloatingBlockFreq(Node); |
650 | } |
651 | |
652 | raw_ostream & |
653 | BlockFrequencyInfoImplBase::printBlockFreq(raw_ostream &OS, |
654 | const BlockFrequency &Freq) const { |
655 | Scaled64 Block(Freq.getFrequency(), 0); |
656 | Scaled64 Entry(getEntryFreq(), 0); |
657 | |
658 | return OS << Block / Entry; |
659 | } |
660 | |
661 | void IrreducibleGraph::addNodesInLoop(const BFIBase::LoopData &OuterLoop) { |
662 | Start = OuterLoop.getHeader(); |
663 | Nodes.reserve(OuterLoop.Nodes.size()); |
664 | for (auto N : OuterLoop.Nodes) |
665 | addNode(N); |
666 | indexNodes(); |
667 | } |
668 | |
669 | void IrreducibleGraph::addNodesInFunction() { |
670 | Start = 0; |
671 | for (uint32_t Index = 0; Index < BFI.Working.size(); ++Index) |
672 | if (!BFI.Working[Index].isPackaged()) |
673 | addNode(Index); |
674 | indexNodes(); |
675 | } |
676 | |
677 | void IrreducibleGraph::indexNodes() { |
678 | for (auto &I : Nodes) |
679 | Lookup[I.Node.Index] = &I; |
680 | } |
681 | |
682 | void IrreducibleGraph::addEdge(IrrNode &Irr, const BlockNode &Succ, |
683 | const BFIBase::LoopData *OuterLoop) { |
684 | if (OuterLoop && OuterLoop->isHeader(Succ)) |
685 | return; |
686 | auto L = Lookup.find(Succ.Index); |
687 | if (L == Lookup.end()) |
688 | return; |
689 | IrrNode &SuccIrr = *L->second; |
690 | Irr.Edges.push_back(&SuccIrr); |
691 | SuccIrr.Edges.push_front(&Irr); |
692 | ++SuccIrr.NumIn; |
693 | } |
694 | |
695 | namespace llvm { |
696 | |
697 | template <> struct GraphTraits<IrreducibleGraph> { |
698 | using GraphT = bfi_detail::IrreducibleGraph; |
699 | using NodeRef = const GraphT::IrrNode *; |
700 | using ChildIteratorType = GraphT::IrrNode::iterator; |
701 | |
702 | static NodeRef getEntryNode(const GraphT &G) { return G.StartIrr; } |
703 | static ChildIteratorType child_begin(NodeRef N) { return N->succ_begin(); } |
704 | static ChildIteratorType child_end(NodeRef N) { return N->succ_end(); } |
705 | }; |
706 | |
707 | } |
708 | |
709 | |
710 | |
711 | |
712 | |
713 | static void findIrreducibleHeaders( |
714 | const BlockFrequencyInfoImplBase &BFI, |
715 | const IrreducibleGraph &G, |
716 | const std::vector<const IrreducibleGraph::IrrNode *> &SCC, |
717 | LoopData::NodeList &Headers, LoopData::NodeList &Others) { |
718 | |
719 | SmallDenseMap<const IrreducibleGraph::IrrNode *, bool, 8> InSCC; |
720 | |
721 | |
722 | for (const auto *I : SCC) |
723 | InSCC[I] = false; |
724 | |
725 | for (auto I = InSCC.begin(), E = InSCC.end(); I != E; ++I) { |
726 | auto &Irr = *I->first; |
727 | for (const auto *P : make_range(Irr.pred_begin(), Irr.pred_end())) { |
728 | if (InSCC.count(P)) |
729 | continue; |
730 | |
731 | |
732 | I->second = true; |
733 | Headers.push_back(Irr.Node); |
734 | LLVM_DEBUG(dbgs() << " => entry = " << BFI.getBlockName(Irr.Node) |
735 | << "\n"); |
736 | break; |
737 | } |
738 | } |
739 | assert(Headers.size() >= 2 && |
740 | "Expected irreducible CFG; -loop-info is likely invalid"); |
741 | if (Headers.size() == InSCC.size()) { |
742 | |
743 | llvm::sort(Headers); |
744 | return; |
745 | } |
746 | |
747 | |
748 | for (const auto &I : InSCC) { |
749 | |
750 | if (I.second) |
751 | continue; |
752 | |
753 | auto &Irr = *I.first; |
754 | for (const auto *P : make_range(Irr.pred_begin(), Irr.pred_end())) { |
755 | |
756 | if (P->Node < Irr.Node) |
757 | continue; |
758 | |
759 | |
760 | |
761 | if (InSCC.lookup(P)) |
762 | continue; |
763 | |
764 | |
765 | Headers.push_back(Irr.Node); |
766 | LLVM_DEBUG(dbgs() << " => extra = " << BFI.getBlockName(Irr.Node) |
767 | << "\n"); |
768 | break; |
769 | } |
770 | if (Headers.back() == Irr.Node) |
771 | |
772 | continue; |
773 | |
774 | |
775 | Others.push_back(Irr.Node); |
776 | LLVM_DEBUG(dbgs() << " => other = " << BFI.getBlockName(Irr.Node) << "\n"); |
777 | } |
778 | llvm::sort(Headers); |
779 | llvm::sort(Others); |
780 | } |
781 | |
782 | static void createIrreducibleLoop( |
783 | BlockFrequencyInfoImplBase &BFI, const IrreducibleGraph &G, |
784 | LoopData *OuterLoop, std::list<LoopData>::iterator Insert, |
785 | const std::vector<const IrreducibleGraph::IrrNode *> &SCC) { |
786 | |
787 | LLVM_DEBUG(dbgs() << " - found-scc\n"); |
788 | |
789 | LoopData::NodeList Headers; |
790 | LoopData::NodeList Others; |
791 | findIrreducibleHeaders(BFI, G, SCC, Headers, Others); |
792 | |
793 | auto Loop = BFI.Loops.emplace(Insert, OuterLoop, Headers.begin(), |
794 | Headers.end(), Others.begin(), Others.end()); |
795 | |
796 | |
797 | for (const auto &N : Loop->Nodes) |
798 | if (BFI.Working[N.Index].isLoopHeader()) |
799 | BFI.Working[N.Index].Loop->Parent = &*Loop; |
800 | else |
801 | BFI.Working[N.Index].Loop = &*Loop; |
802 | } |
803 | |
804 | iterator_range<std::list<LoopData>::iterator> |
805 | BlockFrequencyInfoImplBase::analyzeIrreducible( |
806 | const IrreducibleGraph &G, LoopData *OuterLoop, |
807 | std::list<LoopData>::iterator Insert) { |
808 | assert((OuterLoop == nullptr) == (Insert == Loops.begin())); |
809 | auto Prev = OuterLoop ? std::prev(Insert) : Loops.end(); |
810 | |
811 | for (auto I = scc_begin(G); !I.isAtEnd(); ++I) { |
812 | if (I->size() < 2) |
813 | continue; |
814 | |
815 | |
816 | createIrreducibleLoop(*this, G, OuterLoop, Insert, *I); |
817 | } |
818 | |
819 | if (OuterLoop) |
820 | return make_range(std::next(Prev), Insert); |
821 | return make_range(Loops.begin(), Insert); |
822 | } |
823 | |
824 | void |
825 | BlockFrequencyInfoImplBase::updateLoopWithIrreducible(LoopData &OuterLoop) { |
826 | OuterLoop.Exits.clear(); |
827 | for (auto &Mass : OuterLoop.BackedgeMass) |
828 | Mass = BlockMass::getEmpty(); |
829 | auto O = OuterLoop.Nodes.begin() + 1; |
830 | for (auto I = O, E = OuterLoop.Nodes.end(); I != E; ++I) |
831 | if (!Working[I->Index].isPackaged()) |
832 | *O++ = *I; |
833 | OuterLoop.Nodes.erase(O, OuterLoop.Nodes.end()); |
834 | } |
835 | |
836 | void BlockFrequencyInfoImplBase::adjustLoopHeaderMass(LoopData &Loop) { |
837 | assert(Loop.isIrreducible() && "this only makes sense on irreducible loops"); |
838 | |
839 | |
840 | |
841 | |
842 | |
843 | |
844 | |
845 | BlockMass LoopMass = BlockMass::getFull(); |
846 | Distribution Dist; |
847 | |
848 | LLVM_DEBUG(dbgs() << "adjust-loop-header-mass:\n"); |
849 | for (uint32_t H = 0; H < Loop.NumHeaders; ++H) { |
850 | auto &HeaderNode = Loop.Nodes[H]; |
851 | auto &BackedgeMass = Loop.BackedgeMass[Loop.getHeaderIndex(HeaderNode)]; |
852 | LLVM_DEBUG(dbgs() << " - Add back edge mass for node " |
853 | << getBlockName(HeaderNode) << ": " << BackedgeMass |
854 | << "\n"); |
855 | if (BackedgeMass.getMass() > 0) |
856 | Dist.addLocal(HeaderNode, BackedgeMass.getMass()); |
857 | else |
858 | LLVM_DEBUG(dbgs() << " Nothing added. Back edge mass is zero\n"); |
859 | } |
860 | |
861 | DitheringDistributer D(Dist, LoopMass); |
862 | |
863 | LLVM_DEBUG(dbgs() << " Distribute loop mass " << LoopMass |
864 | << " to headers using above weights\n"); |
865 | for (const Weight &W : Dist.Weights) { |
866 | BlockMass Taken = D.takeMass(W.Amount); |
867 | assert(W.Type == Weight::Local && "all weights should be local"); |
868 | Working[W.TargetNode.Index].getMass() = Taken; |
869 | LLVM_DEBUG(debugAssign(*this, D, W.TargetNode, Taken, nullptr)); |
870 | } |
871 | } |
872 | |
873 | void BlockFrequencyInfoImplBase::distributeIrrLoopHeaderMass(Distribution &Dist) { |
874 | BlockMass LoopMass = BlockMass::getFull(); |
875 | DitheringDistributer D(Dist, LoopMass); |
876 | for (const Weight &W : Dist.Weights) { |
877 | BlockMass Taken = D.takeMass(W.Amount); |
878 | assert(W.Type == Weight::Local && "all weights should be local"); |
879 | Working[W.TargetNode.Index].getMass() = Taken; |
880 | LLVM_DEBUG(debugAssign(*this, D, W.TargetNode, Taken, nullptr)); |
881 | } |
882 | } |
1 | |
2 | |
3 | |
4 | |
5 | |
6 | |
7 | |
8 | |
9 | |
10 | |
11 | |
12 | |
13 | |
14 | |
15 | |
16 | |
17 | |
18 | |
19 | |
20 | |
21 | #ifndef LLVM_SUPPORT_SCALEDNUMBER_H |
22 | #define LLVM_SUPPORT_SCALEDNUMBER_H |
23 | |
24 | #include "llvm/Support/MathExtras.h" |
25 | #include <algorithm> |
26 | #include <cstdint> |
27 | #include <limits> |
28 | #include <string> |
29 | #include <tuple> |
30 | #include <utility> |
31 | |
32 | namespace llvm { |
33 | namespace ScaledNumbers { |
34 | |
35 | |
36 | const int32_t MaxScale = 16383; |
37 | |
38 | |
39 | const int32_t MinScale = -16382; |
40 | |
41 | |
42 | template <class DigitsT> inline int getWidth() { return sizeof(DigitsT) * 8; } |
43 | |
44 | |
45 | |
46 | |
47 | |
48 | |
49 | |
50 | |
51 | template <class DigitsT> |
52 | inline std::pair<DigitsT, int16_t> getRounded(DigitsT Digits, int16_t Scale, |
53 | bool ShouldRound) { |
54 | static_assert(!std::numeric_limits<DigitsT>::is_signed, "expected unsigned"); |
55 | |
56 | if (ShouldRound) |
57 | if (!++Digits) |
58 | |
59 | return std::make_pair(DigitsT(1) << (getWidth<DigitsT>() - 1), Scale + 1); |
60 | return std::make_pair(Digits, Scale); |
61 | } |
62 | |
63 | |
64 | inline std::pair<uint32_t, int16_t> getRounded32(uint32_t Digits, int16_t Scale, |
65 | bool ShouldRound) { |
66 | return getRounded(Digits, Scale, ShouldRound); |
67 | } |
68 | |
69 | |
70 | inline std::pair<uint64_t, int16_t> getRounded64(uint64_t Digits, int16_t Scale, |
71 | bool ShouldRound) { |
72 | return getRounded(Digits, Scale, ShouldRound); |
73 | } |
74 | |
75 | |
76 | |
77 | |
78 | template <class DigitsT> |
79 | inline std::pair<DigitsT, int16_t> getAdjusted(uint64_t Digits, |
80 | int16_t Scale = 0) { |
81 | static_assert(!std::numeric_limits<DigitsT>::is_signed, "expected unsigned"); |
82 | |
83 | const int Width = getWidth<DigitsT>(); |
84 | if (Width == 64 || Digits <= std::numeric_limits<DigitsT>::max()) |
85 | return std::make_pair(Digits, Scale); |
86 | |
87 | |
88 | int Shift = 64 - Width - countLeadingZeros(Digits); |
89 | return getRounded<DigitsT>(Digits >> Shift, Scale + Shift, |
90 | Digits & (UINT64_C(1) << (Shift - 1))); |
91 | } |
92 | |
93 | |
94 | inline std::pair<uint32_t, int16_t> getAdjusted32(uint64_t Digits, |
95 | int16_t Scale = 0) { |
96 | return getAdjusted<uint32_t>(Digits, Scale); |
97 | } |
98 | |
99 | |
100 | inline std::pair<uint64_t, int16_t> getAdjusted64(uint64_t Digits, |
101 | int16_t Scale = 0) { |
102 | return getAdjusted<uint64_t>(Digits, Scale); |
103 | } |
104 | |
105 | |
106 | |
107 | |
108 | std::pair<uint64_t, int16_t> multiply64(uint64_t LHS, uint64_t RHS); |
109 | |
110 | |
111 | |
112 | |
113 | template <class DigitsT> |
114 | inline std::pair<DigitsT, int16_t> getProduct(DigitsT LHS, DigitsT RHS) { |
115 | static_assert(!std::numeric_limits<DigitsT>::is_signed, "expected unsigned"); |
116 | |
117 | if (getWidth<DigitsT>() <= 32 || (LHS <= UINT32_MAX && RHS <= UINT32_MAX)) |
118 | return getAdjusted<DigitsT>(uint64_t(LHS) * RHS); |
119 | |
120 | return multiply64(LHS, RHS); |
121 | } |
122 | |
123 | |
124 | inline std::pair<uint32_t, int16_t> getProduct32(uint32_t LHS, uint32_t RHS) { |
125 | return getProduct(LHS, RHS); |
126 | } |
127 | |
128 | |
129 | inline std::pair<uint64_t, int16_t> getProduct64(uint64_t LHS, uint64_t RHS) { |
130 | return getProduct(LHS, RHS); |
131 | } |
132 | |
133 | |
134 | |
135 | |
136 | |
137 | |
138 | std::pair<uint64_t, int16_t> divide64(uint64_t Dividend, uint64_t Divisor); |
139 | |
140 | |
141 | |
142 | |
143 | |
144 | |
145 | std::pair<uint32_t, int16_t> divide32(uint32_t Dividend, uint32_t Divisor); |
146 | |
147 | |
148 | |
149 | |
150 | |
151 | |
152 | template <class DigitsT> |
153 | std::pair<DigitsT, int16_t> getQuotient(DigitsT Dividend, DigitsT Divisor) { |
154 | static_assert(!std::numeric_limits<DigitsT>::is_signed, "expected unsigned"); |
155 | static_assert(sizeof(DigitsT) == 4 || sizeof(DigitsT) == 8, |
156 | "expected 32-bit or 64-bit digits"); |
157 | |
158 | |
159 | if (!Dividend) |
160 | return std::make_pair(0, 0); |
161 | if (!Divisor) |
162 | return std::make_pair(std::numeric_limits<DigitsT>::max(), MaxScale); |
163 | |
164 | if (getWidth<DigitsT>() == 64) |
165 | return divide64(Dividend, Divisor); |
166 | return divide32(Dividend, Divisor); |
167 | } |
168 | |
169 | |
170 | inline std::pair<uint32_t, int16_t> getQuotient32(uint32_t Dividend, |
171 | uint32_t Divisor) { |
172 | return getQuotient(Dividend, Divisor); |
173 | } |
174 | |
175 | |
176 | inline std::pair<uint64_t, int16_t> getQuotient64(uint64_t Dividend, |
177 | uint64_t Divisor) { |
178 | return getQuotient(Dividend, Divisor); |
179 | } |
180 | |
181 | |
182 | |
183 | |
184 | |
185 | |
186 | |
187 | template <class DigitsT> |
188 | inline std::pair<int32_t, int> getLgImpl(DigitsT Digits, int16_t Scale) { |
189 | static_assert(!std::numeric_limits<DigitsT>::is_signed, "expected unsigned"); |
190 | |
191 | if (!Digits) |
192 | return std::make_pair(INT32_MIN, 0); |
193 | |
194 | |
195 | int32_t LocalFloor = sizeof(Digits) * 8 - countLeadingZeros(Digits) - 1; |
196 | |
197 | |
198 | int32_t Floor = Scale + LocalFloor; |
199 | if (Digits == UINT64_C(1) << LocalFloor) |
200 | return std::make_pair(Floor, 0); |
201 | |
202 | |
203 | assert(LocalFloor >= 1); |
204 | bool Round = Digits & UINT64_C(1) << (LocalFloor - 1); |
205 | return std::make_pair(Floor + Round, Round ? 1 : -1); |
206 | } |
207 | |
208 | |
209 | |
210 | |
211 | |
212 | |
213 | template <class DigitsT> int32_t getLg(DigitsT Digits, int16_t Scale) { |
214 | return getLgImpl(Digits, Scale).first; |
215 | } |
216 | |
217 | |
218 | |
219 | |
220 | |
221 | |
222 | template <class DigitsT> int32_t getLgFloor(DigitsT Digits, int16_t Scale) { |
223 | auto Lg = getLgImpl(Digits, Scale); |
224 | return Lg.first - (Lg.second > 0); |
225 | } |
226 | |
227 | |
228 | |
229 | |
230 | |
231 | |
232 | template <class DigitsT> int32_t getLgCeiling(DigitsT Digits, int16_t Scale) { |
233 | auto Lg = getLgImpl(Digits, Scale); |
234 | return Lg.first + (Lg.second < 0); |
235 | } |
236 | |
237 | |
238 | |
239 | |
240 | |
241 | |
242 | |
243 | |
244 | int compareImpl(uint64_t L, uint64_t R, int ScaleDiff); |
245 | |
246 | |
247 | |
248 | |
249 | |
250 | template <class DigitsT> |
251 | int compare(DigitsT LDigits, int16_t LScale, DigitsT RDigits, int16_t RScale) { |
252 | static_assert(!std::numeric_limits<DigitsT>::is_signed, "expected unsigned"); |
253 | |
254 | |
255 | if (!LDigits) |
256 | return RDigits ? -1 : 0; |
257 | if (!RDigits) |
258 | return 1; |
259 | |
260 | |
261 | |
262 | int32_t lgL = getLgFloor(LDigits, LScale), lgR = getLgFloor(RDigits, RScale); |
263 | if (lgL != lgR) |
264 | return lgL < lgR ? -1 : 1; |
265 | |
266 | |
267 | if (LScale < RScale) |
268 | return compareImpl(LDigits, RDigits, RScale - LScale); |
269 | |
270 | return -compareImpl(RDigits, LDigits, LScale - RScale); |
271 | } |
272 | |
273 | |
274 | |
275 | |
276 | |
277 | |
278 | |
279 | |
280 | |
281 | |
282 | |
283 | |
284 | |
285 | template <class DigitsT> |
286 | int16_t matchScales(DigitsT &LDigits, int16_t &LScale, DigitsT &RDigits, |
287 | int16_t &RScale) { |
288 | static_assert(!std::numeric_limits<DigitsT>::is_signed, "expected unsigned"); |
289 | |
290 | if (LScale < RScale) |
291 | |
292 | return matchScales(RDigits, RScale, LDigits, LScale); |
293 | if (!LDigits) |
294 | return RScale; |
295 | if (!RDigits || LScale == RScale) |
296 | return LScale; |
297 | |
298 | |
299 | int32_t ScaleDiff = int32_t(LScale) - RScale; |
300 | if (ScaleDiff >= 2 * getWidth<DigitsT>()) { |
301 | |
302 | RDigits = 0; |
303 | return LScale; |
304 | } |
305 | |
306 | |
307 | int32_t ShiftL = std::min<int32_t>(countLeadingZeros(LDigits), ScaleDiff); |
308 | assert(ShiftL < getWidth<DigitsT>() && "can't shift more than width"); |
309 | |
310 | int32_t ShiftR = ScaleDiff - ShiftL; |
311 | if (ShiftR >= getWidth<DigitsT>()) { |
312 | |
313 | RDigits = 0; |
314 | return LScale; |
315 | } |
316 | |
317 | LDigits <<= ShiftL; |
318 | RDigits >>= ShiftR; |
319 | |
320 | LScale -= ShiftL; |
321 | RScale += ShiftR; |
322 | assert(LScale == RScale && "scales should match"); |
323 | return LScale; |
324 | } |
325 | |
326 | |
327 | |
328 | |
329 | |
330 | |
331 | template <class DigitsT> |
332 | std::pair<DigitsT, int16_t> getSum(DigitsT LDigits, int16_t LScale, |
333 | DigitsT RDigits, int16_t RScale) { |
334 | static_assert(!std::numeric_limits<DigitsT>::is_signed, "expected unsigned"); |
335 | |
336 | |
337 | |
338 | assert(LScale < INT16_MAX && "scale too large"); |
339 | assert(RScale < INT16_MAX && "scale too large"); |
340 | |
341 | |
342 | int16_t Scale = matchScales(LDigits, LScale, RDigits, RScale); |
343 | |
344 | |
345 | DigitsT Sum = LDigits + RDigits; |
346 | if (Sum >= RDigits) |
347 | return std::make_pair(Sum, Scale); |
348 | |
349 | |
350 | DigitsT HighBit = DigitsT(1) << (getWidth<DigitsT>() - 1); |
351 | return std::make_pair(HighBit | Sum >> 1, Scale + 1); |
352 | } |
353 | |
354 | |
355 | inline std::pair<uint32_t, int16_t> getSum32(uint32_t LDigits, int16_t LScale, |
356 | uint32_t RDigits, int16_t RScale) { |
357 | return getSum(LDigits, LScale, RDigits, RScale); |
358 | } |
359 | |
360 | |
361 | inline std::pair<uint64_t, int16_t> getSum64(uint64_t LDigits, int16_t LScale, |
362 | uint64_t RDigits, int16_t RScale) { |
363 | return getSum(LDigits, LScale, RDigits, RScale); |
364 | } |
365 | |
366 | |
367 | |
368 | |
369 | |
370 | |
371 | template <class DigitsT> |
372 | std::pair<DigitsT, int16_t> getDifference(DigitsT LDigits, int16_t LScale, |
373 | DigitsT RDigits, int16_t RScale) { |
374 | static_assert(!std::numeric_limits<DigitsT>::is_signed, "expected unsigned"); |
375 | |
376 | |
377 | const DigitsT SavedRDigits = RDigits; |
378 | const int16_t SavedRScale = RScale; |
379 | matchScales(LDigits, LScale, RDigits, RScale); |
380 | |
381 | |
382 | if (LDigits <= RDigits) |
383 | return std::make_pair(0, 0); |
384 | if (RDigits || !SavedRDigits) |
385 | return std::make_pair(LDigits - RDigits, LScale); |
386 | |
387 | |
388 | |
389 | |
390 | const auto RLgFloor = getLgFloor(SavedRDigits, SavedRScale); |
391 | if (!compare(LDigits, LScale, DigitsT(1), RLgFloor + getWidth<DigitsT>())) |
392 | return std::make_pair(std::numeric_limits<DigitsT>::max(), RLgFloor); |
393 | |
394 | return std::make_pair(LDigits, LScale); |
395 | } |
396 | |
397 | |
398 | inline std::pair<uint32_t, int16_t> getDifference32(uint32_t LDigits, |
399 | int16_t LScale, |
400 | uint32_t RDigits, |
401 | int16_t RScale) { |
402 | return getDifference(LDigits, LScale, RDigits, RScale); |
403 | } |
404 | |
405 | |
406 | inline std::pair<uint64_t, int16_t> getDifference64(uint64_t LDigits, |
407 | int16_t LScale, |
408 | uint64_t RDigits, |
409 | int16_t RScale) { |
410 | return getDifference(LDigits, LScale, RDigits, RScale); |
411 | } |
412 | |
413 | } |
414 | } |
415 | |
416 | namespace llvm { |
417 | |
418 | class raw_ostream; |
419 | class ScaledNumberBase { |
420 | public: |
421 | static constexpr int DefaultPrecision = 10; |
422 | |
423 | static void dump(uint64_t D, int16_t E, int Width); |
424 | static raw_ostream &print(raw_ostream &OS, uint64_t D, int16_t E, int Width, |
425 | unsigned Precision); |
426 | static std::string toString(uint64_t D, int16_t E, int Width, |
427 | unsigned Precision); |
428 | static int countLeadingZeros32(uint32_t N) { return countLeadingZeros(N); } |
429 | static int countLeadingZeros64(uint64_t N) { return countLeadingZeros(N); } |
430 | static uint64_t getHalf(uint64_t N) { return (N >> 1) + (N & 1); } |
431 | |
432 | static std::pair<uint64_t, bool> splitSigned(int64_t N) { |
433 | if (N >= 0) |
434 | return std::make_pair(N, false); |
435 | uint64_t Unsigned = N == INT64_MIN ? UINT64_C(1) << 63 : uint64_t(-N); |
436 | return std::make_pair(Unsigned, true); |
437 | } |
438 | static int64_t joinSigned(uint64_t U, bool IsNeg) { |
439 | if (U > uint64_t(INT64_MAX)) |
440 | return IsNeg ? INT64_MIN : INT64_MAX; |
441 | return IsNeg ? -int64_t(U) : int64_t(U); |
442 | } |
443 | }; |
444 | |
445 | |
446 | |
447 | |
448 | |
449 | |
450 | |
451 | |
452 | |
453 | |
454 | |
455 | |
456 | |
457 | |
458 | |
459 | |
460 | |
461 | |
462 | |
463 | |
464 | |
465 | |
466 | |
467 | |
468 | |
469 | |
470 | |
471 | |
472 | |
473 | |
474 | |
475 | |
476 | |
477 | |
478 | |
479 | |
480 | |
481 | |
482 | |
483 | |
484 | |
485 | |
486 | |
487 | |
488 | |
489 | |
490 | |
491 | |
492 | template <class DigitsT> class ScaledNumber : ScaledNumberBase { |
493 | public: |
494 | static_assert(!std::numeric_limits<DigitsT>::is_signed, |
495 | "only unsigned floats supported"); |
496 | |
497 | typedef DigitsT DigitsType; |
498 | |
499 | private: |
500 | typedef std::numeric_limits<DigitsType> DigitsLimits; |
501 | |
502 | static constexpr int Width = sizeof(DigitsType) * 8; |
503 | static_assert(Width <= 64, "invalid integer width for digits"); |
504 | |
505 | private: |
506 | DigitsType Digits = 0; |
507 | int16_t Scale = 0; |
508 | |
509 | public: |
510 | ScaledNumber() = default; |
511 | |
512 | constexpr ScaledNumber(DigitsType Digits, int16_t Scale) |
513 | : Digits(Digits), Scale(Scale) {} |
514 | |
515 | private: |
516 | ScaledNumber(const std::pair<DigitsT, int16_t> &X) |
517 | : Digits(X.first), Scale(X.second) {} |
518 | |
519 | public: |
520 | static ScaledNumber getZero() { return ScaledNumber(0, 0); } |
521 | static ScaledNumber getOne() { return ScaledNumber(1, 0); } |
522 | static ScaledNumber getLargest() { |
523 | return ScaledNumber(DigitsLimits::max(), ScaledNumbers::MaxScale); |
524 | } |
525 | static ScaledNumber get(uint64_t N) { return adjustToWidth(N, 0); } |
526 | static ScaledNumber getInverse(uint64_t N) { |
527 | return get(N).invert(); |
528 | } |
529 | static ScaledNumber getFraction(DigitsType N, DigitsType D) { |
530 | return getQuotient(N, D); |
531 | } |
532 | |
533 | int16_t getScale() const { return Scale; } |
534 | DigitsType getDigits() const { return Digits; } |
535 | |
536 | |
537 | |
538 | |
539 | |
540 | template <class IntT> IntT toInt() const; |
541 | |
542 | bool isZero() const { return !Digits; } |
543 | bool isLargest() const { return *this == getLargest(); } |
544 | bool isOne() const { |
545 | if (Scale > 0 || Scale <= -Width) |
546 | return false; |
547 | return Digits == DigitsType(1) << -Scale; |
548 | } |
549 | |
550 | |
551 | |
552 | |
553 | int32_t lg() const { return ScaledNumbers::getLg(Digits, Scale); } |
554 | |
555 | |
556 | |
557 | |
558 | int32_t lgFloor() const { return ScaledNumbers::getLgFloor(Digits, Scale); } |
559 | |
560 | |
561 | |
562 | |
563 | int32_t lgCeiling() const { |
564 | return ScaledNumbers::getLgCeiling(Digits, Scale); |
565 | } |
566 | |
567 | bool operator==(const ScaledNumber &X) const { return compare(X) == 0; } |
568 | bool operator<(const ScaledNumber &X) const { return compare(X) < 0; } |
569 | bool operator!=(const ScaledNumber &X) const { return compare(X) != 0; } |
570 | bool operator>(const ScaledNumber &X) const { return compare(X) > 0; } |
571 | bool operator<=(const ScaledNumber &X) const { return compare(X) <= 0; } |
572 | bool operator>=(const ScaledNumber &X) const { return compare(X) >= 0; } |
573 | |
574 | bool operator!() const { return isZero(); } |
575 | |
576 | |
577 | |
578 | |
579 | |
580 | |
581 | |
582 | |
583 | |
584 | |
585 | |
586 | |
587 | |
588 | |
589 | |
590 | |
591 | |
592 | |
593 | |
594 | |
595 | std::string toString(unsigned Precision = DefaultPrecision) { |
596 | return ScaledNumberBase::toString(Digits, Scale, Width, Precision); |
597 | } |
598 | |
599 | |
600 | |
601 | |
602 | raw_ostream &print(raw_ostream &OS, |
603 | unsigned Precision = DefaultPrecision) const { |
604 | return ScaledNumberBase::print(OS, Digits, Scale, Width, Precision); |
605 | } |
606 | void dump() const { return ScaledNumberBase::dump(Digits, Scale, Width); } |
607 | |
608 | ScaledNumber &operator+=(const ScaledNumber &X) { |
609 | std::tie(Digits, Scale) = |
610 | ScaledNumbers::getSum(Digits, Scale, X.Digits, X.Scale); |
611 | |
612 | if (Scale > ScaledNumbers::MaxScale) |
613 | *this = getLargest(); |
614 | return *this; |
615 | } |
616 | ScaledNumber &operator-=(const ScaledNumber &X) { |
617 | std::tie(Digits, Scale) = |
618 | ScaledNumbers::getDifference(Digits, Scale, X.Digits, X.Scale); |
619 | return *this; |
620 | } |
621 | ScaledNumber &operator*=(const ScaledNumber &X); |
622 | ScaledNumber &operator/=(const ScaledNumber &X); |
623 | ScaledNumber &operator<<=(int16_t Shift) { |
624 | shiftLeft(Shift); |
625 | return *this; |
626 | } |
627 | ScaledNumber &operator>>=(int16_t Shift) { |
628 | shiftRight(Shift); |
629 | return *this; |
630 | } |
631 | |
632 | private: |
633 | void shiftLeft(int32_t Shift); |
634 | void shiftRight(int32_t Shift); |
635 | |
636 | |
637 | |
638 | |
639 | |
640 | |
641 | |
642 | |
643 | ScaledNumber matchScales(ScaledNumber X) { |
644 | ScaledNumbers::matchScales(Digits, Scale, X.Digits, X.Scale); |
645 | return X; |
646 | } |
647 | |
648 | public: |
649 | |
650 | |
651 | |
652 | |
653 | uint64_t scale(uint64_t N) const; |
654 | uint64_t scaleByInverse(uint64_t N) const { |
655 | |
656 | |
657 | return inverse().scale(N); |
658 | } |
659 | int64_t scale(int64_t N) const { |
660 | std::pair<uint64_t, bool> Unsigned = splitSigned(N); |
661 | return joinSigned(scale(Unsigned.first), Unsigned.second); |
662 | } |
663 | int64_t scaleByInverse(int64_t N) const { |
664 | std::pair<uint64_t, bool> Unsigned = splitSigned(N); |
665 | return joinSigned(scaleByInverse(Unsigned.first), Unsigned.second); |
666 | } |
667 | |
668 | int compare(const ScaledNumber &X) const { |
669 | return ScaledNumbers::compare(Digits, Scale, X.Digits, X.Scale); |
670 | } |
671 | int compareTo(uint64_t N) const { |
672 | return ScaledNumbers::compare<uint64_t>(Digits, Scale, N, 0); |
673 | } |
674 | int compareTo(int64_t N) const { return N < 0 ? 1 : compareTo(uint64_t(N)); } |
675 | |
676 | ScaledNumber &invert() { return *this = ScaledNumber::get(1) / *this; } |
677 | ScaledNumber inverse() const { return ScaledNumber(*this).invert(); } |
678 | |
679 | private: |
680 | static ScaledNumber getProduct(DigitsType LHS, DigitsType RHS) { |
681 | return ScaledNumbers::getProduct(LHS, RHS); |
682 | } |
683 | static ScaledNumber getQuotient(DigitsType Dividend, DigitsType Divisor) { |
684 | return ScaledNumbers::getQuotient(Dividend, Divisor); |
685 | } |
686 | |
687 | static int countLeadingZerosWidth(DigitsType Digits) { |
688 | if (Width == 64) |
689 | return countLeadingZeros64(Digits); |
690 | if (Width == 32) |
691 | return countLeadingZeros32(Digits); |
692 | return countLeadingZeros32(Digits) + Width - 32; |
693 | } |
694 | |
695 | |
696 | |
697 | |
698 | |
699 | |
700 | static ScaledNumber adjustToWidth(uint64_t N, int32_t Shift) { |
701 | assert(Shift >= ScaledNumbers::MinScale && "Shift should be close to 0"); |
702 | assert(Shift <= ScaledNumbers::MaxScale - 64 && |
703 | "Shift should be close to 0"); |
704 | auto Adjusted = ScaledNumbers::getAdjusted<DigitsT>(N, Shift); |
705 | return Adjusted; |
706 | } |
707 | |
708 | static ScaledNumber getRounded(ScaledNumber P, bool Round) { |
709 | |
710 | if (P.isLargest()) |
711 | return P; |
712 | |
713 | return ScaledNumbers::getRounded(P.Digits, P.Scale, Round); |
714 | } |
715 | }; |
716 | |
717 | #define SCALED_NUMBER_BOP(op, base) \ |
718 | template <class DigitsT> \ |
719 | ScaledNumber<DigitsT> operator op(const ScaledNumber<DigitsT> &L, \ |
720 | const ScaledNumber<DigitsT> &R) { \ |
721 | return ScaledNumber<DigitsT>(L) base R; \ |
722 | } |
723 | SCALED_NUMBER_BOP(+, += ) |
724 | SCALED_NUMBER_BOP(-, -= ) |
725 | SCALED_NUMBER_BOP(*, *= ) |
| 9 | | The value 16383 is assigned to 'Scaled.Scale' | |
|
726 | SCALED_NUMBER_BOP(/, /= ) |
727 | #undef SCALED_NUMBER_BOP |
728 | |
729 | template <class DigitsT> |
730 | ScaledNumber<DigitsT> operator<<(const ScaledNumber<DigitsT> &L, |
731 | int16_t Shift) { |
732 | return ScaledNumber<DigitsT>(L) <<= Shift; |
733 | } |
734 | |
735 | template <class DigitsT> |
736 | ScaledNumber<DigitsT> operator>>(const ScaledNumber<DigitsT> &L, |
737 | int16_t Shift) { |
738 | return ScaledNumber<DigitsT>(L) >>= Shift; |
739 | } |
740 | |
741 | template <class DigitsT> |
742 | raw_ostream &operator<<(raw_ostream &OS, const ScaledNumber<DigitsT> &X) { |
743 | return X.print(OS, 10); |
744 | } |
745 | |
746 | #define SCALED_NUMBER_COMPARE_TO_TYPE(op, T1, T2) \ |
747 | template <class DigitsT> \ |
748 | bool operator op(const ScaledNumber<DigitsT> &L, T1 R) { \ |
749 | return L.compareTo(T2(R)) op 0; \ |
750 | } \ |
751 | template <class DigitsT> \ |
752 | bool operator op(T1 L, const ScaledNumber<DigitsT> &R) { \ |
753 | return 0 op R.compareTo(T2(L)); \ |
754 | } |
755 | #define SCALED_NUMBER_COMPARE_TO(op) \ |
756 | SCALED_NUMBER_COMPARE_TO_TYPE(op, uint64_t, uint64_t) \ |
757 | SCALED_NUMBER_COMPARE_TO_TYPE(op, uint32_t, uint64_t) \ |
758 | SCALED_NUMBER_COMPARE_TO_TYPE(op, int64_t, int64_t) \ |
759 | SCALED_NUMBER_COMPARE_TO_TYPE(op, int32_t, int64_t) |
760 | SCALED_NUMBER_COMPARE_TO(< ) |
| 13 | | Assuming the condition is false | |
|
| 14 | | Returning zero, which participates in a condition later | |
|
761 | SCALED_NUMBER_COMPARE_TO(> ) |
762 | SCALED_NUMBER_COMPARE_TO(== ) |
763 | SCALED_NUMBER_COMPARE_TO(!= ) |
764 | SCALED_NUMBER_COMPARE_TO(<= ) |
765 | SCALED_NUMBER_COMPARE_TO(>= ) |
| 18 | | Assuming the condition is false | |
|
| 19 | | Returning zero, which participates in a condition later | |
|
766 | #undef SCALED_NUMBER_COMPARE_TO |
767 | #undef SCALED_NUMBER_COMPARE_TO_TYPE |
768 | |
769 | template <class DigitsT> |
770 | uint64_t ScaledNumber<DigitsT>::scale(uint64_t N) const { |
771 | if (Width == 64 || N <= DigitsLimits::max()) |
772 | return (get(N) * *this).template toInt<uint64_t>(); |
773 | |
774 | |
775 | return ScaledNumber<uint64_t>(Digits, Scale).scale(N); |
776 | } |
777 | |
778 | template <class DigitsT> |
779 | template <class IntT> |
780 | IntT ScaledNumber<DigitsT>::toInt() const { |
781 | typedef std::numeric_limits<IntT> Limits; |
782 | if (*this < 1) |
| 12 | | Calling 'operator<<unsigned long long>' | |
|
| 15 | | Returning from 'operator<<unsigned long long>' | |
|
| |
783 | return 0; |
784 | if (*this >= Limits::max()) |
| 17 | | Calling 'operator>=<unsigned long long>' | |
|
| 20 | | Returning from 'operator>=<unsigned long long>' | |
|
| |
785 | return Limits::max(); |
786 | |
787 | IntT N = Digits; |
788 | if (Scale > 0) { |
| |
789 | assert(size_t(Scale) < sizeof(IntT) * 8); |
790 | return N << Scale; |
| 23 | | The result of the left shift is undefined due to shifting by '16383', which is greater or equal to the width of type 'unsigned long long' |
|
791 | } |
792 | if (Scale < 0) { |
793 | assert(size_t(-Scale) < sizeof(IntT) * 8); |
794 | return N >> -Scale; |
795 | } |
796 | return N; |
797 | } |
798 | |
799 | template <class DigitsT> |
800 | ScaledNumber<DigitsT> &ScaledNumber<DigitsT>:: |
801 | operator*=(const ScaledNumber &X) { |
802 | if (isZero()) |
803 | return *this; |
804 | if (X.isZero()) |
805 | return *this = X; |
806 | |
807 | |
808 | int32_t Scales = int32_t(Scale) + int32_t(X.Scale); |
809 | |
810 | |
811 | *this = getProduct(Digits, X.Digits); |
812 | |
813 | |
814 | return *this <<= Scales; |
815 | } |
816 | template <class DigitsT> |
817 | ScaledNumber<DigitsT> &ScaledNumber<DigitsT>:: |
818 | operator/=(const ScaledNumber &X) { |
819 | if (isZero()) |
820 | return *this; |
821 | if (X.isZero()) |
822 | return *this = getLargest(); |
823 | |
824 | |
825 | int32_t Scales = int32_t(Scale) - int32_t(X.Scale); |
826 | |
827 | |
828 | *this = getQuotient(Digits, X.Digits); |
829 | |
830 | |
831 | return *this <<= Scales; |
832 | } |
833 | template <class DigitsT> void ScaledNumber<DigitsT>::shiftLeft(int32_t Shift) { |
834 | if (!Shift || isZero()) |
835 | return; |
836 | assert(Shift != INT32_MIN); |
837 | if (Shift < 0) { |
838 | shiftRight(-Shift); |
839 | return; |
840 | } |
841 | |
842 | |
843 | int32_t ScaleShift = std::min(Shift, ScaledNumbers::MaxScale - Scale); |
844 | Scale += ScaleShift; |
845 | if (ScaleShift == Shift) |
846 | return; |
847 | |
848 | |
849 | if (isLargest()) |
850 | return; |
851 | |
852 | |
853 | Shift -= ScaleShift; |
854 | if (Shift > countLeadingZerosWidth(Digits)) { |
855 | |
856 | *this = getLargest(); |
857 | return; |
858 | } |
859 | |
860 | Digits <<= Shift; |
861 | } |
862 | |
863 | template <class DigitsT> void ScaledNumber<DigitsT>::shiftRight(int32_t Shift) { |
864 | if (!Shift || isZero()) |
865 | return; |
866 | assert(Shift != INT32_MIN); |
867 | if (Shift < 0) { |
868 | shiftLeft(-Shift); |
869 | return; |
870 | } |
871 | |
872 | |
873 | int32_t ScaleShift = std::min(Shift, Scale - ScaledNumbers::MinScale); |
874 | Scale -= ScaleShift; |
875 | if (ScaleShift == Shift) |
876 | return; |
877 | |
878 | |
879 | Shift -= ScaleShift; |
880 | if (Shift >= Width) { |
881 | |
882 | *this = getZero(); |
883 | return; |
884 | } |
885 | |
886 | Digits >>= Shift; |
887 | } |
888 | |
889 | |
890 | } |
891 | |
892 | #endif // LLVM_SUPPORT_SCALEDNUMBER_H |