clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CGBlocks.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/gnu/usr.bin/clang/libclangCodeGen/obj -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/gnu/usr.bin/clang/libclangCodeGen/../../../llvm/clang/include -I /usr/src/gnu/usr.bin/clang/libclangCodeGen/../../../llvm/llvm/include -I /usr/src/gnu/usr.bin/clang/libclangCodeGen/../include -I /usr/src/gnu/usr.bin/clang/libclangCodeGen/obj -I /usr/src/gnu/usr.bin/clang/libclangCodeGen/obj/../include -D NDEBUG -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D LLVM_PREFIX="/usr" -internal-isystem /usr/include/c++/v1 -internal-isystem /usr/local/lib/clang/13.0.0/include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/usr/src/gnu/usr.bin/clang/libclangCodeGen/obj -ferror-limit 19 -fvisibility-inlines-hidden -fwrapv -stack-protector 2 -fno-rtti -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/ben/Projects/vmm/scan-build/2022-01-12-194120-40624-1 -x c++ /usr/src/gnu/usr.bin/clang/libclangCodeGen/../../../llvm/clang/lib/CodeGen/CGBlocks.cpp
1 | |
2 | |
3 | |
4 | |
5 | |
6 | |
7 | |
8 | |
9 | |
10 | |
11 | |
12 | |
13 | #include "CGBlocks.h" |
14 | #include "CGCXXABI.h" |
15 | #include "CGDebugInfo.h" |
16 | #include "CGObjCRuntime.h" |
17 | #include "CGOpenCLRuntime.h" |
18 | #include "CodeGenFunction.h" |
19 | #include "CodeGenModule.h" |
20 | #include "ConstantEmitter.h" |
21 | #include "TargetInfo.h" |
22 | #include "clang/AST/Attr.h" |
23 | #include "clang/AST/DeclObjC.h" |
24 | #include "clang/CodeGen/ConstantInitBuilder.h" |
25 | #include "llvm/ADT/SmallSet.h" |
26 | #include "llvm/IR/DataLayout.h" |
27 | #include "llvm/IR/Module.h" |
28 | #include "llvm/Support/ScopedPrinter.h" |
29 | #include <algorithm> |
30 | #include <cstdio> |
31 | |
32 | using namespace clang; |
33 | using namespace CodeGen; |
34 | |
35 | CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name) |
36 | : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false), |
37 | HasCXXObject(false), UsesStret(false), HasCapturedVariableLayout(false), |
38 | CapturesNonExternalType(false), LocalAddress(Address::invalid()), |
39 | StructureType(nullptr), Block(block) { |
40 | |
41 | |
42 | |
43 | if (!name.empty() && name[0] == '\01') |
44 | name = name.substr(1); |
45 | } |
46 | |
47 | |
48 | BlockByrefHelpers::~BlockByrefHelpers() {} |
49 | |
50 | |
51 | static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM, |
52 | const CGBlockInfo &blockInfo, |
53 | llvm::Constant *blockFn); |
54 | |
55 | |
56 | static llvm::Constant *buildCopyHelper(CodeGenModule &CGM, |
57 | const CGBlockInfo &blockInfo) { |
58 | return CodeGenFunction(CGM).GenerateCopyHelperFunction(blockInfo); |
59 | } |
60 | |
61 | |
62 | static llvm::Constant *buildDisposeHelper(CodeGenModule &CGM, |
63 | const CGBlockInfo &blockInfo) { |
64 | return CodeGenFunction(CGM).GenerateDestroyHelperFunction(blockInfo); |
65 | } |
66 | |
67 | namespace { |
68 | |
69 | |
70 | |
71 | enum class BlockCaptureEntityKind { |
72 | CXXRecord, |
73 | ARCWeak, |
74 | ARCStrong, |
75 | NonTrivialCStruct, |
76 | BlockObject, |
77 | None |
78 | }; |
79 | |
80 | |
81 | |
82 | struct BlockCaptureManagedEntity { |
83 | BlockCaptureEntityKind CopyKind, DisposeKind; |
84 | BlockFieldFlags CopyFlags, DisposeFlags; |
85 | const BlockDecl::Capture *CI; |
86 | const CGBlockInfo::Capture *Capture; |
87 | |
88 | BlockCaptureManagedEntity(BlockCaptureEntityKind CopyType, |
89 | BlockCaptureEntityKind DisposeType, |
90 | BlockFieldFlags CopyFlags, |
91 | BlockFieldFlags DisposeFlags, |
92 | const BlockDecl::Capture &CI, |
93 | const CGBlockInfo::Capture &Capture) |
94 | : CopyKind(CopyType), DisposeKind(DisposeType), CopyFlags(CopyFlags), |
95 | DisposeFlags(DisposeFlags), CI(&CI), Capture(&Capture) {} |
96 | |
97 | bool operator<(const BlockCaptureManagedEntity &Other) const { |
98 | return Capture->getOffset() < Other.Capture->getOffset(); |
99 | } |
100 | }; |
101 | |
102 | enum class CaptureStrKind { |
103 | |
104 | CopyHelper, |
105 | |
106 | DisposeHelper, |
107 | |
108 | Merged |
109 | }; |
110 | |
111 | } |
112 | |
113 | static void findBlockCapturedManagedEntities( |
114 | const CGBlockInfo &BlockInfo, const LangOptions &LangOpts, |
115 | SmallVectorImpl<BlockCaptureManagedEntity> &ManagedCaptures); |
116 | |
117 | static std::string getBlockCaptureStr(const BlockCaptureManagedEntity &E, |
118 | CaptureStrKind StrKind, |
119 | CharUnits BlockAlignment, |
120 | CodeGenModule &CGM); |
121 | |
122 | static std::string getBlockDescriptorName(const CGBlockInfo &BlockInfo, |
123 | CodeGenModule &CGM) { |
124 | std::string Name = "__block_descriptor_"; |
125 | Name += llvm::to_string(BlockInfo.BlockSize.getQuantity()) + "_"; |
126 | |
127 | if (BlockInfo.needsCopyDisposeHelpers()) { |
128 | if (CGM.getLangOpts().Exceptions) |
129 | Name += "e"; |
130 | if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions) |
131 | Name += "a"; |
132 | Name += llvm::to_string(BlockInfo.BlockAlign.getQuantity()) + "_"; |
133 | |
134 | SmallVector<BlockCaptureManagedEntity, 4> ManagedCaptures; |
135 | findBlockCapturedManagedEntities(BlockInfo, CGM.getContext().getLangOpts(), |
136 | ManagedCaptures); |
137 | |
138 | for (const BlockCaptureManagedEntity &E : ManagedCaptures) { |
139 | Name += llvm::to_string(E.Capture->getOffset().getQuantity()); |
140 | |
141 | if (E.CopyKind == E.DisposeKind) { |
142 | |
143 | |
144 | assert(E.CopyKind != BlockCaptureEntityKind::None && |
145 | "shouldn't see BlockCaptureManagedEntity that is None"); |
146 | Name += getBlockCaptureStr(E, CaptureStrKind::Merged, |
147 | BlockInfo.BlockAlign, CGM); |
148 | } else { |
149 | |
150 | |
151 | |
152 | Name += getBlockCaptureStr(E, CaptureStrKind::CopyHelper, |
153 | BlockInfo.BlockAlign, CGM); |
154 | Name += getBlockCaptureStr(E, CaptureStrKind::DisposeHelper, |
155 | BlockInfo.BlockAlign, CGM); |
156 | } |
157 | } |
158 | Name += "_"; |
159 | } |
160 | |
161 | std::string TypeAtEncoding = |
162 | CGM.getContext().getObjCEncodingForBlock(BlockInfo.getBlockExpr()); |
163 | |
164 | |
165 | std::replace(TypeAtEncoding.begin(), TypeAtEncoding.end(), '@', '\1'); |
166 | Name += "e" + llvm::to_string(TypeAtEncoding.size()) + "_" + TypeAtEncoding; |
167 | Name += "l" + CGM.getObjCRuntime().getRCBlockLayoutStr(CGM, BlockInfo); |
168 | return Name; |
169 | } |
170 | |
171 | |
172 | |
173 | |
174 | |
175 | |
176 | |
177 | |
178 | |
179 | |
180 | |
181 | |
182 | |
183 | |
184 | |
185 | static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM, |
186 | const CGBlockInfo &blockInfo) { |
187 | ASTContext &C = CGM.getContext(); |
188 | |
189 | llvm::IntegerType *ulong = |
190 | cast<llvm::IntegerType>(CGM.getTypes().ConvertType(C.UnsignedLongTy)); |
191 | llvm::PointerType *i8p = nullptr; |
192 | if (CGM.getLangOpts().OpenCL) |
193 | i8p = |
194 | llvm::Type::getInt8PtrTy( |
195 | CGM.getLLVMContext(), C.getTargetAddressSpace(LangAS::opencl_constant)); |
196 | else |
197 | i8p = CGM.VoidPtrTy; |
198 | |
199 | std::string descName; |
200 | |
201 | |
202 | if (C.getLangOpts().ObjC && |
203 | CGM.getLangOpts().getGC() == LangOptions::NonGC) { |
204 | descName = getBlockDescriptorName(blockInfo, CGM); |
205 | if (llvm::GlobalValue *desc = CGM.getModule().getNamedValue(descName)) |
206 | return llvm::ConstantExpr::getBitCast(desc, |
207 | CGM.getBlockDescriptorType()); |
208 | } |
209 | |
210 | |
211 | |
212 | ConstantInitBuilder builder(CGM); |
213 | auto elements = builder.beginStruct(); |
214 | |
215 | |
216 | elements.addInt(ulong, 0); |
217 | |
218 | |
219 | |
220 | |
221 | |
222 | elements.addInt(ulong, blockInfo.BlockSize.getQuantity()); |
223 | |
224 | |
225 | bool hasInternalHelper = false; |
226 | if (blockInfo.needsCopyDisposeHelpers()) { |
227 | |
228 | llvm::Constant *copyHelper = buildCopyHelper(CGM, blockInfo); |
229 | elements.add(copyHelper); |
230 | |
231 | |
232 | llvm::Constant *disposeHelper = buildDisposeHelper(CGM, blockInfo); |
233 | elements.add(disposeHelper); |
234 | |
235 | if (cast<llvm::Function>(copyHelper->getOperand(0))->hasInternalLinkage() || |
236 | cast<llvm::Function>(disposeHelper->getOperand(0)) |
237 | ->hasInternalLinkage()) |
238 | hasInternalHelper = true; |
239 | } |
240 | |
241 | |
242 | std::string typeAtEncoding = |
243 | CGM.getContext().getObjCEncodingForBlock(blockInfo.getBlockExpr()); |
244 | elements.add(llvm::ConstantExpr::getBitCast( |
245 | CGM.GetAddrOfConstantCString(typeAtEncoding).getPointer(), i8p)); |
246 | |
247 | |
248 | if (C.getLangOpts().ObjC) { |
249 | if (CGM.getLangOpts().getGC() != LangOptions::NonGC) |
250 | elements.add(CGM.getObjCRuntime().BuildGCBlockLayout(CGM, blockInfo)); |
251 | else |
252 | elements.add(CGM.getObjCRuntime().BuildRCBlockLayout(CGM, blockInfo)); |
253 | } |
254 | else |
255 | elements.addNullPointer(i8p); |
256 | |
257 | unsigned AddrSpace = 0; |
258 | if (C.getLangOpts().OpenCL) |
259 | AddrSpace = C.getTargetAddressSpace(LangAS::opencl_constant); |
260 | |
261 | llvm::GlobalValue::LinkageTypes linkage; |
262 | if (descName.empty()) { |
263 | linkage = llvm::GlobalValue::InternalLinkage; |
264 | descName = "__block_descriptor_tmp"; |
265 | } else if (hasInternalHelper) { |
266 | |
267 | |
268 | linkage = llvm::GlobalValue::InternalLinkage; |
269 | } else { |
270 | linkage = llvm::GlobalValue::LinkOnceODRLinkage; |
271 | } |
272 | |
273 | llvm::GlobalVariable *global = |
274 | elements.finishAndCreateGlobal(descName, CGM.getPointerAlign(), |
275 | true, linkage, AddrSpace); |
276 | |
277 | if (linkage == llvm::GlobalValue::LinkOnceODRLinkage) { |
278 | if (CGM.supportsCOMDAT()) |
279 | global->setComdat(CGM.getModule().getOrInsertComdat(descName)); |
280 | global->setVisibility(llvm::GlobalValue::HiddenVisibility); |
281 | global->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); |
282 | } |
283 | |
284 | return llvm::ConstantExpr::getBitCast(global, CGM.getBlockDescriptorType()); |
285 | } |
286 | |
287 | |
288 | |
289 | |
290 | |
291 | |
292 | |
293 | |
294 | |
295 | |
296 | |
297 | |
298 | |
299 | |
300 | |
301 | |
302 | |
303 | |
304 | |
305 | |
306 | |
307 | |
308 | |
309 | |
310 | |
311 | |
312 | |
313 | |
314 | |
315 | |
316 | |
317 | |
318 | |
319 | |
320 | |
321 | |
322 | |
323 | |
324 | |
325 | |
326 | |
327 | |
328 | |
329 | |
330 | |
331 | |
332 | |
333 | |
334 | |
335 | |
336 | |
337 | |
338 | namespace { |
339 | |
340 | struct BlockLayoutChunk { |
341 | CharUnits Alignment; |
342 | CharUnits Size; |
343 | Qualifiers::ObjCLifetime Lifetime; |
344 | const BlockDecl::Capture *Capture; |
345 | llvm::Type *Type; |
346 | QualType FieldType; |
347 | |
348 | BlockLayoutChunk(CharUnits align, CharUnits size, |
349 | Qualifiers::ObjCLifetime lifetime, |
350 | const BlockDecl::Capture *capture, |
351 | llvm::Type *type, QualType fieldType) |
352 | : Alignment(align), Size(size), Lifetime(lifetime), |
353 | Capture(capture), Type(type), FieldType(fieldType) {} |
354 | |
355 | |
356 | void setIndex(CGBlockInfo &info, unsigned index, CharUnits offset) { |
357 | if (!Capture) { |
358 | info.CXXThisIndex = index; |
359 | info.CXXThisOffset = offset; |
360 | } else { |
361 | auto C = CGBlockInfo::Capture::makeIndex(index, offset, FieldType); |
362 | info.Captures.insert({Capture->getVariable(), C}); |
363 | } |
364 | } |
365 | }; |
366 | |
367 | |
368 | |
369 | bool operator<(const BlockLayoutChunk &left, const BlockLayoutChunk &right) { |
370 | if (left.Alignment != right.Alignment) |
371 | return left.Alignment > right.Alignment; |
372 | |
373 | auto getPrefOrder = [](const BlockLayoutChunk &chunk) { |
374 | if (chunk.Capture && chunk.Capture->isByRef()) |
375 | return 1; |
376 | if (chunk.Lifetime == Qualifiers::OCL_Strong) |
377 | return 0; |
378 | if (chunk.Lifetime == Qualifiers::OCL_Weak) |
379 | return 2; |
380 | return 3; |
381 | }; |
382 | |
383 | return getPrefOrder(left) < getPrefOrder(right); |
384 | } |
385 | } |
386 | |
387 | |
388 | static bool isSafeForCXXConstantCapture(QualType type) { |
389 | const RecordType *recordType = |
390 | type->getBaseElementTypeUnsafe()->getAs<RecordType>(); |
391 | |
392 | |
393 | if (!recordType) return true; |
394 | |
395 | const auto *record = cast<CXXRecordDecl>(recordType->getDecl()); |
396 | |
397 | |
398 | if (!record->hasTrivialDestructor()) return false; |
399 | if (record->hasNonTrivialCopyConstructor()) return false; |
400 | |
401 | |
402 | |
403 | return !record->hasMutableFields(); |
404 | } |
405 | |
406 | |
407 | |
408 | |
409 | |
410 | |
411 | |
412 | static llvm::Constant *tryCaptureAsConstant(CodeGenModule &CGM, |
413 | CodeGenFunction *CGF, |
414 | const VarDecl *var) { |
415 | |
416 | |
417 | if (isa<ParmVarDecl>(var)) |
418 | return nullptr; |
419 | |
420 | QualType type = var->getType(); |
421 | |
422 | |
423 | if (!type.isConstQualified()) return nullptr; |
424 | |
425 | |
426 | |
427 | |
428 | |
429 | |
430 | if (CGM.getLangOpts().CPlusPlus && !isSafeForCXXConstantCapture(type)) |
431 | return nullptr; |
432 | |
433 | |
434 | |
435 | |
436 | const Expr *init = var->getInit(); |
437 | if (!init) return nullptr; |
438 | |
439 | return ConstantEmitter(CGM, CGF).tryEmitAbstractForInitializer(*var); |
440 | } |
441 | |
442 | |
443 | |
444 | static CharUnits getLowBit(CharUnits v) { |
445 | return CharUnits::fromQuantity(v.getQuantity() & (~v.getQuantity() + 1)); |
446 | } |
447 | |
448 | static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info, |
449 | SmallVectorImpl<llvm::Type*> &elementTypes) { |
450 | |
451 | assert(elementTypes.empty()); |
452 | if (CGM.getLangOpts().OpenCL) { |
453 | |
454 | |
455 | auto GenericAS = |
456 | CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic); |
457 | auto GenPtrAlign = |
458 | CharUnits::fromQuantity(CGM.getTarget().getPointerAlign(GenericAS) / 8); |
459 | auto GenPtrSize = |
460 | CharUnits::fromQuantity(CGM.getTarget().getPointerWidth(GenericAS) / 8); |
461 | assert(CGM.getIntSize() <= GenPtrSize); |
462 | assert(CGM.getIntAlign() <= GenPtrAlign); |
463 | assert((2 * CGM.getIntSize()).isMultipleOf(GenPtrAlign)); |
464 | elementTypes.push_back(CGM.IntTy); |
465 | elementTypes.push_back(CGM.IntTy); |
466 | elementTypes.push_back( |
467 | CGM.getOpenCLRuntime() |
468 | .getGenericVoidPointerType()); |
469 | unsigned Offset = |
470 | 2 * CGM.getIntSize().getQuantity() + GenPtrSize.getQuantity(); |
471 | unsigned BlockAlign = GenPtrAlign.getQuantity(); |
472 | if (auto *Helper = |
473 | CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) { |
474 | for (auto I : Helper->getCustomFieldTypes()) { |
475 | |
476 | |
477 | unsigned Align = CGM.getDataLayout().getABITypeAlignment(I); |
478 | if (BlockAlign < Align) |
479 | BlockAlign = Align; |
480 | assert(Offset % Align == 0); |
481 | Offset += CGM.getDataLayout().getTypeAllocSize(I); |
482 | elementTypes.push_back(I); |
483 | } |
484 | } |
485 | info.BlockAlign = CharUnits::fromQuantity(BlockAlign); |
486 | info.BlockSize = CharUnits::fromQuantity(Offset); |
487 | } else { |
488 | |
489 | |
490 | assert(CGM.getIntSize() <= CGM.getPointerSize()); |
491 | assert(CGM.getIntAlign() <= CGM.getPointerAlign()); |
492 | assert((2 * CGM.getIntSize()).isMultipleOf(CGM.getPointerAlign())); |
493 | info.BlockAlign = CGM.getPointerAlign(); |
494 | info.BlockSize = 3 * CGM.getPointerSize() + 2 * CGM.getIntSize(); |
495 | elementTypes.push_back(CGM.VoidPtrTy); |
496 | elementTypes.push_back(CGM.IntTy); |
497 | elementTypes.push_back(CGM.IntTy); |
498 | elementTypes.push_back(CGM.VoidPtrTy); |
499 | elementTypes.push_back(CGM.getBlockDescriptorType()); |
500 | } |
501 | } |
502 | |
503 | static QualType getCaptureFieldType(const CodeGenFunction &CGF, |
504 | const BlockDecl::Capture &CI) { |
505 | const VarDecl *VD = CI.getVariable(); |
506 | |
507 | |
508 | |
509 | if (CGF.BlockInfo && CI.isNested()) |
510 | return CGF.BlockInfo->getCapture(VD).fieldType(); |
511 | if (auto *FD = CGF.LambdaCaptureFields.lookup(VD)) |
512 | return FD->getType(); |
513 | |
514 | |
515 | |
516 | return VD->isNonEscapingByref() ? |
517 | CGF.getContext().getLValueReferenceType(VD->getType()) : VD->getType(); |
518 | } |
519 | |
520 | |
521 | |
522 | static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF, |
523 | CGBlockInfo &info) { |
524 | ASTContext &C = CGM.getContext(); |
525 | const BlockDecl *block = info.getBlockDecl(); |
526 | |
527 | SmallVector<llvm::Type*, 8> elementTypes; |
528 | initializeForBlockHeader(CGM, info, elementTypes); |
529 | bool hasNonConstantCustomFields = false; |
530 | if (auto *OpenCLHelper = |
| 4 | | Assuming 'OpenCLHelper' is null | |
|
| |
531 | CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) |
532 | hasNonConstantCustomFields = |
533 | !OpenCLHelper->areAllCustomFieldValuesConstant(info); |
534 | if (!block->hasCaptures() && !hasNonConstantCustomFields) { |
535 | info.StructureType = |
536 | llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true); |
537 | info.CanBeGlobal = true; |
538 | return; |
539 | } |
540 | else if (C.getLangOpts().ObjC && |
| 6 | | Assuming field 'ObjC' is 0 | |
|
541 | CGM.getLangOpts().getGC() == LangOptions::NonGC) |
542 | info.HasCapturedVariableLayout = true; |
543 | |
544 | |
545 | SmallVector<BlockLayoutChunk, 16> layout; |
546 | layout.reserve(block->capturesCXXThis() + |
547 | (block->capture_end() - block->capture_begin())); |
548 | |
549 | CharUnits maxFieldAlign; |
| 7 | | Calling defaulted default constructor for 'CharUnits' | |
|
| 9 | | Returning from default constructor for 'CharUnits' | |
|
550 | |
551 | |
552 | if (block->capturesCXXThis()) { |
| 10 | | Assuming the condition is false | |
|
| |
553 | assert(CGF && CGF->CurFuncDecl && isa<CXXMethodDecl>(CGF->CurFuncDecl) && |
554 | "Can't capture 'this' outside a method"); |
555 | QualType thisType = cast<CXXMethodDecl>(CGF->CurFuncDecl)->getThisType(); |
556 | |
557 | |
558 | |
559 | llvm::Type *llvmType = CGM.getTypes().ConvertType(thisType); |
560 | auto TInfo = CGM.getContext().getTypeInfoInChars(thisType); |
561 | maxFieldAlign = std::max(maxFieldAlign, TInfo.Align); |
562 | |
563 | layout.push_back(BlockLayoutChunk(TInfo.Align, TInfo.Width, |
564 | Qualifiers::OCL_None, |
565 | nullptr, llvmType, thisType)); |
566 | } |
567 | |
568 | |
569 | for (const auto &CI : block->captures()) { |
| 12 | | Assuming '__begin1' is equal to '__end1' | |
|
570 | const VarDecl *variable = CI.getVariable(); |
571 | |
572 | if (CI.isEscapingByref()) { |
573 | |
574 | info.NeedsCopyDispose = true; |
575 | |
576 | |
577 | CharUnits align = CGM.getPointerAlign(); |
578 | maxFieldAlign = std::max(maxFieldAlign, align); |
579 | |
580 | |
581 | |
582 | assert(CGF && getCaptureFieldType(*CGF, CI) == variable->getType() && |
583 | "capture type differs from the variable type"); |
584 | layout.push_back(BlockLayoutChunk(align, CGM.getPointerSize(), |
585 | Qualifiers::OCL_None, &CI, |
586 | CGM.VoidPtrTy, variable->getType())); |
587 | continue; |
588 | } |
589 | |
590 | |
591 | |
592 | if (llvm::Constant *constant = tryCaptureAsConstant(CGM, CGF, variable)) { |
593 | info.Captures[variable] = CGBlockInfo::Capture::makeConstant(constant); |
594 | continue; |
595 | } |
596 | |
597 | QualType VT = getCaptureFieldType(*CGF, CI); |
598 | |
599 | |
600 | |
601 | Qualifiers::ObjCLifetime lifetime = VT.getObjCLifetime(); |
602 | if (lifetime) { |
603 | switch (lifetime) { |
604 | case Qualifiers::OCL_None: llvm_unreachable("impossible"); |
605 | case Qualifiers::OCL_ExplicitNone: |
606 | case Qualifiers::OCL_Autoreleasing: |
607 | break; |
608 | |
609 | case Qualifiers::OCL_Strong: |
610 | case Qualifiers::OCL_Weak: |
611 | info.NeedsCopyDispose = true; |
612 | } |
613 | |
614 | |
615 | } else if (VT->isObjCRetainableType()) { |
616 | |
617 | |
618 | if (VT->isObjCInertUnsafeUnretainedType()) { |
619 | lifetime = Qualifiers::OCL_ExplicitNone; |
620 | } else { |
621 | info.NeedsCopyDispose = true; |
622 | |
623 | lifetime = Qualifiers::OCL_Strong; |
624 | } |
625 | |
626 | |
627 | } else if (CI.hasCopyExpr()) { |
628 | info.NeedsCopyDispose = true; |
629 | info.HasCXXObject = true; |
630 | if (!VT->getAsCXXRecordDecl()->isExternallyVisible()) |
631 | info.CapturesNonExternalType = true; |
632 | |
633 | |
634 | |
635 | } else if (VT.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct || |
636 | VT.isDestructedType() == QualType::DK_nontrivial_c_struct) { |
637 | info.NeedsCopyDispose = true; |
638 | |
639 | |
640 | } else if (CGM.getLangOpts().CPlusPlus) { |
641 | if (const CXXRecordDecl *record = VT->getAsCXXRecordDecl()) { |
642 | if (!record->hasTrivialDestructor()) { |
643 | info.HasCXXObject = true; |
644 | info.NeedsCopyDispose = true; |
645 | if (!record->isExternallyVisible()) |
646 | info.CapturesNonExternalType = true; |
647 | } |
648 | } |
649 | } |
650 | |
651 | CharUnits size = C.getTypeSizeInChars(VT); |
652 | CharUnits align = C.getDeclAlign(variable); |
653 | |
654 | maxFieldAlign = std::max(maxFieldAlign, align); |
655 | |
656 | llvm::Type *llvmType = |
657 | CGM.getTypes().ConvertTypeForMem(VT); |
658 | |
659 | layout.push_back( |
660 | BlockLayoutChunk(align, size, lifetime, &CI, llvmType, VT)); |
661 | } |
662 | |
663 | |
664 | if (layout.empty()) { |
| |
665 | info.StructureType = |
666 | llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true); |
667 | info.CanBeGlobal = true; |
668 | return; |
669 | } |
670 | |
671 | |
672 | |
673 | |
674 | llvm::stable_sort(layout); |
675 | |
676 | |
677 | info.BlockHeaderForcedGapOffset = info.BlockSize; |
678 | info.BlockHeaderForcedGapSize = CharUnits::Zero(); |
679 | |
680 | CharUnits &blockSize = info.BlockSize; |
681 | info.BlockAlign = std::max(maxFieldAlign, info.BlockAlign); |
682 | |
683 | |
684 | |
685 | CharUnits endAlign = getLowBit(blockSize); |
686 | |
687 | |
688 | |
689 | |
690 | |
691 | |
692 | |
693 | |
694 | |
695 | |
696 | |
697 | |
698 | if (endAlign < maxFieldAlign) { |
| |
699 | SmallVectorImpl<BlockLayoutChunk>::iterator |
700 | li = layout.begin() + 1, le = layout.end(); |
701 | |
702 | |
703 | |
704 | for (; li != le && endAlign < li->Alignment; ++li) |
| 15 | | Assuming 'li' is equal to 'le' | |
|
705 | ; |
706 | |
707 | |
708 | |
709 | if (li != le) { |
| 16 | | Assuming 'li' is equal to 'le' | |
|
| |
710 | SmallVectorImpl<BlockLayoutChunk>::iterator first = li; |
711 | for (; li != le; ++li) { |
712 | assert(endAlign >= li->Alignment); |
713 | |
714 | li->setIndex(info, elementTypes.size(), blockSize); |
715 | elementTypes.push_back(li->Type); |
716 | blockSize += li->Size; |
717 | endAlign = getLowBit(blockSize); |
718 | |
719 | |
720 | if (endAlign >= maxFieldAlign) { |
721 | break; |
722 | } |
723 | } |
724 | |
725 | layout.erase(first, li); |
726 | } |
727 | } |
728 | |
729 | assert(endAlign == getLowBit(blockSize)); |
730 | |
731 | |
732 | |
733 | if (endAlign < maxFieldAlign) { |
| |
734 | CharUnits newBlockSize = blockSize.alignTo(maxFieldAlign); |
| 19 | | Calling 'CharUnits::alignTo' | |
|
735 | CharUnits padding = newBlockSize - blockSize; |
736 | |
737 | |
738 | |
739 | if (blockSize == info.BlockHeaderForcedGapOffset) { |
740 | info.BlockHeaderForcedGapSize = padding; |
741 | } |
742 | |
743 | elementTypes.push_back(llvm::ArrayType::get(CGM.Int8Ty, |
744 | padding.getQuantity())); |
745 | blockSize = newBlockSize; |
746 | endAlign = getLowBit(blockSize); |
747 | } |
748 | |
749 | assert(endAlign >= maxFieldAlign); |
750 | assert(endAlign == getLowBit(blockSize)); |
751 | |
752 | |
753 | |
754 | for (SmallVectorImpl<BlockLayoutChunk>::iterator |
755 | li = layout.begin(), le = layout.end(); li != le; ++li) { |
756 | if (endAlign < li->Alignment) { |
757 | |
758 | |
759 | |
760 | CharUnits padding = li->Alignment - endAlign; |
761 | elementTypes.push_back(llvm::ArrayType::get(CGM.Int8Ty, |
762 | padding.getQuantity())); |
763 | blockSize += padding; |
764 | endAlign = getLowBit(blockSize); |
765 | } |
766 | assert(endAlign >= li->Alignment); |
767 | li->setIndex(info, elementTypes.size(), blockSize); |
768 | elementTypes.push_back(li->Type); |
769 | blockSize += li->Size; |
770 | endAlign = getLowBit(blockSize); |
771 | } |
772 | |
773 | info.StructureType = |
774 | llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true); |
775 | } |
776 | |
777 | |
778 | llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) { |
779 | |
780 | |
781 | if (!blockExpr->getBlockDecl()->hasCaptures()) |
782 | |
783 | |
784 | if (llvm::Constant *Block = CGM.getAddrOfGlobalBlockIfEmitted(blockExpr)) |
785 | return Block; |
786 | |
787 | CGBlockInfo blockInfo(blockExpr->getBlockDecl(), CurFn->getName()); |
788 | computeBlockInfo(CGM, this, blockInfo); |
789 | blockInfo.BlockExpression = blockExpr; |
790 | if (!blockInfo.CanBeGlobal) |
791 | blockInfo.LocalAddress = CreateTempAlloca(blockInfo.StructureType, |
792 | blockInfo.BlockAlign, "block"); |
793 | return EmitBlockLiteral(blockInfo); |
794 | } |
795 | |
796 | llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { |
797 | bool IsOpenCL = CGM.getContext().getLangOpts().OpenCL; |
798 | auto GenVoidPtrTy = |
799 | IsOpenCL ? CGM.getOpenCLRuntime().getGenericVoidPointerType() : VoidPtrTy; |
800 | LangAS GenVoidPtrAddr = IsOpenCL ? LangAS::opencl_generic : LangAS::Default; |
801 | auto GenVoidPtrSize = CharUnits::fromQuantity( |
802 | CGM.getTarget().getPointerWidth( |
803 | CGM.getContext().getTargetAddressSpace(GenVoidPtrAddr)) / |
804 | 8); |
805 | |
806 | bool isLambdaConv = blockInfo.getBlockDecl()->isConversionFromLambda(); |
807 | CodeGenFunction BlockCGF{CGM, true}; |
808 | BlockCGF.SanOpts = SanOpts; |
809 | auto *InvokeFn = BlockCGF.GenerateBlockFunction( |
810 | CurGD, blockInfo, LocalDeclMap, isLambdaConv, blockInfo.CanBeGlobal); |
811 | auto *blockFn = llvm::ConstantExpr::getPointerCast(InvokeFn, GenVoidPtrTy); |
812 | |
813 | |
814 | if (blockInfo.CanBeGlobal) |
815 | return CGM.getAddrOfGlobalBlockIfEmitted(blockInfo.BlockExpression); |
816 | |
817 | |
818 | |
819 | Address blockAddr = blockInfo.LocalAddress; |
820 | assert(blockAddr.isValid() && "block has no address!"); |
821 | |
822 | llvm::Constant *isa; |
823 | llvm::Constant *descriptor; |
824 | BlockFlags flags; |
825 | if (!IsOpenCL) { |
826 | |
827 | |
828 | |
829 | llvm::Constant *blockISA = blockInfo.getBlockDecl()->doesNotEscape() |
830 | ? CGM.getNSConcreteGlobalBlock() |
831 | : CGM.getNSConcreteStackBlock(); |
832 | isa = llvm::ConstantExpr::getBitCast(blockISA, VoidPtrTy); |
833 | |
834 | |
835 | descriptor = buildBlockDescriptor(CGM, blockInfo); |
836 | |
837 | |
838 | flags = BLOCK_HAS_SIGNATURE; |
839 | if (blockInfo.HasCapturedVariableLayout) |
840 | flags |= BLOCK_HAS_EXTENDED_LAYOUT; |
841 | if (blockInfo.needsCopyDisposeHelpers()) |
842 | flags |= BLOCK_HAS_COPY_DISPOSE; |
843 | if (blockInfo.HasCXXObject) |
844 | flags |= BLOCK_HAS_CXX_OBJ; |
845 | if (blockInfo.UsesStret) |
846 | flags |= BLOCK_USE_STRET; |
847 | if (blockInfo.getBlockDecl()->doesNotEscape()) |
848 | flags |= BLOCK_IS_NOESCAPE | BLOCK_IS_GLOBAL; |
849 | } |
850 | |
851 | auto projectField = [&](unsigned index, const Twine &name) -> Address { |
852 | return Builder.CreateStructGEP(blockAddr, index, name); |
853 | }; |
854 | auto storeField = [&](llvm::Value *value, unsigned index, const Twine &name) { |
855 | Builder.CreateStore(value, projectField(index, name)); |
856 | }; |
857 | |
858 | |
859 | { |
860 | |
861 | unsigned index = 0; |
862 | CharUnits offset; |
863 | auto addHeaderField = [&](llvm::Value *value, CharUnits size, |
864 | const Twine &name) { |
865 | storeField(value, index, name); |
866 | offset += size; |
867 | index++; |
868 | }; |
869 | |
870 | if (!IsOpenCL) { |
871 | addHeaderField(isa, getPointerSize(), "block.isa"); |
872 | addHeaderField(llvm::ConstantInt::get(IntTy, flags.getBitMask()), |
873 | getIntSize(), "block.flags"); |
874 | addHeaderField(llvm::ConstantInt::get(IntTy, 0), getIntSize(), |
875 | "block.reserved"); |
876 | } else { |
877 | addHeaderField( |
878 | llvm::ConstantInt::get(IntTy, blockInfo.BlockSize.getQuantity()), |
879 | getIntSize(), "block.size"); |
880 | addHeaderField( |
881 | llvm::ConstantInt::get(IntTy, blockInfo.BlockAlign.getQuantity()), |
882 | getIntSize(), "block.align"); |
883 | } |
884 | addHeaderField(blockFn, GenVoidPtrSize, "block.invoke"); |
885 | if (!IsOpenCL) |
886 | addHeaderField(descriptor, getPointerSize(), "block.descriptor"); |
887 | else if (auto *Helper = |
888 | CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) { |
889 | for (auto I : Helper->getCustomFieldValues(*this, blockInfo)) { |
890 | addHeaderField( |
891 | I.first, |
892 | CharUnits::fromQuantity( |
893 | CGM.getDataLayout().getTypeAllocSize(I.first->getType())), |
894 | I.second); |
895 | } |
896 | } |
897 | } |
898 | |
899 | |
900 | const BlockDecl *blockDecl = blockInfo.getBlockDecl(); |
901 | |
902 | |
903 | if (blockDecl->capturesCXXThis()) { |
904 | Address addr = |
905 | projectField(blockInfo.CXXThisIndex, "block.captured-this.addr"); |
906 | Builder.CreateStore(LoadCXXThis(), addr); |
907 | } |
908 | |
909 | |
910 | for (const auto &CI : blockDecl->captures()) { |
911 | const VarDecl *variable = CI.getVariable(); |
912 | const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable); |
913 | |
914 | |
915 | if (capture.isConstant()) continue; |
916 | |
917 | QualType type = capture.fieldType(); |
918 | |
919 | |
920 | |
921 | Address blockField = projectField(capture.getIndex(), "block.captured"); |
922 | |
923 | |
924 | |
925 | Address src = Address::invalid(); |
926 | |
927 | if (blockDecl->isConversionFromLambda()) { |
928 | |
929 | |
930 | src = Address::invalid(); |
931 | } else if (CI.isEscapingByref()) { |
932 | if (BlockInfo && CI.isNested()) { |
933 | |
934 | const CGBlockInfo::Capture &enclosingCapture = |
935 | BlockInfo->getCapture(variable); |
936 | |
937 | |
938 | src = Builder.CreateStructGEP(LoadBlockStruct(), |
939 | enclosingCapture.getIndex(), |
940 | "block.capture.addr"); |
941 | } else { |
942 | auto I = LocalDeclMap.find(variable); |
943 | assert(I != LocalDeclMap.end()); |
944 | src = I->second; |
945 | } |
946 | } else { |
947 | DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable), |
948 | CI.isNested(), |
949 | type.getNonReferenceType(), VK_LValue, |
950 | SourceLocation()); |
951 | src = EmitDeclRefLValue(&declRef).getAddress(*this); |
952 | }; |
953 | |
954 | |
955 | |
956 | |
957 | |
958 | if (CI.isEscapingByref()) { |
959 | |
960 | llvm::Value *byrefPointer; |
961 | if (CI.isNested()) |
962 | byrefPointer = Builder.CreateLoad(src, "byref.capture"); |
963 | else |
964 | byrefPointer = Builder.CreateBitCast(src.getPointer(), VoidPtrTy); |
965 | |
966 | |
967 | Builder.CreateStore(byrefPointer, blockField); |
968 | |
969 | |
970 | } else if (const Expr *copyExpr = CI.getCopyExpr()) { |
971 | if (blockDecl->isConversionFromLambda()) { |
972 | |
973 | |
974 | AggValueSlot Slot = |
975 | AggValueSlot::forAddr(blockField, Qualifiers(), |
976 | AggValueSlot::IsDestructed, |
977 | AggValueSlot::DoesNotNeedGCBarriers, |
978 | AggValueSlot::IsNotAliased, |
979 | AggValueSlot::DoesNotOverlap); |
980 | EmitAggExpr(copyExpr, Slot); |
981 | } else { |
982 | EmitSynthesizedCXXCopyCtor(blockField, src, copyExpr); |
983 | } |
984 | |
985 | |
986 | } else if (type->isReferenceType()) { |
987 | Builder.CreateStore(src.getPointer(), blockField); |
988 | |
989 | |
990 | } else if (type.isConstQualified() && |
991 | type.getObjCLifetime() == Qualifiers::OCL_Strong && |
992 | CGM.getCodeGenOpts().OptimizationLevel != 0) { |
993 | llvm::Value *value = Builder.CreateLoad(src, "captured"); |
994 | Builder.CreateStore(value, blockField); |
995 | |
996 | |
997 | |
998 | |
999 | |
1000 | |
1001 | |
1002 | |
1003 | } else if (type.getObjCLifetime() == Qualifiers::OCL_Strong && |
1004 | type->isBlockPointerType()) { |
1005 | |
1006 | llvm::Value *value = Builder.CreateLoad(src, "block.captured_block"); |
1007 | value = EmitARCRetainNonBlock(value); |
1008 | |
1009 | |
1010 | Builder.CreateStore(value, blockField); |
1011 | |
1012 | |
1013 | } else { |
1014 | |
1015 | |
1016 | ImplicitParamDecl BlockFieldPseudoVar(getContext(), type, |
1017 | ImplicitParamDecl::Other); |
1018 | |
1019 | |
1020 | |
1021 | DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable), |
1022 | CI.isNested(), |
1023 | type, VK_LValue, SourceLocation()); |
1024 | |
1025 | ImplicitCastExpr l2r(ImplicitCastExpr::OnStack, type, CK_LValueToRValue, |
1026 | &declRef, VK_PRValue, FPOptionsOverride()); |
1027 | |
1028 | |
1029 | |
1030 | EmitExprAsInit(&l2r, &BlockFieldPseudoVar, |
1031 | MakeAddrLValue(blockField, type, AlignmentSource::Decl), |
1032 | false); |
1033 | } |
1034 | |
1035 | |
1036 | if (!blockInfo.NeedsCopyDispose) |
1037 | continue; |
1038 | |
1039 | |
1040 | |
1041 | if (CI.isByRef()) |
1042 | continue; |
1043 | |
1044 | |
1045 | QualType::DestructionKind dtorKind = type.isDestructedType(); |
1046 | if (dtorKind == QualType::DK_none) |
1047 | continue; |
1048 | |
1049 | CodeGenFunction::Destroyer *destroyer; |
1050 | |
1051 | |
1052 | |
1053 | |
1054 | |
1055 | |
1056 | |
1057 | if (type.isConstQualified() && |
1058 | type.getObjCLifetime() == Qualifiers::OCL_Strong && |
1059 | CGM.getCodeGenOpts().OptimizationLevel != 0) { |
1060 | assert(CGM.getLangOpts().ObjCAutoRefCount && |
1061 | "expected ObjC ARC to be enabled"); |
1062 | destroyer = emitARCIntrinsicUse; |
1063 | } else if (dtorKind == QualType::DK_objc_strong_lifetime) { |
1064 | destroyer = destroyARCStrongImprecise; |
1065 | } else { |
1066 | destroyer = getDestroyer(dtorKind); |
1067 | } |
1068 | |
1069 | CleanupKind cleanupKind = NormalCleanup; |
1070 | bool useArrayEHCleanup = needsEHCleanup(dtorKind); |
1071 | if (useArrayEHCleanup) |
1072 | cleanupKind = NormalAndEHCleanup; |
1073 | |
1074 | |
1075 | |
1076 | |
1077 | |
1078 | auto IsBlockDeclInRetExpr = [&]() { |
1079 | auto *EWC = llvm::dyn_cast_or_null<ExprWithCleanups>(RetExpr); |
1080 | if (EWC) |
1081 | for (auto &C : EWC->getObjects()) |
1082 | if (auto *BD = C.dyn_cast<BlockDecl *>()) |
1083 | if (BD == blockDecl) |
1084 | return true; |
1085 | return false; |
1086 | }; |
1087 | |
1088 | if (IsBlockDeclInRetExpr()) |
1089 | pushDestroy(cleanupKind, blockField, type, destroyer, useArrayEHCleanup); |
1090 | else |
1091 | pushLifetimeExtendedDestroy(cleanupKind, blockField, type, destroyer, |
1092 | useArrayEHCleanup); |
1093 | } |
1094 | |
1095 | |
1096 | |
1097 | llvm::Value *result = Builder.CreatePointerCast( |
1098 | blockAddr.getPointer(), ConvertType(blockInfo.getBlockExpr()->getType())); |
1099 | |
1100 | if (IsOpenCL) { |
1101 | CGM.getOpenCLRuntime().recordBlockInfo(blockInfo.BlockExpression, InvokeFn, |
1102 | result); |
1103 | } |
1104 | |
1105 | return result; |
1106 | } |
1107 | |
1108 | |
1109 | llvm::Type *CodeGenModule::getBlockDescriptorType() { |
1110 | if (BlockDescriptorType) |
1111 | return BlockDescriptorType; |
1112 | |
1113 | llvm::Type *UnsignedLongTy = |
1114 | getTypes().ConvertType(getContext().UnsignedLongTy); |
1115 | |
1116 | |
1117 | |
1118 | |
1119 | |
1120 | |
1121 | |
1122 | |
1123 | |
1124 | |
1125 | |
1126 | |
1127 | |
1128 | |
1129 | |
1130 | BlockDescriptorType = llvm::StructType::create( |
1131 | "struct.__block_descriptor", UnsignedLongTy, UnsignedLongTy); |
1132 | |
1133 | |
1134 | unsigned AddrSpace = 0; |
1135 | if (getLangOpts().OpenCL) |
1136 | AddrSpace = getContext().getTargetAddressSpace(LangAS::opencl_constant); |
1137 | BlockDescriptorType = llvm::PointerType::get(BlockDescriptorType, AddrSpace); |
1138 | return BlockDescriptorType; |
1139 | } |
1140 | |
1141 | llvm::Type *CodeGenModule::getGenericBlockLiteralType() { |
1142 | if (GenericBlockLiteralType) |
1143 | return GenericBlockLiteralType; |
1144 | |
1145 | llvm::Type *BlockDescPtrTy = getBlockDescriptorType(); |
1146 | |
1147 | if (getLangOpts().OpenCL) { |
1148 | |
1149 | |
1150 | |
1151 | |
1152 | |
1153 | |
1154 | SmallVector<llvm::Type *, 8> StructFields( |
1155 | {IntTy, IntTy, getOpenCLRuntime().getGenericVoidPointerType()}); |
1156 | if (auto *Helper = getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) { |
1157 | for (auto I : Helper->getCustomFieldTypes()) |
1158 | StructFields.push_back(I); |
1159 | } |
1160 | GenericBlockLiteralType = llvm::StructType::create( |
1161 | StructFields, "struct.__opencl_block_literal_generic"); |
1162 | } else { |
1163 | |
1164 | |
1165 | |
1166 | |
1167 | |
1168 | |
1169 | |
1170 | GenericBlockLiteralType = |
1171 | llvm::StructType::create("struct.__block_literal_generic", VoidPtrTy, |
1172 | IntTy, IntTy, VoidPtrTy, BlockDescPtrTy); |
1173 | } |
1174 | |
1175 | return GenericBlockLiteralType; |
1176 | } |
1177 | |
1178 | RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E, |
1179 | ReturnValueSlot ReturnValue) { |
1180 | const auto *BPT = E->getCallee()->getType()->castAs<BlockPointerType>(); |
1181 | llvm::Value *BlockPtr = EmitScalarExpr(E->getCallee()); |
1182 | llvm::Type *GenBlockTy = CGM.getGenericBlockLiteralType(); |
1183 | llvm::Value *Func = nullptr; |
1184 | QualType FnType = BPT->getPointeeType(); |
1185 | ASTContext &Ctx = getContext(); |
1186 | CallArgList Args; |
1187 | |
1188 | if (getLangOpts().OpenCL) { |
1189 | |
1190 | |
1191 | |
1192 | |
1193 | llvm::Type *GenericVoidPtrTy = |
1194 | CGM.getOpenCLRuntime().getGenericVoidPointerType(); |
1195 | llvm::Value *BlockDescriptor = Builder.CreatePointerCast( |
1196 | BlockPtr, GenericVoidPtrTy); |
1197 | QualType VoidPtrQualTy = Ctx.getPointerType( |
1198 | Ctx.getAddrSpaceQualType(Ctx.VoidTy, LangAS::opencl_generic)); |
1199 | Args.add(RValue::get(BlockDescriptor), VoidPtrQualTy); |
1200 | |
1201 | EmitCallArgs(Args, FnType->getAs<FunctionProtoType>(), E->arguments()); |
1202 | |
1203 | |
1204 | if (!isa<ParmVarDecl>(E->getCalleeDecl())) |
1205 | Func = CGM.getOpenCLRuntime().getInvokeFunction(E->getCallee()); |
1206 | else { |
1207 | llvm::Value *FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 2); |
1208 | Func = Builder.CreateAlignedLoad(GenericVoidPtrTy, FuncPtr, |
1209 | getPointerAlign()); |
1210 | } |
1211 | } else { |
1212 | |
1213 | BlockPtr = Builder.CreatePointerCast( |
1214 | BlockPtr, llvm::PointerType::get(GenBlockTy, 0), "block.literal"); |
1215 | |
1216 | llvm::Value *FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 3); |
1217 | |
1218 | |
1219 | BlockPtr = Builder.CreatePointerCast(BlockPtr, VoidPtrTy); |
1220 | Args.add(RValue::get(BlockPtr), Ctx.VoidPtrTy); |
1221 | |
1222 | EmitCallArgs(Args, FnType->getAs<FunctionProtoType>(), E->arguments()); |
1223 | |
1224 | |
1225 | Func = Builder.CreateAlignedLoad(VoidPtrTy, FuncPtr, getPointerAlign()); |
1226 | } |
1227 | |
1228 | const FunctionType *FuncTy = FnType->castAs<FunctionType>(); |
1229 | const CGFunctionInfo &FnInfo = |
1230 | CGM.getTypes().arrangeBlockFunctionCall(Args, FuncTy); |
1231 | |
1232 | |
1233 | llvm::Type *BlockFTy = CGM.getTypes().GetFunctionType(FnInfo); |
1234 | |
1235 | llvm::Type *BlockFTyPtr = llvm::PointerType::getUnqual(BlockFTy); |
1236 | Func = Builder.CreatePointerCast(Func, BlockFTyPtr); |
1237 | |
1238 | |
1239 | CGCallee Callee(CGCalleeInfo(), Func); |
1240 | |
1241 | |
1242 | return EmitCall(FnInfo, Callee, ReturnValue, Args); |
1243 | } |
1244 | |
1245 | Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable) { |
1246 | assert(BlockInfo && "evaluating block ref without block information?"); |
1247 | const CGBlockInfo::Capture &capture = BlockInfo->getCapture(variable); |
1248 | |
1249 | |
1250 | if (capture.isConstant()) return LocalDeclMap.find(variable)->second; |
1251 | |
1252 | Address addr = Builder.CreateStructGEP(LoadBlockStruct(), capture.getIndex(), |
1253 | "block.capture.addr"); |
1254 | |
1255 | if (variable->isEscapingByref()) { |
1256 | |
1257 | |
1258 | |
1259 | auto &byrefInfo = getBlockByrefInfo(variable); |
1260 | addr = Address(Builder.CreateLoad(addr), byrefInfo.ByrefAlignment); |
1261 | |
1262 | auto byrefPointerType = llvm::PointerType::get(byrefInfo.Type, 0); |
1263 | addr = Builder.CreateBitCast(addr, byrefPointerType, "byref.addr"); |
1264 | |
1265 | addr = emitBlockByrefAddress(addr, byrefInfo, true, |
1266 | variable->getName()); |
1267 | } |
1268 | |
1269 | assert((!variable->isNonEscapingByref() || |
1270 | capture.fieldType()->isReferenceType()) && |
1271 | "the capture field of a non-escaping variable should have a " |
1272 | "reference type"); |
1273 | if (capture.fieldType()->isReferenceType()) |
1274 | addr = EmitLoadOfReference(MakeAddrLValue(addr, capture.fieldType())); |
1275 | |
1276 | return addr; |
1277 | } |
1278 | |
1279 | void CodeGenModule::setAddrOfGlobalBlock(const BlockExpr *BE, |
1280 | llvm::Constant *Addr) { |
1281 | bool Ok = EmittedGlobalBlocks.insert(std::make_pair(BE, Addr)).second; |
1282 | (void)Ok; |
1283 | assert(Ok && "Trying to replace an already-existing global block!"); |
1284 | } |
1285 | |
1286 | llvm::Constant * |
1287 | CodeGenModule::GetAddrOfGlobalBlock(const BlockExpr *BE, |
1288 | StringRef Name) { |
1289 | if (llvm::Constant *Block = getAddrOfGlobalBlockIfEmitted(BE)) |
| 1 | Assuming 'Block' is null | |
|
| |
1290 | return Block; |
1291 | |
1292 | CGBlockInfo blockInfo(BE->getBlockDecl(), Name); |
1293 | blockInfo.BlockExpression = BE; |
1294 | |
1295 | |
1296 | computeBlockInfo(*this, nullptr, blockInfo); |
| 3 | | Calling 'computeBlockInfo' | |
|
1297 | |
1298 | |
1299 | { |
1300 | CodeGenFunction::DeclMapTy LocalDeclMap; |
1301 | CodeGenFunction(*this).GenerateBlockFunction( |
1302 | GlobalDecl(), blockInfo, LocalDeclMap, |
1303 | false, true); |
1304 | } |
1305 | |
1306 | return getAddrOfGlobalBlockIfEmitted(BE); |
1307 | } |
1308 | |
1309 | static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM, |
1310 | const CGBlockInfo &blockInfo, |
1311 | llvm::Constant *blockFn) { |
1312 | assert(blockInfo.CanBeGlobal); |
1313 | |
1314 | |
1315 | |
1316 | assert(!CGM.getAddrOfGlobalBlockIfEmitted(blockInfo.BlockExpression) && |
1317 | "Refusing to re-emit a global block."); |
1318 | |
1319 | |
1320 | ConstantInitBuilder builder(CGM); |
1321 | auto fields = builder.beginStruct(); |
1322 | |
1323 | bool IsOpenCL = CGM.getLangOpts().OpenCL; |
1324 | bool IsWindows = CGM.getTarget().getTriple().isOSWindows(); |
1325 | if (!IsOpenCL) { |
1326 | |
1327 | if (IsWindows) |
1328 | fields.addNullPointer(CGM.Int8PtrPtrTy); |
1329 | else |
1330 | fields.add(CGM.getNSConcreteGlobalBlock()); |
1331 | |
1332 | |
1333 | BlockFlags flags = BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE; |
1334 | if (blockInfo.UsesStret) |
1335 | flags |= BLOCK_USE_STRET; |
1336 | |
1337 | fields.addInt(CGM.IntTy, flags.getBitMask()); |
1338 | |
1339 | |
1340 | fields.addInt(CGM.IntTy, 0); |
1341 | } else { |
1342 | fields.addInt(CGM.IntTy, blockInfo.BlockSize.getQuantity()); |
1343 | fields.addInt(CGM.IntTy, blockInfo.BlockAlign.getQuantity()); |
1344 | } |
1345 | |
1346 | |
1347 | fields.add(blockFn); |
1348 | |
1349 | if (!IsOpenCL) { |
1350 | |
1351 | fields.add(buildBlockDescriptor(CGM, blockInfo)); |
1352 | } else if (auto *Helper = |
1353 | CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) { |
1354 | for (auto I : Helper->getCustomFieldValues(CGM, blockInfo)) { |
1355 | fields.add(I); |
1356 | } |
1357 | } |
1358 | |
1359 | unsigned AddrSpace = 0; |
1360 | if (CGM.getContext().getLangOpts().OpenCL) |
1361 | AddrSpace = CGM.getContext().getTargetAddressSpace(LangAS::opencl_global); |
1362 | |
1363 | llvm::GlobalVariable *literal = fields.finishAndCreateGlobal( |
1364 | "__block_literal_global", blockInfo.BlockAlign, |
1365 | !IsWindows, llvm::GlobalVariable::InternalLinkage, AddrSpace); |
1366 | |
1367 | literal->addAttribute("objc_arc_inert"); |
1368 | |
1369 | |
1370 | |
1371 | if (IsWindows) { |
1372 | auto *Init = llvm::Function::Create(llvm::FunctionType::get(CGM.VoidTy, |
1373 | {}), llvm::GlobalValue::InternalLinkage, ".block_isa_init", |
1374 | &CGM.getModule()); |
1375 | llvm::IRBuilder<> b(llvm::BasicBlock::Create(CGM.getLLVMContext(), "entry", |
1376 | Init)); |
1377 | b.CreateAlignedStore(CGM.getNSConcreteGlobalBlock(), |
1378 | b.CreateStructGEP(literal->getValueType(), literal, 0), |
1379 | CGM.getPointerAlign().getAsAlign()); |
1380 | b.CreateRetVoid(); |
1381 | |
1382 | |
1383 | auto *InitVar = new llvm::GlobalVariable(CGM.getModule(), Init->getType(), |
1384 | true, llvm::GlobalValue::InternalLinkage, |
1385 | Init, ".block_isa_init_ptr"); |
1386 | InitVar->setSection(".CRT$XCLa"); |
1387 | CGM.addUsedGlobal(InitVar); |
1388 | } |
1389 | |
1390 | |
1391 | llvm::Type *RequiredType = |
1392 | CGM.getTypes().ConvertType(blockInfo.getBlockExpr()->getType()); |
1393 | llvm::Constant *Result = |
1394 | llvm::ConstantExpr::getPointerCast(literal, RequiredType); |
1395 | CGM.setAddrOfGlobalBlock(blockInfo.BlockExpression, Result); |
1396 | if (CGM.getContext().getLangOpts().OpenCL) |
1397 | CGM.getOpenCLRuntime().recordBlockInfo( |
1398 | blockInfo.BlockExpression, |
1399 | cast<llvm::Function>(blockFn->stripPointerCasts()), Result); |
1400 | return Result; |
1401 | } |
1402 | |
1403 | void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D, |
1404 | unsigned argNum, |
1405 | llvm::Value *arg) { |
1406 | assert(BlockInfo && "not emitting prologue of block invocation function?!"); |
1407 | |
1408 | |
1409 | |
1410 | Address alloc = CreateMemTemp(D->getType(), D->getName() + ".addr"); |
1411 | Builder.CreateStore(arg, alloc); |
1412 | if (CGDebugInfo *DI = getDebugInfo()) { |
1413 | if (CGM.getCodeGenOpts().hasReducedDebugInfo()) { |
1414 | DI->setLocation(D->getLocation()); |
1415 | DI->EmitDeclareOfBlockLiteralArgVariable( |
1416 | *BlockInfo, D->getName(), argNum, |
1417 | cast<llvm::AllocaInst>(alloc.getPointer()), Builder); |
1418 | } |
1419 | } |
1420 | |
1421 | SourceLocation StartLoc = BlockInfo->getBlockExpr()->getBody()->getBeginLoc(); |
1422 | ApplyDebugLocation Scope(*this, StartLoc); |
1423 | |
1424 | |
1425 | |
1426 | BlockPointer = Builder.CreatePointerCast( |
1427 | arg, |
1428 | BlockInfo->StructureType->getPointerTo( |
1429 | getContext().getLangOpts().OpenCL |
1430 | ? getContext().getTargetAddressSpace(LangAS::opencl_generic) |
1431 | : 0), |
1432 | "block"); |
1433 | } |
1434 | |
1435 | Address CodeGenFunction::LoadBlockStruct() { |
1436 | assert(BlockInfo && "not in a block invocation function!"); |
1437 | assert(BlockPointer && "no block pointer set!"); |
1438 | return Address(BlockPointer, BlockInfo->BlockAlign); |
1439 | } |
1440 | |
1441 | llvm::Function * |
1442 | CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, |
1443 | const CGBlockInfo &blockInfo, |
1444 | const DeclMapTy &ldm, |
1445 | bool IsLambdaConversionToBlock, |
1446 | bool BuildGlobalBlock) { |
1447 | const BlockDecl *blockDecl = blockInfo.getBlockDecl(); |
1448 | |
1449 | CurGD = GD; |
1450 | |
1451 | CurEHLocation = blockInfo.getBlockExpr()->getEndLoc(); |
1452 | |
1453 | BlockInfo = &blockInfo; |
1454 | |
1455 | |
1456 | |
1457 | |
1458 | for (DeclMapTy::const_iterator i = ldm.begin(), e = ldm.end(); i != e; ++i) { |
1459 | const auto *var = dyn_cast<VarDecl>(i->first); |
1460 | if (var && !var->hasLocalStorage()) |
1461 | setAddrOfLocalVar(var, i->second); |
1462 | } |
1463 | |
1464 | |
1465 | |
1466 | |
1467 | FunctionArgList args; |
1468 | |
1469 | |
1470 | |
1471 | QualType selfTy = getContext().VoidPtrTy; |
1472 | |
1473 | |
1474 | |
1475 | |
1476 | |
1477 | if (getLangOpts().OpenCL) |
1478 | selfTy = getContext().getPointerType(getContext().getAddrSpaceQualType( |
1479 | getContext().VoidTy, LangAS::opencl_generic)); |
1480 | |
1481 | IdentifierInfo *II = &CGM.getContext().Idents.get(".block_descriptor"); |
1482 | |
1483 | ImplicitParamDecl SelfDecl(getContext(), const_cast<BlockDecl *>(blockDecl), |
1484 | SourceLocation(), II, selfTy, |
1485 | ImplicitParamDecl::ObjCSelf); |
1486 | args.push_back(&SelfDecl); |
1487 | |
1488 | |
1489 | args.append(blockDecl->param_begin(), blockDecl->param_end()); |
1490 | |
1491 | |
1492 | const FunctionProtoType *fnType = blockInfo.getBlockExpr()->getFunctionType(); |
1493 | const CGFunctionInfo &fnInfo = |
1494 | CGM.getTypes().arrangeBlockFunctionDeclaration(fnType, args); |
1495 | if (CGM.ReturnSlotInterferesWithArgs(fnInfo)) |
1496 | blockInfo.UsesStret = true; |
1497 | |
1498 | llvm::FunctionType *fnLLVMType = CGM.getTypes().GetFunctionType(fnInfo); |
1499 | |
1500 | StringRef name = CGM.getBlockMangledName(GD, blockDecl); |
1501 | llvm::Function *fn = llvm::Function::Create( |
1502 | fnLLVMType, llvm::GlobalValue::InternalLinkage, name, &CGM.getModule()); |
1503 | CGM.SetInternalFunctionAttributes(blockDecl, fn, fnInfo); |
1504 | |
1505 | if (BuildGlobalBlock) { |
1506 | auto GenVoidPtrTy = getContext().getLangOpts().OpenCL |
1507 | ? CGM.getOpenCLRuntime().getGenericVoidPointerType() |
1508 | : VoidPtrTy; |
1509 | buildGlobalBlock(CGM, blockInfo, |
1510 | llvm::ConstantExpr::getPointerCast(fn, GenVoidPtrTy)); |
1511 | } |
1512 | |
1513 | |
1514 | StartFunction(blockDecl, fnType->getReturnType(), fn, fnInfo, args, |
1515 | blockDecl->getLocation(), |
1516 | blockInfo.getBlockExpr()->getBody()->getBeginLoc()); |
1517 | |
1518 | |
1519 | |
1520 | |
1521 | |
1522 | llvm::Value *BlockPointerDbgLoc = BlockPointer; |
1523 | if (CGM.getCodeGenOpts().OptimizationLevel == 0) { |
1524 | |
1525 | Address Alloca = CreateTempAlloca(BlockPointer->getType(), |
1526 | getPointerAlign(), |
1527 | "block.addr"); |
1528 | |
1529 | |
1530 | auto NL = ApplyDebugLocation::CreateEmpty(*this); |
1531 | Builder.CreateStore(BlockPointer, Alloca); |
1532 | BlockPointerDbgLoc = Alloca.getPointer(); |
1533 | } |
1534 | |
1535 | |
1536 | |
1537 | if (blockDecl->capturesCXXThis()) { |
1538 | Address addr = Builder.CreateStructGEP( |
1539 | LoadBlockStruct(), blockInfo.CXXThisIndex, "block.captured-this"); |
1540 | CXXThisValue = Builder.CreateLoad(addr, "this"); |
1541 | } |
1542 | |
1543 | |
1544 | for (const auto &CI : blockDecl->captures()) { |
1545 | const VarDecl *variable = CI.getVariable(); |
1546 | const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable); |
1547 | if (!capture.isConstant()) continue; |
1548 | |
1549 | CharUnits align = getContext().getDeclAlign(variable); |
1550 | Address alloca = |
1551 | CreateMemTemp(variable->getType(), align, "block.captured-const"); |
1552 | |
1553 | Builder.CreateStore(capture.getConstant(), alloca); |
1554 | |
1555 | setAddrOfLocalVar(variable, alloca); |
1556 | } |
1557 | |
1558 | |
1559 | llvm::BasicBlock *entry = Builder.GetInsertBlock(); |
1560 | llvm::BasicBlock::iterator entry_ptr = Builder.GetInsertPoint(); |
1561 | --entry_ptr; |
1562 | |
1563 | if (IsLambdaConversionToBlock) |
1564 | EmitLambdaBlockInvokeBody(); |
1565 | else { |
1566 | PGO.assignRegionCounters(GlobalDecl(blockDecl), fn); |
1567 | incrementProfileCounter(blockDecl->getBody()); |
1568 | EmitStmt(blockDecl->getBody()); |
1569 | } |
1570 | |
1571 | |
1572 | llvm::BasicBlock *resume = Builder.GetInsertBlock(); |
1573 | |
1574 | |
1575 | ++entry_ptr; |
1576 | Builder.SetInsertPoint(entry, entry_ptr); |
1577 | |
1578 | |
1579 | |
1580 | if (CGDebugInfo *DI = getDebugInfo()) { |
1581 | for (const auto &CI : blockDecl->captures()) { |
1582 | const VarDecl *variable = CI.getVariable(); |
1583 | DI->EmitLocation(Builder, variable->getLocation()); |
1584 | |
1585 | if (CGM.getCodeGenOpts().hasReducedDebugInfo()) { |
1586 | const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable); |
1587 | if (capture.isConstant()) { |
1588 | auto addr = LocalDeclMap.find(variable)->second; |
1589 | (void)DI->EmitDeclareOfAutoVariable(variable, addr.getPointer(), |
1590 | Builder); |
1591 | continue; |
1592 | } |
1593 | |
1594 | DI->EmitDeclareOfBlockDeclRefVariable( |
1595 | variable, BlockPointerDbgLoc, Builder, blockInfo, |
1596 | entry_ptr == entry->end() ? nullptr : &*entry_ptr); |
1597 | } |
1598 | } |
1599 | |
1600 | DI->EmitLocation(Builder, |
1601 | cast<CompoundStmt>(blockDecl->getBody())->getRBracLoc()); |
1602 | } |
1603 | |
1604 | |
1605 | if (resume == nullptr) |
1606 | Builder.ClearInsertionPoint(); |
1607 | else |
1608 | Builder.SetInsertPoint(resume); |
1609 | |
1610 | FinishFunction(cast<CompoundStmt>(blockDecl->getBody())->getRBracLoc()); |
1611 | |
1612 | return fn; |
1613 | } |
1614 | |
1615 | static std::pair<BlockCaptureEntityKind, BlockFieldFlags> |
1616 | computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T, |
1617 | const LangOptions &LangOpts) { |
1618 | if (CI.getCopyExpr()) { |
1619 | assert(!CI.isByRef()); |
1620 | |
1621 | return std::make_pair(BlockCaptureEntityKind::CXXRecord, BlockFieldFlags()); |
1622 | } |
1623 | BlockFieldFlags Flags; |
1624 | if (CI.isEscapingByref()) { |
1625 | Flags = BLOCK_FIELD_IS_BYREF; |
1626 | if (T.isObjCGCWeak()) |
1627 | Flags |= BLOCK_FIELD_IS_WEAK; |
1628 | return std::make_pair(BlockCaptureEntityKind::BlockObject, Flags); |
1629 | } |
1630 | |
1631 | Flags = BLOCK_FIELD_IS_OBJECT; |
1632 | bool isBlockPointer = T->isBlockPointerType(); |
1633 | if (isBlockPointer) |
1634 | Flags = BLOCK_FIELD_IS_BLOCK; |
1635 | |
1636 | switch (T.isNonTrivialToPrimitiveCopy()) { |
1637 | case QualType::PCK_Struct: |
1638 | return std::make_pair(BlockCaptureEntityKind::NonTrivialCStruct, |
1639 | BlockFieldFlags()); |
1640 | case QualType::PCK_ARCWeak: |
1641 | |
1642 | return std::make_pair(BlockCaptureEntityKind::ARCWeak, Flags); |
1643 | case QualType::PCK_ARCStrong: |
1644 | |
1645 | |
1646 | |
1647 | |
1648 | return std::make_pair(!isBlockPointer ? BlockCaptureEntityKind::ARCStrong |
1649 | : BlockCaptureEntityKind::BlockObject, |
1650 | Flags); |
1651 | case QualType::PCK_Trivial: |
1652 | case QualType::PCK_VolatileTrivial: { |
1653 | if (!T->isObjCRetainableType()) |
1654 | |
1655 | return std::make_pair(BlockCaptureEntityKind::None, BlockFieldFlags()); |
1656 | |
1657 | |
1658 | Qualifiers QS = T.getQualifiers(); |
1659 | |
1660 | |
1661 | |
1662 | if (!QS.getObjCLifetime() && !LangOpts.ObjCAutoRefCount) |
1663 | return std::make_pair(BlockCaptureEntityKind::BlockObject, Flags); |
1664 | |
1665 | |
1666 | return std::make_pair(BlockCaptureEntityKind::None, BlockFieldFlags()); |
1667 | } |
1668 | } |
1669 | llvm_unreachable("after exhaustive PrimitiveCopyKind switch"); |
1670 | } |
1671 | |
1672 | static std::pair<BlockCaptureEntityKind, BlockFieldFlags> |
1673 | computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T, |
1674 | const LangOptions &LangOpts); |
1675 | |
1676 | |
1677 | static void findBlockCapturedManagedEntities( |
1678 | const CGBlockInfo &BlockInfo, const LangOptions &LangOpts, |
1679 | SmallVectorImpl<BlockCaptureManagedEntity> &ManagedCaptures) { |
1680 | for (const auto &CI : BlockInfo.getBlockDecl()->captures()) { |
1681 | const VarDecl *Variable = CI.getVariable(); |
1682 | const CGBlockInfo::Capture &Capture = BlockInfo.getCapture(Variable); |
1683 | if (Capture.isConstant()) |
1684 | continue; |
1685 | |
1686 | QualType VT = Capture.fieldType(); |
1687 | auto CopyInfo = computeCopyInfoForBlockCapture(CI, VT, LangOpts); |
1688 | auto DisposeInfo = computeDestroyInfoForBlockCapture(CI, VT, LangOpts); |
1689 | if (CopyInfo.first != BlockCaptureEntityKind::None || |
1690 | DisposeInfo.first != BlockCaptureEntityKind::None) |
1691 | ManagedCaptures.emplace_back(CopyInfo.first, DisposeInfo.first, |
1692 | CopyInfo.second, DisposeInfo.second, CI, |
1693 | Capture); |
1694 | } |
1695 | |
1696 | |
1697 | llvm::sort(ManagedCaptures); |
1698 | } |
1699 | |
1700 | namespace { |
1701 | |
1702 | struct CallBlockRelease final : EHScopeStack::Cleanup { |
1703 | Address Addr; |
1704 | BlockFieldFlags FieldFlags; |
1705 | bool LoadBlockVarAddr, CanThrow; |
1706 | |
1707 | CallBlockRelease(Address Addr, BlockFieldFlags Flags, bool LoadValue, |
1708 | bool CT) |
1709 | : Addr(Addr), FieldFlags(Flags), LoadBlockVarAddr(LoadValue), |
1710 | CanThrow(CT) {} |
1711 | |
1712 | void Emit(CodeGenFunction &CGF, Flags flags) override { |
1713 | llvm::Value *BlockVarAddr; |
1714 | if (LoadBlockVarAddr) { |
1715 | BlockVarAddr = CGF.Builder.CreateLoad(Addr); |
1716 | BlockVarAddr = CGF.Builder.CreateBitCast(BlockVarAddr, CGF.VoidPtrTy); |
1717 | } else { |
1718 | BlockVarAddr = Addr.getPointer(); |
1719 | } |
1720 | |
1721 | CGF.BuildBlockRelease(BlockVarAddr, FieldFlags, CanThrow); |
1722 | } |
1723 | }; |
1724 | } |
1725 | |
1726 | |
1727 | bool CodeGenFunction::cxxDestructorCanThrow(QualType T) { |
1728 | if (const auto *RD = T->getAsCXXRecordDecl()) |
1729 | if (const CXXDestructorDecl *DD = RD->getDestructor()) |
1730 | return DD->getType()->castAs<FunctionProtoType>()->canThrow(); |
1731 | return false; |
1732 | } |
1733 | |
1734 | |
1735 | static std::string getBlockCaptureStr(const BlockCaptureManagedEntity &E, |
1736 | CaptureStrKind StrKind, |
1737 | CharUnits BlockAlignment, |
1738 | CodeGenModule &CGM) { |
1739 | std::string Str; |
1740 | ASTContext &Ctx = CGM.getContext(); |
1741 | const BlockDecl::Capture &CI = *E.CI; |
1742 | QualType CaptureTy = CI.getVariable()->getType(); |
1743 | |
1744 | BlockCaptureEntityKind Kind; |
1745 | BlockFieldFlags Flags; |
1746 | |
1747 | |
1748 | |
1749 | assert((StrKind != CaptureStrKind::Merged || |
1750 | (E.CopyKind == E.DisposeKind && E.CopyFlags == E.DisposeFlags)) && |
1751 | "different operations and flags"); |
1752 | |
1753 | if (StrKind == CaptureStrKind::DisposeHelper) { |
1754 | Kind = E.DisposeKind; |
1755 | Flags = E.DisposeFlags; |
1756 | } else { |
1757 | Kind = E.CopyKind; |
1758 | Flags = E.CopyFlags; |
1759 | } |
1760 | |
1761 | switch (Kind) { |
1762 | case BlockCaptureEntityKind::CXXRecord: { |
1763 | Str += "c"; |
1764 | SmallString<256> TyStr; |
1765 | llvm::raw_svector_ostream Out(TyStr); |
1766 | CGM.getCXXABI().getMangleContext().mangleTypeName(CaptureTy, Out); |
1767 | Str += llvm::to_string(TyStr.size()) + TyStr.c_str(); |
1768 | break; |
1769 | } |
1770 | case BlockCaptureEntityKind::ARCWeak: |
1771 | Str += "w"; |
1772 | break; |
1773 | case BlockCaptureEntityKind::ARCStrong: |
1774 | Str += "s"; |
1775 | break; |
1776 | case BlockCaptureEntityKind::BlockObject: { |
1777 | const VarDecl *Var = CI.getVariable(); |
1778 | unsigned F = Flags.getBitMask(); |
1779 | if (F & BLOCK_FIELD_IS_BYREF) { |
1780 | Str += "r"; |
1781 | if (F & BLOCK_FIELD_IS_WEAK) |
1782 | Str += "w"; |
1783 | else { |
1784 | |
1785 | |
1786 | if (StrKind != CaptureStrKind::DisposeHelper) { |
1787 | if (Ctx.getBlockVarCopyInit(Var).canThrow()) |
1788 | Str += "c"; |
1789 | } |
1790 | if (StrKind != CaptureStrKind::CopyHelper) { |
1791 | if (CodeGenFunction::cxxDestructorCanThrow(CaptureTy)) |
1792 | Str += "d"; |
1793 | } |
1794 | } |
1795 | } else { |
1796 | assert((F & BLOCK_FIELD_IS_OBJECT) && "unexpected flag value"); |
1797 | if (F == BLOCK_FIELD_IS_BLOCK) |
1798 | Str += "b"; |
1799 | else |
1800 | Str += "o"; |
1801 | } |
1802 | break; |
1803 | } |
1804 | case BlockCaptureEntityKind::NonTrivialCStruct: { |
1805 | bool IsVolatile = CaptureTy.isVolatileQualified(); |
1806 | CharUnits Alignment = |
1807 | BlockAlignment.alignmentAtOffset(E.Capture->getOffset()); |
1808 | |
1809 | Str += "n"; |
1810 | std::string FuncStr; |
1811 | if (StrKind == CaptureStrKind::DisposeHelper) |
1812 | FuncStr = CodeGenFunction::getNonTrivialDestructorStr( |
1813 | CaptureTy, Alignment, IsVolatile, Ctx); |
1814 | else |
1815 | |
1816 | |
1817 | FuncStr = CodeGenFunction::getNonTrivialCopyConstructorStr( |
1818 | CaptureTy, Alignment, IsVolatile, Ctx); |
1819 | |
1820 | |
1821 | Str += llvm::to_string(FuncStr.size()) + "_" + FuncStr; |
1822 | break; |
1823 | } |
1824 | case BlockCaptureEntityKind::None: |
1825 | break; |
1826 | } |
1827 | |
1828 | return Str; |
1829 | } |
1830 | |
1831 | static std::string getCopyDestroyHelperFuncName( |
1832 | const SmallVectorImpl<BlockCaptureManagedEntity> &Captures, |
1833 | CharUnits BlockAlignment, CaptureStrKind StrKind, CodeGenModule &CGM) { |
1834 | assert((StrKind == CaptureStrKind::CopyHelper || |
1835 | StrKind == CaptureStrKind::DisposeHelper) && |
1836 | "unexpected CaptureStrKind"); |
1837 | std::string Name = StrKind == CaptureStrKind::CopyHelper |
1838 | ? "__copy_helper_block_" |
1839 | : "__destroy_helper_block_"; |
1840 | if (CGM.getLangOpts().Exceptions) |
1841 | Name += "e"; |
1842 | if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions) |
1843 | Name += "a"; |
1844 | Name += llvm::to_string(BlockAlignment.getQuantity()) + "_"; |
1845 | |
1846 | for (const BlockCaptureManagedEntity &E : Captures) { |
1847 | Name += llvm::to_string(E.Capture->getOffset().getQuantity()); |
1848 | Name += getBlockCaptureStr(E, StrKind, BlockAlignment, CGM); |
1849 | } |
1850 | |
1851 | return Name; |
1852 | } |
1853 | |
1854 | static void pushCaptureCleanup(BlockCaptureEntityKind CaptureKind, |
1855 | Address Field, QualType CaptureType, |
1856 | BlockFieldFlags Flags, bool ForCopyHelper, |
1857 | VarDecl *Var, CodeGenFunction &CGF) { |
1858 | bool EHOnly = ForCopyHelper; |
1859 | |
1860 | switch (CaptureKind) { |
1861 | case BlockCaptureEntityKind::CXXRecord: |
1862 | case BlockCaptureEntityKind::ARCWeak: |
1863 | case BlockCaptureEntityKind::NonTrivialCStruct: |
1864 | case BlockCaptureEntityKind::ARCStrong: { |
1865 | if (CaptureType.isDestructedType() && |
1866 | (!EHOnly || CGF.needsEHCleanup(CaptureType.isDestructedType()))) { |
1867 | CodeGenFunction::Destroyer *Destroyer = |
1868 | CaptureKind == BlockCaptureEntityKind::ARCStrong |
1869 | ? CodeGenFunction::destroyARCStrongImprecise |
1870 | : CGF.getDestroyer(CaptureType.isDestructedType()); |
1871 | CleanupKind Kind = |
1872 | EHOnly ? EHCleanup |
1873 | : CGF.getCleanupKind(CaptureType.isDestructedType()); |
1874 | CGF.pushDestroy(Kind, Field, CaptureType, Destroyer, Kind & EHCleanup); |
1875 | } |
1876 | break; |
1877 | } |
1878 | case BlockCaptureEntityKind::BlockObject: { |
1879 | if (!EHOnly || CGF.getLangOpts().Exceptions) { |
1880 | CleanupKind Kind = EHOnly ? EHCleanup : NormalAndEHCleanup; |
1881 | |
1882 | |
1883 | |
1884 | bool CanThrow = |
1885 | !ForCopyHelper && CGF.cxxDestructorCanThrow(CaptureType); |
1886 | CGF.enterByrefCleanup(Kind, Field, Flags, true, |
1887 | CanThrow); |
1888 | } |
1889 | break; |
1890 | } |
1891 | case BlockCaptureEntityKind::None: |
1892 | break; |
1893 | } |
1894 | } |
1895 | |
1896 | static void setBlockHelperAttributesVisibility(bool CapturesNonExternalType, |
1897 | llvm::Function *Fn, |
1898 | const CGFunctionInfo &FI, |
1899 | CodeGenModule &CGM) { |
1900 | if (CapturesNonExternalType) { |
1901 | CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FI); |
1902 | } else { |
1903 | Fn->setVisibility(llvm::GlobalValue::HiddenVisibility); |
1904 | Fn->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); |
1905 | CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, Fn, false); |
1906 | CGM.SetLLVMFunctionAttributesForDefinition(nullptr, Fn); |
1907 | } |
1908 | } |
1909 | |
1910 | |
1911 | |
1912 | |
1913 | |
1914 | |
1915 | |
1916 | |
1917 | llvm::Constant * |
1918 | CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) { |
1919 | SmallVector<BlockCaptureManagedEntity, 4> CopiedCaptures; |
1920 | findBlockCapturedManagedEntities(blockInfo, getLangOpts(), CopiedCaptures); |
1921 | std::string FuncName = |
1922 | getCopyDestroyHelperFuncName(CopiedCaptures, blockInfo.BlockAlign, |
1923 | CaptureStrKind::CopyHelper, CGM); |
1924 | |
1925 | if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(FuncName)) |
1926 | return llvm::ConstantExpr::getBitCast(Func, VoidPtrTy); |
1927 | |
1928 | ASTContext &C = getContext(); |
1929 | |
1930 | QualType ReturnTy = C.VoidTy; |
1931 | |
1932 | FunctionArgList args; |
1933 | ImplicitParamDecl DstDecl(C, C.VoidPtrTy, ImplicitParamDecl::Other); |
1934 | args.push_back(&DstDecl); |
1935 | ImplicitParamDecl SrcDecl(C, C.VoidPtrTy, ImplicitParamDecl::Other); |
1936 | args.push_back(&SrcDecl); |
1937 | |
1938 | const CGFunctionInfo &FI = |
1939 | CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, args); |
1940 | |
1941 | |
1942 | |
1943 | llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI); |
1944 | |
1945 | llvm::Function *Fn = |
1946 | llvm::Function::Create(LTy, llvm::GlobalValue::LinkOnceODRLinkage, |
1947 | FuncName, &CGM.getModule()); |
1948 | if (CGM.supportsCOMDAT()) |
1949 | Fn->setComdat(CGM.getModule().getOrInsertComdat(FuncName)); |
1950 | |
1951 | SmallVector<QualType, 2> ArgTys; |
1952 | ArgTys.push_back(C.VoidPtrTy); |
1953 | ArgTys.push_back(C.VoidPtrTy); |
1954 | |
1955 | setBlockHelperAttributesVisibility(blockInfo.CapturesNonExternalType, Fn, FI, |
1956 | CGM); |
1957 | StartFunction(GlobalDecl(), ReturnTy, Fn, FI, args); |
1958 | auto AL = ApplyDebugLocation::CreateArtificial(*this); |
1959 | |
1960 | llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo(); |
1961 | |
1962 | Address src = GetAddrOfLocalVar(&SrcDecl); |
1963 | src = Address(Builder.CreateLoad(src), blockInfo.BlockAlign); |
1964 | src = Builder.CreateBitCast(src, structPtrTy, "block.source"); |
1965 | |
1966 | Address dst = GetAddrOfLocalVar(&DstDecl); |
1967 | dst = Address(Builder.CreateLoad(dst), blockInfo.BlockAlign); |
1968 | dst = Builder.CreateBitCast(dst, structPtrTy, "block.dest"); |
1969 | |
1970 | for (const auto &CopiedCapture : CopiedCaptures) { |
1971 | const BlockDecl::Capture &CI = *CopiedCapture.CI; |
1972 | const CGBlockInfo::Capture &capture = *CopiedCapture.Capture; |
1973 | QualType captureType = CI.getVariable()->getType(); |
1974 | BlockFieldFlags flags = CopiedCapture.CopyFlags; |
1975 | |
1976 | unsigned index = capture.getIndex(); |
1977 | Address srcField = Builder.CreateStructGEP(src, index); |
1978 | Address dstField = Builder.CreateStructGEP(dst, index); |
1979 | |
1980 | switch (CopiedCapture.CopyKind) { |
1981 | case BlockCaptureEntityKind::CXXRecord: |
1982 | |
1983 | assert(CI.getCopyExpr() && "copy expression for variable is missing"); |
1984 | EmitSynthesizedCXXCopyCtor(dstField, srcField, CI.getCopyExpr()); |
1985 | break; |
1986 | case BlockCaptureEntityKind::ARCWeak: |
1987 | EmitARCCopyWeak(dstField, srcField); |
1988 | break; |
1989 | case BlockCaptureEntityKind::NonTrivialCStruct: { |
1990 | |
1991 | |
1992 | QualType varType = CI.getVariable()->getType(); |
1993 | callCStructCopyConstructor(MakeAddrLValue(dstField, varType), |
1994 | MakeAddrLValue(srcField, varType)); |
1995 | break; |
1996 | } |
1997 | case BlockCaptureEntityKind::ARCStrong: { |
1998 | llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src"); |
1999 | |
2000 | |
2001 | |
2002 | if (CGM.getCodeGenOpts().OptimizationLevel == 0) { |
2003 | auto *ty = cast<llvm::PointerType>(srcValue->getType()); |
2004 | llvm::Value *null = llvm::ConstantPointerNull::get(ty); |
2005 | Builder.CreateStore(null, dstField); |
2006 | EmitARCStoreStrongCall(dstField, srcValue, true); |
2007 | |
2008 | |
2009 | |
2010 | |
2011 | } else { |
2012 | EmitARCRetainNonBlock(srcValue); |
2013 | |
2014 | |
2015 | |
2016 | |
2017 | if (!needsEHCleanup(captureType.isDestructedType())) |
2018 | cast<llvm::Instruction>(dstField.getPointer())->eraseFromParent(); |
2019 | } |
2020 | break; |
2021 | } |
2022 | case BlockCaptureEntityKind::BlockObject: { |
2023 | llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src"); |
2024 | srcValue = Builder.CreateBitCast(srcValue, VoidPtrTy); |
2025 | llvm::Value *dstAddr = |
2026 | Builder.CreateBitCast(dstField.getPointer(), VoidPtrTy); |
2027 | llvm::Value *args[] = { |
2028 | dstAddr, srcValue, llvm::ConstantInt::get(Int32Ty, flags.getBitMask()) |
2029 | }; |
2030 | |
2031 | if (CI.isByRef() && C.getBlockVarCopyInit(CI.getVariable()).canThrow()) |
2032 | EmitRuntimeCallOrInvoke(CGM.getBlockObjectAssign(), args); |
2033 | else |
2034 | EmitNounwindRuntimeCall(CGM.getBlockObjectAssign(), args); |
2035 | break; |
2036 | } |
2037 | case BlockCaptureEntityKind::None: |
2038 | continue; |
2039 | } |
2040 | |
2041 | |
2042 | |
2043 | pushCaptureCleanup(CopiedCapture.CopyKind, dstField, captureType, flags, |
2044 | true, CI.getVariable(), *this); |
2045 | } |
2046 | |
2047 | FinishFunction(); |
2048 | |
2049 | return llvm::ConstantExpr::getBitCast(Fn, VoidPtrTy); |
2050 | } |
2051 | |
2052 | static BlockFieldFlags |
2053 | getBlockFieldFlagsForObjCObjectPointer(const BlockDecl::Capture &CI, |
2054 | QualType T) { |
2055 | BlockFieldFlags Flags = BLOCK_FIELD_IS_OBJECT; |
2056 | if (T->isBlockPointerType()) |
2057 | Flags = BLOCK_FIELD_IS_BLOCK; |
2058 | return Flags; |
2059 | } |
2060 | |
2061 | static std::pair<BlockCaptureEntityKind, BlockFieldFlags> |
2062 | computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T, |
2063 | const LangOptions &LangOpts) { |
2064 | if (CI.isEscapingByref()) { |
2065 | BlockFieldFlags Flags = BLOCK_FIELD_IS_BYREF; |
2066 | if (T.isObjCGCWeak()) |
2067 | Flags |= BLOCK_FIELD_IS_WEAK; |
2068 | return std::make_pair(BlockCaptureEntityKind::BlockObject, Flags); |
2069 | } |
2070 | |
2071 | switch (T.isDestructedType()) { |
2072 | case QualType::DK_cxx_destructor: |
2073 | return std::make_pair(BlockCaptureEntityKind::CXXRecord, BlockFieldFlags()); |
2074 | case QualType::DK_objc_strong_lifetime: |
2075 | |
2076 | |
2077 | return std::make_pair(BlockCaptureEntityKind::ARCStrong, |
2078 | getBlockFieldFlagsForObjCObjectPointer(CI, T)); |
2079 | case QualType::DK_objc_weak_lifetime: |
2080 | |
2081 | return std::make_pair(BlockCaptureEntityKind::ARCWeak, |
2082 | getBlockFieldFlagsForObjCObjectPointer(CI, T)); |
2083 | case QualType::DK_nontrivial_c_struct: |
2084 | return std::make_pair(BlockCaptureEntityKind::NonTrivialCStruct, |
2085 | BlockFieldFlags()); |
2086 | case QualType::DK_none: { |
2087 | |
2088 | if (T->isObjCRetainableType() && !T.getQualifiers().hasObjCLifetime() && |
2089 | !LangOpts.ObjCAutoRefCount) |
2090 | return std::make_pair(BlockCaptureEntityKind::BlockObject, |
2091 | getBlockFieldFlagsForObjCObjectPointer(CI, T)); |
2092 | |
2093 | return std::make_pair(BlockCaptureEntityKind::None, BlockFieldFlags()); |
2094 | } |
2095 | } |
2096 | llvm_unreachable("after exhaustive DestructionKind switch"); |
2097 | } |
2098 | |
2099 | |
2100 | |
2101 | |
2102 | |
2103 | |
2104 | |
2105 | |
2106 | llvm::Constant * |
2107 | CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) { |
2108 | SmallVector<BlockCaptureManagedEntity, 4> DestroyedCaptures; |
2109 | findBlockCapturedManagedEntities(blockInfo, getLangOpts(), DestroyedCaptures); |
2110 | std::string FuncName = |
2111 | getCopyDestroyHelperFuncName(DestroyedCaptures, blockInfo.BlockAlign, |
2112 | CaptureStrKind::DisposeHelper, CGM); |
2113 | |
2114 | if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(FuncName)) |
2115 | return llvm::ConstantExpr::getBitCast(Func, VoidPtrTy); |
2116 | |
2117 | ASTContext &C = getContext(); |
2118 | |
2119 | QualType ReturnTy = C.VoidTy; |
2120 | |
2121 | FunctionArgList args; |
2122 | ImplicitParamDecl SrcDecl(C, C.VoidPtrTy, ImplicitParamDecl::Other); |
2123 | args.push_back(&SrcDecl); |
2124 | |
2125 | const CGFunctionInfo &FI = |
2126 | CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, args); |
2127 | |
2128 | |
2129 | |
2130 | llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI); |
2131 | |
2132 | llvm::Function *Fn = |
2133 | llvm::Function::Create(LTy, llvm::GlobalValue::LinkOnceODRLinkage, |
2134 | FuncName, &CGM.getModule()); |
2135 | if (CGM.supportsCOMDAT()) |
2136 | Fn->setComdat(CGM.getModule().getOrInsertComdat(FuncName)); |
2137 | |
2138 | SmallVector<QualType, 1> ArgTys; |
2139 | ArgTys.push_back(C.VoidPtrTy); |
2140 | |
2141 | setBlockHelperAttributesVisibility(blockInfo.CapturesNonExternalType, Fn, FI, |
2142 | CGM); |
2143 | StartFunction(GlobalDecl(), ReturnTy, Fn, FI, args); |
2144 | markAsIgnoreThreadCheckingAtRuntime(Fn); |
2145 | |
2146 | auto AL = ApplyDebugLocation::CreateArtificial(*this); |
2147 | |
2148 | llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo(); |
2149 | |
2150 | Address src = GetAddrOfLocalVar(&SrcDecl); |
2151 | src = Address(Builder.CreateLoad(src), blockInfo.BlockAlign); |
2152 | src = Builder.CreateBitCast(src, structPtrTy, "block"); |
2153 | |
2154 | CodeGenFunction::RunCleanupsScope cleanups(*this); |
2155 | |
2156 | for (const auto &DestroyedCapture : DestroyedCaptures) { |
2157 | const BlockDecl::Capture &CI = *DestroyedCapture.CI; |
2158 | const CGBlockInfo::Capture &capture = *DestroyedCapture.Capture; |
2159 | BlockFieldFlags flags = DestroyedCapture.DisposeFlags; |
2160 | |
2161 | Address srcField = Builder.CreateStructGEP(src, capture.getIndex()); |
2162 | |
2163 | pushCaptureCleanup(DestroyedCapture.DisposeKind, srcField, |
2164 | CI.getVariable()->getType(), flags, |
2165 | false, CI.getVariable(), *this); |
2166 | } |
2167 | |
2168 | cleanups.ForceCleanup(); |
2169 | |
2170 | FinishFunction(); |
2171 | |
2172 | return llvm::ConstantExpr::getBitCast(Fn, VoidPtrTy); |
2173 | } |
2174 | |
2175 | namespace { |
2176 | |
2177 | |
2178 | class ObjectByrefHelpers final : public BlockByrefHelpers { |
2179 | BlockFieldFlags Flags; |
2180 | |
2181 | public: |
2182 | ObjectByrefHelpers(CharUnits alignment, BlockFieldFlags flags) |
2183 | : BlockByrefHelpers(alignment), Flags(flags) {} |
2184 | |
2185 | void emitCopy(CodeGenFunction &CGF, Address destField, |
2186 | Address srcField) override { |
2187 | destField = CGF.Builder.CreateBitCast(destField, CGF.VoidPtrTy); |
2188 | |
2189 | srcField = CGF.Builder.CreateBitCast(srcField, CGF.VoidPtrPtrTy); |
2190 | llvm::Value *srcValue = CGF.Builder.CreateLoad(srcField); |
2191 | |
2192 | unsigned flags = (Flags | BLOCK_BYREF_CALLER).getBitMask(); |
2193 | |
2194 | llvm::Value *flagsVal = llvm::ConstantInt::get(CGF.Int32Ty, flags); |
2195 | llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign(); |
2196 | |
2197 | llvm::Value *args[] = { destField.getPointer(), srcValue, flagsVal }; |
2198 | CGF.EmitNounwindRuntimeCall(fn, args); |
2199 | } |
2200 | |
2201 | void emitDispose(CodeGenFunction &CGF, Address field) override { |
2202 | field = CGF.Builder.CreateBitCast(field, CGF.Int8PtrTy->getPointerTo(0)); |
2203 | llvm::Value *value = CGF.Builder.CreateLoad(field); |
2204 | |
2205 | CGF.BuildBlockRelease(value, Flags | BLOCK_BYREF_CALLER, false); |
2206 | } |
2207 | |
2208 | void profileImpl(llvm::FoldingSetNodeID &id) const override { |
2209 | id.AddInteger(Flags.getBitMask()); |
2210 | } |
2211 | }; |
2212 | |
2213 | |
2214 | class ARCWeakByrefHelpers final : public BlockByrefHelpers { |
2215 | public: |
2216 | ARCWeakByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {} |
2217 | |
2218 | void emitCopy(CodeGenFunction &CGF, Address destField, |
2219 | Address srcField) override { |
2220 | CGF.EmitARCMoveWeak(destField, srcField); |
2221 | } |
2222 | |
2223 | void emitDispose(CodeGenFunction &CGF, Address field) override { |
2224 | CGF.EmitARCDestroyWeak(field); |
2225 | } |
2226 | |
2227 | void profileImpl(llvm::FoldingSetNodeID &id) const override { |
2228 | |
2229 | id.AddInteger(0); |
2230 | } |
2231 | }; |
2232 | |
2233 | |
2234 | |
2235 | class ARCStrongByrefHelpers final : public BlockByrefHelpers { |
2236 | public: |
2237 | ARCStrongByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {} |
2238 | |
2239 | void emitCopy(CodeGenFunction &CGF, Address destField, |
2240 | Address srcField) override { |
2241 | |
2242 | |
2243 | |
2244 | llvm::Value *value = CGF.Builder.CreateLoad(srcField); |
2245 | |
2246 | llvm::Value *null = |
2247 | llvm::ConstantPointerNull::get(cast<llvm::PointerType>(value->getType())); |
2248 | |
2249 | if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0) { |
2250 | CGF.Builder.CreateStore(null, destField); |
2251 | CGF.EmitARCStoreStrongCall(destField, value, true); |
2252 | CGF.EmitARCStoreStrongCall(srcField, null, true); |
2253 | return; |
2254 | } |
2255 | CGF.Builder.CreateStore(value, destField); |
2256 | CGF.Builder.CreateStore(null, srcField); |
2257 | } |
2258 | |
2259 | void emitDispose(CodeGenFunction &CGF, Address field) override { |
2260 | CGF.EmitARCDestroyStrong(field, ARCImpreciseLifetime); |
2261 | } |
2262 | |
2263 | void profileImpl(llvm::FoldingSetNodeID &id) const override { |
2264 | |
2265 | id.AddInteger(1); |
2266 | } |
2267 | }; |
2268 | |
2269 | |
2270 | |
2271 | class ARCStrongBlockByrefHelpers final : public BlockByrefHelpers { |
2272 | public: |
2273 | ARCStrongBlockByrefHelpers(CharUnits alignment) |
2274 | : BlockByrefHelpers(alignment) {} |
2275 | |
2276 | void emitCopy(CodeGenFunction &CGF, Address destField, |
2277 | Address srcField) override { |
2278 | |
2279 | |
2280 | |
2281 | llvm::Value *oldValue = CGF.Builder.CreateLoad(srcField); |
2282 | llvm::Value *copy = CGF.EmitARCRetainBlock(oldValue, true); |
2283 | CGF.Builder.CreateStore(copy, destField); |
2284 | } |
2285 | |
2286 | void emitDispose(CodeGenFunction &CGF, Address field) override { |
2287 | CGF.EmitARCDestroyStrong(field, ARCImpreciseLifetime); |
2288 | } |
2289 | |
2290 | void profileImpl(llvm::FoldingSetNodeID &id) const override { |
2291 | |
2292 | id.AddInteger(2); |
2293 | } |
2294 | }; |
2295 | |
2296 | |
2297 | |
2298 | class CXXByrefHelpers final : public BlockByrefHelpers { |
2299 | QualType VarType; |
2300 | const Expr *CopyExpr; |
2301 | |
2302 | public: |
2303 | CXXByrefHelpers(CharUnits alignment, QualType type, |
2304 | const Expr *copyExpr) |
2305 | : BlockByrefHelpers(alignment), VarType(type), CopyExpr(copyExpr) {} |
2306 | |
2307 | bool needsCopy() const override { return CopyExpr != nullptr; } |
2308 | void emitCopy(CodeGenFunction &CGF, Address destField, |
2309 | Address srcField) override { |
2310 | if (!CopyExpr) return; |
2311 | CGF.EmitSynthesizedCXXCopyCtor(destField, srcField, CopyExpr); |
2312 | } |
2313 | |
2314 | void emitDispose(CodeGenFunction &CGF, Address field) override { |
2315 | EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin(); |
2316 | CGF.PushDestructorCleanup(VarType, field); |
2317 | CGF.PopCleanupBlocks(cleanupDepth); |
2318 | } |
2319 | |
2320 | void profileImpl(llvm::FoldingSetNodeID &id) const override { |
2321 | id.AddPointer(VarType.getCanonicalType().getAsOpaquePtr()); |
2322 | } |
2323 | }; |
2324 | |
2325 | |
2326 | |
2327 | class NonTrivialCStructByrefHelpers final : public BlockByrefHelpers { |
2328 | QualType VarType; |
2329 | |
2330 | public: |
2331 | NonTrivialCStructByrefHelpers(CharUnits alignment, QualType type) |
2332 | : BlockByrefHelpers(alignment), VarType(type) {} |
2333 | |
2334 | void emitCopy(CodeGenFunction &CGF, Address destField, |
2335 | Address srcField) override { |
2336 | CGF.callCStructMoveConstructor(CGF.MakeAddrLValue(destField, VarType), |
2337 | CGF.MakeAddrLValue(srcField, VarType)); |
2338 | } |
2339 | |
2340 | bool needsDispose() const override { |
2341 | return VarType.isDestructedType(); |
2342 | } |
2343 | |
2344 | void emitDispose(CodeGenFunction &CGF, Address field) override { |
2345 | EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin(); |
2346 | CGF.pushDestroy(VarType.isDestructedType(), field, VarType); |
2347 | CGF.PopCleanupBlocks(cleanupDepth); |
2348 | } |
2349 | |
2350 | void profileImpl(llvm::FoldingSetNodeID &id) const override { |
2351 | id.AddPointer(VarType.getCanonicalType().getAsOpaquePtr()); |
2352 | } |
2353 | }; |
2354 | } |
2355 | |
2356 | static llvm::Constant * |
2357 | generateByrefCopyHelper(CodeGenFunction &CGF, const BlockByrefInfo &byrefInfo, |
2358 | BlockByrefHelpers &generator) { |
2359 | ASTContext &Context = CGF.getContext(); |
2360 | |
2361 | QualType ReturnTy = Context.VoidTy; |
2362 | |
2363 | FunctionArgList args; |
2364 | ImplicitParamDecl Dst(Context, Context.VoidPtrTy, ImplicitParamDecl::Other); |
2365 | args.push_back(&Dst); |
2366 | |
2367 | ImplicitParamDecl Src(Context, Context.VoidPtrTy, ImplicitParamDecl::Other); |
2368 | args.push_back(&Src); |
2369 | |
2370 | const CGFunctionInfo &FI = |
2371 | CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, args); |
2372 | |
2373 | llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(FI); |
2374 | |
2375 | |
2376 | |
2377 | llvm::Function *Fn = |
2378 | llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage, |
2379 | "__Block_byref_object_copy_", &CGF.CGM.getModule()); |
2380 | |
2381 | SmallVector<QualType, 2> ArgTys; |
2382 | ArgTys.push_back(Context.VoidPtrTy); |
2383 | ArgTys.push_back(Context.VoidPtrTy); |
2384 | |
2385 | CGF.CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FI); |
2386 | |
2387 | CGF.StartFunction(GlobalDecl(), ReturnTy, Fn, FI, args); |
2388 | |
2389 | auto AL = ApplyDebugLocation::CreateArtificial(CGF); |
2390 | |
2391 | if (generator.needsCopy()) { |
2392 | llvm::Type *byrefPtrType = byrefInfo.Type->getPointerTo(0); |
2393 | |
2394 | |
2395 | Address destField = CGF.GetAddrOfLocalVar(&Dst); |
2396 | destField = Address(CGF.Builder.CreateLoad(destField), |
2397 | byrefInfo.ByrefAlignment); |
2398 | destField = CGF.Builder.CreateBitCast(destField, byrefPtrType); |
2399 | destField = CGF.emitBlockByrefAddress(destField, byrefInfo, false, |
2400 | "dest-object"); |
2401 | |
2402 | |
2403 | Address srcField = CGF.GetAddrOfLocalVar(&Src); |
2404 | srcField = Address(CGF.Builder.CreateLoad(srcField), |
2405 | byrefInfo.ByrefAlignment); |
2406 | srcField = CGF.Builder.CreateBitCast(srcField, byrefPtrType); |
2407 | srcField = CGF.emitBlockByrefAddress(srcField, byrefInfo, false, |
2408 | "src-object"); |
2409 | |
2410 | generator.emitCopy(CGF, destField, srcField); |
2411 | } |
2412 | |
2413 | CGF.FinishFunction(); |
2414 | |
2415 | return llvm::ConstantExpr::getBitCast(Fn, CGF.Int8PtrTy); |
2416 | } |
2417 | |
2418 | |
2419 | static llvm::Constant *buildByrefCopyHelper(CodeGenModule &CGM, |
2420 | const BlockByrefInfo &byrefInfo, |
2421 | BlockByrefHelpers &generator) { |
2422 | CodeGenFunction CGF(CGM); |
2423 | return generateByrefCopyHelper(CGF, byrefInfo, generator); |
2424 | } |
2425 | |
2426 | |
2427 | static llvm::Constant * |
2428 | generateByrefDisposeHelper(CodeGenFunction &CGF, |
2429 | const BlockByrefInfo &byrefInfo, |
2430 | BlockByrefHelpers &generator) { |
2431 | ASTContext &Context = CGF.getContext(); |
2432 | QualType R = Context.VoidTy; |
2433 | |
2434 | FunctionArgList args; |
2435 | ImplicitParamDecl Src(CGF.getContext(), Context.VoidPtrTy, |
2436 | ImplicitParamDecl::Other); |
2437 | args.push_back(&Src); |
2438 | |
2439 | const CGFunctionInfo &FI = |
2440 | CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(R, args); |
2441 | |
2442 | llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(FI); |
2443 | |
2444 | |
2445 | |
2446 | llvm::Function *Fn = |
2447 | llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage, |
2448 | "__Block_byref_object_dispose_", |
2449 | &CGF.CGM.getModule()); |
2450 | |
2451 | SmallVector<QualType, 1> ArgTys; |
2452 | ArgTys.push_back(Context.VoidPtrTy); |
2453 | |
2454 | CGF.CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FI); |
2455 | |
2456 | CGF.StartFunction(GlobalDecl(), R, Fn, FI, args); |
2457 | |
2458 | auto AL = ApplyDebugLocation::CreateArtificial(CGF); |
2459 | |
2460 | if (generator.needsDispose()) { |
2461 | Address addr = CGF.GetAddrOfLocalVar(&Src); |
2462 | addr = Address(CGF.Builder.CreateLoad(addr), byrefInfo.ByrefAlignment); |
2463 | auto byrefPtrType = byrefInfo.Type->getPointerTo(0); |
2464 | addr = CGF.Builder.CreateBitCast(addr, byrefPtrType); |
2465 | addr = CGF.emitBlockByrefAddress(addr, byrefInfo, false, "object"); |
2466 | |
2467 | generator.emitDispose(CGF, addr); |
2468 | } |
2469 | |
2470 | CGF.FinishFunction(); |
2471 | |
2472 | return llvm::ConstantExpr::getBitCast(Fn, CGF.Int8PtrTy); |
2473 | } |
2474 | |
2475 | |
2476 | static llvm::Constant *buildByrefDisposeHelper(CodeGenModule &CGM, |
2477 | const BlockByrefInfo &byrefInfo, |
2478 | BlockByrefHelpers &generator) { |
2479 | CodeGenFunction CGF(CGM); |
2480 | return generateByrefDisposeHelper(CGF, byrefInfo, generator); |
2481 | } |
2482 | |
2483 | |
2484 | |
2485 | template <class T> |
2486 | static T *buildByrefHelpers(CodeGenModule &CGM, const BlockByrefInfo &byrefInfo, |
2487 | T &&generator) { |
2488 | llvm::FoldingSetNodeID id; |
2489 | generator.Profile(id); |
2490 | |
2491 | void *insertPos; |
2492 | BlockByrefHelpers *node |
2493 | = CGM.ByrefHelpersCache.FindNodeOrInsertPos(id, insertPos); |
2494 | if (node) return static_cast<T*>(node); |
2495 | |
2496 | generator.CopyHelper = buildByrefCopyHelper(CGM, byrefInfo, generator); |
2497 | generator.DisposeHelper = buildByrefDisposeHelper(CGM, byrefInfo, generator); |
2498 | |
2499 | T *copy = new (CGM.getContext()) T(std::forward<T>(generator)); |
2500 | CGM.ByrefHelpersCache.InsertNode(copy, insertPos); |
2501 | return copy; |
2502 | } |
2503 | |
2504 | |
2505 | |
2506 | |
2507 | BlockByrefHelpers * |
2508 | CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType, |
2509 | const AutoVarEmission &emission) { |
2510 | const VarDecl &var = *emission.Variable; |
2511 | assert(var.isEscapingByref() && |
2512 | "only escaping __block variables need byref helpers"); |
2513 | |
2514 | QualType type = var.getType(); |
2515 | |
2516 | auto &byrefInfo = getBlockByrefInfo(&var); |
2517 | |
2518 | |
2519 | |
2520 | CharUnits valueAlignment = |
2521 | byrefInfo.ByrefAlignment.alignmentAtOffset(byrefInfo.FieldOffset); |
2522 | |
2523 | if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) { |
2524 | const Expr *copyExpr = |
2525 | CGM.getContext().getBlockVarCopyInit(&var).getCopyExpr(); |
2526 | if (!copyExpr && record->hasTrivialDestructor()) return nullptr; |
2527 | |
2528 | return ::buildByrefHelpers( |
2529 | CGM, byrefInfo, CXXByrefHelpers(valueAlignment, type, copyExpr)); |
2530 | } |
2531 | |
2532 | |
2533 | |
2534 | if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct || |
2535 | type.isDestructedType() == QualType::DK_nontrivial_c_struct) |
2536 | return ::buildByrefHelpers( |
2537 | CGM, byrefInfo, NonTrivialCStructByrefHelpers(valueAlignment, type)); |
2538 | |
2539 | |
2540 | |
2541 | if (!type->isObjCRetainableType()) return nullptr; |
2542 | |
2543 | Qualifiers qs = type.getQualifiers(); |
2544 | |
2545 | |
2546 | if (Qualifiers::ObjCLifetime lifetime = qs.getObjCLifetime()) { |
2547 | switch (lifetime) { |
2548 | case Qualifiers::OCL_None: llvm_unreachable("impossible"); |
2549 | |
2550 | |
2551 | case Qualifiers::OCL_ExplicitNone: |
2552 | case Qualifiers::OCL_Autoreleasing: |
2553 | return nullptr; |
2554 | |
2555 | |
2556 | |
2557 | case Qualifiers::OCL_Weak: |
2558 | return ::buildByrefHelpers(CGM, byrefInfo, |
2559 | ARCWeakByrefHelpers(valueAlignment)); |
2560 | |
2561 | |
2562 | case Qualifiers::OCL_Strong: |
2563 | |
2564 | |
2565 | if (type->isBlockPointerType()) { |
2566 | return ::buildByrefHelpers(CGM, byrefInfo, |
2567 | ARCStrongBlockByrefHelpers(valueAlignment)); |
2568 | |
2569 | |
2570 | |
2571 | } else { |
2572 | return ::buildByrefHelpers(CGM, byrefInfo, |
2573 | ARCStrongByrefHelpers(valueAlignment)); |
2574 | } |
2575 | } |
2576 | llvm_unreachable("fell out of lifetime switch!"); |
2577 | } |
2578 | |
2579 | BlockFieldFlags flags; |
2580 | if (type->isBlockPointerType()) { |
2581 | flags |= BLOCK_FIELD_IS_BLOCK; |
2582 | } else if (CGM.getContext().isObjCNSObjectType(type) || |
2583 | type->isObjCObjectPointerType()) { |
2584 | flags |= BLOCK_FIELD_IS_OBJECT; |
2585 | } else { |
2586 | return nullptr; |
2587 | } |
2588 | |
2589 | if (type.isObjCGCWeak()) |
2590 | flags |= BLOCK_FIELD_IS_WEAK; |
2591 | |
2592 | return ::buildByrefHelpers(CGM, byrefInfo, |
2593 | ObjectByrefHelpers(valueAlignment, flags)); |
2594 | } |
2595 | |
2596 | Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr, |
2597 | const VarDecl *var, |
2598 | bool followForward) { |
2599 | auto &info = getBlockByrefInfo(var); |
2600 | return emitBlockByrefAddress(baseAddr, info, followForward, var->getName()); |
2601 | } |
2602 | |
2603 | Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr, |
2604 | const BlockByrefInfo &info, |
2605 | bool followForward, |
2606 | const llvm::Twine &name) { |
2607 | |
2608 | if (followForward) { |
2609 | Address forwardingAddr = Builder.CreateStructGEP(baseAddr, 1, "forwarding"); |
2610 | baseAddr = Address(Builder.CreateLoad(forwardingAddr), info.ByrefAlignment); |
2611 | } |
2612 | |
2613 | return Builder.CreateStructGEP(baseAddr, info.FieldIndex, name); |
2614 | } |
2615 | |
2616 | |
2617 | |
2618 | |
2619 | |
2620 | |
2621 | |
2622 | |
2623 | |
2624 | |
2625 | |
2626 | |
2627 | |
2628 | |
2629 | |
2630 | |
2631 | const BlockByrefInfo &CodeGenFunction::getBlockByrefInfo(const VarDecl *D) { |
2632 | auto it = BlockByrefInfos.find(D); |
2633 | if (it != BlockByrefInfos.end()) |
2634 | return it->second; |
2635 | |
2636 | llvm::StructType *byrefType = |
2637 | llvm::StructType::create(getLLVMContext(), |
2638 | "struct.__block_byref_" + D->getNameAsString()); |
2639 | |
2640 | QualType Ty = D->getType(); |
2641 | |
2642 | CharUnits size; |
2643 | SmallVector<llvm::Type *, 8> types; |
2644 | |
2645 | |
2646 | types.push_back(Int8PtrTy); |
2647 | size += getPointerSize(); |
2648 | |
2649 | |
2650 | types.push_back(llvm::PointerType::getUnqual(byrefType)); |
2651 | size += getPointerSize(); |
2652 | |
2653 | |
2654 | types.push_back(Int32Ty); |
2655 | size += CharUnits::fromQuantity(4); |
2656 | |
2657 | |
2658 | types.push_back(Int32Ty); |
2659 | size += CharUnits::fromQuantity(4); |
2660 | |
2661 | |
2662 | bool hasCopyAndDispose = getContext().BlockRequiresCopying(Ty, D); |
2663 | if (hasCopyAndDispose) { |
2664 | |
2665 | types.push_back(Int8PtrTy); |
2666 | size += getPointerSize(); |
2667 | |
2668 | |
2669 | types.push_back(Int8PtrTy); |
2670 | size += getPointerSize(); |
2671 | } |
2672 | |
2673 | bool HasByrefExtendedLayout = false; |
2674 | Qualifiers::ObjCLifetime Lifetime = Qualifiers::OCL_None; |
2675 | if (getContext().getByrefLifetime(Ty, Lifetime, HasByrefExtendedLayout) && |
2676 | HasByrefExtendedLayout) { |
2677 | |
2678 | types.push_back(Int8PtrTy); |
2679 | size += CharUnits::fromQuantity(PointerSizeInBytes); |
2680 | } |
2681 | |
2682 | |
2683 | llvm::Type *varTy = ConvertTypeForMem(Ty); |
2684 | |
2685 | bool packed = false; |
2686 | CharUnits varAlign = getContext().getDeclAlign(D); |
2687 | CharUnits varOffset = size.alignTo(varAlign); |
2688 | |
2689 | |
2690 | if (varOffset != size) { |
2691 | llvm::Type *paddingTy = |
2692 | llvm::ArrayType::get(Int8Ty, (varOffset - size).getQuantity()); |
2693 | |
2694 | types.push_back(paddingTy); |
2695 | size = varOffset; |
2696 | |
2697 | |
2698 | } else if (CGM.getDataLayout().getABITypeAlignment(varTy) |
2699 | > varAlign.getQuantity()) { |
2700 | packed = true; |
2701 | } |
2702 | types.push_back(varTy); |
2703 | |
2704 | byrefType->setBody(types, packed); |
2705 | |
2706 | BlockByrefInfo info; |
2707 | info.Type = byrefType; |
2708 | info.FieldIndex = types.size() - 1; |
2709 | info.FieldOffset = varOffset; |
2710 | info.ByrefAlignment = std::max(varAlign, getPointerAlign()); |
2711 | |
2712 | auto pair = BlockByrefInfos.insert({D, info}); |
2713 | assert(pair.second && "info was inserted recursively?"); |
2714 | return pair.first->second; |
2715 | } |
2716 | |
2717 | |
2718 | |
2719 | void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { |
2720 | |
2721 | Address addr = emission.Addr; |
2722 | |
2723 | |
2724 | llvm::StructType *byrefType = cast<llvm::StructType>( |
2725 | cast<llvm::PointerType>(addr.getPointer()->getType())->getElementType()); |
2726 | |
2727 | unsigned nextHeaderIndex = 0; |
2728 | CharUnits nextHeaderOffset; |
2729 | auto storeHeaderField = [&](llvm::Value *value, CharUnits fieldSize, |
2730 | const Twine &name) { |
2731 | auto fieldAddr = Builder.CreateStructGEP(addr, nextHeaderIndex, name); |
2732 | Builder.CreateStore(value, fieldAddr); |
2733 | |
2734 | nextHeaderIndex++; |
2735 | nextHeaderOffset += fieldSize; |
2736 | }; |
2737 | |
2738 | |
2739 | BlockByrefHelpers *helpers = buildByrefHelpers(*byrefType, emission); |
2740 | |
2741 | const VarDecl &D = *emission.Variable; |
2742 | QualType type = D.getType(); |
2743 | |
2744 | bool HasByrefExtendedLayout = false; |
2745 | Qualifiers::ObjCLifetime ByrefLifetime = Qualifiers::OCL_None; |
2746 | bool ByRefHasLifetime = |
2747 | getContext().getByrefLifetime(type, ByrefLifetime, HasByrefExtendedLayout); |
2748 | |
2749 | llvm::Value *V; |
2750 | |
2751 | |
2752 | int isa = 0; |
2753 | if (type.isObjCGCWeak()) |
2754 | isa = 1; |
2755 | V = Builder.CreateIntToPtr(Builder.getInt32(isa), Int8PtrTy, "isa"); |
2756 | storeHeaderField(V, getPointerSize(), "byref.isa"); |
2757 | |
2758 | |
2759 | storeHeaderField(addr.getPointer(), getPointerSize(), "byref.forwarding"); |
2760 | |
2761 | |
2762 | |
2763 | |
2764 | BlockFlags flags; |
2765 | if (helpers) flags |= BLOCK_BYREF_HAS_COPY_DISPOSE; |
2766 | if (ByRefHasLifetime) { |
2767 | if (HasByrefExtendedLayout) flags |= BLOCK_BYREF_LAYOUT_EXTENDED; |
2768 | else switch (ByrefLifetime) { |
2769 | case Qualifiers::OCL_Strong: |
2770 | flags |= BLOCK_BYREF_LAYOUT_STRONG; |
2771 | break; |
2772 | case Qualifiers::OCL_Weak: |
2773 | flags |= BLOCK_BYREF_LAYOUT_WEAK; |
2774 | break; |
2775 | case Qualifiers::OCL_ExplicitNone: |
2776 | flags |= BLOCK_BYREF_LAYOUT_UNRETAINED; |
2777 | break; |
2778 | case Qualifiers::OCL_None: |
2779 | if (!type->isObjCObjectPointerType() && !type->isBlockPointerType()) |
2780 | flags |= BLOCK_BYREF_LAYOUT_NON_OBJECT; |
2781 | break; |
2782 | default: |
2783 | break; |
2784 | } |
2785 | if (CGM.getLangOpts().ObjCGCBitmapPrint) { |
2786 | printf("\n Inline flag for BYREF variable layout (%d):", flags.getBitMask()); |
2787 | if (flags & BLOCK_BYREF_HAS_COPY_DISPOSE) |
2788 | printf(" BLOCK_BYREF_HAS_COPY_DISPOSE"); |
2789 | if (flags & BLOCK_BYREF_LAYOUT_MASK) { |
2790 | BlockFlags ThisFlag(flags.getBitMask() & BLOCK_BYREF_LAYOUT_MASK); |
2791 | if (ThisFlag == BLOCK_BYREF_LAYOUT_EXTENDED) |
2792 | printf(" BLOCK_BYREF_LAYOUT_EXTENDED"); |
2793 | if (ThisFlag == BLOCK_BYREF_LAYOUT_STRONG) |
2794 | printf(" BLOCK_BYREF_LAYOUT_STRONG"); |
2795 | if (ThisFlag == BLOCK_BYREF_LAYOUT_WEAK) |
2796 | printf(" BLOCK_BYREF_LAYOUT_WEAK"); |
2797 | if (ThisFlag == BLOCK_BYREF_LAYOUT_UNRETAINED) |
2798 | printf(" BLOCK_BYREF_LAYOUT_UNRETAINED"); |
2799 | if (ThisFlag == BLOCK_BYREF_LAYOUT_NON_OBJECT) |
2800 | printf(" BLOCK_BYREF_LAYOUT_NON_OBJECT"); |
2801 | } |
2802 | printf("\n"); |
2803 | } |
2804 | } |
2805 | storeHeaderField(llvm::ConstantInt::get(IntTy, flags.getBitMask()), |
2806 | getIntSize(), "byref.flags"); |
2807 | |
2808 | CharUnits byrefSize = CGM.GetTargetTypeStoreSize(byrefType); |
2809 | V = llvm::ConstantInt::get(IntTy, byrefSize.getQuantity()); |
2810 | storeHeaderField(V, getIntSize(), "byref.size"); |
2811 | |
2812 | if (helpers) { |
2813 | storeHeaderField(helpers->CopyHelper, getPointerSize(), |
2814 | "byref.copyHelper"); |
2815 | storeHeaderField(helpers->DisposeHelper, getPointerSize(), |
2816 | "byref.disposeHelper"); |
2817 | } |
2818 | |
2819 | if (ByRefHasLifetime && HasByrefExtendedLayout) { |
2820 | auto layoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, type); |
2821 | storeHeaderField(layoutInfo, getPointerSize(), "byref.layout"); |
2822 | } |
2823 | } |
2824 | |
2825 | void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags, |
2826 | bool CanThrow) { |
2827 | llvm::FunctionCallee F = CGM.getBlockObjectDispose(); |
2828 | llvm::Value *args[] = { |
2829 | Builder.CreateBitCast(V, Int8PtrTy), |
2830 | llvm::ConstantInt::get(Int32Ty, flags.getBitMask()) |
2831 | }; |
2832 | |
2833 | if (CanThrow) |
2834 | EmitRuntimeCallOrInvoke(F, args); |
2835 | else |
2836 | EmitNounwindRuntimeCall(F, args); |
2837 | } |
2838 | |
2839 | void CodeGenFunction::enterByrefCleanup(CleanupKind Kind, Address Addr, |
2840 | BlockFieldFlags Flags, |
2841 | bool LoadBlockVarAddr, bool CanThrow) { |
2842 | EHStack.pushCleanup<CallBlockRelease>(Kind, Addr, Flags, LoadBlockVarAddr, |
2843 | CanThrow); |
2844 | } |
2845 | |
2846 | |
2847 | static void configureBlocksRuntimeObject(CodeGenModule &CGM, |
2848 | llvm::Constant *C) { |
2849 | auto *GV = cast<llvm::GlobalValue>(C->stripPointerCasts()); |
2850 | |
2851 | if (CGM.getTarget().getTriple().isOSBinFormatCOFF()) { |
2852 | IdentifierInfo &II = CGM.getContext().Idents.get(C->getName()); |
2853 | TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl(); |
2854 | DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl); |
2855 | |
2856 | assert((isa<llvm::Function>(C->stripPointerCasts()) || |
2857 | isa<llvm::GlobalVariable>(C->stripPointerCasts())) && |
2858 | "expected Function or GlobalVariable"); |
2859 | |
2860 | const NamedDecl *ND = nullptr; |
2861 | for (const auto *Result : DC->lookup(&II)) |
2862 | if ((ND = dyn_cast<FunctionDecl>(Result)) || |
2863 | (ND = dyn_cast<VarDecl>(Result))) |
2864 | break; |
2865 | |
2866 | |
2867 | if (GV->isDeclaration() && (!ND || !ND->hasAttr<DLLExportAttr>())) { |
2868 | GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); |
2869 | GV->setLinkage(llvm::GlobalValue::ExternalLinkage); |
2870 | } else { |
2871 | GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); |
2872 | GV->setLinkage(llvm::GlobalValue::ExternalLinkage); |
2873 | } |
2874 | } |
2875 | |
2876 | if (CGM.getLangOpts().BlocksRuntimeOptional && GV->isDeclaration() && |
2877 | GV->hasExternalLinkage()) |
2878 | GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage); |
2879 | |
2880 | CGM.setDSOLocal(GV); |
2881 | } |
2882 | |
2883 | llvm::FunctionCallee CodeGenModule::getBlockObjectDispose() { |
2884 | if (BlockObjectDispose) |
2885 | return BlockObjectDispose; |
2886 | |
2887 | llvm::Type *args[] = { Int8PtrTy, Int32Ty }; |
2888 | llvm::FunctionType *fty |
2889 | = llvm::FunctionType::get(VoidTy, args, false); |
2890 | BlockObjectDispose = CreateRuntimeFunction(fty, "_Block_object_dispose"); |
2891 | configureBlocksRuntimeObject( |
2892 | *this, cast<llvm::Constant>(BlockObjectDispose.getCallee())); |
2893 | return BlockObjectDispose; |
2894 | } |
2895 | |
2896 | llvm::FunctionCallee CodeGenModule::getBlockObjectAssign() { |
2897 | if (BlockObjectAssign) |
2898 | return BlockObjectAssign; |
2899 | |
2900 | llvm::Type *args[] = { Int8PtrTy, Int8PtrTy, Int32Ty }; |
2901 | llvm::FunctionType *fty |
2902 | = llvm::FunctionType::get(VoidTy, args, false); |
2903 | BlockObjectAssign = CreateRuntimeFunction(fty, "_Block_object_assign"); |
2904 | configureBlocksRuntimeObject( |
2905 | *this, cast<llvm::Constant>(BlockObjectAssign.getCallee())); |
2906 | return BlockObjectAssign; |
2907 | } |
2908 | |
2909 | llvm::Constant *CodeGenModule::getNSConcreteGlobalBlock() { |
2910 | if (NSConcreteGlobalBlock) |
2911 | return NSConcreteGlobalBlock; |
2912 | |
2913 | NSConcreteGlobalBlock = |
2914 | GetOrCreateLLVMGlobal("_NSConcreteGlobalBlock", Int8PtrTy, 0, nullptr); |
2915 | configureBlocksRuntimeObject(*this, NSConcreteGlobalBlock); |
2916 | return NSConcreteGlobalBlock; |
2917 | } |
2918 | |
2919 | llvm::Constant *CodeGenModule::getNSConcreteStackBlock() { |
2920 | if (NSConcreteStackBlock) |
2921 | return NSConcreteStackBlock; |
2922 | |
2923 | NSConcreteStackBlock = |
2924 | GetOrCreateLLVMGlobal("_NSConcreteStackBlock", Int8PtrTy, 0, nullptr); |
2925 | configureBlocksRuntimeObject(*this, NSConcreteStackBlock); |
2926 | return NSConcreteStackBlock; |
2927 | } |
1 | |
2 | |
3 | |
4 | |
5 | |
6 | |
7 | |
8 | |
9 | |
10 | |
11 | |
12 | |
13 | #ifndef LLVM_SUPPORT_MATHEXTRAS_H |
14 | #define LLVM_SUPPORT_MATHEXTRAS_H |
15 | |
16 | #include "llvm/Support/Compiler.h" |
17 | #include <cassert> |
18 | #include <climits> |
19 | #include <cmath> |
20 | #include <cstdint> |
21 | #include <cstring> |
22 | #include <limits> |
23 | #include <type_traits> |
24 | |
25 | #ifdef __ANDROID_NDK__ |
26 | #include <android/api-level.h> |
27 | #endif |
28 | |
29 | #ifdef _MSC_VER |
30 | |
31 | |
32 | |
33 | extern "C" { |
34 | unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask); |
35 | unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask); |
36 | unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask); |
37 | unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); |
38 | } |
39 | #endif |
40 | |
41 | namespace llvm { |
42 | |
43 | |
44 | enum ZeroBehavior { |
45 | |
46 | ZB_Undefined, |
47 | |
48 | ZB_Max, |
49 | |
50 | ZB_Width |
51 | }; |
52 | |
53 | |
54 | namespace numbers { |
55 | |
56 | |
57 | constexpr double e = 2.7182818284590452354, |
58 | egamma = .57721566490153286061, |
59 | ln2 = .69314718055994530942, |
60 | ln10 = 2.3025850929940456840, |
61 | log2e = 1.4426950408889634074, |
62 | log10e = .43429448190325182765, |
63 | pi = 3.1415926535897932385, |
64 | inv_pi = .31830988618379067154, |
65 | sqrtpi = 1.7724538509055160273, |
66 | inv_sqrtpi = .56418958354775628695, |
67 | sqrt2 = 1.4142135623730950488, |
68 | inv_sqrt2 = .70710678118654752440, |
69 | sqrt3 = 1.7320508075688772935, |
70 | inv_sqrt3 = .57735026918962576451, |
71 | phi = 1.6180339887498948482; |
72 | constexpr float ef = 2.71828183F, |
73 | egammaf = .577215665F, |
74 | ln2f = .693147181F, |
75 | ln10f = 2.30258509F, |
76 | log2ef = 1.44269504F, |
77 | log10ef = .434294482F, |
78 | pif = 3.14159265F, |
79 | inv_pif = .318309886F, |
80 | sqrtpif = 1.77245385F, |
81 | inv_sqrtpif = .564189584F, |
82 | sqrt2f = 1.41421356F, |
83 | inv_sqrt2f = .707106781F, |
84 | sqrt3f = 1.73205081F, |
85 | inv_sqrt3f = .577350269F, |
86 | phif = 1.61803399F; |
87 | } |
88 | |
89 | namespace detail { |
90 | template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter { |
91 | static unsigned count(T Val, ZeroBehavior) { |
92 | if (!Val) |
93 | return std::numeric_limits<T>::digits; |
94 | if (Val & 0x1) |
95 | return 0; |
96 | |
97 | |
98 | unsigned ZeroBits = 0; |
99 | T Shift = std::numeric_limits<T>::digits >> 1; |
100 | T Mask = std::numeric_limits<T>::max() >> Shift; |
101 | while (Shift) { |
102 | if ((Val & Mask) == 0) { |
103 | Val >>= Shift; |
104 | ZeroBits |= Shift; |
105 | } |
106 | Shift >>= 1; |
107 | Mask >>= Shift; |
108 | } |
109 | return ZeroBits; |
110 | } |
111 | }; |
112 | |
113 | #if defined(__GNUC__) || defined(_MSC_VER) |
114 | template <typename T> struct TrailingZerosCounter<T, 4> { |
115 | static unsigned count(T Val, ZeroBehavior ZB) { |
116 | if (ZB != ZB_Undefined && Val == 0) |
117 | return 32; |
118 | |
119 | #if __has_builtin(__builtin_ctz) || defined(__GNUC__) |
120 | return __builtin_ctz(Val); |
121 | #elif defined(_MSC_VER) |
122 | unsigned long Index; |
123 | _BitScanForward(&Index, Val); |
124 | return Index; |
125 | #endif |
126 | } |
127 | }; |
128 | |
129 | #if !defined(_MSC_VER) || defined(_M_X64) |
130 | template <typename T> struct TrailingZerosCounter<T, 8> { |
131 | static unsigned count(T Val, ZeroBehavior ZB) { |
132 | if (ZB != ZB_Undefined && Val == 0) |
133 | return 64; |
134 | |
135 | #if __has_builtin(__builtin_ctzll) || defined(__GNUC__) |
136 | return __builtin_ctzll(Val); |
137 | #elif defined(_MSC_VER) |
138 | unsigned long Index; |
139 | _BitScanForward64(&Index, Val); |
140 | return Index; |
141 | #endif |
142 | } |
143 | }; |
144 | #endif |
145 | #endif |
146 | } |
147 | |
148 | |
149 | |
150 | |
151 | |
152 | |
153 | |
154 | |
155 | template <typename T> |
156 | unsigned countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) { |
157 | static_assert(std::numeric_limits<T>::is_integer && |
158 | !std::numeric_limits<T>::is_signed, |
159 | "Only unsigned integral types are allowed."); |
160 | return llvm::detail::TrailingZerosCounter<T, sizeof(T)>::count(Val, ZB); |
161 | } |
162 | |
163 | namespace detail { |
164 | template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter { |
165 | static unsigned count(T Val, ZeroBehavior) { |
166 | if (!Val) |
167 | return std::numeric_limits<T>::digits; |
168 | |
169 | |
170 | unsigned ZeroBits = 0; |
171 | for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) { |
172 | T Tmp = Val >> Shift; |
173 | if (Tmp) |
174 | Val = Tmp; |
175 | else |
176 | ZeroBits |= Shift; |
177 | } |
178 | return ZeroBits; |
179 | } |
180 | }; |
181 | |
182 | #if defined(__GNUC__) || defined(_MSC_VER) |
183 | template <typename T> struct LeadingZerosCounter<T, 4> { |
184 | static unsigned count(T Val, ZeroBehavior ZB) { |
185 | if (ZB != ZB_Undefined && Val == 0) |
186 | return 32; |
187 | |
188 | #if __has_builtin(__builtin_clz) || defined(__GNUC__) |
189 | return __builtin_clz(Val); |
190 | #elif defined(_MSC_VER) |
191 | unsigned long Index; |
192 | _BitScanReverse(&Index, Val); |
193 | return Index ^ 31; |
194 | #endif |
195 | } |
196 | }; |
197 | |
198 | #if !defined(_MSC_VER) || defined(_M_X64) |
199 | template <typename T> struct LeadingZerosCounter<T, 8> { |
200 | static unsigned count(T Val, ZeroBehavior ZB) { |
201 | if (ZB != ZB_Undefined && Val == 0) |
202 | return 64; |
203 | |
204 | #if __has_builtin(__builtin_clzll) || defined(__GNUC__) |
205 | return __builtin_clzll(Val); |
206 | #elif defined(_MSC_VER) |
207 | unsigned long Index; |
208 | _BitScanReverse64(&Index, Val); |
209 | return Index ^ 63; |
210 | #endif |
211 | } |
212 | }; |
213 | #endif |
214 | #endif |
215 | } |
216 | |
217 | |
218 | |
219 | |
220 | |
221 | |
222 | |
223 | |
224 | template <typename T> |
225 | unsigned countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) { |
226 | static_assert(std::numeric_limits<T>::is_integer && |
227 | !std::numeric_limits<T>::is_signed, |
228 | "Only unsigned integral types are allowed."); |
229 | return llvm::detail::LeadingZerosCounter<T, sizeof(T)>::count(Val, ZB); |
230 | } |
231 | |
232 | |
233 | |
234 | |
235 | |
236 | |
237 | |
238 | |
239 | template <typename T> T findFirstSet(T Val, ZeroBehavior ZB = ZB_Max) { |
240 | if (ZB == ZB_Max && Val == 0) |
241 | return std::numeric_limits<T>::max(); |
242 | |
243 | return countTrailingZeros(Val, ZB_Undefined); |
244 | } |
245 | |
246 | |
247 | |
248 | template <typename T> T maskTrailingOnes(unsigned N) { |
249 | static_assert(std::is_unsigned<T>::value, "Invalid type!"); |
250 | const unsigned Bits = CHAR_BIT * sizeof(T); |
251 | assert(N <= Bits && "Invalid bit index"); |
252 | return N == 0 ? 0 : (T(-1) >> (Bits - N)); |
253 | } |
254 | |
255 | |
256 | |
257 | template <typename T> T maskLeadingOnes(unsigned N) { |
258 | return ~maskTrailingOnes<T>(CHAR_BIT * sizeof(T) - N); |
259 | } |
260 | |
261 | |
262 | |
263 | template <typename T> T maskTrailingZeros(unsigned N) { |
264 | return maskLeadingOnes<T>(CHAR_BIT * sizeof(T) - N); |
265 | } |
266 | |
267 | |
268 | |
269 | template <typename T> T maskLeadingZeros(unsigned N) { |
270 | return maskTrailingOnes<T>(CHAR_BIT * sizeof(T) - N); |
271 | } |
272 | |
273 | |
274 | |
275 | |
276 | |
277 | |
278 | |
279 | |
280 | template <typename T> T findLastSet(T Val, ZeroBehavior ZB = ZB_Max) { |
281 | if (ZB == ZB_Max && Val == 0) |
282 | return std::numeric_limits<T>::max(); |
283 | |
284 | |
285 | |
286 | return countLeadingZeros(Val, ZB_Undefined) ^ |
287 | (std::numeric_limits<T>::digits - 1); |
288 | } |
289 | |
290 | |
291 | |
292 | |
293 | static const unsigned char BitReverseTable256[256] = { |
294 | #define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64 |
295 | #define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16) |
296 | #define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4) |
297 | R6(0), R6(2), R6(1), R6(3) |
298 | #undef R2 |
299 | #undef R4 |
300 | #undef R6 |
301 | }; |
302 | |
303 | |
304 | template <typename T> |
305 | T reverseBits(T Val) { |
306 | unsigned char in[sizeof(Val)]; |
307 | unsigned char out[sizeof(Val)]; |
308 | std::memcpy(in, &Val, sizeof(Val)); |
309 | for (unsigned i = 0; i < sizeof(Val); ++i) |
310 | out[(sizeof(Val) - i) - 1] = BitReverseTable256[in[i]]; |
311 | std::memcpy(&Val, out, sizeof(Val)); |
312 | return Val; |
313 | } |
314 | |
315 | #if __has_builtin(__builtin_bitreverse8) |
316 | template<> |
317 | inline uint8_t reverseBits<uint8_t>(uint8_t Val) { |
318 | return __builtin_bitreverse8(Val); |
319 | } |
320 | #endif |
321 | |
322 | #if __has_builtin(__builtin_bitreverse16) |
323 | template<> |
324 | inline uint16_t reverseBits<uint16_t>(uint16_t Val) { |
325 | return __builtin_bitreverse16(Val); |
326 | } |
327 | #endif |
328 | |
329 | #if __has_builtin(__builtin_bitreverse32) |
330 | template<> |
331 | inline uint32_t reverseBits<uint32_t>(uint32_t Val) { |
332 | return __builtin_bitreverse32(Val); |
333 | } |
334 | #endif |
335 | |
336 | #if __has_builtin(__builtin_bitreverse64) |
337 | template<> |
338 | inline uint64_t reverseBits<uint64_t>(uint64_t Val) { |
339 | return __builtin_bitreverse64(Val); |
340 | } |
341 | #endif |
342 | |
343 | |
344 | |
345 | |
346 | |
347 | |
348 | constexpr inline uint32_t Hi_32(uint64_t Value) { |
349 | return static_cast<uint32_t>(Value >> 32); |
350 | } |
351 | |
352 | |
353 | constexpr inline uint32_t Lo_32(uint64_t Value) { |
354 | return static_cast<uint32_t>(Value); |
355 | } |
356 | |
357 | |
358 | constexpr inline uint64_t Make_64(uint32_t High, uint32_t Low) { |
359 | return ((uint64_t)High << 32) | (uint64_t)Low; |
360 | } |
361 | |
362 | |
363 | template <unsigned N> constexpr inline bool isInt(int64_t x) { |
364 | return N >= 64 || (-(INT64_C(1)<<(N-1)) <= x && x < (INT64_C(1)<<(N-1))); |
365 | } |
366 | |
367 | template <> constexpr inline bool isInt<8>(int64_t x) { |
368 | return static_cast<int8_t>(x) == x; |
369 | } |
370 | template <> constexpr inline bool isInt<16>(int64_t x) { |
371 | return static_cast<int16_t>(x) == x; |
372 | } |
373 | template <> constexpr inline bool isInt<32>(int64_t x) { |
374 | return static_cast<int32_t>(x) == x; |
375 | } |
376 | |
377 | |
378 | template <unsigned N, unsigned S> |
379 | constexpr inline bool isShiftedInt(int64_t x) { |
380 | static_assert( |
381 | N > 0, "isShiftedInt<0> doesn't make sense (refers to a 0-bit number."); |
382 | static_assert(N + S <= 64, "isShiftedInt<N, S> with N + S > 64 is too wide."); |
383 | return isInt<N + S>(x) && (x % (UINT64_C(1) << S) == 0); |
384 | } |
385 | |
386 | |
387 | |
388 | |
389 | |
390 | |
391 | |
392 | |
393 | |
394 | template <unsigned N> |
395 | constexpr inline std::enable_if_t<(N < 64), bool> isUInt(uint64_t X) { |
396 | static_assert(N > 0, "isUInt<0> doesn't make sense"); |
397 | return X < (UINT64_C(1) << (N)); |
398 | } |
399 | template <unsigned N> |
400 | constexpr inline std::enable_if_t<N >= 64, bool> isUInt(uint64_t) { |
401 | return true; |
402 | } |
403 | |
404 | |
405 | template <> constexpr inline bool isUInt<8>(uint64_t x) { |
406 | return static_cast<uint8_t>(x) == x; |
407 | } |
408 | template <> constexpr inline bool isUInt<16>(uint64_t x) { |
409 | return static_cast<uint16_t>(x) == x; |
410 | } |
411 | template <> constexpr inline bool isUInt<32>(uint64_t x) { |
412 | return static_cast<uint32_t>(x) == x; |
413 | } |
414 | |
415 | |
416 | template <unsigned N, unsigned S> |
417 | constexpr inline bool isShiftedUInt(uint64_t x) { |
418 | static_assert( |
419 | N > 0, "isShiftedUInt<0> doesn't make sense (refers to a 0-bit number)"); |
420 | static_assert(N + S <= 64, |
421 | "isShiftedUInt<N, S> with N + S > 64 is too wide."); |
422 | |
423 | |
424 | return isUInt<N + S>(x) && (x % (UINT64_C(1) << S) == 0); |
425 | } |
426 | |
427 | |
428 | inline uint64_t maxUIntN(uint64_t N) { |
429 | assert(N > 0 && N <= 64 && "integer width out of range"); |
430 | |
431 | |
432 | |
433 | |
434 | |
435 | return UINT64_MAX >> (64 - N); |
436 | } |
437 | |
438 | |
439 | inline int64_t minIntN(int64_t N) { |
440 | assert(N > 0 && N <= 64 && "integer width out of range"); |
441 | |
442 | return UINT64_C(1) + ~(UINT64_C(1) << (N - 1)); |
443 | } |
444 | |
445 | |
446 | inline int64_t maxIntN(int64_t N) { |
447 | assert(N > 0 && N <= 64 && "integer width out of range"); |
448 | |
449 | |
450 | |
451 | return (UINT64_C(1) << (N - 1)) - 1; |
452 | } |
453 | |
454 | |
455 | inline bool isUIntN(unsigned N, uint64_t x) { |
456 | return N >= 64 || x <= maxUIntN(N); |
457 | } |
458 | |
459 | |
460 | inline bool isIntN(unsigned N, int64_t x) { |
461 | return N >= 64 || (minIntN(N) <= x && x <= maxIntN(N)); |
462 | } |
463 | |
464 | |
465 | |
466 | |
467 | constexpr inline bool isMask_32(uint32_t Value) { |
468 | return Value && ((Value + 1) & Value) == 0; |
469 | } |
470 | |
471 | |
472 | |
473 | constexpr inline bool isMask_64(uint64_t Value) { |
474 | return Value && ((Value + 1) & Value) == 0; |
475 | } |
476 | |
477 | |
478 | |
479 | constexpr inline bool isShiftedMask_32(uint32_t Value) { |
480 | return Value && isMask_32((Value - 1) | Value); |
481 | } |
482 | |
483 | |
484 | |
485 | constexpr inline bool isShiftedMask_64(uint64_t Value) { |
486 | return Value && isMask_64((Value - 1) | Value); |
487 | } |
488 | |
489 | |
490 | |
491 | constexpr inline bool isPowerOf2_32(uint32_t Value) { |
492 | return Value && !(Value & (Value - 1)); |
493 | } |
494 | |
495 | |
496 | constexpr inline bool isPowerOf2_64(uint64_t Value) { |
497 | return Value && !(Value & (Value - 1)); |
498 | } |
499 | |
500 | |
501 | |
502 | |
503 | |
504 | |
505 | |
506 | |
507 | |
508 | template <typename T> |
509 | unsigned countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) { |
510 | static_assert(std::numeric_limits<T>::is_integer && |
511 | !std::numeric_limits<T>::is_signed, |
512 | "Only unsigned integral types are allowed."); |
513 | return countLeadingZeros<T>(~Value, ZB); |
514 | } |
515 | |
516 | |
517 | |
518 | |
519 | |
520 | |
521 | |
522 | |
523 | |
524 | template <typename T> |
525 | unsigned countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) { |
526 | static_assert(std::numeric_limits<T>::is_integer && |
527 | !std::numeric_limits<T>::is_signed, |
528 | "Only unsigned integral types are allowed."); |
529 | return countTrailingZeros<T>(~Value, ZB); |
530 | } |
531 | |
532 | namespace detail { |
533 | template <typename T, std::size_t SizeOfT> struct PopulationCounter { |
534 | static unsigned count(T Value) { |
535 | |
536 | static_assert(SizeOfT <= 4, "Not implemented!"); |
537 | #if defined(__GNUC__) |
538 | return __builtin_popcount(Value); |
539 | #else |
540 | uint32_t v = Value; |
541 | v = v - ((v >> 1) & 0x55555555); |
542 | v = (v & 0x33333333) + ((v >> 2) & 0x33333333); |
543 | return ((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24; |
544 | #endif |
545 | } |
546 | }; |
547 | |
548 | template <typename T> struct PopulationCounter<T, 8> { |
549 | static unsigned count(T Value) { |
550 | #if defined(__GNUC__) |
551 | return __builtin_popcountll(Value); |
552 | #else |
553 | uint64_t v = Value; |
554 | v = v - ((v >> 1) & 0x5555555555555555ULL); |
555 | v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); |
556 | v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; |
557 | return unsigned((uint64_t)(v * 0x0101010101010101ULL) >> 56); |
558 | #endif |
559 | } |
560 | }; |
561 | } |
562 | |
563 | |
564 | |
565 | |
566 | template <typename T> |
567 | inline unsigned countPopulation(T Value) { |
568 | static_assert(std::numeric_limits<T>::is_integer && |
569 | !std::numeric_limits<T>::is_signed, |
570 | "Only unsigned integral types are allowed."); |
571 | return detail::PopulationCounter<T, sizeof(T)>::count(Value); |
572 | } |
573 | |
574 | |
575 | |
576 | template <size_t kValue> constexpr inline size_t CTLog2() { |
577 | static_assert(kValue > 0 && llvm::isPowerOf2_64(kValue), |
578 | "Value is not a valid power of 2"); |
579 | return 1 + CTLog2<kValue / 2>(); |
580 | } |
581 | |
582 | template <> constexpr inline size_t CTLog2<1>() { return 0; } |
583 | |
584 | |
585 | inline double Log2(double Value) { |
586 | #if defined(__ANDROID_API__) && __ANDROID_API__ < 18 |
587 | return __builtin_log(Value) / __builtin_log(2.0); |
588 | #else |
589 | return log2(Value); |
590 | #endif |
591 | } |
592 | |
593 | |
594 | |
595 | |
596 | inline unsigned Log2_32(uint32_t Value) { |
597 | return 31 - countLeadingZeros(Value); |
598 | } |
599 | |
600 | |
601 | |
602 | inline unsigned Log2_64(uint64_t Value) { |
603 | return 63 - countLeadingZeros(Value); |
604 | } |
605 | |
606 | |
607 | |
608 | |
609 | inline unsigned Log2_32_Ceil(uint32_t Value) { |
610 | return 32 - countLeadingZeros(Value - 1); |
611 | } |
612 | |
613 | |
614 | |
615 | inline unsigned Log2_64_Ceil(uint64_t Value) { |
616 | return 64 - countLeadingZeros(Value - 1); |
617 | } |
618 | |
619 | |
620 | template <typename T> |
621 | inline T greatestCommonDivisor(T A, T B) { |
622 | while (B) { |
623 | T Tmp = B; |
624 | B = A % B; |
625 | A = Tmp; |
626 | } |
627 | return A; |
628 | } |
629 | |
630 | inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) { |
631 | return greatestCommonDivisor<uint64_t>(A, B); |
632 | } |
633 | |
634 | |
635 | inline double BitsToDouble(uint64_t Bits) { |
636 | double D; |
637 | static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes"); |
638 | memcpy(&D, &Bits, sizeof(Bits)); |
639 | return D; |
640 | } |
641 | |
642 | |
643 | inline float BitsToFloat(uint32_t Bits) { |
644 | float F; |
645 | static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes"); |
646 | memcpy(&F, &Bits, sizeof(Bits)); |
647 | return F; |
648 | } |
649 | |
650 | |
651 | |
652 | |
653 | inline uint64_t DoubleToBits(double Double) { |
654 | uint64_t Bits; |
655 | static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes"); |
656 | memcpy(&Bits, &Double, sizeof(Double)); |
657 | return Bits; |
658 | } |
659 | |
660 | |
661 | |
662 | |
663 | inline uint32_t FloatToBits(float Float) { |
664 | uint32_t Bits; |
665 | static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes"); |
666 | memcpy(&Bits, &Float, sizeof(Float)); |
667 | return Bits; |
668 | } |
669 | |
670 | |
671 | |
672 | constexpr inline uint64_t MinAlign(uint64_t A, uint64_t B) { |
673 | |
674 | |
675 | |
676 | |
677 | |
678 | return (A | B) & (1 + ~(A | B)); |
679 | } |
680 | |
681 | |
682 | |
683 | inline uint64_t NextPowerOf2(uint64_t A) { |
684 | A |= (A >> 1); |
685 | A |= (A >> 2); |
686 | A |= (A >> 4); |
687 | A |= (A >> 8); |
688 | A |= (A >> 16); |
689 | A |= (A >> 32); |
690 | return A + 1; |
691 | } |
692 | |
693 | |
694 | |
695 | inline uint64_t PowerOf2Floor(uint64_t A) { |
696 | if (!A) return 0; |
697 | return 1ull << (63 - countLeadingZeros(A, ZB_Undefined)); |
698 | } |
699 | |
700 | |
701 | |
702 | inline uint64_t PowerOf2Ceil(uint64_t A) { |
703 | if (!A) |
704 | return 0; |
705 | return NextPowerOf2(A - 1); |
706 | } |
707 | |
708 | |
709 | |
710 | |
711 | |
712 | |
713 | |
714 | |
715 | |
716 | |
717 | |
718 | |
719 | |
720 | |
721 | |
722 | |
723 | |
724 | |
725 | |
726 | |
727 | |
728 | inline uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew = 0) { |
729 | assert(Align != 0u && "Align can't be 0."); |
730 | Skew %= Align; |
| |
731 | return (Value + Align - 1 - Skew) / Align * Align + Skew; |
732 | } |
733 | |
734 | |
735 | |
736 | template <uint64_t Align> constexpr inline uint64_t alignTo(uint64_t Value) { |
737 | static_assert(Align != 0u, "Align must be non-zero"); |
738 | return (Value + Align - 1) / Align * Align; |
739 | } |
740 | |
741 | |
742 | inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) { |
743 | return alignTo(Numerator, Denominator) / Denominator; |
744 | } |
745 | |
746 | |
747 | inline uint64_t divideNearest(uint64_t Numerator, uint64_t Denominator) { |
748 | return (Numerator + (Denominator / 2)) / Denominator; |
749 | } |
750 | |
751 | |
752 | |
753 | inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) { |
754 | assert(Align != 0u && "Align can't be 0."); |
755 | Skew %= Align; |
756 | return (Value - Skew) / Align * Align + Skew; |
757 | } |
758 | |
759 | |
760 | |
761 | template <unsigned B> constexpr inline int32_t SignExtend32(uint32_t X) { |
762 | static_assert(B > 0, "Bit width can't be 0."); |
763 | static_assert(B <= 32, "Bit width out of range."); |
764 | return int32_t(X << (32 - B)) >> (32 - B); |
765 | } |
766 | |
767 | |
768 | |
769 | inline int32_t SignExtend32(uint32_t X, unsigned B) { |
770 | assert(B > 0 && "Bit width can't be 0."); |
771 | assert(B <= 32 && "Bit width out of range."); |
772 | return int32_t(X << (32 - B)) >> (32 - B); |
773 | } |
774 | |
775 | |
776 | |
777 | template <unsigned B> constexpr inline int64_t SignExtend64(uint64_t x) { |
778 | static_assert(B > 0, "Bit width can't be 0."); |
779 | static_assert(B <= 64, "Bit width out of range."); |
780 | return int64_t(x << (64 - B)) >> (64 - B); |
781 | } |
782 | |
783 | |
784 | |
785 | inline int64_t SignExtend64(uint64_t X, unsigned B) { |
786 | assert(B > 0 && "Bit width can't be 0."); |
787 | assert(B <= 64 && "Bit width out of range."); |
788 | return int64_t(X << (64 - B)) >> (64 - B); |
789 | } |
790 | |
791 | |
792 | |
793 | template <typename T> |
794 | std::enable_if_t<std::is_unsigned<T>::value, T> AbsoluteDifference(T X, T Y) { |
795 | return X > Y ? (X - Y) : (Y - X); |
796 | } |
797 | |
798 | |
799 | |
800 | |
801 | template <typename T> |
802 | std::enable_if_t<std::is_unsigned<T>::value, T> |
803 | SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) { |
804 | bool Dummy; |
805 | bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy; |
806 | |
807 | T Z = X + Y; |
808 | Overflowed = (Z < X || Z < Y); |
809 | if (Overflowed) |
810 | return std::numeric_limits<T>::max(); |
811 | else |
812 | return Z; |
813 | } |
814 | |
815 | |
816 | |
817 | |
818 | template <typename T> |
819 | std::enable_if_t<std::is_unsigned<T>::value, T> |
820 | SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) { |
821 | bool Dummy; |
822 | bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy; |
823 | |
824 | |
825 | |
826 | |
827 | |
828 | |
829 | Overflowed = false; |
830 | |
831 | |
832 | |
833 | |
834 | int Log2Z = Log2_64(X) + Log2_64(Y); |
835 | const T Max = std::numeric_limits<T>::max(); |
836 | int Log2Max = Log2_64(Max); |
837 | if (Log2Z < Log2Max) { |
838 | return X * Y; |
839 | } |
840 | if (Log2Z > Log2Max) { |
841 | Overflowed = true; |
842 | return Max; |
843 | } |
844 | |
845 | |
846 | |
847 | |
848 | T Z = (X >> 1) * Y; |
849 | if (Z & ~(Max >> 1)) { |
850 | Overflowed = true; |
851 | return Max; |
852 | } |
853 | Z <<= 1; |
854 | if (X & 1) |
855 | return SaturatingAdd(Z, Y, ResultOverflowed); |
856 | |
857 | return Z; |
858 | } |
859 | |
860 | |
861 | |
862 | |
863 | |
864 | template <typename T> |
865 | std::enable_if_t<std::is_unsigned<T>::value, T> |
866 | SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) { |
867 | bool Dummy; |
868 | bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy; |
869 | |
870 | T Product = SaturatingMultiply(X, Y, &Overflowed); |
871 | if (Overflowed) |
872 | return Product; |
873 | |
874 | return SaturatingAdd(A, Product, &Overflowed); |
875 | } |
876 | |
877 | |
878 | extern const float huge_valf; |
879 | |
880 | |
881 | |
882 | |
883 | template <typename T> |
884 | std::enable_if_t<std::is_signed<T>::value, T> AddOverflow(T X, T Y, T &Result) { |
885 | #if __has_builtin(__builtin_add_overflow) |
886 | return __builtin_add_overflow(X, Y, &Result); |
887 | #else |
888 | |
889 | using U = std::make_unsigned_t<T>; |
890 | const U UX = static_cast<U>(X); |
891 | const U UY = static_cast<U>(Y); |
892 | const U UResult = UX + UY; |
893 | |
894 | |
895 | Result = static_cast<T>(UResult); |
896 | |
897 | |
898 | if (X > 0 && Y > 0) |
899 | return Result <= 0; |
900 | |
901 | if (X < 0 && Y < 0) |
902 | return Result >= 0; |
903 | return false; |
904 | #endif |
905 | } |
906 | |
907 | |
908 | |
909 | template <typename T> |
910 | std::enable_if_t<std::is_signed<T>::value, T> SubOverflow(T X, T Y, T &Result) { |
911 | #if __has_builtin(__builtin_sub_overflow) |
912 | return __builtin_sub_overflow(X, Y, &Result); |
913 | #else |
914 | |
915 | using U = std::make_unsigned_t<T>; |
916 | const U UX = static_cast<U>(X); |
917 | const U UY = static_cast<U>(Y); |
918 | const U UResult = UX - UY; |
919 | |
920 | |
921 | Result = static_cast<T>(UResult); |
922 | |
923 | |
924 | if (X <= 0 && Y > 0) |
925 | return Result >= 0; |
926 | |
927 | if (X >= 0 && Y < 0) |
928 | return Result <= 0; |
929 | return false; |
930 | #endif |
931 | } |
932 | |
933 | |
934 | |
935 | template <typename T> |
936 | std::enable_if_t<std::is_signed<T>::value, T> MulOverflow(T X, T Y, T &Result) { |
937 | |
938 | using U = std::make_unsigned_t<T>; |
939 | const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X); |
940 | const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y); |
941 | const U UResult = UX * UY; |
942 | |
943 | |
944 | const bool IsNegative = (X < 0) ^ (Y < 0); |
945 | Result = IsNegative ? (0 - UResult) : UResult; |
946 | |
947 | |
948 | if (UX == 0 || UY == 0) |
949 | return false; |
950 | |
951 | |
952 | |
953 | |
954 | if (IsNegative) |
955 | return UX > (static_cast<U>(std::numeric_limits<T>::max()) + U(1)) / UY; |
956 | else |
957 | return UX > (static_cast<U>(std::numeric_limits<T>::max())) / UY; |
958 | } |
959 | |
960 | } |
961 | |
962 | #endif |