Bug Summary

File:dev/pci/drm/i915/gem/i915_gem_object.h
Warning:line 130, column 17
Access to field 'contended' results in a dereference of a null pointer (loaded from variable 'ww')

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name i915_gem_domain.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/dev/pci/drm/i915/gem/i915_gem_domain.c

/usr/src/sys/dev/pci/drm/i915/gem/i915_gem_domain.c

→
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2016 Intel Corporation
5 */
6
7#include "display/intel_frontbuffer.h"
8
9#include "i915_drv.h"
10#include "i915_gem_clflush.h"
11#include "i915_gem_gtt.h"
12#include "i915_gem_ioctls.h"
13#include "i915_gem_object.h"
14#include "i915_vma.h"
15#include "i915_gem_lmem.h"
16#include "i915_gem_mman.h"
17
18static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
19{
20 /*
21 * We manually flush the CPU domain so that we can override and
22 * force the flush for the display, and perform it asyncrhonously.
23 */
24 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU0x00000001);
25 if (obj->cache_dirty)
26 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE(1UL << (0)));
27 obj->write_domain = 0;
28}
29
30void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
31{
32 if (!i915_gem_object_is_framebuffer(obj))
33 return;
34
35 i915_gem_object_lock(obj, NULL((void *)0));
36 __i915_gem_object_flush_for_display(obj);
37 i915_gem_object_unlock(obj);
38}
39
40void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj)
41{
42 if (i915_gem_object_is_framebuffer(obj))
43 __i915_gem_object_flush_for_display(obj);
44}
45
46/**
47 * Moves a single object to the WC read, and possibly write domain.
48 * @obj: object to act on
49 * @write: ask for write access or read only
50 *
51 * This function returns when the move is complete, including waiting on
52 * flushes to occur.
53 */
54int
55i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool_Bool write)
56{
57 int ret;
58
59 assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0);
60
61 ret = i915_gem_object_wait(obj,
62 I915_WAIT_INTERRUPTIBLE(1UL << (0)) |
63 (write ? I915_WAIT_ALL(1UL << (2)) : 0),
64 MAX_SCHEDULE_TIMEOUT(0x7fffffff));
65 if (ret)
66 return ret;
67
68 if (obj->write_domain == I915_GEM_DOMAIN_WC0x00000080)
69 return 0;
70
71 /* Flush and acquire obj->pages so that we are coherent through
72 * direct access in memory with previous cached writes through
73 * shmemfs and that our cache domain tracking remains valid.
74 * For example, if the obj->filp was moved to swap without us
75 * being notified and releasing the pages, we would mistakenly
76 * continue to assume that the obj remained out of the CPU cached
77 * domain.
78 */
79 ret = i915_gem_object_pin_pages(obj);
80 if (ret)
81 return ret;
82
83 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_WC0x00000080);
84
85 /* Serialise direct access to this object with the barriers for
86 * coherent writes from the GPU, by effectively invalidating the
87 * WC domain upon first access.
88 */
89 if ((obj->read_domains & I915_GEM_DOMAIN_WC0x00000080) == 0)
90 mb()do { __asm volatile("mfence" ::: "memory"); } while (0);
91
92 /* It should now be out of any other write domains, and we can update
93 * the domain values for our changes.
94 */
95 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0)((void)0);
96 obj->read_domains |= I915_GEM_DOMAIN_WC0x00000080;
97 if (write) {
98 obj->read_domains = I915_GEM_DOMAIN_WC0x00000080;
99 obj->write_domain = I915_GEM_DOMAIN_WC0x00000080;
100 obj->mm.dirty = true1;
101 }
102
103 i915_gem_object_unpin_pages(obj);
104 return 0;
105}
106
107/**
108 * Moves a single object to the GTT read, and possibly write domain.
109 * @obj: object to act on
110 * @write: ask for write access or read only
111 *
112 * This function returns when the move is complete, including waiting on
113 * flushes to occur.
114 */
115int
116i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool_Bool write)
117{
118 int ret;
119
120 assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0);
121
122 ret = i915_gem_object_wait(obj,
123 I915_WAIT_INTERRUPTIBLE(1UL << (0)) |
124 (write ? I915_WAIT_ALL(1UL << (2)) : 0),
125 MAX_SCHEDULE_TIMEOUT(0x7fffffff));
126 if (ret)
127 return ret;
128
129 if (obj->write_domain == I915_GEM_DOMAIN_GTT0x00000040)
130 return 0;
131
132 /* Flush and acquire obj->pages so that we are coherent through
133 * direct access in memory with previous cached writes through
134 * shmemfs and that our cache domain tracking remains valid.
135 * For example, if the obj->filp was moved to swap without us
136 * being notified and releasing the pages, we would mistakenly
137 * continue to assume that the obj remained out of the CPU cached
138 * domain.
139 */
140 ret = i915_gem_object_pin_pages(obj);
141 if (ret)
142 return ret;
143
144 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT0x00000040);
145
146 /* Serialise direct access to this object with the barriers for
147 * coherent writes from the GPU, by effectively invalidating the
148 * GTT domain upon first access.
149 */
150 if ((obj->read_domains & I915_GEM_DOMAIN_GTT0x00000040) == 0)
151 mb()do { __asm volatile("mfence" ::: "memory"); } while (0);
152
153 /* It should now be out of any other write domains, and we can update
154 * the domain values for our changes.
155 */
156 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0)((void)0);
157 obj->read_domains |= I915_GEM_DOMAIN_GTT0x00000040;
158 if (write) {
159 struct i915_vma *vma;
160
161 obj->read_domains = I915_GEM_DOMAIN_GTT0x00000040;
162 obj->write_domain = I915_GEM_DOMAIN_GTT0x00000040;
163 obj->mm.dirty = true1;
164
165 spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock);
166 for_each_ggtt_vma(vma, obj)for (vma = ({ const __typeof( ((__typeof(*vma) *)0)->obj_link
) *__mptr = ((&(obj)->vma.list)->next); (__typeof(
*vma) *)( (char *)__mptr - __builtin_offsetof(__typeof(*vma),
obj_link) );}); &vma->obj_link != (&(obj)->vma
.list); vma = ({ const __typeof( ((__typeof(*vma) *)0)->obj_link
) *__mptr = (vma->obj_link.next); (__typeof(*vma) *)( (char
*)__mptr - __builtin_offsetof(__typeof(*vma), obj_link) );})
) if (!i915_vma_is_ggtt(vma)) break; else
167 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND((int)(1UL << (10)))))
168 i915_vma_set_ggtt_write(vma);
169 spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock);
170 }
171
172 i915_gem_object_unpin_pages(obj);
173 return 0;
174}
175
176/**
177 * Changes the cache-level of an object across all VMA.
178 * @obj: object to act on
179 * @cache_level: new cache level to set for the object
180 *
181 * After this function returns, the object will be in the new cache-level
182 * across all GTT and the contents of the backing storage will be coherent,
183 * with respect to the new cache-level. In order to keep the backing storage
184 * coherent for all users, we only allow a single cache level to be set
185 * globally on the object and prevent it from being changed whilst the
186 * hardware is reading from the object. That is if the object is currently
187 * on the scanout it will be set to uncached (or equivalent display
188 * cache coherency) and all non-MOCS GPU access will also be uncached so
189 * that all direct access to the scanout remains coherent.
190 */
191int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
192 enum i915_cache_level cache_level)
193{
194 int ret;
195
196 if (obj->cache_level == cache_level)
197 return 0;
198
199 ret = i915_gem_object_wait(obj,
200 I915_WAIT_INTERRUPTIBLE(1UL << (0)) |
201 I915_WAIT_ALL(1UL << (2)),
202 MAX_SCHEDULE_TIMEOUT(0x7fffffff));
203 if (ret)
204 return ret;
205
206 /* Always invalidate stale cachelines */
207 if (obj->cache_level != cache_level) {
208 i915_gem_object_set_cache_coherency(obj, cache_level);
209 obj->cache_dirty = true1;
210 }
211
212 /* The cache-level will be applied when each vma is rebound. */
213 return i915_gem_object_unbind(obj,
214 I915_GEM_OBJECT_UNBIND_ACTIVE(1UL << (0)) |
215 I915_GEM_OBJECT_UNBIND_BARRIER(1UL << (1)));
216}
217
218int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
219 struct drm_file *file)
220{
221 struct drm_i915_gem_caching *args = data;
222 struct drm_i915_gem_object *obj;
223 int err = 0;
224
225 rcu_read_lock();
226 obj = i915_gem_object_lookup_rcu(file, args->handle);
227 if (!obj) {
228 err = -ENOENT2;
229 goto out;
230 }
231
232 switch (obj->cache_level) {
233 case I915_CACHE_LLC:
234 case I915_CACHE_L3_LLC:
235 args->caching = I915_CACHING_CACHED1;
236 break;
237
238 case I915_CACHE_WT:
239 args->caching = I915_CACHING_DISPLAY2;
240 break;
241
242 default:
243 args->caching = I915_CACHING_NONE0;
244 break;
245 }
246out:
247 rcu_read_unlock();
248 return err;
249}
250
251int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
252 struct drm_file *file)
253{
254 struct drm_i915_privateinteldrm_softc *i915 = to_i915(dev);
255 struct drm_i915_gem_caching *args = data;
256 struct drm_i915_gem_object *obj;
257 enum i915_cache_level level;
258 int ret = 0;
259
260 switch (args->caching) {
261 case I915_CACHING_NONE0:
262 level = I915_CACHE_NONE;
263 break;
264 case I915_CACHING_CACHED1:
265 /*
266 * Due to a HW issue on BXT A stepping, GPU stores via a
267 * snooped mapping may leave stale data in a corresponding CPU
268 * cacheline, whereas normally such cachelines would get
269 * invalidated.
270 */
271 if (!HAS_LLC(i915)((&(i915)->__info)->has_llc) && !HAS_SNOOP(i915)((&(i915)->__info)->has_snoop))
272 return -ENODEV19;
273
274 level = I915_CACHE_LLC;
275 break;
276 case I915_CACHING_DISPLAY2:
277 level = HAS_WT(i915)((IS_PLATFORM(i915, INTEL_HASWELL) || IS_PLATFORM(i915, INTEL_BROADWELL
)) && ((i915)->edram_size_mb)) ? I915_CACHE_WT : I915_CACHE_NONE;
278 break;
279 default:
280 return -EINVAL22;
281 }
282
283 obj = i915_gem_object_lookup(file, args->handle);
284 if (!obj)
285 return -ENOENT2;
286
287 /*
288 * The caching mode of proxy object is handled by its generator, and
289 * not allowed to be changed by userspace.
290 */
291 if (i915_gem_object_is_proxy(obj)) {
292 ret = -ENXIO6;
293 goto out;
294 }
295
296 ret = i915_gem_object_lock_interruptible(obj, NULL((void *)0));
297 if (ret)
298 goto out;
299
300 ret = i915_gem_object_set_cache_level(obj, level);
301 i915_gem_object_unlock(obj);
302
303out:
304 i915_gem_object_put(obj);
305 return ret;
306}
307
308/*
309 * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
310 * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
311 * (for pageflips). We only flush the caches while preparing the buffer for
312 * display, the callers are responsible for frontbuffer flush.
313 */
314struct i915_vma *
315i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
316 u32 alignment,
317 const struct i915_ggtt_view *view,
318 unsigned int flags)
319{
320 struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
321 struct i915_gem_ww_ctx ww;
322 struct i915_vma *vma;
323 int ret;
324
325 /* Frame buffer must be in LMEM (no migration yet) */
326 if (HAS_LMEM(i915)((&(i915)->__info)->memory_regions & ((1UL <<
(INTEL_REGION_LMEM)))) && !i915_gem_object_is_lmem(obj))
327 return ERR_PTR(-EINVAL22);
328
329 i915_gem_ww_ctx_init(&ww, true1);
330retry:
331 ret = i915_gem_object_lock(obj, &ww);
332 if (ret)
333 goto err;
334 /*
335 * The display engine is not coherent with the LLC cache on gen6. As
336 * a result, we make sure that the pinning that is about to occur is
337 * done with uncached PTEs. This is lowest common denominator for all
338 * chipsets.
339 *
340 * However for gen6+, we could do better by using the GFDT bit instead
341 * of uncaching, which would allow us to flush all the LLC-cached data
342 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
343 */
344 ret = i915_gem_object_set_cache_level(obj,
345 HAS_WT(i915)((IS_PLATFORM(i915, INTEL_HASWELL) || IS_PLATFORM(i915, INTEL_BROADWELL
)) && ((i915)->edram_size_mb)) ?
346 I915_CACHE_WT : I915_CACHE_NONE);
347 if (ret)
348 goto err;
349
350 /*
351 * As the user may map the buffer once pinned in the display plane
352 * (e.g. libkms for the bootup splash), we have to ensure that we
353 * always use map_and_fenceable for all scanout buffers. However,
354 * it may simply be too big to fit into mappable, in which case
355 * put it anyway and hope that userspace can cope (but always first
356 * try to preserve the existing ABI).
357 */
358 vma = ERR_PTR(-ENOSPC28);
359 if ((flags & PIN_MAPPABLE(1ULL << (3))) == 0 &&
360 (!view || view->type == I915_GGTT_VIEW_NORMAL))
361 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0, alignment,
362 flags | PIN_MAPPABLE(1ULL << (3)) |
363 PIN_NONBLOCK(1ULL << (2)));
364 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK11))
365 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0,
366 alignment, flags);
367 if (IS_ERR(vma)) {
368 ret = PTR_ERR(vma);
369 goto err;
370 }
371
372 vma->display_alignment = max_t(u64, vma->display_alignment, alignment)({ u64 __max_a = (vma->display_alignment); u64 __max_b = (
alignment); __max_a > __max_b ? __max_a : __max_b; });
373
374 i915_gem_object_flush_if_display_locked(obj);
375
376err:
377 if (ret == -EDEADLK11) {
378 ret = i915_gem_ww_ctx_backoff(&ww);
379 if (!ret)
380 goto retry;
381 }
382 i915_gem_ww_ctx_fini(&ww);
383
384 if (ret)
385 return ERR_PTR(ret);
386
387 return vma;
388}
389
390/**
391 * Moves a single object to the CPU read, and possibly write domain.
392 * @obj: object to act on
393 * @write: requesting write or read-only access
394 *
395 * This function returns when the move is complete, including waiting on
396 * flushes to occur.
397 */
398int
399i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool_Bool write)
400{
401 int ret;
402
403 assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0);
404
405 ret = i915_gem_object_wait(obj,
406 I915_WAIT_INTERRUPTIBLE(1UL << (0)) |
407 (write ? I915_WAIT_ALL(1UL << (2)) : 0),
408 MAX_SCHEDULE_TIMEOUT(0x7fffffff));
409 if (ret)
410 return ret;
411
412 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU0x00000001);
413
414 /* Flush the CPU cache if it's still invalid. */
415 if ((obj->read_domains & I915_GEM_DOMAIN_CPU0x00000001) == 0) {
416 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC(1UL << (1)));
417 obj->read_domains |= I915_GEM_DOMAIN_CPU0x00000001;
418 }
419
420 /* It should now be out of any other write domains, and we can update
421 * the domain values for our changes.
422 */
423 GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU)((void)0);
424
425 /* If we're writing through the CPU, then the GPU read domains will
426 * need to be invalidated at next use.
427 */
428 if (write)
429 __start_cpu_write(obj);
430
431 return 0;
432}
433
434/**
435 * Called when user space prepares to use an object with the CPU, either
436 * through the mmap ioctl's mapping or a GTT mapping.
437 * @dev: drm device
438 * @data: ioctl data blob
439 * @file: drm file
440 */
441int
442i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
443 struct drm_file *file)
444{
445 struct drm_i915_gem_set_domain *args = data;
446 struct drm_i915_gem_object *obj;
447 u32 read_domains = args->read_domains;
448 u32 write_domain = args->write_domain;
449 int err;
450
451 /* Only handle setting domains to types used by the CPU. */
452 if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS(0x00000002 | 0x00000004 | 0x00000008 | 0x00000010 | 0x00000020
))
1
Assuming the condition is false
→
2
←
Taking false branch
→
453 return -EINVAL22;
454
455 /*
456 * Having something in the write domain implies it's in the read
457 * domain, and only that read domain. Enforce that in the request.
458 */
459 if (write_domain && read_domains != write_domain)
3
←
Assuming 'write_domain' is 0
→
460 return -EINVAL22;
461
462 if (!read_domains)
4
←
Assuming 'read_domains' is not equal to 0
→
5
←
Taking false branch
→
463 return 0;
464
465 obj = i915_gem_object_lookup(file, args->handle);
466 if (!obj
5.1
'obj' is non-null
5.1
'obj' is non-null
)
6
←
Taking false branch
→
467 return -ENOENT2;
468
469 /*
470 * Try to flush the object off the GPU without holding the lock.
471 * We will repeat the flush holding the lock in the normal manner
472 * to catch cases where we are gazumped.
473 */
474 err = i915_gem_object_wait(obj,
475 I915_WAIT_INTERRUPTIBLE(1UL << (0)) |
476 I915_WAIT_PRIORITY(1UL << (1)) |
477 (write_domain
6.1
'write_domain' is 0
6.1
'write_domain' is 0
 ? I915_WAIT_ALL(1UL << (2)) : 0),
7
←
'?' condition is false
→
478 MAX_SCHEDULE_TIMEOUT(0x7fffffff));
479 if (err)
8
←
Assuming 'err' is 0
→
9
←
Taking false branch
→
480 goto out;
481
482 /*
483 * Proxy objects do not control access to the backing storage, ergo
484 * they cannot be used as a means to manipulate the cache domain
485 * tracking for that backing storage. The proxy object is always
486 * considered to be outside of any cache domain.
487 */
488 if (i915_gem_object_is_proxy(obj)) {
10
←
Assuming the condition is false
→
11
←
Taking false branch
→
489 err = -ENXIO6;
490 goto out;
491 }
492
493 /*
494 * Flush and acquire obj->pages so that we are coherent through
495 * direct access in memory with previous cached writes through
496 * shmemfs and that our cache domain tracking remains valid.
497 * For example, if the obj->filp was moved to swap without us
498 * being notified and releasing the pages, we would mistakenly
499 * continue to assume that the obj remained out of the CPU cached
500 * domain.
501 */
502 err = i915_gem_object_pin_pages(obj);
503 if (err
11.1
'err' is 0
11.1
'err' is 0
)
12
←
Taking false branch
→
504 goto out;
505
506 /*
507 * Already in the desired write domain? Nothing for us to do!
508 *
509 * We apply a little bit of cunning here to catch a broader set of
510 * no-ops. If obj->write_domain is set, we must be in the same
511 * obj->read_domains, and only that domain. Therefore, if that
512 * obj->write_domain matches the request read_domains, we are
513 * already in the same read/write domain and can skip the operation,
514 * without having to further check the requested write_domain.
515 */
516 if (READ_ONCE(obj->write_domain)({ typeof(obj->write_domain) __tmp = *(volatile typeof(obj
->write_domain) *)&(obj->write_domain); membar_datadep_consumer
(); __tmp; }) == read_domains)
13
←
Assuming the condition is false
→
14
←
Taking false branch
→
517 goto out_unpin;
518
519 err = i915_gem_object_lock_interruptible(obj, NULL((void *)0));
15
←
Passing null pointer value via 2nd parameter 'ww'
→
16
←
Calling 'i915_gem_object_lock_interruptible'
→
520 if (err)
521 goto out_unpin;
522
523 if (read_domains & I915_GEM_DOMAIN_WC0x00000080)
524 err = i915_gem_object_set_to_wc_domain(obj, write_domain);
525 else if (read_domains & I915_GEM_DOMAIN_GTT0x00000040)
526 err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
527 else
528 err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
529
530 i915_gem_object_unlock(obj);
531
532 if (write_domain)
533 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
534
535out_unpin:
536 i915_gem_object_unpin_pages(obj);
537out:
538 i915_gem_object_put(obj);
539 return err;
540}
541
542/*
543 * Pins the specified object's pages and synchronizes the object with
544 * GPU accesses. Sets needs_clflush to non-zero if the caller should
545 * flush the object from the CPU cache.
546 */
547int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,
548 unsigned int *needs_clflush)
549{
550 int ret;
551
552 *needs_clflush = 0;
553 if (!i915_gem_object_has_struct_page(obj))
554 return -ENODEV19;
555
556 assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0);
557
558 ret = i915_gem_object_wait(obj,
559 I915_WAIT_INTERRUPTIBLE(1UL << (0)),
560 MAX_SCHEDULE_TIMEOUT(0x7fffffff));
561 if (ret)
562 return ret;
563
564 ret = i915_gem_object_pin_pages(obj);
565 if (ret)
566 return ret;
567
568 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ(1UL << (0)) ||
569 !static_cpu_has(X86_FEATURE_CLFLUSH1)) {
570 ret = i915_gem_object_set_to_cpu_domain(obj, false0);
571 if (ret)
572 goto err_unpin;
573 else
574 goto out;
575 }
576
577 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU0x00000001);
578
579 /* If we're not in the cpu read domain, set ourself into the gtt
580 * read domain and manually flush cachelines (if required). This
581 * optimizes for the case when the gpu will dirty the data
582 * anyway again before the next pread happens.
583 */
584 if (!obj->cache_dirty &&
585 !(obj->read_domains & I915_GEM_DOMAIN_CPU0x00000001))
586 *needs_clflush = CLFLUSH_BEFORE(1UL << (0));
587
588out:
589 /* return with the pages pinned */
590 return 0;
591
592err_unpin:
593 i915_gem_object_unpin_pages(obj);
594 return ret;
595}
596
597int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
598 unsigned int *needs_clflush)
599{
600 int ret;
601
602 *needs_clflush = 0;
603 if (!i915_gem_object_has_struct_page(obj))
604 return -ENODEV19;
605
606 assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0);
607
608 ret = i915_gem_object_wait(obj,
609 I915_WAIT_INTERRUPTIBLE(1UL << (0)) |
610 I915_WAIT_ALL(1UL << (2)),
611 MAX_SCHEDULE_TIMEOUT(0x7fffffff));
612 if (ret)
613 return ret;
614
615 ret = i915_gem_object_pin_pages(obj);
616 if (ret)
617 return ret;
618
619 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE(1UL << (1)) ||
620 !static_cpu_has(X86_FEATURE_CLFLUSH1)) {
621 ret = i915_gem_object_set_to_cpu_domain(obj, true1);
622 if (ret)
623 goto err_unpin;
624 else
625 goto out;
626 }
627
628 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU0x00000001);
629
630 /* If we're not in the cpu write domain, set ourself into the
631 * gtt write domain and manually flush cachelines (as required).
632 * This optimizes for the case when the gpu will use the data
633 * right away and we therefore have to clflush anyway.
634 */
635 if (!obj->cache_dirty) {
636 *needs_clflush |= CLFLUSH_AFTER(1UL << (1));
637
638 /*
639 * Same trick applies to invalidate partially written
640 * cachelines read before writing.
641 */
642 if (!(obj->read_domains & I915_GEM_DOMAIN_CPU0x00000001))
643 *needs_clflush |= CLFLUSH_BEFORE(1UL << (0));
644 }
645
646out:
647 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
648 obj->mm.dirty = true1;
649 /* return with the pages pinned */
650 return 0;
651
652err_unpin:
653 i915_gem_object_unpin_pages(obj);
654 return ret;
655}
←

/usr/src/sys/dev/pci/drm/i915/gem/i915_gem_object.h

1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2016 Intel Corporation
5 */
6
7#ifndef __I915_GEM_OBJECT_H__
8#define __I915_GEM_OBJECT_H__
9
10#include <drm/drm_gem.h>
11#include <drm/drm_file.h>
12#include <drm/drm_device.h>
13
14#include "display/intel_frontbuffer.h"
15#include "i915_gem_object_types.h"
16#include "i915_gem_gtt.h"
17#include "i915_vma_types.h"
18
19void i915_gem_init__objects(struct drm_i915_privateinteldrm_softc *i915);
20
21struct drm_i915_gem_object *i915_gem_object_alloc(void);
22void i915_gem_object_free(struct drm_i915_gem_object *obj);
23
24void i915_gem_object_init(struct drm_i915_gem_object *obj,
25 const struct drm_i915_gem_object_ops *ops,
26 struct lock_class_key *key);
27struct drm_i915_gem_object *
28i915_gem_object_create_shmem(struct drm_i915_privateinteldrm_softc *i915,
29 resource_size_t size);
30struct drm_i915_gem_object *
31i915_gem_object_create_shmem_from_data(struct drm_i915_privateinteldrm_softc *i915,
32 const void *data, resource_size_t size);
33
34extern const struct drm_i915_gem_object_ops i915_gem_shmem_ops;
35void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
36 struct sg_table *pages,
37 bool_Bool needs_clflush);
38
39int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align);
40
41void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file);
42void i915_gem_free_object(struct drm_gem_object *obj);
43
44void i915_gem_flush_free_objects(struct drm_i915_privateinteldrm_softc *i915);
45
46struct sg_table *
47__i915_gem_object_unset_pages(struct drm_i915_gem_object *obj);
48void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
49
50/**
51 * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle
52 * @filp: DRM file private date
53 * @handle: userspace handle
54 *
55 * Returns:
56 *
57 * A pointer to the object named by the handle if such exists on @filp, NULL
58 * otherwise. This object is only valid whilst under the RCU read lock, and
59 * note carefully the object may be in the process of being destroyed.
60 */
61static inline struct drm_i915_gem_object *
62i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle)
63{
64#ifdef CONFIG_LOCKDEP
65 WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map))({ int __ret = !!((debug_locks && !lock_is_held(&
rcu_lock_map))); if (__ret) printf("%s", "WARN_ON(" "debug_locks && !lock_is_held(&rcu_lock_map)"
")"); __builtin_expect(!!(__ret), 0); });
66#endif
67 return idr_find(&file->object_idr, handle);
68}
69
70static inline struct drm_i915_gem_object *
71i915_gem_object_get_rcu(struct drm_i915_gem_object *obj)
72{
73 if (obj && !kref_get_unless_zero(&obj->base.refcount))
74 obj = NULL((void *)0);
75
76 return obj;
77}
78
79static inline struct drm_i915_gem_object *
80i915_gem_object_lookup(struct drm_file *file, u32 handle)
81{
82 struct drm_i915_gem_object *obj;
83
84 rcu_read_lock();
85 obj = i915_gem_object_lookup_rcu(file, handle);
86 obj = i915_gem_object_get_rcu(obj);
87 rcu_read_unlock();
88
89 return obj;
90}
91
92__deprecated
93struct drm_gem_object *
94drm_gem_object_lookup(struct drm_file *file, u32 handle);
95
96__attribute__((nonnull))
97static inline struct drm_i915_gem_object *
98i915_gem_object_get(struct drm_i915_gem_object *obj)
99{
100 drm_gem_object_get(&obj->base);
101 return obj;
102}
103
104__attribute__((nonnull))
105static inline void
106i915_gem_object_put(struct drm_i915_gem_object *obj)
107{
108 __drm_gem_object_put(&obj->base);
109}
110
111#define assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0) dma_resv_assert_held((obj)->base.resv)do { (void)(&((obj)->base.resv)->lock.base); } while
(0)
112
113static inline int __i915_gem_object_lock(struct drm_i915_gem_object *obj,
114 struct i915_gem_ww_ctx *ww,
115 bool_Bool intr)
116{
117 int ret;
118
119 if (intr
19.1
'intr' is true
19.1
'intr' is true
)
20
←
Taking true branch
→
120 ret = dma_resv_lock_interruptible(obj->base.resv, ww
20.1
'ww' is null
20.1
'ww' is null
 ? &ww->ctx : NULL((void *)0));
21
←
'?' condition is false
→
121 else
122 ret = dma_resv_lock(obj->base.resv, ww ? &ww->ctx : NULL((void *)0));
123
124 if (!ret && ww)
22
←
Assuming 'ret' is not equal to 0
→
125 list_add_tail(&obj->obj_link, &ww->obj_list);
126 if (ret == -EALREADY37)
23
←
Assuming the condition is false
→
24
←
Taking false branch
→
127 ret = 0;
128
129 if (ret == -EDEADLK11)
25
←
Assuming the condition is true
→
26
←
Taking true branch
→
130 ww->contended = obj;
27
←
Access to field 'contended' results in a dereference of a null pointer (loaded from variable 'ww')
131
132 return ret;
133}
134
135static inline int i915_gem_object_lock(struct drm_i915_gem_object *obj,
136 struct i915_gem_ww_ctx *ww)
137{
138 return __i915_gem_object_lock(obj, ww, ww && ww->intr);
139}
140
141static inline int i915_gem_object_lock_interruptible(struct drm_i915_gem_object *obj,
142 struct i915_gem_ww_ctx *ww)
143{
144 WARN_ON(ww && !ww->intr)({ int __ret = !!((ww && !ww->intr)); if (__ret) printf
("%s", "WARN_ON(" "ww && !ww->intr" ")"); __builtin_expect
(!!(__ret), 0); });
17
←
Taking false branch
→
145 return __i915_gem_object_lock(obj, ww, true1);
18
←
Passing null pointer value via 2nd parameter 'ww'
→
19
←
Calling '__i915_gem_object_lock'
→
146}
147
148static inline bool_Bool i915_gem_object_trylock(struct drm_i915_gem_object *obj)
149{
150 return dma_resv_trylock(obj->base.resv);
151}
152
153static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj)
154{
155 dma_resv_unlock(obj->base.resv);
156}
157
158struct dma_fence *
159i915_gem_object_lock_fence(struct drm_i915_gem_object *obj);
160void i915_gem_object_unlock_fence(struct drm_i915_gem_object *obj,
161 struct dma_fence *fence);
162
163static inline void
164i915_gem_object_set_readonly(struct drm_i915_gem_object *obj)
165{
166 obj->flags |= I915_BO_READONLY(1UL << (2));
167}
168
169static inline bool_Bool
170i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj)
171{
172 return obj->flags & I915_BO_READONLY(1UL << (2));
173}
174
175static inline bool_Bool
176i915_gem_object_is_contiguous(const struct drm_i915_gem_object *obj)
177{
178 return obj->flags & I915_BO_ALLOC_CONTIGUOUS(1UL << (0));
179}
180
181static inline bool_Bool
182i915_gem_object_is_volatile(const struct drm_i915_gem_object *obj)
183{
184 return obj->flags & I915_BO_ALLOC_VOLATILE(1UL << (1));
185}
186
187static inline void
188i915_gem_object_set_volatile(struct drm_i915_gem_object *obj)
189{
190 obj->flags |= I915_BO_ALLOC_VOLATILE(1UL << (1));
191}
192
193static inline bool_Bool
194i915_gem_object_type_has(const struct drm_i915_gem_object *obj,
195 unsigned long flags)
196{
197 return obj->ops->flags & flags;
198}
199
200static inline bool_Bool
201i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
202{
203 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_STRUCT_PAGE(1UL << (0)));
204}
205
206static inline bool_Bool
207i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj)
208{
209 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_SHRINKABLE(1UL << (2)));
210}
211
212static inline bool_Bool
213i915_gem_object_is_proxy(const struct drm_i915_gem_object *obj)
214{
215 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_PROXY(1UL << (3)));
216}
217
218static inline bool_Bool
219i915_gem_object_never_mmap(const struct drm_i915_gem_object *obj)
220{
221 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_NO_MMAP(1UL << (4)));
222}
223
224static inline bool_Bool
225i915_gem_object_needs_async_cancel(const struct drm_i915_gem_object *obj)
226{
227 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_ASYNC_CANCEL(1UL << (5)));
228}
229
230static inline bool_Bool
231i915_gem_object_is_framebuffer(const struct drm_i915_gem_object *obj)
232{
233 return READ_ONCE(obj->frontbuffer)({ typeof(obj->frontbuffer) __tmp = *(volatile typeof(obj->
frontbuffer) *)&(obj->frontbuffer); membar_datadep_consumer
(); __tmp; });
234}
235
236static inline unsigned int
237i915_gem_object_get_tiling(const struct drm_i915_gem_object *obj)
238{
239 return obj->tiling_and_stride & TILING_MASK(128 - 1);
240}
241
242static inline bool_Bool
243i915_gem_object_is_tiled(const struct drm_i915_gem_object *obj)
244{
245 return i915_gem_object_get_tiling(obj) != I915_TILING_NONE0;
246}
247
248static inline unsigned int
249i915_gem_object_get_stride(const struct drm_i915_gem_object *obj)
250{
251 return obj->tiling_and_stride & STRIDE_MASK(~(128 - 1));
252}
253
254static inline unsigned int
255i915_gem_tile_height(unsigned int tiling)
256{
257 GEM_BUG_ON(!tiling)((void)0);
258 return tiling == I915_TILING_Y2 ? 32 : 8;
259}
260
261static inline unsigned int
262i915_gem_object_get_tile_height(const struct drm_i915_gem_object *obj)
263{
264 return i915_gem_tile_height(i915_gem_object_get_tiling(obj));
265}
266
267static inline unsigned int
268i915_gem_object_get_tile_row_size(const struct drm_i915_gem_object *obj)
269{
270 return (i915_gem_object_get_stride(obj) *
271 i915_gem_object_get_tile_height(obj));
272}
273
274int i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
275 unsigned int tiling, unsigned int stride);
276
277struct scatterlist *
278i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
279 unsigned int n, unsigned int *offset);
280
281struct vm_page *
282i915_gem_object_get_page(struct drm_i915_gem_object *obj,
283 unsigned int n);
284
285struct vm_page *
286i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
287 unsigned int n);
288
289dma_addr_t
290i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
291 unsigned long n,
292 unsigned int *len);
293
294dma_addr_t
295i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
296 unsigned long n);
297
298void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
299 struct sg_table *pages,
300 unsigned int sg_page_sizes);
301
302int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
303int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
304
305enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock/struct_mutex */
306 I915_MM_NORMAL = 0,
307 /*
308 * Only used by struct_mutex, when called "recursively" from
309 * direct-reclaim-esque. Safe because there is only every one
310 * struct_mutex in the entire system.
311 */
312 I915_MM_SHRINKER = 1,
313 /*
314 * Used for obj->mm.lock when allocating pages. Safe because the object
315 * isn't yet on any LRU, and therefore the shrinker can't deadlock on
316 * it. As soon as the object has pages, obj->mm.lock nests within
317 * fs_reclaim.
318 */
319 I915_MM_GET_PAGES = 1,
320};
321
322static inline int __must_check
323i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
324{
325 might_lock_nested(&obj->mm.lock, I915_MM_GET_PAGES);
326
327 if (atomic_inc_not_zero(&obj->mm.pages_pin_count)atomic_add_unless((&obj->mm.pages_pin_count), 1, 0))
328 return 0;
329
330 return __i915_gem_object_get_pages(obj);
331}
332
333static inline bool_Bool
334i915_gem_object_has_pages(struct drm_i915_gem_object *obj)
335{
336 return !IS_ERR_OR_NULL(READ_ONCE(obj->mm.pages)({ typeof(obj->mm.pages) __tmp = *(volatile typeof(obj->
mm.pages) *)&(obj->mm.pages); membar_datadep_consumer(
); __tmp; }));
337}
338
339static inline void
340__i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
341{
342 GEM_BUG_ON(!i915_gem_object_has_pages(obj))((void)0);
343
344 atomic_inc(&obj->mm.pages_pin_count)__sync_fetch_and_add(&obj->mm.pages_pin_count, 1);
345}
346
347static inline bool_Bool
348i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj)
349{
350 return atomic_read(&obj->mm.pages_pin_count)({ typeof(*(&obj->mm.pages_pin_count)) __tmp = *(volatile
typeof(*(&obj->mm.pages_pin_count)) *)&(*(&obj
->mm.pages_pin_count)); membar_datadep_consumer(); __tmp; }
);
351}
352
353static inline void
354__i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
355{
356 GEM_BUG_ON(!i915_gem_object_has_pages(obj))((void)0);
357 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj))((void)0);
358
359 atomic_dec(&obj->mm.pages_pin_count)__sync_fetch_and_sub(&obj->mm.pages_pin_count, 1);
360}
361
362static inline void
363i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
364{
365 __i915_gem_object_unpin_pages(obj);
366}
367
368int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
369void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
370void i915_gem_object_writeback(struct drm_i915_gem_object *obj);
371
372enum i915_map_type {
373 I915_MAP_WB = 0,
374 I915_MAP_WC,
375#define I915_MAP_OVERRIDE(1UL << (31)) BIT(31)(1UL << (31))
376 I915_MAP_FORCE_WB = I915_MAP_WB | I915_MAP_OVERRIDE(1UL << (31)),
377 I915_MAP_FORCE_WC = I915_MAP_WC | I915_MAP_OVERRIDE(1UL << (31)),
378};
379
380/**
381 * i915_gem_object_pin_map - return a contiguous mapping of the entire object
382 * @obj: the object to map into kernel address space
383 * @type: the type of mapping, used to select pgprot_t
384 *
385 * Calls i915_gem_object_pin_pages() to prevent reaping of the object's
386 * pages and then returns a contiguous mapping of the backing storage into
387 * the kernel address space. Based on the @type of mapping, the PTE will be
388 * set to either WriteBack or WriteCombine (via pgprot_t).
389 *
390 * The caller is responsible for calling i915_gem_object_unpin_map() when the
391 * mapping is no longer required.
392 *
393 * Returns the pointer through which to access the mapped object, or an
394 * ERR_PTR() on error.
395 */
396void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
397 enum i915_map_type type);
398
399void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
400 unsigned long offset,
401 unsigned long size);
402static inline void i915_gem_object_flush_map(struct drm_i915_gem_object *obj)
403{
404 __i915_gem_object_flush_map(obj, 0, obj->base.size);
405}
406
407/**
408 * i915_gem_object_unpin_map - releases an earlier mapping
409 * @obj: the object to unmap
410 *
411 * After pinning the object and mapping its pages, once you are finished
412 * with your access, call i915_gem_object_unpin_map() to release the pin
413 * upon the mapping. Once the pin count reaches zero, that mapping may be
414 * removed.
415 */
416static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj)
417{
418 i915_gem_object_unpin_pages(obj);
419}
420
421void __i915_gem_object_release_map(struct drm_i915_gem_object *obj);
422
423void
424i915_gem_object_flush_write_domain(struct drm_i915_gem_object *obj,
425 unsigned int flush_domains);
426
427int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,
428 unsigned int *needs_clflush);
429int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
430 unsigned int *needs_clflush);
431#define CLFLUSH_BEFORE(1UL << (0)) BIT(0)(1UL << (0))
432#define CLFLUSH_AFTER(1UL << (1)) BIT(1)(1UL << (1))
433#define CLFLUSH_FLAGS((1UL << (0)) | (1UL << (1))) (CLFLUSH_BEFORE(1UL << (0)) | CLFLUSH_AFTER(1UL << (1)))
434
435static inline void
436i915_gem_object_finish_access(struct drm_i915_gem_object *obj)
437{
438 i915_gem_object_unpin_pages(obj);
439}
440
441static inline struct intel_engine_cs *
442i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj)
443{
444 struct intel_engine_cs *engine = NULL((void *)0);
445 struct dma_fence *fence;
446
447 rcu_read_lock();
448 fence = dma_resv_get_excl_rcu(obj->base.resv);
449 rcu_read_unlock();
450
451 if (fence && dma_fence_is_i915(fence) && !dma_fence_is_signaled(fence))
452 engine = to_request(fence)->engine;
453 dma_fence_put(fence);
454
455 return engine;
456}
457
458void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
459 unsigned int cache_level);
460void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj);
461void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj);
462
463int __must_check
464i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool_Bool write);
465int __must_check
466i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool_Bool write);
467int __must_check
468i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool_Bool write);
469struct i915_vma * __must_check
470i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
471 u32 alignment,
472 const struct i915_ggtt_view *view,
473 unsigned int flags);
474
475void i915_gem_object_make_unshrinkable(struct drm_i915_gem_object *obj);
476void i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj);
477void i915_gem_object_make_purgeable(struct drm_i915_gem_object *obj);
478
479static inline bool_Bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
480{
481 if (obj->cache_dirty)
482 return false0;
483
484 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE(1UL << (1))))
485 return true1;
486
487 /* Currently in use by HW (display engine)? Keep flushed. */
488 return i915_gem_object_is_framebuffer(obj);
489}
490
491static inline void __start_cpu_write(struct drm_i915_gem_object *obj)
492{
493 obj->read_domains = I915_GEM_DOMAIN_CPU0x00000001;
494 obj->write_domain = I915_GEM_DOMAIN_CPU0x00000001;
495 if (cpu_write_needs_clflush(obj))
496 obj->cache_dirty = true1;
497}
498
499int i915_gem_object_wait(struct drm_i915_gem_object *obj,
500 unsigned int flags,
501 long timeout);
502int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
503 unsigned int flags,
504 const struct i915_sched_attr *attr);
505
506void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj,
507 enum fb_op_origin origin);
508void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj,
509 enum fb_op_origin origin);
510
511static inline void
512i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj,
513 enum fb_op_origin origin)
514{
515 if (unlikely(rcu_access_pointer(obj->frontbuffer))__builtin_expect(!!((obj->frontbuffer)), 0))
516 __i915_gem_object_flush_frontbuffer(obj, origin);
517}
518
519static inline void
520i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj,
521 enum fb_op_origin origin)
522{
523 if (unlikely(rcu_access_pointer(obj->frontbuffer))__builtin_expect(!!((obj->frontbuffer)), 0))
524 __i915_gem_object_invalidate_frontbuffer(obj, origin);
525}
526
527#endif