Bug Summary

File:dev/pci/drm/i915/gt/intel_engine_cs.c
Warning:line 1329, column 2
Value stored to 'x' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name intel_engine_cs.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/dev/pci/drm/i915/gt/intel_engine_cs.c
1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include <drm/drm_print.h>
26
27#include "gem/i915_gem_context.h"
28
29#include "i915_drv.h"
30
31#include "intel_breadcrumbs.h"
32#include "intel_context.h"
33#include "intel_engine.h"
34#include "intel_engine_pm.h"
35#include "intel_engine_user.h"
36#include "intel_gt.h"
37#include "intel_gt_requests.h"
38#include "intel_gt_pm.h"
39#include "intel_lrc.h"
40#include "intel_reset.h"
41#include "intel_ring.h"
42
43/* Haswell does have the CXT_SIZE register however it does not appear to be
44 * valid. Now, docs explain in dwords what is in the context object. The full
45 * size is 70720 bytes, however, the power context and execlist context will
46 * never be saved (power context is stored elsewhere, and execlists don't work
47 * on HSW) - so the final size, including the extra state required for the
48 * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
49 */
50#define HSW_CXT_TOTAL_SIZE(17 * (1 << 12)) (17 * PAGE_SIZE(1 << 12))
51
52#define DEFAULT_LR_CONTEXT_RENDER_SIZE(22 * (1 << 12)) (22 * PAGE_SIZE(1 << 12))
53#define GEN8_LR_CONTEXT_RENDER_SIZE(20 * (1 << 12)) (20 * PAGE_SIZE(1 << 12))
54#define GEN9_LR_CONTEXT_RENDER_SIZE(22 * (1 << 12)) (22 * PAGE_SIZE(1 << 12))
55#define GEN10_LR_CONTEXT_RENDER_SIZE(18 * (1 << 12)) (18 * PAGE_SIZE(1 << 12))
56#define GEN11_LR_CONTEXT_RENDER_SIZE(14 * (1 << 12)) (14 * PAGE_SIZE(1 << 12))
57
58#define GEN8_LR_CONTEXT_OTHER_SIZE( 2 * (1 << 12)) ( 2 * PAGE_SIZE(1 << 12))
59
60#define MAX_MMIO_BASES3 3
61struct engine_info {
62 unsigned int hw_id;
63 u8 class;
64 u8 instance;
65 /* mmio bases table *must* be sorted in reverse gen order */
66 struct engine_mmio_base {
67 u32 gen : 8;
68 u32 base : 24;
69 } mmio_bases[MAX_MMIO_BASES3];
70};
71
72static const struct engine_info intel_engines[] = {
73 [RCS0] = {
74 .hw_id = RCS0_HW0,
75 .class = RENDER_CLASS0,
76 .instance = 0,
77 .mmio_bases = {
78 { .gen = 1, .base = RENDER_RING_BASE0x02000 }
79 },
80 },
81 [BCS0] = {
82 .hw_id = BCS0_HW2,
83 .class = COPY_ENGINE_CLASS3,
84 .instance = 0,
85 .mmio_bases = {
86 { .gen = 6, .base = BLT_RING_BASE0x22000 }
87 },
88 },
89 [VCS0] = {
90 .hw_id = VCS0_HW1,
91 .class = VIDEO_DECODE_CLASS1,
92 .instance = 0,
93 .mmio_bases = {
94 { .gen = 11, .base = GEN11_BSD_RING_BASE0x1c0000 },
95 { .gen = 6, .base = GEN6_BSD_RING_BASE0x12000 },
96 { .gen = 4, .base = BSD_RING_BASE0x04000 }
97 },
98 },
99 [VCS1] = {
100 .hw_id = VCS1_HW4,
101 .class = VIDEO_DECODE_CLASS1,
102 .instance = 1,
103 .mmio_bases = {
104 { .gen = 11, .base = GEN11_BSD2_RING_BASE0x1c4000 },
105 { .gen = 8, .base = GEN8_BSD2_RING_BASE0x1c000 }
106 },
107 },
108 [VCS2] = {
109 .hw_id = VCS2_HW6,
110 .class = VIDEO_DECODE_CLASS1,
111 .instance = 2,
112 .mmio_bases = {
113 { .gen = 11, .base = GEN11_BSD3_RING_BASE0x1d0000 }
114 },
115 },
116 [VCS3] = {
117 .hw_id = VCS3_HW7,
118 .class = VIDEO_DECODE_CLASS1,
119 .instance = 3,
120 .mmio_bases = {
121 { .gen = 11, .base = GEN11_BSD4_RING_BASE0x1d4000 }
122 },
123 },
124 [VECS0] = {
125 .hw_id = VECS0_HW3,
126 .class = VIDEO_ENHANCEMENT_CLASS2,
127 .instance = 0,
128 .mmio_bases = {
129 { .gen = 11, .base = GEN11_VEBOX_RING_BASE0x1c8000 },
130 { .gen = 7, .base = VEBOX_RING_BASE0x1a000 }
131 },
132 },
133 [VECS1] = {
134 .hw_id = VECS1_HW12,
135 .class = VIDEO_ENHANCEMENT_CLASS2,
136 .instance = 1,
137 .mmio_bases = {
138 { .gen = 11, .base = GEN11_VEBOX2_RING_BASE0x1d8000 }
139 },
140 },
141};
142
143/**
144 * intel_engine_context_size() - return the size of the context for an engine
145 * @gt: the gt
146 * @class: engine class
147 *
148 * Each engine class may require a different amount of space for a context
149 * image.
150 *
151 * Return: size (in bytes) of an engine class specific context image
152 *
153 * Note: this size includes the HWSP, which is part of the context image
154 * in LRC mode, but does not include the "shared data page" used with
155 * GuC submission. The caller should account for this if using the GuC.
156 */
157u32 intel_engine_context_size(struct intel_gt *gt, u8 class)
158{
159 struct intel_uncore *uncore = gt->uncore;
160 u32 cxt_size;
161
162 BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE)extern char _ctassert[(!((1ULL << (12)) != (1 << 12
))) ? 1 : -1 ] __attribute__((__unused__));
163
164 switch (class) {
165 case RENDER_CLASS0:
166 switch (INTEL_GEN(gt->i915)((&(gt->i915)->__info)->gen)) {
167 default:
168 MISSING_CASE(INTEL_GEN(gt->i915))({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n"
, "((&(gt->i915)->__info)->gen)", (long)(((&
(gt->i915)->__info)->gen))); __builtin_expect(!!(__ret
), 0); });
169 return DEFAULT_LR_CONTEXT_RENDER_SIZE(22 * (1 << 12));
170 case 12:
171 case 11:
172 return GEN11_LR_CONTEXT_RENDER_SIZE(14 * (1 << 12));
173 case 10:
174 return GEN10_LR_CONTEXT_RENDER_SIZE(18 * (1 << 12));
175 case 9:
176 return GEN9_LR_CONTEXT_RENDER_SIZE(22 * (1 << 12));
177 case 8:
178 return GEN8_LR_CONTEXT_RENDER_SIZE(20 * (1 << 12));
179 case 7:
180 if (IS_HASWELL(gt->i915)IS_PLATFORM(gt->i915, INTEL_HASWELL))
181 return HSW_CXT_TOTAL_SIZE(17 * (1 << 12));
182
183 cxt_size = intel_uncore_read(uncore, GEN7_CXT_SIZE((const i915_reg_t){ .reg = (0x21a8) }));
184 return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,((((((((cxt_size) >> 9) & 0x7f) + (((cxt_size) >>
0) & 0x3f)) * 64) + (((1 << 12)) - 1)) / ((1 <<
12))) * ((1 << 12)))
185 PAGE_SIZE)((((((((cxt_size) >> 9) & 0x7f) + (((cxt_size) >>
0) & 0x3f)) * 64) + (((1 << 12)) - 1)) / ((1 <<
12))) * ((1 << 12)));
186 case 6:
187 cxt_size = intel_uncore_read(uncore, CXT_SIZE((const i915_reg_t){ .reg = (0x21a0) }));
188 return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,((((((((cxt_size) >> 18) & 0x3f) + (((cxt_size) >>
6) & 0x3f) + (((cxt_size) >> 0) & 0x3f)) * 64)
+ (((1 << 12)) - 1)) / ((1 << 12))) * ((1 <<
12)))
189 PAGE_SIZE)((((((((cxt_size) >> 18) & 0x3f) + (((cxt_size) >>
6) & 0x3f) + (((cxt_size) >> 0) & 0x3f)) * 64)
+ (((1 << 12)) - 1)) / ((1 << 12))) * ((1 <<
12)));
190 case 5:
191 case 4:
192 /*
193 * There is a discrepancy here between the size reported
194 * by the register and the size of the context layout
195 * in the docs. Both are described as authorative!
196 *
197 * The discrepancy is on the order of a few cachelines,
198 * but the total is under one page (4k), which is our
199 * minimum allocation anyway so it should all come
200 * out in the wash.
201 */
202 cxt_size = intel_uncore_read(uncore, CXT_SIZE((const i915_reg_t){ .reg = (0x21a0) })) + 1;
203 drm_dbg(&gt->i915->drm,drm_dev_dbg((&gt->i915->drm)->dev, DRM_UT_DRIVER
, "gen%d CXT_SIZE = %d bytes [0x%08x]\n", ((&(gt->i915
)->__info)->gen), cxt_size * 64, cxt_size - 1)
204 "gen%d CXT_SIZE = %d bytes [0x%08x]\n",drm_dev_dbg((&gt->i915->drm)->dev, DRM_UT_DRIVER
, "gen%d CXT_SIZE = %d bytes [0x%08x]\n", ((&(gt->i915
)->__info)->gen), cxt_size * 64, cxt_size - 1)
205 INTEL_GEN(gt->i915), cxt_size * 64,drm_dev_dbg((&gt->i915->drm)->dev, DRM_UT_DRIVER
, "gen%d CXT_SIZE = %d bytes [0x%08x]\n", ((&(gt->i915
)->__info)->gen), cxt_size * 64, cxt_size - 1)
206 cxt_size - 1)drm_dev_dbg((&gt->i915->drm)->dev, DRM_UT_DRIVER
, "gen%d CXT_SIZE = %d bytes [0x%08x]\n", ((&(gt->i915
)->__info)->gen), cxt_size * 64, cxt_size - 1);
207 return round_up(cxt_size * 64, PAGE_SIZE)((((cxt_size * 64) + (((1 << 12)) - 1)) / ((1 << 12
))) * ((1 << 12)));
208 case 3:
209 case 2:
210 /* For the special day when i810 gets merged. */
211 case 1:
212 return 0;
213 }
214 break;
215 default:
216 MISSING_CASE(class)({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n"
, "class", (long)(class)); __builtin_expect(!!(__ret), 0); });
217 fallthroughdo {} while (0);
218 case VIDEO_DECODE_CLASS1:
219 case VIDEO_ENHANCEMENT_CLASS2:
220 case COPY_ENGINE_CLASS3:
221 if (INTEL_GEN(gt->i915)((&(gt->i915)->__info)->gen) < 8)
222 return 0;
223 return GEN8_LR_CONTEXT_OTHER_SIZE( 2 * (1 << 12));
224 }
225}
226
227static u32 __engine_mmio_base(struct drm_i915_privateinteldrm_softc *i915,
228 const struct engine_mmio_base *bases)
229{
230 int i;
231
232 for (i = 0; i < MAX_MMIO_BASES3; i++)
233 if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= bases[i].gen)
234 break;
235
236 GEM_BUG_ON(i == MAX_MMIO_BASES)((void)0);
237 GEM_BUG_ON(!bases[i].base)((void)0);
238
239 return bases[i].base;
240}
241
242static void __sprint_engine_name(struct intel_engine_cs *engine)
243{
244 /*
245 * Before we know what the uABI name for this engine will be,
246 * we still would like to keep track of this engine in the debug logs.
247 * We throw in a ' here as a reminder that this isn't its final name.
248 */
249 GEM_WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s'%u",({ __builtin_expect(!!(!!(snprintf(engine->name, sizeof(engine
->name), "%s'%u", intel_engine_class_repr(engine->class
), engine->instance) >= sizeof(engine->name))), 0); }
)
250 intel_engine_class_repr(engine->class),({ __builtin_expect(!!(!!(snprintf(engine->name, sizeof(engine
->name), "%s'%u", intel_engine_class_repr(engine->class
), engine->instance) >= sizeof(engine->name))), 0); }
)
251 engine->instance) >= sizeof(engine->name))({ __builtin_expect(!!(!!(snprintf(engine->name, sizeof(engine
->name), "%s'%u", intel_engine_class_repr(engine->class
), engine->instance) >= sizeof(engine->name))), 0); }
);
252}
253
254void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask)
255{
256 /*
257 * Though they added more rings on g4x/ilk, they did not add
258 * per-engine HWSTAM until gen6.
259 */
260 if (INTEL_GEN(engine->i915)((&(engine->i915)->__info)->gen) < 6 && engine->class != RENDER_CLASS0)
261 return;
262
263 if (INTEL_GEN(engine->i915)((&(engine->i915)->__info)->gen) >= 3)
264 ENGINE_WRITE(engine, RING_HWSTAM, mask)intel_uncore_write(((engine))->uncore, ((const i915_reg_t)
{ .reg = (((engine)->mmio_base) + 0x98) }), (mask));
265 else
266 ENGINE_WRITE16(engine, RING_HWSTAM, mask)intel_uncore_write16(((engine))->uncore, ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x98) }), (mask));
267}
268
269static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine)
270{
271 /* Mask off all writes into the unknown HWSP */
272 intel_engine_set_hwsp_writemask(engine, ~0u);
273}
274
275static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id)
276{
277 const struct engine_info *info = &intel_engines[id];
278 struct drm_i915_privateinteldrm_softc *i915 = gt->i915;
279 struct intel_engine_cs *engine;
280
281 BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH))extern char _ctassert[(!(4 >= (1UL << (3)))) ? 1 : -
1 ] __attribute__((__unused__));
282 BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH))extern char _ctassert[(!(3 >= (1UL << (6)))) ? 1 : -
1 ] __attribute__((__unused__));
283
284 if (GEM_DEBUG_WARN_ON(id >= ARRAY_SIZE(gt->engine))({ ((void)0); 0; }))
285 return -EINVAL22;
286
287 if (GEM_DEBUG_WARN_ON(info->class > MAX_ENGINE_CLASS)({ ((void)0); 0; }))
288 return -EINVAL22;
289
290 if (GEM_DEBUG_WARN_ON(info->instance > MAX_ENGINE_INSTANCE)({ ((void)0); 0; }))
291 return -EINVAL22;
292
293 if (GEM_DEBUG_WARN_ON(gt->engine_class[info->class][info->instance])({ ((void)0); 0; }))
294 return -EINVAL22;
295
296 engine = kzalloc(sizeof(*engine), GFP_KERNEL(0x0001 | 0x0004));
297 if (!engine)
298 return -ENOMEM12;
299
300 BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES)extern char _ctassert[(!((8 * sizeof(engine->mask)) < I915_NUM_ENGINES
)) ? 1 : -1 ] __attribute__((__unused__));
301
302 engine->id = id;
303 engine->legacy_idx = INVALID_ENGINE((enum intel_engine_id)-1);
304 engine->mask = BIT(id)(1UL << (id));
305 engine->i915 = i915;
306 engine->gt = gt;
307 engine->uncore = gt->uncore;
308 engine->hw_id = engine->guc_id = info->hw_id;
309 engine->mmio_base = __engine_mmio_base(i915, info->mmio_bases);
310
311 engine->class = info->class;
312 engine->instance = info->instance;
313 __sprint_engine_name(engine);
314
315 engine->props.heartbeat_interval_ms =
316 CONFIG_DRM_I915_HEARTBEAT_INTERVAL2500;
317 engine->props.max_busywait_duration_ns =
318 CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT8000;
319 engine->props.preempt_timeout_ms =
320 CONFIG_DRM_I915_PREEMPT_TIMEOUT640;
321 engine->props.stop_timeout_ms =
322 CONFIG_DRM_I915_STOP_TIMEOUT100;
323 engine->props.timeslice_duration_ms =
324 CONFIG_DRM_I915_TIMESLICE_DURATION1;
325
326 /* Override to uninterruptible for OpenCL workloads. */
327 if (INTEL_GEN(i915)((&(i915)->__info)->gen) == 12 && engine->class == RENDER_CLASS0)
328 engine->props.preempt_timeout_ms = 0;
329
330 engine->defaults = engine->props; /* never to change again */
331
332 engine->context_size = intel_engine_context_size(gt, engine->class);
333 if (WARN_ON(engine->context_size > BIT(20))({ int __ret = !!((engine->context_size > (1UL <<
(20)))); if (__ret) printf("%s", "WARN_ON(" "engine->context_size > (1UL << (20))"
")"); __builtin_expect(!!(__ret), 0); }))
334 engine->context_size = 0;
335 if (engine->context_size)
336 DRIVER_CAPS(i915)(&(i915)->caps)->has_logical_contexts = true1;
337
338 /* Nothing to do here, execute in order of dependencies */
339 engine->schedule = NULL((void *)0);
340
341 ewma__engine_latency_init(&engine->latency);
342 seqlock_init(&engine->stats.lock, IPL_TTY0x9);
343
344 ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
345
346 /* Scrub mmio state on takeover */
347 intel_engine_sanitize_mmio(engine);
348
349 gt->engine_class[info->class][info->instance] = engine;
350 gt->engine[id] = engine;
351
352 return 0;
353}
354
355static void __setup_engine_capabilities(struct intel_engine_cs *engine)
356{
357 struct drm_i915_privateinteldrm_softc *i915 = engine->i915;
358
359 if (engine->class == VIDEO_DECODE_CLASS1) {
360 /*
361 * HEVC support is present on first engine instance
362 * before Gen11 and on all instances afterwards.
363 */
364 if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 11 ||
365 (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 9 && engine->instance == 0))
366 engine->uabi_capabilities |=
367 I915_VIDEO_CLASS_CAPABILITY_HEVC(1 << 0);
368
369 /*
370 * SFC block is present only on even logical engine
371 * instances.
372 */
373 if ((INTEL_GEN(i915)((&(i915)->__info)->gen) >= 11 &&
374 (engine->gt->info.vdbox_sfc_access &
375 BIT(engine->instance)(1UL << (engine->instance)))) ||
376 (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 9 && engine->instance == 0))
377 engine->uabi_capabilities |=
378 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC(1 << 1);
379 } else if (engine->class == VIDEO_ENHANCEMENT_CLASS2) {
380 if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 9)
381 engine->uabi_capabilities |=
382 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC(1 << 1);
383 }
384}
385
386static void intel_setup_engine_capabilities(struct intel_gt *gt)
387{
388 struct intel_engine_cs *engine;
389 enum intel_engine_id id;
390
391 for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else
392 __setup_engine_capabilities(engine);
393}
394
395/**
396 * intel_engines_release() - free the resources allocated for Command Streamers
397 * @gt: pointer to struct intel_gt
398 */
399void intel_engines_release(struct intel_gt *gt)
400{
401 struct intel_engine_cs *engine;
402 enum intel_engine_id id;
403
404 /*
405 * Before we release the resources held by engine, we must be certain
406 * that the HW is no longer accessing them -- having the GPU scribble
407 * to or read from a page being used for something else causes no end
408 * of fun.
409 *
410 * The GPU should be reset by this point, but assume the worst just
411 * in case we aborted before completely initialising the engines.
412 */
413 GEM_BUG_ON(intel_gt_pm_is_awake(gt))((void)0);
414 if (!INTEL_INFO(gt->i915)(&(gt->i915)->__info)->gpu_reset_clobbers_display)
415 __intel_gt_reset(gt, ALL_ENGINES((intel_engine_mask_t)~0ul));
416
417 /* Decouple the backend; but keep the layout for late GPU resets */
418 for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else {
419 if (!engine->release)
420 continue;
421
422 intel_wakeref_wait_for_idle(&engine->wakeref);
423 GEM_BUG_ON(intel_engine_pm_is_awake(engine))((void)0);
424
425 engine->release(engine);
426 engine->release = NULL((void *)0);
427
428 memset(&engine->reset, 0, sizeof(engine->reset))__builtin_memset((&engine->reset), (0), (sizeof(engine
->reset)));
429 }
430}
431
432void intel_engine_free_request_pool(struct intel_engine_cs *engine)
433{
434 if (!engine->request_pool)
435 return;
436
437#ifdef __linux__
438 kmem_cache_free(i915_request_slab_cache(), engine->request_pool);
439#else
440 pool_put(i915_request_slab_cache(), engine->request_pool);
441#endif
442}
443
444void intel_engines_free(struct intel_gt *gt)
445{
446 struct intel_engine_cs *engine;
447 enum intel_engine_id id;
448
449 /* Free the requests! dma-resv keeps fences around for an eternity */
450 rcu_barrier()__asm volatile("" : : : "memory");
451
452 for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else {
453 intel_engine_free_request_pool(engine);
454 kfree(engine);
455 gt->engine[id] = NULL((void *)0);
456 }
457}
458
459/*
460 * Determine which engines are fused off in our particular hardware.
461 * Note that we have a catch-22 situation where we need to be able to access
462 * the blitter forcewake domain to read the engine fuses, but at the same time
463 * we need to know which engines are available on the system to know which
464 * forcewake domains are present. We solve this by intializing the forcewake
465 * domains based on the full engine mask in the platform capabilities before
466 * calling this function and pruning the domains for fused-off engines
467 * afterwards.
468 */
469static intel_engine_mask_t init_engine_mask(struct intel_gt *gt)
470{
471 struct drm_i915_privateinteldrm_softc *i915 = gt->i915;
472 struct intel_gt_info *info = &gt->info;
473 struct intel_uncore *uncore = gt->uncore;
474 unsigned int logical_vdbox = 0;
475 unsigned int i;
476 u32 media_fuse;
477 u16 vdbox_mask;
478 u16 vebox_mask;
479
480 info->engine_mask = INTEL_INFO(i915)(&(i915)->__info)->platform_engine_mask;
481
482 if (INTEL_GEN(i915)((&(i915)->__info)->gen) < 11)
483 return info->engine_mask;
484
485 media_fuse = ~intel_uncore_read(uncore, GEN11_GT_VEBOX_VDBOX_DISABLE((const i915_reg_t){ .reg = (0x9140) }));
486
487 vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK0xff;
488 vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK(0x0f << 16)) >>
489 GEN11_GT_VEBOX_DISABLE_SHIFT16;
490
491 for (i = 0; i < I915_MAX_VCS4; i++) {
492 if (!HAS_ENGINE(gt, _VCS(i))(((gt)->info.engine_mask) & (1UL << ((VCS0 + (i)
))))) {
493 vdbox_mask &= ~BIT(i)(1UL << (i));
494 continue;
495 }
496
497 if (!(BIT(i)(1UL << (i)) & vdbox_mask)) {
498 info->engine_mask &= ~BIT(_VCS(i))(1UL << ((VCS0 + (i))));
499 drm_dbg(&i915->drm, "vcs%u fused off\n", i)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "vcs%u fused off\n"
, i);
500 continue;
501 }
502
503 /*
504 * In Gen11, only even numbered logical VDBOXes are
505 * hooked up to an SFC (Scaler & Format Converter) unit.
506 * In TGL each VDBOX has access to an SFC.
507 */
508 if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 12 || logical_vdbox++ % 2 == 0)
509 gt->info.vdbox_sfc_access |= BIT(i)(1UL << (i));
510 }
511 drm_dbg(&i915->drm, "vdbox enable: %04x, instances: %04lx\n",drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "vdbox enable: %04x, instances: %04lx\n"
, vdbox_mask, ({ unsigned int first__ = (VCS0); unsigned int count__
= (4); ((gt)->info.engine_mask & (((~0UL) >> (64
- (first__ + count__ - 1) - 1)) & ((~0UL) << (first__
)))) >> first__; }))
512 vdbox_mask, VDBOX_MASK(gt))drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "vdbox enable: %04x, instances: %04lx\n"
, vdbox_mask, ({ unsigned int first__ = (VCS0); unsigned int count__
= (4); ((gt)->info.engine_mask & (((~0UL) >> (64
- (first__ + count__ - 1) - 1)) & ((~0UL) << (first__
)))) >> first__; }));
513 GEM_BUG_ON(vdbox_mask != VDBOX_MASK(gt))((void)0);
514
515 for (i = 0; i < I915_MAX_VECS2; i++) {
516 if (!HAS_ENGINE(gt, _VECS(i))(((gt)->info.engine_mask) & (1UL << ((VECS0 + (i
)))))) {
517 vebox_mask &= ~BIT(i)(1UL << (i));
518 continue;
519 }
520
521 if (!(BIT(i)(1UL << (i)) & vebox_mask)) {
522 info->engine_mask &= ~BIT(_VECS(i))(1UL << ((VECS0 + (i))));
523 drm_dbg(&i915->drm, "vecs%u fused off\n", i)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "vecs%u fused off\n"
, i);
524 }
525 }
526 drm_dbg(&i915->drm, "vebox enable: %04x, instances: %04lx\n",drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "vebox enable: %04x, instances: %04lx\n"
, vebox_mask, ({ unsigned int first__ = (VECS0); unsigned int
count__ = (2); ((gt)->info.engine_mask & (((~0UL) >>
(64 - (first__ + count__ - 1) - 1)) & ((~0UL) << (
first__)))) >> first__; }))
527 vebox_mask, VEBOX_MASK(gt))drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "vebox enable: %04x, instances: %04lx\n"
, vebox_mask, ({ unsigned int first__ = (VECS0); unsigned int
count__ = (2); ((gt)->info.engine_mask & (((~0UL) >>
(64 - (first__ + count__ - 1) - 1)) & ((~0UL) << (
first__)))) >> first__; }));
528 GEM_BUG_ON(vebox_mask != VEBOX_MASK(gt))((void)0);
529
530 return info->engine_mask;
531}
532
533/**
534 * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
535 * @gt: pointer to struct intel_gt
536 *
537 * Return: non-zero if the initialization failed.
538 */
539int intel_engines_init_mmio(struct intel_gt *gt)
540{
541 struct drm_i915_privateinteldrm_softc *i915 = gt->i915;
542 const unsigned int engine_mask = init_engine_mask(gt);
543 unsigned int mask = 0;
544 unsigned int i;
545 int err;
546
547 drm_WARN_ON(&i915->drm, engine_mask == 0)({ int __ret = !!((engine_mask == 0)); if (__ret) printf("%s %s: "
"%s", dev_driver_string(((&i915->drm))->dev), "", "drm_WARN_ON("
"engine_mask == 0" ")"); __builtin_expect(!!(__ret), 0); });
548 drm_WARN_ON(&i915->drm, engine_mask &({ int __ret = !!((engine_mask & (((~0UL) >> (64 - (
(8 * sizeof(mask)) - 1) - 1)) & ((~0UL) << (I915_NUM_ENGINES
))))); if (__ret) printf("%s %s: " "%s", dev_driver_string(((
&i915->drm))->dev), "", "drm_WARN_ON(" "engine_mask & (((~0UL) >> (64 - ((8 * sizeof(mask)) - 1) - 1)) & ((~0UL) << (I915_NUM_ENGINES)))"
")"); __builtin_expect(!!(__ret), 0); })
549 GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES))({ int __ret = !!((engine_mask & (((~0UL) >> (64 - (
(8 * sizeof(mask)) - 1) - 1)) & ((~0UL) << (I915_NUM_ENGINES
))))); if (__ret) printf("%s %s: " "%s", dev_driver_string(((
&i915->drm))->dev), "", "drm_WARN_ON(" "engine_mask & (((~0UL) >> (64 - ((8 * sizeof(mask)) - 1) - 1)) & ((~0UL) << (I915_NUM_ENGINES)))"
")"); __builtin_expect(!!(__ret), 0); });
550
551 if (i915_inject_probe_failure(i915)({ ((void)0); 0; }))
552 return -ENODEV19;
553
554 for (i = 0; i < ARRAY_SIZE(intel_engines)(sizeof((intel_engines)) / sizeof((intel_engines)[0])); i++) {
555 if (!HAS_ENGINE(gt, i)(((gt)->info.engine_mask) & (1UL << (i))))
556 continue;
557
558 err = intel_engine_setup(gt, i);
559 if (err)
560 goto cleanup;
561
562 mask |= BIT(i)(1UL << (i));
563 }
564
565 /*
566 * Catch failures to update intel_engines table when the new engines
567 * are added to the driver by a warning and disabling the forgotten
568 * engines.
569 */
570 if (drm_WARN_ON(&i915->drm, mask != engine_mask)({ int __ret = !!((mask != engine_mask)); if (__ret) printf("%s %s: "
"%s", dev_driver_string(((&i915->drm))->dev), "", "drm_WARN_ON("
"mask != engine_mask" ")"); __builtin_expect(!!(__ret), 0); }
))
571 gt->info.engine_mask = mask;
572
573 gt->info.num_engines = hweight32(mask);
574
575 intel_gt_check_and_clear_faults(gt);
576
577 intel_setup_engine_capabilities(gt);
578
579 intel_uncore_prune_engine_fw_domains(gt->uncore, gt);
580
581 return 0;
582
583cleanup:
584 intel_engines_free(gt);
585 return err;
586}
587
588void intel_engine_init_execlists(struct intel_engine_cs *engine)
589{
590 struct intel_engine_execlists * const execlists = &engine->execlists;
591
592 execlists->port_mask = 1;
593 GEM_BUG_ON(!is_power_of_2(execlists_num_ports(execlists)))((void)0);
594 GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS)((void)0);
595
596 memset(execlists->pending, 0, sizeof(execlists->pending))__builtin_memset((execlists->pending), (0), (sizeof(execlists
->pending)));
597 execlists->active =
598 memset(execlists->inflight, 0, sizeof(execlists->inflight))__builtin_memset((execlists->inflight), (0), (sizeof(execlists
->inflight)));
599
600 execlists->queue_priority_hint = INT_MIN(-0x7fffffff-1);
601 execlists->queue = RB_ROOT_CACHED(struct rb_root_cached) { ((void *)0) };
602}
603
604static void cleanup_status_page(struct intel_engine_cs *engine)
605{
606 struct i915_vma *vma;
607
608 /* Prevent writes into HWSP after returning the page to the system */
609 intel_engine_set_hwsp_writemask(engine, ~0u);
610
611 vma = fetch_and_zero(&engine->status_page.vma)({ typeof(*&engine->status_page.vma) __T = *(&engine
->status_page.vma); *(&engine->status_page.vma) = (
typeof(*&engine->status_page.vma))0; __T; });
612 if (!vma)
613 return;
614
615 if (!HWS_NEEDS_PHYSICAL(engine->i915)((&(engine->i915)->__info)->hws_needs_physical))
616 i915_vma_unpin(vma);
617
618 i915_gem_object_unpin_map(vma->obj);
619 i915_gem_object_put(vma->obj);
620}
621
622static int pin_ggtt_status_page(struct intel_engine_cs *engine,
623 struct i915_vma *vma)
624{
625 unsigned int flags;
626
627 if (!HAS_LLC(engine->i915)((&(engine->i915)->__info)->has_llc) && i915_ggtt_has_aperture(engine->gt->ggtt))
628 /*
629 * On g33, we cannot place HWS above 256MiB, so
630 * restrict its pinning to the low mappable arena.
631 * Though this restriction is not documented for
632 * gen4, gen5, or byt, they also behave similarly
633 * and hang if the HWS is placed at the top of the
634 * GTT. To generalise, it appears that all !llc
635 * platforms have issues with us placing the HWS
636 * above the mappable region (even though we never
637 * actually map it).
638 */
639 flags = PIN_MAPPABLE(1ULL << (3));
640 else
641 flags = PIN_HIGH(1ULL << (5));
642
643 return i915_ggtt_pin(vma, NULL((void *)0), 0, flags);
644}
645
646static int init_status_page(struct intel_engine_cs *engine)
647{
648 struct drm_i915_gem_object *obj;
649 struct i915_vma *vma;
650 void *vaddr;
651 int ret;
652
653 /*
654 * Though the HWS register does support 36bit addresses, historically
655 * we have had hangs and corruption reported due to wild writes if
656 * the HWS is placed above 4G. We only allow objects to be allocated
657 * in GFP_DMA32 for i965, and no earlier physical address users had
658 * access to more than 4G.
659 */
660 obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE(1 << 12));
661 if (IS_ERR(obj)) {
662 drm_err(&engine->i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Failed to allocate status page\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__)
663 "Failed to allocate status page\n")printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Failed to allocate status page\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
664 return PTR_ERR(obj);
665 }
666
667 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
668
669 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL((void *)0));
670 if (IS_ERR(vma)) {
671 ret = PTR_ERR(vma);
672 goto err;
673 }
674
675 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
676 if (IS_ERR(vaddr)) {
677 ret = PTR_ERR(vaddr);
678 goto err;
679 }
680
681 engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE)__builtin_memset((vaddr), (0), ((1 << 12)));
682 engine->status_page.vma = vma;
683
684 if (!HWS_NEEDS_PHYSICAL(engine->i915)((&(engine->i915)->__info)->hws_needs_physical)) {
685 ret = pin_ggtt_status_page(engine, vma);
686 if (ret)
687 goto err_unpin;
688 }
689
690 return 0;
691
692err_unpin:
693 i915_gem_object_unpin_map(obj);
694err:
695 i915_gem_object_put(obj);
696 return ret;
697}
698
699static int engine_setup_common(struct intel_engine_cs *engine)
700{
701 int err;
702
703 init_llist_head(&engine->barrier_tasks);
704
705 err = init_status_page(engine);
706 if (err)
707 return err;
708
709 engine->breadcrumbs = intel_breadcrumbs_create(engine);
710 if (!engine->breadcrumbs) {
711 err = -ENOMEM12;
712 goto err_status;
713 }
714
715 err = intel_engine_init_cmd_parser(engine);
716 if (err)
717 goto err_cmd_parser;
718
719 intel_engine_init_active(engine, ENGINE_PHYSICAL0);
720 intel_engine_init_execlists(engine);
721 intel_engine_init__pm(engine);
722 intel_engine_init_retire(engine);
723
724 /* Use the whole device by default */
725 engine->sseu =
726 intel_sseu_from_device_info(&engine->gt->info.sseu);
727
728 intel_engine_init_workarounds(engine);
729 intel_engine_init_whitelist(engine);
730 intel_engine_init_ctx_wa(engine);
731
732 return 0;
733
734err_cmd_parser:
735 intel_breadcrumbs_free(engine->breadcrumbs);
736err_status:
737 cleanup_status_page(engine);
738 return err;
739}
740
741struct measure_breadcrumb {
742 struct i915_request rq;
743 struct intel_ring ring;
744 u32 cs[2048];
745};
746
747static int measure_breadcrumb_dw(struct intel_context *ce)
748{
749 struct intel_engine_cs *engine = ce->engine;
750 struct measure_breadcrumb *frame;
751 int dw;
752
753 GEM_BUG_ON(!engine->gt->scratch)((void)0);
754
755 frame = kzalloc(sizeof(*frame), GFP_KERNEL(0x0001 | 0x0004));
756 if (!frame)
757 return -ENOMEM12;
758
759 frame->rq.engine = engine;
760 frame->rq.context = ce;
761 rcu_assign_pointer(frame->rq.timeline, ce->timeline)do { (frame->rq.timeline) = (ce->timeline); } while(0);
762
763 frame->ring.vaddr = frame->cs;
764 frame->ring.size = sizeof(frame->cs);
765 frame->ring.wrap =
766 BITS_PER_TYPE(frame->ring.size)(8 * sizeof(frame->ring.size)) - ilog2(frame->ring.size)((sizeof(frame->ring.size) <= 4) ? (fls(frame->ring.
size) - 1) : (flsl(frame->ring.size) - 1));
767 frame->ring.effective_size = frame->ring.size;
768 intel_ring_update_space(&frame->ring);
769 frame->rq.ring = &frame->ring;
770
771 mutex_lock(&ce->timeline->mutex)rw_enter_write(&ce->timeline->mutex);
772 spin_lock_irq(&engine->active.lock)mtx_enter(&engine->active.lock);
773
774 dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs;
775
776 spin_unlock_irq(&engine->active.lock)mtx_leave(&engine->active.lock);
777 mutex_unlock(&ce->timeline->mutex)rw_exit_write(&ce->timeline->mutex);
778
779 GEM_BUG_ON(dw & 1)((void)0); /* RING_TAIL must be qword aligned */
780
781 kfree(frame);
782 return dw;
783}
784
785void
786intel_engine_init_active(struct intel_engine_cs *engine, unsigned int subclass)
787{
788 INIT_LIST_HEAD(&engine->active.requests);
789 INIT_LIST_HEAD(&engine->active.hold);
790
791 mtx_init(&engine->active.lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&engine->
active.lock), ((((0x9)) > 0x0 && ((0x9)) < 0x9)
? 0x9 : ((0x9)))); } while (0);
792 lockdep_set_subclass(&engine->active.lock, subclass);
793
794 /*
795 * Due to an interesting quirk in lockdep's internal debug tracking,
796 * after setting a subclass we must ensure the lock is used. Otherwise,
797 * nr_unused_locks is incremented once too often.
798 */
799#ifdef CONFIG_DEBUG_LOCK_ALLOC
800 local_irq_disable()intr_disable();
801 lock_map_acquire(&engine->active.lock.dep_map);
802 lock_map_release(&engine->active.lock.dep_map);
803 local_irq_enable()intr_enable();
804#endif
805}
806
807static struct intel_context *
808create_pinned_context(struct intel_engine_cs *engine,
809 unsigned int hwsp,
810 struct lock_class_key *key,
811 const char *name)
812{
813 struct intel_context *ce;
814 int err;
815
816 ce = intel_context_create(engine);
817 if (IS_ERR(ce))
818 return ce;
819
820 __set_bit(CONTEXT_BARRIER_BIT0, &ce->flags);
821 ce->timeline = page_pack_bits(NULL, hwsp)({ unsigned long __bits = (hwsp); ((void)0); ((typeof(((void *
)0)))((unsigned long)(((void *)0)) | __bits)); });
822
823 err = intel_context_pin(ce); /* perma-pin so it is always available */
824 if (err) {
825 intel_context_put(ce);
826 return ERR_PTR(err);
827 }
828
829 /*
830 * Give our perma-pinned kernel timelines a separate lockdep class,
831 * so that we can use them from within the normal user timelines
832 * should we need to inject GPU operations during their request
833 * construction.
834 */
835 lockdep_set_class_and_name(&ce->timeline->mutex, key, name);
836
837 return ce;
838}
839
840static struct intel_context *
841create_kernel_context(struct intel_engine_cs *engine)
842{
843 static struct lock_class_key kernel;
844
845 return create_pinned_context(engine, I915_GEM_HWS_SEQNO_ADDR(0x40 * sizeof(u32)),
846 &kernel, "kernel_context");
847}
848
849/**
850 * intel_engines_init_common - initialize cengine state which might require hw access
851 * @engine: Engine to initialize.
852 *
853 * Initializes @engine@ structure members shared between legacy and execlists
854 * submission modes which do require hardware access.
855 *
856 * Typcally done at later stages of submission mode specific engine setup.
857 *
858 * Returns zero on success or an error code on failure.
859 */
860static int engine_init_common(struct intel_engine_cs *engine)
861{
862 struct intel_context *ce;
863 int ret;
864
865 engine->set_default_submission(engine);
866
867 /*
868 * We may need to do things with the shrinker which
869 * require us to immediately switch back to the default
870 * context. This can cause a problem as pinning the
871 * default context also requires GTT space which may not
872 * be available. To avoid this we always pin the default
873 * context.
874 */
875 ce = create_kernel_context(engine);
876 if (IS_ERR(ce))
877 return PTR_ERR(ce);
878
879 ret = measure_breadcrumb_dw(ce);
880 if (ret < 0)
881 goto err_context;
882
883 engine->emit_fini_breadcrumb_dw = ret;
884 engine->kernel_context = ce;
885
886 return 0;
887
888err_context:
889 intel_context_put(ce);
890 return ret;
891}
892
893int intel_engines_init(struct intel_gt *gt)
894{
895 int (*setup)(struct intel_engine_cs *engine);
896 struct intel_engine_cs *engine;
897 enum intel_engine_id id;
898 int err;
899
900 if (HAS_EXECLISTS(gt->i915)((&(gt->i915)->__info)->has_logical_ring_contexts
))
901 setup = intel_execlists_submission_setup;
902 else
903 setup = intel_ring_submission_setup;
904
905 for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else {
906 err = engine_setup_common(engine);
907 if (err)
908 return err;
909
910 err = setup(engine);
911 if (err)
912 return err;
913
914 err = engine_init_common(engine);
915 if (err)
916 return err;
917
918 intel_engine_add_user(engine);
919 }
920
921 return 0;
922}
923
924/**
925 * intel_engines_cleanup_common - cleans up the engine state created by
926 * the common initiailizers.
927 * @engine: Engine to cleanup.
928 *
929 * This cleans up everything created by the common helpers.
930 */
931void intel_engine_cleanup_common(struct intel_engine_cs *engine)
932{
933 GEM_BUG_ON(!list_empty(&engine->active.requests))((void)0);
934 tasklet_kill(&engine->execlists.tasklet); /* flush the callback */
935
936 cleanup_status_page(engine);
937 intel_breadcrumbs_free(engine->breadcrumbs);
938
939 intel_engine_fini_retire(engine);
940 intel_engine_cleanup_cmd_parser(engine);
941
942 if (engine->default_state)
943 uao_detach(engine->default_state);
944
945 if (engine->kernel_context) {
946 intel_context_unpin(engine->kernel_context);
947 intel_context_put(engine->kernel_context);
948 }
949 GEM_BUG_ON(!llist_empty(&engine->barrier_tasks))((void)0);
950
951 intel_wa_list_free(&engine->ctx_wa_list);
952 intel_wa_list_free(&engine->wa_list);
953 intel_wa_list_free(&engine->whitelist);
954}
955
956/**
957 * intel_engine_resume - re-initializes the HW state of the engine
958 * @engine: Engine to resume.
959 *
960 * Returns zero on success or an error code on failure.
961 */
962int intel_engine_resume(struct intel_engine_cs *engine)
963{
964 intel_engine_apply_workarounds(engine);
965 intel_engine_apply_whitelist(engine);
966
967 return engine->resume(engine);
968}
969
970u64 intel_engine_get_active_head(const struct intel_engine_cs *engine)
971{
972 struct drm_i915_privateinteldrm_softc *i915 = engine->i915;
973
974 u64 acthd;
975
976 if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 8)
977 acthd = ENGINE_READ64(engine, RING_ACTHD, RING_ACTHD_UDW)intel_uncore_read64_2x32(((engine))->uncore, ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x74) }), ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x5c) }));
978 else if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 4)
979 acthd = ENGINE_READ(engine, RING_ACTHD)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x74) }));
980 else
981 acthd = ENGINE_READ(engine, ACTHD)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0xc8) }));
982
983 return acthd;
984}
985
986u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine)
987{
988 u64 bbaddr;
989
990 if (INTEL_GEN(engine->i915)((&(engine->i915)->__info)->gen) >= 8)
991 bbaddr = ENGINE_READ64(engine, RING_BBADDR, RING_BBADDR_UDW)intel_uncore_read64_2x32(((engine))->uncore, ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x140) }), ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x168) }));
992 else
993 bbaddr = ENGINE_READ(engine, RING_BBADDR)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x140) }));
994
995 return bbaddr;
996}
997
998static unsigned long stop_timeout(const struct intel_engine_cs *engine)
999{
1000 if (in_atomic()0 || irqs_disabled()) /* inside atomic preempt-reset? */
1001 return 0;
1002
1003 /*
1004 * If we are doing a normal GPU reset, we can take our time and allow
1005 * the engine to quiesce. We've stopped submission to the engine, and
1006 * if we wait long enough an innocent context should complete and
1007 * leave the engine idle. So they should not be caught unaware by
1008 * the forthcoming GPU reset (which usually follows the stop_cs)!
1009 */
1010 return READ_ONCE(engine->props.stop_timeout_ms)({ typeof(engine->props.stop_timeout_ms) __tmp = *(volatile
typeof(engine->props.stop_timeout_ms) *)&(engine->
props.stop_timeout_ms); membar_datadep_consumer(); __tmp; });
1011}
1012
1013int intel_engine_stop_cs(struct intel_engine_cs *engine)
1014{
1015 struct intel_uncore *uncore = engine->uncore;
1016 const u32 base = engine->mmio_base;
1017 const i915_reg_t mode = RING_MI_MODE(base)((const i915_reg_t){ .reg = ((base) + 0x9c) });
1018 int err;
1019
1020 if (INTEL_GEN(engine->i915)((&(engine->i915)->__info)->gen) < 3)
1021 return -ENODEV19;
1022
1023 ENGINE_TRACE(engine, "\n")do { const struct intel_engine_cs *e__ __attribute__((__unused__
)) = (engine); do { } while (0); } while (0);
1024
1025 intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING))__raw_uncore_write32(uncore, mode, ({ typeof((1 << 8)) _a
= ((1 << 8)); ({ if (__builtin_constant_p(_a)) do { } while
(0); if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p
(_a) && __builtin_constant_p(_a)) do { } while (0); (
(_a) << 16 | (_a)); }); }));
1026
1027 err = 0;
1028 if (__intel_wait_for_register_fw(uncore,
1029 mode, MODE_IDLE(1 << 9), MODE_IDLE(1 << 9),
1030 1000, stop_timeout(engine),
1031 NULL((void *)0))) {
1032 ENGINE_TRACE(engine, "timed out on STOP_RING -> IDLE\n")do { const struct intel_engine_cs *e__ __attribute__((__unused__
)) = (engine); do { } while (0); } while (0);
1033 err = -ETIMEDOUT60;
1034 }
1035
1036 /* A final mmio read to let GPU writes be hopefully flushed to memory */
1037 intel_uncore_posting_read_fw(uncore, mode)((void)__raw_uncore_read32(uncore, mode));
1038
1039 return err;
1040}
1041
1042void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine)
1043{
1044 ENGINE_TRACE(engine, "\n")do { const struct intel_engine_cs *e__ __attribute__((__unused__
)) = (engine); do { } while (0); } while (0);
1045
1046 ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING))__raw_uncore_write32(((engine))->uncore, ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x9c) }), ((({ if (__builtin_constant_p
(((1 << 8)))) do { } while (0); if (__builtin_constant_p
(0)) do { } while (0); if (__builtin_constant_p(((1 << 8
))) && __builtin_constant_p(0)) do { } while (0); (((
(1 << 8))) << 16 | (0)); }))));
1047}
1048
1049const char *i915_cache_level_str(struct drm_i915_privateinteldrm_softc *i915, int type)
1050{
1051 switch (type) {
1052 case I915_CACHE_NONE: return " uncached";
1053 case I915_CACHE_LLC: return HAS_LLC(i915)((&(i915)->__info)->has_llc) ? " LLC" : " snooped";
1054 case I915_CACHE_L3_LLC: return " L3+LLC";
1055 case I915_CACHE_WT: return " WT";
1056 default: return "";
1057 }
1058}
1059
1060static u32
1061read_subslice_reg(const struct intel_engine_cs *engine,
1062 int slice, int subslice, i915_reg_t reg)
1063{
1064 struct drm_i915_privateinteldrm_softc *i915 = engine->i915;
1065 struct intel_uncore *uncore = engine->uncore;
1066 u32 mcr_mask, mcr_ss, mcr, old_mcr, val;
1067 enum forcewake_domains fw_domains;
1068
1069 if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 11) {
1070 mcr_mask = GEN11_MCR_SLICE_MASK(((0xf) & 0xf) << 27) | GEN11_MCR_SUBSLICE_MASK(((0x7) & 0x7) << 24);
1071 mcr_ss = GEN11_MCR_SLICE(slice)(((slice) & 0xf) << 27) | GEN11_MCR_SUBSLICE(subslice)(((subslice) & 0x7) << 24);
1072 } else {
1073 mcr_mask = GEN8_MCR_SLICE_MASK(((3) & 3) << 26) | GEN8_MCR_SUBSLICE_MASK(((3) & 3) << 24);
1074 mcr_ss = GEN8_MCR_SLICE(slice)(((slice) & 3) << 26) | GEN8_MCR_SUBSLICE(subslice)(((subslice) & 3) << 24);
1075 }
1076
1077 fw_domains = intel_uncore_forcewake_for_reg(uncore, reg,
1078 FW_REG_READ(1));
1079 fw_domains |= intel_uncore_forcewake_for_reg(uncore,
1080 GEN8_MCR_SELECTOR((const i915_reg_t){ .reg = (0xfdc) }),
1081 FW_REG_READ(1) | FW_REG_WRITE(2));
1082
1083 spin_lock_irq(&uncore->lock)mtx_enter(&uncore->lock);
1084 intel_uncore_forcewake_get__locked(uncore, fw_domains);
1085
1086 old_mcr = mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR)__raw_uncore_read32(uncore, ((const i915_reg_t){ .reg = (0xfdc
) }));
1087
1088 mcr &= ~mcr_mask;
1089 mcr |= mcr_ss;
1090 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr)__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0xfdc
) }), mcr);
1091
1092 val = intel_uncore_read_fw(uncore, reg)__raw_uncore_read32(uncore, reg);
1093
1094 mcr &= ~mcr_mask;
1095 mcr |= old_mcr & mcr_mask;
1096
1097 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr)__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0xfdc
) }), mcr);
1098
1099 intel_uncore_forcewake_put__locked(uncore, fw_domains);
1100 spin_unlock_irq(&uncore->lock)mtx_leave(&uncore->lock);
1101
1102 return val;
1103}
1104
1105/* NB: please notice the memset */
1106void intel_engine_get_instdone(const struct intel_engine_cs *engine,
1107 struct intel_instdone *instdone)
1108{
1109 struct drm_i915_privateinteldrm_softc *i915 = engine->i915;
1110 const struct sseu_dev_info *sseu = &engine->gt->info.sseu;
1111 struct intel_uncore *uncore = engine->uncore;
1112 u32 mmio_base = engine->mmio_base;
1113 int slice;
1114 int subslice;
1115
1116 memset(instdone, 0, sizeof(*instdone))__builtin_memset((instdone), (0), (sizeof(*instdone)));
1117
1118 switch (INTEL_GEN(i915)((&(i915)->__info)->gen)) {
1119 default:
1120 instdone->instdone =
1121 intel_uncore_read(uncore, RING_INSTDONE(mmio_base)((const i915_reg_t){ .reg = ((mmio_base) + 0x6c) }));
1122
1123 if (engine->id != RCS0)
1124 break;
1125
1126 instdone->slice_common =
1127 intel_uncore_read(uncore, GEN7_SC_INSTDONE((const i915_reg_t){ .reg = (0x7100) }));
1128 if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 12) {
1129 instdone->slice_common_extra[0] =
1130 intel_uncore_read(uncore, GEN12_SC_INSTDONE_EXTRA((const i915_reg_t){ .reg = (0x7104) }));
1131 instdone->slice_common_extra[1] =
1132 intel_uncore_read(uncore, GEN12_SC_INSTDONE_EXTRA2((const i915_reg_t){ .reg = (0x7108) }));
1133 }
1134 for_each_instdone_slice_subslice(i915, sseu, slice, subslice)for ((slice) = 0, (subslice) = 0; (slice) < 3; (subslice) =
((subslice) + 1) % 8, (slice) += ((subslice) == 0)) if (!(((
((0 + (&(i915)->__info)->gen == (7)) ? 1 : ((sseu)->
slice_mask)) & (1UL << (slice)))) && (((0 +
(&(i915)->__info)->gen == (7)) ? (1 & (1UL <<
(subslice))) : intel_sseu_has_subslice(sseu, 0, subslice))))
) {} else {
1135 instdone->sampler[slice][subslice] =
1136 read_subslice_reg(engine, slice, subslice,
1137 GEN7_SAMPLER_INSTDONE((const i915_reg_t){ .reg = (0xe160) }));
1138 instdone->row[slice][subslice] =
1139 read_subslice_reg(engine, slice, subslice,
1140 GEN7_ROW_INSTDONE((const i915_reg_t){ .reg = (0xe164) }));
1141 }
1142 break;
1143 case 7:
1144 instdone->instdone =
1145 intel_uncore_read(uncore, RING_INSTDONE(mmio_base)((const i915_reg_t){ .reg = ((mmio_base) + 0x6c) }));
1146
1147 if (engine->id != RCS0)
1148 break;
1149
1150 instdone->slice_common =
1151 intel_uncore_read(uncore, GEN7_SC_INSTDONE((const i915_reg_t){ .reg = (0x7100) }));
1152 instdone->sampler[0][0] =
1153 intel_uncore_read(uncore, GEN7_SAMPLER_INSTDONE((const i915_reg_t){ .reg = (0xe160) }));
1154 instdone->row[0][0] =
1155 intel_uncore_read(uncore, GEN7_ROW_INSTDONE((const i915_reg_t){ .reg = (0xe164) }));
1156
1157 break;
1158 case 6:
1159 case 5:
1160 case 4:
1161 instdone->instdone =
1162 intel_uncore_read(uncore, RING_INSTDONE(mmio_base)((const i915_reg_t){ .reg = ((mmio_base) + 0x6c) }));
1163 if (engine->id == RCS0)
1164 /* HACK: Using the wrong struct member */
1165 instdone->slice_common =
1166 intel_uncore_read(uncore, GEN4_INSTDONE1((const i915_reg_t){ .reg = (0x207c) }));
1167 break;
1168 case 3:
1169 case 2:
1170 instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE((const i915_reg_t){ .reg = (0x2090) }));
1171 break;
1172 }
1173}
1174
1175static bool_Bool ring_is_idle(struct intel_engine_cs *engine)
1176{
1177 bool_Bool idle = true1;
1178
1179 if (I915_SELFTEST_ONLY(!engine->mmio_base)0)
1180 return true1;
1181
1182 if (!intel_engine_pm_get_if_awake(engine))
1183 return true1;
1184
1185 /* First check that no commands are left in the ring */
1186 if ((ENGINE_READ(engine, RING_HEAD)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x34) })) & HEAD_ADDR0x001FFFFC) !=
1187 (ENGINE_READ(engine, RING_TAIL)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x30) })) & TAIL_ADDR0x001FFFF8))
1188 idle = false0;
1189
1190 /* No bit for gen2, so assume the CS parser is idle */
1191 if (INTEL_GEN(engine->i915)((&(engine->i915)->__info)->gen) > 2 &&
1192 !(ENGINE_READ(engine, RING_MI_MODE)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x9c) })) & MODE_IDLE(1 << 9)))
1193 idle = false0;
1194
1195 intel_engine_pm_put(engine);
1196
1197 return idle;
1198}
1199
1200void intel_engine_flush_submission(struct intel_engine_cs *engine)
1201{
1202 struct tasklet_struct *t = &engine->execlists.tasklet;
1203
1204 if (!t->func)
1205 return;
1206
1207 /* Synchronise and wait for the tasklet on another CPU */
1208 tasklet_kill(t);
1209
1210 /* Having cancelled the tasklet, ensure that is run */
1211 local_bh_disable();
1212 if (tasklet_trylock(t)) {
1213 /* Must wait for any GPU reset in progress. */
1214 if (__tasklet_is_enabled(t))
1215 t->func(t->data);
1216 tasklet_unlock(t);
1217 }
1218 local_bh_enable();
1219}
1220
1221/**
1222 * intel_engine_is_idle() - Report if the engine has finished process all work
1223 * @engine: the intel_engine_cs
1224 *
1225 * Return true if there are no requests pending, nothing left to be submitted
1226 * to hardware, and that the engine is idle.
1227 */
1228bool_Bool intel_engine_is_idle(struct intel_engine_cs *engine)
1229{
1230 /* More white lies, if wedged, hw state is inconsistent */
1231 if (intel_gt_is_wedged(engine->gt))
1232 return true1;
1233
1234 if (!intel_engine_pm_is_awake(engine))
1235 return true1;
1236
1237 /* Waiting to drain ELSP? */
1238 if (execlists_active(&engine->execlists)) {
1239#ifdef __linux__
1240 synchronize_hardirq(engine->i915->drm.pdev->irq);
1241#else
1242 intr_barrier(engine->i915->irqh);
1243#endif
1244
1245 intel_engine_flush_submission(engine);
1246
1247 if (execlists_active(&engine->execlists))
1248 return false0;
1249 }
1250
1251 /* ELSP is empty, but there are ready requests? E.g. after reset */
1252 if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)((&engine->execlists.queue.rb_root)->rb_node == ((void
*)0)))
1253 return false0;
1254
1255 /* Ring stopped? */
1256 return ring_is_idle(engine);
1257}
1258
1259bool_Bool intel_engines_are_idle(struct intel_gt *gt)
1260{
1261 struct intel_engine_cs *engine;
1262 enum intel_engine_id id;
1263
1264 /*
1265 * If the driver is wedged, HW state may be very inconsistent and
1266 * report that it is still busy, even though we have stopped using it.
1267 */
1268 if (intel_gt_is_wedged(gt))
1269 return true1;
1270
1271 /* Already parked (and passed an idleness test); must still be idle */
1272 if (!READ_ONCE(gt->awake)({ typeof(gt->awake) __tmp = *(volatile typeof(gt->awake
) *)&(gt->awake); membar_datadep_consumer(); __tmp; }))
1273 return true1;
1274
1275 for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else {
1276 if (!intel_engine_is_idle(engine))
1277 return false0;
1278 }
1279
1280 return true1;
1281}
1282
1283void intel_engines_reset_default_submission(struct intel_gt *gt)
1284{
1285 struct intel_engine_cs *engine;
1286 enum intel_engine_id id;
1287
1288 for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else
1289 engine->set_default_submission(engine);
1290}
1291
1292bool_Bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
1293{
1294 switch (INTEL_GEN(engine->i915)((&(engine->i915)->__info)->gen)) {
1295 case 2:
1296 return false0; /* uses physical not virtual addresses */
1297 case 3:
1298 /* maybe only uses physical not virtual addresses */
1299 return !(IS_I915G(engine->i915)IS_PLATFORM(engine->i915, INTEL_I915G) || IS_I915GM(engine->i915)IS_PLATFORM(engine->i915, INTEL_I915GM));
1300 case 4:
1301 return !IS_I965G(engine->i915)IS_PLATFORM(engine->i915, INTEL_I965G); /* who knows! */
1302 case 6:
1303 return engine->class != VIDEO_DECODE_CLASS1; /* b0rked */
1304 default:
1305 return true1;
1306 }
1307}
1308
1309static int print_sched_attr(const struct i915_sched_attr *attr,
1310 char *buf, int x, int len)
1311{
1312 if (attr->priority == I915_PRIORITY_INVALID((-0x7fffffff-1) | (u8)((1UL << (0)) - 1)))
1313 return x;
1314
1315 x += snprintf(buf + x, len - x,
1316 " prio=%d", attr->priority);
1317
1318 return x;
1319}
1320
1321static void print_request(struct drm_printer *m,
1322 struct i915_request *rq,
1323 const char *prefix)
1324{
1325 const char *name = rq->fence.ops->get_timeline_name(&rq->fence);
1326 char buf[80] = "";
1327 int x = 0;
1328
1329 x = print_sched_attr(&rq->sched.attr, buf, x, sizeof(buf));
Value stored to 'x' is never read
1330
1331 drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n",
1332 prefix,
1333 rq->fence.context, rq->fence.seqno,
1334 i915_request_completed(rq) ? "!" :
1335 i915_request_started(rq) ? "*" :
1336 "",
1337 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
1338 &rq->fence.flags) ? "+" :
1339 test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
1340 &rq->fence.flags) ? "-" :
1341 "",
1342 buf,
1343 jiffies_to_msecs(jiffies - rq->emitted_jiffies),
1344 name);
1345}
1346
1347static struct intel_timeline *get_timeline(struct i915_request *rq)
1348{
1349 struct intel_timeline *tl;
1350
1351 /*
1352 * Even though we are holding the engine->active.lock here, there
1353 * is no control over the submission queue per-se and we are
1354 * inspecting the active state at a random point in time, with an
1355 * unknown queue. Play safe and make sure the timeline remains valid.
1356 * (Only being used for pretty printing, one extra kref shouldn't
1357 * cause a camel stampede!)
1358 */
1359 rcu_read_lock();
1360 tl = rcu_dereference(rq->timeline)(rq->timeline);
1361 if (!kref_get_unless_zero(&tl->kref))
1362 tl = NULL((void *)0);
1363 rcu_read_unlock();
1364
1365 return tl;
1366}
1367
1368static int print_ring(char *buf, int sz, struct i915_request *rq)
1369{
1370 int len = 0;
1371
1372 if (!i915_request_signaled(rq)) {
1373 struct intel_timeline *tl = get_timeline(rq);
1374
1375 len = scnprintf(buf, sz,snprintf(buf, sz, "ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, "
, i915_ggtt_offset(rq->ring->vma), tl ? tl->hwsp_offset
: 0, hwsp_seqno(rq), (((intel_context_get_total_runtime_ns(rq
->context)) + ((1000 * 1000) / 2)) / (1000 * 1000)))
1376 "ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, ",snprintf(buf, sz, "ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, "
, i915_ggtt_offset(rq->ring->vma), tl ? tl->hwsp_offset
: 0, hwsp_seqno(rq), (((intel_context_get_total_runtime_ns(rq
->context)) + ((1000 * 1000) / 2)) / (1000 * 1000)))
1377 i915_ggtt_offset(rq->ring->vma),snprintf(buf, sz, "ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, "
, i915_ggtt_offset(rq->ring->vma), tl ? tl->hwsp_offset
: 0, hwsp_seqno(rq), (((intel_context_get_total_runtime_ns(rq
->context)) + ((1000 * 1000) / 2)) / (1000 * 1000)))
1378 tl ? tl->hwsp_offset : 0,snprintf(buf, sz, "ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, "
, i915_ggtt_offset(rq->ring->vma), tl ? tl->hwsp_offset
: 0, hwsp_seqno(rq), (((intel_context_get_total_runtime_ns(rq
->context)) + ((1000 * 1000) / 2)) / (1000 * 1000)))
1379 hwsp_seqno(rq),snprintf(buf, sz, "ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, "
, i915_ggtt_offset(rq->ring->vma), tl ? tl->hwsp_offset
: 0, hwsp_seqno(rq), (((intel_context_get_total_runtime_ns(rq
->context)) + ((1000 * 1000) / 2)) / (1000 * 1000)))
1380 DIV_ROUND_CLOSEST_ULL(intel_context_get_total_runtime_ns(rq->context),snprintf(buf, sz, "ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, "
, i915_ggtt_offset(rq->ring->vma), tl ? tl->hwsp_offset
: 0, hwsp_seqno(rq), (((intel_context_get_total_runtime_ns(rq
->context)) + ((1000 * 1000) / 2)) / (1000 * 1000)))
1381 1000 * 1000))snprintf(buf, sz, "ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, "
, i915_ggtt_offset(rq->ring->vma), tl ? tl->hwsp_offset
: 0, hwsp_seqno(rq), (((intel_context_get_total_runtime_ns(rq
->context)) + ((1000 * 1000) / 2)) / (1000 * 1000)));
1382
1383 if (tl)
1384 intel_timeline_put(tl);
1385 }
1386
1387 return len;
1388}
1389
1390static void hexdump(struct drm_printer *m, const void *buf, size_t len)
1391{
1392 STUB()do { printf("%s: stub\n", __func__); } while(0);
1393#ifdef notyet
1394 const size_t rowsize = 8 * sizeof(u32);
1395 const void *prev = NULL((void *)0);
1396 bool_Bool skip = false0;
1397 size_t pos;
1398
1399 for (pos = 0; pos < len; pos += rowsize) {
1400 char line[128];
1401
1402 if (prev && !memcmp(prev, buf + pos, rowsize)__builtin_memcmp((prev), (buf + pos), (rowsize))) {
1403 if (!skip) {
1404 drm_printf(m, "*\n");
1405 skip = true1;
1406 }
1407 continue;
1408 }
1409
1410 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,({ static int __warned; int __ret = !!((hex_dump_to_buffer(buf
+ pos, len - pos, rowsize, sizeof(u32), line, sizeof(line), 0
) >= sizeof(line))); if (__ret && !__warned) { printf
("%s", "WARN_ON_ONCE(" "hex_dump_to_buffer(buf + pos, len - pos, rowsize, sizeof(u32), line, sizeof(line), 0) >= sizeof(line)"
")"); __warned = 1; } __builtin_expect(!!(__ret), 0); })
1411 rowsize, sizeof(u32),({ static int __warned; int __ret = !!((hex_dump_to_buffer(buf
+ pos, len - pos, rowsize, sizeof(u32), line, sizeof(line), 0
) >= sizeof(line))); if (__ret && !__warned) { printf
("%s", "WARN_ON_ONCE(" "hex_dump_to_buffer(buf + pos, len - pos, rowsize, sizeof(u32), line, sizeof(line), 0) >= sizeof(line)"
")"); __warned = 1; } __builtin_expect(!!(__ret), 0); })
1412 line, sizeof(line),({ static int __warned; int __ret = !!((hex_dump_to_buffer(buf
+ pos, len - pos, rowsize, sizeof(u32), line, sizeof(line), 0
) >= sizeof(line))); if (__ret && !__warned) { printf
("%s", "WARN_ON_ONCE(" "hex_dump_to_buffer(buf + pos, len - pos, rowsize, sizeof(u32), line, sizeof(line), 0) >= sizeof(line)"
")"); __warned = 1; } __builtin_expect(!!(__ret), 0); })
1413 false) >= sizeof(line))({ static int __warned; int __ret = !!((hex_dump_to_buffer(buf
+ pos, len - pos, rowsize, sizeof(u32), line, sizeof(line), 0
) >= sizeof(line))); if (__ret && !__warned) { printf
("%s", "WARN_ON_ONCE(" "hex_dump_to_buffer(buf + pos, len - pos, rowsize, sizeof(u32), line, sizeof(line), 0) >= sizeof(line)"
")"); __warned = 1; } __builtin_expect(!!(__ret), 0); });
1414 drm_printf(m, "[%04zx] %s\n", pos, line);
1415
1416 prev = buf + pos;
1417 skip = false0;
1418 }
1419#endif
1420}
1421
1422static const char *repr_timer(const struct timeout *t)
1423{
1424 if (!READ_ONCE(t->to_time)({ typeof(t->to_time) __tmp = *(volatile typeof(t->to_time
) *)&(t->to_time); membar_datadep_consumer(); __tmp; }
))
1425 return "inactive";
1426
1427 if (timer_pending(t)(((t))->to_flags & 0x02))
1428 return "active";
1429
1430 return "expired";
1431}
1432
1433static void intel_engine_print_registers(struct intel_engine_cs *engine,
1434 struct drm_printer *m)
1435{
1436 struct drm_i915_privateinteldrm_softc *dev_priv = engine->i915;
1437 struct intel_engine_execlists * const execlists = &engine->execlists;
1438 u64 addr;
1439
1440 if (engine->id == RENDER_CLASS0 && IS_GEN_RANGE(dev_priv, 4, 7)(!!((&(dev_priv)->__info)->gen_mask & ( 0 + 0 +
(((~0UL) >> (64 - (((7)) - 1) - 1)) & ((~0UL) <<
(((4)) - 1)))))))
1441 drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x180) })));
1442 if (HAS_EXECLISTS(dev_priv)((&(dev_priv)->__info)->has_logical_ring_contexts)) {
1443 drm_printf(m, "\tEL_STAT_HI: 0x%08x\n",
1444 ENGINE_READ(engine, RING_EXECLIST_STATUS_HI)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x234 + 4) })));
1445 drm_printf(m, "\tEL_STAT_LO: 0x%08x\n",
1446 ENGINE_READ(engine, RING_EXECLIST_STATUS_LO)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x234) })));
1447 }
1448 drm_printf(m, "\tRING_START: 0x%08x\n",
1449 ENGINE_READ(engine, RING_START)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x38) })));
1450 drm_printf(m, "\tRING_HEAD: 0x%08x\n",
1451 ENGINE_READ(engine, RING_HEAD)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x34) })) & HEAD_ADDR0x001FFFFC);
1452 drm_printf(m, "\tRING_TAIL: 0x%08x\n",
1453 ENGINE_READ(engine, RING_TAIL)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x30) })) & TAIL_ADDR0x001FFFF8);
1454 drm_printf(m, "\tRING_CTL: 0x%08x%s\n",
1455 ENGINE_READ(engine, RING_CTL)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x3c) })),
1456 ENGINE_READ(engine, RING_CTL)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x3c) })) & (RING_WAIT(1 << 11) | RING_WAIT_SEMAPHORE(1 << 10)) ? " [waiting]" : "");
1457 if (INTEL_GEN(engine->i915)((&(engine->i915)->__info)->gen) > 2) {
1458 drm_printf(m, "\tRING_MODE: 0x%08x%s\n",
1459 ENGINE_READ(engine, RING_MI_MODE)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x9c) })),
1460 ENGINE_READ(engine, RING_MI_MODE)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x9c) })) & (MODE_IDLE(1 << 9)) ? " [idle]" : "");
1461 }
1462
1463 if (INTEL_GEN(dev_priv)((&(dev_priv)->__info)->gen) >= 6) {
1464 drm_printf(m, "\tRING_IMR: 0x%08x\n",
1465 ENGINE_READ(engine, RING_IMR)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0xa8) })));
1466 drm_printf(m, "\tRING_ESR: 0x%08x\n",
1467 ENGINE_READ(engine, RING_ESR)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0xb8) })));
1468 drm_printf(m, "\tRING_EMR: 0x%08x\n",
1469 ENGINE_READ(engine, RING_EMR)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0xb4) })));
1470 drm_printf(m, "\tRING_EIR: 0x%08x\n",
1471 ENGINE_READ(engine, RING_EIR)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0xb0) })));
1472 }
1473
1474 addr = intel_engine_get_active_head(engine);
1475 drm_printf(m, "\tACTHD: 0x%08x_%08x\n",
1476 upper_32_bits(addr)((u32)(((addr) >> 16) >> 16)), lower_32_bits(addr)((u32)(addr)));
1477 addr = intel_engine_get_last_batch_head(engine);
1478 drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
1479 upper_32_bits(addr)((u32)(((addr) >> 16) >> 16)), lower_32_bits(addr)((u32)(addr)));
1480 if (INTEL_GEN(dev_priv)((&(dev_priv)->__info)->gen) >= 8)
1481 addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW)intel_uncore_read64_2x32(((engine))->uncore, ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x78) }), ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x60) }));
1482 else if (INTEL_GEN(dev_priv)((&(dev_priv)->__info)->gen) >= 4)
1483 addr = ENGINE_READ(engine, RING_DMA_FADD)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x78) }));
1484 else
1485 addr = ENGINE_READ(engine, DMA_FADD_I8XX)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0xd0) }));
1486 drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n",
1487 upper_32_bits(addr)((u32)(((addr) >> 16) >> 16)), lower_32_bits(addr)((u32)(addr)));
1488 if (INTEL_GEN(dev_priv)((&(dev_priv)->__info)->gen) >= 4) {
1489 drm_printf(m, "\tIPEIR: 0x%08x\n",
1490 ENGINE_READ(engine, RING_IPEIR)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x64) })));
1491 drm_printf(m, "\tIPEHR: 0x%08x\n",
1492 ENGINE_READ(engine, RING_IPEHR)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x68) })));
1493 } else {
1494 drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x88) })));
1495 drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x8c) })));
1496 }
1497
1498 if (HAS_EXECLISTS(dev_priv)((&(dev_priv)->__info)->has_logical_ring_contexts)) {
1499 struct i915_request * const *port, *rq;
1500 const u32 *hws =
1501 &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX0x10];
1502 const u8 num_entries = execlists->csb_size;
1503 unsigned int idx;
1504 u8 read, write;
1505
1506 drm_printf(m, "\tExeclist tasklet queued? %s (%s), preempt? %s, timeslice? %s\n",
1507 yesno(test_bit(TASKLET_STATE_SCHED1,
1508 &engine->execlists.tasklet.state)),
1509 enableddisabled(!atomic_read(&engine->execlists.tasklet.count)({ typeof(*(&engine->execlists.tasklet.count)) __tmp =
*(volatile typeof(*(&engine->execlists.tasklet.count)
) *)&(*(&engine->execlists.tasklet.count)); membar_datadep_consumer
(); __tmp; })),
1510 repr_timer(&engine->execlists.preempt),
1511 repr_timer(&engine->execlists.timer));
1512
1513 read = execlists->csb_head;
1514 write = READ_ONCE(*execlists->csb_write)({ typeof(*execlists->csb_write) __tmp = *(volatile typeof
(*execlists->csb_write) *)&(*execlists->csb_write);
membar_datadep_consumer(); __tmp; });
1515
1516 drm_printf(m, "\tExeclist status: 0x%08x %08x; CSB read:%d, write:%d, entries:%d\n",
1517 ENGINE_READ(engine, RING_EXECLIST_STATUS_LO)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x234) })),
1518 ENGINE_READ(engine, RING_EXECLIST_STATUS_HI)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x234 + 4) })),
1519 read, write, num_entries);
1520
1521 if (read >= num_entries)
1522 read = 0;
1523 if (write >= num_entries)
1524 write = 0;
1525 if (read > write)
1526 write += num_entries;
1527 while (read < write) {
1528 idx = ++read % num_entries;
1529 drm_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n",
1530 idx, hws[idx * 2], hws[idx * 2 + 1]);
1531 }
1532
1533 execlists_active_lock_bh(execlists);
1534 rcu_read_lock();
1535 for (port = execlists->active; (rq = *port); port++) {
1536 char hdr[160];
1537 int len;
1538
1539 len = scnprintf(hdr, sizeof(hdr),snprintf(hdr, sizeof(hdr), "\t\tActive[%d]: ccid:%08x%s%s, "
, (int)(port - execlists->active), rq->context->lrc.
ccid, intel_context_is_closed(rq->context) ? "!" : "", intel_context_is_banned
(rq->context) ? "*" : "")
1540 "\t\tActive[%d]: ccid:%08x%s%s, ",snprintf(hdr, sizeof(hdr), "\t\tActive[%d]: ccid:%08x%s%s, "
, (int)(port - execlists->active), rq->context->lrc.
ccid, intel_context_is_closed(rq->context) ? "!" : "", intel_context_is_banned
(rq->context) ? "*" : "")
1541 (int)(port - execlists->active),snprintf(hdr, sizeof(hdr), "\t\tActive[%d]: ccid:%08x%s%s, "
, (int)(port - execlists->active), rq->context->lrc.
ccid, intel_context_is_closed(rq->context) ? "!" : "", intel_context_is_banned
(rq->context) ? "*" : "")
1542 rq->context->lrc.ccid,snprintf(hdr, sizeof(hdr), "\t\tActive[%d]: ccid:%08x%s%s, "
, (int)(port - execlists->active), rq->context->lrc.
ccid, intel_context_is_closed(rq->context) ? "!" : "", intel_context_is_banned
(rq->context) ? "*" : "")
1543 intel_context_is_closed(rq->context) ? "!" : "",snprintf(hdr, sizeof(hdr), "\t\tActive[%d]: ccid:%08x%s%s, "
, (int)(port - execlists->active), rq->context->lrc.
ccid, intel_context_is_closed(rq->context) ? "!" : "", intel_context_is_banned
(rq->context) ? "*" : "")
1544 intel_context_is_banned(rq->context) ? "*" : "")snprintf(hdr, sizeof(hdr), "\t\tActive[%d]: ccid:%08x%s%s, "
, (int)(port - execlists->active), rq->context->lrc.
ccid, intel_context_is_closed(rq->context) ? "!" : "", intel_context_is_banned
(rq->context) ? "*" : "");
1545 len += print_ring(hdr + len, sizeof(hdr) - len, rq);
1546 scnprintf(hdr + len, sizeof(hdr) - len, "rq: ")snprintf(hdr + len, sizeof(hdr) - len, "rq: ");
1547 print_request(m, rq, hdr);
1548 }
1549 for (port = execlists->pending; (rq = *port); port++) {
1550 char hdr[160];
1551 int len;
1552
1553 len = scnprintf(hdr, sizeof(hdr),snprintf(hdr, sizeof(hdr), "\t\tPending[%d]: ccid:%08x%s%s, "
, (int)(port - execlists->pending), rq->context->lrc
.ccid, intel_context_is_closed(rq->context) ? "!" : "", intel_context_is_banned
(rq->context) ? "*" : "")
1554 "\t\tPending[%d]: ccid:%08x%s%s, ",snprintf(hdr, sizeof(hdr), "\t\tPending[%d]: ccid:%08x%s%s, "
, (int)(port - execlists->pending), rq->context->lrc
.ccid, intel_context_is_closed(rq->context) ? "!" : "", intel_context_is_banned
(rq->context) ? "*" : "")
1555 (int)(port - execlists->pending),snprintf(hdr, sizeof(hdr), "\t\tPending[%d]: ccid:%08x%s%s, "
, (int)(port - execlists->pending), rq->context->lrc
.ccid, intel_context_is_closed(rq->context) ? "!" : "", intel_context_is_banned
(rq->context) ? "*" : "")
1556 rq->context->lrc.ccid,snprintf(hdr, sizeof(hdr), "\t\tPending[%d]: ccid:%08x%s%s, "
, (int)(port - execlists->pending), rq->context->lrc
.ccid, intel_context_is_closed(rq->context) ? "!" : "", intel_context_is_banned
(rq->context) ? "*" : "")
1557 intel_context_is_closed(rq->context) ? "!" : "",snprintf(hdr, sizeof(hdr), "\t\tPending[%d]: ccid:%08x%s%s, "
, (int)(port - execlists->pending), rq->context->lrc
.ccid, intel_context_is_closed(rq->context) ? "!" : "", intel_context_is_banned
(rq->context) ? "*" : "")
1558 intel_context_is_banned(rq->context) ? "*" : "")snprintf(hdr, sizeof(hdr), "\t\tPending[%d]: ccid:%08x%s%s, "
, (int)(port - execlists->pending), rq->context->lrc
.ccid, intel_context_is_closed(rq->context) ? "!" : "", intel_context_is_banned
(rq->context) ? "*" : "");
1559 len += print_ring(hdr + len, sizeof(hdr) - len, rq);
1560 scnprintf(hdr + len, sizeof(hdr) - len, "rq: ")snprintf(hdr + len, sizeof(hdr) - len, "rq: ");
1561 print_request(m, rq, hdr);
1562 }
1563 rcu_read_unlock();
1564 execlists_active_unlock_bh(execlists);
1565 } else if (INTEL_GEN(dev_priv)((&(dev_priv)->__info)->gen) > 6) {
1566 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
1567 ENGINE_READ(engine, RING_PP_DIR_BASE)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x228) })));
1568 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
1569 ENGINE_READ(engine, RING_PP_DIR_BASE_READ)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x518) })));
1570 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
1571 ENGINE_READ(engine, RING_PP_DIR_DCLV)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){
.reg = (((engine)->mmio_base) + 0x220) })));
1572 }
1573}
1574
1575static void print_request_ring(struct drm_printer *m, struct i915_request *rq)
1576{
1577 void *ring;
1578 int size;
1579
1580 drm_printf(m,
1581 "[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]:\n",
1582 rq->head, rq->postfix, rq->tail,
1583 rq->batch ? upper_32_bits(rq->batch->node.start)((u32)(((rq->batch->node.start) >> 16) >> 16
)) : ~0u,
1584 rq->batch ? lower_32_bits(rq->batch->node.start)((u32)(rq->batch->node.start)) : ~0u);
1585
1586 size = rq->tail - rq->head;
1587 if (rq->tail < rq->head)
1588 size += rq->ring->size;
1589
1590 ring = kmalloc(size, GFP_ATOMIC0x0002);
1591 if (ring) {
1592 const void *vaddr = rq->ring->vaddr;
1593 unsigned int head = rq->head;
1594 unsigned int len = 0;
1595
1596 if (rq->tail < head) {
1597 len = rq->ring->size - head;
1598 memcpy(ring, vaddr + head, len)__builtin_memcpy((ring), (vaddr + head), (len));
1599 head = 0;
1600 }
1601 memcpy(ring + len, vaddr + head, size - len)__builtin_memcpy((ring + len), (vaddr + head), (size - len));
1602
1603 hexdump(m, ring, size);
1604 kfree(ring);
1605 }
1606}
1607
1608static unsigned long list_count(struct list_head *list)
1609{
1610 struct list_head *pos;
1611 unsigned long count = 0;
1612
1613 list_for_each(pos, list)for (pos = (list)->next; pos != list; pos = (pos)->next
)
1614 count++;
1615
1616 return count;
1617}
1618
1619void intel_engine_dump(struct intel_engine_cs *engine,
1620 struct drm_printer *m,
1621 const char *header, ...)
1622{
1623 struct i915_gpu_error * const error = &engine->i915->gpu_error;
1624 struct i915_request *rq;
1625 intel_wakeref_t wakeref;
1626 unsigned long flags;
1627 ktime_t dummy;
1628
1629 if (header) {
1630 va_list ap;
1631
1632 va_start(ap, header)__builtin_va_start((ap), header);
1633 drm_vprintf(m, header, &ap);
1634 va_end(ap)__builtin_va_end((ap));
1635 }
1636
1637 if (intel_gt_is_wedged(engine->gt))
1638 drm_printf(m, "*** WEDGED ***\n");
1639
1640 drm_printf(m, "\tAwake? %d\n", atomic_read(&engine->wakeref.count)({ typeof(*(&engine->wakeref.count)) __tmp = *(volatile
typeof(*(&engine->wakeref.count)) *)&(*(&engine
->wakeref.count)); membar_datadep_consumer(); __tmp; }));
1641 drm_printf(m, "\tBarriers?: %s\n",
1642 yesno(!llist_empty(&engine->barrier_tasks)));
1643 drm_printf(m, "\tLatency: %luus\n",
1644 ewma__engine_latency_read(&engine->latency));
1645 if (intel_engine_supports_stats(engine))
1646 drm_printf(m, "\tRuntime: %llums\n",
1647 ktime_to_ms(intel_engine_get_busy_time(engine,
1648 &dummy)));
1649 drm_printf(m, "\tForcewake: %x domains, %d active\n",
1650 engine->fw_domain, atomic_read(&engine->fw_active)({ typeof(*(&engine->fw_active)) __tmp = *(volatile typeof
(*(&engine->fw_active)) *)&(*(&engine->fw_active
)); membar_datadep_consumer(); __tmp; }));
1651
1652 rcu_read_lock();
1653 rq = READ_ONCE(engine->heartbeat.systole)({ typeof(engine->heartbeat.systole) __tmp = *(volatile typeof
(engine->heartbeat.systole) *)&(engine->heartbeat.systole
); membar_datadep_consumer(); __tmp; });
1654 if (rq)
1655 drm_printf(m, "\tHeartbeat: %d ms ago\n",
1656 jiffies_to_msecs(jiffies - rq->emitted_jiffies));
1657 rcu_read_unlock();
1658 drm_printf(m, "\tReset count: %d (global %d)\n",
1659 i915_reset_engine_count(error, engine),
1660 i915_reset_count(error));
1661
1662 drm_printf(m, "\tRequests:\n");
1663
1664 spin_lock_irqsave(&engine->active.lock, flags)do { flags = 0; mtx_enter(&engine->active.lock); } while
(0);
1665 rq = intel_engine_find_active_request(engine);
1666 if (rq) {
1667 struct intel_timeline *tl = get_timeline(rq);
1668
1669 print_request(m, rq, "\t\tactive ");
1670
1671 drm_printf(m, "\t\tring->start: 0x%08x\n",
1672 i915_ggtt_offset(rq->ring->vma));
1673 drm_printf(m, "\t\tring->head: 0x%08x\n",
1674 rq->ring->head);
1675 drm_printf(m, "\t\tring->tail: 0x%08x\n",
1676 rq->ring->tail);
1677 drm_printf(m, "\t\tring->emit: 0x%08x\n",
1678 rq->ring->emit);
1679 drm_printf(m, "\t\tring->space: 0x%08x\n",
1680 rq->ring->space);
1681
1682 if (tl) {
1683 drm_printf(m, "\t\tring->hwsp: 0x%08x\n",
1684 tl->hwsp_offset);
1685 intel_timeline_put(tl);
1686 }
1687
1688 print_request_ring(m, rq);
1689
1690 if (rq->context->lrc_reg_state) {
1691 drm_printf(m, "Logical Ring Context:\n");
1692 hexdump(m, rq->context->lrc_reg_state, PAGE_SIZE(1 << 12));
1693 }
1694 }
1695 drm_printf(m, "\tOn hold?: %lu\n", list_count(&engine->active.hold));
1696 spin_unlock_irqrestore(&engine->active.lock, flags)do { (void)(flags); mtx_leave(&engine->active.lock); }
while (0);
1697
1698 drm_printf(m, "\tMMIO base: 0x%08x\n", engine->mmio_base);
1699 wakeref = intel_runtime_pm_get_if_in_use(engine->uncore->rpm);
1700 if (wakeref) {
1701 intel_engine_print_registers(engine, m);
1702 intel_runtime_pm_put(engine->uncore->rpm, wakeref);
1703 } else {
1704 drm_printf(m, "\tDevice is asleep; skipping register dump\n");
1705 }
1706
1707 intel_execlists_show_requests(engine, m, print_request, 8);
1708
1709 drm_printf(m, "HWSP:\n");
1710 hexdump(m, engine->status_page.addr, PAGE_SIZE(1 << 12));
1711
1712 drm_printf(m, "Idle? %s\n", yesno(intel_engine_is_idle(engine)));
1713
1714 intel_engine_print_breadcrumbs(engine, m);
1715}
1716
1717static ktime_t __intel_engine_get_busy_time(struct intel_engine_cs *engine,
1718 ktime_t *now)
1719{
1720 ktime_t total = engine->stats.total;
1721
1722 /*
1723 * If the engine is executing something at the moment
1724 * add it to the total.
1725 */
1726 *now = ktime_get();
1727 if (atomic_read(&engine->stats.active)({ typeof(*(&engine->stats.active)) __tmp = *(volatile
typeof(*(&engine->stats.active)) *)&(*(&engine
->stats.active)); membar_datadep_consumer(); __tmp; }))
1728 total = ktime_add(total, ktime_sub(*now, engine->stats.start));
1729
1730 return total;
1731}
1732
1733/**
1734 * intel_engine_get_busy_time() - Return current accumulated engine busyness
1735 * @engine: engine to report on
1736 * @now: monotonic timestamp of sampling
1737 *
1738 * Returns accumulated time @engine was busy since engine stats were enabled.
1739 */
1740ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine, ktime_t *now)
1741{
1742 unsigned int seq;
1743 ktime_t total;
1744
1745 do {
1746 seq = read_seqbegin(&engine->stats.lock);
1747 total = __intel_engine_get_busy_time(engine, now);
1748 } while (read_seqretry(&engine->stats.lock, seq));
1749
1750 return total;
1751}
1752
1753static bool_Bool match_ring(struct i915_request *rq)
1754{
1755 u32 ring = ENGINE_READ(rq->engine, RING_START)intel_uncore_read(((rq->engine))->uncore, ((const i915_reg_t
){ .reg = (((rq->engine)->mmio_base) + 0x38) }));
1756
1757 return ring == i915_ggtt_offset(rq->ring->vma);
1758}
1759
1760struct i915_request *
1761intel_engine_find_active_request(struct intel_engine_cs *engine)
1762{
1763 struct i915_request *request, *active = NULL((void *)0);
1764
1765 /*
1766 * We are called by the error capture, reset and to dump engine
1767 * state at random points in time. In particular, note that neither is
1768 * crucially ordered with an interrupt. After a hang, the GPU is dead
1769 * and we assume that no more writes can happen (we waited long enough
1770 * for all writes that were in transaction to be flushed) - adding an
1771 * extra delay for a recent interrupt is pointless. Hence, we do
1772 * not need an engine->irq_seqno_barrier() before the seqno reads.
1773 * At all other times, we must assume the GPU is still running, but
1774 * we only care about the snapshot of this moment.
1775 */
1776 lockdep_assert_held(&engine->active.lock)do { (void)(&engine->active.lock); } while(0);
1777
1778 rcu_read_lock();
1779 request = execlists_active(&engine->execlists);
1780 if (request) {
1781 struct intel_timeline *tl = request->context->timeline;
1782
1783 list_for_each_entry_from_reverse(request, &tl->requests, link)for (; &request->link != (&tl->requests); request
= ({ const __typeof( ((__typeof(*request) *)0)->link ) *__mptr
= (request->link.prev); (__typeof(*request) *)( (char *)__mptr
- __builtin_offsetof(__typeof(*request), link) );})) {
1784 if (i915_request_completed(request))
1785 break;
1786
1787 active = request;
1788 }
1789 }
1790 rcu_read_unlock();
1791 if (active)
1792 return active;
1793
1794 list_for_each_entry(request, &engine->active.requests, sched.link)for (request = ({ const __typeof( ((__typeof(*request) *)0)->
sched.link ) *__mptr = ((&engine->active.requests)->
next); (__typeof(*request) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*request), sched.link) );}); &request->sched
.link != (&engine->active.requests); request = ({ const
__typeof( ((__typeof(*request) *)0)->sched.link ) *__mptr
= (request->sched.link.next); (__typeof(*request) *)( (char
*)__mptr - __builtin_offsetof(__typeof(*request), sched.link
) );})) {
1795 if (i915_request_completed(request))
1796 continue;
1797
1798 if (!i915_request_started(request))
1799 continue;
1800
1801 /* More than one preemptible request may match! */
1802 if (!match_ring(request))
1803 continue;
1804
1805 active = request;
1806 break;
1807 }
1808
1809 return active;
1810}
1811
1812#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0
1813#include "mock_engine.c"
1814#include "selftest_engine.c"
1815#include "selftest_engine_cs.c"
1816#endif