Bug Summary

File:dev/pci/drm/amd/amdgpu/gmc_v10_0.c
Warning:line 902, column 3
Value stored to 'r' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name gmc_v10_0.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/pci/drm/amd/amdgpu/gmc_v10_0.c
1/*
2 * Copyright 2019 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23#include <linux/firmware.h>
24#include <linux/pci.h>
25
26#include <drm/drm_cache.h>
27
28#include "amdgpu.h"
29#include "amdgpu_atomfirmware.h"
30#include "gmc_v10_0.h"
31#include "umc_v8_7.h"
32
33#include "athub/athub_2_0_0_sh_mask.h"
34#include "athub/athub_2_0_0_offset.h"
35#include "dcn/dcn_2_0_0_offset.h"
36#include "dcn/dcn_2_0_0_sh_mask.h"
37#include "oss/osssys_5_0_0_offset.h"
38#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
39#include "navi10_enum.h"
40
41#include "soc15.h"
42#include "soc15d.h"
43#include "soc15_common.h"
44
45#include "nbio_v2_3.h"
46
47#include "gfxhub_v2_0.h"
48#include "gfxhub_v2_1.h"
49#include "mmhub_v2_0.h"
50#include "mmhub_v2_3.h"
51#include "athub_v2_0.h"
52#include "athub_v2_1.h"
53
54#include "amdgpu_reset.h"
55
56#if 0
57static const struct soc15_reg_golden golden_settings_navi10_hdp[] =
58{
59 /* TODO add golden setting for hdp */
60};
61#endif
62
63static int gmc_v10_0_ecc_interrupt_state(struct amdgpu_device *adev,
64 struct amdgpu_irq_src *src,
65 unsigned type,
66 enum amdgpu_interrupt_state state)
67{
68 return 0;
69}
70
71static int
72gmc_v10_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
73 struct amdgpu_irq_src *src, unsigned type,
74 enum amdgpu_interrupt_state state)
75{
76 switch (state) {
77 case AMDGPU_IRQ_STATE_DISABLE:
78 /* MM HUB */
79 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB_01, false0);
80 /* GFX HUB */
81 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB_00, false0);
82 break;
83 case AMDGPU_IRQ_STATE_ENABLE:
84 /* MM HUB */
85 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB_01, true1);
86 /* GFX HUB */
87 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB_00, true1);
88 break;
89 default:
90 break;
91 }
92
93 return 0;
94}
95
96static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev,
97 struct amdgpu_irq_src *source,
98 struct amdgpu_iv_entry *entry)
99{
100 bool_Bool retry_fault = !!(entry->src_data[1] & 0x80);
101 bool_Bool write_fault = !!(entry->src_data[1] & 0x20);
102 struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src];
103 struct amdgpu_task_info task_info;
104 uint32_t status = 0;
105 u64 addr;
106
107 addr = (u64)entry->src_data[0] << 12;
108 addr |= ((u64)entry->src_data[1] & 0xf) << 44;
109
110 if (retry_fault) {
111 /* Returning 1 here also prevents sending the IV to the KFD */
112
113 /* Process it onyl if it's the first fault for this address */
114 if (entry->ih != &adev->irq.ih_soft &&
115 amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
116 entry->timestamp))
117 return 1;
118
119 /* Delegate it to a different ring if the hardware hasn't
120 * already done it.
121 */
122 if (entry->ih == &adev->irq.ih) {
123 amdgpu_irq_delegate(adev, entry, 8);
124 return 1;
125 }
126
127 /* Try to handle the recoverable page faults by filling page
128 * tables
129 */
130 if (amdgpu_vm_handle_fault(adev, entry->pasid, addr, write_fault))
131 return 1;
132 }
133
134 if (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
135 /*
136 * Issue a dummy read to wait for the status register to
137 * be updated to avoid reading an incorrect value due to
138 * the new fast GRBM interface.
139 */
140 if ((entry->vmid_src == AMDGPU_GFXHUB_00) &&
141 (adev->ip_versions[GC_HWIP][0] < IP_VERSION(10, 3, 0)(((10) << 16) | ((3) << 8) | (0))))
142 RREG32(hub->vm_l2_pro_fault_status)amdgpu_device_rreg(adev, (hub->vm_l2_pro_fault_status), 0);
143
144 status = RREG32(hub->vm_l2_pro_fault_status)amdgpu_device_rreg(adev, (hub->vm_l2_pro_fault_status), 0);
145 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1)do { uint32_t tmp_ = amdgpu_device_rreg(adev, (hub->vm_l2_pro_fault_cntl
), 0); tmp_ &= (~1); tmp_ |= ((1) & ~(~1)); amdgpu_device_wreg
(adev, (hub->vm_l2_pro_fault_cntl), (tmp_), 0); } while (0
);
146 }
147
148 if (!printk_ratelimit()1)
149 return 0;
150
151 memset(&task_info, 0, sizeof(struct amdgpu_task_info))__builtin_memset((&task_info), (0), (sizeof(struct amdgpu_task_info
)));
152 amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
153
154 dev_err(adev->dev,printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid)
155 "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid)
156 "for process %s pid %d thread %s pid %d)\n",printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid)
157 entry->vmid_src ? "mmhub" : "gfxhub",printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid)
158 entry->src_id, entry->ring_id, entry->vmid,printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid)
159 entry->pasid, task_info.process_name, task_info.tgid,printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid)
160 task_info.task_name, task_info.pid)printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid);
161 dev_err(adev->dev, " in page starting at address 0x%016llx from client 0x%x (%s)\n",printf("drm:pid%d:%s *ERROR* " " in page starting at address 0x%016llx from client 0x%x (%s)\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , addr, entry
->client_id, soc15_ih_clientid_name[entry->client_id])
162 addr, entry->client_id,printf("drm:pid%d:%s *ERROR* " " in page starting at address 0x%016llx from client 0x%x (%s)\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , addr, entry
->client_id, soc15_ih_clientid_name[entry->client_id])
163 soc15_ih_clientid_name[entry->client_id])printf("drm:pid%d:%s *ERROR* " " in page starting at address 0x%016llx from client 0x%x (%s)\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , addr, entry
->client_id, soc15_ih_clientid_name[entry->client_id]);
164
165 if (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
166 hub->vmhub_funcs->print_l2_protection_fault_status(adev,
167 status);
168
169 return 0;
170}
171
172static const struct amdgpu_irq_src_funcs gmc_v10_0_irq_funcs = {
173 .set = gmc_v10_0_vm_fault_interrupt_state,
174 .process = gmc_v10_0_process_interrupt,
175};
176
177static const struct amdgpu_irq_src_funcs gmc_v10_0_ecc_funcs = {
178 .set = gmc_v10_0_ecc_interrupt_state,
179 .process = amdgpu_umc_process_ecc_irq,
180};
181
182static void gmc_v10_0_set_irq_funcs(struct amdgpu_device *adev)
183{
184 adev->gmc.vm_fault.num_types = 1;
185 adev->gmc.vm_fault.funcs = &gmc_v10_0_irq_funcs;
186
187 if (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
188 adev->gmc.ecc_irq.num_types = 1;
189 adev->gmc.ecc_irq.funcs = &gmc_v10_0_ecc_funcs;
190 }
191}
192
193/**
194 * gmc_v10_0_use_invalidate_semaphore - judge whether to use semaphore
195 *
196 * @adev: amdgpu_device pointer
197 * @vmhub: vmhub type
198 *
199 */
200static bool_Bool gmc_v10_0_use_invalidate_semaphore(struct amdgpu_device *adev,
201 uint32_t vmhub)
202{
203 return ((vmhub == AMDGPU_MMHUB_01 ||
204 vmhub == AMDGPU_MMHUB_12) &&
205 (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))));
206}
207
208static bool_Bool gmc_v10_0_get_atc_vmid_pasid_mapping_info(
209 struct amdgpu_device *adev,
210 uint8_t vmid, uint16_t *p_pasid)
211{
212 uint32_t value;
213
214 value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)amdgpu_device_rreg(adev, ((adev->reg_offset[ATHUB_HWIP][0]
[0] + 0x000c) + vmid), 0)
215 + vmid)amdgpu_device_rreg(adev, ((adev->reg_offset[ATHUB_HWIP][0]
[0] + 0x000c) + vmid), 0);
216 *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK0x0000FFFFL;
217
218 return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK0x80000000L);
219}
220
221/*
222 * GART
223 * VMID 0 is the physical GPU addresses as used by the kernel.
224 * VMIDs 1-15 are used for userspace clients and are handled
225 * by the amdgpu vm/hsa code.
226 */
227
228static void gmc_v10_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
229 unsigned int vmhub, uint32_t flush_type)
230{
231 bool_Bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(adev, vmhub);
232 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
233 u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
234 u32 tmp;
235 /* Use register 17 for GART */
236 const unsigned eng = 17;
237 unsigned int i;
238 unsigned char hub_ip = 0;
239
240 hub_ip = (vmhub == AMDGPU_GFXHUB_00) ?
241 GC_HWIP : MMHUB_HWIP;
242
243 spin_lock(&adev->gmc.invalidate_lock)mtx_enter(&adev->gmc.invalidate_lock);
244 /*
245 * It may lose gpuvm invalidate acknowldege state across power-gating
246 * off cycle, add semaphore acquire before invalidation and semaphore
247 * release after invalidation to avoid entering power gated state
248 * to WA the Issue
249 */
250
251 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
252 if (use_semaphore) {
253 for (i = 0; i < adev->usec_timeout; i++) {
254 /* a read return value of 1 means semaphore acuqire */
255 tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem +((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, hub->vm_inv_eng0_sem + hub->
eng_distance * eng, (1<<1) | (1<<2), hub_ip) : amdgpu_device_rreg
(adev, (hub->vm_inv_eng0_sem + hub->eng_distance * eng)
, 0))
256 hub->eng_distance * eng, hub_ip)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, hub->vm_inv_eng0_sem + hub->
eng_distance * eng, (1<<1) | (1<<2), hub_ip) : amdgpu_device_rreg
(adev, (hub->vm_inv_eng0_sem + hub->eng_distance * eng)
, 0));
257
258 if (tmp & 0x1)
259 break;
260 udelay(1);
261 }
262
263 if (i >= adev->usec_timeout)
264 DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n")__drm_err("Timeout waiting for sem acquire in VM flush!\n");
265 }
266
267 WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req +((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, hub->vm_inv_eng0_req + hub->
eng_distance * eng, inv_req, (1<<1) | (1<<2), hub_ip
) : amdgpu_device_wreg(adev, (hub->vm_inv_eng0_req + hub->
eng_distance * eng), (inv_req), 0))
268 hub->eng_distance * eng,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, hub->vm_inv_eng0_req + hub->
eng_distance * eng, inv_req, (1<<1) | (1<<2), hub_ip
) : amdgpu_device_wreg(adev, (hub->vm_inv_eng0_req + hub->
eng_distance * eng), (inv_req), 0))
269 inv_req, hub_ip)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, hub->vm_inv_eng0_req + hub->
eng_distance * eng, inv_req, (1<<1) | (1<<2), hub_ip
) : amdgpu_device_wreg(adev, (hub->vm_inv_eng0_req + hub->
eng_distance * eng), (inv_req), 0));
270
271 /*
272 * Issue a dummy read to wait for the ACK register to be cleared
273 * to avoid a false ACK due to the new fast GRBM interface.
274 */
275 if ((vmhub == AMDGPU_GFXHUB_00) &&
276 (adev->ip_versions[GC_HWIP][0] < IP_VERSION(10, 3, 0)(((10) << 16) | ((3) << 8) | (0))))
277 RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req +((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, hub->vm_inv_eng0_req + hub->
eng_distance * eng, (1<<1) | (1<<2), hub_ip) : amdgpu_device_rreg
(adev, (hub->vm_inv_eng0_req + hub->eng_distance * eng)
, 0))
278 hub->eng_distance * eng, hub_ip)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, hub->vm_inv_eng0_req + hub->
eng_distance * eng, (1<<1) | (1<<2), hub_ip) : amdgpu_device_rreg
(adev, (hub->vm_inv_eng0_req + hub->eng_distance * eng)
, 0));
279
280 /* Wait for ACK with a delay.*/
281 for (i = 0; i < adev->usec_timeout; i++) {
282 tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack +((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, hub->vm_inv_eng0_ack + hub->
eng_distance * eng, (1<<1) | (1<<2), hub_ip) : amdgpu_device_rreg
(adev, (hub->vm_inv_eng0_ack + hub->eng_distance * eng)
, 0))
283 hub->eng_distance * eng, hub_ip)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, hub->vm_inv_eng0_ack + hub->
eng_distance * eng, (1<<1) | (1<<2), hub_ip) : amdgpu_device_rreg
(adev, (hub->vm_inv_eng0_ack + hub->eng_distance * eng)
, 0));
284
285 tmp &= 1 << vmid;
286 if (tmp)
287 break;
288
289 udelay(1);
290 }
291
292 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
293 if (use_semaphore)
294 /*
295 * add semaphore release after invalidation,
296 * write with 0 means semaphore release
297 */
298 WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem +((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, hub->vm_inv_eng0_sem + hub->
eng_distance * eng, 0, (1<<1) | (1<<2), hub_ip) :
amdgpu_device_wreg(adev, (hub->vm_inv_eng0_sem + hub->
eng_distance * eng), (0), 0))
299 hub->eng_distance * eng, 0, hub_ip)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, hub->vm_inv_eng0_sem + hub->
eng_distance * eng, 0, (1<<1) | (1<<2), hub_ip) :
amdgpu_device_wreg(adev, (hub->vm_inv_eng0_sem + hub->
eng_distance * eng), (0), 0));
300
301 spin_unlock(&adev->gmc.invalidate_lock)mtx_leave(&adev->gmc.invalidate_lock);
302
303 if (i < adev->usec_timeout)
304 return;
305
306 DRM_ERROR("Timeout waiting for VM flush hub: %d!\n", vmhub)__drm_err("Timeout waiting for VM flush hub: %d!\n", vmhub);
307}
308
309/**
310 * gmc_v10_0_flush_gpu_tlb - gart tlb flush callback
311 *
312 * @adev: amdgpu_device pointer
313 * @vmid: vm instance to flush
314 * @vmhub: vmhub type
315 * @flush_type: the flush type
316 *
317 * Flush the TLB for the requested page table.
318 */
319static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
320 uint32_t vmhub, uint32_t flush_type)
321{
322 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
323 struct dma_fence *fence;
324 struct amdgpu_job *job;
325
326 int r;
327
328 /* flush hdp cache */
329 adev->hdp.funcs->flush_hdp(adev, NULL((void *)0));
330
331 /* For SRIOV run time, driver shouldn't access the register through MMIO
332 * Directly use kiq to do the vm invalidation instead
333 */
334 if (adev->gfx.kiq.ring.sched.ready && !adev->enable_mes &&
335 (amdgpu_sriov_runtime(adev)((adev)->virt.caps & (1 << 4)) || !amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) &&
336 down_read_trylock(&adev->reset_domain->sem)(rw_enter(&adev->reset_domain->sem, 0x0002UL | 0x0040UL
) == 0)) {
337 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
338 const unsigned eng = 17;
339 u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
340 u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
341 u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
342
343 amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,
344 1 << vmid);
345
346 up_read(&adev->reset_domain->sem)rw_exit_read(&adev->reset_domain->sem);
347 return;
348 }
349
350 mutex_lock(&adev->mman.gtt_window_lock)rw_enter_write(&adev->mman.gtt_window_lock);
351
352 if (vmhub == AMDGPU_MMHUB_01) {
353 gmc_v10_0_flush_vm_hub(adev, vmid, AMDGPU_MMHUB_01, 0);
354 mutex_unlock(&adev->mman.gtt_window_lock)rw_exit_write(&adev->mman.gtt_window_lock);
355 return;
356 }
357
358 BUG_ON(vmhub != AMDGPU_GFXHUB_0)((!(vmhub != 0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/pci/drm/amd/amdgpu/gmc_v10_0.c"
, 358, "!(vmhub != 0)"));
359
360 if (!adev->mman.buffer_funcs_enabled ||
361 !adev->ib_pool_ready ||
362 amdgpu_in_reset(adev) ||
363 ring->sched.ready == false0) {
364 gmc_v10_0_flush_vm_hub(adev, vmid, AMDGPU_GFXHUB_00, 0);
365 mutex_unlock(&adev->mman.gtt_window_lock)rw_exit_write(&adev->mman.gtt_window_lock);
366 return;
367 }
368
369 /* The SDMA on Navi has a bug which can theoretically result in memory
370 * corruption if an invalidation happens at the same time as an VA
371 * translation. Avoid this by doing the invalidation from the SDMA
372 * itself.
373 */
374 r = amdgpu_job_alloc_with_ib(adev, 16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
375 &job);
376 if (r)
377 goto error_alloc;
378
379 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
380 job->vm_needs_flush = true1;
381 job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
382 amdgpu_ring_pad_ib(ring, &job->ibs[0])((ring)->funcs->pad_ib((ring), (&job->ibs[0])));
383 r = amdgpu_job_submit(job, &adev->mman.entity,
384 AMDGPU_FENCE_OWNER_UNDEFINED((void *)0ul), &fence);
385 if (r)
386 goto error_submit;
387
388 mutex_unlock(&adev->mman.gtt_window_lock)rw_exit_write(&adev->mman.gtt_window_lock);
389
390 dma_fence_wait(fence, false0);
391 dma_fence_put(fence);
392
393 return;
394
395error_submit:
396 amdgpu_job_free(job);
397
398error_alloc:
399 mutex_unlock(&adev->mman.gtt_window_lock)rw_exit_write(&adev->mman.gtt_window_lock);
400 DRM_ERROR("Error flushing GPU TLB using the SDMA (%d)!\n", r)__drm_err("Error flushing GPU TLB using the SDMA (%d)!\n", r);
401}
402
403/**
404 * gmc_v10_0_flush_gpu_tlb_pasid - tlb flush via pasid
405 *
406 * @adev: amdgpu_device pointer
407 * @pasid: pasid to be flush
408 * @flush_type: the flush type
409 * @all_hub: Used with PACKET3_INVALIDATE_TLBS_ALL_HUB()
410 *
411 * Flush the TLB for the requested pasid.
412 */
413static int gmc_v10_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
414 uint16_t pasid, uint32_t flush_type,
415 bool_Bool all_hub)
416{
417 int vmid, i;
418 signed long r;
419 uint32_t seq;
420 uint16_t queried_pasid;
421 bool_Bool ret;
422 u32 usec_timeout = amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)) ? SRIOV_USEC_TIMEOUT1200000 : adev->usec_timeout;
423 struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
424 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
425
426 if (amdgpu_emu_mode == 0 && ring->sched.ready) {
427 spin_lock(&adev->gfx.kiq.ring_lock)mtx_enter(&adev->gfx.kiq.ring_lock);
428 /* 2 dwords flush + 8 dwords fence */
429 amdgpu_ring_alloc(ring, kiq->pmf->invalidate_tlbs_size + 8);
430 kiq->pmf->kiq_invalidate_tlbs(ring,
431 pasid, flush_type, all_hub);
432 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT5000);
433 if (r) {
434 amdgpu_ring_undo(ring);
435 spin_unlock(&adev->gfx.kiq.ring_lock)mtx_leave(&adev->gfx.kiq.ring_lock);
436 return -ETIME60;
437 }
438
439 amdgpu_ring_commit(ring);
440 spin_unlock(&adev->gfx.kiq.ring_lock)mtx_leave(&adev->gfx.kiq.ring_lock);
441 r = amdgpu_fence_wait_polling(ring, seq, usec_timeout);
442 if (r < 1) {
443 dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r)printf("drm:pid%d:%s *ERROR* " "wait for kiq fence error: %ld.\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r);
444 return -ETIME60;
445 }
446
447 return 0;
448 }
449
450 for (vmid = 1; vmid < AMDGPU_NUM_VMID16; vmid++) {
451
452 ret = gmc_v10_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
453 &queried_pasid);
454 if (ret && queried_pasid == pasid) {
455 if (all_hub) {
456 for (i = 0; i < adev->num_vmhubs; i++)
457 gmc_v10_0_flush_gpu_tlb(adev, vmid,
458 i, flush_type);
459 } else {
460 gmc_v10_0_flush_gpu_tlb(adev, vmid,
461 AMDGPU_GFXHUB_00, flush_type);
462 }
463 if (!adev->enable_mes)
464 break;
465 }
466 }
467
468 return 0;
469}
470
471static uint64_t gmc_v10_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
472 unsigned vmid, uint64_t pd_addr)
473{
474 bool_Bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub);
475 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
476 uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
477 unsigned eng = ring->vm_inv_eng;
478
479 /*
480 * It may lose gpuvm invalidate acknowldege state across power-gating
481 * off cycle, add semaphore acquire before invalidation and semaphore
482 * release after invalidation to avoid entering power gated state
483 * to WA the Issue
484 */
485
486 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
487 if (use_semaphore)
488 /* a read return value of 1 means semaphore acuqire */
489 amdgpu_ring_emit_reg_wait(ring,(ring)->funcs->emit_reg_wait((ring), (hub->vm_inv_eng0_sem
+ hub->eng_distance * eng), (0x1), (0x1))
490 hub->vm_inv_eng0_sem +(ring)->funcs->emit_reg_wait((ring), (hub->vm_inv_eng0_sem
+ hub->eng_distance * eng), (0x1), (0x1))
491 hub->eng_distance * eng, 0x1, 0x1)(ring)->funcs->emit_reg_wait((ring), (hub->vm_inv_eng0_sem
+ hub->eng_distance * eng), (0x1), (0x1));
492
493 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +(ring)->funcs->emit_wreg((ring), (hub->ctx0_ptb_addr_lo32
+ (hub->ctx_addr_distance * vmid)), (((u32)(pd_addr))))
494 (hub->ctx_addr_distance * vmid),(ring)->funcs->emit_wreg((ring), (hub->ctx0_ptb_addr_lo32
+ (hub->ctx_addr_distance * vmid)), (((u32)(pd_addr))))
495 lower_32_bits(pd_addr))(ring)->funcs->emit_wreg((ring), (hub->ctx0_ptb_addr_lo32
+ (hub->ctx_addr_distance * vmid)), (((u32)(pd_addr))));
496
497 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +(ring)->funcs->emit_wreg((ring), (hub->ctx0_ptb_addr_hi32
+ (hub->ctx_addr_distance * vmid)), (((u32)(((pd_addr) >>
16) >> 16))))
498 (hub->ctx_addr_distance * vmid),(ring)->funcs->emit_wreg((ring), (hub->ctx0_ptb_addr_hi32
+ (hub->ctx_addr_distance * vmid)), (((u32)(((pd_addr) >>
16) >> 16))))
499 upper_32_bits(pd_addr))(ring)->funcs->emit_wreg((ring), (hub->ctx0_ptb_addr_hi32
+ (hub->ctx_addr_distance * vmid)), (((u32)(((pd_addr) >>
16) >> 16))));
500
501 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +(ring)->funcs->emit_reg_write_reg_wait((ring), (hub->
vm_inv_eng0_req + hub->eng_distance * eng), (hub->vm_inv_eng0_ack
+ hub->eng_distance * eng), (req), (1 << vmid))
502 hub->eng_distance * eng,(ring)->funcs->emit_reg_write_reg_wait((ring), (hub->
vm_inv_eng0_req + hub->eng_distance * eng), (hub->vm_inv_eng0_ack
+ hub->eng_distance * eng), (req), (1 << vmid))
503 hub->vm_inv_eng0_ack +(ring)->funcs->emit_reg_write_reg_wait((ring), (hub->
vm_inv_eng0_req + hub->eng_distance * eng), (hub->vm_inv_eng0_ack
+ hub->eng_distance * eng), (req), (1 << vmid))
504 hub->eng_distance * eng,(ring)->funcs->emit_reg_write_reg_wait((ring), (hub->
vm_inv_eng0_req + hub->eng_distance * eng), (hub->vm_inv_eng0_ack
+ hub->eng_distance * eng), (req), (1 << vmid))
505 req, 1 << vmid)(ring)->funcs->emit_reg_write_reg_wait((ring), (hub->
vm_inv_eng0_req + hub->eng_distance * eng), (hub->vm_inv_eng0_ack
+ hub->eng_distance * eng), (req), (1 << vmid));
506
507 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
508 if (use_semaphore)
509 /*
510 * add semaphore release after invalidation,
511 * write with 0 means semaphore release
512 */
513 amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +(ring)->funcs->emit_wreg((ring), (hub->vm_inv_eng0_sem
+ hub->eng_distance * eng), (0))
514 hub->eng_distance * eng, 0)(ring)->funcs->emit_wreg((ring), (hub->vm_inv_eng0_sem
+ hub->eng_distance * eng), (0));
515
516 return pd_addr;
517}
518
519static void gmc_v10_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
520 unsigned pasid)
521{
522 struct amdgpu_device *adev = ring->adev;
523 uint32_t reg;
524
525 /* MES fw manages IH_VMID_x_LUT updating */
526 if (ring->is_mes_queue)
527 return;
528
529 if (ring->funcs->vmhub == AMDGPU_GFXHUB_00)
530 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT)(adev->reg_offset[OSSSYS_HWIP][0][0] + 0x0000) + vmid;
531 else
532 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM)(adev->reg_offset[OSSSYS_HWIP][0][0] + 0x0010) + vmid;
533
534 amdgpu_ring_emit_wreg(ring, reg, pasid)(ring)->funcs->emit_wreg((ring), (reg), (pasid));
535}
536
537/*
538 * PTE format on NAVI 10:
539 * 63:59 reserved
540 * 58 reserved and for sienna_cichlid is used for MALL noalloc
541 * 57 reserved
542 * 56 F
543 * 55 L
544 * 54 reserved
545 * 53:52 SW
546 * 51 T
547 * 50:48 mtype
548 * 47:12 4k physical page base address
549 * 11:7 fragment
550 * 6 write
551 * 5 read
552 * 4 exe
553 * 3 Z
554 * 2 snooped
555 * 1 system
556 * 0 valid
557 *
558 * PDE format on NAVI 10:
559 * 63:59 block fragment size
560 * 58:55 reserved
561 * 54 P
562 * 53:48 reserved
563 * 47:6 physical base address of PD or PTE
564 * 5:3 reserved
565 * 2 C
566 * 1 system
567 * 0 valid
568 */
569
570static uint64_t gmc_v10_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
571{
572 switch (flags) {
573 case AMDGPU_VM_MTYPE_DEFAULT(0 << 5):
574 return AMDGPU_PTE_MTYPE_NV10(MTYPE_NC)((uint64_t)(MTYPE_NC) << 48);
575 case AMDGPU_VM_MTYPE_NC(1 << 5):
576 return AMDGPU_PTE_MTYPE_NV10(MTYPE_NC)((uint64_t)(MTYPE_NC) << 48);
577 case AMDGPU_VM_MTYPE_WC(2 << 5):
578 return AMDGPU_PTE_MTYPE_NV10(MTYPE_WC)((uint64_t)(MTYPE_WC) << 48);
579 case AMDGPU_VM_MTYPE_CC(3 << 5):
580 return AMDGPU_PTE_MTYPE_NV10(MTYPE_CC)((uint64_t)(MTYPE_CC) << 48);
581 case AMDGPU_VM_MTYPE_UC(4 << 5):
582 return AMDGPU_PTE_MTYPE_NV10(MTYPE_UC)((uint64_t)(MTYPE_UC) << 48);
583 default:
584 return AMDGPU_PTE_MTYPE_NV10(MTYPE_NC)((uint64_t)(MTYPE_NC) << 48);
585 }
586}
587
588static void gmc_v10_0_get_vm_pde(struct amdgpu_device *adev, int level,
589 uint64_t *addr, uint64_t *flags)
590{
591 if (!(*flags & AMDGPU_PDE_PTE(1ULL << 54)) && !(*flags & AMDGPU_PTE_SYSTEM(1ULL << 1)))
592 *addr = amdgpu_gmc_vram_mc2pa(adev, *addr);
593 BUG_ON(*addr & 0xFFFF00000000003FULL)((!(*addr & 0xFFFF00000000003FULL)) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/dev/pci/drm/amd/amdgpu/gmc_v10_0.c"
, 593, "!(*addr & 0xFFFF00000000003FULL)"));
594
595 if (!adev->gmc.translate_further)
596 return;
597
598 if (level == AMDGPU_VM_PDB1) {
599 /* Set the block fragment size */
600 if (!(*flags & AMDGPU_PDE_PTE(1ULL << 54)))
601 *flags |= AMDGPU_PDE_BFS(0x9)((uint64_t)0x9 << 59);
602
603 } else if (level == AMDGPU_VM_PDB0) {
604 if (*flags & AMDGPU_PDE_PTE(1ULL << 54))
605 *flags &= ~AMDGPU_PDE_PTE(1ULL << 54);
606 else
607 *flags |= AMDGPU_PTE_TF(1ULL << 56);
608 }
609}
610
611static void gmc_v10_0_get_vm_pte(struct amdgpu_device *adev,
612 struct amdgpu_bo_va_mapping *mapping,
613 uint64_t *flags)
614{
615 *flags &= ~AMDGPU_PTE_EXECUTABLE(1ULL << 4);
616 *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE(1ULL << 4);
617
618 *flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK((uint64_t)(7ULL) << 48);
619 *flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK((uint64_t)(7ULL) << 48));
620
621 *flags &= ~AMDGPU_PTE_NOALLOC(1ULL << 58);
622 *flags |= (mapping->flags & AMDGPU_PTE_NOALLOC(1ULL << 58));
623
624 if (mapping->flags & AMDGPU_PTE_PRT(1ULL << 51)) {
625 *flags |= AMDGPU_PTE_PRT(1ULL << 51);
626 *flags |= AMDGPU_PTE_SNOOPED(1ULL << 2);
627 *flags |= AMDGPU_PTE_LOG(1ULL << 55);
628 *flags |= AMDGPU_PTE_SYSTEM(1ULL << 1);
629 *flags &= ~AMDGPU_PTE_VALID(1ULL << 0);
630 }
631}
632
633static unsigned gmc_v10_0_get_vbios_fb_size(struct amdgpu_device *adev)
634{
635 u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[DCE_HWIP][0][
1] + 0x000c, 0, DCE_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[DCE_HWIP][0][1] + 0x000c), 0));
636 unsigned size;
637
638 if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)(((d1vga_control) & 0x00000001L) >> 0x0)) {
639 size = AMDGPU_VBIOS_VGA_ALLOCATION(9 * 1024 * 1024);
640 } else {
641 u32 viewport;
642 u32 pitch;
643
644 viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[DCE_HWIP][0][
2] + 0x05ea, 0, DCE_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[DCE_HWIP][0][2] + 0x05ea), 0));
645 pitch = RREG32_SOC15(DCE, 0, mmHUBPREQ0_DCSURF_SURFACE_PITCH)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[DCE_HWIP][0][
2] + 0x0607, 0, DCE_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[DCE_HWIP][0][2] + 0x0607), 0));
646 size = (REG_GET_FIELD(viewport,(((viewport) & 0x3FFF0000L) >> 0x10)
647 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT)(((viewport) & 0x3FFF0000L) >> 0x10) *
648 REG_GET_FIELD(pitch, HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH)(((pitch) & 0x00003FFFL) >> 0x0) *
649 4);
650 }
651
652 return size;
653}
654
655static const struct amdgpu_gmc_funcs gmc_v10_0_gmc_funcs = {
656 .flush_gpu_tlb = gmc_v10_0_flush_gpu_tlb,
657 .flush_gpu_tlb_pasid = gmc_v10_0_flush_gpu_tlb_pasid,
658 .emit_flush_gpu_tlb = gmc_v10_0_emit_flush_gpu_tlb,
659 .emit_pasid_mapping = gmc_v10_0_emit_pasid_mapping,
660 .map_mtype = gmc_v10_0_map_mtype,
661 .get_vm_pde = gmc_v10_0_get_vm_pde,
662 .get_vm_pte = gmc_v10_0_get_vm_pte,
663 .get_vbios_fb_size = gmc_v10_0_get_vbios_fb_size,
664};
665
666static void gmc_v10_0_set_gmc_funcs(struct amdgpu_device *adev)
667{
668 if (adev->gmc.gmc_funcs == NULL((void *)0))
669 adev->gmc.gmc_funcs = &gmc_v10_0_gmc_funcs;
670}
671
672static void gmc_v10_0_set_umc_funcs(struct amdgpu_device *adev)
673{
674 switch (adev->ip_versions[UMC_HWIP][0]) {
675 case IP_VERSION(8, 7, 0)(((8) << 16) | ((7) << 8) | (0)):
676 adev->umc.max_ras_err_cnt_per_query = UMC_V8_7_TOTAL_CHANNEL_NUM(2 * 8);
677 adev->umc.channel_inst_num = UMC_V8_7_CHANNEL_INSTANCE_NUM2;
678 adev->umc.umc_inst_num = UMC_V8_7_UMC_INSTANCE_NUM8;
679 adev->umc.channel_offs = UMC_V8_7_PER_CHANNEL_OFFSET_SIENNA0x400;
680 adev->umc.channel_idx_tbl = &umc_v8_7_channel_idx_tbl[0][0];
681 adev->umc.ras = &umc_v8_7_ras;
682 break;
683 default:
684 break;
685 }
686 if (adev->umc.ras) {
687 amdgpu_ras_register_ras_block(adev, &adev->umc.ras->ras_block);
688
689 strlcpy(adev->umc.ras->ras_block.ras_comm.name, "umc",
690 sizeof(adev->umc.ras->ras_block.ras_comm.name));
691 adev->umc.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__UMC;
692 adev->umc.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
693 adev->umc.ras_if = &adev->umc.ras->ras_block.ras_comm;
694
695 /* If don't define special ras_late_init function, use default ras_late_init */
696 if (!adev->umc.ras->ras_block.ras_late_init)
697 adev->umc.ras->ras_block.ras_late_init = amdgpu_umc_ras_late_init;
698
699 /* If not defined special ras_cb function, use default ras_cb */
700 if (!adev->umc.ras->ras_block.ras_cb)
701 adev->umc.ras->ras_block.ras_cb = amdgpu_umc_process_ras_data_cb;
702 }
703}
704
705
706static void gmc_v10_0_set_mmhub_funcs(struct amdgpu_device *adev)
707{
708 switch (adev->ip_versions[MMHUB_HWIP][0]) {
709 case IP_VERSION(2, 3, 0)(((2) << 16) | ((3) << 8) | (0)):
710 case IP_VERSION(2, 4, 0)(((2) << 16) | ((4) << 8) | (0)):
711 case IP_VERSION(2, 4, 1)(((2) << 16) | ((4) << 8) | (1)):
712 adev->mmhub.funcs = &mmhub_v2_3_funcs;
713 break;
714 default:
715 adev->mmhub.funcs = &mmhub_v2_0_funcs;
716 break;
717 }
718}
719
720static void gmc_v10_0_set_gfxhub_funcs(struct amdgpu_device *adev)
721{
722 switch (adev->ip_versions[GC_HWIP][0]) {
723 case IP_VERSION(10, 3, 0)(((10) << 16) | ((3) << 8) | (0)):
724 case IP_VERSION(10, 3, 2)(((10) << 16) | ((3) << 8) | (2)):
725 case IP_VERSION(10, 3, 1)(((10) << 16) | ((3) << 8) | (1)):
726 case IP_VERSION(10, 3, 4)(((10) << 16) | ((3) << 8) | (4)):
727 case IP_VERSION(10, 3, 5)(((10) << 16) | ((3) << 8) | (5)):
728 case IP_VERSION(10, 3, 6)(((10) << 16) | ((3) << 8) | (6)):
729 case IP_VERSION(10, 3, 3)(((10) << 16) | ((3) << 8) | (3)):
730 case IP_VERSION(10, 3, 7)(((10) << 16) | ((3) << 8) | (7)):
731 adev->gfxhub.funcs = &gfxhub_v2_1_funcs;
732 break;
733 default:
734 adev->gfxhub.funcs = &gfxhub_v2_0_funcs;
735 break;
736 }
737}
738
739
740static int gmc_v10_0_early_init(void *handle)
741{
742 int r;
743 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
744
745 gmc_v10_0_set_mmhub_funcs(adev);
746 gmc_v10_0_set_gfxhub_funcs(adev);
747 gmc_v10_0_set_gmc_funcs(adev);
748 gmc_v10_0_set_irq_funcs(adev);
749 gmc_v10_0_set_umc_funcs(adev);
750
751 adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
752 adev->gmc.shared_aperture_end =
753 adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
754 adev->gmc.private_aperture_start = 0x1000000000000000ULL;
755 adev->gmc.private_aperture_end =
756 adev->gmc.private_aperture_start + (4ULL << 30) - 1;
757
758 r = amdgpu_gmc_ras_early_init(adev);
759 if (r)
760 return r;
761
762 return 0;
763}
764
765static int gmc_v10_0_late_init(void *handle)
766{
767 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
768 int r;
769
770 r = amdgpu_gmc_allocate_vm_inv_eng(adev);
771 if (r)
772 return r;
773
774 r = amdgpu_gmc_ras_late_init(adev);
775 if (r)
776 return r;
777
778 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
779}
780
781static void gmc_v10_0_vram_gtt_location(struct amdgpu_device *adev,
782 struct amdgpu_gmc *mc)
783{
784 u64 base = 0;
785
786 base = adev->gfxhub.funcs->get_fb_location(adev);
787
788 /* add the xgmi offset of the physical node */
789 base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
790
791 amdgpu_gmc_vram_location(adev, &adev->gmc, base);
792 amdgpu_gmc_gart_location(adev, mc);
793 amdgpu_gmc_agp_location(adev, mc);
794
795 /* base offset of vram pages */
796 adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
797
798 /* add the xgmi offset of the physical node */
799 adev->vm_manager.vram_base_offset +=
800 adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
801}
802
803/**
804 * gmc_v10_0_mc_init - initialize the memory controller driver params
805 *
806 * @adev: amdgpu_device pointer
807 *
808 * Look up the amount of vram, vram width, and decide how to place
809 * vram and gart within the GPU's physical address space.
810 * Returns 0 for success.
811 */
812static int gmc_v10_0_mc_init(struct amdgpu_device *adev)
813{
814 int r;
815
816 /* size in MB on si */
817 adev->gmc.mc_vram_size =
818 adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
819 adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
820
821 if (!(adev->flags & AMD_IS_APU)) {
822 r = amdgpu_device_resize_fb_bar(adev);
823 if (r)
824 return r;
825 }
826 adev->gmc.aper_base = adev->fb_aper_offset;
827 adev->gmc.aper_size = adev->fb_aper_size;
828
829#ifdef CONFIG_X86_641
830 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)((adev)->virt.caps & (1 << 3))) {
831 adev->gmc.aper_base = adev->gfxhub.funcs->get_mc_fb_offset(adev);
832 adev->gmc.aper_size = adev->gmc.real_vram_size;
833 }
834#endif
835
836 /* In case the PCI BAR is larger than the actual amount of vram */
837 adev->gmc.visible_vram_size = adev->gmc.aper_size;
838 if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
839 adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
840
841 /* set the gart size */
842 if (amdgpu_gart_size == -1) {
843 switch (adev->ip_versions[GC_HWIP][0]) {
844 default:
845 adev->gmc.gart_size = 512ULL << 20;
846 break;
847 case IP_VERSION(10, 3, 1)(((10) << 16) | ((3) << 8) | (1)): /* DCE SG support */
848 case IP_VERSION(10, 3, 3)(((10) << 16) | ((3) << 8) | (3)): /* DCE SG support */
849 case IP_VERSION(10, 3, 6)(((10) << 16) | ((3) << 8) | (6)): /* DCE SG support */
850 case IP_VERSION(10, 3, 7)(((10) << 16) | ((3) << 8) | (7)): /* DCE SG support */
851 adev->gmc.gart_size = 1024ULL << 20;
852 break;
853 }
854 } else {
855 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
856 }
857
858 gmc_v10_0_vram_gtt_location(adev, &adev->gmc);
859
860 return 0;
861}
862
863static int gmc_v10_0_gart_init(struct amdgpu_device *adev)
864{
865 int r;
866
867 if (adev->gart.bo) {
868 WARN(1, "NAVI10 PCIE GART already initialized\n")({ int __ret = !!(1); if (__ret) printf("NAVI10 PCIE GART already initialized\n"
); __builtin_expect(!!(__ret), 0); });
869 return 0;
870 }
871
872 /* Initialize common gart structure */
873 r = amdgpu_gart_init(adev);
874 if (r)
875 return r;
876
877 adev->gart.table_size = adev->gart.num_gpu_pages * 8;
878 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_NV10(MTYPE_UC)((uint64_t)(MTYPE_UC) << 48) |
879 AMDGPU_PTE_EXECUTABLE(1ULL << 4);
880
881 return amdgpu_gart_table_vram_alloc(adev);
882}
883
884static int gmc_v10_0_sw_init(void *handle)
885{
886 int r, vram_width = 0, vram_type = 0, vram_vendor = 0;
887 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
888
889 adev->gfxhub.funcs->init(adev);
890
891 adev->mmhub.funcs->init(adev);
892
893 mtx_init(&adev->gmc.invalidate_lock, IPL_NONE)do { (void)(((void *)0)); (void)(0); __mtx_init((&adev->
gmc.invalidate_lock), ((((0x0)) > 0x0 && ((0x0)) <
0x9) ? 0x9 : ((0x0)))); } while (0);
894
895 if ((adev->flags & AMD_IS_APU) && amdgpu_emu_mode == 1) {
896 adev->gmc.vram_type = AMDGPU_VRAM_TYPE_DDR48;
897 adev->gmc.vram_width = 64;
898 } else if (amdgpu_emu_mode == 1) {
899 adev->gmc.vram_type = AMDGPU_VRAM_TYPE_GDDR69;
900 adev->gmc.vram_width = 1 * 128; /* numchan * chansize */
901 } else {
902 r = amdgpu_atomfirmware_get_vram_info(adev,
Value stored to 'r' is never read
903 &vram_width, &vram_type, &vram_vendor);
904 adev->gmc.vram_width = vram_width;
905
906 adev->gmc.vram_type = vram_type;
907 adev->gmc.vram_vendor = vram_vendor;
908 }
909
910 switch (adev->ip_versions[GC_HWIP][0]) {
911 case IP_VERSION(10, 3, 0)(((10) << 16) | ((3) << 8) | (0)):
912 adev->gmc.mall_size = 128 * 1024 * 1024;
913 break;
914 case IP_VERSION(10, 3, 2)(((10) << 16) | ((3) << 8) | (2)):
915 adev->gmc.mall_size = 96 * 1024 * 1024;
916 break;
917 case IP_VERSION(10, 3, 4)(((10) << 16) | ((3) << 8) | (4)):
918 adev->gmc.mall_size = 32 * 1024 * 1024;
919 break;
920 case IP_VERSION(10, 3, 5)(((10) << 16) | ((3) << 8) | (5)):
921 adev->gmc.mall_size = 16 * 1024 * 1024;
922 break;
923 default:
924 adev->gmc.mall_size = 0;
925 break;
926 }
927
928 switch (adev->ip_versions[GC_HWIP][0]) {
929 case IP_VERSION(10, 1, 10)(((10) << 16) | ((1) << 8) | (10)):
930 case IP_VERSION(10, 1, 1)(((10) << 16) | ((1) << 8) | (1)):
931 case IP_VERSION(10, 1, 2)(((10) << 16) | ((1) << 8) | (2)):
932 case IP_VERSION(10, 1, 3)(((10) << 16) | ((1) << 8) | (3)):
933 case IP_VERSION(10, 1, 4)(((10) << 16) | ((1) << 8) | (4)):
934 case IP_VERSION(10, 3, 0)(((10) << 16) | ((3) << 8) | (0)):
935 case IP_VERSION(10, 3, 2)(((10) << 16) | ((3) << 8) | (2)):
936 case IP_VERSION(10, 3, 1)(((10) << 16) | ((3) << 8) | (1)):
937 case IP_VERSION(10, 3, 4)(((10) << 16) | ((3) << 8) | (4)):
938 case IP_VERSION(10, 3, 5)(((10) << 16) | ((3) << 8) | (5)):
939 case IP_VERSION(10, 3, 6)(((10) << 16) | ((3) << 8) | (6)):
940 case IP_VERSION(10, 3, 3)(((10) << 16) | ((3) << 8) | (3)):
941 case IP_VERSION(10, 3, 7)(((10) << 16) | ((3) << 8) | (7)):
942 adev->num_vmhubs = 2;
943 /*
944 * To fulfill 4-level page support,
945 * vm size is 256TB (48bit), maximum size of Navi10/Navi14/Navi12,
946 * block size 512 (9bit)
947 */
948 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
949 break;
950 default:
951 break;
952 }
953
954 /* This interrupt is VMC page fault.*/
955 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC,
956 VMC_1_0__SRCID__VM_FAULT0,
957 &adev->gmc.vm_fault);
958
959 if (r)
960 return r;
961
962 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2,
963 UTCL2_1_0__SRCID__FAULT0,
964 &adev->gmc.vm_fault);
965 if (r)
966 return r;
967
968 if (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
969 /* interrupt sent to DF. */
970 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
971 &adev->gmc.ecc_irq);
972 if (r)
973 return r;
974 }
975
976 /*
977 * Set the internal MC address mask This is the max address of the GPU's
978 * internal address space.
979 */
980 adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
981
982 r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)(((44) == 64) ? ~0ULL : (1ULL<<(44)) -1));
983 if (r) {
984 printk(KERN_WARNING"\0014" "amdgpu: No suitable DMA available.\n");
985 return r;
986 }
987
988 adev->need_swiotlb = drm_need_swiotlb(44);
989
990 r = gmc_v10_0_mc_init(adev);
991 if (r)
992 return r;
993
994 amdgpu_gmc_get_vbios_allocations(adev);
995
996 /* Memory manager */
997 r = amdgpu_bo_init(adev);
998 if (r)
999 return r;
1000
1001 r = gmc_v10_0_gart_init(adev);
1002 if (r)
1003 return r;
1004
1005 /*
1006 * number of VMs
1007 * VMID 0 is reserved for System
1008 * amdgpu graphics/compute will use VMIDs 1-7
1009 * amdkfd will use VMIDs 8-15
1010 */
1011 adev->vm_manager.first_kfd_vmid = 8;
1012
1013 amdgpu_vm_manager_init(adev);
1014
1015 return 0;
1016}
1017
1018/**
1019 * gmc_v10_0_gart_fini - vm fini callback
1020 *
1021 * @adev: amdgpu_device pointer
1022 *
1023 * Tears down the driver GART/VM setup (CIK).
1024 */
1025static void gmc_v10_0_gart_fini(struct amdgpu_device *adev)
1026{
1027 amdgpu_gart_table_vram_free(adev);
1028}
1029
1030static int gmc_v10_0_sw_fini(void *handle)
1031{
1032 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1033
1034 amdgpu_vm_manager_fini(adev);
1035 gmc_v10_0_gart_fini(adev);
1036 amdgpu_gem_force_release(adev);
1037 amdgpu_bo_fini(adev);
1038
1039 return 0;
1040}
1041
1042static void gmc_v10_0_init_golden_registers(struct amdgpu_device *adev)
1043{
1044}
1045
1046/**
1047 * gmc_v10_0_gart_enable - gart enable
1048 *
1049 * @adev: amdgpu_device pointer
1050 */
1051static int gmc_v10_0_gart_enable(struct amdgpu_device *adev)
1052{
1053 int r;
1054 bool_Bool value;
1055
1056 if (adev->gart.bo == NULL((void *)0)) {
1057 dev_err(adev->dev, "No VRAM object for PCIE GART.\n")printf("drm:pid%d:%s *ERROR* " "No VRAM object for PCIE GART.\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
1058 return -EINVAL22;
1059 }
1060
1061 amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);
1062 r = adev->gfxhub.funcs->gart_enable(adev);
1063 if (r)
1064 return r;
1065
1066 r = adev->mmhub.funcs->gart_enable(adev);
1067 if (r)
1068 return r;
1069
1070 adev->hdp.funcs->init_registers(adev);
1071
1072 /* Flush HDP after it is initialized */
1073 adev->hdp.funcs->flush_hdp(adev, NULL((void *)0));
1074
1075 value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS2) ?
1076 false0 : true1;
1077
1078 adev->gfxhub.funcs->set_fault_enable_default(adev, value);
1079 adev->mmhub.funcs->set_fault_enable_default(adev, value);
1080 gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_MMHUB_01, 0);
1081 gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB_00, 0);
1082
1083 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",printk("\0016" "[" "drm" "] " "PCIE GART of %uM enabled (table at 0x%016llX).\n"
, (unsigned)(adev->gmc.gart_size >> 20), (unsigned long
long)amdgpu_bo_gpu_offset(adev->gart.bo))
1084 (unsigned)(adev->gmc.gart_size >> 20),printk("\0016" "[" "drm" "] " "PCIE GART of %uM enabled (table at 0x%016llX).\n"
, (unsigned)(adev->gmc.gart_size >> 20), (unsigned long
long)amdgpu_bo_gpu_offset(adev->gart.bo))
1085 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo))printk("\0016" "[" "drm" "] " "PCIE GART of %uM enabled (table at 0x%016llX).\n"
, (unsigned)(adev->gmc.gart_size >> 20), (unsigned long
long)amdgpu_bo_gpu_offset(adev->gart.bo));
1086
1087 return 0;
1088}
1089
1090static int gmc_v10_0_hw_init(void *handle)
1091{
1092 int r;
1093 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1094
1095 /* The sequence of these two function calls matters.*/
1096 gmc_v10_0_init_golden_registers(adev);
1097
1098 /*
1099 * harvestable groups in gc_utcl2 need to be programmed before any GFX block
1100 * register setup within GMC, or else system hang when harvesting SA.
1101 */
1102 if (adev->gfxhub.funcs && adev->gfxhub.funcs->utcl2_harvest)
1103 adev->gfxhub.funcs->utcl2_harvest(adev);
1104
1105 r = gmc_v10_0_gart_enable(adev);
1106 if (r)
1107 return r;
1108
1109 if (amdgpu_emu_mode == 1) {
1110 r = amdgpu_gmc_vram_checking(adev);
1111 if (r)
1112 return r;
1113 }
1114
1115 if (adev->umc.funcs && adev->umc.funcs->init_registers)
1116 adev->umc.funcs->init_registers(adev);
1117
1118 return 0;
1119}
1120
1121/**
1122 * gmc_v10_0_gart_disable - gart disable
1123 *
1124 * @adev: amdgpu_device pointer
1125 *
1126 * This disables all VM page table.
1127 */
1128static void gmc_v10_0_gart_disable(struct amdgpu_device *adev)
1129{
1130 adev->gfxhub.funcs->gart_disable(adev);
1131 adev->mmhub.funcs->gart_disable(adev);
1132}
1133
1134static int gmc_v10_0_hw_fini(void *handle)
1135{
1136 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1137
1138 gmc_v10_0_gart_disable(adev);
1139
1140 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
1141 /* full access mode, so don't touch any GMC register */
1142 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n")___drm_dbg(((void *)0), DRM_UT_CORE, "For SRIOV client, shouldn't do anything.\n"
);
1143 return 0;
1144 }
1145
1146 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
1147
1148 return 0;
1149}
1150
1151static int gmc_v10_0_suspend(void *handle)
1152{
1153 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1154
1155 gmc_v10_0_hw_fini(adev);
1156
1157 return 0;
1158}
1159
1160static int gmc_v10_0_resume(void *handle)
1161{
1162 int r;
1163 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1164
1165 r = gmc_v10_0_hw_init(adev);
1166 if (r)
1167 return r;
1168
1169 amdgpu_vmid_reset_all(adev);
1170
1171 return 0;
1172}
1173
1174static bool_Bool gmc_v10_0_is_idle(void *handle)
1175{
1176 /* MC is always ready in GMC v10.*/
1177 return true1;
1178}
1179
1180static int gmc_v10_0_wait_for_idle(void *handle)
1181{
1182 /* There is no need to wait for MC idle in GMC v10.*/
1183 return 0;
1184}
1185
1186static int gmc_v10_0_soft_reset(void *handle)
1187{
1188 return 0;
1189}
1190
1191static int gmc_v10_0_set_clockgating_state(void *handle,
1192 enum amd_clockgating_state state)
1193{
1194 int r;
1195 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1196
1197 /*
1198 * The issue mmhub can't disconnect from DF with MMHUB clock gating being disabled
1199 * is a new problem observed at DF 3.0.3, however with the same suspend sequence not
1200 * seen any issue on the DF 3.0.2 series platform.
1201 */
1202 if (adev->in_s0ix && adev->ip_versions[DF_HWIP][0] > IP_VERSION(3, 0, 2)(((3) << 16) | ((0) << 8) | (2))) {
1203 dev_dbg(adev->dev, "keep mmhub clock gating being enabled for s0ix\n")do { } while(0);
1204 return 0;
1205 }
1206
1207 r = adev->mmhub.funcs->set_clockgating(adev, state);
1208 if (r)
1209 return r;
1210
1211 if (adev->ip_versions[ATHUB_HWIP][0] >= IP_VERSION(2, 1, 0)(((2) << 16) | ((1) << 8) | (0)))
1212 return athub_v2_1_set_clockgating(adev, state);
1213 else
1214 return athub_v2_0_set_clockgating(adev, state);
1215}
1216
1217static void gmc_v10_0_get_clockgating_state(void *handle, u64 *flags)
1218{
1219 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1220
1221 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(10, 1, 3)(((10) << 16) | ((1) << 8) | (3)) ||
1222 adev->ip_versions[GC_HWIP][0] == IP_VERSION(10, 1, 4)(((10) << 16) | ((1) << 8) | (4)))
1223 return;
1224
1225 adev->mmhub.funcs->get_clockgating(adev, flags);
1226
1227 if (adev->ip_versions[ATHUB_HWIP][0] >= IP_VERSION(2, 1, 0)(((2) << 16) | ((1) << 8) | (0)))
1228 athub_v2_1_get_clockgating(adev, flags);
1229 else
1230 athub_v2_0_get_clockgating(adev, flags);
1231}
1232
1233static int gmc_v10_0_set_powergating_state(void *handle,
1234 enum amd_powergating_state state)
1235{
1236 return 0;
1237}
1238
1239const struct amd_ip_funcs gmc_v10_0_ip_funcs = {
1240 .name = "gmc_v10_0",
1241 .early_init = gmc_v10_0_early_init,
1242 .late_init = gmc_v10_0_late_init,
1243 .sw_init = gmc_v10_0_sw_init,
1244 .sw_fini = gmc_v10_0_sw_fini,
1245 .hw_init = gmc_v10_0_hw_init,
1246 .hw_fini = gmc_v10_0_hw_fini,
1247 .suspend = gmc_v10_0_suspend,
1248 .resume = gmc_v10_0_resume,
1249 .is_idle = gmc_v10_0_is_idle,
1250 .wait_for_idle = gmc_v10_0_wait_for_idle,
1251 .soft_reset = gmc_v10_0_soft_reset,
1252 .set_clockgating_state = gmc_v10_0_set_clockgating_state,
1253 .set_powergating_state = gmc_v10_0_set_powergating_state,
1254 .get_clockgating_state = gmc_v10_0_get_clockgating_state,
1255};
1256
1257const struct amdgpu_ip_block_version gmc_v10_0_ip_block =
1258{
1259 .type = AMD_IP_BLOCK_TYPE_GMC,
1260 .major = 10,
1261 .minor = 0,
1262 .rev = 0,
1263 .funcs = &gmc_v10_0_ip_funcs,
1264};