Update version for v7.1.0-rc2 release
[qemu.git] / hw / ppc / spapr_pci_nvlink2.c
1 /*
2 * QEMU sPAPR PCI for NVLink2 pass through
3 *
4 * Copyright (c) 2019 Alexey Kardashevskiy, IBM Corporation.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24 #include "qemu/osdep.h"
25 #include "qapi/error.h"
26 #include "hw/pci/pci.h"
27 #include "hw/pci-host/spapr.h"
28 #include "hw/ppc/spapr_numa.h"
29 #include "qemu/error-report.h"
30 #include "hw/ppc/fdt.h"
31 #include "hw/pci/pci_bridge.h"
32
33 #define PHANDLE_PCIDEV(phb, pdev) (0x12000000 | \
34 (((phb)->index) << 16) | ((pdev)->devfn))
35 #define PHANDLE_GPURAM(phb, n) (0x110000FF | ((n) << 8) | \
36 (((phb)->index) << 16))
37 #define PHANDLE_NVLINK(phb, gn, nn) (0x00130000 | (((phb)->index) << 8) | \
38 ((gn) << 4) | (nn))
39
40 typedef struct SpaprPhbPciNvGpuSlot {
41 uint64_t tgt;
42 uint64_t gpa;
43 unsigned numa_id;
44 PCIDevice *gpdev;
45 int linknum;
46 struct {
47 uint64_t atsd_gpa;
48 PCIDevice *npdev;
49 uint32_t link_speed;
50 } links[NVGPU_MAX_LINKS];
51 } SpaprPhbPciNvGpuSlot;
52
53 struct SpaprPhbPciNvGpuConfig {
54 uint64_t nv2_ram_current;
55 uint64_t nv2_atsd_current;
56 int num; /* number of non empty (i.e. tgt!=0) entries in slots[] */
57 SpaprPhbPciNvGpuSlot slots[NVGPU_MAX_NUM];
58 Error *err;
59 };
60
61 static SpaprPhbPciNvGpuSlot *
62 spapr_nvgpu_get_slot(SpaprPhbPciNvGpuConfig *nvgpus, uint64_t tgt)
63 {
64 int i;
65
66 /* Search for partially collected "slot" */
67 for (i = 0; i < nvgpus->num; ++i) {
68 if (nvgpus->slots[i].tgt == tgt) {
69 return &nvgpus->slots[i];
70 }
71 }
72
73 if (nvgpus->num == ARRAY_SIZE(nvgpus->slots)) {
74 return NULL;
75 }
76
77 i = nvgpus->num;
78 nvgpus->slots[i].tgt = tgt;
79 ++nvgpus->num;
80
81 return &nvgpus->slots[i];
82 }
83
84 static void spapr_pci_collect_nvgpu(SpaprPhbPciNvGpuConfig *nvgpus,
85 PCIDevice *pdev, uint64_t tgt,
86 MemoryRegion *mr, Error **errp)
87 {
88 MachineState *machine = MACHINE(qdev_get_machine());
89 SpaprMachineState *spapr = SPAPR_MACHINE(machine);
90 SpaprPhbPciNvGpuSlot *nvslot = spapr_nvgpu_get_slot(nvgpus, tgt);
91
92 if (!nvslot) {
93 error_setg(errp, "Found too many GPUs per vPHB");
94 return;
95 }
96 g_assert(!nvslot->gpdev);
97 nvslot->gpdev = pdev;
98
99 nvslot->gpa = nvgpus->nv2_ram_current;
100 nvgpus->nv2_ram_current += memory_region_size(mr);
101 nvslot->numa_id = spapr->gpu_numa_id;
102 ++spapr->gpu_numa_id;
103 }
104
105 static void spapr_pci_collect_nvnpu(SpaprPhbPciNvGpuConfig *nvgpus,
106 PCIDevice *pdev, uint64_t tgt,
107 MemoryRegion *mr, Error **errp)
108 {
109 SpaprPhbPciNvGpuSlot *nvslot = spapr_nvgpu_get_slot(nvgpus, tgt);
110 int j;
111
112 if (!nvslot) {
113 error_setg(errp, "Found too many NVLink bridges per vPHB");
114 return;
115 }
116
117 j = nvslot->linknum;
118 if (j == ARRAY_SIZE(nvslot->links)) {
119 error_setg(errp, "Found too many NVLink bridges per GPU");
120 return;
121 }
122 ++nvslot->linknum;
123
124 g_assert(!nvslot->links[j].npdev);
125 nvslot->links[j].npdev = pdev;
126 nvslot->links[j].atsd_gpa = nvgpus->nv2_atsd_current;
127 nvgpus->nv2_atsd_current += memory_region_size(mr);
128 nvslot->links[j].link_speed =
129 object_property_get_uint(OBJECT(pdev), "nvlink2-link-speed", NULL);
130 }
131
132 static void spapr_phb_pci_collect_nvgpu(PCIBus *bus, PCIDevice *pdev,
133 void *opaque)
134 {
135 PCIBus *sec_bus;
136 Object *po = OBJECT(pdev);
137 uint64_t tgt = object_property_get_uint(po, "nvlink2-tgt", NULL);
138
139 if (tgt) {
140 Error *local_err = NULL;
141 SpaprPhbPciNvGpuConfig *nvgpus = opaque;
142 Object *mr_gpu = object_property_get_link(po, "nvlink2-mr[0]", NULL);
143 Object *mr_npu = object_property_get_link(po, "nvlink2-atsd-mr[0]",
144 NULL);
145
146 g_assert(mr_gpu || mr_npu);
147 if (mr_gpu) {
148 spapr_pci_collect_nvgpu(nvgpus, pdev, tgt, MEMORY_REGION(mr_gpu),
149 &local_err);
150 } else {
151 spapr_pci_collect_nvnpu(nvgpus, pdev, tgt, MEMORY_REGION(mr_npu),
152 &local_err);
153 }
154 error_propagate(&nvgpus->err, local_err);
155 }
156 if ((pci_default_read_config(pdev, PCI_HEADER_TYPE, 1) !=
157 PCI_HEADER_TYPE_BRIDGE)) {
158 return;
159 }
160
161 sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(pdev));
162 if (!sec_bus) {
163 return;
164 }
165
166 pci_for_each_device_under_bus(sec_bus, spapr_phb_pci_collect_nvgpu, opaque);
167 }
168
169 void spapr_phb_nvgpu_setup(SpaprPhbState *sphb, Error **errp)
170 {
171 int i, j, valid_gpu_num;
172 PCIBus *bus;
173
174 /* Search for GPUs and NPUs */
175 if (!sphb->nv2_gpa_win_addr || !sphb->nv2_atsd_win_addr) {
176 return;
177 }
178
179 sphb->nvgpus = g_new0(SpaprPhbPciNvGpuConfig, 1);
180 sphb->nvgpus->nv2_ram_current = sphb->nv2_gpa_win_addr;
181 sphb->nvgpus->nv2_atsd_current = sphb->nv2_atsd_win_addr;
182
183 bus = PCI_HOST_BRIDGE(sphb)->bus;
184 pci_for_each_device_under_bus(bus, spapr_phb_pci_collect_nvgpu,
185 sphb->nvgpus);
186
187 if (sphb->nvgpus->err) {
188 error_propagate(errp, sphb->nvgpus->err);
189 sphb->nvgpus->err = NULL;
190 goto cleanup_exit;
191 }
192
193 /* Add found GPU RAM and ATSD MRs if found */
194 for (i = 0, valid_gpu_num = 0; i < sphb->nvgpus->num; ++i) {
195 Object *nvmrobj;
196 SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i];
197
198 if (!nvslot->gpdev) {
199 continue;
200 }
201 nvmrobj = object_property_get_link(OBJECT(nvslot->gpdev),
202 "nvlink2-mr[0]", NULL);
203 /* ATSD is pointless without GPU RAM MR so skip those */
204 if (!nvmrobj) {
205 continue;
206 }
207
208 ++valid_gpu_num;
209 memory_region_add_subregion(get_system_memory(), nvslot->gpa,
210 MEMORY_REGION(nvmrobj));
211
212 for (j = 0; j < nvslot->linknum; ++j) {
213 Object *atsdmrobj;
214
215 atsdmrobj = object_property_get_link(OBJECT(nvslot->links[j].npdev),
216 "nvlink2-atsd-mr[0]", NULL);
217 if (!atsdmrobj) {
218 continue;
219 }
220 memory_region_add_subregion(get_system_memory(),
221 nvslot->links[j].atsd_gpa,
222 MEMORY_REGION(atsdmrobj));
223 }
224 }
225
226 if (valid_gpu_num) {
227 return;
228 }
229 /* We did not find any interesting GPU */
230 cleanup_exit:
231 g_free(sphb->nvgpus);
232 sphb->nvgpus = NULL;
233 }
234
235 void spapr_phb_nvgpu_free(SpaprPhbState *sphb)
236 {
237 int i, j;
238
239 if (!sphb->nvgpus) {
240 return;
241 }
242
243 for (i = 0; i < sphb->nvgpus->num; ++i) {
244 SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i];
245 Object *nv_mrobj = object_property_get_link(OBJECT(nvslot->gpdev),
246 "nvlink2-mr[0]", NULL);
247
248 if (nv_mrobj) {
249 memory_region_del_subregion(get_system_memory(),
250 MEMORY_REGION(nv_mrobj));
251 }
252 for (j = 0; j < nvslot->linknum; ++j) {
253 PCIDevice *npdev = nvslot->links[j].npdev;
254 Object *atsd_mrobj;
255 atsd_mrobj = object_property_get_link(OBJECT(npdev),
256 "nvlink2-atsd-mr[0]", NULL);
257 if (atsd_mrobj) {
258 memory_region_del_subregion(get_system_memory(),
259 MEMORY_REGION(atsd_mrobj));
260 }
261 }
262 }
263 g_free(sphb->nvgpus);
264 sphb->nvgpus = NULL;
265 }
266
267 void spapr_phb_nvgpu_populate_dt(SpaprPhbState *sphb, void *fdt, int bus_off,
268 Error **errp)
269 {
270 int i, j, atsdnum = 0;
271 uint64_t atsd[8]; /* The existing limitation of known guests */
272
273 if (!sphb->nvgpus) {
274 return;
275 }
276
277 for (i = 0; (i < sphb->nvgpus->num) && (atsdnum < ARRAY_SIZE(atsd)); ++i) {
278 SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i];
279
280 if (!nvslot->gpdev) {
281 continue;
282 }
283 for (j = 0; j < nvslot->linknum; ++j) {
284 if (!nvslot->links[j].atsd_gpa) {
285 continue;
286 }
287
288 if (atsdnum == ARRAY_SIZE(atsd)) {
289 error_report("Only %"PRIuPTR" ATSD registers supported",
290 ARRAY_SIZE(atsd));
291 break;
292 }
293 atsd[atsdnum] = cpu_to_be64(nvslot->links[j].atsd_gpa);
294 ++atsdnum;
295 }
296 }
297
298 if (!atsdnum) {
299 error_setg(errp, "No ATSD registers found");
300 return;
301 }
302
303 if (!spapr_phb_eeh_available(sphb)) {
304 /*
305 * ibm,mmio-atsd contains ATSD registers; these belong to an NPU PHB
306 * which we do not emulate as a separate device. Instead we put
307 * ibm,mmio-atsd to the vPHB with GPU and make sure that we do not
308 * put GPUs from different IOMMU groups to the same vPHB to ensure
309 * that the guest will use ATSDs from the corresponding NPU.
310 */
311 error_setg(errp, "ATSD requires separate vPHB per GPU IOMMU group");
312 return;
313 }
314
315 _FDT((fdt_setprop(fdt, bus_off, "ibm,mmio-atsd", atsd,
316 atsdnum * sizeof(atsd[0]))));
317 }
318
319 void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt)
320 {
321 int i, j, linkidx, npuoff;
322 g_autofree char *npuname = NULL;
323
324 if (!sphb->nvgpus) {
325 return;
326 }
327
328 npuname = g_strdup_printf("npuphb%d", sphb->index);
329 npuoff = fdt_add_subnode(fdt, 0, npuname);
330 _FDT(npuoff);
331 _FDT(fdt_setprop_cell(fdt, npuoff, "#address-cells", 1));
332 _FDT(fdt_setprop_cell(fdt, npuoff, "#size-cells", 0));
333 /* Advertise NPU as POWER9 so the guest can enable NPU2 contexts */
334 _FDT((fdt_setprop_string(fdt, npuoff, "compatible", "ibm,power9-npu")));
335
336 for (i = 0, linkidx = 0; i < sphb->nvgpus->num; ++i) {
337 for (j = 0; j < sphb->nvgpus->slots[i].linknum; ++j) {
338 g_autofree char *linkname = g_strdup_printf("link@%d", linkidx);
339 int off = fdt_add_subnode(fdt, npuoff, linkname);
340
341 _FDT(off);
342 /* _FDT((fdt_setprop_cell(fdt, off, "reg", linkidx))); */
343 _FDT((fdt_setprop_string(fdt, off, "compatible",
344 "ibm,npu-link")));
345 _FDT((fdt_setprop_cell(fdt, off, "phandle",
346 PHANDLE_NVLINK(sphb, i, j))));
347 _FDT((fdt_setprop_cell(fdt, off, "ibm,npu-link-index", linkidx)));
348 ++linkidx;
349 }
350 }
351
352 /* Add memory nodes for GPU RAM and mark them unusable */
353 for (i = 0; i < sphb->nvgpus->num; ++i) {
354 SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i];
355 Object *nv_mrobj = object_property_get_link(OBJECT(nvslot->gpdev),
356 "nvlink2-mr[0]",
357 &error_abort);
358 uint64_t size = object_property_get_uint(nv_mrobj, "size", NULL);
359 uint64_t mem_reg[2] = { cpu_to_be64(nvslot->gpa), cpu_to_be64(size) };
360 g_autofree char *mem_name = g_strdup_printf("memory@%"PRIx64,
361 nvslot->gpa);
362 int off = fdt_add_subnode(fdt, 0, mem_name);
363
364 _FDT(off);
365 _FDT((fdt_setprop_string(fdt, off, "device_type", "memory")));
366 _FDT((fdt_setprop(fdt, off, "reg", mem_reg, sizeof(mem_reg))));
367
368 spapr_numa_write_associativity_dt(SPAPR_MACHINE(qdev_get_machine()),
369 fdt, off, nvslot->numa_id);
370
371 _FDT((fdt_setprop_string(fdt, off, "compatible",
372 "ibm,coherent-device-memory")));
373
374 mem_reg[1] = cpu_to_be64(0);
375 _FDT((fdt_setprop(fdt, off, "linux,usable-memory", mem_reg,
376 sizeof(mem_reg))));
377 _FDT((fdt_setprop_cell(fdt, off, "phandle",
378 PHANDLE_GPURAM(sphb, i))));
379 }
380
381 }
382
383 void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset,
384 SpaprPhbState *sphb)
385 {
386 int i, j;
387
388 if (!sphb->nvgpus) {
389 return;
390 }
391
392 for (i = 0; i < sphb->nvgpus->num; ++i) {
393 SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i];
394
395 /* Skip "slot" without attached GPU */
396 if (!nvslot->gpdev) {
397 continue;
398 }
399 if (dev == nvslot->gpdev) {
400 uint32_t npus[nvslot->linknum];
401
402 for (j = 0; j < nvslot->linknum; ++j) {
403 PCIDevice *npdev = nvslot->links[j].npdev;
404
405 npus[j] = cpu_to_be32(PHANDLE_PCIDEV(sphb, npdev));
406 }
407 _FDT(fdt_setprop(fdt, offset, "ibm,npu", npus,
408 j * sizeof(npus[0])));
409 _FDT((fdt_setprop_cell(fdt, offset, "phandle",
410 PHANDLE_PCIDEV(sphb, dev))));
411 continue;
412 }
413
414 for (j = 0; j < nvslot->linknum; ++j) {
415 if (dev != nvslot->links[j].npdev) {
416 continue;
417 }
418
419 _FDT((fdt_setprop_cell(fdt, offset, "phandle",
420 PHANDLE_PCIDEV(sphb, dev))));
421 _FDT(fdt_setprop_cell(fdt, offset, "ibm,gpu",
422 PHANDLE_PCIDEV(sphb, nvslot->gpdev)));
423 _FDT((fdt_setprop_cell(fdt, offset, "ibm,nvlink",
424 PHANDLE_NVLINK(sphb, i, j))));
425 /*
426 * If we ever want to emulate GPU RAM at the same location as on
427 * the host - here is the encoding GPA->TGT:
428 *
429 * gta = ((sphb->nv2_gpa >> 42) & 0x1) << 42;
430 * gta |= ((sphb->nv2_gpa >> 45) & 0x3) << 43;
431 * gta |= ((sphb->nv2_gpa >> 49) & 0x3) << 45;
432 * gta |= sphb->nv2_gpa & ((1UL << 43) - 1);
433 */
434 _FDT(fdt_setprop_cell(fdt, offset, "memory-region",
435 PHANDLE_GPURAM(sphb, i)));
436 _FDT(fdt_setprop_u64(fdt, offset, "ibm,device-tgt-addr",
437 nvslot->tgt));
438 _FDT(fdt_setprop_cell(fdt, offset, "ibm,nvlink-speed",
439 nvslot->links[j].link_speed));
440 }
441 }
442 }