hw/rdma/vmw/pvrdma_cmd.c

   1 /*
   2  * QEMU paravirtual RDMA - Command channel
   3  *
   4  * Copyright (C) 2018 Oracle
   5  * Copyright (C) 2018 Red Hat Inc
   6  *
   7  * Authors:
   8  *     Yuval Shaia <yuval.shaia@oracle.com>
   9  *     Marcel Apfelbaum <marcel@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "cpu.h"
  18 #include "hw/pci/pci.h"
  19 #include "hw/pci/pci_ids.h"
  20
  21 #include "../rdma_backend.h"
  22 #include "../rdma_rm.h"
  23 #include "../rdma_utils.h"
  24
  25 #include "trace.h"
  26 #include "pvrdma.h"
  27 #include "standard-headers/rdma/vmw_pvrdma-abi.h"
  28
  29 static void *pvrdma_map_to_pdir(PCIDevice *pdev, uint64_t pdir_dma,
  30                                 uint32_t nchunks, size_t length)
  31 {
  32     uint64_t *dir, *tbl;
  33     int tbl_idx, dir_idx, addr_idx;
  34     void *host_virt = NULL, *curr_page;
  35
  36     if (!nchunks) {
  37         rdma_error_report("Got nchunks=0");
  38         return NULL;
  39     }
  40
  41     length = ROUND_UP(length, TARGET_PAGE_SIZE);
  42     if (nchunks * TARGET_PAGE_SIZE != length) {
  43         rdma_error_report("Invalid nchunks/length (%u, %lu)", nchunks,
  44                           (unsigned long)length);
  45         return NULL;
  46     }
  47
  48     dir = rdma_pci_dma_map(pdev, pdir_dma, TARGET_PAGE_SIZE);
  49     if (!dir) {
  50         rdma_error_report("Failed to map to page directory");
  51         return NULL;
  52     }
  53
  54     tbl = rdma_pci_dma_map(pdev, dir[0], TARGET_PAGE_SIZE);
  55     if (!tbl) {
  56         rdma_error_report("Failed to map to page table 0");
  57         goto out_unmap_dir;
  58     }
  59
  60     curr_page = rdma_pci_dma_map(pdev, (dma_addr_t)tbl[0], TARGET_PAGE_SIZE);
  61     if (!curr_page) {
  62         rdma_error_report("Failed to map the page 0");
  63         goto out_unmap_tbl;
  64     }
  65
  66     host_virt = mremap(curr_page, 0, length, MREMAP_MAYMOVE);
  67     if (host_virt == MAP_FAILED) {
  68         host_virt = NULL;
  69         rdma_error_report("Failed to remap memory for host_virt");
  70         goto out_unmap_tbl;
  71     }
  72     trace_pvrdma_map_to_pdir_host_virt(curr_page, host_virt);
  73
  74     rdma_pci_dma_unmap(pdev, curr_page, TARGET_PAGE_SIZE);
  75
  76     dir_idx = 0;
  77     tbl_idx = 1;
  78     addr_idx = 1;
  79     while (addr_idx < nchunks) {
  80         if (tbl_idx == TARGET_PAGE_SIZE / sizeof(uint64_t)) {
  81             tbl_idx = 0;
  82             dir_idx++;
  83             rdma_pci_dma_unmap(pdev, tbl, TARGET_PAGE_SIZE);
  84             tbl = rdma_pci_dma_map(pdev, dir[dir_idx], TARGET_PAGE_SIZE);
  85             if (!tbl) {
  86                 rdma_error_report("Failed to map to page table %d", dir_idx);
  87                 goto out_unmap_host_virt;
  88             }
  89         }
  90
  91         curr_page = rdma_pci_dma_map(pdev, (dma_addr_t)tbl[tbl_idx],
  92                                      TARGET_PAGE_SIZE);
  93         if (!curr_page) {
  94             rdma_error_report("Failed to map to page %d, dir %d", tbl_idx,
  95                               dir_idx);
  96             goto out_unmap_host_virt;
  97         }
  98
  99         mremap(curr_page, 0, TARGET_PAGE_SIZE, MREMAP_MAYMOVE | MREMAP_FIXED,
 100                host_virt + TARGET_PAGE_SIZE * addr_idx);
 101
 102         trace_pvrdma_map_to_pdir_next_page(addr_idx, curr_page, host_virt +
 103                                            TARGET_PAGE_SIZE * addr_idx);
 104
 105         rdma_pci_dma_unmap(pdev, curr_page, TARGET_PAGE_SIZE);
 106
 107         addr_idx++;
 108
 109         tbl_idx++;
 110     }
 111
 112     goto out_unmap_tbl;
 113
 114 out_unmap_host_virt:
 115     munmap(host_virt, length);
 116     host_virt = NULL;
 117
 118 out_unmap_tbl:
 119     rdma_pci_dma_unmap(pdev, tbl, TARGET_PAGE_SIZE);
 120
 121 out_unmap_dir:
 122     rdma_pci_dma_unmap(pdev, dir, TARGET_PAGE_SIZE);
 123
 124     return host_virt;
 125 }
 126
 127 static int query_port(PVRDMADev *dev, union pvrdma_cmd_req *req,
 128                       union pvrdma_cmd_resp *rsp)
 129 {
 130     struct pvrdma_cmd_query_port *cmd = &req->query_port;
 131     struct pvrdma_cmd_query_port_resp *resp = &rsp->query_port_resp;
 132     struct pvrdma_port_attr attrs = {};
 133
 134     if (cmd->port_num > MAX_PORTS) {
 135         return -EINVAL;
 136     }
 137
 138     if (rdma_backend_query_port(&dev->backend_dev,
 139                                 (struct ibv_port_attr *)&attrs)) {
 140         return -ENOMEM;
 141     }
 142
 143     memset(resp, 0, sizeof(*resp));
 144
 145     resp->attrs.state = dev->func0->device_active ? attrs.state :
 146                                                     PVRDMA_PORT_DOWN;
 147     resp->attrs.max_mtu = attrs.max_mtu;
 148     resp->attrs.active_mtu = attrs.active_mtu;
 149     resp->attrs.phys_state = attrs.phys_state;
 150     resp->attrs.gid_tbl_len = MIN(MAX_PORT_GIDS, attrs.gid_tbl_len);
 151     resp->attrs.max_msg_sz = 1024;
 152     resp->attrs.pkey_tbl_len = MIN(MAX_PORT_PKEYS, attrs.pkey_tbl_len);
 153     resp->attrs.active_width = 1;
 154     resp->attrs.active_speed = 1;
 155
 156     return 0;
 157 }
 158
 159 static int query_pkey(PVRDMADev *dev, union pvrdma_cmd_req *req,
 160                       union pvrdma_cmd_resp *rsp)
 161 {
 162     struct pvrdma_cmd_query_pkey *cmd = &req->query_pkey;
 163     struct pvrdma_cmd_query_pkey_resp *resp = &rsp->query_pkey_resp;
 164
 165     if (cmd->port_num > MAX_PORTS) {
 166         return -EINVAL;
 167     }
 168
 169     if (cmd->index > MAX_PKEYS) {
 170         return -EINVAL;
 171     }
 172
 173     memset(resp, 0, sizeof(*resp));
 174
 175     resp->pkey = PVRDMA_PKEY;
 176
 177     return 0;
 178 }
 179
 180 static int create_pd(PVRDMADev *dev, union pvrdma_cmd_req *req,
 181                      union pvrdma_cmd_resp *rsp)
 182 {
 183     struct pvrdma_cmd_create_pd *cmd = &req->create_pd;
 184     struct pvrdma_cmd_create_pd_resp *resp = &rsp->create_pd_resp;
 185     int rc;
 186
 187     memset(resp, 0, sizeof(*resp));
 188     rc = rdma_rm_alloc_pd(&dev->rdma_dev_res, &dev->backend_dev,
 189                           &resp->pd_handle, cmd->ctx_handle);
 190
 191     return rc;
 192 }
 193
 194 static int destroy_pd(PVRDMADev *dev, union pvrdma_cmd_req *req,
 195                       union pvrdma_cmd_resp *rsp)
 196 {
 197     struct pvrdma_cmd_destroy_pd *cmd = &req->destroy_pd;
 198
 199     rdma_rm_dealloc_pd(&dev->rdma_dev_res, cmd->pd_handle);
 200
 201     return 0;
 202 }
 203
 204 static int create_mr(PVRDMADev *dev, union pvrdma_cmd_req *req,
 205                      union pvrdma_cmd_resp *rsp)
 206 {
 207     struct pvrdma_cmd_create_mr *cmd = &req->create_mr;
 208     struct pvrdma_cmd_create_mr_resp *resp = &rsp->create_mr_resp;
 209     PCIDevice *pci_dev = PCI_DEVICE(dev);
 210     void *host_virt = NULL;
 211     int rc = 0;
 212
 213     memset(resp, 0, sizeof(*resp));
 214
 215     if (!(cmd->flags & PVRDMA_MR_FLAG_DMA)) {
 216         host_virt = pvrdma_map_to_pdir(pci_dev, cmd->pdir_dma, cmd->nchunks,
 217                                        cmd->length);
 218         if (!host_virt) {
 219             rdma_error_report("Failed to map to pdir");
 220             return -EINVAL;
 221         }
 222     }
 223
 224     rc = rdma_rm_alloc_mr(&dev->rdma_dev_res, cmd->pd_handle, cmd->start,
 225                           cmd->length, host_virt, cmd->access_flags,
 226                           &resp->mr_handle, &resp->lkey, &resp->rkey);
 227     if (rc && host_virt) {
 228         munmap(host_virt, cmd->length);
 229     }
 230
 231     return rc;
 232 }
 233
 234 static int destroy_mr(PVRDMADev *dev, union pvrdma_cmd_req *req,
 235                       union pvrdma_cmd_resp *rsp)
 236 {
 237     struct pvrdma_cmd_destroy_mr *cmd = &req->destroy_mr;
 238
 239     rdma_rm_dealloc_mr(&dev->rdma_dev_res, cmd->mr_handle);
 240
 241     return 0;
 242 }
 243
 244 static int create_cq_ring(PCIDevice *pci_dev , PvrdmaRing **ring,
 245                           uint64_t pdir_dma, uint32_t nchunks, uint32_t cqe)
 246 {
 247     uint64_t *dir = NULL, *tbl = NULL;
 248     PvrdmaRing *r;
 249     int rc = -EINVAL;
 250     char ring_name[MAX_RING_NAME_SZ];
 251
 252     if (!nchunks || nchunks > PVRDMA_MAX_FAST_REG_PAGES) {
 253         rdma_error_report("Got invalid nchunks: %d", nchunks);
 254         return rc;
 255     }
 256
 257     dir = rdma_pci_dma_map(pci_dev, pdir_dma, TARGET_PAGE_SIZE);
 258     if (!dir) {
 259         rdma_error_report("Failed to map to CQ page directory");
 260         goto out;
 261     }
 262
 263     tbl = rdma_pci_dma_map(pci_dev, dir[0], TARGET_PAGE_SIZE);
 264     if (!tbl) {
 265         rdma_error_report("Failed to map to CQ page table");
 266         goto out;
 267     }
 268
 269     r = g_malloc(sizeof(*r));
 270     *ring = r;
 271
 272     r->ring_state = (PvrdmaRingState *)
 273         rdma_pci_dma_map(pci_dev, tbl[0], TARGET_PAGE_SIZE);
 274
 275     if (!r->ring_state) {
 276         rdma_error_report("Failed to map to CQ ring state");
 277         goto out_free_ring;
 278     }
 279
 280     sprintf(ring_name, "cq_ring_%" PRIx64, pdir_dma);
 281     rc = pvrdma_ring_init(r, ring_name, pci_dev, &r->ring_state[1],
 282                           cqe, sizeof(struct pvrdma_cqe),
 283                           /* first page is ring state */
 284                           (dma_addr_t *)&tbl[1], nchunks - 1);
 285     if (rc) {
 286         goto out_unmap_ring_state;
 287     }
 288
 289     goto out;
 290
 291 out_unmap_ring_state:
 292     /* ring_state was in slot 1, not 0 so need to jump back */
 293     rdma_pci_dma_unmap(pci_dev, --r->ring_state, TARGET_PAGE_SIZE);
 294
 295 out_free_ring:
 296     g_free(r);
 297
 298 out:
 299     rdma_pci_dma_unmap(pci_dev, tbl, TARGET_PAGE_SIZE);
 300     rdma_pci_dma_unmap(pci_dev, dir, TARGET_PAGE_SIZE);
 301
 302     return rc;
 303 }
 304
 305 static void destroy_cq_ring(PvrdmaRing *ring)
 306 {
 307     pvrdma_ring_free(ring);
 308     /* ring_state was in slot 1, not 0 so need to jump back */
 309     rdma_pci_dma_unmap(ring->dev, --ring->ring_state, TARGET_PAGE_SIZE);
 310     g_free(ring);
 311 }
 312
 313 static int create_cq(PVRDMADev *dev, union pvrdma_cmd_req *req,
 314                      union pvrdma_cmd_resp *rsp)
 315 {
 316     struct pvrdma_cmd_create_cq *cmd = &req->create_cq;
 317     struct pvrdma_cmd_create_cq_resp *resp = &rsp->create_cq_resp;
 318     PvrdmaRing *ring = NULL;
 319     int rc;
 320
 321     memset(resp, 0, sizeof(*resp));
 322
 323     resp->cqe = cmd->cqe;
 324
 325     rc = create_cq_ring(PCI_DEVICE(dev), &ring, cmd->pdir_dma, cmd->nchunks,
 326                         cmd->cqe);
 327     if (rc) {
 328         return rc;
 329     }
 330
 331     rc = rdma_rm_alloc_cq(&dev->rdma_dev_res, &dev->backend_dev, cmd->cqe,
 332                           &resp->cq_handle, ring);
 333     if (rc) {
 334         destroy_cq_ring(ring);
 335     }
 336
 337     resp->cqe = cmd->cqe;
 338
 339     return rc;
 340 }
 341
 342 static int destroy_cq(PVRDMADev *dev, union pvrdma_cmd_req *req,
 343                       union pvrdma_cmd_resp *rsp)
 344 {
 345     struct pvrdma_cmd_destroy_cq *cmd = &req->destroy_cq;
 346     RdmaRmCQ *cq;
 347     PvrdmaRing *ring;
 348
 349     cq = rdma_rm_get_cq(&dev->rdma_dev_res, cmd->cq_handle);
 350     if (!cq) {
 351         rdma_error_report("Got invalid CQ handle");
 352         return -EINVAL;
 353     }
 354
 355     ring = (PvrdmaRing *)cq->opaque;
 356     destroy_cq_ring(ring);
 357
 358     rdma_rm_dealloc_cq(&dev->rdma_dev_res, cmd->cq_handle);
 359
 360     return 0;
 361 }
 362
 363 static int create_qp_rings(PCIDevice *pci_dev, uint64_t pdir_dma,
 364                            PvrdmaRing **rings, uint32_t scqe, uint32_t smax_sge,
 365                            uint32_t spages, uint32_t rcqe, uint32_t rmax_sge,
 366                            uint32_t rpages, uint8_t is_srq)
 367 {
 368     uint64_t *dir = NULL, *tbl = NULL;
 369     PvrdmaRing *sr, *rr;
 370     int rc = -EINVAL;
 371     char ring_name[MAX_RING_NAME_SZ];
 372     uint32_t wqe_sz;
 373
 374     if (!spages || spages > PVRDMA_MAX_FAST_REG_PAGES) {
 375         rdma_error_report("Got invalid send page count for QP ring: %d",
 376                           spages);
 377         return rc;
 378     }
 379
 380     if (!is_srq && (!rpages || rpages > PVRDMA_MAX_FAST_REG_PAGES)) {
 381         rdma_error_report("Got invalid recv page count for QP ring: %d",
 382                           rpages);
 383         return rc;
 384     }
 385
 386     dir = rdma_pci_dma_map(pci_dev, pdir_dma, TARGET_PAGE_SIZE);
 387     if (!dir) {
 388         rdma_error_report("Failed to map to QP page directory");
 389         goto out;
 390     }
 391
 392     tbl = rdma_pci_dma_map(pci_dev, dir[0], TARGET_PAGE_SIZE);
 393     if (!tbl) {
 394         rdma_error_report("Failed to map to QP page table");
 395         goto out;
 396     }
 397
 398     if (!is_srq) {
 399         sr = g_malloc(2 * sizeof(*rr));
 400         rr = &sr[1];
 401     } else {
 402         sr = g_malloc(sizeof(*sr));
 403     }
 404
 405     *rings = sr;
 406
 407     /* Create send ring */
 408     sr->ring_state = (PvrdmaRingState *)
 409         rdma_pci_dma_map(pci_dev, tbl[0], TARGET_PAGE_SIZE);
 410     if (!sr->ring_state) {
 411         rdma_error_report("Failed to map to QP ring state");
 412         goto out_free_sr_mem;
 413     }
 414
 415     wqe_sz = pow2ceil(sizeof(struct pvrdma_sq_wqe_hdr) +
 416                       sizeof(struct pvrdma_sge) * smax_sge - 1);
 417
 418     sprintf(ring_name, "qp_sring_%" PRIx64, pdir_dma);
 419     rc = pvrdma_ring_init(sr, ring_name, pci_dev, sr->ring_state,
 420                           scqe, wqe_sz, (dma_addr_t *)&tbl[1], spages);
 421     if (rc) {
 422         goto out_unmap_ring_state;
 423     }
 424
 425     if (!is_srq) {
 426         /* Create recv ring */
 427         rr->ring_state = &sr->ring_state[1];
 428         wqe_sz = pow2ceil(sizeof(struct pvrdma_rq_wqe_hdr) +
 429                           sizeof(struct pvrdma_sge) * rmax_sge - 1);
 430         sprintf(ring_name, "qp_rring_%" PRIx64, pdir_dma);
 431         rc = pvrdma_ring_init(rr, ring_name, pci_dev, rr->ring_state,
 432                               rcqe, wqe_sz, (dma_addr_t *)&tbl[1 + spages],
 433                               rpages);
 434         if (rc) {
 435             goto out_free_sr;
 436         }
 437     }
 438
 439     goto out;
 440
 441 out_free_sr:
 442     pvrdma_ring_free(sr);
 443
 444 out_unmap_ring_state:
 445     rdma_pci_dma_unmap(pci_dev, sr->ring_state, TARGET_PAGE_SIZE);
 446
 447 out_free_sr_mem:
 448     g_free(sr);
 449
 450 out:
 451     rdma_pci_dma_unmap(pci_dev, tbl, TARGET_PAGE_SIZE);
 452     rdma_pci_dma_unmap(pci_dev, dir, TARGET_PAGE_SIZE);
 453
 454     return rc;
 455 }
 456
 457 static void destroy_qp_rings(PvrdmaRing *ring, uint8_t is_srq)
 458 {
 459     pvrdma_ring_free(&ring[0]);
 460     if (!is_srq) {
 461         pvrdma_ring_free(&ring[1]);
 462     }
 463
 464     rdma_pci_dma_unmap(ring->dev, ring->ring_state, TARGET_PAGE_SIZE);
 465     g_free(ring);
 466 }
 467
 468 static int create_qp(PVRDMADev *dev, union pvrdma_cmd_req *req,
 469                      union pvrdma_cmd_resp *rsp)
 470 {
 471     struct pvrdma_cmd_create_qp *cmd = &req->create_qp;
 472     struct pvrdma_cmd_create_qp_resp *resp = &rsp->create_qp_resp;
 473     PvrdmaRing *rings = NULL;
 474     int rc;
 475
 476     memset(resp, 0, sizeof(*resp));
 477
 478     rc = create_qp_rings(PCI_DEVICE(dev), cmd->pdir_dma, &rings,
 479                          cmd->max_send_wr, cmd->max_send_sge, cmd->send_chunks,
 480                          cmd->max_recv_wr, cmd->max_recv_sge,
 481                          cmd->total_chunks - cmd->send_chunks - 1, cmd->is_srq);
 482     if (rc) {
 483         return rc;
 484     }
 485
 486     rc = rdma_rm_alloc_qp(&dev->rdma_dev_res, cmd->pd_handle, cmd->qp_type,
 487                           cmd->max_send_wr, cmd->max_send_sge,
 488                           cmd->send_cq_handle, cmd->max_recv_wr,
 489                           cmd->max_recv_sge, cmd->recv_cq_handle, rings,
 490                           &resp->qpn, cmd->is_srq, cmd->srq_handle);
 491     if (rc) {
 492         destroy_qp_rings(rings, cmd->is_srq);
 493         return rc;
 494     }
 495
 496     resp->max_send_wr = cmd->max_send_wr;
 497     resp->max_recv_wr = cmd->max_recv_wr;
 498     resp->max_send_sge = cmd->max_send_sge;
 499     resp->max_recv_sge = cmd->max_recv_sge;
 500     resp->max_inline_data = cmd->max_inline_data;
 501
 502     return 0;
 503 }
 504
 505 static int modify_qp(PVRDMADev *dev, union pvrdma_cmd_req *req,
 506                      union pvrdma_cmd_resp *rsp)
 507 {
 508     struct pvrdma_cmd_modify_qp *cmd = &req->modify_qp;
 509     int rc;
 510
 511     /* No need to verify sgid_index since it is u8 */
 512
 513     rc = rdma_rm_modify_qp(&dev->rdma_dev_res, &dev->backend_dev,
 514                            cmd->qp_handle, cmd->attr_mask,
 515                            cmd->attrs.ah_attr.grh.sgid_index,
 516                            (union ibv_gid *)&cmd->attrs.ah_attr.grh.dgid,
 517                            cmd->attrs.dest_qp_num,
 518                            (enum ibv_qp_state)cmd->attrs.qp_state,
 519                            cmd->attrs.qkey, cmd->attrs.rq_psn,
 520                            cmd->attrs.sq_psn);
 521
 522     return rc;
 523 }
 524
 525 static int query_qp(PVRDMADev *dev, union pvrdma_cmd_req *req,
 526                      union pvrdma_cmd_resp *rsp)
 527 {
 528     struct pvrdma_cmd_query_qp *cmd = &req->query_qp;
 529     struct pvrdma_cmd_query_qp_resp *resp = &rsp->query_qp_resp;
 530     struct ibv_qp_init_attr init_attr;
 531     int rc;
 532
 533     memset(resp, 0, sizeof(*resp));
 534
 535     rc = rdma_rm_query_qp(&dev->rdma_dev_res, &dev->backend_dev, cmd->qp_handle,
 536                           (struct ibv_qp_attr *)&resp->attrs, cmd->attr_mask,
 537                           &init_attr);
 538
 539     return rc;
 540 }
 541
 542 static int destroy_qp(PVRDMADev *dev, union pvrdma_cmd_req *req,
 543                       union pvrdma_cmd_resp *rsp)
 544 {
 545     struct pvrdma_cmd_destroy_qp *cmd = &req->destroy_qp;
 546     RdmaRmQP *qp;
 547     PvrdmaRing *ring;
 548
 549     qp = rdma_rm_get_qp(&dev->rdma_dev_res, cmd->qp_handle);
 550     if (!qp) {
 551         return -EINVAL;
 552     }
 553
 554     ring = (PvrdmaRing *)qp->opaque;
 555     destroy_qp_rings(ring, qp->is_srq);
 556     rdma_rm_dealloc_qp(&dev->rdma_dev_res, cmd->qp_handle);
 557
 558     return 0;
 559 }
 560
 561 static int create_bind(PVRDMADev *dev, union pvrdma_cmd_req *req,
 562                        union pvrdma_cmd_resp *rsp)
 563 {
 564     struct pvrdma_cmd_create_bind *cmd = &req->create_bind;
 565     int rc;
 566     union ibv_gid *gid = (union ibv_gid *)&cmd->new_gid;
 567
 568     if (cmd->index >= MAX_PORT_GIDS) {
 569         return -EINVAL;
 570     }
 571
 572     rc = rdma_rm_add_gid(&dev->rdma_dev_res, &dev->backend_dev,
 573                          dev->backend_eth_device_name, gid, cmd->index);
 574
 575     return rc;
 576 }
 577
 578 static int destroy_bind(PVRDMADev *dev, union pvrdma_cmd_req *req,
 579                         union pvrdma_cmd_resp *rsp)
 580 {
 581     int rc;
 582
 583     struct pvrdma_cmd_destroy_bind *cmd = &req->destroy_bind;
 584
 585     if (cmd->index >= MAX_PORT_GIDS) {
 586         return -EINVAL;
 587     }
 588
 589     rc = rdma_rm_del_gid(&dev->rdma_dev_res, &dev->backend_dev,
 590                         dev->backend_eth_device_name, cmd->index);
 591
 592     return rc;
 593 }
 594
 595 static int create_uc(PVRDMADev *dev, union pvrdma_cmd_req *req,
 596                      union pvrdma_cmd_resp *rsp)
 597 {
 598     struct pvrdma_cmd_create_uc *cmd = &req->create_uc;
 599     struct pvrdma_cmd_create_uc_resp *resp = &rsp->create_uc_resp;
 600     int rc;
 601
 602     memset(resp, 0, sizeof(*resp));
 603     rc = rdma_rm_alloc_uc(&dev->rdma_dev_res, cmd->pfn, &resp->ctx_handle);
 604
 605     return rc;
 606 }
 607
 608 static int destroy_uc(PVRDMADev *dev, union pvrdma_cmd_req *req,
 609                       union pvrdma_cmd_resp *rsp)
 610 {
 611     struct pvrdma_cmd_destroy_uc *cmd = &req->destroy_uc;
 612
 613     rdma_rm_dealloc_uc(&dev->rdma_dev_res, cmd->ctx_handle);
 614
 615     return 0;
 616 }
 617
 618 static int create_srq_ring(PCIDevice *pci_dev, PvrdmaRing **ring,
 619                            uint64_t pdir_dma, uint32_t max_wr,
 620                            uint32_t max_sge, uint32_t nchunks)
 621 {
 622     uint64_t *dir = NULL, *tbl = NULL;
 623     PvrdmaRing *r;
 624     int rc = -EINVAL;
 625     char ring_name[MAX_RING_NAME_SZ];
 626     uint32_t wqe_sz;
 627
 628     if (!nchunks || nchunks > PVRDMA_MAX_FAST_REG_PAGES) {
 629         rdma_error_report("Got invalid page count for SRQ ring: %d",
 630                           nchunks);
 631         return rc;
 632     }
 633
 634     dir = rdma_pci_dma_map(pci_dev, pdir_dma, TARGET_PAGE_SIZE);
 635     if (!dir) {
 636         rdma_error_report("Failed to map to SRQ page directory");
 637         goto out;
 638     }
 639
 640     tbl = rdma_pci_dma_map(pci_dev, dir[0], TARGET_PAGE_SIZE);
 641     if (!tbl) {
 642         rdma_error_report("Failed to map to SRQ page table");
 643         goto out;
 644     }
 645
 646     r = g_malloc(sizeof(*r));
 647     *ring = r;
 648
 649     r->ring_state = (PvrdmaRingState *)
 650             rdma_pci_dma_map(pci_dev, tbl[0], TARGET_PAGE_SIZE);
 651     if (!r->ring_state) {
 652         rdma_error_report("Failed to map tp SRQ ring state");
 653         goto out_free_ring_mem;
 654     }
 655
 656     wqe_sz = pow2ceil(sizeof(struct pvrdma_rq_wqe_hdr) +
 657                       sizeof(struct pvrdma_sge) * max_sge - 1);
 658     sprintf(ring_name, "srq_ring_%" PRIx64, pdir_dma);
 659     rc = pvrdma_ring_init(r, ring_name, pci_dev, &r->ring_state[1], max_wr,
 660                           wqe_sz, (dma_addr_t *)&tbl[1], nchunks - 1);
 661     if (rc) {
 662         goto out_unmap_ring_state;
 663     }
 664
 665     goto out;
 666
 667 out_unmap_ring_state:
 668     rdma_pci_dma_unmap(pci_dev, r->ring_state, TARGET_PAGE_SIZE);
 669
 670 out_free_ring_mem:
 671     g_free(r);
 672
 673 out:
 674     rdma_pci_dma_unmap(pci_dev, tbl, TARGET_PAGE_SIZE);
 675     rdma_pci_dma_unmap(pci_dev, dir, TARGET_PAGE_SIZE);
 676
 677     return rc;
 678 }
 679
 680 static void destroy_srq_ring(PvrdmaRing *ring)
 681 {
 682     pvrdma_ring_free(ring);
 683     rdma_pci_dma_unmap(ring->dev, ring->ring_state, TARGET_PAGE_SIZE);
 684     g_free(ring);
 685 }
 686
 687 static int create_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
 688                       union pvrdma_cmd_resp *rsp)
 689 {
 690     struct pvrdma_cmd_create_srq *cmd = &req->create_srq;
 691     struct pvrdma_cmd_create_srq_resp *resp = &rsp->create_srq_resp;
 692     PvrdmaRing *ring = NULL;
 693     int rc;
 694
 695     memset(resp, 0, sizeof(*resp));
 696
 697     rc = create_srq_ring(PCI_DEVICE(dev), &ring, cmd->pdir_dma,
 698                          cmd->attrs.max_wr, cmd->attrs.max_sge,
 699                          cmd->nchunks);
 700     if (rc) {
 701         return rc;
 702     }
 703
 704     rc = rdma_rm_alloc_srq(&dev->rdma_dev_res, cmd->pd_handle,
 705                            cmd->attrs.max_wr, cmd->attrs.max_sge,
 706                            cmd->attrs.srq_limit, &resp->srqn, ring);
 707     if (rc) {
 708         destroy_srq_ring(ring);
 709         return rc;
 710     }
 711
 712     return 0;
 713 }
 714
 715 static int query_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
 716                      union pvrdma_cmd_resp *rsp)
 717 {
 718     struct pvrdma_cmd_query_srq *cmd = &req->query_srq;
 719     struct pvrdma_cmd_query_srq_resp *resp = &rsp->query_srq_resp;
 720
 721     memset(resp, 0, sizeof(*resp));
 722
 723     return rdma_rm_query_srq(&dev->rdma_dev_res, cmd->srq_handle,
 724                              (struct ibv_srq_attr *)&resp->attrs);
 725 }
 726
 727 static int modify_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
 728                       union pvrdma_cmd_resp *rsp)
 729 {
 730     struct pvrdma_cmd_modify_srq *cmd = &req->modify_srq;
 731
 732     /* Only support SRQ limit */
 733     if (!(cmd->attr_mask & IBV_SRQ_LIMIT) ||
 734         (cmd->attr_mask & IBV_SRQ_MAX_WR))
 735             return -EINVAL;
 736
 737     return rdma_rm_modify_srq(&dev->rdma_dev_res, cmd->srq_handle,
 738                               (struct ibv_srq_attr *)&cmd->attrs,
 739                               cmd->attr_mask);
 740 }
 741
 742 static int destroy_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
 743                        union pvrdma_cmd_resp *rsp)
 744 {
 745     struct pvrdma_cmd_destroy_srq *cmd = &req->destroy_srq;
 746     RdmaRmSRQ *srq;
 747     PvrdmaRing *ring;
 748
 749     srq = rdma_rm_get_srq(&dev->rdma_dev_res, cmd->srq_handle);
 750     if (!srq) {
 751         return -EINVAL;
 752     }
 753
 754     ring = (PvrdmaRing *)srq->opaque;
 755     destroy_srq_ring(ring);
 756     rdma_rm_dealloc_srq(&dev->rdma_dev_res, cmd->srq_handle);
 757
 758     return 0;
 759 }
 760
 761 struct cmd_handler {
 762     uint32_t cmd;
 763     uint32_t ack;
 764     int (*exec)(PVRDMADev *dev, union pvrdma_cmd_req *req,
 765             union pvrdma_cmd_resp *rsp);
 766 };
 767
 768 static struct cmd_handler cmd_handlers[] = {
 769     {PVRDMA_CMD_QUERY_PORT,   PVRDMA_CMD_QUERY_PORT_RESP,        query_port},
 770     {PVRDMA_CMD_QUERY_PKEY,   PVRDMA_CMD_QUERY_PKEY_RESP,        query_pkey},
 771     {PVRDMA_CMD_CREATE_PD,    PVRDMA_CMD_CREATE_PD_RESP,         create_pd},
 772     {PVRDMA_CMD_DESTROY_PD,   PVRDMA_CMD_DESTROY_PD_RESP_NOOP,   destroy_pd},
 773     {PVRDMA_CMD_CREATE_MR,    PVRDMA_CMD_CREATE_MR_RESP,         create_mr},
 774     {PVRDMA_CMD_DESTROY_MR,   PVRDMA_CMD_DESTROY_MR_RESP_NOOP,   destroy_mr},
 775     {PVRDMA_CMD_CREATE_CQ,    PVRDMA_CMD_CREATE_CQ_RESP,         create_cq},
 776     {PVRDMA_CMD_RESIZE_CQ,    PVRDMA_CMD_RESIZE_CQ_RESP,         NULL},
 777     {PVRDMA_CMD_DESTROY_CQ,   PVRDMA_CMD_DESTROY_CQ_RESP_NOOP,   destroy_cq},
 778     {PVRDMA_CMD_CREATE_QP,    PVRDMA_CMD_CREATE_QP_RESP,         create_qp},
 779     {PVRDMA_CMD_MODIFY_QP,    PVRDMA_CMD_MODIFY_QP_RESP,         modify_qp},
 780     {PVRDMA_CMD_QUERY_QP,     PVRDMA_CMD_QUERY_QP_RESP,          query_qp},
 781     {PVRDMA_CMD_DESTROY_QP,   PVRDMA_CMD_DESTROY_QP_RESP,        destroy_qp},
 782     {PVRDMA_CMD_CREATE_UC,    PVRDMA_CMD_CREATE_UC_RESP,         create_uc},
 783     {PVRDMA_CMD_DESTROY_UC,   PVRDMA_CMD_DESTROY_UC_RESP_NOOP,   destroy_uc},
 784     {PVRDMA_CMD_CREATE_BIND,  PVRDMA_CMD_CREATE_BIND_RESP_NOOP,  create_bind},
 785     {PVRDMA_CMD_DESTROY_BIND, PVRDMA_CMD_DESTROY_BIND_RESP_NOOP, destroy_bind},
 786     {PVRDMA_CMD_CREATE_SRQ,   PVRDMA_CMD_CREATE_SRQ_RESP,        create_srq},
 787     {PVRDMA_CMD_QUERY_SRQ,    PVRDMA_CMD_QUERY_SRQ_RESP,         query_srq},
 788     {PVRDMA_CMD_MODIFY_SRQ,   PVRDMA_CMD_MODIFY_SRQ_RESP,        modify_srq},
 789     {PVRDMA_CMD_DESTROY_SRQ,  PVRDMA_CMD_DESTROY_SRQ_RESP,       destroy_srq},
 790 };
 791
 792 int pvrdma_exec_cmd(PVRDMADev *dev)
 793 {
 794     int err = 0xFFFF;
 795     DSRInfo *dsr_info;
 796
 797     dsr_info = &dev->dsr_info;
 798
 799     if (dsr_info->req->hdr.cmd >= sizeof(cmd_handlers) /
 800                       sizeof(struct cmd_handler)) {
 801         rdma_error_report("Unsupported command");
 802         goto out;
 803     }
 804
 805     if (!cmd_handlers[dsr_info->req->hdr.cmd].exec) {
 806         rdma_error_report("Unsupported command (not implemented yet)");
 807         goto out;
 808     }
 809
 810     err = cmd_handlers[dsr_info->req->hdr.cmd].exec(dev, dsr_info->req,
 811                                                     dsr_info->rsp);
 812     dsr_info->rsp->hdr.response = dsr_info->req->hdr.response;
 813     dsr_info->rsp->hdr.ack = cmd_handlers[dsr_info->req->hdr.cmd].ack;
 814     dsr_info->rsp->hdr.err = err < 0 ? -err : 0;
 815
 816     trace_pvrdma_exec_cmd(dsr_info->req->hdr.cmd, dsr_info->rsp->hdr.err);
 817
 818     dev->stats.commands++;
 819
 820 out:
 821     set_reg_val(dev, PVRDMA_REG_ERR, err);
 822     post_interrupt(dev, INTR_VEC_CMD_RING);
 823
 824     return (err == 0) ? 0 : -EINVAL;
 825 }