util/vfio-helpers: Pass page protections to qemu_vfio_pci_map_bar()
[qemu.git] / block / nvme.c
1 /*
2 * NVMe block driver based on vfio
3 *
4 * Copyright 2016 - 2018 Red Hat, Inc.
5 *
6 * Authors:
7 * Fam Zheng <famz@redhat.com>
8 * Paolo Bonzini <pbonzini@redhat.com>
9 *
10 * This work is licensed under the terms of the GNU GPL, version 2 or later.
11 * See the COPYING file in the top-level directory.
12 */
13
14 #include "qemu/osdep.h"
15 #include <linux/vfio.h>
16 #include "qapi/error.h"
17 #include "qapi/qmp/qdict.h"
18 #include "qapi/qmp/qstring.h"
19 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qemu/module.h"
22 #include "qemu/cutils.h"
23 #include "qemu/option.h"
24 #include "qemu/vfio-helpers.h"
25 #include "block/block_int.h"
26 #include "sysemu/replay.h"
27 #include "trace.h"
28
29 #include "block/nvme.h"
30
31 #define NVME_SQ_ENTRY_BYTES 64
32 #define NVME_CQ_ENTRY_BYTES 16
33 #define NVME_QUEUE_SIZE 128
34 #define NVME_BAR_SIZE 8192
35
36 /*
37 * We have to leave one slot empty as that is the full queue case where
38 * head == tail + 1.
39 */
40 #define NVME_NUM_REQS (NVME_QUEUE_SIZE - 1)
41
42 typedef struct BDRVNVMeState BDRVNVMeState;
43
44 typedef struct {
45 int32_t head, tail;
46 uint8_t *queue;
47 uint64_t iova;
48 /* Hardware MMIO register */
49 volatile uint32_t *doorbell;
50 } NVMeQueue;
51
52 typedef struct {
53 BlockCompletionFunc *cb;
54 void *opaque;
55 int cid;
56 void *prp_list_page;
57 uint64_t prp_list_iova;
58 int free_req_next; /* q->reqs[] index of next free req */
59 } NVMeRequest;
60
61 typedef struct {
62 QemuMutex lock;
63
64 /* Read from I/O code path, initialized under BQL */
65 BDRVNVMeState *s;
66 int index;
67
68 /* Fields protected by BQL */
69 uint8_t *prp_list_pages;
70
71 /* Fields protected by @lock */
72 CoQueue free_req_queue;
73 NVMeQueue sq, cq;
74 int cq_phase;
75 int free_req_head;
76 NVMeRequest reqs[NVME_NUM_REQS];
77 int need_kick;
78 int inflight;
79
80 /* Thread-safe, no lock necessary */
81 QEMUBH *completion_bh;
82 } NVMeQueuePair;
83
84 /* Memory mapped registers */
85 typedef volatile struct {
86 NvmeBar ctrl;
87 struct {
88 uint32_t sq_tail;
89 uint32_t cq_head;
90 } doorbells[];
91 } NVMeRegs;
92
93 #define INDEX_ADMIN 0
94 #define INDEX_IO(n) (1 + n)
95
96 /* This driver shares a single MSIX IRQ for the admin and I/O queues */
97 enum {
98 MSIX_SHARED_IRQ_IDX = 0,
99 MSIX_IRQ_COUNT = 1
100 };
101
102 struct BDRVNVMeState {
103 AioContext *aio_context;
104 QEMUVFIOState *vfio;
105 NVMeRegs *regs;
106 /* The submission/completion queue pairs.
107 * [0]: admin queue.
108 * [1..]: io queues.
109 */
110 NVMeQueuePair **queues;
111 int nr_queues;
112 size_t page_size;
113 /* How many uint32_t elements does each doorbell entry take. */
114 size_t doorbell_scale;
115 bool write_cache_supported;
116 EventNotifier irq_notifier[MSIX_IRQ_COUNT];
117
118 uint64_t nsze; /* Namespace size reported by identify command */
119 int nsid; /* The namespace id to read/write data. */
120 int blkshift;
121
122 uint64_t max_transfer;
123 bool plugged;
124
125 bool supports_write_zeroes;
126 bool supports_discard;
127
128 CoMutex dma_map_lock;
129 CoQueue dma_flush_queue;
130
131 /* Total size of mapped qiov, accessed under dma_map_lock */
132 int dma_map_count;
133
134 /* PCI address (required for nvme_refresh_filename()) */
135 char *device;
136 };
137
138 #define NVME_BLOCK_OPT_DEVICE "device"
139 #define NVME_BLOCK_OPT_NAMESPACE "namespace"
140
141 static void nvme_process_completion_bh(void *opaque);
142
143 static QemuOptsList runtime_opts = {
144 .name = "nvme",
145 .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
146 .desc = {
147 {
148 .name = NVME_BLOCK_OPT_DEVICE,
149 .type = QEMU_OPT_STRING,
150 .help = "NVMe PCI device address",
151 },
152 {
153 .name = NVME_BLOCK_OPT_NAMESPACE,
154 .type = QEMU_OPT_NUMBER,
155 .help = "NVMe namespace",
156 },
157 { /* end of list */ }
158 },
159 };
160
161 static void nvme_init_queue(BDRVNVMeState *s, NVMeQueue *q,
162 int nentries, int entry_bytes, Error **errp)
163 {
164 size_t bytes;
165 int r;
166
167 bytes = ROUND_UP(nentries * entry_bytes, s->page_size);
168 q->head = q->tail = 0;
169 q->queue = qemu_try_memalign(s->page_size, bytes);
170 if (!q->queue) {
171 error_setg(errp, "Cannot allocate queue");
172 return;
173 }
174 memset(q->queue, 0, bytes);
175 r = qemu_vfio_dma_map(s->vfio, q->queue, bytes, false, &q->iova);
176 if (r) {
177 error_setg(errp, "Cannot map queue");
178 }
179 }
180
181 static void nvme_free_queue_pair(NVMeQueuePair *q)
182 {
183 if (q->completion_bh) {
184 qemu_bh_delete(q->completion_bh);
185 }
186 qemu_vfree(q->prp_list_pages);
187 qemu_vfree(q->sq.queue);
188 qemu_vfree(q->cq.queue);
189 qemu_mutex_destroy(&q->lock);
190 g_free(q);
191 }
192
193 static void nvme_free_req_queue_cb(void *opaque)
194 {
195 NVMeQueuePair *q = opaque;
196
197 qemu_mutex_lock(&q->lock);
198 while (qemu_co_enter_next(&q->free_req_queue, &q->lock)) {
199 /* Retry all pending requests */
200 }
201 qemu_mutex_unlock(&q->lock);
202 }
203
204 static NVMeQueuePair *nvme_create_queue_pair(BDRVNVMeState *s,
205 AioContext *aio_context,
206 int idx, int size,
207 Error **errp)
208 {
209 int i, r;
210 Error *local_err = NULL;
211 NVMeQueuePair *q;
212 uint64_t prp_list_iova;
213
214 q = g_try_new0(NVMeQueuePair, 1);
215 if (!q) {
216 return NULL;
217 }
218 q->prp_list_pages = qemu_try_memalign(s->page_size,
219 s->page_size * NVME_NUM_REQS);
220 if (!q->prp_list_pages) {
221 goto fail;
222 }
223 memset(q->prp_list_pages, 0, s->page_size * NVME_NUM_REQS);
224 qemu_mutex_init(&q->lock);
225 q->s = s;
226 q->index = idx;
227 qemu_co_queue_init(&q->free_req_queue);
228 q->completion_bh = aio_bh_new(aio_context, nvme_process_completion_bh, q);
229 r = qemu_vfio_dma_map(s->vfio, q->prp_list_pages,
230 s->page_size * NVME_NUM_REQS,
231 false, &prp_list_iova);
232 if (r) {
233 goto fail;
234 }
235 q->free_req_head = -1;
236 for (i = 0; i < NVME_NUM_REQS; i++) {
237 NVMeRequest *req = &q->reqs[i];
238 req->cid = i + 1;
239 req->free_req_next = q->free_req_head;
240 q->free_req_head = i;
241 req->prp_list_page = q->prp_list_pages + i * s->page_size;
242 req->prp_list_iova = prp_list_iova + i * s->page_size;
243 }
244
245 nvme_init_queue(s, &q->sq, size, NVME_SQ_ENTRY_BYTES, &local_err);
246 if (local_err) {
247 error_propagate(errp, local_err);
248 goto fail;
249 }
250 q->sq.doorbell = &s->regs->doorbells[idx * s->doorbell_scale].sq_tail;
251
252 nvme_init_queue(s, &q->cq, size, NVME_CQ_ENTRY_BYTES, &local_err);
253 if (local_err) {
254 error_propagate(errp, local_err);
255 goto fail;
256 }
257 q->cq.doorbell = &s->regs->doorbells[idx * s->doorbell_scale].cq_head;
258
259 return q;
260 fail:
261 nvme_free_queue_pair(q);
262 return NULL;
263 }
264
265 /* With q->lock */
266 static void nvme_kick(NVMeQueuePair *q)
267 {
268 BDRVNVMeState *s = q->s;
269
270 if (s->plugged || !q->need_kick) {
271 return;
272 }
273 trace_nvme_kick(s, q->index);
274 assert(!(q->sq.tail & 0xFF00));
275 /* Fence the write to submission queue entry before notifying the device. */
276 smp_wmb();
277 *q->sq.doorbell = cpu_to_le32(q->sq.tail);
278 q->inflight += q->need_kick;
279 q->need_kick = 0;
280 }
281
282 /* Find a free request element if any, otherwise:
283 * a) if in coroutine context, try to wait for one to become available;
284 * b) if not in coroutine, return NULL;
285 */
286 static NVMeRequest *nvme_get_free_req(NVMeQueuePair *q)
287 {
288 NVMeRequest *req;
289
290 qemu_mutex_lock(&q->lock);
291
292 while (q->free_req_head == -1) {
293 if (qemu_in_coroutine()) {
294 trace_nvme_free_req_queue_wait(q);
295 qemu_co_queue_wait(&q->free_req_queue, &q->lock);
296 } else {
297 qemu_mutex_unlock(&q->lock);
298 return NULL;
299 }
300 }
301
302 req = &q->reqs[q->free_req_head];
303 q->free_req_head = req->free_req_next;
304 req->free_req_next = -1;
305
306 qemu_mutex_unlock(&q->lock);
307 return req;
308 }
309
310 /* With q->lock */
311 static void nvme_put_free_req_locked(NVMeQueuePair *q, NVMeRequest *req)
312 {
313 req->free_req_next = q->free_req_head;
314 q->free_req_head = req - q->reqs;
315 }
316
317 /* With q->lock */
318 static void nvme_wake_free_req_locked(NVMeQueuePair *q)
319 {
320 if (!qemu_co_queue_empty(&q->free_req_queue)) {
321 replay_bh_schedule_oneshot_event(q->s->aio_context,
322 nvme_free_req_queue_cb, q);
323 }
324 }
325
326 /* Insert a request in the freelist and wake waiters */
327 static void nvme_put_free_req_and_wake(NVMeQueuePair *q, NVMeRequest *req)
328 {
329 qemu_mutex_lock(&q->lock);
330 nvme_put_free_req_locked(q, req);
331 nvme_wake_free_req_locked(q);
332 qemu_mutex_unlock(&q->lock);
333 }
334
335 static inline int nvme_translate_error(const NvmeCqe *c)
336 {
337 uint16_t status = (le16_to_cpu(c->status) >> 1) & 0xFF;
338 if (status) {
339 trace_nvme_error(le32_to_cpu(c->result),
340 le16_to_cpu(c->sq_head),
341 le16_to_cpu(c->sq_id),
342 le16_to_cpu(c->cid),
343 le16_to_cpu(status));
344 }
345 switch (status) {
346 case 0:
347 return 0;
348 case 1:
349 return -ENOSYS;
350 case 2:
351 return -EINVAL;
352 default:
353 return -EIO;
354 }
355 }
356
357 /* With q->lock */
358 static bool nvme_process_completion(NVMeQueuePair *q)
359 {
360 BDRVNVMeState *s = q->s;
361 bool progress = false;
362 NVMeRequest *preq;
363 NVMeRequest req;
364 NvmeCqe *c;
365
366 trace_nvme_process_completion(s, q->index, q->inflight);
367 if (s->plugged) {
368 trace_nvme_process_completion_queue_plugged(s, q->index);
369 return false;
370 }
371
372 /*
373 * Support re-entrancy when a request cb() function invokes aio_poll().
374 * Pending completions must be visible to aio_poll() so that a cb()
375 * function can wait for the completion of another request.
376 *
377 * The aio_poll() loop will execute our BH and we'll resume completion
378 * processing there.
379 */
380 qemu_bh_schedule(q->completion_bh);
381
382 assert(q->inflight >= 0);
383 while (q->inflight) {
384 int ret;
385 int16_t cid;
386
387 c = (NvmeCqe *)&q->cq.queue[q->cq.head * NVME_CQ_ENTRY_BYTES];
388 if ((le16_to_cpu(c->status) & 0x1) == q->cq_phase) {
389 break;
390 }
391 ret = nvme_translate_error(c);
392 q->cq.head = (q->cq.head + 1) % NVME_QUEUE_SIZE;
393 if (!q->cq.head) {
394 q->cq_phase = !q->cq_phase;
395 }
396 cid = le16_to_cpu(c->cid);
397 if (cid == 0 || cid > NVME_QUEUE_SIZE) {
398 fprintf(stderr, "Unexpected CID in completion queue: %" PRIu32 "\n",
399 cid);
400 continue;
401 }
402 trace_nvme_complete_command(s, q->index, cid);
403 preq = &q->reqs[cid - 1];
404 req = *preq;
405 assert(req.cid == cid);
406 assert(req.cb);
407 nvme_put_free_req_locked(q, preq);
408 preq->cb = preq->opaque = NULL;
409 q->inflight--;
410 qemu_mutex_unlock(&q->lock);
411 req.cb(req.opaque, ret);
412 qemu_mutex_lock(&q->lock);
413 progress = true;
414 }
415 if (progress) {
416 /* Notify the device so it can post more completions. */
417 smp_mb_release();
418 *q->cq.doorbell = cpu_to_le32(q->cq.head);
419 nvme_wake_free_req_locked(q);
420 }
421
422 qemu_bh_cancel(q->completion_bh);
423
424 return progress;
425 }
426
427 static void nvme_process_completion_bh(void *opaque)
428 {
429 NVMeQueuePair *q = opaque;
430
431 /*
432 * We're being invoked because a nvme_process_completion() cb() function
433 * called aio_poll(). The callback may be waiting for further completions
434 * so notify the device that it has space to fill in more completions now.
435 */
436 smp_mb_release();
437 *q->cq.doorbell = cpu_to_le32(q->cq.head);
438 nvme_wake_free_req_locked(q);
439
440 nvme_process_completion(q);
441 }
442
443 static void nvme_trace_command(const NvmeCmd *cmd)
444 {
445 int i;
446
447 if (!trace_event_get_state_backends(TRACE_NVME_SUBMIT_COMMAND_RAW)) {
448 return;
449 }
450 for (i = 0; i < 8; ++i) {
451 uint8_t *cmdp = (uint8_t *)cmd + i * 8;
452 trace_nvme_submit_command_raw(cmdp[0], cmdp[1], cmdp[2], cmdp[3],
453 cmdp[4], cmdp[5], cmdp[6], cmdp[7]);
454 }
455 }
456
457 static void nvme_submit_command(NVMeQueuePair *q, NVMeRequest *req,
458 NvmeCmd *cmd, BlockCompletionFunc cb,
459 void *opaque)
460 {
461 assert(!req->cb);
462 req->cb = cb;
463 req->opaque = opaque;
464 cmd->cid = cpu_to_le32(req->cid);
465
466 trace_nvme_submit_command(q->s, q->index, req->cid);
467 nvme_trace_command(cmd);
468 qemu_mutex_lock(&q->lock);
469 memcpy((uint8_t *)q->sq.queue +
470 q->sq.tail * NVME_SQ_ENTRY_BYTES, cmd, sizeof(*cmd));
471 q->sq.tail = (q->sq.tail + 1) % NVME_QUEUE_SIZE;
472 q->need_kick++;
473 nvme_kick(q);
474 nvme_process_completion(q);
475 qemu_mutex_unlock(&q->lock);
476 }
477
478 static void nvme_cmd_sync_cb(void *opaque, int ret)
479 {
480 int *pret = opaque;
481 *pret = ret;
482 aio_wait_kick();
483 }
484
485 static int nvme_cmd_sync(BlockDriverState *bs, NVMeQueuePair *q,
486 NvmeCmd *cmd)
487 {
488 AioContext *aio_context = bdrv_get_aio_context(bs);
489 NVMeRequest *req;
490 int ret = -EINPROGRESS;
491 req = nvme_get_free_req(q);
492 if (!req) {
493 return -EBUSY;
494 }
495 nvme_submit_command(q, req, cmd, nvme_cmd_sync_cb, &ret);
496
497 AIO_WAIT_WHILE(aio_context, ret == -EINPROGRESS);
498 return ret;
499 }
500
501 static void nvme_identify(BlockDriverState *bs, int namespace, Error **errp)
502 {
503 BDRVNVMeState *s = bs->opaque;
504 union {
505 NvmeIdCtrl ctrl;
506 NvmeIdNs ns;
507 } *id;
508 NvmeLBAF *lbaf;
509 uint16_t oncs;
510 int r;
511 uint64_t iova;
512 NvmeCmd cmd = {
513 .opcode = NVME_ADM_CMD_IDENTIFY,
514 .cdw10 = cpu_to_le32(0x1),
515 };
516
517 id = qemu_try_memalign(s->page_size, sizeof(*id));
518 if (!id) {
519 error_setg(errp, "Cannot allocate buffer for identify response");
520 goto out;
521 }
522 r = qemu_vfio_dma_map(s->vfio, id, sizeof(*id), true, &iova);
523 if (r) {
524 error_setg(errp, "Cannot map buffer for DMA");
525 goto out;
526 }
527
528 memset(id, 0, sizeof(*id));
529 cmd.dptr.prp1 = cpu_to_le64(iova);
530 if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
531 error_setg(errp, "Failed to identify controller");
532 goto out;
533 }
534
535 if (le32_to_cpu(id->ctrl.nn) < namespace) {
536 error_setg(errp, "Invalid namespace");
537 goto out;
538 }
539 s->write_cache_supported = le32_to_cpu(id->ctrl.vwc) & 0x1;
540 s->max_transfer = (id->ctrl.mdts ? 1 << id->ctrl.mdts : 0) * s->page_size;
541 /* For now the page list buffer per command is one page, to hold at most
542 * s->page_size / sizeof(uint64_t) entries. */
543 s->max_transfer = MIN_NON_ZERO(s->max_transfer,
544 s->page_size / sizeof(uint64_t) * s->page_size);
545
546 oncs = le16_to_cpu(id->ctrl.oncs);
547 s->supports_write_zeroes = !!(oncs & NVME_ONCS_WRITE_ZEROES);
548 s->supports_discard = !!(oncs & NVME_ONCS_DSM);
549
550 memset(id, 0, sizeof(*id));
551 cmd.cdw10 = 0;
552 cmd.nsid = cpu_to_le32(namespace);
553 if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
554 error_setg(errp, "Failed to identify namespace");
555 goto out;
556 }
557
558 s->nsze = le64_to_cpu(id->ns.nsze);
559 lbaf = &id->ns.lbaf[NVME_ID_NS_FLBAS_INDEX(id->ns.flbas)];
560
561 if (NVME_ID_NS_DLFEAT_WRITE_ZEROES(id->ns.dlfeat) &&
562 NVME_ID_NS_DLFEAT_READ_BEHAVIOR(id->ns.dlfeat) ==
563 NVME_ID_NS_DLFEAT_READ_BEHAVIOR_ZEROES) {
564 bs->supported_write_flags |= BDRV_REQ_MAY_UNMAP;
565 }
566
567 if (lbaf->ms) {
568 error_setg(errp, "Namespaces with metadata are not yet supported");
569 goto out;
570 }
571
572 if (lbaf->ds < BDRV_SECTOR_BITS || lbaf->ds > 12 ||
573 (1 << lbaf->ds) > s->page_size)
574 {
575 error_setg(errp, "Namespace has unsupported block size (2^%d)",
576 lbaf->ds);
577 goto out;
578 }
579
580 s->blkshift = lbaf->ds;
581 out:
582 qemu_vfio_dma_unmap(s->vfio, id);
583 qemu_vfree(id);
584 }
585
586 static bool nvme_poll_queue(NVMeQueuePair *q)
587 {
588 bool progress = false;
589
590 const size_t cqe_offset = q->cq.head * NVME_CQ_ENTRY_BYTES;
591 NvmeCqe *cqe = (NvmeCqe *)&q->cq.queue[cqe_offset];
592
593 /*
594 * Do an early check for completions. q->lock isn't needed because
595 * nvme_process_completion() only runs in the event loop thread and
596 * cannot race with itself.
597 */
598 if ((le16_to_cpu(cqe->status) & 0x1) == q->cq_phase) {
599 return false;
600 }
601
602 qemu_mutex_lock(&q->lock);
603 while (nvme_process_completion(q)) {
604 /* Keep polling */
605 progress = true;
606 }
607 qemu_mutex_unlock(&q->lock);
608
609 return progress;
610 }
611
612 static bool nvme_poll_queues(BDRVNVMeState *s)
613 {
614 bool progress = false;
615 int i;
616
617 for (i = 0; i < s->nr_queues; i++) {
618 if (nvme_poll_queue(s->queues[i])) {
619 progress = true;
620 }
621 }
622 return progress;
623 }
624
625 static void nvme_handle_event(EventNotifier *n)
626 {
627 BDRVNVMeState *s = container_of(n, BDRVNVMeState,
628 irq_notifier[MSIX_SHARED_IRQ_IDX]);
629
630 trace_nvme_handle_event(s);
631 event_notifier_test_and_clear(n);
632 nvme_poll_queues(s);
633 }
634
635 static bool nvme_add_io_queue(BlockDriverState *bs, Error **errp)
636 {
637 BDRVNVMeState *s = bs->opaque;
638 int n = s->nr_queues;
639 NVMeQueuePair *q;
640 NvmeCmd cmd;
641 int queue_size = NVME_QUEUE_SIZE;
642
643 q = nvme_create_queue_pair(s, bdrv_get_aio_context(bs),
644 n, queue_size, errp);
645 if (!q) {
646 return false;
647 }
648 cmd = (NvmeCmd) {
649 .opcode = NVME_ADM_CMD_CREATE_CQ,
650 .dptr.prp1 = cpu_to_le64(q->cq.iova),
651 .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)),
652 .cdw11 = cpu_to_le32(0x3),
653 };
654 if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
655 error_setg(errp, "Failed to create CQ io queue [%d]", n);
656 goto out_error;
657 }
658 cmd = (NvmeCmd) {
659 .opcode = NVME_ADM_CMD_CREATE_SQ,
660 .dptr.prp1 = cpu_to_le64(q->sq.iova),
661 .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)),
662 .cdw11 = cpu_to_le32(0x1 | (n << 16)),
663 };
664 if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
665 error_setg(errp, "Failed to create SQ io queue [%d]", n);
666 goto out_error;
667 }
668 s->queues = g_renew(NVMeQueuePair *, s->queues, n + 1);
669 s->queues[n] = q;
670 s->nr_queues++;
671 return true;
672 out_error:
673 nvme_free_queue_pair(q);
674 return false;
675 }
676
677 static bool nvme_poll_cb(void *opaque)
678 {
679 EventNotifier *e = opaque;
680 BDRVNVMeState *s = container_of(e, BDRVNVMeState,
681 irq_notifier[MSIX_SHARED_IRQ_IDX]);
682
683 trace_nvme_poll_cb(s);
684 return nvme_poll_queues(s);
685 }
686
687 static int nvme_init(BlockDriverState *bs, const char *device, int namespace,
688 Error **errp)
689 {
690 BDRVNVMeState *s = bs->opaque;
691 AioContext *aio_context = bdrv_get_aio_context(bs);
692 int ret;
693 uint64_t cap;
694 uint64_t timeout_ms;
695 uint64_t deadline, now;
696 Error *local_err = NULL;
697
698 qemu_co_mutex_init(&s->dma_map_lock);
699 qemu_co_queue_init(&s->dma_flush_queue);
700 s->device = g_strdup(device);
701 s->nsid = namespace;
702 s->aio_context = bdrv_get_aio_context(bs);
703 ret = event_notifier_init(&s->irq_notifier[MSIX_SHARED_IRQ_IDX], 0);
704 if (ret) {
705 error_setg(errp, "Failed to init event notifier");
706 return ret;
707 }
708
709 s->vfio = qemu_vfio_open_pci(device, errp);
710 if (!s->vfio) {
711 ret = -EINVAL;
712 goto out;
713 }
714
715 s->regs = qemu_vfio_pci_map_bar(s->vfio, 0, 0, NVME_BAR_SIZE,
716 PROT_READ | PROT_WRITE, errp);
717 if (!s->regs) {
718 ret = -EINVAL;
719 goto out;
720 }
721
722 /* Perform initialize sequence as described in NVMe spec "7.6.1
723 * Initialization". */
724
725 cap = le64_to_cpu(s->regs->ctrl.cap);
726 if (!(cap & (1ULL << 37))) {
727 error_setg(errp, "Device doesn't support NVMe command set");
728 ret = -EINVAL;
729 goto out;
730 }
731
732 s->page_size = MAX(4096, 1 << (12 + ((cap >> 48) & 0xF)));
733 s->doorbell_scale = (4 << (((cap >> 32) & 0xF))) / sizeof(uint32_t);
734 bs->bl.opt_mem_alignment = s->page_size;
735 timeout_ms = MIN(500 * ((cap >> 24) & 0xFF), 30000);
736
737 /* Reset device to get a clean state. */
738 s->regs->ctrl.cc = cpu_to_le32(le32_to_cpu(s->regs->ctrl.cc) & 0xFE);
739 /* Wait for CSTS.RDY = 0. */
740 deadline = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + timeout_ms * SCALE_MS;
741 while (le32_to_cpu(s->regs->ctrl.csts) & 0x1) {
742 if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
743 error_setg(errp, "Timeout while waiting for device to reset (%"
744 PRId64 " ms)",
745 timeout_ms);
746 ret = -ETIMEDOUT;
747 goto out;
748 }
749 }
750
751 /* Set up admin queue. */
752 s->queues = g_new(NVMeQueuePair *, 1);
753 s->queues[INDEX_ADMIN] = nvme_create_queue_pair(s, aio_context, 0,
754 NVME_QUEUE_SIZE,
755 errp);
756 if (!s->queues[INDEX_ADMIN]) {
757 ret = -EINVAL;
758 goto out;
759 }
760 s->nr_queues = 1;
761 QEMU_BUILD_BUG_ON(NVME_QUEUE_SIZE & 0xF000);
762 s->regs->ctrl.aqa = cpu_to_le32((NVME_QUEUE_SIZE << 16) | NVME_QUEUE_SIZE);
763 s->regs->ctrl.asq = cpu_to_le64(s->queues[INDEX_ADMIN]->sq.iova);
764 s->regs->ctrl.acq = cpu_to_le64(s->queues[INDEX_ADMIN]->cq.iova);
765
766 /* After setting up all control registers we can enable device now. */
767 s->regs->ctrl.cc = cpu_to_le32((ctz32(NVME_CQ_ENTRY_BYTES) << 20) |
768 (ctz32(NVME_SQ_ENTRY_BYTES) << 16) |
769 0x1);
770 /* Wait for CSTS.RDY = 1. */
771 now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
772 deadline = now + timeout_ms * 1000000;
773 while (!(le32_to_cpu(s->regs->ctrl.csts) & 0x1)) {
774 if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
775 error_setg(errp, "Timeout while waiting for device to start (%"
776 PRId64 " ms)",
777 timeout_ms);
778 ret = -ETIMEDOUT;
779 goto out;
780 }
781 }
782
783 ret = qemu_vfio_pci_init_irq(s->vfio, s->irq_notifier,
784 VFIO_PCI_MSIX_IRQ_INDEX, errp);
785 if (ret) {
786 goto out;
787 }
788 aio_set_event_notifier(bdrv_get_aio_context(bs),
789 &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
790 false, nvme_handle_event, nvme_poll_cb);
791
792 nvme_identify(bs, namespace, &local_err);
793 if (local_err) {
794 error_propagate(errp, local_err);
795 ret = -EIO;
796 goto out;
797 }
798
799 /* Set up command queues. */
800 if (!nvme_add_io_queue(bs, errp)) {
801 ret = -EIO;
802 }
803 out:
804 /* Cleaning up is done in nvme_file_open() upon error. */
805 return ret;
806 }
807
808 /* Parse a filename in the format of nvme://XXXX:XX:XX.X/X. Example:
809 *
810 * nvme://0000:44:00.0/1
811 *
812 * where the "nvme://" is a fixed form of the protocol prefix, the middle part
813 * is the PCI address, and the last part is the namespace number starting from
814 * 1 according to the NVMe spec. */
815 static void nvme_parse_filename(const char *filename, QDict *options,
816 Error **errp)
817 {
818 int pref = strlen("nvme://");
819
820 if (strlen(filename) > pref && !strncmp(filename, "nvme://", pref)) {
821 const char *tmp = filename + pref;
822 char *device;
823 const char *namespace;
824 unsigned long ns;
825 const char *slash = strchr(tmp, '/');
826 if (!slash) {
827 qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, tmp);
828 return;
829 }
830 device = g_strndup(tmp, slash - tmp);
831 qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, device);
832 g_free(device);
833 namespace = slash + 1;
834 if (*namespace && qemu_strtoul(namespace, NULL, 10, &ns)) {
835 error_setg(errp, "Invalid namespace '%s', positive number expected",
836 namespace);
837 return;
838 }
839 qdict_put_str(options, NVME_BLOCK_OPT_NAMESPACE,
840 *namespace ? namespace : "1");
841 }
842 }
843
844 static int nvme_enable_disable_write_cache(BlockDriverState *bs, bool enable,
845 Error **errp)
846 {
847 int ret;
848 BDRVNVMeState *s = bs->opaque;
849 NvmeCmd cmd = {
850 .opcode = NVME_ADM_CMD_SET_FEATURES,
851 .nsid = cpu_to_le32(s->nsid),
852 .cdw10 = cpu_to_le32(0x06),
853 .cdw11 = cpu_to_le32(enable ? 0x01 : 0x00),
854 };
855
856 ret = nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd);
857 if (ret) {
858 error_setg(errp, "Failed to configure NVMe write cache");
859 }
860 return ret;
861 }
862
863 static void nvme_close(BlockDriverState *bs)
864 {
865 int i;
866 BDRVNVMeState *s = bs->opaque;
867
868 for (i = 0; i < s->nr_queues; ++i) {
869 nvme_free_queue_pair(s->queues[i]);
870 }
871 g_free(s->queues);
872 aio_set_event_notifier(bdrv_get_aio_context(bs),
873 &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
874 false, NULL, NULL);
875 event_notifier_cleanup(&s->irq_notifier[MSIX_SHARED_IRQ_IDX]);
876 qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)s->regs, 0, NVME_BAR_SIZE);
877 qemu_vfio_close(s->vfio);
878
879 g_free(s->device);
880 }
881
882 static int nvme_file_open(BlockDriverState *bs, QDict *options, int flags,
883 Error **errp)
884 {
885 const char *device;
886 QemuOpts *opts;
887 int namespace;
888 int ret;
889 BDRVNVMeState *s = bs->opaque;
890
891 bs->supported_write_flags = BDRV_REQ_FUA;
892
893 opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
894 qemu_opts_absorb_qdict(opts, options, &error_abort);
895 device = qemu_opt_get(opts, NVME_BLOCK_OPT_DEVICE);
896 if (!device) {
897 error_setg(errp, "'" NVME_BLOCK_OPT_DEVICE "' option is required");
898 qemu_opts_del(opts);
899 return -EINVAL;
900 }
901
902 namespace = qemu_opt_get_number(opts, NVME_BLOCK_OPT_NAMESPACE, 1);
903 ret = nvme_init(bs, device, namespace, errp);
904 qemu_opts_del(opts);
905 if (ret) {
906 goto fail;
907 }
908 if (flags & BDRV_O_NOCACHE) {
909 if (!s->write_cache_supported) {
910 error_setg(errp,
911 "NVMe controller doesn't support write cache configuration");
912 ret = -EINVAL;
913 } else {
914 ret = nvme_enable_disable_write_cache(bs, !(flags & BDRV_O_NOCACHE),
915 errp);
916 }
917 if (ret) {
918 goto fail;
919 }
920 }
921 return 0;
922 fail:
923 nvme_close(bs);
924 return ret;
925 }
926
927 static int64_t nvme_getlength(BlockDriverState *bs)
928 {
929 BDRVNVMeState *s = bs->opaque;
930 return s->nsze << s->blkshift;
931 }
932
933 static uint32_t nvme_get_blocksize(BlockDriverState *bs)
934 {
935 BDRVNVMeState *s = bs->opaque;
936 assert(s->blkshift >= BDRV_SECTOR_BITS && s->blkshift <= 12);
937 return UINT32_C(1) << s->blkshift;
938 }
939
940 static int nvme_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz)
941 {
942 uint32_t blocksize = nvme_get_blocksize(bs);
943 bsz->phys = blocksize;
944 bsz->log = blocksize;
945 return 0;
946 }
947
948 /* Called with s->dma_map_lock */
949 static coroutine_fn int nvme_cmd_unmap_qiov(BlockDriverState *bs,
950 QEMUIOVector *qiov)
951 {
952 int r = 0;
953 BDRVNVMeState *s = bs->opaque;
954
955 s->dma_map_count -= qiov->size;
956 if (!s->dma_map_count && !qemu_co_queue_empty(&s->dma_flush_queue)) {
957 r = qemu_vfio_dma_reset_temporary(s->vfio);
958 if (!r) {
959 qemu_co_queue_restart_all(&s->dma_flush_queue);
960 }
961 }
962 return r;
963 }
964
965 /* Called with s->dma_map_lock */
966 static coroutine_fn int nvme_cmd_map_qiov(BlockDriverState *bs, NvmeCmd *cmd,
967 NVMeRequest *req, QEMUIOVector *qiov)
968 {
969 BDRVNVMeState *s = bs->opaque;
970 uint64_t *pagelist = req->prp_list_page;
971 int i, j, r;
972 int entries = 0;
973
974 assert(qiov->size);
975 assert(QEMU_IS_ALIGNED(qiov->size, s->page_size));
976 assert(qiov->size / s->page_size <= s->page_size / sizeof(uint64_t));
977 for (i = 0; i < qiov->niov; ++i) {
978 bool retry = true;
979 uint64_t iova;
980 try_map:
981 r = qemu_vfio_dma_map(s->vfio,
982 qiov->iov[i].iov_base,
983 qiov->iov[i].iov_len,
984 true, &iova);
985 if (r == -ENOMEM && retry) {
986 retry = false;
987 trace_nvme_dma_flush_queue_wait(s);
988 if (s->dma_map_count) {
989 trace_nvme_dma_map_flush(s);
990 qemu_co_queue_wait(&s->dma_flush_queue, &s->dma_map_lock);
991 } else {
992 r = qemu_vfio_dma_reset_temporary(s->vfio);
993 if (r) {
994 goto fail;
995 }
996 }
997 goto try_map;
998 }
999 if (r) {
1000 goto fail;
1001 }
1002
1003 for (j = 0; j < qiov->iov[i].iov_len / s->page_size; j++) {
1004 pagelist[entries++] = cpu_to_le64(iova + j * s->page_size);
1005 }
1006 trace_nvme_cmd_map_qiov_iov(s, i, qiov->iov[i].iov_base,
1007 qiov->iov[i].iov_len / s->page_size);
1008 }
1009
1010 s->dma_map_count += qiov->size;
1011
1012 assert(entries <= s->page_size / sizeof(uint64_t));
1013 switch (entries) {
1014 case 0:
1015 abort();
1016 case 1:
1017 cmd->dptr.prp1 = pagelist[0];
1018 cmd->dptr.prp2 = 0;
1019 break;
1020 case 2:
1021 cmd->dptr.prp1 = pagelist[0];
1022 cmd->dptr.prp2 = pagelist[1];
1023 break;
1024 default:
1025 cmd->dptr.prp1 = pagelist[0];
1026 cmd->dptr.prp2 = cpu_to_le64(req->prp_list_iova + sizeof(uint64_t));
1027 break;
1028 }
1029 trace_nvme_cmd_map_qiov(s, cmd, req, qiov, entries);
1030 for (i = 0; i < entries; ++i) {
1031 trace_nvme_cmd_map_qiov_pages(s, i, pagelist[i]);
1032 }
1033 return 0;
1034 fail:
1035 /* No need to unmap [0 - i) iovs even if we've failed, since we don't
1036 * increment s->dma_map_count. This is okay for fixed mapping memory areas
1037 * because they are already mapped before calling this function; for
1038 * temporary mappings, a later nvme_cmd_(un)map_qiov will reclaim by
1039 * calling qemu_vfio_dma_reset_temporary when necessary. */
1040 return r;
1041 }
1042
1043 typedef struct {
1044 Coroutine *co;
1045 int ret;
1046 AioContext *ctx;
1047 } NVMeCoData;
1048
1049 static void nvme_rw_cb_bh(void *opaque)
1050 {
1051 NVMeCoData *data = opaque;
1052 qemu_coroutine_enter(data->co);
1053 }
1054
1055 static void nvme_rw_cb(void *opaque, int ret)
1056 {
1057 NVMeCoData *data = opaque;
1058 data->ret = ret;
1059 if (!data->co) {
1060 /* The rw coroutine hasn't yielded, don't try to enter. */
1061 return;
1062 }
1063 replay_bh_schedule_oneshot_event(data->ctx, nvme_rw_cb_bh, data);
1064 }
1065
1066 static coroutine_fn int nvme_co_prw_aligned(BlockDriverState *bs,
1067 uint64_t offset, uint64_t bytes,
1068 QEMUIOVector *qiov,
1069 bool is_write,
1070 int flags)
1071 {
1072 int r;
1073 BDRVNVMeState *s = bs->opaque;
1074 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1075 NVMeRequest *req;
1076
1077 uint32_t cdw12 = (((bytes >> s->blkshift) - 1) & 0xFFFF) |
1078 (flags & BDRV_REQ_FUA ? 1 << 30 : 0);
1079 NvmeCmd cmd = {
1080 .opcode = is_write ? NVME_CMD_WRITE : NVME_CMD_READ,
1081 .nsid = cpu_to_le32(s->nsid),
1082 .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
1083 .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
1084 .cdw12 = cpu_to_le32(cdw12),
1085 };
1086 NVMeCoData data = {
1087 .ctx = bdrv_get_aio_context(bs),
1088 .ret = -EINPROGRESS,
1089 };
1090
1091 trace_nvme_prw_aligned(s, is_write, offset, bytes, flags, qiov->niov);
1092 assert(s->nr_queues > 1);
1093 req = nvme_get_free_req(ioq);
1094 assert(req);
1095
1096 qemu_co_mutex_lock(&s->dma_map_lock);
1097 r = nvme_cmd_map_qiov(bs, &cmd, req, qiov);
1098 qemu_co_mutex_unlock(&s->dma_map_lock);
1099 if (r) {
1100 nvme_put_free_req_and_wake(ioq, req);
1101 return r;
1102 }
1103 nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1104
1105 data.co = qemu_coroutine_self();
1106 while (data.ret == -EINPROGRESS) {
1107 qemu_coroutine_yield();
1108 }
1109
1110 qemu_co_mutex_lock(&s->dma_map_lock);
1111 r = nvme_cmd_unmap_qiov(bs, qiov);
1112 qemu_co_mutex_unlock(&s->dma_map_lock);
1113 if (r) {
1114 return r;
1115 }
1116
1117 trace_nvme_rw_done(s, is_write, offset, bytes, data.ret);
1118 return data.ret;
1119 }
1120
1121 static inline bool nvme_qiov_aligned(BlockDriverState *bs,
1122 const QEMUIOVector *qiov)
1123 {
1124 int i;
1125 BDRVNVMeState *s = bs->opaque;
1126
1127 for (i = 0; i < qiov->niov; ++i) {
1128 if (!QEMU_PTR_IS_ALIGNED(qiov->iov[i].iov_base, s->page_size) ||
1129 !QEMU_IS_ALIGNED(qiov->iov[i].iov_len, s->page_size)) {
1130 trace_nvme_qiov_unaligned(qiov, i, qiov->iov[i].iov_base,
1131 qiov->iov[i].iov_len, s->page_size);
1132 return false;
1133 }
1134 }
1135 return true;
1136 }
1137
1138 static int nvme_co_prw(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
1139 QEMUIOVector *qiov, bool is_write, int flags)
1140 {
1141 BDRVNVMeState *s = bs->opaque;
1142 int r;
1143 uint8_t *buf = NULL;
1144 QEMUIOVector local_qiov;
1145
1146 assert(QEMU_IS_ALIGNED(offset, s->page_size));
1147 assert(QEMU_IS_ALIGNED(bytes, s->page_size));
1148 assert(bytes <= s->max_transfer);
1149 if (nvme_qiov_aligned(bs, qiov)) {
1150 return nvme_co_prw_aligned(bs, offset, bytes, qiov, is_write, flags);
1151 }
1152 trace_nvme_prw_buffered(s, offset, bytes, qiov->niov, is_write);
1153 buf = qemu_try_memalign(s->page_size, bytes);
1154
1155 if (!buf) {
1156 return -ENOMEM;
1157 }
1158 qemu_iovec_init(&local_qiov, 1);
1159 if (is_write) {
1160 qemu_iovec_to_buf(qiov, 0, buf, bytes);
1161 }
1162 qemu_iovec_add(&local_qiov, buf, bytes);
1163 r = nvme_co_prw_aligned(bs, offset, bytes, &local_qiov, is_write, flags);
1164 qemu_iovec_destroy(&local_qiov);
1165 if (!r && !is_write) {
1166 qemu_iovec_from_buf(qiov, 0, buf, bytes);
1167 }
1168 qemu_vfree(buf);
1169 return r;
1170 }
1171
1172 static coroutine_fn int nvme_co_preadv(BlockDriverState *bs,
1173 uint64_t offset, uint64_t bytes,
1174 QEMUIOVector *qiov, int flags)
1175 {
1176 return nvme_co_prw(bs, offset, bytes, qiov, false, flags);
1177 }
1178
1179 static coroutine_fn int nvme_co_pwritev(BlockDriverState *bs,
1180 uint64_t offset, uint64_t bytes,
1181 QEMUIOVector *qiov, int flags)
1182 {
1183 return nvme_co_prw(bs, offset, bytes, qiov, true, flags);
1184 }
1185
1186 static coroutine_fn int nvme_co_flush(BlockDriverState *bs)
1187 {
1188 BDRVNVMeState *s = bs->opaque;
1189 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1190 NVMeRequest *req;
1191 NvmeCmd cmd = {
1192 .opcode = NVME_CMD_FLUSH,
1193 .nsid = cpu_to_le32(s->nsid),
1194 };
1195 NVMeCoData data = {
1196 .ctx = bdrv_get_aio_context(bs),
1197 .ret = -EINPROGRESS,
1198 };
1199
1200 assert(s->nr_queues > 1);
1201 req = nvme_get_free_req(ioq);
1202 assert(req);
1203 nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1204
1205 data.co = qemu_coroutine_self();
1206 if (data.ret == -EINPROGRESS) {
1207 qemu_coroutine_yield();
1208 }
1209
1210 return data.ret;
1211 }
1212
1213
1214 static coroutine_fn int nvme_co_pwrite_zeroes(BlockDriverState *bs,
1215 int64_t offset,
1216 int bytes,
1217 BdrvRequestFlags flags)
1218 {
1219 BDRVNVMeState *s = bs->opaque;
1220 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1221 NVMeRequest *req;
1222
1223 uint32_t cdw12 = ((bytes >> s->blkshift) - 1) & 0xFFFF;
1224
1225 if (!s->supports_write_zeroes) {
1226 return -ENOTSUP;
1227 }
1228
1229 NvmeCmd cmd = {
1230 .opcode = NVME_CMD_WRITE_ZEROES,
1231 .nsid = cpu_to_le32(s->nsid),
1232 .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
1233 .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
1234 };
1235
1236 NVMeCoData data = {
1237 .ctx = bdrv_get_aio_context(bs),
1238 .ret = -EINPROGRESS,
1239 };
1240
1241 if (flags & BDRV_REQ_MAY_UNMAP) {
1242 cdw12 |= (1 << 25);
1243 }
1244
1245 if (flags & BDRV_REQ_FUA) {
1246 cdw12 |= (1 << 30);
1247 }
1248
1249 cmd.cdw12 = cpu_to_le32(cdw12);
1250
1251 trace_nvme_write_zeroes(s, offset, bytes, flags);
1252 assert(s->nr_queues > 1);
1253 req = nvme_get_free_req(ioq);
1254 assert(req);
1255
1256 nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1257
1258 data.co = qemu_coroutine_self();
1259 while (data.ret == -EINPROGRESS) {
1260 qemu_coroutine_yield();
1261 }
1262
1263 trace_nvme_rw_done(s, true, offset, bytes, data.ret);
1264 return data.ret;
1265 }
1266
1267
1268 static int coroutine_fn nvme_co_pdiscard(BlockDriverState *bs,
1269 int64_t offset,
1270 int bytes)
1271 {
1272 BDRVNVMeState *s = bs->opaque;
1273 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1274 NVMeRequest *req;
1275 NvmeDsmRange *buf;
1276 QEMUIOVector local_qiov;
1277 int ret;
1278
1279 NvmeCmd cmd = {
1280 .opcode = NVME_CMD_DSM,
1281 .nsid = cpu_to_le32(s->nsid),
1282 .cdw10 = cpu_to_le32(0), /*number of ranges - 0 based*/
1283 .cdw11 = cpu_to_le32(1 << 2), /*deallocate bit*/
1284 };
1285
1286 NVMeCoData data = {
1287 .ctx = bdrv_get_aio_context(bs),
1288 .ret = -EINPROGRESS,
1289 };
1290
1291 if (!s->supports_discard) {
1292 return -ENOTSUP;
1293 }
1294
1295 assert(s->nr_queues > 1);
1296
1297 buf = qemu_try_memalign(s->page_size, s->page_size);
1298 if (!buf) {
1299 return -ENOMEM;
1300 }
1301 memset(buf, 0, s->page_size);
1302 buf->nlb = cpu_to_le32(bytes >> s->blkshift);
1303 buf->slba = cpu_to_le64(offset >> s->blkshift);
1304 buf->cattr = 0;
1305
1306 qemu_iovec_init(&local_qiov, 1);
1307 qemu_iovec_add(&local_qiov, buf, 4096);
1308
1309 req = nvme_get_free_req(ioq);
1310 assert(req);
1311
1312 qemu_co_mutex_lock(&s->dma_map_lock);
1313 ret = nvme_cmd_map_qiov(bs, &cmd, req, &local_qiov);
1314 qemu_co_mutex_unlock(&s->dma_map_lock);
1315
1316 if (ret) {
1317 nvme_put_free_req_and_wake(ioq, req);
1318 goto out;
1319 }
1320
1321 trace_nvme_dsm(s, offset, bytes);
1322
1323 nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1324
1325 data.co = qemu_coroutine_self();
1326 while (data.ret == -EINPROGRESS) {
1327 qemu_coroutine_yield();
1328 }
1329
1330 qemu_co_mutex_lock(&s->dma_map_lock);
1331 ret = nvme_cmd_unmap_qiov(bs, &local_qiov);
1332 qemu_co_mutex_unlock(&s->dma_map_lock);
1333
1334 if (ret) {
1335 goto out;
1336 }
1337
1338 ret = data.ret;
1339 trace_nvme_dsm_done(s, offset, bytes, ret);
1340 out:
1341 qemu_iovec_destroy(&local_qiov);
1342 qemu_vfree(buf);
1343 return ret;
1344
1345 }
1346
1347
1348 static int nvme_reopen_prepare(BDRVReopenState *reopen_state,
1349 BlockReopenQueue *queue, Error **errp)
1350 {
1351 return 0;
1352 }
1353
1354 static void nvme_refresh_filename(BlockDriverState *bs)
1355 {
1356 BDRVNVMeState *s = bs->opaque;
1357
1358 snprintf(bs->exact_filename, sizeof(bs->exact_filename), "nvme://%s/%i",
1359 s->device, s->nsid);
1360 }
1361
1362 static void nvme_refresh_limits(BlockDriverState *bs, Error **errp)
1363 {
1364 BDRVNVMeState *s = bs->opaque;
1365
1366 bs->bl.opt_mem_alignment = s->page_size;
1367 bs->bl.request_alignment = s->page_size;
1368 bs->bl.max_transfer = s->max_transfer;
1369 }
1370
1371 static void nvme_detach_aio_context(BlockDriverState *bs)
1372 {
1373 BDRVNVMeState *s = bs->opaque;
1374
1375 for (int i = 0; i < s->nr_queues; i++) {
1376 NVMeQueuePair *q = s->queues[i];
1377
1378 qemu_bh_delete(q->completion_bh);
1379 q->completion_bh = NULL;
1380 }
1381
1382 aio_set_event_notifier(bdrv_get_aio_context(bs),
1383 &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
1384 false, NULL, NULL);
1385 }
1386
1387 static void nvme_attach_aio_context(BlockDriverState *bs,
1388 AioContext *new_context)
1389 {
1390 BDRVNVMeState *s = bs->opaque;
1391
1392 s->aio_context = new_context;
1393 aio_set_event_notifier(new_context, &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
1394 false, nvme_handle_event, nvme_poll_cb);
1395
1396 for (int i = 0; i < s->nr_queues; i++) {
1397 NVMeQueuePair *q = s->queues[i];
1398
1399 q->completion_bh =
1400 aio_bh_new(new_context, nvme_process_completion_bh, q);
1401 }
1402 }
1403
1404 static void nvme_aio_plug(BlockDriverState *bs)
1405 {
1406 BDRVNVMeState *s = bs->opaque;
1407 assert(!s->plugged);
1408 s->plugged = true;
1409 }
1410
1411 static void nvme_aio_unplug(BlockDriverState *bs)
1412 {
1413 int i;
1414 BDRVNVMeState *s = bs->opaque;
1415 assert(s->plugged);
1416 s->plugged = false;
1417 for (i = INDEX_IO(0); i < s->nr_queues; i++) {
1418 NVMeQueuePair *q = s->queues[i];
1419 qemu_mutex_lock(&q->lock);
1420 nvme_kick(q);
1421 nvme_process_completion(q);
1422 qemu_mutex_unlock(&q->lock);
1423 }
1424 }
1425
1426 static void nvme_register_buf(BlockDriverState *bs, void *host, size_t size)
1427 {
1428 int ret;
1429 BDRVNVMeState *s = bs->opaque;
1430
1431 ret = qemu_vfio_dma_map(s->vfio, host, size, false, NULL);
1432 if (ret) {
1433 /* FIXME: we may run out of IOVA addresses after repeated
1434 * bdrv_register_buf/bdrv_unregister_buf, because nvme_vfio_dma_unmap
1435 * doesn't reclaim addresses for fixed mappings. */
1436 error_report("nvme_register_buf failed: %s", strerror(-ret));
1437 }
1438 }
1439
1440 static void nvme_unregister_buf(BlockDriverState *bs, void *host)
1441 {
1442 BDRVNVMeState *s = bs->opaque;
1443
1444 qemu_vfio_dma_unmap(s->vfio, host);
1445 }
1446
1447 static const char *const nvme_strong_runtime_opts[] = {
1448 NVME_BLOCK_OPT_DEVICE,
1449 NVME_BLOCK_OPT_NAMESPACE,
1450
1451 NULL
1452 };
1453
1454 static BlockDriver bdrv_nvme = {
1455 .format_name = "nvme",
1456 .protocol_name = "nvme",
1457 .instance_size = sizeof(BDRVNVMeState),
1458
1459 .bdrv_co_create_opts = bdrv_co_create_opts_simple,
1460 .create_opts = &bdrv_create_opts_simple,
1461
1462 .bdrv_parse_filename = nvme_parse_filename,
1463 .bdrv_file_open = nvme_file_open,
1464 .bdrv_close = nvme_close,
1465 .bdrv_getlength = nvme_getlength,
1466 .bdrv_probe_blocksizes = nvme_probe_blocksizes,
1467
1468 .bdrv_co_preadv = nvme_co_preadv,
1469 .bdrv_co_pwritev = nvme_co_pwritev,
1470
1471 .bdrv_co_pwrite_zeroes = nvme_co_pwrite_zeroes,
1472 .bdrv_co_pdiscard = nvme_co_pdiscard,
1473
1474 .bdrv_co_flush_to_disk = nvme_co_flush,
1475 .bdrv_reopen_prepare = nvme_reopen_prepare,
1476
1477 .bdrv_refresh_filename = nvme_refresh_filename,
1478 .bdrv_refresh_limits = nvme_refresh_limits,
1479 .strong_runtime_opts = nvme_strong_runtime_opts,
1480
1481 .bdrv_detach_aio_context = nvme_detach_aio_context,
1482 .bdrv_attach_aio_context = nvme_attach_aio_context,
1483
1484 .bdrv_io_plug = nvme_aio_plug,
1485 .bdrv_io_unplug = nvme_aio_unplug,
1486
1487 .bdrv_register_buf = nvme_register_buf,
1488 .bdrv_unregister_buf = nvme_unregister_buf,
1489 };
1490
1491 static void bdrv_nvme_init(void)
1492 {
1493 bdrv_register(&bdrv_nvme);
1494 }
1495
1496 block_init(bdrv_nvme_init);