block/nvme: Drop NVMeRegs structure, directly use NvmeBar
[qemu.git] / block / nvme.c
1 /*
2 * NVMe block driver based on vfio
3 *
4 * Copyright 2016 - 2018 Red Hat, Inc.
5 *
6 * Authors:
7 * Fam Zheng <famz@redhat.com>
8 * Paolo Bonzini <pbonzini@redhat.com>
9 *
10 * This work is licensed under the terms of the GNU GPL, version 2 or later.
11 * See the COPYING file in the top-level directory.
12 */
13
14 #include "qemu/osdep.h"
15 #include <linux/vfio.h>
16 #include "qapi/error.h"
17 #include "qapi/qmp/qdict.h"
18 #include "qapi/qmp/qstring.h"
19 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qemu/module.h"
22 #include "qemu/cutils.h"
23 #include "qemu/option.h"
24 #include "qemu/vfio-helpers.h"
25 #include "block/block_int.h"
26 #include "sysemu/replay.h"
27 #include "trace.h"
28
29 #include "block/nvme.h"
30
31 #define NVME_SQ_ENTRY_BYTES 64
32 #define NVME_CQ_ENTRY_BYTES 16
33 #define NVME_QUEUE_SIZE 128
34 #define NVME_DOORBELL_SIZE 4096
35
36 /*
37 * We have to leave one slot empty as that is the full queue case where
38 * head == tail + 1.
39 */
40 #define NVME_NUM_REQS (NVME_QUEUE_SIZE - 1)
41
42 typedef struct BDRVNVMeState BDRVNVMeState;
43
44 typedef struct {
45 int32_t head, tail;
46 uint8_t *queue;
47 uint64_t iova;
48 /* Hardware MMIO register */
49 volatile uint32_t *doorbell;
50 } NVMeQueue;
51
52 typedef struct {
53 BlockCompletionFunc *cb;
54 void *opaque;
55 int cid;
56 void *prp_list_page;
57 uint64_t prp_list_iova;
58 int free_req_next; /* q->reqs[] index of next free req */
59 } NVMeRequest;
60
61 typedef struct {
62 QemuMutex lock;
63
64 /* Read from I/O code path, initialized under BQL */
65 BDRVNVMeState *s;
66 int index;
67
68 /* Fields protected by BQL */
69 uint8_t *prp_list_pages;
70
71 /* Fields protected by @lock */
72 CoQueue free_req_queue;
73 NVMeQueue sq, cq;
74 int cq_phase;
75 int free_req_head;
76 NVMeRequest reqs[NVME_NUM_REQS];
77 int need_kick;
78 int inflight;
79
80 /* Thread-safe, no lock necessary */
81 QEMUBH *completion_bh;
82 } NVMeQueuePair;
83
84 #define INDEX_ADMIN 0
85 #define INDEX_IO(n) (1 + n)
86
87 /* This driver shares a single MSIX IRQ for the admin and I/O queues */
88 enum {
89 MSIX_SHARED_IRQ_IDX = 0,
90 MSIX_IRQ_COUNT = 1
91 };
92
93 struct BDRVNVMeState {
94 AioContext *aio_context;
95 QEMUVFIOState *vfio;
96 /* Memory mapped registers */
97 volatile struct {
98 uint32_t sq_tail;
99 uint32_t cq_head;
100 } *doorbells;
101 /* The submission/completion queue pairs.
102 * [0]: admin queue.
103 * [1..]: io queues.
104 */
105 NVMeQueuePair **queues;
106 int nr_queues;
107 size_t page_size;
108 /* How many uint32_t elements does each doorbell entry take. */
109 size_t doorbell_scale;
110 bool write_cache_supported;
111 EventNotifier irq_notifier[MSIX_IRQ_COUNT];
112
113 uint64_t nsze; /* Namespace size reported by identify command */
114 int nsid; /* The namespace id to read/write data. */
115 int blkshift;
116
117 uint64_t max_transfer;
118 bool plugged;
119
120 bool supports_write_zeroes;
121 bool supports_discard;
122
123 CoMutex dma_map_lock;
124 CoQueue dma_flush_queue;
125
126 /* Total size of mapped qiov, accessed under dma_map_lock */
127 int dma_map_count;
128
129 /* PCI address (required for nvme_refresh_filename()) */
130 char *device;
131 };
132
133 #define NVME_BLOCK_OPT_DEVICE "device"
134 #define NVME_BLOCK_OPT_NAMESPACE "namespace"
135
136 static void nvme_process_completion_bh(void *opaque);
137
138 static QemuOptsList runtime_opts = {
139 .name = "nvme",
140 .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
141 .desc = {
142 {
143 .name = NVME_BLOCK_OPT_DEVICE,
144 .type = QEMU_OPT_STRING,
145 .help = "NVMe PCI device address",
146 },
147 {
148 .name = NVME_BLOCK_OPT_NAMESPACE,
149 .type = QEMU_OPT_NUMBER,
150 .help = "NVMe namespace",
151 },
152 { /* end of list */ }
153 },
154 };
155
156 static void nvme_init_queue(BDRVNVMeState *s, NVMeQueue *q,
157 int nentries, int entry_bytes, Error **errp)
158 {
159 size_t bytes;
160 int r;
161
162 bytes = ROUND_UP(nentries * entry_bytes, s->page_size);
163 q->head = q->tail = 0;
164 q->queue = qemu_try_memalign(s->page_size, bytes);
165 if (!q->queue) {
166 error_setg(errp, "Cannot allocate queue");
167 return;
168 }
169 memset(q->queue, 0, bytes);
170 r = qemu_vfio_dma_map(s->vfio, q->queue, bytes, false, &q->iova);
171 if (r) {
172 error_setg(errp, "Cannot map queue");
173 }
174 }
175
176 static void nvme_free_queue_pair(NVMeQueuePair *q)
177 {
178 if (q->completion_bh) {
179 qemu_bh_delete(q->completion_bh);
180 }
181 qemu_vfree(q->prp_list_pages);
182 qemu_vfree(q->sq.queue);
183 qemu_vfree(q->cq.queue);
184 qemu_mutex_destroy(&q->lock);
185 g_free(q);
186 }
187
188 static void nvme_free_req_queue_cb(void *opaque)
189 {
190 NVMeQueuePair *q = opaque;
191
192 qemu_mutex_lock(&q->lock);
193 while (qemu_co_enter_next(&q->free_req_queue, &q->lock)) {
194 /* Retry all pending requests */
195 }
196 qemu_mutex_unlock(&q->lock);
197 }
198
199 static NVMeQueuePair *nvme_create_queue_pair(BDRVNVMeState *s,
200 AioContext *aio_context,
201 int idx, int size,
202 Error **errp)
203 {
204 int i, r;
205 Error *local_err = NULL;
206 NVMeQueuePair *q;
207 uint64_t prp_list_iova;
208
209 q = g_try_new0(NVMeQueuePair, 1);
210 if (!q) {
211 return NULL;
212 }
213 q->prp_list_pages = qemu_try_memalign(s->page_size,
214 s->page_size * NVME_NUM_REQS);
215 if (!q->prp_list_pages) {
216 goto fail;
217 }
218 memset(q->prp_list_pages, 0, s->page_size * NVME_NUM_REQS);
219 qemu_mutex_init(&q->lock);
220 q->s = s;
221 q->index = idx;
222 qemu_co_queue_init(&q->free_req_queue);
223 q->completion_bh = aio_bh_new(aio_context, nvme_process_completion_bh, q);
224 r = qemu_vfio_dma_map(s->vfio, q->prp_list_pages,
225 s->page_size * NVME_NUM_REQS,
226 false, &prp_list_iova);
227 if (r) {
228 goto fail;
229 }
230 q->free_req_head = -1;
231 for (i = 0; i < NVME_NUM_REQS; i++) {
232 NVMeRequest *req = &q->reqs[i];
233 req->cid = i + 1;
234 req->free_req_next = q->free_req_head;
235 q->free_req_head = i;
236 req->prp_list_page = q->prp_list_pages + i * s->page_size;
237 req->prp_list_iova = prp_list_iova + i * s->page_size;
238 }
239
240 nvme_init_queue(s, &q->sq, size, NVME_SQ_ENTRY_BYTES, &local_err);
241 if (local_err) {
242 error_propagate(errp, local_err);
243 goto fail;
244 }
245 q->sq.doorbell = &s->doorbells[idx * s->doorbell_scale].sq_tail;
246
247 nvme_init_queue(s, &q->cq, size, NVME_CQ_ENTRY_BYTES, &local_err);
248 if (local_err) {
249 error_propagate(errp, local_err);
250 goto fail;
251 }
252 q->cq.doorbell = &s->doorbells[idx * s->doorbell_scale].cq_head;
253
254 return q;
255 fail:
256 nvme_free_queue_pair(q);
257 return NULL;
258 }
259
260 /* With q->lock */
261 static void nvme_kick(NVMeQueuePair *q)
262 {
263 BDRVNVMeState *s = q->s;
264
265 if (s->plugged || !q->need_kick) {
266 return;
267 }
268 trace_nvme_kick(s, q->index);
269 assert(!(q->sq.tail & 0xFF00));
270 /* Fence the write to submission queue entry before notifying the device. */
271 smp_wmb();
272 *q->sq.doorbell = cpu_to_le32(q->sq.tail);
273 q->inflight += q->need_kick;
274 q->need_kick = 0;
275 }
276
277 /* Find a free request element if any, otherwise:
278 * a) if in coroutine context, try to wait for one to become available;
279 * b) if not in coroutine, return NULL;
280 */
281 static NVMeRequest *nvme_get_free_req(NVMeQueuePair *q)
282 {
283 NVMeRequest *req;
284
285 qemu_mutex_lock(&q->lock);
286
287 while (q->free_req_head == -1) {
288 if (qemu_in_coroutine()) {
289 trace_nvme_free_req_queue_wait(q);
290 qemu_co_queue_wait(&q->free_req_queue, &q->lock);
291 } else {
292 qemu_mutex_unlock(&q->lock);
293 return NULL;
294 }
295 }
296
297 req = &q->reqs[q->free_req_head];
298 q->free_req_head = req->free_req_next;
299 req->free_req_next = -1;
300
301 qemu_mutex_unlock(&q->lock);
302 return req;
303 }
304
305 /* With q->lock */
306 static void nvme_put_free_req_locked(NVMeQueuePair *q, NVMeRequest *req)
307 {
308 req->free_req_next = q->free_req_head;
309 q->free_req_head = req - q->reqs;
310 }
311
312 /* With q->lock */
313 static void nvme_wake_free_req_locked(NVMeQueuePair *q)
314 {
315 if (!qemu_co_queue_empty(&q->free_req_queue)) {
316 replay_bh_schedule_oneshot_event(q->s->aio_context,
317 nvme_free_req_queue_cb, q);
318 }
319 }
320
321 /* Insert a request in the freelist and wake waiters */
322 static void nvme_put_free_req_and_wake(NVMeQueuePair *q, NVMeRequest *req)
323 {
324 qemu_mutex_lock(&q->lock);
325 nvme_put_free_req_locked(q, req);
326 nvme_wake_free_req_locked(q);
327 qemu_mutex_unlock(&q->lock);
328 }
329
330 static inline int nvme_translate_error(const NvmeCqe *c)
331 {
332 uint16_t status = (le16_to_cpu(c->status) >> 1) & 0xFF;
333 if (status) {
334 trace_nvme_error(le32_to_cpu(c->result),
335 le16_to_cpu(c->sq_head),
336 le16_to_cpu(c->sq_id),
337 le16_to_cpu(c->cid),
338 le16_to_cpu(status));
339 }
340 switch (status) {
341 case 0:
342 return 0;
343 case 1:
344 return -ENOSYS;
345 case 2:
346 return -EINVAL;
347 default:
348 return -EIO;
349 }
350 }
351
352 /* With q->lock */
353 static bool nvme_process_completion(NVMeQueuePair *q)
354 {
355 BDRVNVMeState *s = q->s;
356 bool progress = false;
357 NVMeRequest *preq;
358 NVMeRequest req;
359 NvmeCqe *c;
360
361 trace_nvme_process_completion(s, q->index, q->inflight);
362 if (s->plugged) {
363 trace_nvme_process_completion_queue_plugged(s, q->index);
364 return false;
365 }
366
367 /*
368 * Support re-entrancy when a request cb() function invokes aio_poll().
369 * Pending completions must be visible to aio_poll() so that a cb()
370 * function can wait for the completion of another request.
371 *
372 * The aio_poll() loop will execute our BH and we'll resume completion
373 * processing there.
374 */
375 qemu_bh_schedule(q->completion_bh);
376
377 assert(q->inflight >= 0);
378 while (q->inflight) {
379 int ret;
380 int16_t cid;
381
382 c = (NvmeCqe *)&q->cq.queue[q->cq.head * NVME_CQ_ENTRY_BYTES];
383 if ((le16_to_cpu(c->status) & 0x1) == q->cq_phase) {
384 break;
385 }
386 ret = nvme_translate_error(c);
387 q->cq.head = (q->cq.head + 1) % NVME_QUEUE_SIZE;
388 if (!q->cq.head) {
389 q->cq_phase = !q->cq_phase;
390 }
391 cid = le16_to_cpu(c->cid);
392 if (cid == 0 || cid > NVME_QUEUE_SIZE) {
393 fprintf(stderr, "Unexpected CID in completion queue: %" PRIu32 "\n",
394 cid);
395 continue;
396 }
397 trace_nvme_complete_command(s, q->index, cid);
398 preq = &q->reqs[cid - 1];
399 req = *preq;
400 assert(req.cid == cid);
401 assert(req.cb);
402 nvme_put_free_req_locked(q, preq);
403 preq->cb = preq->opaque = NULL;
404 q->inflight--;
405 qemu_mutex_unlock(&q->lock);
406 req.cb(req.opaque, ret);
407 qemu_mutex_lock(&q->lock);
408 progress = true;
409 }
410 if (progress) {
411 /* Notify the device so it can post more completions. */
412 smp_mb_release();
413 *q->cq.doorbell = cpu_to_le32(q->cq.head);
414 nvme_wake_free_req_locked(q);
415 }
416
417 qemu_bh_cancel(q->completion_bh);
418
419 return progress;
420 }
421
422 static void nvme_process_completion_bh(void *opaque)
423 {
424 NVMeQueuePair *q = opaque;
425
426 /*
427 * We're being invoked because a nvme_process_completion() cb() function
428 * called aio_poll(). The callback may be waiting for further completions
429 * so notify the device that it has space to fill in more completions now.
430 */
431 smp_mb_release();
432 *q->cq.doorbell = cpu_to_le32(q->cq.head);
433 nvme_wake_free_req_locked(q);
434
435 nvme_process_completion(q);
436 }
437
438 static void nvme_trace_command(const NvmeCmd *cmd)
439 {
440 int i;
441
442 if (!trace_event_get_state_backends(TRACE_NVME_SUBMIT_COMMAND_RAW)) {
443 return;
444 }
445 for (i = 0; i < 8; ++i) {
446 uint8_t *cmdp = (uint8_t *)cmd + i * 8;
447 trace_nvme_submit_command_raw(cmdp[0], cmdp[1], cmdp[2], cmdp[3],
448 cmdp[4], cmdp[5], cmdp[6], cmdp[7]);
449 }
450 }
451
452 static void nvme_submit_command(NVMeQueuePair *q, NVMeRequest *req,
453 NvmeCmd *cmd, BlockCompletionFunc cb,
454 void *opaque)
455 {
456 assert(!req->cb);
457 req->cb = cb;
458 req->opaque = opaque;
459 cmd->cid = cpu_to_le32(req->cid);
460
461 trace_nvme_submit_command(q->s, q->index, req->cid);
462 nvme_trace_command(cmd);
463 qemu_mutex_lock(&q->lock);
464 memcpy((uint8_t *)q->sq.queue +
465 q->sq.tail * NVME_SQ_ENTRY_BYTES, cmd, sizeof(*cmd));
466 q->sq.tail = (q->sq.tail + 1) % NVME_QUEUE_SIZE;
467 q->need_kick++;
468 nvme_kick(q);
469 nvme_process_completion(q);
470 qemu_mutex_unlock(&q->lock);
471 }
472
473 static void nvme_cmd_sync_cb(void *opaque, int ret)
474 {
475 int *pret = opaque;
476 *pret = ret;
477 aio_wait_kick();
478 }
479
480 static int nvme_cmd_sync(BlockDriverState *bs, NVMeQueuePair *q,
481 NvmeCmd *cmd)
482 {
483 AioContext *aio_context = bdrv_get_aio_context(bs);
484 NVMeRequest *req;
485 int ret = -EINPROGRESS;
486 req = nvme_get_free_req(q);
487 if (!req) {
488 return -EBUSY;
489 }
490 nvme_submit_command(q, req, cmd, nvme_cmd_sync_cb, &ret);
491
492 AIO_WAIT_WHILE(aio_context, ret == -EINPROGRESS);
493 return ret;
494 }
495
496 static void nvme_identify(BlockDriverState *bs, int namespace, Error **errp)
497 {
498 BDRVNVMeState *s = bs->opaque;
499 union {
500 NvmeIdCtrl ctrl;
501 NvmeIdNs ns;
502 } *id;
503 NvmeLBAF *lbaf;
504 uint16_t oncs;
505 int r;
506 uint64_t iova;
507 NvmeCmd cmd = {
508 .opcode = NVME_ADM_CMD_IDENTIFY,
509 .cdw10 = cpu_to_le32(0x1),
510 };
511
512 id = qemu_try_memalign(s->page_size, sizeof(*id));
513 if (!id) {
514 error_setg(errp, "Cannot allocate buffer for identify response");
515 goto out;
516 }
517 r = qemu_vfio_dma_map(s->vfio, id, sizeof(*id), true, &iova);
518 if (r) {
519 error_setg(errp, "Cannot map buffer for DMA");
520 goto out;
521 }
522
523 memset(id, 0, sizeof(*id));
524 cmd.dptr.prp1 = cpu_to_le64(iova);
525 if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
526 error_setg(errp, "Failed to identify controller");
527 goto out;
528 }
529
530 if (le32_to_cpu(id->ctrl.nn) < namespace) {
531 error_setg(errp, "Invalid namespace");
532 goto out;
533 }
534 s->write_cache_supported = le32_to_cpu(id->ctrl.vwc) & 0x1;
535 s->max_transfer = (id->ctrl.mdts ? 1 << id->ctrl.mdts : 0) * s->page_size;
536 /* For now the page list buffer per command is one page, to hold at most
537 * s->page_size / sizeof(uint64_t) entries. */
538 s->max_transfer = MIN_NON_ZERO(s->max_transfer,
539 s->page_size / sizeof(uint64_t) * s->page_size);
540
541 oncs = le16_to_cpu(id->ctrl.oncs);
542 s->supports_write_zeroes = !!(oncs & NVME_ONCS_WRITE_ZEROES);
543 s->supports_discard = !!(oncs & NVME_ONCS_DSM);
544
545 memset(id, 0, sizeof(*id));
546 cmd.cdw10 = 0;
547 cmd.nsid = cpu_to_le32(namespace);
548 if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
549 error_setg(errp, "Failed to identify namespace");
550 goto out;
551 }
552
553 s->nsze = le64_to_cpu(id->ns.nsze);
554 lbaf = &id->ns.lbaf[NVME_ID_NS_FLBAS_INDEX(id->ns.flbas)];
555
556 if (NVME_ID_NS_DLFEAT_WRITE_ZEROES(id->ns.dlfeat) &&
557 NVME_ID_NS_DLFEAT_READ_BEHAVIOR(id->ns.dlfeat) ==
558 NVME_ID_NS_DLFEAT_READ_BEHAVIOR_ZEROES) {
559 bs->supported_write_flags |= BDRV_REQ_MAY_UNMAP;
560 }
561
562 if (lbaf->ms) {
563 error_setg(errp, "Namespaces with metadata are not yet supported");
564 goto out;
565 }
566
567 if (lbaf->ds < BDRV_SECTOR_BITS || lbaf->ds > 12 ||
568 (1 << lbaf->ds) > s->page_size)
569 {
570 error_setg(errp, "Namespace has unsupported block size (2^%d)",
571 lbaf->ds);
572 goto out;
573 }
574
575 s->blkshift = lbaf->ds;
576 out:
577 qemu_vfio_dma_unmap(s->vfio, id);
578 qemu_vfree(id);
579 }
580
581 static bool nvme_poll_queue(NVMeQueuePair *q)
582 {
583 bool progress = false;
584
585 const size_t cqe_offset = q->cq.head * NVME_CQ_ENTRY_BYTES;
586 NvmeCqe *cqe = (NvmeCqe *)&q->cq.queue[cqe_offset];
587
588 /*
589 * Do an early check for completions. q->lock isn't needed because
590 * nvme_process_completion() only runs in the event loop thread and
591 * cannot race with itself.
592 */
593 if ((le16_to_cpu(cqe->status) & 0x1) == q->cq_phase) {
594 return false;
595 }
596
597 qemu_mutex_lock(&q->lock);
598 while (nvme_process_completion(q)) {
599 /* Keep polling */
600 progress = true;
601 }
602 qemu_mutex_unlock(&q->lock);
603
604 return progress;
605 }
606
607 static bool nvme_poll_queues(BDRVNVMeState *s)
608 {
609 bool progress = false;
610 int i;
611
612 for (i = 0; i < s->nr_queues; i++) {
613 if (nvme_poll_queue(s->queues[i])) {
614 progress = true;
615 }
616 }
617 return progress;
618 }
619
620 static void nvme_handle_event(EventNotifier *n)
621 {
622 BDRVNVMeState *s = container_of(n, BDRVNVMeState,
623 irq_notifier[MSIX_SHARED_IRQ_IDX]);
624
625 trace_nvme_handle_event(s);
626 event_notifier_test_and_clear(n);
627 nvme_poll_queues(s);
628 }
629
630 static bool nvme_add_io_queue(BlockDriverState *bs, Error **errp)
631 {
632 BDRVNVMeState *s = bs->opaque;
633 int n = s->nr_queues;
634 NVMeQueuePair *q;
635 NvmeCmd cmd;
636 int queue_size = NVME_QUEUE_SIZE;
637
638 q = nvme_create_queue_pair(s, bdrv_get_aio_context(bs),
639 n, queue_size, errp);
640 if (!q) {
641 return false;
642 }
643 cmd = (NvmeCmd) {
644 .opcode = NVME_ADM_CMD_CREATE_CQ,
645 .dptr.prp1 = cpu_to_le64(q->cq.iova),
646 .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)),
647 .cdw11 = cpu_to_le32(0x3),
648 };
649 if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
650 error_setg(errp, "Failed to create CQ io queue [%d]", n);
651 goto out_error;
652 }
653 cmd = (NvmeCmd) {
654 .opcode = NVME_ADM_CMD_CREATE_SQ,
655 .dptr.prp1 = cpu_to_le64(q->sq.iova),
656 .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)),
657 .cdw11 = cpu_to_le32(0x1 | (n << 16)),
658 };
659 if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
660 error_setg(errp, "Failed to create SQ io queue [%d]", n);
661 goto out_error;
662 }
663 s->queues = g_renew(NVMeQueuePair *, s->queues, n + 1);
664 s->queues[n] = q;
665 s->nr_queues++;
666 return true;
667 out_error:
668 nvme_free_queue_pair(q);
669 return false;
670 }
671
672 static bool nvme_poll_cb(void *opaque)
673 {
674 EventNotifier *e = opaque;
675 BDRVNVMeState *s = container_of(e, BDRVNVMeState,
676 irq_notifier[MSIX_SHARED_IRQ_IDX]);
677
678 trace_nvme_poll_cb(s);
679 return nvme_poll_queues(s);
680 }
681
682 static int nvme_init(BlockDriverState *bs, const char *device, int namespace,
683 Error **errp)
684 {
685 BDRVNVMeState *s = bs->opaque;
686 AioContext *aio_context = bdrv_get_aio_context(bs);
687 int ret;
688 uint64_t cap;
689 uint64_t timeout_ms;
690 uint64_t deadline, now;
691 Error *local_err = NULL;
692 volatile NvmeBar *regs = NULL;
693
694 qemu_co_mutex_init(&s->dma_map_lock);
695 qemu_co_queue_init(&s->dma_flush_queue);
696 s->device = g_strdup(device);
697 s->nsid = namespace;
698 s->aio_context = bdrv_get_aio_context(bs);
699 ret = event_notifier_init(&s->irq_notifier[MSIX_SHARED_IRQ_IDX], 0);
700 if (ret) {
701 error_setg(errp, "Failed to init event notifier");
702 return ret;
703 }
704
705 s->vfio = qemu_vfio_open_pci(device, errp);
706 if (!s->vfio) {
707 ret = -EINVAL;
708 goto out;
709 }
710
711 regs = qemu_vfio_pci_map_bar(s->vfio, 0, 0, sizeof(NvmeBar),
712 PROT_READ | PROT_WRITE, errp);
713 if (!regs) {
714 ret = -EINVAL;
715 goto out;
716 }
717 /* Perform initialize sequence as described in NVMe spec "7.6.1
718 * Initialization". */
719
720 cap = le64_to_cpu(regs->cap);
721 if (!(cap & (1ULL << 37))) {
722 error_setg(errp, "Device doesn't support NVMe command set");
723 ret = -EINVAL;
724 goto out;
725 }
726
727 s->page_size = MAX(4096, 1 << (12 + ((cap >> 48) & 0xF)));
728 s->doorbell_scale = (4 << (((cap >> 32) & 0xF))) / sizeof(uint32_t);
729 bs->bl.opt_mem_alignment = s->page_size;
730 timeout_ms = MIN(500 * ((cap >> 24) & 0xFF), 30000);
731
732 /* Reset device to get a clean state. */
733 regs->cc = cpu_to_le32(le32_to_cpu(regs->cc) & 0xFE);
734 /* Wait for CSTS.RDY = 0. */
735 deadline = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + timeout_ms * SCALE_MS;
736 while (le32_to_cpu(regs->csts) & 0x1) {
737 if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
738 error_setg(errp, "Timeout while waiting for device to reset (%"
739 PRId64 " ms)",
740 timeout_ms);
741 ret = -ETIMEDOUT;
742 goto out;
743 }
744 }
745
746 s->doorbells = qemu_vfio_pci_map_bar(s->vfio, 0, sizeof(NvmeBar),
747 NVME_DOORBELL_SIZE, PROT_WRITE, errp);
748 if (!s->doorbells) {
749 ret = -EINVAL;
750 goto out;
751 }
752
753 /* Set up admin queue. */
754 s->queues = g_new(NVMeQueuePair *, 1);
755 s->queues[INDEX_ADMIN] = nvme_create_queue_pair(s, aio_context, 0,
756 NVME_QUEUE_SIZE,
757 errp);
758 if (!s->queues[INDEX_ADMIN]) {
759 ret = -EINVAL;
760 goto out;
761 }
762 s->nr_queues = 1;
763 QEMU_BUILD_BUG_ON(NVME_QUEUE_SIZE & 0xF000);
764 regs->aqa = cpu_to_le32((NVME_QUEUE_SIZE << 16) | NVME_QUEUE_SIZE);
765 regs->asq = cpu_to_le64(s->queues[INDEX_ADMIN]->sq.iova);
766 regs->acq = cpu_to_le64(s->queues[INDEX_ADMIN]->cq.iova);
767
768 /* After setting up all control registers we can enable device now. */
769 regs->cc = cpu_to_le32((ctz32(NVME_CQ_ENTRY_BYTES) << 20) |
770 (ctz32(NVME_SQ_ENTRY_BYTES) << 16) |
771 0x1);
772 /* Wait for CSTS.RDY = 1. */
773 now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
774 deadline = now + timeout_ms * 1000000;
775 while (!(le32_to_cpu(regs->csts) & 0x1)) {
776 if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
777 error_setg(errp, "Timeout while waiting for device to start (%"
778 PRId64 " ms)",
779 timeout_ms);
780 ret = -ETIMEDOUT;
781 goto out;
782 }
783 }
784
785 ret = qemu_vfio_pci_init_irq(s->vfio, s->irq_notifier,
786 VFIO_PCI_MSIX_IRQ_INDEX, errp);
787 if (ret) {
788 goto out;
789 }
790 aio_set_event_notifier(bdrv_get_aio_context(bs),
791 &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
792 false, nvme_handle_event, nvme_poll_cb);
793
794 nvme_identify(bs, namespace, &local_err);
795 if (local_err) {
796 error_propagate(errp, local_err);
797 ret = -EIO;
798 goto out;
799 }
800
801 /* Set up command queues. */
802 if (!nvme_add_io_queue(bs, errp)) {
803 ret = -EIO;
804 }
805 out:
806 if (regs) {
807 qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)regs, 0, sizeof(NvmeBar));
808 }
809
810 /* Cleaning up is done in nvme_file_open() upon error. */
811 return ret;
812 }
813
814 /* Parse a filename in the format of nvme://XXXX:XX:XX.X/X. Example:
815 *
816 * nvme://0000:44:00.0/1
817 *
818 * where the "nvme://" is a fixed form of the protocol prefix, the middle part
819 * is the PCI address, and the last part is the namespace number starting from
820 * 1 according to the NVMe spec. */
821 static void nvme_parse_filename(const char *filename, QDict *options,
822 Error **errp)
823 {
824 int pref = strlen("nvme://");
825
826 if (strlen(filename) > pref && !strncmp(filename, "nvme://", pref)) {
827 const char *tmp = filename + pref;
828 char *device;
829 const char *namespace;
830 unsigned long ns;
831 const char *slash = strchr(tmp, '/');
832 if (!slash) {
833 qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, tmp);
834 return;
835 }
836 device = g_strndup(tmp, slash - tmp);
837 qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, device);
838 g_free(device);
839 namespace = slash + 1;
840 if (*namespace && qemu_strtoul(namespace, NULL, 10, &ns)) {
841 error_setg(errp, "Invalid namespace '%s', positive number expected",
842 namespace);
843 return;
844 }
845 qdict_put_str(options, NVME_BLOCK_OPT_NAMESPACE,
846 *namespace ? namespace : "1");
847 }
848 }
849
850 static int nvme_enable_disable_write_cache(BlockDriverState *bs, bool enable,
851 Error **errp)
852 {
853 int ret;
854 BDRVNVMeState *s = bs->opaque;
855 NvmeCmd cmd = {
856 .opcode = NVME_ADM_CMD_SET_FEATURES,
857 .nsid = cpu_to_le32(s->nsid),
858 .cdw10 = cpu_to_le32(0x06),
859 .cdw11 = cpu_to_le32(enable ? 0x01 : 0x00),
860 };
861
862 ret = nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd);
863 if (ret) {
864 error_setg(errp, "Failed to configure NVMe write cache");
865 }
866 return ret;
867 }
868
869 static void nvme_close(BlockDriverState *bs)
870 {
871 int i;
872 BDRVNVMeState *s = bs->opaque;
873
874 for (i = 0; i < s->nr_queues; ++i) {
875 nvme_free_queue_pair(s->queues[i]);
876 }
877 g_free(s->queues);
878 aio_set_event_notifier(bdrv_get_aio_context(bs),
879 &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
880 false, NULL, NULL);
881 event_notifier_cleanup(&s->irq_notifier[MSIX_SHARED_IRQ_IDX]);
882 qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)s->doorbells,
883 sizeof(NvmeBar), NVME_DOORBELL_SIZE);
884 qemu_vfio_close(s->vfio);
885
886 g_free(s->device);
887 }
888
889 static int nvme_file_open(BlockDriverState *bs, QDict *options, int flags,
890 Error **errp)
891 {
892 const char *device;
893 QemuOpts *opts;
894 int namespace;
895 int ret;
896 BDRVNVMeState *s = bs->opaque;
897
898 bs->supported_write_flags = BDRV_REQ_FUA;
899
900 opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
901 qemu_opts_absorb_qdict(opts, options, &error_abort);
902 device = qemu_opt_get(opts, NVME_BLOCK_OPT_DEVICE);
903 if (!device) {
904 error_setg(errp, "'" NVME_BLOCK_OPT_DEVICE "' option is required");
905 qemu_opts_del(opts);
906 return -EINVAL;
907 }
908
909 namespace = qemu_opt_get_number(opts, NVME_BLOCK_OPT_NAMESPACE, 1);
910 ret = nvme_init(bs, device, namespace, errp);
911 qemu_opts_del(opts);
912 if (ret) {
913 goto fail;
914 }
915 if (flags & BDRV_O_NOCACHE) {
916 if (!s->write_cache_supported) {
917 error_setg(errp,
918 "NVMe controller doesn't support write cache configuration");
919 ret = -EINVAL;
920 } else {
921 ret = nvme_enable_disable_write_cache(bs, !(flags & BDRV_O_NOCACHE),
922 errp);
923 }
924 if (ret) {
925 goto fail;
926 }
927 }
928 return 0;
929 fail:
930 nvme_close(bs);
931 return ret;
932 }
933
934 static int64_t nvme_getlength(BlockDriverState *bs)
935 {
936 BDRVNVMeState *s = bs->opaque;
937 return s->nsze << s->blkshift;
938 }
939
940 static uint32_t nvme_get_blocksize(BlockDriverState *bs)
941 {
942 BDRVNVMeState *s = bs->opaque;
943 assert(s->blkshift >= BDRV_SECTOR_BITS && s->blkshift <= 12);
944 return UINT32_C(1) << s->blkshift;
945 }
946
947 static int nvme_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz)
948 {
949 uint32_t blocksize = nvme_get_blocksize(bs);
950 bsz->phys = blocksize;
951 bsz->log = blocksize;
952 return 0;
953 }
954
955 /* Called with s->dma_map_lock */
956 static coroutine_fn int nvme_cmd_unmap_qiov(BlockDriverState *bs,
957 QEMUIOVector *qiov)
958 {
959 int r = 0;
960 BDRVNVMeState *s = bs->opaque;
961
962 s->dma_map_count -= qiov->size;
963 if (!s->dma_map_count && !qemu_co_queue_empty(&s->dma_flush_queue)) {
964 r = qemu_vfio_dma_reset_temporary(s->vfio);
965 if (!r) {
966 qemu_co_queue_restart_all(&s->dma_flush_queue);
967 }
968 }
969 return r;
970 }
971
972 /* Called with s->dma_map_lock */
973 static coroutine_fn int nvme_cmd_map_qiov(BlockDriverState *bs, NvmeCmd *cmd,
974 NVMeRequest *req, QEMUIOVector *qiov)
975 {
976 BDRVNVMeState *s = bs->opaque;
977 uint64_t *pagelist = req->prp_list_page;
978 int i, j, r;
979 int entries = 0;
980
981 assert(qiov->size);
982 assert(QEMU_IS_ALIGNED(qiov->size, s->page_size));
983 assert(qiov->size / s->page_size <= s->page_size / sizeof(uint64_t));
984 for (i = 0; i < qiov->niov; ++i) {
985 bool retry = true;
986 uint64_t iova;
987 try_map:
988 r = qemu_vfio_dma_map(s->vfio,
989 qiov->iov[i].iov_base,
990 qiov->iov[i].iov_len,
991 true, &iova);
992 if (r == -ENOMEM && retry) {
993 retry = false;
994 trace_nvme_dma_flush_queue_wait(s);
995 if (s->dma_map_count) {
996 trace_nvme_dma_map_flush(s);
997 qemu_co_queue_wait(&s->dma_flush_queue, &s->dma_map_lock);
998 } else {
999 r = qemu_vfio_dma_reset_temporary(s->vfio);
1000 if (r) {
1001 goto fail;
1002 }
1003 }
1004 goto try_map;
1005 }
1006 if (r) {
1007 goto fail;
1008 }
1009
1010 for (j = 0; j < qiov->iov[i].iov_len / s->page_size; j++) {
1011 pagelist[entries++] = cpu_to_le64(iova + j * s->page_size);
1012 }
1013 trace_nvme_cmd_map_qiov_iov(s, i, qiov->iov[i].iov_base,
1014 qiov->iov[i].iov_len / s->page_size);
1015 }
1016
1017 s->dma_map_count += qiov->size;
1018
1019 assert(entries <= s->page_size / sizeof(uint64_t));
1020 switch (entries) {
1021 case 0:
1022 abort();
1023 case 1:
1024 cmd->dptr.prp1 = pagelist[0];
1025 cmd->dptr.prp2 = 0;
1026 break;
1027 case 2:
1028 cmd->dptr.prp1 = pagelist[0];
1029 cmd->dptr.prp2 = pagelist[1];
1030 break;
1031 default:
1032 cmd->dptr.prp1 = pagelist[0];
1033 cmd->dptr.prp2 = cpu_to_le64(req->prp_list_iova + sizeof(uint64_t));
1034 break;
1035 }
1036 trace_nvme_cmd_map_qiov(s, cmd, req, qiov, entries);
1037 for (i = 0; i < entries; ++i) {
1038 trace_nvme_cmd_map_qiov_pages(s, i, pagelist[i]);
1039 }
1040 return 0;
1041 fail:
1042 /* No need to unmap [0 - i) iovs even if we've failed, since we don't
1043 * increment s->dma_map_count. This is okay for fixed mapping memory areas
1044 * because they are already mapped before calling this function; for
1045 * temporary mappings, a later nvme_cmd_(un)map_qiov will reclaim by
1046 * calling qemu_vfio_dma_reset_temporary when necessary. */
1047 return r;
1048 }
1049
1050 typedef struct {
1051 Coroutine *co;
1052 int ret;
1053 AioContext *ctx;
1054 } NVMeCoData;
1055
1056 static void nvme_rw_cb_bh(void *opaque)
1057 {
1058 NVMeCoData *data = opaque;
1059 qemu_coroutine_enter(data->co);
1060 }
1061
1062 static void nvme_rw_cb(void *opaque, int ret)
1063 {
1064 NVMeCoData *data = opaque;
1065 data->ret = ret;
1066 if (!data->co) {
1067 /* The rw coroutine hasn't yielded, don't try to enter. */
1068 return;
1069 }
1070 replay_bh_schedule_oneshot_event(data->ctx, nvme_rw_cb_bh, data);
1071 }
1072
1073 static coroutine_fn int nvme_co_prw_aligned(BlockDriverState *bs,
1074 uint64_t offset, uint64_t bytes,
1075 QEMUIOVector *qiov,
1076 bool is_write,
1077 int flags)
1078 {
1079 int r;
1080 BDRVNVMeState *s = bs->opaque;
1081 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1082 NVMeRequest *req;
1083
1084 uint32_t cdw12 = (((bytes >> s->blkshift) - 1) & 0xFFFF) |
1085 (flags & BDRV_REQ_FUA ? 1 << 30 : 0);
1086 NvmeCmd cmd = {
1087 .opcode = is_write ? NVME_CMD_WRITE : NVME_CMD_READ,
1088 .nsid = cpu_to_le32(s->nsid),
1089 .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
1090 .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
1091 .cdw12 = cpu_to_le32(cdw12),
1092 };
1093 NVMeCoData data = {
1094 .ctx = bdrv_get_aio_context(bs),
1095 .ret = -EINPROGRESS,
1096 };
1097
1098 trace_nvme_prw_aligned(s, is_write, offset, bytes, flags, qiov->niov);
1099 assert(s->nr_queues > 1);
1100 req = nvme_get_free_req(ioq);
1101 assert(req);
1102
1103 qemu_co_mutex_lock(&s->dma_map_lock);
1104 r = nvme_cmd_map_qiov(bs, &cmd, req, qiov);
1105 qemu_co_mutex_unlock(&s->dma_map_lock);
1106 if (r) {
1107 nvme_put_free_req_and_wake(ioq, req);
1108 return r;
1109 }
1110 nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1111
1112 data.co = qemu_coroutine_self();
1113 while (data.ret == -EINPROGRESS) {
1114 qemu_coroutine_yield();
1115 }
1116
1117 qemu_co_mutex_lock(&s->dma_map_lock);
1118 r = nvme_cmd_unmap_qiov(bs, qiov);
1119 qemu_co_mutex_unlock(&s->dma_map_lock);
1120 if (r) {
1121 return r;
1122 }
1123
1124 trace_nvme_rw_done(s, is_write, offset, bytes, data.ret);
1125 return data.ret;
1126 }
1127
1128 static inline bool nvme_qiov_aligned(BlockDriverState *bs,
1129 const QEMUIOVector *qiov)
1130 {
1131 int i;
1132 BDRVNVMeState *s = bs->opaque;
1133
1134 for (i = 0; i < qiov->niov; ++i) {
1135 if (!QEMU_PTR_IS_ALIGNED(qiov->iov[i].iov_base, s->page_size) ||
1136 !QEMU_IS_ALIGNED(qiov->iov[i].iov_len, s->page_size)) {
1137 trace_nvme_qiov_unaligned(qiov, i, qiov->iov[i].iov_base,
1138 qiov->iov[i].iov_len, s->page_size);
1139 return false;
1140 }
1141 }
1142 return true;
1143 }
1144
1145 static int nvme_co_prw(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
1146 QEMUIOVector *qiov, bool is_write, int flags)
1147 {
1148 BDRVNVMeState *s = bs->opaque;
1149 int r;
1150 uint8_t *buf = NULL;
1151 QEMUIOVector local_qiov;
1152
1153 assert(QEMU_IS_ALIGNED(offset, s->page_size));
1154 assert(QEMU_IS_ALIGNED(bytes, s->page_size));
1155 assert(bytes <= s->max_transfer);
1156 if (nvme_qiov_aligned(bs, qiov)) {
1157 return nvme_co_prw_aligned(bs, offset, bytes, qiov, is_write, flags);
1158 }
1159 trace_nvme_prw_buffered(s, offset, bytes, qiov->niov, is_write);
1160 buf = qemu_try_memalign(s->page_size, bytes);
1161
1162 if (!buf) {
1163 return -ENOMEM;
1164 }
1165 qemu_iovec_init(&local_qiov, 1);
1166 if (is_write) {
1167 qemu_iovec_to_buf(qiov, 0, buf, bytes);
1168 }
1169 qemu_iovec_add(&local_qiov, buf, bytes);
1170 r = nvme_co_prw_aligned(bs, offset, bytes, &local_qiov, is_write, flags);
1171 qemu_iovec_destroy(&local_qiov);
1172 if (!r && !is_write) {
1173 qemu_iovec_from_buf(qiov, 0, buf, bytes);
1174 }
1175 qemu_vfree(buf);
1176 return r;
1177 }
1178
1179 static coroutine_fn int nvme_co_preadv(BlockDriverState *bs,
1180 uint64_t offset, uint64_t bytes,
1181 QEMUIOVector *qiov, int flags)
1182 {
1183 return nvme_co_prw(bs, offset, bytes, qiov, false, flags);
1184 }
1185
1186 static coroutine_fn int nvme_co_pwritev(BlockDriverState *bs,
1187 uint64_t offset, uint64_t bytes,
1188 QEMUIOVector *qiov, int flags)
1189 {
1190 return nvme_co_prw(bs, offset, bytes, qiov, true, flags);
1191 }
1192
1193 static coroutine_fn int nvme_co_flush(BlockDriverState *bs)
1194 {
1195 BDRVNVMeState *s = bs->opaque;
1196 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1197 NVMeRequest *req;
1198 NvmeCmd cmd = {
1199 .opcode = NVME_CMD_FLUSH,
1200 .nsid = cpu_to_le32(s->nsid),
1201 };
1202 NVMeCoData data = {
1203 .ctx = bdrv_get_aio_context(bs),
1204 .ret = -EINPROGRESS,
1205 };
1206
1207 assert(s->nr_queues > 1);
1208 req = nvme_get_free_req(ioq);
1209 assert(req);
1210 nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1211
1212 data.co = qemu_coroutine_self();
1213 if (data.ret == -EINPROGRESS) {
1214 qemu_coroutine_yield();
1215 }
1216
1217 return data.ret;
1218 }
1219
1220
1221 static coroutine_fn int nvme_co_pwrite_zeroes(BlockDriverState *bs,
1222 int64_t offset,
1223 int bytes,
1224 BdrvRequestFlags flags)
1225 {
1226 BDRVNVMeState *s = bs->opaque;
1227 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1228 NVMeRequest *req;
1229
1230 uint32_t cdw12 = ((bytes >> s->blkshift) - 1) & 0xFFFF;
1231
1232 if (!s->supports_write_zeroes) {
1233 return -ENOTSUP;
1234 }
1235
1236 NvmeCmd cmd = {
1237 .opcode = NVME_CMD_WRITE_ZEROES,
1238 .nsid = cpu_to_le32(s->nsid),
1239 .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
1240 .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
1241 };
1242
1243 NVMeCoData data = {
1244 .ctx = bdrv_get_aio_context(bs),
1245 .ret = -EINPROGRESS,
1246 };
1247
1248 if (flags & BDRV_REQ_MAY_UNMAP) {
1249 cdw12 |= (1 << 25);
1250 }
1251
1252 if (flags & BDRV_REQ_FUA) {
1253 cdw12 |= (1 << 30);
1254 }
1255
1256 cmd.cdw12 = cpu_to_le32(cdw12);
1257
1258 trace_nvme_write_zeroes(s, offset, bytes, flags);
1259 assert(s->nr_queues > 1);
1260 req = nvme_get_free_req(ioq);
1261 assert(req);
1262
1263 nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1264
1265 data.co = qemu_coroutine_self();
1266 while (data.ret == -EINPROGRESS) {
1267 qemu_coroutine_yield();
1268 }
1269
1270 trace_nvme_rw_done(s, true, offset, bytes, data.ret);
1271 return data.ret;
1272 }
1273
1274
1275 static int coroutine_fn nvme_co_pdiscard(BlockDriverState *bs,
1276 int64_t offset,
1277 int bytes)
1278 {
1279 BDRVNVMeState *s = bs->opaque;
1280 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1281 NVMeRequest *req;
1282 NvmeDsmRange *buf;
1283 QEMUIOVector local_qiov;
1284 int ret;
1285
1286 NvmeCmd cmd = {
1287 .opcode = NVME_CMD_DSM,
1288 .nsid = cpu_to_le32(s->nsid),
1289 .cdw10 = cpu_to_le32(0), /*number of ranges - 0 based*/
1290 .cdw11 = cpu_to_le32(1 << 2), /*deallocate bit*/
1291 };
1292
1293 NVMeCoData data = {
1294 .ctx = bdrv_get_aio_context(bs),
1295 .ret = -EINPROGRESS,
1296 };
1297
1298 if (!s->supports_discard) {
1299 return -ENOTSUP;
1300 }
1301
1302 assert(s->nr_queues > 1);
1303
1304 buf = qemu_try_memalign(s->page_size, s->page_size);
1305 if (!buf) {
1306 return -ENOMEM;
1307 }
1308 memset(buf, 0, s->page_size);
1309 buf->nlb = cpu_to_le32(bytes >> s->blkshift);
1310 buf->slba = cpu_to_le64(offset >> s->blkshift);
1311 buf->cattr = 0;
1312
1313 qemu_iovec_init(&local_qiov, 1);
1314 qemu_iovec_add(&local_qiov, buf, 4096);
1315
1316 req = nvme_get_free_req(ioq);
1317 assert(req);
1318
1319 qemu_co_mutex_lock(&s->dma_map_lock);
1320 ret = nvme_cmd_map_qiov(bs, &cmd, req, &local_qiov);
1321 qemu_co_mutex_unlock(&s->dma_map_lock);
1322
1323 if (ret) {
1324 nvme_put_free_req_and_wake(ioq, req);
1325 goto out;
1326 }
1327
1328 trace_nvme_dsm(s, offset, bytes);
1329
1330 nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1331
1332 data.co = qemu_coroutine_self();
1333 while (data.ret == -EINPROGRESS) {
1334 qemu_coroutine_yield();
1335 }
1336
1337 qemu_co_mutex_lock(&s->dma_map_lock);
1338 ret = nvme_cmd_unmap_qiov(bs, &local_qiov);
1339 qemu_co_mutex_unlock(&s->dma_map_lock);
1340
1341 if (ret) {
1342 goto out;
1343 }
1344
1345 ret = data.ret;
1346 trace_nvme_dsm_done(s, offset, bytes, ret);
1347 out:
1348 qemu_iovec_destroy(&local_qiov);
1349 qemu_vfree(buf);
1350 return ret;
1351
1352 }
1353
1354
1355 static int nvme_reopen_prepare(BDRVReopenState *reopen_state,
1356 BlockReopenQueue *queue, Error **errp)
1357 {
1358 return 0;
1359 }
1360
1361 static void nvme_refresh_filename(BlockDriverState *bs)
1362 {
1363 BDRVNVMeState *s = bs->opaque;
1364
1365 snprintf(bs->exact_filename, sizeof(bs->exact_filename), "nvme://%s/%i",
1366 s->device, s->nsid);
1367 }
1368
1369 static void nvme_refresh_limits(BlockDriverState *bs, Error **errp)
1370 {
1371 BDRVNVMeState *s = bs->opaque;
1372
1373 bs->bl.opt_mem_alignment = s->page_size;
1374 bs->bl.request_alignment = s->page_size;
1375 bs->bl.max_transfer = s->max_transfer;
1376 }
1377
1378 static void nvme_detach_aio_context(BlockDriverState *bs)
1379 {
1380 BDRVNVMeState *s = bs->opaque;
1381
1382 for (int i = 0; i < s->nr_queues; i++) {
1383 NVMeQueuePair *q = s->queues[i];
1384
1385 qemu_bh_delete(q->completion_bh);
1386 q->completion_bh = NULL;
1387 }
1388
1389 aio_set_event_notifier(bdrv_get_aio_context(bs),
1390 &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
1391 false, NULL, NULL);
1392 }
1393
1394 static void nvme_attach_aio_context(BlockDriverState *bs,
1395 AioContext *new_context)
1396 {
1397 BDRVNVMeState *s = bs->opaque;
1398
1399 s->aio_context = new_context;
1400 aio_set_event_notifier(new_context, &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
1401 false, nvme_handle_event, nvme_poll_cb);
1402
1403 for (int i = 0; i < s->nr_queues; i++) {
1404 NVMeQueuePair *q = s->queues[i];
1405
1406 q->completion_bh =
1407 aio_bh_new(new_context, nvme_process_completion_bh, q);
1408 }
1409 }
1410
1411 static void nvme_aio_plug(BlockDriverState *bs)
1412 {
1413 BDRVNVMeState *s = bs->opaque;
1414 assert(!s->plugged);
1415 s->plugged = true;
1416 }
1417
1418 static void nvme_aio_unplug(BlockDriverState *bs)
1419 {
1420 int i;
1421 BDRVNVMeState *s = bs->opaque;
1422 assert(s->plugged);
1423 s->plugged = false;
1424 for (i = INDEX_IO(0); i < s->nr_queues; i++) {
1425 NVMeQueuePair *q = s->queues[i];
1426 qemu_mutex_lock(&q->lock);
1427 nvme_kick(q);
1428 nvme_process_completion(q);
1429 qemu_mutex_unlock(&q->lock);
1430 }
1431 }
1432
1433 static void nvme_register_buf(BlockDriverState *bs, void *host, size_t size)
1434 {
1435 int ret;
1436 BDRVNVMeState *s = bs->opaque;
1437
1438 ret = qemu_vfio_dma_map(s->vfio, host, size, false, NULL);
1439 if (ret) {
1440 /* FIXME: we may run out of IOVA addresses after repeated
1441 * bdrv_register_buf/bdrv_unregister_buf, because nvme_vfio_dma_unmap
1442 * doesn't reclaim addresses for fixed mappings. */
1443 error_report("nvme_register_buf failed: %s", strerror(-ret));
1444 }
1445 }
1446
1447 static void nvme_unregister_buf(BlockDriverState *bs, void *host)
1448 {
1449 BDRVNVMeState *s = bs->opaque;
1450
1451 qemu_vfio_dma_unmap(s->vfio, host);
1452 }
1453
1454 static const char *const nvme_strong_runtime_opts[] = {
1455 NVME_BLOCK_OPT_DEVICE,
1456 NVME_BLOCK_OPT_NAMESPACE,
1457
1458 NULL
1459 };
1460
1461 static BlockDriver bdrv_nvme = {
1462 .format_name = "nvme",
1463 .protocol_name = "nvme",
1464 .instance_size = sizeof(BDRVNVMeState),
1465
1466 .bdrv_co_create_opts = bdrv_co_create_opts_simple,
1467 .create_opts = &bdrv_create_opts_simple,
1468
1469 .bdrv_parse_filename = nvme_parse_filename,
1470 .bdrv_file_open = nvme_file_open,
1471 .bdrv_close = nvme_close,
1472 .bdrv_getlength = nvme_getlength,
1473 .bdrv_probe_blocksizes = nvme_probe_blocksizes,
1474
1475 .bdrv_co_preadv = nvme_co_preadv,
1476 .bdrv_co_pwritev = nvme_co_pwritev,
1477
1478 .bdrv_co_pwrite_zeroes = nvme_co_pwrite_zeroes,
1479 .bdrv_co_pdiscard = nvme_co_pdiscard,
1480
1481 .bdrv_co_flush_to_disk = nvme_co_flush,
1482 .bdrv_reopen_prepare = nvme_reopen_prepare,
1483
1484 .bdrv_refresh_filename = nvme_refresh_filename,
1485 .bdrv_refresh_limits = nvme_refresh_limits,
1486 .strong_runtime_opts = nvme_strong_runtime_opts,
1487
1488 .bdrv_detach_aio_context = nvme_detach_aio_context,
1489 .bdrv_attach_aio_context = nvme_attach_aio_context,
1490
1491 .bdrv_io_plug = nvme_aio_plug,
1492 .bdrv_io_unplug = nvme_aio_unplug,
1493
1494 .bdrv_register_buf = nvme_register_buf,
1495 .bdrv_unregister_buf = nvme_unregister_buf,
1496 };
1497
1498 static void bdrv_nvme_init(void)
1499 {
1500 bdrv_register(&bdrv_nvme);
1501 }
1502
1503 block_init(bdrv_nvme_init);