block/nvme: Map doorbells pages write-only
[qemu.git] / block / nvme.c
1 /*
2 * NVMe block driver based on vfio
3 *
4 * Copyright 2016 - 2018 Red Hat, Inc.
5 *
6 * Authors:
7 * Fam Zheng <famz@redhat.com>
8 * Paolo Bonzini <pbonzini@redhat.com>
9 *
10 * This work is licensed under the terms of the GNU GPL, version 2 or later.
11 * See the COPYING file in the top-level directory.
12 */
13
14 #include "qemu/osdep.h"
15 #include <linux/vfio.h>
16 #include "qapi/error.h"
17 #include "qapi/qmp/qdict.h"
18 #include "qapi/qmp/qstring.h"
19 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qemu/module.h"
22 #include "qemu/cutils.h"
23 #include "qemu/option.h"
24 #include "qemu/vfio-helpers.h"
25 #include "block/block_int.h"
26 #include "sysemu/replay.h"
27 #include "trace.h"
28
29 #include "block/nvme.h"
30
31 #define NVME_SQ_ENTRY_BYTES 64
32 #define NVME_CQ_ENTRY_BYTES 16
33 #define NVME_QUEUE_SIZE 128
34 #define NVME_DOORBELL_SIZE 4096
35
36 /*
37 * We have to leave one slot empty as that is the full queue case where
38 * head == tail + 1.
39 */
40 #define NVME_NUM_REQS (NVME_QUEUE_SIZE - 1)
41
42 typedef struct BDRVNVMeState BDRVNVMeState;
43
44 typedef struct {
45 int32_t head, tail;
46 uint8_t *queue;
47 uint64_t iova;
48 /* Hardware MMIO register */
49 volatile uint32_t *doorbell;
50 } NVMeQueue;
51
52 typedef struct {
53 BlockCompletionFunc *cb;
54 void *opaque;
55 int cid;
56 void *prp_list_page;
57 uint64_t prp_list_iova;
58 int free_req_next; /* q->reqs[] index of next free req */
59 } NVMeRequest;
60
61 typedef struct {
62 QemuMutex lock;
63
64 /* Read from I/O code path, initialized under BQL */
65 BDRVNVMeState *s;
66 int index;
67
68 /* Fields protected by BQL */
69 uint8_t *prp_list_pages;
70
71 /* Fields protected by @lock */
72 CoQueue free_req_queue;
73 NVMeQueue sq, cq;
74 int cq_phase;
75 int free_req_head;
76 NVMeRequest reqs[NVME_NUM_REQS];
77 int need_kick;
78 int inflight;
79
80 /* Thread-safe, no lock necessary */
81 QEMUBH *completion_bh;
82 } NVMeQueuePair;
83
84 /* Memory mapped registers */
85 typedef volatile struct {
86 NvmeBar ctrl;
87 } NVMeRegs;
88
89 #define INDEX_ADMIN 0
90 #define INDEX_IO(n) (1 + n)
91
92 /* This driver shares a single MSIX IRQ for the admin and I/O queues */
93 enum {
94 MSIX_SHARED_IRQ_IDX = 0,
95 MSIX_IRQ_COUNT = 1
96 };
97
98 struct BDRVNVMeState {
99 AioContext *aio_context;
100 QEMUVFIOState *vfio;
101 NVMeRegs *regs;
102 /* Memory mapped registers */
103 volatile struct {
104 uint32_t sq_tail;
105 uint32_t cq_head;
106 } *doorbells;
107 /* The submission/completion queue pairs.
108 * [0]: admin queue.
109 * [1..]: io queues.
110 */
111 NVMeQueuePair **queues;
112 int nr_queues;
113 size_t page_size;
114 /* How many uint32_t elements does each doorbell entry take. */
115 size_t doorbell_scale;
116 bool write_cache_supported;
117 EventNotifier irq_notifier[MSIX_IRQ_COUNT];
118
119 uint64_t nsze; /* Namespace size reported by identify command */
120 int nsid; /* The namespace id to read/write data. */
121 int blkshift;
122
123 uint64_t max_transfer;
124 bool plugged;
125
126 bool supports_write_zeroes;
127 bool supports_discard;
128
129 CoMutex dma_map_lock;
130 CoQueue dma_flush_queue;
131
132 /* Total size of mapped qiov, accessed under dma_map_lock */
133 int dma_map_count;
134
135 /* PCI address (required for nvme_refresh_filename()) */
136 char *device;
137 };
138
139 #define NVME_BLOCK_OPT_DEVICE "device"
140 #define NVME_BLOCK_OPT_NAMESPACE "namespace"
141
142 static void nvme_process_completion_bh(void *opaque);
143
144 static QemuOptsList runtime_opts = {
145 .name = "nvme",
146 .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
147 .desc = {
148 {
149 .name = NVME_BLOCK_OPT_DEVICE,
150 .type = QEMU_OPT_STRING,
151 .help = "NVMe PCI device address",
152 },
153 {
154 .name = NVME_BLOCK_OPT_NAMESPACE,
155 .type = QEMU_OPT_NUMBER,
156 .help = "NVMe namespace",
157 },
158 { /* end of list */ }
159 },
160 };
161
162 static void nvme_init_queue(BDRVNVMeState *s, NVMeQueue *q,
163 int nentries, int entry_bytes, Error **errp)
164 {
165 size_t bytes;
166 int r;
167
168 bytes = ROUND_UP(nentries * entry_bytes, s->page_size);
169 q->head = q->tail = 0;
170 q->queue = qemu_try_memalign(s->page_size, bytes);
171 if (!q->queue) {
172 error_setg(errp, "Cannot allocate queue");
173 return;
174 }
175 memset(q->queue, 0, bytes);
176 r = qemu_vfio_dma_map(s->vfio, q->queue, bytes, false, &q->iova);
177 if (r) {
178 error_setg(errp, "Cannot map queue");
179 }
180 }
181
182 static void nvme_free_queue_pair(NVMeQueuePair *q)
183 {
184 if (q->completion_bh) {
185 qemu_bh_delete(q->completion_bh);
186 }
187 qemu_vfree(q->prp_list_pages);
188 qemu_vfree(q->sq.queue);
189 qemu_vfree(q->cq.queue);
190 qemu_mutex_destroy(&q->lock);
191 g_free(q);
192 }
193
194 static void nvme_free_req_queue_cb(void *opaque)
195 {
196 NVMeQueuePair *q = opaque;
197
198 qemu_mutex_lock(&q->lock);
199 while (qemu_co_enter_next(&q->free_req_queue, &q->lock)) {
200 /* Retry all pending requests */
201 }
202 qemu_mutex_unlock(&q->lock);
203 }
204
205 static NVMeQueuePair *nvme_create_queue_pair(BDRVNVMeState *s,
206 AioContext *aio_context,
207 int idx, int size,
208 Error **errp)
209 {
210 int i, r;
211 Error *local_err = NULL;
212 NVMeQueuePair *q;
213 uint64_t prp_list_iova;
214
215 q = g_try_new0(NVMeQueuePair, 1);
216 if (!q) {
217 return NULL;
218 }
219 q->prp_list_pages = qemu_try_memalign(s->page_size,
220 s->page_size * NVME_NUM_REQS);
221 if (!q->prp_list_pages) {
222 goto fail;
223 }
224 memset(q->prp_list_pages, 0, s->page_size * NVME_NUM_REQS);
225 qemu_mutex_init(&q->lock);
226 q->s = s;
227 q->index = idx;
228 qemu_co_queue_init(&q->free_req_queue);
229 q->completion_bh = aio_bh_new(aio_context, nvme_process_completion_bh, q);
230 r = qemu_vfio_dma_map(s->vfio, q->prp_list_pages,
231 s->page_size * NVME_NUM_REQS,
232 false, &prp_list_iova);
233 if (r) {
234 goto fail;
235 }
236 q->free_req_head = -1;
237 for (i = 0; i < NVME_NUM_REQS; i++) {
238 NVMeRequest *req = &q->reqs[i];
239 req->cid = i + 1;
240 req->free_req_next = q->free_req_head;
241 q->free_req_head = i;
242 req->prp_list_page = q->prp_list_pages + i * s->page_size;
243 req->prp_list_iova = prp_list_iova + i * s->page_size;
244 }
245
246 nvme_init_queue(s, &q->sq, size, NVME_SQ_ENTRY_BYTES, &local_err);
247 if (local_err) {
248 error_propagate(errp, local_err);
249 goto fail;
250 }
251 q->sq.doorbell = &s->doorbells[idx * s->doorbell_scale].sq_tail;
252
253 nvme_init_queue(s, &q->cq, size, NVME_CQ_ENTRY_BYTES, &local_err);
254 if (local_err) {
255 error_propagate(errp, local_err);
256 goto fail;
257 }
258 q->cq.doorbell = &s->doorbells[idx * s->doorbell_scale].cq_head;
259
260 return q;
261 fail:
262 nvme_free_queue_pair(q);
263 return NULL;
264 }
265
266 /* With q->lock */
267 static void nvme_kick(NVMeQueuePair *q)
268 {
269 BDRVNVMeState *s = q->s;
270
271 if (s->plugged || !q->need_kick) {
272 return;
273 }
274 trace_nvme_kick(s, q->index);
275 assert(!(q->sq.tail & 0xFF00));
276 /* Fence the write to submission queue entry before notifying the device. */
277 smp_wmb();
278 *q->sq.doorbell = cpu_to_le32(q->sq.tail);
279 q->inflight += q->need_kick;
280 q->need_kick = 0;
281 }
282
283 /* Find a free request element if any, otherwise:
284 * a) if in coroutine context, try to wait for one to become available;
285 * b) if not in coroutine, return NULL;
286 */
287 static NVMeRequest *nvme_get_free_req(NVMeQueuePair *q)
288 {
289 NVMeRequest *req;
290
291 qemu_mutex_lock(&q->lock);
292
293 while (q->free_req_head == -1) {
294 if (qemu_in_coroutine()) {
295 trace_nvme_free_req_queue_wait(q);
296 qemu_co_queue_wait(&q->free_req_queue, &q->lock);
297 } else {
298 qemu_mutex_unlock(&q->lock);
299 return NULL;
300 }
301 }
302
303 req = &q->reqs[q->free_req_head];
304 q->free_req_head = req->free_req_next;
305 req->free_req_next = -1;
306
307 qemu_mutex_unlock(&q->lock);
308 return req;
309 }
310
311 /* With q->lock */
312 static void nvme_put_free_req_locked(NVMeQueuePair *q, NVMeRequest *req)
313 {
314 req->free_req_next = q->free_req_head;
315 q->free_req_head = req - q->reqs;
316 }
317
318 /* With q->lock */
319 static void nvme_wake_free_req_locked(NVMeQueuePair *q)
320 {
321 if (!qemu_co_queue_empty(&q->free_req_queue)) {
322 replay_bh_schedule_oneshot_event(q->s->aio_context,
323 nvme_free_req_queue_cb, q);
324 }
325 }
326
327 /* Insert a request in the freelist and wake waiters */
328 static void nvme_put_free_req_and_wake(NVMeQueuePair *q, NVMeRequest *req)
329 {
330 qemu_mutex_lock(&q->lock);
331 nvme_put_free_req_locked(q, req);
332 nvme_wake_free_req_locked(q);
333 qemu_mutex_unlock(&q->lock);
334 }
335
336 static inline int nvme_translate_error(const NvmeCqe *c)
337 {
338 uint16_t status = (le16_to_cpu(c->status) >> 1) & 0xFF;
339 if (status) {
340 trace_nvme_error(le32_to_cpu(c->result),
341 le16_to_cpu(c->sq_head),
342 le16_to_cpu(c->sq_id),
343 le16_to_cpu(c->cid),
344 le16_to_cpu(status));
345 }
346 switch (status) {
347 case 0:
348 return 0;
349 case 1:
350 return -ENOSYS;
351 case 2:
352 return -EINVAL;
353 default:
354 return -EIO;
355 }
356 }
357
358 /* With q->lock */
359 static bool nvme_process_completion(NVMeQueuePair *q)
360 {
361 BDRVNVMeState *s = q->s;
362 bool progress = false;
363 NVMeRequest *preq;
364 NVMeRequest req;
365 NvmeCqe *c;
366
367 trace_nvme_process_completion(s, q->index, q->inflight);
368 if (s->plugged) {
369 trace_nvme_process_completion_queue_plugged(s, q->index);
370 return false;
371 }
372
373 /*
374 * Support re-entrancy when a request cb() function invokes aio_poll().
375 * Pending completions must be visible to aio_poll() so that a cb()
376 * function can wait for the completion of another request.
377 *
378 * The aio_poll() loop will execute our BH and we'll resume completion
379 * processing there.
380 */
381 qemu_bh_schedule(q->completion_bh);
382
383 assert(q->inflight >= 0);
384 while (q->inflight) {
385 int ret;
386 int16_t cid;
387
388 c = (NvmeCqe *)&q->cq.queue[q->cq.head * NVME_CQ_ENTRY_BYTES];
389 if ((le16_to_cpu(c->status) & 0x1) == q->cq_phase) {
390 break;
391 }
392 ret = nvme_translate_error(c);
393 q->cq.head = (q->cq.head + 1) % NVME_QUEUE_SIZE;
394 if (!q->cq.head) {
395 q->cq_phase = !q->cq_phase;
396 }
397 cid = le16_to_cpu(c->cid);
398 if (cid == 0 || cid > NVME_QUEUE_SIZE) {
399 fprintf(stderr, "Unexpected CID in completion queue: %" PRIu32 "\n",
400 cid);
401 continue;
402 }
403 trace_nvme_complete_command(s, q->index, cid);
404 preq = &q->reqs[cid - 1];
405 req = *preq;
406 assert(req.cid == cid);
407 assert(req.cb);
408 nvme_put_free_req_locked(q, preq);
409 preq->cb = preq->opaque = NULL;
410 q->inflight--;
411 qemu_mutex_unlock(&q->lock);
412 req.cb(req.opaque, ret);
413 qemu_mutex_lock(&q->lock);
414 progress = true;
415 }
416 if (progress) {
417 /* Notify the device so it can post more completions. */
418 smp_mb_release();
419 *q->cq.doorbell = cpu_to_le32(q->cq.head);
420 nvme_wake_free_req_locked(q);
421 }
422
423 qemu_bh_cancel(q->completion_bh);
424
425 return progress;
426 }
427
428 static void nvme_process_completion_bh(void *opaque)
429 {
430 NVMeQueuePair *q = opaque;
431
432 /*
433 * We're being invoked because a nvme_process_completion() cb() function
434 * called aio_poll(). The callback may be waiting for further completions
435 * so notify the device that it has space to fill in more completions now.
436 */
437 smp_mb_release();
438 *q->cq.doorbell = cpu_to_le32(q->cq.head);
439 nvme_wake_free_req_locked(q);
440
441 nvme_process_completion(q);
442 }
443
444 static void nvme_trace_command(const NvmeCmd *cmd)
445 {
446 int i;
447
448 if (!trace_event_get_state_backends(TRACE_NVME_SUBMIT_COMMAND_RAW)) {
449 return;
450 }
451 for (i = 0; i < 8; ++i) {
452 uint8_t *cmdp = (uint8_t *)cmd + i * 8;
453 trace_nvme_submit_command_raw(cmdp[0], cmdp[1], cmdp[2], cmdp[3],
454 cmdp[4], cmdp[5], cmdp[6], cmdp[7]);
455 }
456 }
457
458 static void nvme_submit_command(NVMeQueuePair *q, NVMeRequest *req,
459 NvmeCmd *cmd, BlockCompletionFunc cb,
460 void *opaque)
461 {
462 assert(!req->cb);
463 req->cb = cb;
464 req->opaque = opaque;
465 cmd->cid = cpu_to_le32(req->cid);
466
467 trace_nvme_submit_command(q->s, q->index, req->cid);
468 nvme_trace_command(cmd);
469 qemu_mutex_lock(&q->lock);
470 memcpy((uint8_t *)q->sq.queue +
471 q->sq.tail * NVME_SQ_ENTRY_BYTES, cmd, sizeof(*cmd));
472 q->sq.tail = (q->sq.tail + 1) % NVME_QUEUE_SIZE;
473 q->need_kick++;
474 nvme_kick(q);
475 nvme_process_completion(q);
476 qemu_mutex_unlock(&q->lock);
477 }
478
479 static void nvme_cmd_sync_cb(void *opaque, int ret)
480 {
481 int *pret = opaque;
482 *pret = ret;
483 aio_wait_kick();
484 }
485
486 static int nvme_cmd_sync(BlockDriverState *bs, NVMeQueuePair *q,
487 NvmeCmd *cmd)
488 {
489 AioContext *aio_context = bdrv_get_aio_context(bs);
490 NVMeRequest *req;
491 int ret = -EINPROGRESS;
492 req = nvme_get_free_req(q);
493 if (!req) {
494 return -EBUSY;
495 }
496 nvme_submit_command(q, req, cmd, nvme_cmd_sync_cb, &ret);
497
498 AIO_WAIT_WHILE(aio_context, ret == -EINPROGRESS);
499 return ret;
500 }
501
502 static void nvme_identify(BlockDriverState *bs, int namespace, Error **errp)
503 {
504 BDRVNVMeState *s = bs->opaque;
505 union {
506 NvmeIdCtrl ctrl;
507 NvmeIdNs ns;
508 } *id;
509 NvmeLBAF *lbaf;
510 uint16_t oncs;
511 int r;
512 uint64_t iova;
513 NvmeCmd cmd = {
514 .opcode = NVME_ADM_CMD_IDENTIFY,
515 .cdw10 = cpu_to_le32(0x1),
516 };
517
518 id = qemu_try_memalign(s->page_size, sizeof(*id));
519 if (!id) {
520 error_setg(errp, "Cannot allocate buffer for identify response");
521 goto out;
522 }
523 r = qemu_vfio_dma_map(s->vfio, id, sizeof(*id), true, &iova);
524 if (r) {
525 error_setg(errp, "Cannot map buffer for DMA");
526 goto out;
527 }
528
529 memset(id, 0, sizeof(*id));
530 cmd.dptr.prp1 = cpu_to_le64(iova);
531 if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
532 error_setg(errp, "Failed to identify controller");
533 goto out;
534 }
535
536 if (le32_to_cpu(id->ctrl.nn) < namespace) {
537 error_setg(errp, "Invalid namespace");
538 goto out;
539 }
540 s->write_cache_supported = le32_to_cpu(id->ctrl.vwc) & 0x1;
541 s->max_transfer = (id->ctrl.mdts ? 1 << id->ctrl.mdts : 0) * s->page_size;
542 /* For now the page list buffer per command is one page, to hold at most
543 * s->page_size / sizeof(uint64_t) entries. */
544 s->max_transfer = MIN_NON_ZERO(s->max_transfer,
545 s->page_size / sizeof(uint64_t) * s->page_size);
546
547 oncs = le16_to_cpu(id->ctrl.oncs);
548 s->supports_write_zeroes = !!(oncs & NVME_ONCS_WRITE_ZEROES);
549 s->supports_discard = !!(oncs & NVME_ONCS_DSM);
550
551 memset(id, 0, sizeof(*id));
552 cmd.cdw10 = 0;
553 cmd.nsid = cpu_to_le32(namespace);
554 if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
555 error_setg(errp, "Failed to identify namespace");
556 goto out;
557 }
558
559 s->nsze = le64_to_cpu(id->ns.nsze);
560 lbaf = &id->ns.lbaf[NVME_ID_NS_FLBAS_INDEX(id->ns.flbas)];
561
562 if (NVME_ID_NS_DLFEAT_WRITE_ZEROES(id->ns.dlfeat) &&
563 NVME_ID_NS_DLFEAT_READ_BEHAVIOR(id->ns.dlfeat) ==
564 NVME_ID_NS_DLFEAT_READ_BEHAVIOR_ZEROES) {
565 bs->supported_write_flags |= BDRV_REQ_MAY_UNMAP;
566 }
567
568 if (lbaf->ms) {
569 error_setg(errp, "Namespaces with metadata are not yet supported");
570 goto out;
571 }
572
573 if (lbaf->ds < BDRV_SECTOR_BITS || lbaf->ds > 12 ||
574 (1 << lbaf->ds) > s->page_size)
575 {
576 error_setg(errp, "Namespace has unsupported block size (2^%d)",
577 lbaf->ds);
578 goto out;
579 }
580
581 s->blkshift = lbaf->ds;
582 out:
583 qemu_vfio_dma_unmap(s->vfio, id);
584 qemu_vfree(id);
585 }
586
587 static bool nvme_poll_queue(NVMeQueuePair *q)
588 {
589 bool progress = false;
590
591 const size_t cqe_offset = q->cq.head * NVME_CQ_ENTRY_BYTES;
592 NvmeCqe *cqe = (NvmeCqe *)&q->cq.queue[cqe_offset];
593
594 /*
595 * Do an early check for completions. q->lock isn't needed because
596 * nvme_process_completion() only runs in the event loop thread and
597 * cannot race with itself.
598 */
599 if ((le16_to_cpu(cqe->status) & 0x1) == q->cq_phase) {
600 return false;
601 }
602
603 qemu_mutex_lock(&q->lock);
604 while (nvme_process_completion(q)) {
605 /* Keep polling */
606 progress = true;
607 }
608 qemu_mutex_unlock(&q->lock);
609
610 return progress;
611 }
612
613 static bool nvme_poll_queues(BDRVNVMeState *s)
614 {
615 bool progress = false;
616 int i;
617
618 for (i = 0; i < s->nr_queues; i++) {
619 if (nvme_poll_queue(s->queues[i])) {
620 progress = true;
621 }
622 }
623 return progress;
624 }
625
626 static void nvme_handle_event(EventNotifier *n)
627 {
628 BDRVNVMeState *s = container_of(n, BDRVNVMeState,
629 irq_notifier[MSIX_SHARED_IRQ_IDX]);
630
631 trace_nvme_handle_event(s);
632 event_notifier_test_and_clear(n);
633 nvme_poll_queues(s);
634 }
635
636 static bool nvme_add_io_queue(BlockDriverState *bs, Error **errp)
637 {
638 BDRVNVMeState *s = bs->opaque;
639 int n = s->nr_queues;
640 NVMeQueuePair *q;
641 NvmeCmd cmd;
642 int queue_size = NVME_QUEUE_SIZE;
643
644 q = nvme_create_queue_pair(s, bdrv_get_aio_context(bs),
645 n, queue_size, errp);
646 if (!q) {
647 return false;
648 }
649 cmd = (NvmeCmd) {
650 .opcode = NVME_ADM_CMD_CREATE_CQ,
651 .dptr.prp1 = cpu_to_le64(q->cq.iova),
652 .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)),
653 .cdw11 = cpu_to_le32(0x3),
654 };
655 if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
656 error_setg(errp, "Failed to create CQ io queue [%d]", n);
657 goto out_error;
658 }
659 cmd = (NvmeCmd) {
660 .opcode = NVME_ADM_CMD_CREATE_SQ,
661 .dptr.prp1 = cpu_to_le64(q->sq.iova),
662 .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)),
663 .cdw11 = cpu_to_le32(0x1 | (n << 16)),
664 };
665 if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
666 error_setg(errp, "Failed to create SQ io queue [%d]", n);
667 goto out_error;
668 }
669 s->queues = g_renew(NVMeQueuePair *, s->queues, n + 1);
670 s->queues[n] = q;
671 s->nr_queues++;
672 return true;
673 out_error:
674 nvme_free_queue_pair(q);
675 return false;
676 }
677
678 static bool nvme_poll_cb(void *opaque)
679 {
680 EventNotifier *e = opaque;
681 BDRVNVMeState *s = container_of(e, BDRVNVMeState,
682 irq_notifier[MSIX_SHARED_IRQ_IDX]);
683
684 trace_nvme_poll_cb(s);
685 return nvme_poll_queues(s);
686 }
687
688 static int nvme_init(BlockDriverState *bs, const char *device, int namespace,
689 Error **errp)
690 {
691 BDRVNVMeState *s = bs->opaque;
692 AioContext *aio_context = bdrv_get_aio_context(bs);
693 int ret;
694 uint64_t cap;
695 uint64_t timeout_ms;
696 uint64_t deadline, now;
697 Error *local_err = NULL;
698
699 qemu_co_mutex_init(&s->dma_map_lock);
700 qemu_co_queue_init(&s->dma_flush_queue);
701 s->device = g_strdup(device);
702 s->nsid = namespace;
703 s->aio_context = bdrv_get_aio_context(bs);
704 ret = event_notifier_init(&s->irq_notifier[MSIX_SHARED_IRQ_IDX], 0);
705 if (ret) {
706 error_setg(errp, "Failed to init event notifier");
707 return ret;
708 }
709
710 s->vfio = qemu_vfio_open_pci(device, errp);
711 if (!s->vfio) {
712 ret = -EINVAL;
713 goto out;
714 }
715
716 s->regs = qemu_vfio_pci_map_bar(s->vfio, 0, 0, sizeof(NvmeBar),
717 PROT_READ | PROT_WRITE, errp);
718 if (!s->regs) {
719 ret = -EINVAL;
720 goto out;
721 }
722 /* Perform initialize sequence as described in NVMe spec "7.6.1
723 * Initialization". */
724
725 cap = le64_to_cpu(s->regs->ctrl.cap);
726 if (!(cap & (1ULL << 37))) {
727 error_setg(errp, "Device doesn't support NVMe command set");
728 ret = -EINVAL;
729 goto out;
730 }
731
732 s->page_size = MAX(4096, 1 << (12 + ((cap >> 48) & 0xF)));
733 s->doorbell_scale = (4 << (((cap >> 32) & 0xF))) / sizeof(uint32_t);
734 bs->bl.opt_mem_alignment = s->page_size;
735 timeout_ms = MIN(500 * ((cap >> 24) & 0xFF), 30000);
736
737 /* Reset device to get a clean state. */
738 s->regs->ctrl.cc = cpu_to_le32(le32_to_cpu(s->regs->ctrl.cc) & 0xFE);
739 /* Wait for CSTS.RDY = 0. */
740 deadline = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + timeout_ms * SCALE_MS;
741 while (le32_to_cpu(s->regs->ctrl.csts) & 0x1) {
742 if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
743 error_setg(errp, "Timeout while waiting for device to reset (%"
744 PRId64 " ms)",
745 timeout_ms);
746 ret = -ETIMEDOUT;
747 goto out;
748 }
749 }
750
751 s->doorbells = qemu_vfio_pci_map_bar(s->vfio, 0, sizeof(NvmeBar),
752 NVME_DOORBELL_SIZE, PROT_WRITE, errp);
753 if (!s->doorbells) {
754 ret = -EINVAL;
755 goto out;
756 }
757
758 /* Set up admin queue. */
759 s->queues = g_new(NVMeQueuePair *, 1);
760 s->queues[INDEX_ADMIN] = nvme_create_queue_pair(s, aio_context, 0,
761 NVME_QUEUE_SIZE,
762 errp);
763 if (!s->queues[INDEX_ADMIN]) {
764 ret = -EINVAL;
765 goto out;
766 }
767 s->nr_queues = 1;
768 QEMU_BUILD_BUG_ON(NVME_QUEUE_SIZE & 0xF000);
769 s->regs->ctrl.aqa = cpu_to_le32((NVME_QUEUE_SIZE << 16) | NVME_QUEUE_SIZE);
770 s->regs->ctrl.asq = cpu_to_le64(s->queues[INDEX_ADMIN]->sq.iova);
771 s->regs->ctrl.acq = cpu_to_le64(s->queues[INDEX_ADMIN]->cq.iova);
772
773 /* After setting up all control registers we can enable device now. */
774 s->regs->ctrl.cc = cpu_to_le32((ctz32(NVME_CQ_ENTRY_BYTES) << 20) |
775 (ctz32(NVME_SQ_ENTRY_BYTES) << 16) |
776 0x1);
777 /* Wait for CSTS.RDY = 1. */
778 now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
779 deadline = now + timeout_ms * 1000000;
780 while (!(le32_to_cpu(s->regs->ctrl.csts) & 0x1)) {
781 if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
782 error_setg(errp, "Timeout while waiting for device to start (%"
783 PRId64 " ms)",
784 timeout_ms);
785 ret = -ETIMEDOUT;
786 goto out;
787 }
788 }
789
790 ret = qemu_vfio_pci_init_irq(s->vfio, s->irq_notifier,
791 VFIO_PCI_MSIX_IRQ_INDEX, errp);
792 if (ret) {
793 goto out;
794 }
795 aio_set_event_notifier(bdrv_get_aio_context(bs),
796 &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
797 false, nvme_handle_event, nvme_poll_cb);
798
799 nvme_identify(bs, namespace, &local_err);
800 if (local_err) {
801 error_propagate(errp, local_err);
802 ret = -EIO;
803 goto out;
804 }
805
806 /* Set up command queues. */
807 if (!nvme_add_io_queue(bs, errp)) {
808 ret = -EIO;
809 }
810 out:
811 /* Cleaning up is done in nvme_file_open() upon error. */
812 return ret;
813 }
814
815 /* Parse a filename in the format of nvme://XXXX:XX:XX.X/X. Example:
816 *
817 * nvme://0000:44:00.0/1
818 *
819 * where the "nvme://" is a fixed form of the protocol prefix, the middle part
820 * is the PCI address, and the last part is the namespace number starting from
821 * 1 according to the NVMe spec. */
822 static void nvme_parse_filename(const char *filename, QDict *options,
823 Error **errp)
824 {
825 int pref = strlen("nvme://");
826
827 if (strlen(filename) > pref && !strncmp(filename, "nvme://", pref)) {
828 const char *tmp = filename + pref;
829 char *device;
830 const char *namespace;
831 unsigned long ns;
832 const char *slash = strchr(tmp, '/');
833 if (!slash) {
834 qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, tmp);
835 return;
836 }
837 device = g_strndup(tmp, slash - tmp);
838 qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, device);
839 g_free(device);
840 namespace = slash + 1;
841 if (*namespace && qemu_strtoul(namespace, NULL, 10, &ns)) {
842 error_setg(errp, "Invalid namespace '%s', positive number expected",
843 namespace);
844 return;
845 }
846 qdict_put_str(options, NVME_BLOCK_OPT_NAMESPACE,
847 *namespace ? namespace : "1");
848 }
849 }
850
851 static int nvme_enable_disable_write_cache(BlockDriverState *bs, bool enable,
852 Error **errp)
853 {
854 int ret;
855 BDRVNVMeState *s = bs->opaque;
856 NvmeCmd cmd = {
857 .opcode = NVME_ADM_CMD_SET_FEATURES,
858 .nsid = cpu_to_le32(s->nsid),
859 .cdw10 = cpu_to_le32(0x06),
860 .cdw11 = cpu_to_le32(enable ? 0x01 : 0x00),
861 };
862
863 ret = nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd);
864 if (ret) {
865 error_setg(errp, "Failed to configure NVMe write cache");
866 }
867 return ret;
868 }
869
870 static void nvme_close(BlockDriverState *bs)
871 {
872 int i;
873 BDRVNVMeState *s = bs->opaque;
874
875 for (i = 0; i < s->nr_queues; ++i) {
876 nvme_free_queue_pair(s->queues[i]);
877 }
878 g_free(s->queues);
879 aio_set_event_notifier(bdrv_get_aio_context(bs),
880 &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
881 false, NULL, NULL);
882 event_notifier_cleanup(&s->irq_notifier[MSIX_SHARED_IRQ_IDX]);
883 qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)s->doorbells,
884 sizeof(NvmeBar), NVME_DOORBELL_SIZE);
885 qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)s->regs, 0, sizeof(NvmeBar));
886 qemu_vfio_close(s->vfio);
887
888 g_free(s->device);
889 }
890
891 static int nvme_file_open(BlockDriverState *bs, QDict *options, int flags,
892 Error **errp)
893 {
894 const char *device;
895 QemuOpts *opts;
896 int namespace;
897 int ret;
898 BDRVNVMeState *s = bs->opaque;
899
900 bs->supported_write_flags = BDRV_REQ_FUA;
901
902 opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
903 qemu_opts_absorb_qdict(opts, options, &error_abort);
904 device = qemu_opt_get(opts, NVME_BLOCK_OPT_DEVICE);
905 if (!device) {
906 error_setg(errp, "'" NVME_BLOCK_OPT_DEVICE "' option is required");
907 qemu_opts_del(opts);
908 return -EINVAL;
909 }
910
911 namespace = qemu_opt_get_number(opts, NVME_BLOCK_OPT_NAMESPACE, 1);
912 ret = nvme_init(bs, device, namespace, errp);
913 qemu_opts_del(opts);
914 if (ret) {
915 goto fail;
916 }
917 if (flags & BDRV_O_NOCACHE) {
918 if (!s->write_cache_supported) {
919 error_setg(errp,
920 "NVMe controller doesn't support write cache configuration");
921 ret = -EINVAL;
922 } else {
923 ret = nvme_enable_disable_write_cache(bs, !(flags & BDRV_O_NOCACHE),
924 errp);
925 }
926 if (ret) {
927 goto fail;
928 }
929 }
930 return 0;
931 fail:
932 nvme_close(bs);
933 return ret;
934 }
935
936 static int64_t nvme_getlength(BlockDriverState *bs)
937 {
938 BDRVNVMeState *s = bs->opaque;
939 return s->nsze << s->blkshift;
940 }
941
942 static uint32_t nvme_get_blocksize(BlockDriverState *bs)
943 {
944 BDRVNVMeState *s = bs->opaque;
945 assert(s->blkshift >= BDRV_SECTOR_BITS && s->blkshift <= 12);
946 return UINT32_C(1) << s->blkshift;
947 }
948
949 static int nvme_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz)
950 {
951 uint32_t blocksize = nvme_get_blocksize(bs);
952 bsz->phys = blocksize;
953 bsz->log = blocksize;
954 return 0;
955 }
956
957 /* Called with s->dma_map_lock */
958 static coroutine_fn int nvme_cmd_unmap_qiov(BlockDriverState *bs,
959 QEMUIOVector *qiov)
960 {
961 int r = 0;
962 BDRVNVMeState *s = bs->opaque;
963
964 s->dma_map_count -= qiov->size;
965 if (!s->dma_map_count && !qemu_co_queue_empty(&s->dma_flush_queue)) {
966 r = qemu_vfio_dma_reset_temporary(s->vfio);
967 if (!r) {
968 qemu_co_queue_restart_all(&s->dma_flush_queue);
969 }
970 }
971 return r;
972 }
973
974 /* Called with s->dma_map_lock */
975 static coroutine_fn int nvme_cmd_map_qiov(BlockDriverState *bs, NvmeCmd *cmd,
976 NVMeRequest *req, QEMUIOVector *qiov)
977 {
978 BDRVNVMeState *s = bs->opaque;
979 uint64_t *pagelist = req->prp_list_page;
980 int i, j, r;
981 int entries = 0;
982
983 assert(qiov->size);
984 assert(QEMU_IS_ALIGNED(qiov->size, s->page_size));
985 assert(qiov->size / s->page_size <= s->page_size / sizeof(uint64_t));
986 for (i = 0; i < qiov->niov; ++i) {
987 bool retry = true;
988 uint64_t iova;
989 try_map:
990 r = qemu_vfio_dma_map(s->vfio,
991 qiov->iov[i].iov_base,
992 qiov->iov[i].iov_len,
993 true, &iova);
994 if (r == -ENOMEM && retry) {
995 retry = false;
996 trace_nvme_dma_flush_queue_wait(s);
997 if (s->dma_map_count) {
998 trace_nvme_dma_map_flush(s);
999 qemu_co_queue_wait(&s->dma_flush_queue, &s->dma_map_lock);
1000 } else {
1001 r = qemu_vfio_dma_reset_temporary(s->vfio);
1002 if (r) {
1003 goto fail;
1004 }
1005 }
1006 goto try_map;
1007 }
1008 if (r) {
1009 goto fail;
1010 }
1011
1012 for (j = 0; j < qiov->iov[i].iov_len / s->page_size; j++) {
1013 pagelist[entries++] = cpu_to_le64(iova + j * s->page_size);
1014 }
1015 trace_nvme_cmd_map_qiov_iov(s, i, qiov->iov[i].iov_base,
1016 qiov->iov[i].iov_len / s->page_size);
1017 }
1018
1019 s->dma_map_count += qiov->size;
1020
1021 assert(entries <= s->page_size / sizeof(uint64_t));
1022 switch (entries) {
1023 case 0:
1024 abort();
1025 case 1:
1026 cmd->dptr.prp1 = pagelist[0];
1027 cmd->dptr.prp2 = 0;
1028 break;
1029 case 2:
1030 cmd->dptr.prp1 = pagelist[0];
1031 cmd->dptr.prp2 = pagelist[1];
1032 break;
1033 default:
1034 cmd->dptr.prp1 = pagelist[0];
1035 cmd->dptr.prp2 = cpu_to_le64(req->prp_list_iova + sizeof(uint64_t));
1036 break;
1037 }
1038 trace_nvme_cmd_map_qiov(s, cmd, req, qiov, entries);
1039 for (i = 0; i < entries; ++i) {
1040 trace_nvme_cmd_map_qiov_pages(s, i, pagelist[i]);
1041 }
1042 return 0;
1043 fail:
1044 /* No need to unmap [0 - i) iovs even if we've failed, since we don't
1045 * increment s->dma_map_count. This is okay for fixed mapping memory areas
1046 * because they are already mapped before calling this function; for
1047 * temporary mappings, a later nvme_cmd_(un)map_qiov will reclaim by
1048 * calling qemu_vfio_dma_reset_temporary when necessary. */
1049 return r;
1050 }
1051
1052 typedef struct {
1053 Coroutine *co;
1054 int ret;
1055 AioContext *ctx;
1056 } NVMeCoData;
1057
1058 static void nvme_rw_cb_bh(void *opaque)
1059 {
1060 NVMeCoData *data = opaque;
1061 qemu_coroutine_enter(data->co);
1062 }
1063
1064 static void nvme_rw_cb(void *opaque, int ret)
1065 {
1066 NVMeCoData *data = opaque;
1067 data->ret = ret;
1068 if (!data->co) {
1069 /* The rw coroutine hasn't yielded, don't try to enter. */
1070 return;
1071 }
1072 replay_bh_schedule_oneshot_event(data->ctx, nvme_rw_cb_bh, data);
1073 }
1074
1075 static coroutine_fn int nvme_co_prw_aligned(BlockDriverState *bs,
1076 uint64_t offset, uint64_t bytes,
1077 QEMUIOVector *qiov,
1078 bool is_write,
1079 int flags)
1080 {
1081 int r;
1082 BDRVNVMeState *s = bs->opaque;
1083 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1084 NVMeRequest *req;
1085
1086 uint32_t cdw12 = (((bytes >> s->blkshift) - 1) & 0xFFFF) |
1087 (flags & BDRV_REQ_FUA ? 1 << 30 : 0);
1088 NvmeCmd cmd = {
1089 .opcode = is_write ? NVME_CMD_WRITE : NVME_CMD_READ,
1090 .nsid = cpu_to_le32(s->nsid),
1091 .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
1092 .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
1093 .cdw12 = cpu_to_le32(cdw12),
1094 };
1095 NVMeCoData data = {
1096 .ctx = bdrv_get_aio_context(bs),
1097 .ret = -EINPROGRESS,
1098 };
1099
1100 trace_nvme_prw_aligned(s, is_write, offset, bytes, flags, qiov->niov);
1101 assert(s->nr_queues > 1);
1102 req = nvme_get_free_req(ioq);
1103 assert(req);
1104
1105 qemu_co_mutex_lock(&s->dma_map_lock);
1106 r = nvme_cmd_map_qiov(bs, &cmd, req, qiov);
1107 qemu_co_mutex_unlock(&s->dma_map_lock);
1108 if (r) {
1109 nvme_put_free_req_and_wake(ioq, req);
1110 return r;
1111 }
1112 nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1113
1114 data.co = qemu_coroutine_self();
1115 while (data.ret == -EINPROGRESS) {
1116 qemu_coroutine_yield();
1117 }
1118
1119 qemu_co_mutex_lock(&s->dma_map_lock);
1120 r = nvme_cmd_unmap_qiov(bs, qiov);
1121 qemu_co_mutex_unlock(&s->dma_map_lock);
1122 if (r) {
1123 return r;
1124 }
1125
1126 trace_nvme_rw_done(s, is_write, offset, bytes, data.ret);
1127 return data.ret;
1128 }
1129
1130 static inline bool nvme_qiov_aligned(BlockDriverState *bs,
1131 const QEMUIOVector *qiov)
1132 {
1133 int i;
1134 BDRVNVMeState *s = bs->opaque;
1135
1136 for (i = 0; i < qiov->niov; ++i) {
1137 if (!QEMU_PTR_IS_ALIGNED(qiov->iov[i].iov_base, s->page_size) ||
1138 !QEMU_IS_ALIGNED(qiov->iov[i].iov_len, s->page_size)) {
1139 trace_nvme_qiov_unaligned(qiov, i, qiov->iov[i].iov_base,
1140 qiov->iov[i].iov_len, s->page_size);
1141 return false;
1142 }
1143 }
1144 return true;
1145 }
1146
1147 static int nvme_co_prw(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
1148 QEMUIOVector *qiov, bool is_write, int flags)
1149 {
1150 BDRVNVMeState *s = bs->opaque;
1151 int r;
1152 uint8_t *buf = NULL;
1153 QEMUIOVector local_qiov;
1154
1155 assert(QEMU_IS_ALIGNED(offset, s->page_size));
1156 assert(QEMU_IS_ALIGNED(bytes, s->page_size));
1157 assert(bytes <= s->max_transfer);
1158 if (nvme_qiov_aligned(bs, qiov)) {
1159 return nvme_co_prw_aligned(bs, offset, bytes, qiov, is_write, flags);
1160 }
1161 trace_nvme_prw_buffered(s, offset, bytes, qiov->niov, is_write);
1162 buf = qemu_try_memalign(s->page_size, bytes);
1163
1164 if (!buf) {
1165 return -ENOMEM;
1166 }
1167 qemu_iovec_init(&local_qiov, 1);
1168 if (is_write) {
1169 qemu_iovec_to_buf(qiov, 0, buf, bytes);
1170 }
1171 qemu_iovec_add(&local_qiov, buf, bytes);
1172 r = nvme_co_prw_aligned(bs, offset, bytes, &local_qiov, is_write, flags);
1173 qemu_iovec_destroy(&local_qiov);
1174 if (!r && !is_write) {
1175 qemu_iovec_from_buf(qiov, 0, buf, bytes);
1176 }
1177 qemu_vfree(buf);
1178 return r;
1179 }
1180
1181 static coroutine_fn int nvme_co_preadv(BlockDriverState *bs,
1182 uint64_t offset, uint64_t bytes,
1183 QEMUIOVector *qiov, int flags)
1184 {
1185 return nvme_co_prw(bs, offset, bytes, qiov, false, flags);
1186 }
1187
1188 static coroutine_fn int nvme_co_pwritev(BlockDriverState *bs,
1189 uint64_t offset, uint64_t bytes,
1190 QEMUIOVector *qiov, int flags)
1191 {
1192 return nvme_co_prw(bs, offset, bytes, qiov, true, flags);
1193 }
1194
1195 static coroutine_fn int nvme_co_flush(BlockDriverState *bs)
1196 {
1197 BDRVNVMeState *s = bs->opaque;
1198 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1199 NVMeRequest *req;
1200 NvmeCmd cmd = {
1201 .opcode = NVME_CMD_FLUSH,
1202 .nsid = cpu_to_le32(s->nsid),
1203 };
1204 NVMeCoData data = {
1205 .ctx = bdrv_get_aio_context(bs),
1206 .ret = -EINPROGRESS,
1207 };
1208
1209 assert(s->nr_queues > 1);
1210 req = nvme_get_free_req(ioq);
1211 assert(req);
1212 nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1213
1214 data.co = qemu_coroutine_self();
1215 if (data.ret == -EINPROGRESS) {
1216 qemu_coroutine_yield();
1217 }
1218
1219 return data.ret;
1220 }
1221
1222
1223 static coroutine_fn int nvme_co_pwrite_zeroes(BlockDriverState *bs,
1224 int64_t offset,
1225 int bytes,
1226 BdrvRequestFlags flags)
1227 {
1228 BDRVNVMeState *s = bs->opaque;
1229 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1230 NVMeRequest *req;
1231
1232 uint32_t cdw12 = ((bytes >> s->blkshift) - 1) & 0xFFFF;
1233
1234 if (!s->supports_write_zeroes) {
1235 return -ENOTSUP;
1236 }
1237
1238 NvmeCmd cmd = {
1239 .opcode = NVME_CMD_WRITE_ZEROES,
1240 .nsid = cpu_to_le32(s->nsid),
1241 .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
1242 .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
1243 };
1244
1245 NVMeCoData data = {
1246 .ctx = bdrv_get_aio_context(bs),
1247 .ret = -EINPROGRESS,
1248 };
1249
1250 if (flags & BDRV_REQ_MAY_UNMAP) {
1251 cdw12 |= (1 << 25);
1252 }
1253
1254 if (flags & BDRV_REQ_FUA) {
1255 cdw12 |= (1 << 30);
1256 }
1257
1258 cmd.cdw12 = cpu_to_le32(cdw12);
1259
1260 trace_nvme_write_zeroes(s, offset, bytes, flags);
1261 assert(s->nr_queues > 1);
1262 req = nvme_get_free_req(ioq);
1263 assert(req);
1264
1265 nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1266
1267 data.co = qemu_coroutine_self();
1268 while (data.ret == -EINPROGRESS) {
1269 qemu_coroutine_yield();
1270 }
1271
1272 trace_nvme_rw_done(s, true, offset, bytes, data.ret);
1273 return data.ret;
1274 }
1275
1276
1277 static int coroutine_fn nvme_co_pdiscard(BlockDriverState *bs,
1278 int64_t offset,
1279 int bytes)
1280 {
1281 BDRVNVMeState *s = bs->opaque;
1282 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1283 NVMeRequest *req;
1284 NvmeDsmRange *buf;
1285 QEMUIOVector local_qiov;
1286 int ret;
1287
1288 NvmeCmd cmd = {
1289 .opcode = NVME_CMD_DSM,
1290 .nsid = cpu_to_le32(s->nsid),
1291 .cdw10 = cpu_to_le32(0), /*number of ranges - 0 based*/
1292 .cdw11 = cpu_to_le32(1 << 2), /*deallocate bit*/
1293 };
1294
1295 NVMeCoData data = {
1296 .ctx = bdrv_get_aio_context(bs),
1297 .ret = -EINPROGRESS,
1298 };
1299
1300 if (!s->supports_discard) {
1301 return -ENOTSUP;
1302 }
1303
1304 assert(s->nr_queues > 1);
1305
1306 buf = qemu_try_memalign(s->page_size, s->page_size);
1307 if (!buf) {
1308 return -ENOMEM;
1309 }
1310 memset(buf, 0, s->page_size);
1311 buf->nlb = cpu_to_le32(bytes >> s->blkshift);
1312 buf->slba = cpu_to_le64(offset >> s->blkshift);
1313 buf->cattr = 0;
1314
1315 qemu_iovec_init(&local_qiov, 1);
1316 qemu_iovec_add(&local_qiov, buf, 4096);
1317
1318 req = nvme_get_free_req(ioq);
1319 assert(req);
1320
1321 qemu_co_mutex_lock(&s->dma_map_lock);
1322 ret = nvme_cmd_map_qiov(bs, &cmd, req, &local_qiov);
1323 qemu_co_mutex_unlock(&s->dma_map_lock);
1324
1325 if (ret) {
1326 nvme_put_free_req_and_wake(ioq, req);
1327 goto out;
1328 }
1329
1330 trace_nvme_dsm(s, offset, bytes);
1331
1332 nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1333
1334 data.co = qemu_coroutine_self();
1335 while (data.ret == -EINPROGRESS) {
1336 qemu_coroutine_yield();
1337 }
1338
1339 qemu_co_mutex_lock(&s->dma_map_lock);
1340 ret = nvme_cmd_unmap_qiov(bs, &local_qiov);
1341 qemu_co_mutex_unlock(&s->dma_map_lock);
1342
1343 if (ret) {
1344 goto out;
1345 }
1346
1347 ret = data.ret;
1348 trace_nvme_dsm_done(s, offset, bytes, ret);
1349 out:
1350 qemu_iovec_destroy(&local_qiov);
1351 qemu_vfree(buf);
1352 return ret;
1353
1354 }
1355
1356
1357 static int nvme_reopen_prepare(BDRVReopenState *reopen_state,
1358 BlockReopenQueue *queue, Error **errp)
1359 {
1360 return 0;
1361 }
1362
1363 static void nvme_refresh_filename(BlockDriverState *bs)
1364 {
1365 BDRVNVMeState *s = bs->opaque;
1366
1367 snprintf(bs->exact_filename, sizeof(bs->exact_filename), "nvme://%s/%i",
1368 s->device, s->nsid);
1369 }
1370
1371 static void nvme_refresh_limits(BlockDriverState *bs, Error **errp)
1372 {
1373 BDRVNVMeState *s = bs->opaque;
1374
1375 bs->bl.opt_mem_alignment = s->page_size;
1376 bs->bl.request_alignment = s->page_size;
1377 bs->bl.max_transfer = s->max_transfer;
1378 }
1379
1380 static void nvme_detach_aio_context(BlockDriverState *bs)
1381 {
1382 BDRVNVMeState *s = bs->opaque;
1383
1384 for (int i = 0; i < s->nr_queues; i++) {
1385 NVMeQueuePair *q = s->queues[i];
1386
1387 qemu_bh_delete(q->completion_bh);
1388 q->completion_bh = NULL;
1389 }
1390
1391 aio_set_event_notifier(bdrv_get_aio_context(bs),
1392 &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
1393 false, NULL, NULL);
1394 }
1395
1396 static void nvme_attach_aio_context(BlockDriverState *bs,
1397 AioContext *new_context)
1398 {
1399 BDRVNVMeState *s = bs->opaque;
1400
1401 s->aio_context = new_context;
1402 aio_set_event_notifier(new_context, &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
1403 false, nvme_handle_event, nvme_poll_cb);
1404
1405 for (int i = 0; i < s->nr_queues; i++) {
1406 NVMeQueuePair *q = s->queues[i];
1407
1408 q->completion_bh =
1409 aio_bh_new(new_context, nvme_process_completion_bh, q);
1410 }
1411 }
1412
1413 static void nvme_aio_plug(BlockDriverState *bs)
1414 {
1415 BDRVNVMeState *s = bs->opaque;
1416 assert(!s->plugged);
1417 s->plugged = true;
1418 }
1419
1420 static void nvme_aio_unplug(BlockDriverState *bs)
1421 {
1422 int i;
1423 BDRVNVMeState *s = bs->opaque;
1424 assert(s->plugged);
1425 s->plugged = false;
1426 for (i = INDEX_IO(0); i < s->nr_queues; i++) {
1427 NVMeQueuePair *q = s->queues[i];
1428 qemu_mutex_lock(&q->lock);
1429 nvme_kick(q);
1430 nvme_process_completion(q);
1431 qemu_mutex_unlock(&q->lock);
1432 }
1433 }
1434
1435 static void nvme_register_buf(BlockDriverState *bs, void *host, size_t size)
1436 {
1437 int ret;
1438 BDRVNVMeState *s = bs->opaque;
1439
1440 ret = qemu_vfio_dma_map(s->vfio, host, size, false, NULL);
1441 if (ret) {
1442 /* FIXME: we may run out of IOVA addresses after repeated
1443 * bdrv_register_buf/bdrv_unregister_buf, because nvme_vfio_dma_unmap
1444 * doesn't reclaim addresses for fixed mappings. */
1445 error_report("nvme_register_buf failed: %s", strerror(-ret));
1446 }
1447 }
1448
1449 static void nvme_unregister_buf(BlockDriverState *bs, void *host)
1450 {
1451 BDRVNVMeState *s = bs->opaque;
1452
1453 qemu_vfio_dma_unmap(s->vfio, host);
1454 }
1455
1456 static const char *const nvme_strong_runtime_opts[] = {
1457 NVME_BLOCK_OPT_DEVICE,
1458 NVME_BLOCK_OPT_NAMESPACE,
1459
1460 NULL
1461 };
1462
1463 static BlockDriver bdrv_nvme = {
1464 .format_name = "nvme",
1465 .protocol_name = "nvme",
1466 .instance_size = sizeof(BDRVNVMeState),
1467
1468 .bdrv_co_create_opts = bdrv_co_create_opts_simple,
1469 .create_opts = &bdrv_create_opts_simple,
1470
1471 .bdrv_parse_filename = nvme_parse_filename,
1472 .bdrv_file_open = nvme_file_open,
1473 .bdrv_close = nvme_close,
1474 .bdrv_getlength = nvme_getlength,
1475 .bdrv_probe_blocksizes = nvme_probe_blocksizes,
1476
1477 .bdrv_co_preadv = nvme_co_preadv,
1478 .bdrv_co_pwritev = nvme_co_pwritev,
1479
1480 .bdrv_co_pwrite_zeroes = nvme_co_pwrite_zeroes,
1481 .bdrv_co_pdiscard = nvme_co_pdiscard,
1482
1483 .bdrv_co_flush_to_disk = nvme_co_flush,
1484 .bdrv_reopen_prepare = nvme_reopen_prepare,
1485
1486 .bdrv_refresh_filename = nvme_refresh_filename,
1487 .bdrv_refresh_limits = nvme_refresh_limits,
1488 .strong_runtime_opts = nvme_strong_runtime_opts,
1489
1490 .bdrv_detach_aio_context = nvme_detach_aio_context,
1491 .bdrv_attach_aio_context = nvme_attach_aio_context,
1492
1493 .bdrv_io_plug = nvme_aio_plug,
1494 .bdrv_io_unplug = nvme_aio_unplug,
1495
1496 .bdrv_register_buf = nvme_register_buf,
1497 .bdrv_unregister_buf = nvme_unregister_buf,
1498 };
1499
1500 static void bdrv_nvme_init(void)
1501 {
1502 bdrv_register(&bdrv_nvme);
1503 }
1504
1505 block_init(bdrv_nvme_init);