hw/block/nvme: bump supported version to v1.3
[qemu.git] / hw / block / nvme.c
1 /*
2 * QEMU NVM Express Controller
3 *
4 * Copyright (c) 2012, Intel Corporation
5 *
6 * Written by Keith Busch <keith.busch@intel.com>
7 *
8 * This code is licensed under the GNU GPL v2 or later.
9 */
10
11 /**
12 * Reference Specs: http://www.nvmexpress.org, 1.2, 1.1, 1.0e
13 *
14 * https://nvmexpress.org/developers/nvme-specification/
15 */
16
17 /**
18 * Usage: add options:
19 * -drive file=<file>,if=none,id=<drive_id>
20 * -device nvme,drive=<drive_id>,serial=<serial>,id=<id[optional]>, \
21 * cmb_size_mb=<cmb_size_mb[optional]>, \
22 * [pmrdev=<mem_backend_file_id>,] \
23 * max_ioqpairs=<N[optional]>, \
24 * aerl=<N[optional]>, aer_max_queued=<N[optional]>
25 *
26 * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
27 * offset 0 in BAR2 and supports only WDS, RDS and SQS for now.
28 *
29 * cmb_size_mb= and pmrdev= options are mutually exclusive due to limitation
30 * in available BAR's. cmb_size_mb= will take precedence over pmrdev= when
31 * both provided.
32 * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
33 * For example:
34 * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
35 * size=<size> .... -device nvme,...,pmrdev=<mem_id>
36 *
37 *
38 * nvme device parameters
39 * ~~~~~~~~~~~~~~~~~~~~~~
40 * - `aerl`
41 * The Asynchronous Event Request Limit (AERL). Indicates the maximum number
42 * of concurrently outstanding Asynchronous Event Request commands suppoert
43 * by the controller. This is a 0's based value.
44 *
45 * - `aer_max_queued`
46 * This is the maximum number of events that the device will enqueue for
47 * completion when there are no oustanding AERs. When the maximum number of
48 * enqueued events are reached, subsequent events will be dropped.
49 *
50 */
51
52 #include "qemu/osdep.h"
53 #include "qemu/units.h"
54 #include "qemu/error-report.h"
55 #include "hw/block/block.h"
56 #include "hw/pci/msix.h"
57 #include "hw/pci/pci.h"
58 #include "hw/qdev-properties.h"
59 #include "migration/vmstate.h"
60 #include "sysemu/sysemu.h"
61 #include "qapi/error.h"
62 #include "qapi/visitor.h"
63 #include "sysemu/hostmem.h"
64 #include "sysemu/block-backend.h"
65 #include "exec/memory.h"
66 #include "qemu/log.h"
67 #include "qemu/module.h"
68 #include "qemu/cutils.h"
69 #include "trace.h"
70 #include "nvme.h"
71
72 #define NVME_MAX_IOQPAIRS 0xffff
73 #define NVME_DB_SIZE 4
74 #define NVME_SPEC_VER 0x00010300
75 #define NVME_CMB_BIR 2
76 #define NVME_PMR_BIR 2
77 #define NVME_TEMPERATURE 0x143
78 #define NVME_TEMPERATURE_WARNING 0x157
79 #define NVME_TEMPERATURE_CRITICAL 0x175
80 #define NVME_NUM_FW_SLOTS 1
81
82 #define NVME_GUEST_ERR(trace, fmt, ...) \
83 do { \
84 (trace_##trace)(__VA_ARGS__); \
85 qemu_log_mask(LOG_GUEST_ERROR, #trace \
86 " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
87 } while (0)
88
89 static const bool nvme_feature_support[NVME_FID_MAX] = {
90 [NVME_ARBITRATION] = true,
91 [NVME_POWER_MANAGEMENT] = true,
92 [NVME_TEMPERATURE_THRESHOLD] = true,
93 [NVME_ERROR_RECOVERY] = true,
94 [NVME_VOLATILE_WRITE_CACHE] = true,
95 [NVME_NUMBER_OF_QUEUES] = true,
96 [NVME_INTERRUPT_COALESCING] = true,
97 [NVME_INTERRUPT_VECTOR_CONF] = true,
98 [NVME_WRITE_ATOMICITY] = true,
99 [NVME_ASYNCHRONOUS_EVENT_CONF] = true,
100 [NVME_TIMESTAMP] = true,
101 };
102
103 static const uint32_t nvme_feature_cap[NVME_FID_MAX] = {
104 [NVME_TEMPERATURE_THRESHOLD] = NVME_FEAT_CAP_CHANGE,
105 [NVME_VOLATILE_WRITE_CACHE] = NVME_FEAT_CAP_CHANGE,
106 [NVME_NUMBER_OF_QUEUES] = NVME_FEAT_CAP_CHANGE,
107 [NVME_ASYNCHRONOUS_EVENT_CONF] = NVME_FEAT_CAP_CHANGE,
108 [NVME_TIMESTAMP] = NVME_FEAT_CAP_CHANGE,
109 };
110
111 static void nvme_process_sq(void *opaque);
112
113 static uint16_t nvme_cid(NvmeRequest *req)
114 {
115 if (!req) {
116 return 0xffff;
117 }
118
119 return le16_to_cpu(req->cqe.cid);
120 }
121
122 static uint16_t nvme_sqid(NvmeRequest *req)
123 {
124 return le16_to_cpu(req->sq->sqid);
125 }
126
127 static bool nvme_addr_is_cmb(NvmeCtrl *n, hwaddr addr)
128 {
129 hwaddr low = n->ctrl_mem.addr;
130 hwaddr hi = n->ctrl_mem.addr + int128_get64(n->ctrl_mem.size);
131
132 return addr >= low && addr < hi;
133 }
134
135 static void nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
136 {
137 if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr)) {
138 memcpy(buf, (void *)&n->cmbuf[addr - n->ctrl_mem.addr], size);
139 return;
140 }
141
142 pci_dma_read(&n->parent_obj, addr, buf, size);
143 }
144
145 static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
146 {
147 return sqid < n->params.max_ioqpairs + 1 && n->sq[sqid] != NULL ? 0 : -1;
148 }
149
150 static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
151 {
152 return cqid < n->params.max_ioqpairs + 1 && n->cq[cqid] != NULL ? 0 : -1;
153 }
154
155 static void nvme_inc_cq_tail(NvmeCQueue *cq)
156 {
157 cq->tail++;
158 if (cq->tail >= cq->size) {
159 cq->tail = 0;
160 cq->phase = !cq->phase;
161 }
162 }
163
164 static void nvme_inc_sq_head(NvmeSQueue *sq)
165 {
166 sq->head = (sq->head + 1) % sq->size;
167 }
168
169 static uint8_t nvme_cq_full(NvmeCQueue *cq)
170 {
171 return (cq->tail + 1) % cq->size == cq->head;
172 }
173
174 static uint8_t nvme_sq_empty(NvmeSQueue *sq)
175 {
176 return sq->head == sq->tail;
177 }
178
179 static void nvme_irq_check(NvmeCtrl *n)
180 {
181 if (msix_enabled(&(n->parent_obj))) {
182 return;
183 }
184 if (~n->bar.intms & n->irq_status) {
185 pci_irq_assert(&n->parent_obj);
186 } else {
187 pci_irq_deassert(&n->parent_obj);
188 }
189 }
190
191 static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq)
192 {
193 if (cq->irq_enabled) {
194 if (msix_enabled(&(n->parent_obj))) {
195 trace_pci_nvme_irq_msix(cq->vector);
196 msix_notify(&(n->parent_obj), cq->vector);
197 } else {
198 trace_pci_nvme_irq_pin();
199 assert(cq->vector < 32);
200 n->irq_status |= 1 << cq->vector;
201 nvme_irq_check(n);
202 }
203 } else {
204 trace_pci_nvme_irq_masked();
205 }
206 }
207
208 static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
209 {
210 if (cq->irq_enabled) {
211 if (msix_enabled(&(n->parent_obj))) {
212 return;
213 } else {
214 assert(cq->vector < 32);
215 n->irq_status &= ~(1 << cq->vector);
216 nvme_irq_check(n);
217 }
218 }
219 }
220
221 static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,
222 uint64_t prp2, uint32_t len, NvmeCtrl *n)
223 {
224 hwaddr trans_len = n->page_size - (prp1 % n->page_size);
225 trans_len = MIN(len, trans_len);
226 int num_prps = (len >> n->page_bits) + 1;
227
228 if (unlikely(!prp1)) {
229 trace_pci_nvme_err_invalid_prp();
230 return NVME_INVALID_FIELD | NVME_DNR;
231 } else if (n->bar.cmbsz && prp1 >= n->ctrl_mem.addr &&
232 prp1 < n->ctrl_mem.addr + int128_get64(n->ctrl_mem.size)) {
233 qsg->nsg = 0;
234 qemu_iovec_init(iov, num_prps);
235 qemu_iovec_add(iov, (void *)&n->cmbuf[prp1 - n->ctrl_mem.addr], trans_len);
236 } else {
237 pci_dma_sglist_init(qsg, &n->parent_obj, num_prps);
238 qemu_sglist_add(qsg, prp1, trans_len);
239 }
240 len -= trans_len;
241 if (len) {
242 if (unlikely(!prp2)) {
243 trace_pci_nvme_err_invalid_prp2_missing();
244 goto unmap;
245 }
246 if (len > n->page_size) {
247 uint64_t prp_list[n->max_prp_ents];
248 uint32_t nents, prp_trans;
249 int i = 0;
250
251 nents = (len + n->page_size - 1) >> n->page_bits;
252 prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
253 nvme_addr_read(n, prp2, (void *)prp_list, prp_trans);
254 while (len != 0) {
255 uint64_t prp_ent = le64_to_cpu(prp_list[i]);
256
257 if (i == n->max_prp_ents - 1 && len > n->page_size) {
258 if (unlikely(!prp_ent || prp_ent & (n->page_size - 1))) {
259 trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
260 goto unmap;
261 }
262
263 i = 0;
264 nents = (len + n->page_size - 1) >> n->page_bits;
265 prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
266 nvme_addr_read(n, prp_ent, (void *)prp_list,
267 prp_trans);
268 prp_ent = le64_to_cpu(prp_list[i]);
269 }
270
271 if (unlikely(!prp_ent || prp_ent & (n->page_size - 1))) {
272 trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
273 goto unmap;
274 }
275
276 trans_len = MIN(len, n->page_size);
277 if (qsg->nsg){
278 qemu_sglist_add(qsg, prp_ent, trans_len);
279 } else {
280 qemu_iovec_add(iov, (void *)&n->cmbuf[prp_ent - n->ctrl_mem.addr], trans_len);
281 }
282 len -= trans_len;
283 i++;
284 }
285 } else {
286 if (unlikely(prp2 & (n->page_size - 1))) {
287 trace_pci_nvme_err_invalid_prp2_align(prp2);
288 goto unmap;
289 }
290 if (qsg->nsg) {
291 qemu_sglist_add(qsg, prp2, len);
292 } else {
293 qemu_iovec_add(iov, (void *)&n->cmbuf[prp2 - n->ctrl_mem.addr], trans_len);
294 }
295 }
296 }
297 return NVME_SUCCESS;
298
299 unmap:
300 qemu_sglist_destroy(qsg);
301 return NVME_INVALID_FIELD | NVME_DNR;
302 }
303
304 static uint16_t nvme_dma_write_prp(NvmeCtrl *n, uint8_t *ptr, uint32_t len,
305 uint64_t prp1, uint64_t prp2)
306 {
307 QEMUSGList qsg;
308 QEMUIOVector iov;
309 uint16_t status = NVME_SUCCESS;
310
311 if (nvme_map_prp(&qsg, &iov, prp1, prp2, len, n)) {
312 return NVME_INVALID_FIELD | NVME_DNR;
313 }
314 if (qsg.nsg > 0) {
315 if (dma_buf_write(ptr, len, &qsg)) {
316 status = NVME_INVALID_FIELD | NVME_DNR;
317 }
318 qemu_sglist_destroy(&qsg);
319 } else {
320 if (qemu_iovec_to_buf(&iov, 0, ptr, len) != len) {
321 status = NVME_INVALID_FIELD | NVME_DNR;
322 }
323 qemu_iovec_destroy(&iov);
324 }
325 return status;
326 }
327
328 static uint16_t nvme_dma_read_prp(NvmeCtrl *n, uint8_t *ptr, uint32_t len,
329 uint64_t prp1, uint64_t prp2)
330 {
331 QEMUSGList qsg;
332 QEMUIOVector iov;
333 uint16_t status = NVME_SUCCESS;
334
335 trace_pci_nvme_dma_read(prp1, prp2);
336
337 if (nvme_map_prp(&qsg, &iov, prp1, prp2, len, n)) {
338 return NVME_INVALID_FIELD | NVME_DNR;
339 }
340 if (qsg.nsg > 0) {
341 if (unlikely(dma_buf_read(ptr, len, &qsg))) {
342 trace_pci_nvme_err_invalid_dma();
343 status = NVME_INVALID_FIELD | NVME_DNR;
344 }
345 qemu_sglist_destroy(&qsg);
346 } else {
347 if (unlikely(qemu_iovec_from_buf(&iov, 0, ptr, len) != len)) {
348 trace_pci_nvme_err_invalid_dma();
349 status = NVME_INVALID_FIELD | NVME_DNR;
350 }
351 qemu_iovec_destroy(&iov);
352 }
353 return status;
354 }
355
356 static void nvme_post_cqes(void *opaque)
357 {
358 NvmeCQueue *cq = opaque;
359 NvmeCtrl *n = cq->ctrl;
360 NvmeRequest *req, *next;
361
362 QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
363 NvmeSQueue *sq;
364 hwaddr addr;
365
366 if (nvme_cq_full(cq)) {
367 break;
368 }
369
370 QTAILQ_REMOVE(&cq->req_list, req, entry);
371 sq = req->sq;
372 req->cqe.status = cpu_to_le16((req->status << 1) | cq->phase);
373 req->cqe.sq_id = cpu_to_le16(sq->sqid);
374 req->cqe.sq_head = cpu_to_le16(sq->head);
375 addr = cq->dma_addr + cq->tail * n->cqe_size;
376 nvme_inc_cq_tail(cq);
377 pci_dma_write(&n->parent_obj, addr, (void *)&req->cqe,
378 sizeof(req->cqe));
379 QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
380 }
381 if (cq->tail != cq->head) {
382 nvme_irq_assert(n, cq);
383 }
384 }
385
386 static void nvme_enqueue_req_completion(NvmeCQueue *cq, NvmeRequest *req)
387 {
388 assert(cq->cqid == req->sq->cqid);
389 trace_pci_nvme_enqueue_req_completion(nvme_cid(req), cq->cqid,
390 req->status);
391 QTAILQ_REMOVE(&req->sq->out_req_list, req, entry);
392 QTAILQ_INSERT_TAIL(&cq->req_list, req, entry);
393 timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
394 }
395
396 static void nvme_process_aers(void *opaque)
397 {
398 NvmeCtrl *n = opaque;
399 NvmeAsyncEvent *event, *next;
400
401 trace_pci_nvme_process_aers(n->aer_queued);
402
403 QTAILQ_FOREACH_SAFE(event, &n->aer_queue, entry, next) {
404 NvmeRequest *req;
405 NvmeAerResult *result;
406
407 /* can't post cqe if there is nothing to complete */
408 if (!n->outstanding_aers) {
409 trace_pci_nvme_no_outstanding_aers();
410 break;
411 }
412
413 /* ignore if masked (cqe posted, but event not cleared) */
414 if (n->aer_mask & (1 << event->result.event_type)) {
415 trace_pci_nvme_aer_masked(event->result.event_type, n->aer_mask);
416 continue;
417 }
418
419 QTAILQ_REMOVE(&n->aer_queue, event, entry);
420 n->aer_queued--;
421
422 n->aer_mask |= 1 << event->result.event_type;
423 n->outstanding_aers--;
424
425 req = n->aer_reqs[n->outstanding_aers];
426
427 result = (NvmeAerResult *) &req->cqe.result;
428 result->event_type = event->result.event_type;
429 result->event_info = event->result.event_info;
430 result->log_page = event->result.log_page;
431 g_free(event);
432
433 req->status = NVME_SUCCESS;
434
435 trace_pci_nvme_aer_post_cqe(result->event_type, result->event_info,
436 result->log_page);
437
438 nvme_enqueue_req_completion(&n->admin_cq, req);
439 }
440 }
441
442 static void nvme_enqueue_event(NvmeCtrl *n, uint8_t event_type,
443 uint8_t event_info, uint8_t log_page)
444 {
445 NvmeAsyncEvent *event;
446
447 trace_pci_nvme_enqueue_event(event_type, event_info, log_page);
448
449 if (n->aer_queued == n->params.aer_max_queued) {
450 trace_pci_nvme_enqueue_event_noqueue(n->aer_queued);
451 return;
452 }
453
454 event = g_new(NvmeAsyncEvent, 1);
455 event->result = (NvmeAerResult) {
456 .event_type = event_type,
457 .event_info = event_info,
458 .log_page = log_page,
459 };
460
461 QTAILQ_INSERT_TAIL(&n->aer_queue, event, entry);
462 n->aer_queued++;
463
464 nvme_process_aers(n);
465 }
466
467 static void nvme_clear_events(NvmeCtrl *n, uint8_t event_type)
468 {
469 n->aer_mask &= ~(1 << event_type);
470 if (!QTAILQ_EMPTY(&n->aer_queue)) {
471 nvme_process_aers(n);
472 }
473 }
474
475 static void nvme_rw_cb(void *opaque, int ret)
476 {
477 NvmeRequest *req = opaque;
478 NvmeSQueue *sq = req->sq;
479 NvmeCtrl *n = sq->ctrl;
480 NvmeCQueue *cq = n->cq[sq->cqid];
481
482 trace_pci_nvme_rw_cb(nvme_cid(req));
483
484 if (!ret) {
485 block_acct_done(blk_get_stats(n->conf.blk), &req->acct);
486 req->status = NVME_SUCCESS;
487 } else {
488 block_acct_failed(blk_get_stats(n->conf.blk), &req->acct);
489 req->status = NVME_INTERNAL_DEV_ERROR;
490 }
491 if (req->has_sg) {
492 qemu_sglist_destroy(&req->qsg);
493 }
494 nvme_enqueue_req_completion(cq, req);
495 }
496
497 static uint16_t nvme_flush(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
498 NvmeRequest *req)
499 {
500 req->has_sg = false;
501 block_acct_start(blk_get_stats(n->conf.blk), &req->acct, 0,
502 BLOCK_ACCT_FLUSH);
503 req->aiocb = blk_aio_flush(n->conf.blk, nvme_rw_cb, req);
504
505 return NVME_NO_COMPLETE;
506 }
507
508 static uint16_t nvme_write_zeros(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
509 NvmeRequest *req)
510 {
511 NvmeRwCmd *rw = (NvmeRwCmd *)cmd;
512 const uint8_t lba_index = NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas);
513 const uint8_t data_shift = ns->id_ns.lbaf[lba_index].ds;
514 uint64_t slba = le64_to_cpu(rw->slba);
515 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
516 uint64_t offset = slba << data_shift;
517 uint32_t count = nlb << data_shift;
518
519 trace_pci_nvme_write_zeroes(nvme_cid(req), slba, nlb);
520
521 if (unlikely(slba + nlb > ns->id_ns.nsze)) {
522 trace_pci_nvme_err_invalid_lba_range(slba, nlb, ns->id_ns.nsze);
523 return NVME_LBA_RANGE | NVME_DNR;
524 }
525
526 req->has_sg = false;
527 block_acct_start(blk_get_stats(n->conf.blk), &req->acct, 0,
528 BLOCK_ACCT_WRITE);
529 req->aiocb = blk_aio_pwrite_zeroes(n->conf.blk, offset, count,
530 BDRV_REQ_MAY_UNMAP, nvme_rw_cb, req);
531 return NVME_NO_COMPLETE;
532 }
533
534 static uint16_t nvme_rw(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
535 NvmeRequest *req)
536 {
537 NvmeRwCmd *rw = (NvmeRwCmd *)cmd;
538 uint32_t nlb = le32_to_cpu(rw->nlb) + 1;
539 uint64_t slba = le64_to_cpu(rw->slba);
540 uint64_t prp1 = le64_to_cpu(rw->dptr.prp1);
541 uint64_t prp2 = le64_to_cpu(rw->dptr.prp2);
542
543 uint8_t lba_index = NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas);
544 uint8_t data_shift = ns->id_ns.lbaf[lba_index].ds;
545 uint64_t data_size = (uint64_t)nlb << data_shift;
546 uint64_t data_offset = slba << data_shift;
547 int is_write = rw->opcode == NVME_CMD_WRITE ? 1 : 0;
548 enum BlockAcctType acct = is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ;
549
550 trace_pci_nvme_rw(is_write ? "write" : "read", nlb, data_size, slba);
551
552 if (unlikely((slba + nlb) > ns->id_ns.nsze)) {
553 block_acct_invalid(blk_get_stats(n->conf.blk), acct);
554 trace_pci_nvme_err_invalid_lba_range(slba, nlb, ns->id_ns.nsze);
555 return NVME_LBA_RANGE | NVME_DNR;
556 }
557
558 if (nvme_map_prp(&req->qsg, &req->iov, prp1, prp2, data_size, n)) {
559 block_acct_invalid(blk_get_stats(n->conf.blk), acct);
560 return NVME_INVALID_FIELD | NVME_DNR;
561 }
562
563 dma_acct_start(n->conf.blk, &req->acct, &req->qsg, acct);
564 if (req->qsg.nsg > 0) {
565 req->has_sg = true;
566 req->aiocb = is_write ?
567 dma_blk_write(n->conf.blk, &req->qsg, data_offset, BDRV_SECTOR_SIZE,
568 nvme_rw_cb, req) :
569 dma_blk_read(n->conf.blk, &req->qsg, data_offset, BDRV_SECTOR_SIZE,
570 nvme_rw_cb, req);
571 } else {
572 req->has_sg = false;
573 req->aiocb = is_write ?
574 blk_aio_pwritev(n->conf.blk, data_offset, &req->iov, 0, nvme_rw_cb,
575 req) :
576 blk_aio_preadv(n->conf.blk, data_offset, &req->iov, 0, nvme_rw_cb,
577 req);
578 }
579
580 return NVME_NO_COMPLETE;
581 }
582
583 static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
584 {
585 NvmeNamespace *ns;
586 uint32_t nsid = le32_to_cpu(cmd->nsid);
587
588 trace_pci_nvme_io_cmd(nvme_cid(req), nsid, nvme_sqid(req), cmd->opcode);
589
590 if (unlikely(nsid == 0 || nsid > n->num_namespaces)) {
591 trace_pci_nvme_err_invalid_ns(nsid, n->num_namespaces);
592 return NVME_INVALID_NSID | NVME_DNR;
593 }
594
595 ns = &n->namespaces[nsid - 1];
596 switch (cmd->opcode) {
597 case NVME_CMD_FLUSH:
598 return nvme_flush(n, ns, cmd, req);
599 case NVME_CMD_WRITE_ZEROS:
600 return nvme_write_zeros(n, ns, cmd, req);
601 case NVME_CMD_WRITE:
602 case NVME_CMD_READ:
603 return nvme_rw(n, ns, cmd, req);
604 default:
605 trace_pci_nvme_err_invalid_opc(cmd->opcode);
606 return NVME_INVALID_OPCODE | NVME_DNR;
607 }
608 }
609
610 static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)
611 {
612 n->sq[sq->sqid] = NULL;
613 timer_del(sq->timer);
614 timer_free(sq->timer);
615 g_free(sq->io_req);
616 if (sq->sqid) {
617 g_free(sq);
618 }
619 }
620
621 static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeCmd *cmd)
622 {
623 NvmeDeleteQ *c = (NvmeDeleteQ *)cmd;
624 NvmeRequest *req, *next;
625 NvmeSQueue *sq;
626 NvmeCQueue *cq;
627 uint16_t qid = le16_to_cpu(c->qid);
628
629 if (unlikely(!qid || nvme_check_sqid(n, qid))) {
630 trace_pci_nvme_err_invalid_del_sq(qid);
631 return NVME_INVALID_QID | NVME_DNR;
632 }
633
634 trace_pci_nvme_del_sq(qid);
635
636 sq = n->sq[qid];
637 while (!QTAILQ_EMPTY(&sq->out_req_list)) {
638 req = QTAILQ_FIRST(&sq->out_req_list);
639 assert(req->aiocb);
640 blk_aio_cancel(req->aiocb);
641 }
642 if (!nvme_check_cqid(n, sq->cqid)) {
643 cq = n->cq[sq->cqid];
644 QTAILQ_REMOVE(&cq->sq_list, sq, entry);
645
646 nvme_post_cqes(cq);
647 QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
648 if (req->sq == sq) {
649 QTAILQ_REMOVE(&cq->req_list, req, entry);
650 QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
651 }
652 }
653 }
654
655 nvme_free_sq(sq, n);
656 return NVME_SUCCESS;
657 }
658
659 static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
660 uint16_t sqid, uint16_t cqid, uint16_t size)
661 {
662 int i;
663 NvmeCQueue *cq;
664
665 sq->ctrl = n;
666 sq->dma_addr = dma_addr;
667 sq->sqid = sqid;
668 sq->size = size;
669 sq->cqid = cqid;
670 sq->head = sq->tail = 0;
671 sq->io_req = g_new(NvmeRequest, sq->size);
672
673 QTAILQ_INIT(&sq->req_list);
674 QTAILQ_INIT(&sq->out_req_list);
675 for (i = 0; i < sq->size; i++) {
676 sq->io_req[i].sq = sq;
677 QTAILQ_INSERT_TAIL(&(sq->req_list), &sq->io_req[i], entry);
678 }
679 sq->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, nvme_process_sq, sq);
680
681 assert(n->cq[cqid]);
682 cq = n->cq[cqid];
683 QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry);
684 n->sq[sqid] = sq;
685 }
686
687 static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeCmd *cmd)
688 {
689 NvmeSQueue *sq;
690 NvmeCreateSq *c = (NvmeCreateSq *)cmd;
691
692 uint16_t cqid = le16_to_cpu(c->cqid);
693 uint16_t sqid = le16_to_cpu(c->sqid);
694 uint16_t qsize = le16_to_cpu(c->qsize);
695 uint16_t qflags = le16_to_cpu(c->sq_flags);
696 uint64_t prp1 = le64_to_cpu(c->prp1);
697
698 trace_pci_nvme_create_sq(prp1, sqid, cqid, qsize, qflags);
699
700 if (unlikely(!cqid || nvme_check_cqid(n, cqid))) {
701 trace_pci_nvme_err_invalid_create_sq_cqid(cqid);
702 return NVME_INVALID_CQID | NVME_DNR;
703 }
704 if (unlikely(!sqid || !nvme_check_sqid(n, sqid))) {
705 trace_pci_nvme_err_invalid_create_sq_sqid(sqid);
706 return NVME_INVALID_QID | NVME_DNR;
707 }
708 if (unlikely(!qsize || qsize > NVME_CAP_MQES(n->bar.cap))) {
709 trace_pci_nvme_err_invalid_create_sq_size(qsize);
710 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
711 }
712 if (unlikely(!prp1 || prp1 & (n->page_size - 1))) {
713 trace_pci_nvme_err_invalid_create_sq_addr(prp1);
714 return NVME_INVALID_FIELD | NVME_DNR;
715 }
716 if (unlikely(!(NVME_SQ_FLAGS_PC(qflags)))) {
717 trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags));
718 return NVME_INVALID_FIELD | NVME_DNR;
719 }
720 sq = g_malloc0(sizeof(*sq));
721 nvme_init_sq(sq, n, prp1, sqid, cqid, qsize + 1);
722 return NVME_SUCCESS;
723 }
724
725 static uint16_t nvme_smart_info(NvmeCtrl *n, NvmeCmd *cmd, uint8_t rae,
726 uint32_t buf_len, uint64_t off,
727 NvmeRequest *req)
728 {
729 uint64_t prp1 = le64_to_cpu(cmd->dptr.prp1);
730 uint64_t prp2 = le64_to_cpu(cmd->dptr.prp2);
731 uint32_t nsid = le32_to_cpu(cmd->nsid);
732
733 uint32_t trans_len;
734 time_t current_ms;
735 uint64_t units_read = 0, units_written = 0;
736 uint64_t read_commands = 0, write_commands = 0;
737 NvmeSmartLog smart;
738 BlockAcctStats *s;
739
740 if (nsid && nsid != 0xffffffff) {
741 return NVME_INVALID_FIELD | NVME_DNR;
742 }
743
744 s = blk_get_stats(n->conf.blk);
745
746 units_read = s->nr_bytes[BLOCK_ACCT_READ] >> BDRV_SECTOR_BITS;
747 units_written = s->nr_bytes[BLOCK_ACCT_WRITE] >> BDRV_SECTOR_BITS;
748 read_commands = s->nr_ops[BLOCK_ACCT_READ];
749 write_commands = s->nr_ops[BLOCK_ACCT_WRITE];
750
751 if (off > sizeof(smart)) {
752 return NVME_INVALID_FIELD | NVME_DNR;
753 }
754
755 trans_len = MIN(sizeof(smart) - off, buf_len);
756
757 memset(&smart, 0x0, sizeof(smart));
758
759 smart.data_units_read[0] = cpu_to_le64(DIV_ROUND_UP(units_read, 1000));
760 smart.data_units_written[0] = cpu_to_le64(DIV_ROUND_UP(units_written,
761 1000));
762 smart.host_read_commands[0] = cpu_to_le64(read_commands);
763 smart.host_write_commands[0] = cpu_to_le64(write_commands);
764
765 smart.temperature = cpu_to_le16(n->temperature);
766
767 if ((n->temperature >= n->features.temp_thresh_hi) ||
768 (n->temperature <= n->features.temp_thresh_low)) {
769 smart.critical_warning |= NVME_SMART_TEMPERATURE;
770 }
771
772 current_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
773 smart.power_on_hours[0] =
774 cpu_to_le64((((current_ms - n->starttime_ms) / 1000) / 60) / 60);
775
776 if (!rae) {
777 nvme_clear_events(n, NVME_AER_TYPE_SMART);
778 }
779
780 return nvme_dma_read_prp(n, (uint8_t *) &smart + off, trans_len, prp1,
781 prp2);
782 }
783
784 static uint16_t nvme_fw_log_info(NvmeCtrl *n, NvmeCmd *cmd, uint32_t buf_len,
785 uint64_t off, NvmeRequest *req)
786 {
787 uint32_t trans_len;
788 uint64_t prp1 = le64_to_cpu(cmd->dptr.prp1);
789 uint64_t prp2 = le64_to_cpu(cmd->dptr.prp2);
790 NvmeFwSlotInfoLog fw_log = {
791 .afi = 0x1,
792 };
793
794 strpadcpy((char *)&fw_log.frs1, sizeof(fw_log.frs1), "1.0", ' ');
795
796 if (off > sizeof(fw_log)) {
797 return NVME_INVALID_FIELD | NVME_DNR;
798 }
799
800 trans_len = MIN(sizeof(fw_log) - off, buf_len);
801
802 return nvme_dma_read_prp(n, (uint8_t *) &fw_log + off, trans_len, prp1,
803 prp2);
804 }
805
806 static uint16_t nvme_error_info(NvmeCtrl *n, NvmeCmd *cmd, uint8_t rae,
807 uint32_t buf_len, uint64_t off,
808 NvmeRequest *req)
809 {
810 uint32_t trans_len;
811 uint64_t prp1 = le64_to_cpu(cmd->dptr.prp1);
812 uint64_t prp2 = le64_to_cpu(cmd->dptr.prp2);
813 NvmeErrorLog errlog;
814
815 if (!rae) {
816 nvme_clear_events(n, NVME_AER_TYPE_ERROR);
817 }
818
819 if (off > sizeof(errlog)) {
820 return NVME_INVALID_FIELD | NVME_DNR;
821 }
822
823 memset(&errlog, 0x0, sizeof(errlog));
824
825 trans_len = MIN(sizeof(errlog) - off, buf_len);
826
827 return nvme_dma_read_prp(n, (uint8_t *)&errlog, trans_len, prp1, prp2);
828 }
829
830 static uint16_t nvme_get_log(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
831 {
832 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
833 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
834 uint32_t dw12 = le32_to_cpu(cmd->cdw12);
835 uint32_t dw13 = le32_to_cpu(cmd->cdw13);
836 uint8_t lid = dw10 & 0xff;
837 uint8_t lsp = (dw10 >> 8) & 0xf;
838 uint8_t rae = (dw10 >> 15) & 0x1;
839 uint32_t numdl, numdu;
840 uint64_t off, lpol, lpou;
841 size_t len;
842
843 numdl = (dw10 >> 16);
844 numdu = (dw11 & 0xffff);
845 lpol = dw12;
846 lpou = dw13;
847
848 len = (((numdu << 16) | numdl) + 1) << 2;
849 off = (lpou << 32ULL) | lpol;
850
851 if (off & 0x3) {
852 return NVME_INVALID_FIELD | NVME_DNR;
853 }
854
855 trace_pci_nvme_get_log(nvme_cid(req), lid, lsp, rae, len, off);
856
857 switch (lid) {
858 case NVME_LOG_ERROR_INFO:
859 return nvme_error_info(n, cmd, rae, len, off, req);
860 case NVME_LOG_SMART_INFO:
861 return nvme_smart_info(n, cmd, rae, len, off, req);
862 case NVME_LOG_FW_SLOT_INFO:
863 return nvme_fw_log_info(n, cmd, len, off, req);
864 default:
865 trace_pci_nvme_err_invalid_log_page(nvme_cid(req), lid);
866 return NVME_INVALID_FIELD | NVME_DNR;
867 }
868 }
869
870 static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n)
871 {
872 n->cq[cq->cqid] = NULL;
873 timer_del(cq->timer);
874 timer_free(cq->timer);
875 msix_vector_unuse(&n->parent_obj, cq->vector);
876 if (cq->cqid) {
877 g_free(cq);
878 }
879 }
880
881 static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeCmd *cmd)
882 {
883 NvmeDeleteQ *c = (NvmeDeleteQ *)cmd;
884 NvmeCQueue *cq;
885 uint16_t qid = le16_to_cpu(c->qid);
886
887 if (unlikely(!qid || nvme_check_cqid(n, qid))) {
888 trace_pci_nvme_err_invalid_del_cq_cqid(qid);
889 return NVME_INVALID_CQID | NVME_DNR;
890 }
891
892 cq = n->cq[qid];
893 if (unlikely(!QTAILQ_EMPTY(&cq->sq_list))) {
894 trace_pci_nvme_err_invalid_del_cq_notempty(qid);
895 return NVME_INVALID_QUEUE_DEL;
896 }
897 nvme_irq_deassert(n, cq);
898 trace_pci_nvme_del_cq(qid);
899 nvme_free_cq(cq, n);
900 return NVME_SUCCESS;
901 }
902
903 static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,
904 uint16_t cqid, uint16_t vector, uint16_t size, uint16_t irq_enabled)
905 {
906 int ret;
907
908 ret = msix_vector_use(&n->parent_obj, vector);
909 assert(ret == 0);
910 cq->ctrl = n;
911 cq->cqid = cqid;
912 cq->size = size;
913 cq->dma_addr = dma_addr;
914 cq->phase = 1;
915 cq->irq_enabled = irq_enabled;
916 cq->vector = vector;
917 cq->head = cq->tail = 0;
918 QTAILQ_INIT(&cq->req_list);
919 QTAILQ_INIT(&cq->sq_list);
920 n->cq[cqid] = cq;
921 cq->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, nvme_post_cqes, cq);
922 }
923
924 static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeCmd *cmd)
925 {
926 NvmeCQueue *cq;
927 NvmeCreateCq *c = (NvmeCreateCq *)cmd;
928 uint16_t cqid = le16_to_cpu(c->cqid);
929 uint16_t vector = le16_to_cpu(c->irq_vector);
930 uint16_t qsize = le16_to_cpu(c->qsize);
931 uint16_t qflags = le16_to_cpu(c->cq_flags);
932 uint64_t prp1 = le64_to_cpu(c->prp1);
933
934 trace_pci_nvme_create_cq(prp1, cqid, vector, qsize, qflags,
935 NVME_CQ_FLAGS_IEN(qflags) != 0);
936
937 if (unlikely(!cqid || !nvme_check_cqid(n, cqid))) {
938 trace_pci_nvme_err_invalid_create_cq_cqid(cqid);
939 return NVME_INVALID_CQID | NVME_DNR;
940 }
941 if (unlikely(!qsize || qsize > NVME_CAP_MQES(n->bar.cap))) {
942 trace_pci_nvme_err_invalid_create_cq_size(qsize);
943 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
944 }
945 if (unlikely(!prp1)) {
946 trace_pci_nvme_err_invalid_create_cq_addr(prp1);
947 return NVME_INVALID_FIELD | NVME_DNR;
948 }
949 if (unlikely(!msix_enabled(&n->parent_obj) && vector)) {
950 trace_pci_nvme_err_invalid_create_cq_vector(vector);
951 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
952 }
953 if (unlikely(vector >= n->params.msix_qsize)) {
954 trace_pci_nvme_err_invalid_create_cq_vector(vector);
955 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
956 }
957 if (unlikely(!(NVME_CQ_FLAGS_PC(qflags)))) {
958 trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags));
959 return NVME_INVALID_FIELD | NVME_DNR;
960 }
961
962 cq = g_malloc0(sizeof(*cq));
963 nvme_init_cq(cq, n, prp1, cqid, vector, qsize + 1,
964 NVME_CQ_FLAGS_IEN(qflags));
965
966 /*
967 * It is only required to set qs_created when creating a completion queue;
968 * creating a submission queue without a matching completion queue will
969 * fail.
970 */
971 n->qs_created = true;
972 return NVME_SUCCESS;
973 }
974
975 static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeIdentify *c)
976 {
977 uint64_t prp1 = le64_to_cpu(c->prp1);
978 uint64_t prp2 = le64_to_cpu(c->prp2);
979
980 trace_pci_nvme_identify_ctrl();
981
982 return nvme_dma_read_prp(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl),
983 prp1, prp2);
984 }
985
986 static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeIdentify *c)
987 {
988 NvmeNamespace *ns;
989 uint32_t nsid = le32_to_cpu(c->nsid);
990 uint64_t prp1 = le64_to_cpu(c->prp1);
991 uint64_t prp2 = le64_to_cpu(c->prp2);
992
993 trace_pci_nvme_identify_ns(nsid);
994
995 if (unlikely(nsid == 0 || nsid > n->num_namespaces)) {
996 trace_pci_nvme_err_invalid_ns(nsid, n->num_namespaces);
997 return NVME_INVALID_NSID | NVME_DNR;
998 }
999
1000 ns = &n->namespaces[nsid - 1];
1001
1002 return nvme_dma_read_prp(n, (uint8_t *)&ns->id_ns, sizeof(ns->id_ns),
1003 prp1, prp2);
1004 }
1005
1006 static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeIdentify *c)
1007 {
1008 static const int data_len = NVME_IDENTIFY_DATA_SIZE;
1009 uint32_t min_nsid = le32_to_cpu(c->nsid);
1010 uint64_t prp1 = le64_to_cpu(c->prp1);
1011 uint64_t prp2 = le64_to_cpu(c->prp2);
1012 uint32_t *list;
1013 uint16_t ret;
1014 int i, j = 0;
1015
1016 trace_pci_nvme_identify_nslist(min_nsid);
1017
1018 /*
1019 * Both 0xffffffff (NVME_NSID_BROADCAST) and 0xfffffffe are invalid values
1020 * since the Active Namespace ID List should return namespaces with ids
1021 * *higher* than the NSID specified in the command. This is also specified
1022 * in the spec (NVM Express v1.3d, Section 5.15.4).
1023 */
1024 if (min_nsid >= NVME_NSID_BROADCAST - 1) {
1025 return NVME_INVALID_NSID | NVME_DNR;
1026 }
1027
1028 list = g_malloc0(data_len);
1029 for (i = 0; i < n->num_namespaces; i++) {
1030 if (i < min_nsid) {
1031 continue;
1032 }
1033 list[j++] = cpu_to_le32(i + 1);
1034 if (j == data_len / sizeof(uint32_t)) {
1035 break;
1036 }
1037 }
1038 ret = nvme_dma_read_prp(n, (uint8_t *)list, data_len, prp1, prp2);
1039 g_free(list);
1040 return ret;
1041 }
1042
1043 static uint16_t nvme_identify_ns_descr_list(NvmeCtrl *n, NvmeIdentify *c)
1044 {
1045 uint32_t nsid = le32_to_cpu(c->nsid);
1046 uint64_t prp1 = le64_to_cpu(c->prp1);
1047 uint64_t prp2 = le64_to_cpu(c->prp2);
1048
1049 uint8_t list[NVME_IDENTIFY_DATA_SIZE];
1050
1051 struct data {
1052 struct {
1053 NvmeIdNsDescr hdr;
1054 uint8_t v[16];
1055 } uuid;
1056 };
1057
1058 struct data *ns_descrs = (struct data *)list;
1059
1060 trace_pci_nvme_identify_ns_descr_list(nsid);
1061
1062 if (unlikely(nsid == 0 || nsid > n->num_namespaces)) {
1063 trace_pci_nvme_err_invalid_ns(nsid, n->num_namespaces);
1064 return NVME_INVALID_NSID | NVME_DNR;
1065 }
1066
1067 memset(list, 0x0, sizeof(list));
1068
1069 /*
1070 * Because the NGUID and EUI64 fields are 0 in the Identify Namespace data
1071 * structure, a Namespace UUID (nidt = 0x3) must be reported in the
1072 * Namespace Identification Descriptor. Add a very basic Namespace UUID
1073 * here.
1074 */
1075 ns_descrs->uuid.hdr.nidt = NVME_NIDT_UUID;
1076 ns_descrs->uuid.hdr.nidl = NVME_NIDT_UUID_LEN;
1077 stl_be_p(&ns_descrs->uuid.v, nsid);
1078
1079 return nvme_dma_read_prp(n, list, NVME_IDENTIFY_DATA_SIZE, prp1, prp2);
1080 }
1081
1082 static uint16_t nvme_identify(NvmeCtrl *n, NvmeCmd *cmd)
1083 {
1084 NvmeIdentify *c = (NvmeIdentify *)cmd;
1085
1086 switch (le32_to_cpu(c->cns)) {
1087 case NVME_ID_CNS_NS:
1088 return nvme_identify_ns(n, c);
1089 case NVME_ID_CNS_CTRL:
1090 return nvme_identify_ctrl(n, c);
1091 case NVME_ID_CNS_NS_ACTIVE_LIST:
1092 return nvme_identify_nslist(n, c);
1093 case NVME_ID_CNS_NS_DESCR_LIST:
1094 return nvme_identify_ns_descr_list(n, c);
1095 default:
1096 trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns));
1097 return NVME_INVALID_FIELD | NVME_DNR;
1098 }
1099 }
1100
1101 static uint16_t nvme_abort(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
1102 {
1103 uint16_t sqid = le32_to_cpu(cmd->cdw10) & 0xffff;
1104
1105 req->cqe.result = 1;
1106 if (nvme_check_sqid(n, sqid)) {
1107 return NVME_INVALID_FIELD | NVME_DNR;
1108 }
1109
1110 return NVME_SUCCESS;
1111 }
1112
1113 static inline void nvme_set_timestamp(NvmeCtrl *n, uint64_t ts)
1114 {
1115 trace_pci_nvme_setfeat_timestamp(ts);
1116
1117 n->host_timestamp = le64_to_cpu(ts);
1118 n->timestamp_set_qemu_clock_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
1119 }
1120
1121 static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n)
1122 {
1123 uint64_t current_time = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
1124 uint64_t elapsed_time = current_time - n->timestamp_set_qemu_clock_ms;
1125
1126 union nvme_timestamp {
1127 struct {
1128 uint64_t timestamp:48;
1129 uint64_t sync:1;
1130 uint64_t origin:3;
1131 uint64_t rsvd1:12;
1132 };
1133 uint64_t all;
1134 };
1135
1136 union nvme_timestamp ts;
1137 ts.all = 0;
1138
1139 /*
1140 * If the sum of the Timestamp value set by the host and the elapsed
1141 * time exceeds 2^48, the value returned should be reduced modulo 2^48.
1142 */
1143 ts.timestamp = (n->host_timestamp + elapsed_time) & 0xffffffffffff;
1144
1145 /* If the host timestamp is non-zero, set the timestamp origin */
1146 ts.origin = n->host_timestamp ? 0x01 : 0x00;
1147
1148 trace_pci_nvme_getfeat_timestamp(ts.all);
1149
1150 return cpu_to_le64(ts.all);
1151 }
1152
1153 static uint16_t nvme_get_feature_timestamp(NvmeCtrl *n, NvmeCmd *cmd)
1154 {
1155 uint64_t prp1 = le64_to_cpu(cmd->dptr.prp1);
1156 uint64_t prp2 = le64_to_cpu(cmd->dptr.prp2);
1157
1158 uint64_t timestamp = nvme_get_timestamp(n);
1159
1160 return nvme_dma_read_prp(n, (uint8_t *)&timestamp,
1161 sizeof(timestamp), prp1, prp2);
1162 }
1163
1164 static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
1165 {
1166 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
1167 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
1168 uint32_t nsid = le32_to_cpu(cmd->nsid);
1169 uint32_t result;
1170 uint8_t fid = NVME_GETSETFEAT_FID(dw10);
1171 NvmeGetFeatureSelect sel = NVME_GETFEAT_SELECT(dw10);
1172 uint16_t iv;
1173
1174 static const uint32_t nvme_feature_default[NVME_FID_MAX] = {
1175 [NVME_ARBITRATION] = NVME_ARB_AB_NOLIMIT,
1176 };
1177
1178 trace_pci_nvme_getfeat(nvme_cid(req), fid, sel, dw11);
1179
1180 if (!nvme_feature_support[fid]) {
1181 return NVME_INVALID_FIELD | NVME_DNR;
1182 }
1183
1184 if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
1185 if (!nsid || nsid > n->num_namespaces) {
1186 /*
1187 * The Reservation Notification Mask and Reservation Persistence
1188 * features require a status code of Invalid Field in Command when
1189 * NSID is 0xFFFFFFFF. Since the device does not support those
1190 * features we can always return Invalid Namespace or Format as we
1191 * should do for all other features.
1192 */
1193 return NVME_INVALID_NSID | NVME_DNR;
1194 }
1195 }
1196
1197 switch (sel) {
1198 case NVME_GETFEAT_SELECT_CURRENT:
1199 break;
1200 case NVME_GETFEAT_SELECT_SAVED:
1201 /* no features are saveable by the controller; fallthrough */
1202 case NVME_GETFEAT_SELECT_DEFAULT:
1203 goto defaults;
1204 case NVME_GETFEAT_SELECT_CAP:
1205 result = nvme_feature_cap[fid];
1206 goto out;
1207 }
1208
1209 switch (fid) {
1210 case NVME_TEMPERATURE_THRESHOLD:
1211 result = 0;
1212
1213 /*
1214 * The controller only implements the Composite Temperature sensor, so
1215 * return 0 for all other sensors.
1216 */
1217 if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
1218 goto out;
1219 }
1220
1221 switch (NVME_TEMP_THSEL(dw11)) {
1222 case NVME_TEMP_THSEL_OVER:
1223 result = n->features.temp_thresh_hi;
1224 goto out;
1225 case NVME_TEMP_THSEL_UNDER:
1226 result = n->features.temp_thresh_low;
1227 goto out;
1228 }
1229
1230 return NVME_INVALID_FIELD | NVME_DNR;
1231 case NVME_VOLATILE_WRITE_CACHE:
1232 result = blk_enable_write_cache(n->conf.blk);
1233 trace_pci_nvme_getfeat_vwcache(result ? "enabled" : "disabled");
1234 goto out;
1235 case NVME_ASYNCHRONOUS_EVENT_CONF:
1236 result = n->features.async_config;
1237 goto out;
1238 case NVME_TIMESTAMP:
1239 return nvme_get_feature_timestamp(n, cmd);
1240 default:
1241 break;
1242 }
1243
1244 defaults:
1245 switch (fid) {
1246 case NVME_TEMPERATURE_THRESHOLD:
1247 result = 0;
1248
1249 if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
1250 break;
1251 }
1252
1253 if (NVME_TEMP_THSEL(dw11) == NVME_TEMP_THSEL_OVER) {
1254 result = NVME_TEMPERATURE_WARNING;
1255 }
1256
1257 break;
1258 case NVME_NUMBER_OF_QUEUES:
1259 result = (n->params.max_ioqpairs - 1) |
1260 ((n->params.max_ioqpairs - 1) << 16);
1261 trace_pci_nvme_getfeat_numq(result);
1262 break;
1263 case NVME_INTERRUPT_VECTOR_CONF:
1264 iv = dw11 & 0xffff;
1265 if (iv >= n->params.max_ioqpairs + 1) {
1266 return NVME_INVALID_FIELD | NVME_DNR;
1267 }
1268
1269 result = iv;
1270 if (iv == n->admin_cq.vector) {
1271 result |= NVME_INTVC_NOCOALESCING;
1272 }
1273
1274 break;
1275 default:
1276 result = nvme_feature_default[fid];
1277 break;
1278 }
1279
1280 out:
1281 req->cqe.result = cpu_to_le32(result);
1282 return NVME_SUCCESS;
1283 }
1284
1285 static uint16_t nvme_set_feature_timestamp(NvmeCtrl *n, NvmeCmd *cmd)
1286 {
1287 uint16_t ret;
1288 uint64_t timestamp;
1289 uint64_t prp1 = le64_to_cpu(cmd->dptr.prp1);
1290 uint64_t prp2 = le64_to_cpu(cmd->dptr.prp2);
1291
1292 ret = nvme_dma_write_prp(n, (uint8_t *)&timestamp,
1293 sizeof(timestamp), prp1, prp2);
1294 if (ret != NVME_SUCCESS) {
1295 return ret;
1296 }
1297
1298 nvme_set_timestamp(n, timestamp);
1299
1300 return NVME_SUCCESS;
1301 }
1302
1303 static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
1304 {
1305 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
1306 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
1307 uint32_t nsid = le32_to_cpu(cmd->nsid);
1308 uint8_t fid = NVME_GETSETFEAT_FID(dw10);
1309 uint8_t save = NVME_SETFEAT_SAVE(dw10);
1310
1311 trace_pci_nvme_setfeat(nvme_cid(req), fid, save, dw11);
1312
1313 if (save) {
1314 return NVME_FID_NOT_SAVEABLE | NVME_DNR;
1315 }
1316
1317 if (!nvme_feature_support[fid]) {
1318 return NVME_INVALID_FIELD | NVME_DNR;
1319 }
1320
1321 if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
1322 if (!nsid || (nsid != NVME_NSID_BROADCAST &&
1323 nsid > n->num_namespaces)) {
1324 return NVME_INVALID_NSID | NVME_DNR;
1325 }
1326 } else if (nsid && nsid != NVME_NSID_BROADCAST) {
1327 if (nsid > n->num_namespaces) {
1328 return NVME_INVALID_NSID | NVME_DNR;
1329 }
1330
1331 return NVME_FEAT_NOT_NS_SPEC | NVME_DNR;
1332 }
1333
1334 if (!(nvme_feature_cap[fid] & NVME_FEAT_CAP_CHANGE)) {
1335 return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
1336 }
1337
1338 switch (fid) {
1339 case NVME_TEMPERATURE_THRESHOLD:
1340 if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
1341 break;
1342 }
1343
1344 switch (NVME_TEMP_THSEL(dw11)) {
1345 case NVME_TEMP_THSEL_OVER:
1346 n->features.temp_thresh_hi = NVME_TEMP_TMPTH(dw11);
1347 break;
1348 case NVME_TEMP_THSEL_UNDER:
1349 n->features.temp_thresh_low = NVME_TEMP_TMPTH(dw11);
1350 break;
1351 default:
1352 return NVME_INVALID_FIELD | NVME_DNR;
1353 }
1354
1355 if (((n->temperature >= n->features.temp_thresh_hi) ||
1356 (n->temperature <= n->features.temp_thresh_low)) &&
1357 NVME_AEC_SMART(n->features.async_config) & NVME_SMART_TEMPERATURE) {
1358 nvme_enqueue_event(n, NVME_AER_TYPE_SMART,
1359 NVME_AER_INFO_SMART_TEMP_THRESH,
1360 NVME_LOG_SMART_INFO);
1361 }
1362
1363 break;
1364 case NVME_VOLATILE_WRITE_CACHE:
1365 if (!(dw11 & 0x1) && blk_enable_write_cache(n->conf.blk)) {
1366 blk_flush(n->conf.blk);
1367 }
1368
1369 blk_set_enable_write_cache(n->conf.blk, dw11 & 1);
1370 break;
1371 case NVME_NUMBER_OF_QUEUES:
1372 if (n->qs_created) {
1373 return NVME_CMD_SEQ_ERROR | NVME_DNR;
1374 }
1375
1376 /*
1377 * NVMe v1.3, Section 5.21.1.7: 0xffff is not an allowed value for NCQR
1378 * and NSQR.
1379 */
1380 if ((dw11 & 0xffff) == 0xffff || ((dw11 >> 16) & 0xffff) == 0xffff) {
1381 return NVME_INVALID_FIELD | NVME_DNR;
1382 }
1383
1384 trace_pci_nvme_setfeat_numq((dw11 & 0xFFFF) + 1,
1385 ((dw11 >> 16) & 0xFFFF) + 1,
1386 n->params.max_ioqpairs,
1387 n->params.max_ioqpairs);
1388 req->cqe.result = cpu_to_le32((n->params.max_ioqpairs - 1) |
1389 ((n->params.max_ioqpairs - 1) << 16));
1390 break;
1391 case NVME_ASYNCHRONOUS_EVENT_CONF:
1392 n->features.async_config = dw11;
1393 break;
1394 case NVME_TIMESTAMP:
1395 return nvme_set_feature_timestamp(n, cmd);
1396 default:
1397 return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
1398 }
1399 return NVME_SUCCESS;
1400 }
1401
1402 static uint16_t nvme_aer(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
1403 {
1404 trace_pci_nvme_aer(nvme_cid(req));
1405
1406 if (n->outstanding_aers > n->params.aerl) {
1407 trace_pci_nvme_aer_aerl_exceeded();
1408 return NVME_AER_LIMIT_EXCEEDED;
1409 }
1410
1411 n->aer_reqs[n->outstanding_aers] = req;
1412 n->outstanding_aers++;
1413
1414 if (!QTAILQ_EMPTY(&n->aer_queue)) {
1415 nvme_process_aers(n);
1416 }
1417
1418 return NVME_NO_COMPLETE;
1419 }
1420
1421 static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
1422 {
1423 trace_pci_nvme_admin_cmd(nvme_cid(req), nvme_sqid(req), cmd->opcode);
1424
1425 switch (cmd->opcode) {
1426 case NVME_ADM_CMD_DELETE_SQ:
1427 return nvme_del_sq(n, cmd);
1428 case NVME_ADM_CMD_CREATE_SQ:
1429 return nvme_create_sq(n, cmd);
1430 case NVME_ADM_CMD_GET_LOG_PAGE:
1431 return nvme_get_log(n, cmd, req);
1432 case NVME_ADM_CMD_DELETE_CQ:
1433 return nvme_del_cq(n, cmd);
1434 case NVME_ADM_CMD_CREATE_CQ:
1435 return nvme_create_cq(n, cmd);
1436 case NVME_ADM_CMD_IDENTIFY:
1437 return nvme_identify(n, cmd);
1438 case NVME_ADM_CMD_ABORT:
1439 return nvme_abort(n, cmd, req);
1440 case NVME_ADM_CMD_SET_FEATURES:
1441 return nvme_set_feature(n, cmd, req);
1442 case NVME_ADM_CMD_GET_FEATURES:
1443 return nvme_get_feature(n, cmd, req);
1444 case NVME_ADM_CMD_ASYNC_EV_REQ:
1445 return nvme_aer(n, cmd, req);
1446 default:
1447 trace_pci_nvme_err_invalid_admin_opc(cmd->opcode);
1448 return NVME_INVALID_OPCODE | NVME_DNR;
1449 }
1450 }
1451
1452 static void nvme_process_sq(void *opaque)
1453 {
1454 NvmeSQueue *sq = opaque;
1455 NvmeCtrl *n = sq->ctrl;
1456 NvmeCQueue *cq = n->cq[sq->cqid];
1457
1458 uint16_t status;
1459 hwaddr addr;
1460 NvmeCmd cmd;
1461 NvmeRequest *req;
1462
1463 while (!(nvme_sq_empty(sq) || QTAILQ_EMPTY(&sq->req_list))) {
1464 addr = sq->dma_addr + sq->head * n->sqe_size;
1465 nvme_addr_read(n, addr, (void *)&cmd, sizeof(cmd));
1466 nvme_inc_sq_head(sq);
1467
1468 req = QTAILQ_FIRST(&sq->req_list);
1469 QTAILQ_REMOVE(&sq->req_list, req, entry);
1470 QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry);
1471 memset(&req->cqe, 0, sizeof(req->cqe));
1472 req->cqe.cid = cmd.cid;
1473
1474 status = sq->sqid ? nvme_io_cmd(n, &cmd, req) :
1475 nvme_admin_cmd(n, &cmd, req);
1476 if (status != NVME_NO_COMPLETE) {
1477 req->status = status;
1478 nvme_enqueue_req_completion(cq, req);
1479 }
1480 }
1481 }
1482
1483 static void nvme_clear_ctrl(NvmeCtrl *n)
1484 {
1485 int i;
1486
1487 blk_drain(n->conf.blk);
1488
1489 for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
1490 if (n->sq[i] != NULL) {
1491 nvme_free_sq(n->sq[i], n);
1492 }
1493 }
1494 for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
1495 if (n->cq[i] != NULL) {
1496 nvme_free_cq(n->cq[i], n);
1497 }
1498 }
1499
1500 while (!QTAILQ_EMPTY(&n->aer_queue)) {
1501 NvmeAsyncEvent *event = QTAILQ_FIRST(&n->aer_queue);
1502 QTAILQ_REMOVE(&n->aer_queue, event, entry);
1503 g_free(event);
1504 }
1505
1506 n->aer_queued = 0;
1507 n->outstanding_aers = 0;
1508 n->qs_created = false;
1509
1510 blk_flush(n->conf.blk);
1511 n->bar.cc = 0;
1512 }
1513
1514 static int nvme_start_ctrl(NvmeCtrl *n)
1515 {
1516 uint32_t page_bits = NVME_CC_MPS(n->bar.cc) + 12;
1517 uint32_t page_size = 1 << page_bits;
1518
1519 if (unlikely(n->cq[0])) {
1520 trace_pci_nvme_err_startfail_cq();
1521 return -1;
1522 }
1523 if (unlikely(n->sq[0])) {
1524 trace_pci_nvme_err_startfail_sq();
1525 return -1;
1526 }
1527 if (unlikely(!n->bar.asq)) {
1528 trace_pci_nvme_err_startfail_nbarasq();
1529 return -1;
1530 }
1531 if (unlikely(!n->bar.acq)) {
1532 trace_pci_nvme_err_startfail_nbaracq();
1533 return -1;
1534 }
1535 if (unlikely(n->bar.asq & (page_size - 1))) {
1536 trace_pci_nvme_err_startfail_asq_misaligned(n->bar.asq);
1537 return -1;
1538 }
1539 if (unlikely(n->bar.acq & (page_size - 1))) {
1540 trace_pci_nvme_err_startfail_acq_misaligned(n->bar.acq);
1541 return -1;
1542 }
1543 if (unlikely(NVME_CC_MPS(n->bar.cc) <
1544 NVME_CAP_MPSMIN(n->bar.cap))) {
1545 trace_pci_nvme_err_startfail_page_too_small(
1546 NVME_CC_MPS(n->bar.cc),
1547 NVME_CAP_MPSMIN(n->bar.cap));
1548 return -1;
1549 }
1550 if (unlikely(NVME_CC_MPS(n->bar.cc) >
1551 NVME_CAP_MPSMAX(n->bar.cap))) {
1552 trace_pci_nvme_err_startfail_page_too_large(
1553 NVME_CC_MPS(n->bar.cc),
1554 NVME_CAP_MPSMAX(n->bar.cap));
1555 return -1;
1556 }
1557 if (unlikely(NVME_CC_IOCQES(n->bar.cc) <
1558 NVME_CTRL_CQES_MIN(n->id_ctrl.cqes))) {
1559 trace_pci_nvme_err_startfail_cqent_too_small(
1560 NVME_CC_IOCQES(n->bar.cc),
1561 NVME_CTRL_CQES_MIN(n->bar.cap));
1562 return -1;
1563 }
1564 if (unlikely(NVME_CC_IOCQES(n->bar.cc) >
1565 NVME_CTRL_CQES_MAX(n->id_ctrl.cqes))) {
1566 trace_pci_nvme_err_startfail_cqent_too_large(
1567 NVME_CC_IOCQES(n->bar.cc),
1568 NVME_CTRL_CQES_MAX(n->bar.cap));
1569 return -1;
1570 }
1571 if (unlikely(NVME_CC_IOSQES(n->bar.cc) <
1572 NVME_CTRL_SQES_MIN(n->id_ctrl.sqes))) {
1573 trace_pci_nvme_err_startfail_sqent_too_small(
1574 NVME_CC_IOSQES(n->bar.cc),
1575 NVME_CTRL_SQES_MIN(n->bar.cap));
1576 return -1;
1577 }
1578 if (unlikely(NVME_CC_IOSQES(n->bar.cc) >
1579 NVME_CTRL_SQES_MAX(n->id_ctrl.sqes))) {
1580 trace_pci_nvme_err_startfail_sqent_too_large(
1581 NVME_CC_IOSQES(n->bar.cc),
1582 NVME_CTRL_SQES_MAX(n->bar.cap));
1583 return -1;
1584 }
1585 if (unlikely(!NVME_AQA_ASQS(n->bar.aqa))) {
1586 trace_pci_nvme_err_startfail_asqent_sz_zero();
1587 return -1;
1588 }
1589 if (unlikely(!NVME_AQA_ACQS(n->bar.aqa))) {
1590 trace_pci_nvme_err_startfail_acqent_sz_zero();
1591 return -1;
1592 }
1593
1594 n->page_bits = page_bits;
1595 n->page_size = page_size;
1596 n->max_prp_ents = n->page_size / sizeof(uint64_t);
1597 n->cqe_size = 1 << NVME_CC_IOCQES(n->bar.cc);
1598 n->sqe_size = 1 << NVME_CC_IOSQES(n->bar.cc);
1599 nvme_init_cq(&n->admin_cq, n, n->bar.acq, 0, 0,
1600 NVME_AQA_ACQS(n->bar.aqa) + 1, 1);
1601 nvme_init_sq(&n->admin_sq, n, n->bar.asq, 0, 0,
1602 NVME_AQA_ASQS(n->bar.aqa) + 1);
1603
1604 nvme_set_timestamp(n, 0ULL);
1605
1606 QTAILQ_INIT(&n->aer_queue);
1607
1608 return 0;
1609 }
1610
1611 static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
1612 unsigned size)
1613 {
1614 if (unlikely(offset & (sizeof(uint32_t) - 1))) {
1615 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32,
1616 "MMIO write not 32-bit aligned,"
1617 " offset=0x%"PRIx64"", offset);
1618 /* should be ignored, fall through for now */
1619 }
1620
1621 if (unlikely(size < sizeof(uint32_t))) {
1622 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall,
1623 "MMIO write smaller than 32-bits,"
1624 " offset=0x%"PRIx64", size=%u",
1625 offset, size);
1626 /* should be ignored, fall through for now */
1627 }
1628
1629 switch (offset) {
1630 case 0xc: /* INTMS */
1631 if (unlikely(msix_enabled(&(n->parent_obj)))) {
1632 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
1633 "undefined access to interrupt mask set"
1634 " when MSI-X is enabled");
1635 /* should be ignored, fall through for now */
1636 }
1637 n->bar.intms |= data & 0xffffffff;
1638 n->bar.intmc = n->bar.intms;
1639 trace_pci_nvme_mmio_intm_set(data & 0xffffffff, n->bar.intmc);
1640 nvme_irq_check(n);
1641 break;
1642 case 0x10: /* INTMC */
1643 if (unlikely(msix_enabled(&(n->parent_obj)))) {
1644 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
1645 "undefined access to interrupt mask clr"
1646 " when MSI-X is enabled");
1647 /* should be ignored, fall through for now */
1648 }
1649 n->bar.intms &= ~(data & 0xffffffff);
1650 n->bar.intmc = n->bar.intms;
1651 trace_pci_nvme_mmio_intm_clr(data & 0xffffffff, n->bar.intmc);
1652 nvme_irq_check(n);
1653 break;
1654 case 0x14: /* CC */
1655 trace_pci_nvme_mmio_cfg(data & 0xffffffff);
1656 /* Windows first sends data, then sends enable bit */
1657 if (!NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc) &&
1658 !NVME_CC_SHN(data) && !NVME_CC_SHN(n->bar.cc))
1659 {
1660 n->bar.cc = data;
1661 }
1662
1663 if (NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc)) {
1664 n->bar.cc = data;
1665 if (unlikely(nvme_start_ctrl(n))) {
1666 trace_pci_nvme_err_startfail();
1667 n->bar.csts = NVME_CSTS_FAILED;
1668 } else {
1669 trace_pci_nvme_mmio_start_success();
1670 n->bar.csts = NVME_CSTS_READY;
1671 }
1672 } else if (!NVME_CC_EN(data) && NVME_CC_EN(n->bar.cc)) {
1673 trace_pci_nvme_mmio_stopped();
1674 nvme_clear_ctrl(n);
1675 n->bar.csts &= ~NVME_CSTS_READY;
1676 }
1677 if (NVME_CC_SHN(data) && !(NVME_CC_SHN(n->bar.cc))) {
1678 trace_pci_nvme_mmio_shutdown_set();
1679 nvme_clear_ctrl(n);
1680 n->bar.cc = data;
1681 n->bar.csts |= NVME_CSTS_SHST_COMPLETE;
1682 } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(n->bar.cc)) {
1683 trace_pci_nvme_mmio_shutdown_cleared();
1684 n->bar.csts &= ~NVME_CSTS_SHST_COMPLETE;
1685 n->bar.cc = data;
1686 }
1687 break;
1688 case 0x1C: /* CSTS */
1689 if (data & (1 << 4)) {
1690 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported,
1691 "attempted to W1C CSTS.NSSRO"
1692 " but CAP.NSSRS is zero (not supported)");
1693 } else if (data != 0) {
1694 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts,
1695 "attempted to set a read only bit"
1696 " of controller status");
1697 }
1698 break;
1699 case 0x20: /* NSSR */
1700 if (data == 0x4E564D65) {
1701 trace_pci_nvme_ub_mmiowr_ssreset_unsupported();
1702 } else {
1703 /* The spec says that writes of other values have no effect */
1704 return;
1705 }
1706 break;
1707 case 0x24: /* AQA */
1708 n->bar.aqa = data & 0xffffffff;
1709 trace_pci_nvme_mmio_aqattr(data & 0xffffffff);
1710 break;
1711 case 0x28: /* ASQ */
1712 n->bar.asq = data;
1713 trace_pci_nvme_mmio_asqaddr(data);
1714 break;
1715 case 0x2c: /* ASQ hi */
1716 n->bar.asq |= data << 32;
1717 trace_pci_nvme_mmio_asqaddr_hi(data, n->bar.asq);
1718 break;
1719 case 0x30: /* ACQ */
1720 trace_pci_nvme_mmio_acqaddr(data);
1721 n->bar.acq = data;
1722 break;
1723 case 0x34: /* ACQ hi */
1724 n->bar.acq |= data << 32;
1725 trace_pci_nvme_mmio_acqaddr_hi(data, n->bar.acq);
1726 break;
1727 case 0x38: /* CMBLOC */
1728 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved,
1729 "invalid write to reserved CMBLOC"
1730 " when CMBSZ is zero, ignored");
1731 return;
1732 case 0x3C: /* CMBSZ */
1733 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly,
1734 "invalid write to read only CMBSZ, ignored");
1735 return;
1736 case 0xE00: /* PMRCAP */
1737 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly,
1738 "invalid write to PMRCAP register, ignored");
1739 return;
1740 case 0xE04: /* TODO PMRCTL */
1741 break;
1742 case 0xE08: /* PMRSTS */
1743 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly,
1744 "invalid write to PMRSTS register, ignored");
1745 return;
1746 case 0xE0C: /* PMREBS */
1747 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly,
1748 "invalid write to PMREBS register, ignored");
1749 return;
1750 case 0xE10: /* PMRSWTP */
1751 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly,
1752 "invalid write to PMRSWTP register, ignored");
1753 return;
1754 case 0xE14: /* TODO PMRMSC */
1755 break;
1756 default:
1757 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid,
1758 "invalid MMIO write,"
1759 " offset=0x%"PRIx64", data=%"PRIx64"",
1760 offset, data);
1761 break;
1762 }
1763 }
1764
1765 static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
1766 {
1767 NvmeCtrl *n = (NvmeCtrl *)opaque;
1768 uint8_t *ptr = (uint8_t *)&n->bar;
1769 uint64_t val = 0;
1770
1771 trace_pci_nvme_mmio_read(addr);
1772
1773 if (unlikely(addr & (sizeof(uint32_t) - 1))) {
1774 NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32,
1775 "MMIO read not 32-bit aligned,"
1776 " offset=0x%"PRIx64"", addr);
1777 /* should RAZ, fall through for now */
1778 } else if (unlikely(size < sizeof(uint32_t))) {
1779 NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall,
1780 "MMIO read smaller than 32-bits,"
1781 " offset=0x%"PRIx64"", addr);
1782 /* should RAZ, fall through for now */
1783 }
1784
1785 if (addr < sizeof(n->bar)) {
1786 /*
1787 * When PMRWBM bit 1 is set then read from
1788 * from PMRSTS should ensure prior writes
1789 * made it to persistent media
1790 */
1791 if (addr == 0xE08 &&
1792 (NVME_PMRCAP_PMRWBM(n->bar.pmrcap) & 0x02)) {
1793 memory_region_msync(&n->pmrdev->mr, 0, n->pmrdev->size);
1794 }
1795 memcpy(&val, ptr + addr, size);
1796 } else {
1797 NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs,
1798 "MMIO read beyond last register,"
1799 " offset=0x%"PRIx64", returning 0", addr);
1800 }
1801
1802 return val;
1803 }
1804
1805 static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
1806 {
1807 uint32_t qid;
1808
1809 if (unlikely(addr & ((1 << 2) - 1))) {
1810 NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned,
1811 "doorbell write not 32-bit aligned,"
1812 " offset=0x%"PRIx64", ignoring", addr);
1813 return;
1814 }
1815
1816 if (((addr - 0x1000) >> 2) & 1) {
1817 /* Completion queue doorbell write */
1818
1819 uint16_t new_head = val & 0xffff;
1820 int start_sqs;
1821 NvmeCQueue *cq;
1822
1823 qid = (addr - (0x1000 + (1 << 2))) >> 3;
1824 if (unlikely(nvme_check_cqid(n, qid))) {
1825 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq,
1826 "completion queue doorbell write"
1827 " for nonexistent queue,"
1828 " sqid=%"PRIu32", ignoring", qid);
1829
1830 /*
1831 * NVM Express v1.3d, Section 4.1 state: "If host software writes
1832 * an invalid value to the Submission Queue Tail Doorbell or
1833 * Completion Queue Head Doorbell regiter and an Asynchronous Event
1834 * Request command is outstanding, then an asynchronous event is
1835 * posted to the Admin Completion Queue with a status code of
1836 * Invalid Doorbell Write Value."
1837 *
1838 * Also note that the spec includes the "Invalid Doorbell Register"
1839 * status code, but nowhere does it specify when to use it.
1840 * However, it seems reasonable to use it here in a similar
1841 * fashion.
1842 */
1843 if (n->outstanding_aers) {
1844 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
1845 NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
1846 NVME_LOG_ERROR_INFO);
1847 }
1848
1849 return;
1850 }
1851
1852 cq = n->cq[qid];
1853 if (unlikely(new_head >= cq->size)) {
1854 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead,
1855 "completion queue doorbell write value"
1856 " beyond queue size, sqid=%"PRIu32","
1857 " new_head=%"PRIu16", ignoring",
1858 qid, new_head);
1859
1860 if (n->outstanding_aers) {
1861 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
1862 NVME_AER_INFO_ERR_INVALID_DB_VALUE,
1863 NVME_LOG_ERROR_INFO);
1864 }
1865
1866 return;
1867 }
1868
1869 trace_pci_nvme_mmio_doorbell_cq(cq->cqid, new_head);
1870
1871 start_sqs = nvme_cq_full(cq) ? 1 : 0;
1872 cq->head = new_head;
1873 if (start_sqs) {
1874 NvmeSQueue *sq;
1875 QTAILQ_FOREACH(sq, &cq->sq_list, entry) {
1876 timer_mod(sq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
1877 }
1878 timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
1879 }
1880
1881 if (cq->tail == cq->head) {
1882 nvme_irq_deassert(n, cq);
1883 }
1884 } else {
1885 /* Submission queue doorbell write */
1886
1887 uint16_t new_tail = val & 0xffff;
1888 NvmeSQueue *sq;
1889
1890 qid = (addr - 0x1000) >> 3;
1891 if (unlikely(nvme_check_sqid(n, qid))) {
1892 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq,
1893 "submission queue doorbell write"
1894 " for nonexistent queue,"
1895 " sqid=%"PRIu32", ignoring", qid);
1896
1897 if (n->outstanding_aers) {
1898 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
1899 NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
1900 NVME_LOG_ERROR_INFO);
1901 }
1902
1903 return;
1904 }
1905
1906 sq = n->sq[qid];
1907 if (unlikely(new_tail >= sq->size)) {
1908 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail,
1909 "submission queue doorbell write value"
1910 " beyond queue size, sqid=%"PRIu32","
1911 " new_tail=%"PRIu16", ignoring",
1912 qid, new_tail);
1913
1914 if (n->outstanding_aers) {
1915 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
1916 NVME_AER_INFO_ERR_INVALID_DB_VALUE,
1917 NVME_LOG_ERROR_INFO);
1918 }
1919
1920 return;
1921 }
1922
1923 trace_pci_nvme_mmio_doorbell_sq(sq->sqid, new_tail);
1924
1925 sq->tail = new_tail;
1926 timer_mod(sq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
1927 }
1928 }
1929
1930 static void nvme_mmio_write(void *opaque, hwaddr addr, uint64_t data,
1931 unsigned size)
1932 {
1933 NvmeCtrl *n = (NvmeCtrl *)opaque;
1934
1935 trace_pci_nvme_mmio_write(addr, data);
1936
1937 if (addr < sizeof(n->bar)) {
1938 nvme_write_bar(n, addr, data, size);
1939 } else {
1940 nvme_process_db(n, addr, data);
1941 }
1942 }
1943
1944 static const MemoryRegionOps nvme_mmio_ops = {
1945 .read = nvme_mmio_read,
1946 .write = nvme_mmio_write,
1947 .endianness = DEVICE_LITTLE_ENDIAN,
1948 .impl = {
1949 .min_access_size = 2,
1950 .max_access_size = 8,
1951 },
1952 };
1953
1954 static void nvme_cmb_write(void *opaque, hwaddr addr, uint64_t data,
1955 unsigned size)
1956 {
1957 NvmeCtrl *n = (NvmeCtrl *)opaque;
1958 stn_le_p(&n->cmbuf[addr], size, data);
1959 }
1960
1961 static uint64_t nvme_cmb_read(void *opaque, hwaddr addr, unsigned size)
1962 {
1963 NvmeCtrl *n = (NvmeCtrl *)opaque;
1964 return ldn_le_p(&n->cmbuf[addr], size);
1965 }
1966
1967 static const MemoryRegionOps nvme_cmb_ops = {
1968 .read = nvme_cmb_read,
1969 .write = nvme_cmb_write,
1970 .endianness = DEVICE_LITTLE_ENDIAN,
1971 .impl = {
1972 .min_access_size = 1,
1973 .max_access_size = 8,
1974 },
1975 };
1976
1977 static void nvme_check_constraints(NvmeCtrl *n, Error **errp)
1978 {
1979 NvmeParams *params = &n->params;
1980
1981 if (params->num_queues) {
1982 warn_report("num_queues is deprecated; please use max_ioqpairs "
1983 "instead");
1984
1985 params->max_ioqpairs = params->num_queues - 1;
1986 }
1987
1988 if (params->max_ioqpairs < 1 ||
1989 params->max_ioqpairs > NVME_MAX_IOQPAIRS) {
1990 error_setg(errp, "max_ioqpairs must be between 1 and %d",
1991 NVME_MAX_IOQPAIRS);
1992 return;
1993 }
1994
1995 if (params->msix_qsize < 1 ||
1996 params->msix_qsize > PCI_MSIX_FLAGS_QSIZE + 1) {
1997 error_setg(errp, "msix_qsize must be between 1 and %d",
1998 PCI_MSIX_FLAGS_QSIZE + 1);
1999 return;
2000 }
2001
2002 if (!n->conf.blk) {
2003 error_setg(errp, "drive property not set");
2004 return;
2005 }
2006
2007 if (!params->serial) {
2008 error_setg(errp, "serial property not set");
2009 return;
2010 }
2011
2012 if (!n->params.cmb_size_mb && n->pmrdev) {
2013 if (host_memory_backend_is_mapped(n->pmrdev)) {
2014 error_setg(errp, "can't use already busy memdev: %s",
2015 object_get_canonical_path_component(OBJECT(n->pmrdev)));
2016 return;
2017 }
2018
2019 if (!is_power_of_2(n->pmrdev->size)) {
2020 error_setg(errp, "pmr backend size needs to be power of 2 in size");
2021 return;
2022 }
2023
2024 host_memory_backend_set_mapped(n->pmrdev, true);
2025 }
2026 }
2027
2028 static void nvme_init_state(NvmeCtrl *n)
2029 {
2030 n->num_namespaces = 1;
2031 /* add one to max_ioqpairs to account for the admin queue pair */
2032 n->reg_size = pow2ceil(sizeof(NvmeBar) +
2033 2 * (n->params.max_ioqpairs + 1) * NVME_DB_SIZE);
2034 n->namespaces = g_new0(NvmeNamespace, n->num_namespaces);
2035 n->sq = g_new0(NvmeSQueue *, n->params.max_ioqpairs + 1);
2036 n->cq = g_new0(NvmeCQueue *, n->params.max_ioqpairs + 1);
2037 n->temperature = NVME_TEMPERATURE;
2038 n->features.temp_thresh_hi = NVME_TEMPERATURE_WARNING;
2039 n->starttime_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
2040 n->aer_reqs = g_new0(NvmeRequest *, n->params.aerl + 1);
2041 }
2042
2043 static void nvme_init_blk(NvmeCtrl *n, Error **errp)
2044 {
2045 if (!blkconf_blocksizes(&n->conf, errp)) {
2046 return;
2047 }
2048 blkconf_apply_backend_options(&n->conf, blk_is_read_only(n->conf.blk),
2049 false, errp);
2050 }
2051
2052 static void nvme_init_namespace(NvmeCtrl *n, NvmeNamespace *ns, Error **errp)
2053 {
2054 int64_t bs_size;
2055 NvmeIdNs *id_ns = &ns->id_ns;
2056
2057 bs_size = blk_getlength(n->conf.blk);
2058 if (bs_size < 0) {
2059 error_setg_errno(errp, -bs_size, "could not get backing file size");
2060 return;
2061 }
2062
2063 n->ns_size = bs_size;
2064
2065 id_ns->lbaf[0].ds = BDRV_SECTOR_BITS;
2066 id_ns->nsze = cpu_to_le64(nvme_ns_nlbas(n, ns));
2067
2068 /* no thin provisioning */
2069 id_ns->ncap = id_ns->nsze;
2070 id_ns->nuse = id_ns->ncap;
2071 }
2072
2073 static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev)
2074 {
2075 NVME_CMBLOC_SET_BIR(n->bar.cmbloc, NVME_CMB_BIR);
2076 NVME_CMBLOC_SET_OFST(n->bar.cmbloc, 0);
2077
2078 NVME_CMBSZ_SET_SQS(n->bar.cmbsz, 1);
2079 NVME_CMBSZ_SET_CQS(n->bar.cmbsz, 0);
2080 NVME_CMBSZ_SET_LISTS(n->bar.cmbsz, 0);
2081 NVME_CMBSZ_SET_RDS(n->bar.cmbsz, 1);
2082 NVME_CMBSZ_SET_WDS(n->bar.cmbsz, 1);
2083 NVME_CMBSZ_SET_SZU(n->bar.cmbsz, 2); /* MBs */
2084 NVME_CMBSZ_SET_SZ(n->bar.cmbsz, n->params.cmb_size_mb);
2085
2086 n->cmbuf = g_malloc0(NVME_CMBSZ_GETSIZE(n->bar.cmbsz));
2087 memory_region_init_io(&n->ctrl_mem, OBJECT(n), &nvme_cmb_ops, n,
2088 "nvme-cmb", NVME_CMBSZ_GETSIZE(n->bar.cmbsz));
2089 pci_register_bar(pci_dev, NVME_CMBLOC_BIR(n->bar.cmbloc),
2090 PCI_BASE_ADDRESS_SPACE_MEMORY |
2091 PCI_BASE_ADDRESS_MEM_TYPE_64 |
2092 PCI_BASE_ADDRESS_MEM_PREFETCH, &n->ctrl_mem);
2093 }
2094
2095 static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
2096 {
2097 /* Controller Capabilities register */
2098 NVME_CAP_SET_PMRS(n->bar.cap, 1);
2099
2100 /* PMR Capabities register */
2101 n->bar.pmrcap = 0;
2102 NVME_PMRCAP_SET_RDS(n->bar.pmrcap, 0);
2103 NVME_PMRCAP_SET_WDS(n->bar.pmrcap, 0);
2104 NVME_PMRCAP_SET_BIR(n->bar.pmrcap, NVME_PMR_BIR);
2105 NVME_PMRCAP_SET_PMRTU(n->bar.pmrcap, 0);
2106 /* Turn on bit 1 support */
2107 NVME_PMRCAP_SET_PMRWBM(n->bar.pmrcap, 0x02);
2108 NVME_PMRCAP_SET_PMRTO(n->bar.pmrcap, 0);
2109 NVME_PMRCAP_SET_CMSS(n->bar.pmrcap, 0);
2110
2111 /* PMR Control register */
2112 n->bar.pmrctl = 0;
2113 NVME_PMRCTL_SET_EN(n->bar.pmrctl, 0);
2114
2115 /* PMR Status register */
2116 n->bar.pmrsts = 0;
2117 NVME_PMRSTS_SET_ERR(n->bar.pmrsts, 0);
2118 NVME_PMRSTS_SET_NRDY(n->bar.pmrsts, 0);
2119 NVME_PMRSTS_SET_HSTS(n->bar.pmrsts, 0);
2120 NVME_PMRSTS_SET_CBAI(n->bar.pmrsts, 0);
2121
2122 /* PMR Elasticity Buffer Size register */
2123 n->bar.pmrebs = 0;
2124 NVME_PMREBS_SET_PMRSZU(n->bar.pmrebs, 0);
2125 NVME_PMREBS_SET_RBB(n->bar.pmrebs, 0);
2126 NVME_PMREBS_SET_PMRWBZ(n->bar.pmrebs, 0);
2127
2128 /* PMR Sustained Write Throughput register */
2129 n->bar.pmrswtp = 0;
2130 NVME_PMRSWTP_SET_PMRSWTU(n->bar.pmrswtp, 0);
2131 NVME_PMRSWTP_SET_PMRSWTV(n->bar.pmrswtp, 0);
2132
2133 /* PMR Memory Space Control register */
2134 n->bar.pmrmsc = 0;
2135 NVME_PMRMSC_SET_CMSE(n->bar.pmrmsc, 0);
2136 NVME_PMRMSC_SET_CBA(n->bar.pmrmsc, 0);
2137
2138 pci_register_bar(pci_dev, NVME_PMRCAP_BIR(n->bar.pmrcap),
2139 PCI_BASE_ADDRESS_SPACE_MEMORY |
2140 PCI_BASE_ADDRESS_MEM_TYPE_64 |
2141 PCI_BASE_ADDRESS_MEM_PREFETCH, &n->pmrdev->mr);
2142 }
2143
2144 static void nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
2145 {
2146 uint8_t *pci_conf = pci_dev->config;
2147
2148 pci_conf[PCI_INTERRUPT_PIN] = 1;
2149 pci_config_set_prog_interface(pci_conf, 0x2);
2150 pci_config_set_class(pci_conf, PCI_CLASS_STORAGE_EXPRESS);
2151 pcie_endpoint_cap_init(pci_dev, 0x80);
2152
2153 memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
2154 n->reg_size);
2155 pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
2156 PCI_BASE_ADDRESS_MEM_TYPE_64, &n->iomem);
2157 if (msix_init_exclusive_bar(pci_dev, n->params.msix_qsize, 4, errp)) {
2158 return;
2159 }
2160
2161 if (n->params.cmb_size_mb) {
2162 nvme_init_cmb(n, pci_dev);
2163 } else if (n->pmrdev) {
2164 nvme_init_pmr(n, pci_dev);
2165 }
2166 }
2167
2168 static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
2169 {
2170 NvmeIdCtrl *id = &n->id_ctrl;
2171 uint8_t *pci_conf = pci_dev->config;
2172 char *subnqn;
2173
2174 id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
2175 id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
2176 strpadcpy((char *)id->mn, sizeof(id->mn), "QEMU NVMe Ctrl", ' ');
2177 strpadcpy((char *)id->fr, sizeof(id->fr), "1.0", ' ');
2178 strpadcpy((char *)id->sn, sizeof(id->sn), n->params.serial, ' ');
2179 id->rab = 6;
2180 id->ieee[0] = 0x00;
2181 id->ieee[1] = 0x02;
2182 id->ieee[2] = 0xb3;
2183 id->ver = cpu_to_le32(NVME_SPEC_VER);
2184 id->oacs = cpu_to_le16(0);
2185
2186 /*
2187 * Because the controller always completes the Abort command immediately,
2188 * there can never be more than one concurrently executing Abort command,
2189 * so this value is never used for anything. Note that there can easily be
2190 * many Abort commands in the queues, but they are not considered
2191 * "executing" until processed by nvme_abort.
2192 *
2193 * The specification recommends a value of 3 for Abort Command Limit (four
2194 * concurrently outstanding Abort commands), so lets use that though it is
2195 * inconsequential.
2196 */
2197 id->acl = 3;
2198 id->aerl = n->params.aerl;
2199 id->frmw = (NVME_NUM_FW_SLOTS << 1) | NVME_FRMW_SLOT1_RO;
2200 id->lpa = NVME_LPA_EXTENDED;
2201
2202 /* recommended default value (~70 C) */
2203 id->wctemp = cpu_to_le16(NVME_TEMPERATURE_WARNING);
2204 id->cctemp = cpu_to_le16(NVME_TEMPERATURE_CRITICAL);
2205
2206 id->sqes = (0x6 << 4) | 0x6;
2207 id->cqes = (0x4 << 4) | 0x4;
2208 id->nn = cpu_to_le32(n->num_namespaces);
2209 id->oncs = cpu_to_le16(NVME_ONCS_WRITE_ZEROS | NVME_ONCS_TIMESTAMP |
2210 NVME_ONCS_FEATURES);
2211
2212 subnqn = g_strdup_printf("nqn.2019-08.org.qemu:%s", n->params.serial);
2213 strpadcpy((char *)id->subnqn, sizeof(id->subnqn), subnqn, '\0');
2214 g_free(subnqn);
2215
2216 id->psd[0].mp = cpu_to_le16(0x9c4);
2217 id->psd[0].enlat = cpu_to_le32(0x10);
2218 id->psd[0].exlat = cpu_to_le32(0x4);
2219 if (blk_enable_write_cache(n->conf.blk)) {
2220 id->vwc = 1;
2221 }
2222
2223 n->bar.cap = 0;
2224 NVME_CAP_SET_MQES(n->bar.cap, 0x7ff);
2225 NVME_CAP_SET_CQR(n->bar.cap, 1);
2226 NVME_CAP_SET_TO(n->bar.cap, 0xf);
2227 NVME_CAP_SET_CSS(n->bar.cap, 1);
2228 NVME_CAP_SET_MPSMAX(n->bar.cap, 4);
2229
2230 n->bar.vs = NVME_SPEC_VER;
2231 n->bar.intmc = n->bar.intms = 0;
2232 }
2233
2234 static void nvme_realize(PCIDevice *pci_dev, Error **errp)
2235 {
2236 NvmeCtrl *n = NVME(pci_dev);
2237 Error *local_err = NULL;
2238
2239 int i;
2240
2241 nvme_check_constraints(n, &local_err);
2242 if (local_err) {
2243 error_propagate(errp, local_err);
2244 return;
2245 }
2246
2247 nvme_init_state(n);
2248 nvme_init_blk(n, &local_err);
2249 if (local_err) {
2250 error_propagate(errp, local_err);
2251 return;
2252 }
2253
2254 nvme_init_pci(n, pci_dev, &local_err);
2255 if (local_err) {
2256 error_propagate(errp, local_err);
2257 return;
2258 }
2259
2260 nvme_init_ctrl(n, pci_dev);
2261
2262 for (i = 0; i < n->num_namespaces; i++) {
2263 nvme_init_namespace(n, &n->namespaces[i], &local_err);
2264 if (local_err) {
2265 error_propagate(errp, local_err);
2266 return;
2267 }
2268 }
2269 }
2270
2271 static void nvme_exit(PCIDevice *pci_dev)
2272 {
2273 NvmeCtrl *n = NVME(pci_dev);
2274
2275 nvme_clear_ctrl(n);
2276 g_free(n->namespaces);
2277 g_free(n->cq);
2278 g_free(n->sq);
2279 g_free(n->aer_reqs);
2280
2281 if (n->params.cmb_size_mb) {
2282 g_free(n->cmbuf);
2283 }
2284
2285 if (n->pmrdev) {
2286 host_memory_backend_set_mapped(n->pmrdev, false);
2287 }
2288 msix_uninit_exclusive_bar(pci_dev);
2289 }
2290
2291 static Property nvme_props[] = {
2292 DEFINE_BLOCK_PROPERTIES(NvmeCtrl, conf),
2293 DEFINE_PROP_LINK("pmrdev", NvmeCtrl, pmrdev, TYPE_MEMORY_BACKEND,
2294 HostMemoryBackend *),
2295 DEFINE_PROP_STRING("serial", NvmeCtrl, params.serial),
2296 DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl, params.cmb_size_mb, 0),
2297 DEFINE_PROP_UINT32("num_queues", NvmeCtrl, params.num_queues, 0),
2298 DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl, params.max_ioqpairs, 64),
2299 DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl, params.msix_qsize, 65),
2300 DEFINE_PROP_UINT8("aerl", NvmeCtrl, params.aerl, 3),
2301 DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl, params.aer_max_queued, 64),
2302 DEFINE_PROP_END_OF_LIST(),
2303 };
2304
2305 static const VMStateDescription nvme_vmstate = {
2306 .name = "nvme",
2307 .unmigratable = 1,
2308 };
2309
2310 static void nvme_class_init(ObjectClass *oc, void *data)
2311 {
2312 DeviceClass *dc = DEVICE_CLASS(oc);
2313 PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
2314
2315 pc->realize = nvme_realize;
2316 pc->exit = nvme_exit;
2317 pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
2318 pc->vendor_id = PCI_VENDOR_ID_INTEL;
2319 pc->device_id = 0x5845;
2320 pc->revision = 2;
2321
2322 set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
2323 dc->desc = "Non-Volatile Memory Express";
2324 device_class_set_props(dc, nvme_props);
2325 dc->vmsd = &nvme_vmstate;
2326 }
2327
2328 static void nvme_instance_init(Object *obj)
2329 {
2330 NvmeCtrl *s = NVME(obj);
2331
2332 device_add_bootindex_property(obj, &s->conf.bootindex,
2333 "bootindex", "/namespace@1,0",
2334 DEVICE(obj));
2335 }
2336
2337 static const TypeInfo nvme_info = {
2338 .name = TYPE_NVME,
2339 .parent = TYPE_PCI_DEVICE,
2340 .instance_size = sizeof(NvmeCtrl),
2341 .class_init = nvme_class_init,
2342 .instance_init = nvme_instance_init,
2343 .interfaces = (InterfaceInfo[]) {
2344 { INTERFACE_PCIE_DEVICE },
2345 { }
2346 },
2347 };
2348
2349 static void nvme_register_types(void)
2350 {
2351 type_register_static(&nvme_info);
2352 }
2353
2354 type_init(nvme_register_types)