PPC: e500mc: add missing IVORs to bitmap
[qemu.git] / block / sheepdog.c
1 /*
2 * Copyright (C) 2009-2010 Nippon Telegraph and Telephone Corporation.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License version
6 * 2 as published by the Free Software Foundation.
7 *
8 * You should have received a copy of the GNU General Public License
9 * along with this program. If not, see <http://www.gnu.org/licenses/>.
10 *
11 * Contributions after 2012-01-13 are licensed under the terms of the
12 * GNU GPL, version 2 or (at your option) any later version.
13 */
14
15 #include "qemu-common.h"
16 #include "qemu-error.h"
17 #include "qemu_socket.h"
18 #include "block_int.h"
19 #include "bitops.h"
20
21 #define SD_PROTO_VER 0x01
22
23 #define SD_DEFAULT_ADDR "localhost"
24 #define SD_DEFAULT_PORT "7000"
25
26 #define SD_OP_CREATE_AND_WRITE_OBJ 0x01
27 #define SD_OP_READ_OBJ 0x02
28 #define SD_OP_WRITE_OBJ 0x03
29
30 #define SD_OP_NEW_VDI 0x11
31 #define SD_OP_LOCK_VDI 0x12
32 #define SD_OP_RELEASE_VDI 0x13
33 #define SD_OP_GET_VDI_INFO 0x14
34 #define SD_OP_READ_VDIS 0x15
35
36 #define SD_FLAG_CMD_WRITE 0x01
37 #define SD_FLAG_CMD_COW 0x02
38
39 #define SD_RES_SUCCESS 0x00 /* Success */
40 #define SD_RES_UNKNOWN 0x01 /* Unknown error */
41 #define SD_RES_NO_OBJ 0x02 /* No object found */
42 #define SD_RES_EIO 0x03 /* I/O error */
43 #define SD_RES_VDI_EXIST 0x04 /* Vdi exists already */
44 #define SD_RES_INVALID_PARMS 0x05 /* Invalid parameters */
45 #define SD_RES_SYSTEM_ERROR 0x06 /* System error */
46 #define SD_RES_VDI_LOCKED 0x07 /* Vdi is locked */
47 #define SD_RES_NO_VDI 0x08 /* No vdi found */
48 #define SD_RES_NO_BASE_VDI 0x09 /* No base vdi found */
49 #define SD_RES_VDI_READ 0x0A /* Cannot read requested vdi */
50 #define SD_RES_VDI_WRITE 0x0B /* Cannot write requested vdi */
51 #define SD_RES_BASE_VDI_READ 0x0C /* Cannot read base vdi */
52 #define SD_RES_BASE_VDI_WRITE 0x0D /* Cannot write base vdi */
53 #define SD_RES_NO_TAG 0x0E /* Requested tag is not found */
54 #define SD_RES_STARTUP 0x0F /* Sheepdog is on starting up */
55 #define SD_RES_VDI_NOT_LOCKED 0x10 /* Vdi is not locked */
56 #define SD_RES_SHUTDOWN 0x11 /* Sheepdog is shutting down */
57 #define SD_RES_NO_MEM 0x12 /* Cannot allocate memory */
58 #define SD_RES_FULL_VDI 0x13 /* we already have the maximum vdis */
59 #define SD_RES_VER_MISMATCH 0x14 /* Protocol version mismatch */
60 #define SD_RES_NO_SPACE 0x15 /* Server has no room for new objects */
61 #define SD_RES_WAIT_FOR_FORMAT 0x16 /* Waiting for a format operation */
62 #define SD_RES_WAIT_FOR_JOIN 0x17 /* Waiting for other nodes joining */
63 #define SD_RES_JOIN_FAILED 0x18 /* Target node had failed to join sheepdog */
64
65 /*
66 * Object ID rules
67 *
68 * 0 - 19 (20 bits): data object space
69 * 20 - 31 (12 bits): reserved data object space
70 * 32 - 55 (24 bits): vdi object space
71 * 56 - 59 ( 4 bits): reserved vdi object space
72 * 60 - 63 ( 4 bits): object type identifier space
73 */
74
75 #define VDI_SPACE_SHIFT 32
76 #define VDI_BIT (UINT64_C(1) << 63)
77 #define VMSTATE_BIT (UINT64_C(1) << 62)
78 #define MAX_DATA_OBJS (UINT64_C(1) << 20)
79 #define MAX_CHILDREN 1024
80 #define SD_MAX_VDI_LEN 256
81 #define SD_MAX_VDI_TAG_LEN 256
82 #define SD_NR_VDIS (1U << 24)
83 #define SD_DATA_OBJ_SIZE (UINT64_C(1) << 22)
84 #define SD_MAX_VDI_SIZE (SD_DATA_OBJ_SIZE * MAX_DATA_OBJS)
85 #define SECTOR_SIZE 512
86
87 #define SD_INODE_SIZE (sizeof(SheepdogInode))
88 #define CURRENT_VDI_ID 0
89
90 typedef struct SheepdogReq {
91 uint8_t proto_ver;
92 uint8_t opcode;
93 uint16_t flags;
94 uint32_t epoch;
95 uint32_t id;
96 uint32_t data_length;
97 uint32_t opcode_specific[8];
98 } SheepdogReq;
99
100 typedef struct SheepdogRsp {
101 uint8_t proto_ver;
102 uint8_t opcode;
103 uint16_t flags;
104 uint32_t epoch;
105 uint32_t id;
106 uint32_t data_length;
107 uint32_t result;
108 uint32_t opcode_specific[7];
109 } SheepdogRsp;
110
111 typedef struct SheepdogObjReq {
112 uint8_t proto_ver;
113 uint8_t opcode;
114 uint16_t flags;
115 uint32_t epoch;
116 uint32_t id;
117 uint32_t data_length;
118 uint64_t oid;
119 uint64_t cow_oid;
120 uint32_t copies;
121 uint32_t rsvd;
122 uint64_t offset;
123 } SheepdogObjReq;
124
125 typedef struct SheepdogObjRsp {
126 uint8_t proto_ver;
127 uint8_t opcode;
128 uint16_t flags;
129 uint32_t epoch;
130 uint32_t id;
131 uint32_t data_length;
132 uint32_t result;
133 uint32_t copies;
134 uint32_t pad[6];
135 } SheepdogObjRsp;
136
137 typedef struct SheepdogVdiReq {
138 uint8_t proto_ver;
139 uint8_t opcode;
140 uint16_t flags;
141 uint32_t epoch;
142 uint32_t id;
143 uint32_t data_length;
144 uint64_t vdi_size;
145 uint32_t base_vdi_id;
146 uint32_t copies;
147 uint32_t snapid;
148 uint32_t pad[3];
149 } SheepdogVdiReq;
150
151 typedef struct SheepdogVdiRsp {
152 uint8_t proto_ver;
153 uint8_t opcode;
154 uint16_t flags;
155 uint32_t epoch;
156 uint32_t id;
157 uint32_t data_length;
158 uint32_t result;
159 uint32_t rsvd;
160 uint32_t vdi_id;
161 uint32_t pad[5];
162 } SheepdogVdiRsp;
163
164 typedef struct SheepdogInode {
165 char name[SD_MAX_VDI_LEN];
166 char tag[SD_MAX_VDI_TAG_LEN];
167 uint64_t ctime;
168 uint64_t snap_ctime;
169 uint64_t vm_clock_nsec;
170 uint64_t vdi_size;
171 uint64_t vm_state_size;
172 uint16_t copy_policy;
173 uint8_t nr_copies;
174 uint8_t block_size_shift;
175 uint32_t snap_id;
176 uint32_t vdi_id;
177 uint32_t parent_vdi_id;
178 uint32_t child_vdi_id[MAX_CHILDREN];
179 uint32_t data_vdi_id[MAX_DATA_OBJS];
180 } SheepdogInode;
181
182 /*
183 * 64 bit FNV-1a non-zero initial basis
184 */
185 #define FNV1A_64_INIT ((uint64_t)0xcbf29ce484222325ULL)
186
187 /*
188 * 64 bit Fowler/Noll/Vo FNV-1a hash code
189 */
190 static inline uint64_t fnv_64a_buf(void *buf, size_t len, uint64_t hval)
191 {
192 unsigned char *bp = buf;
193 unsigned char *be = bp + len;
194 while (bp < be) {
195 hval ^= (uint64_t) *bp++;
196 hval += (hval << 1) + (hval << 4) + (hval << 5) +
197 (hval << 7) + (hval << 8) + (hval << 40);
198 }
199 return hval;
200 }
201
202 static inline int is_data_obj_writable(SheepdogInode *inode, unsigned int idx)
203 {
204 return inode->vdi_id == inode->data_vdi_id[idx];
205 }
206
207 static inline int is_data_obj(uint64_t oid)
208 {
209 return !(VDI_BIT & oid);
210 }
211
212 static inline uint64_t data_oid_to_idx(uint64_t oid)
213 {
214 return oid & (MAX_DATA_OBJS - 1);
215 }
216
217 static inline uint64_t vid_to_vdi_oid(uint32_t vid)
218 {
219 return VDI_BIT | ((uint64_t)vid << VDI_SPACE_SHIFT);
220 }
221
222 static inline uint64_t vid_to_vmstate_oid(uint32_t vid, uint32_t idx)
223 {
224 return VMSTATE_BIT | ((uint64_t)vid << VDI_SPACE_SHIFT) | idx;
225 }
226
227 static inline uint64_t vid_to_data_oid(uint32_t vid, uint32_t idx)
228 {
229 return ((uint64_t)vid << VDI_SPACE_SHIFT) | idx;
230 }
231
232 static inline int is_snapshot(struct SheepdogInode *inode)
233 {
234 return !!inode->snap_ctime;
235 }
236
237 #undef dprintf
238 #ifdef DEBUG_SDOG
239 #define dprintf(fmt, args...) \
240 do { \
241 fprintf(stdout, "%s %d: " fmt, __func__, __LINE__, ##args); \
242 } while (0)
243 #else
244 #define dprintf(fmt, args...)
245 #endif
246
247 typedef struct SheepdogAIOCB SheepdogAIOCB;
248
249 typedef struct AIOReq {
250 SheepdogAIOCB *aiocb;
251 unsigned int iov_offset;
252
253 uint64_t oid;
254 uint64_t base_oid;
255 uint64_t offset;
256 unsigned int data_len;
257 uint8_t flags;
258 uint32_t id;
259
260 QLIST_ENTRY(AIOReq) outstanding_aio_siblings;
261 QLIST_ENTRY(AIOReq) aioreq_siblings;
262 } AIOReq;
263
264 enum AIOCBState {
265 AIOCB_WRITE_UDATA,
266 AIOCB_READ_UDATA,
267 };
268
269 struct SheepdogAIOCB {
270 BlockDriverAIOCB common;
271
272 QEMUIOVector *qiov;
273
274 int64_t sector_num;
275 int nb_sectors;
276
277 int ret;
278 enum AIOCBState aiocb_type;
279
280 Coroutine *coroutine;
281 void (*aio_done_func)(SheepdogAIOCB *);
282
283 int canceled;
284
285 QLIST_HEAD(aioreq_head, AIOReq) aioreq_head;
286 };
287
288 typedef struct BDRVSheepdogState {
289 SheepdogInode inode;
290
291 uint32_t min_dirty_data_idx;
292 uint32_t max_dirty_data_idx;
293
294 char name[SD_MAX_VDI_LEN];
295 int is_snapshot;
296
297 char *addr;
298 char *port;
299 int fd;
300
301 CoMutex lock;
302 Coroutine *co_send;
303 Coroutine *co_recv;
304
305 uint32_t aioreq_seq_num;
306 QLIST_HEAD(outstanding_aio_head, AIOReq) outstanding_aio_head;
307 } BDRVSheepdogState;
308
309 static const char * sd_strerror(int err)
310 {
311 int i;
312
313 static const struct {
314 int err;
315 const char *desc;
316 } errors[] = {
317 {SD_RES_SUCCESS, "Success"},
318 {SD_RES_UNKNOWN, "Unknown error"},
319 {SD_RES_NO_OBJ, "No object found"},
320 {SD_RES_EIO, "I/O error"},
321 {SD_RES_VDI_EXIST, "VDI exists already"},
322 {SD_RES_INVALID_PARMS, "Invalid parameters"},
323 {SD_RES_SYSTEM_ERROR, "System error"},
324 {SD_RES_VDI_LOCKED, "VDI is already locked"},
325 {SD_RES_NO_VDI, "No vdi found"},
326 {SD_RES_NO_BASE_VDI, "No base VDI found"},
327 {SD_RES_VDI_READ, "Failed read the requested VDI"},
328 {SD_RES_VDI_WRITE, "Failed to write the requested VDI"},
329 {SD_RES_BASE_VDI_READ, "Failed to read the base VDI"},
330 {SD_RES_BASE_VDI_WRITE, "Failed to write the base VDI"},
331 {SD_RES_NO_TAG, "Failed to find the requested tag"},
332 {SD_RES_STARTUP, "The system is still booting"},
333 {SD_RES_VDI_NOT_LOCKED, "VDI isn't locked"},
334 {SD_RES_SHUTDOWN, "The system is shutting down"},
335 {SD_RES_NO_MEM, "Out of memory on the server"},
336 {SD_RES_FULL_VDI, "We already have the maximum vdis"},
337 {SD_RES_VER_MISMATCH, "Protocol version mismatch"},
338 {SD_RES_NO_SPACE, "Server has no space for new objects"},
339 {SD_RES_WAIT_FOR_FORMAT, "Sheepdog is waiting for a format operation"},
340 {SD_RES_WAIT_FOR_JOIN, "Sheepdog is waiting for other nodes joining"},
341 {SD_RES_JOIN_FAILED, "Target node had failed to join sheepdog"},
342 };
343
344 for (i = 0; i < ARRAY_SIZE(errors); ++i) {
345 if (errors[i].err == err) {
346 return errors[i].desc;
347 }
348 }
349
350 return "Invalid error code";
351 }
352
353 /*
354 * Sheepdog I/O handling:
355 *
356 * 1. In sd_co_rw_vector, we send the I/O requests to the server and
357 * link the requests to the outstanding_list in the
358 * BDRVSheepdogState. The function exits without waiting for
359 * receiving the response.
360 *
361 * 2. We receive the response in aio_read_response, the fd handler to
362 * the sheepdog connection. If metadata update is needed, we send
363 * the write request to the vdi object in sd_write_done, the write
364 * completion function. We switch back to sd_co_readv/writev after
365 * all the requests belonging to the AIOCB are finished.
366 */
367
368 static inline AIOReq *alloc_aio_req(BDRVSheepdogState *s, SheepdogAIOCB *acb,
369 uint64_t oid, unsigned int data_len,
370 uint64_t offset, uint8_t flags,
371 uint64_t base_oid, unsigned int iov_offset)
372 {
373 AIOReq *aio_req;
374
375 aio_req = g_malloc(sizeof(*aio_req));
376 aio_req->aiocb = acb;
377 aio_req->iov_offset = iov_offset;
378 aio_req->oid = oid;
379 aio_req->base_oid = base_oid;
380 aio_req->offset = offset;
381 aio_req->data_len = data_len;
382 aio_req->flags = flags;
383 aio_req->id = s->aioreq_seq_num++;
384
385 QLIST_INSERT_HEAD(&s->outstanding_aio_head, aio_req,
386 outstanding_aio_siblings);
387 QLIST_INSERT_HEAD(&acb->aioreq_head, aio_req, aioreq_siblings);
388
389 return aio_req;
390 }
391
392 static inline int free_aio_req(BDRVSheepdogState *s, AIOReq *aio_req)
393 {
394 SheepdogAIOCB *acb = aio_req->aiocb;
395 QLIST_REMOVE(aio_req, outstanding_aio_siblings);
396 QLIST_REMOVE(aio_req, aioreq_siblings);
397 g_free(aio_req);
398
399 return !QLIST_EMPTY(&acb->aioreq_head);
400 }
401
402 static void coroutine_fn sd_finish_aiocb(SheepdogAIOCB *acb)
403 {
404 if (!acb->canceled) {
405 qemu_coroutine_enter(acb->coroutine, NULL);
406 }
407 qemu_aio_release(acb);
408 }
409
410 static void sd_aio_cancel(BlockDriverAIOCB *blockacb)
411 {
412 SheepdogAIOCB *acb = (SheepdogAIOCB *)blockacb;
413
414 /*
415 * Sheepdog cannot cancel the requests which are already sent to
416 * the servers, so we just complete the request with -EIO here.
417 */
418 acb->ret = -EIO;
419 qemu_coroutine_enter(acb->coroutine, NULL);
420 acb->canceled = 1;
421 }
422
423 static AIOPool sd_aio_pool = {
424 .aiocb_size = sizeof(SheepdogAIOCB),
425 .cancel = sd_aio_cancel,
426 };
427
428 static SheepdogAIOCB *sd_aio_setup(BlockDriverState *bs, QEMUIOVector *qiov,
429 int64_t sector_num, int nb_sectors,
430 BlockDriverCompletionFunc *cb, void *opaque)
431 {
432 SheepdogAIOCB *acb;
433
434 acb = qemu_aio_get(&sd_aio_pool, bs, cb, opaque);
435
436 acb->qiov = qiov;
437
438 acb->sector_num = sector_num;
439 acb->nb_sectors = nb_sectors;
440
441 acb->aio_done_func = NULL;
442 acb->canceled = 0;
443 acb->coroutine = qemu_coroutine_self();
444 acb->ret = 0;
445 QLIST_INIT(&acb->aioreq_head);
446 return acb;
447 }
448
449 static int connect_to_sdog(const char *addr, const char *port)
450 {
451 char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
452 int fd, ret;
453 struct addrinfo hints, *res, *res0;
454
455 if (!addr) {
456 addr = SD_DEFAULT_ADDR;
457 port = SD_DEFAULT_PORT;
458 }
459
460 memset(&hints, 0, sizeof(hints));
461 hints.ai_socktype = SOCK_STREAM;
462
463 ret = getaddrinfo(addr, port, &hints, &res0);
464 if (ret) {
465 error_report("unable to get address info %s, %s",
466 addr, strerror(errno));
467 return -1;
468 }
469
470 for (res = res0; res; res = res->ai_next) {
471 ret = getnameinfo(res->ai_addr, res->ai_addrlen, hbuf, sizeof(hbuf),
472 sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV);
473 if (ret) {
474 continue;
475 }
476
477 fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
478 if (fd < 0) {
479 continue;
480 }
481
482 reconnect:
483 ret = connect(fd, res->ai_addr, res->ai_addrlen);
484 if (ret < 0) {
485 if (errno == EINTR) {
486 goto reconnect;
487 }
488 break;
489 }
490
491 dprintf("connected to %s:%s\n", addr, port);
492 goto success;
493 }
494 fd = -1;
495 error_report("failed connect to %s:%s", addr, port);
496 success:
497 freeaddrinfo(res0);
498 return fd;
499 }
500
501 static int send_req(int sockfd, SheepdogReq *hdr, void *data,
502 unsigned int *wlen)
503 {
504 int ret;
505
506 ret = qemu_send_full(sockfd, hdr, sizeof(*hdr), 0);
507 if (ret < sizeof(*hdr)) {
508 error_report("failed to send a req, %s", strerror(errno));
509 }
510
511 ret = qemu_send_full(sockfd, data, *wlen, 0);
512 if (ret < *wlen) {
513 error_report("failed to send a req, %s", strerror(errno));
514 }
515
516 return ret;
517 }
518
519 static int do_req(int sockfd, SheepdogReq *hdr, void *data,
520 unsigned int *wlen, unsigned int *rlen)
521 {
522 int ret;
523
524 socket_set_block(sockfd);
525 ret = send_req(sockfd, hdr, data, wlen);
526 if (ret < 0) {
527 goto out;
528 }
529
530 ret = qemu_recv_full(sockfd, hdr, sizeof(*hdr), 0);
531 if (ret < sizeof(*hdr)) {
532 error_report("failed to get a rsp, %s", strerror(errno));
533 goto out;
534 }
535
536 if (*rlen > hdr->data_length) {
537 *rlen = hdr->data_length;
538 }
539
540 if (*rlen) {
541 ret = qemu_recv_full(sockfd, data, *rlen, 0);
542 if (ret < *rlen) {
543 error_report("failed to get the data, %s", strerror(errno));
544 goto out;
545 }
546 }
547 ret = 0;
548 out:
549 socket_set_nonblock(sockfd);
550 return ret;
551 }
552
553 static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
554 struct iovec *iov, int niov, int create,
555 enum AIOCBState aiocb_type);
556
557 /*
558 * This function searchs pending requests to the object `oid', and
559 * sends them.
560 */
561 static void coroutine_fn send_pending_req(BDRVSheepdogState *s, uint64_t oid, uint32_t id)
562 {
563 AIOReq *aio_req, *next;
564 SheepdogAIOCB *acb;
565 int ret;
566
567 QLIST_FOREACH_SAFE(aio_req, &s->outstanding_aio_head,
568 outstanding_aio_siblings, next) {
569 if (id == aio_req->id) {
570 continue;
571 }
572 if (aio_req->oid != oid) {
573 continue;
574 }
575
576 acb = aio_req->aiocb;
577 ret = add_aio_request(s, aio_req, acb->qiov->iov,
578 acb->qiov->niov, 0, acb->aiocb_type);
579 if (ret < 0) {
580 error_report("add_aio_request is failed");
581 free_aio_req(s, aio_req);
582 if (QLIST_EMPTY(&acb->aioreq_head)) {
583 sd_finish_aiocb(acb);
584 }
585 }
586 }
587 }
588
589 /*
590 * Receive responses of the I/O requests.
591 *
592 * This function is registered as a fd handler, and called from the
593 * main loop when s->fd is ready for reading responses.
594 */
595 static void coroutine_fn aio_read_response(void *opaque)
596 {
597 SheepdogObjRsp rsp;
598 BDRVSheepdogState *s = opaque;
599 int fd = s->fd;
600 int ret;
601 AIOReq *aio_req = NULL;
602 SheepdogAIOCB *acb;
603 int rest;
604 unsigned long idx;
605
606 if (QLIST_EMPTY(&s->outstanding_aio_head)) {
607 goto out;
608 }
609
610 /* read a header */
611 ret = qemu_co_recv(fd, &rsp, sizeof(rsp));
612 if (ret < 0) {
613 error_report("failed to get the header, %s", strerror(errno));
614 goto out;
615 }
616
617 /* find the right aio_req from the outstanding_aio list */
618 QLIST_FOREACH(aio_req, &s->outstanding_aio_head, outstanding_aio_siblings) {
619 if (aio_req->id == rsp.id) {
620 break;
621 }
622 }
623 if (!aio_req) {
624 error_report("cannot find aio_req %x", rsp.id);
625 goto out;
626 }
627
628 acb = aio_req->aiocb;
629
630 switch (acb->aiocb_type) {
631 case AIOCB_WRITE_UDATA:
632 if (!is_data_obj(aio_req->oid)) {
633 break;
634 }
635 idx = data_oid_to_idx(aio_req->oid);
636
637 if (s->inode.data_vdi_id[idx] != s->inode.vdi_id) {
638 /*
639 * If the object is newly created one, we need to update
640 * the vdi object (metadata object). min_dirty_data_idx
641 * and max_dirty_data_idx are changed to include updated
642 * index between them.
643 */
644 s->inode.data_vdi_id[idx] = s->inode.vdi_id;
645 s->max_dirty_data_idx = MAX(idx, s->max_dirty_data_idx);
646 s->min_dirty_data_idx = MIN(idx, s->min_dirty_data_idx);
647
648 /*
649 * Some requests may be blocked because simultaneous
650 * create requests are not allowed, so we search the
651 * pending requests here.
652 */
653 send_pending_req(s, vid_to_data_oid(s->inode.vdi_id, idx), rsp.id);
654 }
655 break;
656 case AIOCB_READ_UDATA:
657 ret = qemu_co_recvv(fd, acb->qiov->iov, rsp.data_length,
658 aio_req->iov_offset);
659 if (ret < 0) {
660 error_report("failed to get the data, %s", strerror(errno));
661 goto out;
662 }
663 break;
664 }
665
666 if (rsp.result != SD_RES_SUCCESS) {
667 acb->ret = -EIO;
668 error_report("%s", sd_strerror(rsp.result));
669 }
670
671 rest = free_aio_req(s, aio_req);
672 if (!rest) {
673 /*
674 * We've finished all requests which belong to the AIOCB, so
675 * we can switch back to sd_co_readv/writev now.
676 */
677 acb->aio_done_func(acb);
678 }
679 out:
680 s->co_recv = NULL;
681 }
682
683 static void co_read_response(void *opaque)
684 {
685 BDRVSheepdogState *s = opaque;
686
687 if (!s->co_recv) {
688 s->co_recv = qemu_coroutine_create(aio_read_response);
689 }
690
691 qemu_coroutine_enter(s->co_recv, opaque);
692 }
693
694 static void co_write_request(void *opaque)
695 {
696 BDRVSheepdogState *s = opaque;
697
698 qemu_coroutine_enter(s->co_send, NULL);
699 }
700
701 static int aio_flush_request(void *opaque)
702 {
703 BDRVSheepdogState *s = opaque;
704
705 return !QLIST_EMPTY(&s->outstanding_aio_head);
706 }
707
708 static int set_nodelay(int fd)
709 {
710 int ret, opt;
711
712 opt = 1;
713 ret = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char *)&opt, sizeof(opt));
714 return ret;
715 }
716
717 /*
718 * Return a socket discriptor to read/write objects.
719 *
720 * We cannot use this discriptor for other operations because
721 * the block driver may be on waiting response from the server.
722 */
723 static int get_sheep_fd(BDRVSheepdogState *s)
724 {
725 int ret, fd;
726
727 fd = connect_to_sdog(s->addr, s->port);
728 if (fd < 0) {
729 error_report("%s", strerror(errno));
730 return -1;
731 }
732
733 socket_set_nonblock(fd);
734
735 ret = set_nodelay(fd);
736 if (ret) {
737 error_report("%s", strerror(errno));
738 closesocket(fd);
739 return -1;
740 }
741
742 qemu_aio_set_fd_handler(fd, co_read_response, NULL, aio_flush_request,
743 NULL, s);
744 return fd;
745 }
746
747 /*
748 * Parse a filename
749 *
750 * filename must be one of the following formats:
751 * 1. [vdiname]
752 * 2. [vdiname]:[snapid]
753 * 3. [vdiname]:[tag]
754 * 4. [hostname]:[port]:[vdiname]
755 * 5. [hostname]:[port]:[vdiname]:[snapid]
756 * 6. [hostname]:[port]:[vdiname]:[tag]
757 *
758 * You can boot from the snapshot images by specifying `snapid` or
759 * `tag'.
760 *
761 * You can run VMs outside the Sheepdog cluster by specifying
762 * `hostname' and `port' (experimental).
763 */
764 static int parse_vdiname(BDRVSheepdogState *s, const char *filename,
765 char *vdi, uint32_t *snapid, char *tag)
766 {
767 char *p, *q;
768 int nr_sep;
769
770 p = q = g_strdup(filename);
771
772 /* count the number of separators */
773 nr_sep = 0;
774 while (*p) {
775 if (*p == ':') {
776 nr_sep++;
777 }
778 p++;
779 }
780 p = q;
781
782 /* use the first two tokens as hostname and port number. */
783 if (nr_sep >= 2) {
784 s->addr = p;
785 p = strchr(p, ':');
786 *p++ = '\0';
787
788 s->port = p;
789 p = strchr(p, ':');
790 *p++ = '\0';
791 } else {
792 s->addr = NULL;
793 s->port = 0;
794 }
795
796 strncpy(vdi, p, SD_MAX_VDI_LEN);
797
798 p = strchr(vdi, ':');
799 if (p) {
800 *p++ = '\0';
801 *snapid = strtoul(p, NULL, 10);
802 if (*snapid == 0) {
803 strncpy(tag, p, SD_MAX_VDI_TAG_LEN);
804 }
805 } else {
806 *snapid = CURRENT_VDI_ID; /* search current vdi */
807 }
808
809 if (s->addr == NULL) {
810 g_free(q);
811 }
812
813 return 0;
814 }
815
816 static int find_vdi_name(BDRVSheepdogState *s, char *filename, uint32_t snapid,
817 char *tag, uint32_t *vid, int for_snapshot)
818 {
819 int ret, fd;
820 SheepdogVdiReq hdr;
821 SheepdogVdiRsp *rsp = (SheepdogVdiRsp *)&hdr;
822 unsigned int wlen, rlen = 0;
823 char buf[SD_MAX_VDI_LEN + SD_MAX_VDI_TAG_LEN];
824
825 fd = connect_to_sdog(s->addr, s->port);
826 if (fd < 0) {
827 return -1;
828 }
829
830 memset(buf, 0, sizeof(buf));
831 strncpy(buf, filename, SD_MAX_VDI_LEN);
832 strncpy(buf + SD_MAX_VDI_LEN, tag, SD_MAX_VDI_TAG_LEN);
833
834 memset(&hdr, 0, sizeof(hdr));
835 if (for_snapshot) {
836 hdr.opcode = SD_OP_GET_VDI_INFO;
837 } else {
838 hdr.opcode = SD_OP_LOCK_VDI;
839 }
840 wlen = SD_MAX_VDI_LEN + SD_MAX_VDI_TAG_LEN;
841 hdr.proto_ver = SD_PROTO_VER;
842 hdr.data_length = wlen;
843 hdr.snapid = snapid;
844 hdr.flags = SD_FLAG_CMD_WRITE;
845
846 ret = do_req(fd, (SheepdogReq *)&hdr, buf, &wlen, &rlen);
847 if (ret) {
848 ret = -1;
849 goto out;
850 }
851
852 if (rsp->result != SD_RES_SUCCESS) {
853 error_report("cannot get vdi info, %s, %s %d %s",
854 sd_strerror(rsp->result), filename, snapid, tag);
855 ret = -1;
856 goto out;
857 }
858 *vid = rsp->vdi_id;
859
860 ret = 0;
861 out:
862 closesocket(fd);
863 return ret;
864 }
865
866 static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
867 struct iovec *iov, int niov, int create,
868 enum AIOCBState aiocb_type)
869 {
870 int nr_copies = s->inode.nr_copies;
871 SheepdogObjReq hdr;
872 unsigned int wlen;
873 int ret;
874 uint64_t oid = aio_req->oid;
875 unsigned int datalen = aio_req->data_len;
876 uint64_t offset = aio_req->offset;
877 uint8_t flags = aio_req->flags;
878 uint64_t old_oid = aio_req->base_oid;
879
880 if (!nr_copies) {
881 error_report("bug");
882 }
883
884 memset(&hdr, 0, sizeof(hdr));
885
886 if (aiocb_type == AIOCB_READ_UDATA) {
887 wlen = 0;
888 hdr.opcode = SD_OP_READ_OBJ;
889 hdr.flags = flags;
890 } else if (create) {
891 wlen = datalen;
892 hdr.opcode = SD_OP_CREATE_AND_WRITE_OBJ;
893 hdr.flags = SD_FLAG_CMD_WRITE | flags;
894 } else {
895 wlen = datalen;
896 hdr.opcode = SD_OP_WRITE_OBJ;
897 hdr.flags = SD_FLAG_CMD_WRITE | flags;
898 }
899
900 hdr.oid = oid;
901 hdr.cow_oid = old_oid;
902 hdr.copies = s->inode.nr_copies;
903
904 hdr.data_length = datalen;
905 hdr.offset = offset;
906
907 hdr.id = aio_req->id;
908
909 qemu_co_mutex_lock(&s->lock);
910 s->co_send = qemu_coroutine_self();
911 qemu_aio_set_fd_handler(s->fd, co_read_response, co_write_request,
912 aio_flush_request, NULL, s);
913 socket_set_cork(s->fd, 1);
914
915 /* send a header */
916 ret = qemu_co_send(s->fd, &hdr, sizeof(hdr));
917 if (ret < 0) {
918 qemu_co_mutex_unlock(&s->lock);
919 error_report("failed to send a req, %s", strerror(errno));
920 return -EIO;
921 }
922
923 if (wlen) {
924 ret = qemu_co_sendv(s->fd, iov, wlen, aio_req->iov_offset);
925 if (ret < 0) {
926 qemu_co_mutex_unlock(&s->lock);
927 error_report("failed to send a data, %s", strerror(errno));
928 return -EIO;
929 }
930 }
931
932 socket_set_cork(s->fd, 0);
933 qemu_aio_set_fd_handler(s->fd, co_read_response, NULL,
934 aio_flush_request, NULL, s);
935 qemu_co_mutex_unlock(&s->lock);
936
937 return 0;
938 }
939
940 static int read_write_object(int fd, char *buf, uint64_t oid, int copies,
941 unsigned int datalen, uint64_t offset,
942 int write, int create)
943 {
944 SheepdogObjReq hdr;
945 SheepdogObjRsp *rsp = (SheepdogObjRsp *)&hdr;
946 unsigned int wlen, rlen;
947 int ret;
948
949 memset(&hdr, 0, sizeof(hdr));
950
951 if (write) {
952 wlen = datalen;
953 rlen = 0;
954 hdr.flags = SD_FLAG_CMD_WRITE;
955 if (create) {
956 hdr.opcode = SD_OP_CREATE_AND_WRITE_OBJ;
957 } else {
958 hdr.opcode = SD_OP_WRITE_OBJ;
959 }
960 } else {
961 wlen = 0;
962 rlen = datalen;
963 hdr.opcode = SD_OP_READ_OBJ;
964 }
965 hdr.oid = oid;
966 hdr.data_length = datalen;
967 hdr.offset = offset;
968 hdr.copies = copies;
969
970 ret = do_req(fd, (SheepdogReq *)&hdr, buf, &wlen, &rlen);
971 if (ret) {
972 error_report("failed to send a request to the sheep");
973 return -1;
974 }
975
976 switch (rsp->result) {
977 case SD_RES_SUCCESS:
978 return 0;
979 default:
980 error_report("%s", sd_strerror(rsp->result));
981 return -1;
982 }
983 }
984
985 static int read_object(int fd, char *buf, uint64_t oid, int copies,
986 unsigned int datalen, uint64_t offset)
987 {
988 return read_write_object(fd, buf, oid, copies, datalen, offset, 0, 0);
989 }
990
991 static int write_object(int fd, char *buf, uint64_t oid, int copies,
992 unsigned int datalen, uint64_t offset, int create)
993 {
994 return read_write_object(fd, buf, oid, copies, datalen, offset, 1, create);
995 }
996
997 static int sd_open(BlockDriverState *bs, const char *filename, int flags)
998 {
999 int ret, fd;
1000 uint32_t vid = 0;
1001 BDRVSheepdogState *s = bs->opaque;
1002 char vdi[SD_MAX_VDI_LEN], tag[SD_MAX_VDI_TAG_LEN];
1003 uint32_t snapid;
1004 char *buf = NULL;
1005
1006 strstart(filename, "sheepdog:", (const char **)&filename);
1007
1008 QLIST_INIT(&s->outstanding_aio_head);
1009 s->fd = -1;
1010
1011 memset(vdi, 0, sizeof(vdi));
1012 memset(tag, 0, sizeof(tag));
1013 if (parse_vdiname(s, filename, vdi, &snapid, tag) < 0) {
1014 goto out;
1015 }
1016 s->fd = get_sheep_fd(s);
1017 if (s->fd < 0) {
1018 goto out;
1019 }
1020
1021 ret = find_vdi_name(s, vdi, snapid, tag, &vid, 0);
1022 if (ret) {
1023 goto out;
1024 }
1025
1026 if (snapid) {
1027 dprintf("%" PRIx32 " snapshot inode was open.\n", vid);
1028 s->is_snapshot = 1;
1029 }
1030
1031 fd = connect_to_sdog(s->addr, s->port);
1032 if (fd < 0) {
1033 error_report("failed to connect");
1034 goto out;
1035 }
1036
1037 buf = g_malloc(SD_INODE_SIZE);
1038 ret = read_object(fd, buf, vid_to_vdi_oid(vid), 0, SD_INODE_SIZE, 0);
1039
1040 closesocket(fd);
1041
1042 if (ret) {
1043 goto out;
1044 }
1045
1046 memcpy(&s->inode, buf, sizeof(s->inode));
1047 s->min_dirty_data_idx = UINT32_MAX;
1048 s->max_dirty_data_idx = 0;
1049
1050 bs->total_sectors = s->inode.vdi_size / SECTOR_SIZE;
1051 strncpy(s->name, vdi, sizeof(s->name));
1052 qemu_co_mutex_init(&s->lock);
1053 g_free(buf);
1054 return 0;
1055 out:
1056 qemu_aio_set_fd_handler(s->fd, NULL, NULL, NULL, NULL, NULL);
1057 if (s->fd >= 0) {
1058 closesocket(s->fd);
1059 }
1060 g_free(buf);
1061 return -1;
1062 }
1063
1064 static int do_sd_create(char *filename, int64_t vdi_size,
1065 uint32_t base_vid, uint32_t *vdi_id, int snapshot,
1066 const char *addr, const char *port)
1067 {
1068 SheepdogVdiReq hdr;
1069 SheepdogVdiRsp *rsp = (SheepdogVdiRsp *)&hdr;
1070 int fd, ret;
1071 unsigned int wlen, rlen = 0;
1072 char buf[SD_MAX_VDI_LEN];
1073
1074 fd = connect_to_sdog(addr, port);
1075 if (fd < 0) {
1076 return -EIO;
1077 }
1078
1079 memset(buf, 0, sizeof(buf));
1080 strncpy(buf, filename, SD_MAX_VDI_LEN);
1081
1082 memset(&hdr, 0, sizeof(hdr));
1083 hdr.opcode = SD_OP_NEW_VDI;
1084 hdr.base_vdi_id = base_vid;
1085
1086 wlen = SD_MAX_VDI_LEN;
1087
1088 hdr.flags = SD_FLAG_CMD_WRITE;
1089 hdr.snapid = snapshot;
1090
1091 hdr.data_length = wlen;
1092 hdr.vdi_size = vdi_size;
1093
1094 ret = do_req(fd, (SheepdogReq *)&hdr, buf, &wlen, &rlen);
1095
1096 closesocket(fd);
1097
1098 if (ret) {
1099 return -EIO;
1100 }
1101
1102 if (rsp->result != SD_RES_SUCCESS) {
1103 error_report("%s, %s", sd_strerror(rsp->result), filename);
1104 return -EIO;
1105 }
1106
1107 if (vdi_id) {
1108 *vdi_id = rsp->vdi_id;
1109 }
1110
1111 return 0;
1112 }
1113
1114 static int sd_prealloc(const char *filename)
1115 {
1116 BlockDriverState *bs = NULL;
1117 uint32_t idx, max_idx;
1118 int64_t vdi_size;
1119 void *buf = g_malloc0(SD_DATA_OBJ_SIZE);
1120 int ret;
1121
1122 ret = bdrv_file_open(&bs, filename, BDRV_O_RDWR);
1123 if (ret < 0) {
1124 goto out;
1125 }
1126
1127 vdi_size = bdrv_getlength(bs);
1128 if (vdi_size < 0) {
1129 ret = vdi_size;
1130 goto out;
1131 }
1132 max_idx = DIV_ROUND_UP(vdi_size, SD_DATA_OBJ_SIZE);
1133
1134 for (idx = 0; idx < max_idx; idx++) {
1135 /*
1136 * The created image can be a cloned image, so we need to read
1137 * a data from the source image.
1138 */
1139 ret = bdrv_pread(bs, idx * SD_DATA_OBJ_SIZE, buf, SD_DATA_OBJ_SIZE);
1140 if (ret < 0) {
1141 goto out;
1142 }
1143 ret = bdrv_pwrite(bs, idx * SD_DATA_OBJ_SIZE, buf, SD_DATA_OBJ_SIZE);
1144 if (ret < 0) {
1145 goto out;
1146 }
1147 }
1148 out:
1149 if (bs) {
1150 bdrv_delete(bs);
1151 }
1152 g_free(buf);
1153
1154 return ret;
1155 }
1156
1157 static int sd_create(const char *filename, QEMUOptionParameter *options)
1158 {
1159 int ret;
1160 uint32_t vid = 0, base_vid = 0;
1161 int64_t vdi_size = 0;
1162 char *backing_file = NULL;
1163 BDRVSheepdogState s;
1164 char vdi[SD_MAX_VDI_LEN], tag[SD_MAX_VDI_TAG_LEN];
1165 uint32_t snapid;
1166 int prealloc = 0;
1167 const char *vdiname;
1168
1169 strstart(filename, "sheepdog:", &vdiname);
1170
1171 memset(&s, 0, sizeof(s));
1172 memset(vdi, 0, sizeof(vdi));
1173 memset(tag, 0, sizeof(tag));
1174 if (parse_vdiname(&s, vdiname, vdi, &snapid, tag) < 0) {
1175 error_report("invalid filename");
1176 return -EINVAL;
1177 }
1178
1179 while (options && options->name) {
1180 if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
1181 vdi_size = options->value.n;
1182 } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FILE)) {
1183 backing_file = options->value.s;
1184 } else if (!strcmp(options->name, BLOCK_OPT_PREALLOC)) {
1185 if (!options->value.s || !strcmp(options->value.s, "off")) {
1186 prealloc = 0;
1187 } else if (!strcmp(options->value.s, "full")) {
1188 prealloc = 1;
1189 } else {
1190 error_report("Invalid preallocation mode: '%s'",
1191 options->value.s);
1192 return -EINVAL;
1193 }
1194 }
1195 options++;
1196 }
1197
1198 if (vdi_size > SD_MAX_VDI_SIZE) {
1199 error_report("too big image size");
1200 return -EINVAL;
1201 }
1202
1203 if (backing_file) {
1204 BlockDriverState *bs;
1205 BDRVSheepdogState *s;
1206 BlockDriver *drv;
1207
1208 /* Currently, only Sheepdog backing image is supported. */
1209 drv = bdrv_find_protocol(backing_file);
1210 if (!drv || strcmp(drv->protocol_name, "sheepdog") != 0) {
1211 error_report("backing_file must be a sheepdog image");
1212 return -EINVAL;
1213 }
1214
1215 ret = bdrv_file_open(&bs, backing_file, 0);
1216 if (ret < 0)
1217 return -EIO;
1218
1219 s = bs->opaque;
1220
1221 if (!is_snapshot(&s->inode)) {
1222 error_report("cannot clone from a non snapshot vdi");
1223 bdrv_delete(bs);
1224 return -EINVAL;
1225 }
1226
1227 base_vid = s->inode.vdi_id;
1228 bdrv_delete(bs);
1229 }
1230
1231 ret = do_sd_create(vdi, vdi_size, base_vid, &vid, 0, s.addr, s.port);
1232 if (!prealloc || ret) {
1233 return ret;
1234 }
1235
1236 return sd_prealloc(filename);
1237 }
1238
1239 static void sd_close(BlockDriverState *bs)
1240 {
1241 BDRVSheepdogState *s = bs->opaque;
1242 SheepdogVdiReq hdr;
1243 SheepdogVdiRsp *rsp = (SheepdogVdiRsp *)&hdr;
1244 unsigned int wlen, rlen = 0;
1245 int fd, ret;
1246
1247 dprintf("%s\n", s->name);
1248
1249 fd = connect_to_sdog(s->addr, s->port);
1250 if (fd < 0) {
1251 return;
1252 }
1253
1254 memset(&hdr, 0, sizeof(hdr));
1255
1256 hdr.opcode = SD_OP_RELEASE_VDI;
1257 wlen = strlen(s->name) + 1;
1258 hdr.data_length = wlen;
1259 hdr.flags = SD_FLAG_CMD_WRITE;
1260
1261 ret = do_req(fd, (SheepdogReq *)&hdr, s->name, &wlen, &rlen);
1262
1263 closesocket(fd);
1264
1265 if (!ret && rsp->result != SD_RES_SUCCESS &&
1266 rsp->result != SD_RES_VDI_NOT_LOCKED) {
1267 error_report("%s, %s", sd_strerror(rsp->result), s->name);
1268 }
1269
1270 qemu_aio_set_fd_handler(s->fd, NULL, NULL, NULL, NULL, NULL);
1271 closesocket(s->fd);
1272 g_free(s->addr);
1273 }
1274
1275 static int64_t sd_getlength(BlockDriverState *bs)
1276 {
1277 BDRVSheepdogState *s = bs->opaque;
1278
1279 return s->inode.vdi_size;
1280 }
1281
1282 static int sd_truncate(BlockDriverState *bs, int64_t offset)
1283 {
1284 BDRVSheepdogState *s = bs->opaque;
1285 int ret, fd;
1286 unsigned int datalen;
1287
1288 if (offset < s->inode.vdi_size) {
1289 error_report("shrinking is not supported");
1290 return -EINVAL;
1291 } else if (offset > SD_MAX_VDI_SIZE) {
1292 error_report("too big image size");
1293 return -EINVAL;
1294 }
1295
1296 fd = connect_to_sdog(s->addr, s->port);
1297 if (fd < 0) {
1298 return -EIO;
1299 }
1300
1301 /* we don't need to update entire object */
1302 datalen = SD_INODE_SIZE - sizeof(s->inode.data_vdi_id);
1303 s->inode.vdi_size = offset;
1304 ret = write_object(fd, (char *)&s->inode, vid_to_vdi_oid(s->inode.vdi_id),
1305 s->inode.nr_copies, datalen, 0, 0);
1306 close(fd);
1307
1308 if (ret < 0) {
1309 error_report("failed to update an inode.");
1310 return -EIO;
1311 }
1312
1313 return 0;
1314 }
1315
1316 /*
1317 * This function is called after writing data objects. If we need to
1318 * update metadata, this sends a write request to the vdi object.
1319 * Otherwise, this switches back to sd_co_readv/writev.
1320 */
1321 static void coroutine_fn sd_write_done(SheepdogAIOCB *acb)
1322 {
1323 int ret;
1324 BDRVSheepdogState *s = acb->common.bs->opaque;
1325 struct iovec iov;
1326 AIOReq *aio_req;
1327 uint32_t offset, data_len, mn, mx;
1328
1329 mn = s->min_dirty_data_idx;
1330 mx = s->max_dirty_data_idx;
1331 if (mn <= mx) {
1332 /* we need to update the vdi object. */
1333 offset = sizeof(s->inode) - sizeof(s->inode.data_vdi_id) +
1334 mn * sizeof(s->inode.data_vdi_id[0]);
1335 data_len = (mx - mn + 1) * sizeof(s->inode.data_vdi_id[0]);
1336
1337 s->min_dirty_data_idx = UINT32_MAX;
1338 s->max_dirty_data_idx = 0;
1339
1340 iov.iov_base = &s->inode;
1341 iov.iov_len = sizeof(s->inode);
1342 aio_req = alloc_aio_req(s, acb, vid_to_vdi_oid(s->inode.vdi_id),
1343 data_len, offset, 0, 0, offset);
1344 ret = add_aio_request(s, aio_req, &iov, 1, 0, AIOCB_WRITE_UDATA);
1345 if (ret) {
1346 free_aio_req(s, aio_req);
1347 acb->ret = -EIO;
1348 goto out;
1349 }
1350
1351 acb->aio_done_func = sd_finish_aiocb;
1352 acb->aiocb_type = AIOCB_WRITE_UDATA;
1353 return;
1354 }
1355 out:
1356 sd_finish_aiocb(acb);
1357 }
1358
1359 /*
1360 * Create a writable VDI from a snapshot
1361 */
1362 static int sd_create_branch(BDRVSheepdogState *s)
1363 {
1364 int ret, fd;
1365 uint32_t vid;
1366 char *buf;
1367
1368 dprintf("%" PRIx32 " is snapshot.\n", s->inode.vdi_id);
1369
1370 buf = g_malloc(SD_INODE_SIZE);
1371
1372 ret = do_sd_create(s->name, s->inode.vdi_size, s->inode.vdi_id, &vid, 1,
1373 s->addr, s->port);
1374 if (ret) {
1375 goto out;
1376 }
1377
1378 dprintf("%" PRIx32 " is created.\n", vid);
1379
1380 fd = connect_to_sdog(s->addr, s->port);
1381 if (fd < 0) {
1382 error_report("failed to connect");
1383 goto out;
1384 }
1385
1386 ret = read_object(fd, buf, vid_to_vdi_oid(vid), s->inode.nr_copies,
1387 SD_INODE_SIZE, 0);
1388
1389 closesocket(fd);
1390
1391 if (ret < 0) {
1392 goto out;
1393 }
1394
1395 memcpy(&s->inode, buf, sizeof(s->inode));
1396
1397 s->is_snapshot = 0;
1398 ret = 0;
1399 dprintf("%" PRIx32 " was newly created.\n", s->inode.vdi_id);
1400
1401 out:
1402 g_free(buf);
1403
1404 return ret;
1405 }
1406
1407 /*
1408 * Send I/O requests to the server.
1409 *
1410 * This function sends requests to the server, links the requests to
1411 * the outstanding_list in BDRVSheepdogState, and exits without
1412 * waiting the response. The responses are received in the
1413 * `aio_read_response' function which is called from the main loop as
1414 * a fd handler.
1415 *
1416 * Returns 1 when we need to wait a response, 0 when there is no sent
1417 * request and -errno in error cases.
1418 */
1419 static int coroutine_fn sd_co_rw_vector(void *p)
1420 {
1421 SheepdogAIOCB *acb = p;
1422 int ret = 0;
1423 unsigned long len, done = 0, total = acb->nb_sectors * SECTOR_SIZE;
1424 unsigned long idx = acb->sector_num * SECTOR_SIZE / SD_DATA_OBJ_SIZE;
1425 uint64_t oid;
1426 uint64_t offset = (acb->sector_num * SECTOR_SIZE) % SD_DATA_OBJ_SIZE;
1427 BDRVSheepdogState *s = acb->common.bs->opaque;
1428 SheepdogInode *inode = &s->inode;
1429 AIOReq *aio_req;
1430
1431 if (acb->aiocb_type == AIOCB_WRITE_UDATA && s->is_snapshot) {
1432 /*
1433 * In the case we open the snapshot VDI, Sheepdog creates the
1434 * writable VDI when we do a write operation first.
1435 */
1436 ret = sd_create_branch(s);
1437 if (ret) {
1438 acb->ret = -EIO;
1439 goto out;
1440 }
1441 }
1442
1443 while (done != total) {
1444 uint8_t flags = 0;
1445 uint64_t old_oid = 0;
1446 int create = 0;
1447
1448 oid = vid_to_data_oid(inode->data_vdi_id[idx], idx);
1449
1450 len = MIN(total - done, SD_DATA_OBJ_SIZE - offset);
1451
1452 if (!inode->data_vdi_id[idx]) {
1453 if (acb->aiocb_type == AIOCB_READ_UDATA) {
1454 goto done;
1455 }
1456
1457 create = 1;
1458 } else if (acb->aiocb_type == AIOCB_WRITE_UDATA
1459 && !is_data_obj_writable(inode, idx)) {
1460 /* Copy-On-Write */
1461 create = 1;
1462 old_oid = oid;
1463 flags = SD_FLAG_CMD_COW;
1464 }
1465
1466 if (create) {
1467 dprintf("update ino (%" PRIu32") %" PRIu64 " %" PRIu64
1468 " %" PRIu64 "\n", inode->vdi_id, oid,
1469 vid_to_data_oid(inode->data_vdi_id[idx], idx), idx);
1470 oid = vid_to_data_oid(inode->vdi_id, idx);
1471 dprintf("new oid %lx\n", oid);
1472 }
1473
1474 aio_req = alloc_aio_req(s, acb, oid, len, offset, flags, old_oid, done);
1475
1476 if (create) {
1477 AIOReq *areq;
1478 QLIST_FOREACH(areq, &s->outstanding_aio_head,
1479 outstanding_aio_siblings) {
1480 if (areq == aio_req) {
1481 continue;
1482 }
1483 if (areq->oid == oid) {
1484 /*
1485 * Sheepdog cannot handle simultaneous create
1486 * requests to the same object. So we cannot send
1487 * the request until the previous request
1488 * finishes.
1489 */
1490 aio_req->flags = 0;
1491 aio_req->base_oid = 0;
1492 goto done;
1493 }
1494 }
1495 }
1496
1497 ret = add_aio_request(s, aio_req, acb->qiov->iov, acb->qiov->niov,
1498 create, acb->aiocb_type);
1499 if (ret < 0) {
1500 error_report("add_aio_request is failed");
1501 free_aio_req(s, aio_req);
1502 acb->ret = -EIO;
1503 goto out;
1504 }
1505 done:
1506 offset = 0;
1507 idx++;
1508 done += len;
1509 }
1510 out:
1511 if (QLIST_EMPTY(&acb->aioreq_head)) {
1512 return acb->ret;
1513 }
1514 return 1;
1515 }
1516
1517 static coroutine_fn int sd_co_writev(BlockDriverState *bs, int64_t sector_num,
1518 int nb_sectors, QEMUIOVector *qiov)
1519 {
1520 SheepdogAIOCB *acb;
1521 int ret;
1522
1523 if (bs->growable && sector_num + nb_sectors > bs->total_sectors) {
1524 /* TODO: shouldn't block here */
1525 if (sd_truncate(bs, (sector_num + nb_sectors) * SECTOR_SIZE) < 0) {
1526 return -EIO;
1527 }
1528 bs->total_sectors = sector_num + nb_sectors;
1529 }
1530
1531 acb = sd_aio_setup(bs, qiov, sector_num, nb_sectors, NULL, NULL);
1532 acb->aio_done_func = sd_write_done;
1533 acb->aiocb_type = AIOCB_WRITE_UDATA;
1534
1535 ret = sd_co_rw_vector(acb);
1536 if (ret <= 0) {
1537 qemu_aio_release(acb);
1538 return ret;
1539 }
1540
1541 qemu_coroutine_yield();
1542
1543 return acb->ret;
1544 }
1545
1546 static coroutine_fn int sd_co_readv(BlockDriverState *bs, int64_t sector_num,
1547 int nb_sectors, QEMUIOVector *qiov)
1548 {
1549 SheepdogAIOCB *acb;
1550 int i, ret;
1551
1552 acb = sd_aio_setup(bs, qiov, sector_num, nb_sectors, NULL, NULL);
1553 acb->aiocb_type = AIOCB_READ_UDATA;
1554 acb->aio_done_func = sd_finish_aiocb;
1555
1556 /*
1557 * TODO: we can do better; we don't need to initialize
1558 * blindly.
1559 */
1560 for (i = 0; i < qiov->niov; i++) {
1561 memset(qiov->iov[i].iov_base, 0, qiov->iov[i].iov_len);
1562 }
1563
1564 ret = sd_co_rw_vector(acb);
1565 if (ret <= 0) {
1566 qemu_aio_release(acb);
1567 return ret;
1568 }
1569
1570 qemu_coroutine_yield();
1571
1572 return acb->ret;
1573 }
1574
1575 static int sd_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
1576 {
1577 BDRVSheepdogState *s = bs->opaque;
1578 int ret, fd;
1579 uint32_t new_vid;
1580 SheepdogInode *inode;
1581 unsigned int datalen;
1582
1583 dprintf("sn_info: name %s id_str %s s: name %s vm_state_size %d "
1584 "is_snapshot %d\n", sn_info->name, sn_info->id_str,
1585 s->name, sn_info->vm_state_size, s->is_snapshot);
1586
1587 if (s->is_snapshot) {
1588 error_report("You can't create a snapshot of a snapshot VDI, "
1589 "%s (%" PRIu32 ").", s->name, s->inode.vdi_id);
1590
1591 return -EINVAL;
1592 }
1593
1594 dprintf("%s %s\n", sn_info->name, sn_info->id_str);
1595
1596 s->inode.vm_state_size = sn_info->vm_state_size;
1597 s->inode.vm_clock_nsec = sn_info->vm_clock_nsec;
1598 strncpy(s->inode.tag, sn_info->name, sizeof(s->inode.tag));
1599 /* we don't need to update entire object */
1600 datalen = SD_INODE_SIZE - sizeof(s->inode.data_vdi_id);
1601
1602 /* refresh inode. */
1603 fd = connect_to_sdog(s->addr, s->port);
1604 if (fd < 0) {
1605 ret = -EIO;
1606 goto cleanup;
1607 }
1608
1609 ret = write_object(fd, (char *)&s->inode, vid_to_vdi_oid(s->inode.vdi_id),
1610 s->inode.nr_copies, datalen, 0, 0);
1611 if (ret < 0) {
1612 error_report("failed to write snapshot's inode.");
1613 ret = -EIO;
1614 goto cleanup;
1615 }
1616
1617 ret = do_sd_create(s->name, s->inode.vdi_size, s->inode.vdi_id, &new_vid, 1,
1618 s->addr, s->port);
1619 if (ret < 0) {
1620 error_report("failed to create inode for snapshot. %s",
1621 strerror(errno));
1622 ret = -EIO;
1623 goto cleanup;
1624 }
1625
1626 inode = (SheepdogInode *)g_malloc(datalen);
1627
1628 ret = read_object(fd, (char *)inode, vid_to_vdi_oid(new_vid),
1629 s->inode.nr_copies, datalen, 0);
1630
1631 if (ret < 0) {
1632 error_report("failed to read new inode info. %s", strerror(errno));
1633 ret = -EIO;
1634 goto cleanup;
1635 }
1636
1637 memcpy(&s->inode, inode, datalen);
1638 dprintf("s->inode: name %s snap_id %x oid %x\n",
1639 s->inode.name, s->inode.snap_id, s->inode.vdi_id);
1640
1641 cleanup:
1642 closesocket(fd);
1643 return ret;
1644 }
1645
1646 static int sd_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
1647 {
1648 BDRVSheepdogState *s = bs->opaque;
1649 BDRVSheepdogState *old_s;
1650 char vdi[SD_MAX_VDI_LEN], tag[SD_MAX_VDI_TAG_LEN];
1651 char *buf = NULL;
1652 uint32_t vid;
1653 uint32_t snapid = 0;
1654 int ret = -ENOENT, fd;
1655
1656 old_s = g_malloc(sizeof(BDRVSheepdogState));
1657
1658 memcpy(old_s, s, sizeof(BDRVSheepdogState));
1659
1660 memset(vdi, 0, sizeof(vdi));
1661 strncpy(vdi, s->name, sizeof(vdi));
1662
1663 memset(tag, 0, sizeof(tag));
1664 snapid = strtoul(snapshot_id, NULL, 10);
1665 if (!snapid) {
1666 strncpy(tag, s->name, sizeof(tag));
1667 }
1668
1669 ret = find_vdi_name(s, vdi, snapid, tag, &vid, 1);
1670 if (ret) {
1671 error_report("Failed to find_vdi_name");
1672 ret = -ENOENT;
1673 goto out;
1674 }
1675
1676 fd = connect_to_sdog(s->addr, s->port);
1677 if (fd < 0) {
1678 error_report("failed to connect");
1679 goto out;
1680 }
1681
1682 buf = g_malloc(SD_INODE_SIZE);
1683 ret = read_object(fd, buf, vid_to_vdi_oid(vid), s->inode.nr_copies,
1684 SD_INODE_SIZE, 0);
1685
1686 closesocket(fd);
1687
1688 if (ret) {
1689 ret = -ENOENT;
1690 goto out;
1691 }
1692
1693 memcpy(&s->inode, buf, sizeof(s->inode));
1694
1695 if (!s->inode.vm_state_size) {
1696 error_report("Invalid snapshot");
1697 ret = -ENOENT;
1698 goto out;
1699 }
1700
1701 s->is_snapshot = 1;
1702
1703 g_free(buf);
1704 g_free(old_s);
1705
1706 return 0;
1707 out:
1708 /* recover bdrv_sd_state */
1709 memcpy(s, old_s, sizeof(BDRVSheepdogState));
1710 g_free(buf);
1711 g_free(old_s);
1712
1713 error_report("failed to open. recover old bdrv_sd_state.");
1714
1715 return ret;
1716 }
1717
1718 static int sd_snapshot_delete(BlockDriverState *bs, const char *snapshot_id)
1719 {
1720 /* FIXME: Delete specified snapshot id. */
1721 return 0;
1722 }
1723
1724 static int sd_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
1725 {
1726 BDRVSheepdogState *s = bs->opaque;
1727 SheepdogReq req;
1728 int fd, nr = 1024, ret, max = BITS_TO_LONGS(SD_NR_VDIS) * sizeof(long);
1729 QEMUSnapshotInfo *sn_tab = NULL;
1730 unsigned wlen, rlen;
1731 int found = 0;
1732 static SheepdogInode inode;
1733 unsigned long *vdi_inuse;
1734 unsigned int start_nr;
1735 uint64_t hval;
1736 uint32_t vid;
1737
1738 vdi_inuse = g_malloc(max);
1739
1740 fd = connect_to_sdog(s->addr, s->port);
1741 if (fd < 0) {
1742 goto out;
1743 }
1744
1745 rlen = max;
1746 wlen = 0;
1747
1748 memset(&req, 0, sizeof(req));
1749
1750 req.opcode = SD_OP_READ_VDIS;
1751 req.data_length = max;
1752
1753 ret = do_req(fd, (SheepdogReq *)&req, vdi_inuse, &wlen, &rlen);
1754
1755 closesocket(fd);
1756 if (ret) {
1757 goto out;
1758 }
1759
1760 sn_tab = g_malloc0(nr * sizeof(*sn_tab));
1761
1762 /* calculate a vdi id with hash function */
1763 hval = fnv_64a_buf(s->name, strlen(s->name), FNV1A_64_INIT);
1764 start_nr = hval & (SD_NR_VDIS - 1);
1765
1766 fd = connect_to_sdog(s->addr, s->port);
1767 if (fd < 0) {
1768 error_report("failed to connect");
1769 goto out;
1770 }
1771
1772 for (vid = start_nr; found < nr; vid = (vid + 1) % SD_NR_VDIS) {
1773 if (!test_bit(vid, vdi_inuse)) {
1774 break;
1775 }
1776
1777 /* we don't need to read entire object */
1778 ret = read_object(fd, (char *)&inode, vid_to_vdi_oid(vid),
1779 0, SD_INODE_SIZE - sizeof(inode.data_vdi_id), 0);
1780
1781 if (ret) {
1782 continue;
1783 }
1784
1785 if (!strcmp(inode.name, s->name) && is_snapshot(&inode)) {
1786 sn_tab[found].date_sec = inode.snap_ctime >> 32;
1787 sn_tab[found].date_nsec = inode.snap_ctime & 0xffffffff;
1788 sn_tab[found].vm_state_size = inode.vm_state_size;
1789 sn_tab[found].vm_clock_nsec = inode.vm_clock_nsec;
1790
1791 snprintf(sn_tab[found].id_str, sizeof(sn_tab[found].id_str), "%u",
1792 inode.snap_id);
1793 strncpy(sn_tab[found].name, inode.tag,
1794 MIN(sizeof(sn_tab[found].name), sizeof(inode.tag)));
1795 found++;
1796 }
1797 }
1798
1799 closesocket(fd);
1800 out:
1801 *psn_tab = sn_tab;
1802
1803 g_free(vdi_inuse);
1804
1805 return found;
1806 }
1807
1808 static int do_load_save_vmstate(BDRVSheepdogState *s, uint8_t *data,
1809 int64_t pos, int size, int load)
1810 {
1811 int fd, create;
1812 int ret = 0;
1813 unsigned int data_len;
1814 uint64_t vmstate_oid;
1815 uint32_t vdi_index;
1816 uint64_t offset;
1817
1818 fd = connect_to_sdog(s->addr, s->port);
1819 if (fd < 0) {
1820 ret = -EIO;
1821 goto cleanup;
1822 }
1823
1824 while (size) {
1825 vdi_index = pos / SD_DATA_OBJ_SIZE;
1826 offset = pos % SD_DATA_OBJ_SIZE;
1827
1828 data_len = MIN(size, SD_DATA_OBJ_SIZE);
1829
1830 vmstate_oid = vid_to_vmstate_oid(s->inode.vdi_id, vdi_index);
1831
1832 create = (offset == 0);
1833 if (load) {
1834 ret = read_object(fd, (char *)data, vmstate_oid,
1835 s->inode.nr_copies, data_len, offset);
1836 } else {
1837 ret = write_object(fd, (char *)data, vmstate_oid,
1838 s->inode.nr_copies, data_len, offset, create);
1839 }
1840
1841 if (ret < 0) {
1842 error_report("failed to save vmstate %s", strerror(errno));
1843 ret = -EIO;
1844 goto cleanup;
1845 }
1846
1847 pos += data_len;
1848 size -= data_len;
1849 ret += data_len;
1850 }
1851 cleanup:
1852 closesocket(fd);
1853 return ret;
1854 }
1855
1856 static int sd_save_vmstate(BlockDriverState *bs, const uint8_t *data,
1857 int64_t pos, int size)
1858 {
1859 BDRVSheepdogState *s = bs->opaque;
1860
1861 return do_load_save_vmstate(s, (uint8_t *)data, pos, size, 0);
1862 }
1863
1864 static int sd_load_vmstate(BlockDriverState *bs, uint8_t *data,
1865 int64_t pos, int size)
1866 {
1867 BDRVSheepdogState *s = bs->opaque;
1868
1869 return do_load_save_vmstate(s, data, pos, size, 1);
1870 }
1871
1872
1873 static QEMUOptionParameter sd_create_options[] = {
1874 {
1875 .name = BLOCK_OPT_SIZE,
1876 .type = OPT_SIZE,
1877 .help = "Virtual disk size"
1878 },
1879 {
1880 .name = BLOCK_OPT_BACKING_FILE,
1881 .type = OPT_STRING,
1882 .help = "File name of a base image"
1883 },
1884 {
1885 .name = BLOCK_OPT_PREALLOC,
1886 .type = OPT_STRING,
1887 .help = "Preallocation mode (allowed values: off, full)"
1888 },
1889 { NULL }
1890 };
1891
1892 BlockDriver bdrv_sheepdog = {
1893 .format_name = "sheepdog",
1894 .protocol_name = "sheepdog",
1895 .instance_size = sizeof(BDRVSheepdogState),
1896 .bdrv_file_open = sd_open,
1897 .bdrv_close = sd_close,
1898 .bdrv_create = sd_create,
1899 .bdrv_getlength = sd_getlength,
1900 .bdrv_truncate = sd_truncate,
1901
1902 .bdrv_co_readv = sd_co_readv,
1903 .bdrv_co_writev = sd_co_writev,
1904
1905 .bdrv_snapshot_create = sd_snapshot_create,
1906 .bdrv_snapshot_goto = sd_snapshot_goto,
1907 .bdrv_snapshot_delete = sd_snapshot_delete,
1908 .bdrv_snapshot_list = sd_snapshot_list,
1909
1910 .bdrv_save_vmstate = sd_save_vmstate,
1911 .bdrv_load_vmstate = sd_load_vmstate,
1912
1913 .create_options = sd_create_options,
1914 };
1915
1916 static void bdrv_sheepdog_init(void)
1917 {
1918 bdrv_register(&bdrv_sheepdog);
1919 }
1920 block_init(bdrv_sheepdog_init);