vmxnet3: Use common MAC address tracing macros
[qemu.git] / block / linux-aio.c
1 /*
2 * Linux native AIO support.
3 *
4 * Copyright (C) 2009 IBM, Corp.
5 * Copyright (C) 2009 Red Hat, Inc.
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2 or later.
8 * See the COPYING file in the top-level directory.
9 */
10 #include "qemu/osdep.h"
11 #include "qemu-common.h"
12 #include "block/aio.h"
13 #include "qemu/queue.h"
14 #include "block/raw-aio.h"
15 #include "qemu/event_notifier.h"
16
17 #include <libaio.h>
18
19 /*
20 * Queue size (per-device).
21 *
22 * XXX: eventually we need to communicate this to the guest and/or make it
23 * tunable by the guest. If we get more outstanding requests at a time
24 * than this we will get EAGAIN from io_submit which is communicated to
25 * the guest as an I/O error.
26 */
27 #define MAX_EVENTS 128
28
29 #define MAX_QUEUED_IO 128
30
31 struct qemu_laiocb {
32 BlockAIOCB common;
33 LinuxAioState *ctx;
34 struct iocb iocb;
35 ssize_t ret;
36 size_t nbytes;
37 QEMUIOVector *qiov;
38 bool is_read;
39 QSIMPLEQ_ENTRY(qemu_laiocb) next;
40 };
41
42 typedef struct {
43 int plugged;
44 unsigned int n;
45 bool blocked;
46 QSIMPLEQ_HEAD(, qemu_laiocb) pending;
47 } LaioQueue;
48
49 struct LinuxAioState {
50 io_context_t ctx;
51 EventNotifier e;
52
53 /* io queue for submit at batch */
54 LaioQueue io_q;
55
56 /* I/O completion processing */
57 QEMUBH *completion_bh;
58 struct io_event events[MAX_EVENTS];
59 int event_idx;
60 int event_max;
61 };
62
63 static void ioq_submit(LinuxAioState *s);
64
65 static inline ssize_t io_event_ret(struct io_event *ev)
66 {
67 return (ssize_t)(((uint64_t)ev->res2 << 32) | ev->res);
68 }
69
70 /*
71 * Completes an AIO request (calls the callback and frees the ACB).
72 */
73 static void qemu_laio_process_completion(struct qemu_laiocb *laiocb)
74 {
75 int ret;
76
77 ret = laiocb->ret;
78 if (ret != -ECANCELED) {
79 if (ret == laiocb->nbytes) {
80 ret = 0;
81 } else if (ret >= 0) {
82 /* Short reads mean EOF, pad with zeros. */
83 if (laiocb->is_read) {
84 qemu_iovec_memset(laiocb->qiov, ret, 0,
85 laiocb->qiov->size - ret);
86 } else {
87 ret = -EINVAL;
88 }
89 }
90 }
91 laiocb->common.cb(laiocb->common.opaque, ret);
92
93 qemu_aio_unref(laiocb);
94 }
95
96 /* The completion BH fetches completed I/O requests and invokes their
97 * callbacks.
98 *
99 * The function is somewhat tricky because it supports nested event loops, for
100 * example when a request callback invokes aio_poll(). In order to do this,
101 * the completion events array and index are kept in LinuxAioState. The BH
102 * reschedules itself as long as there are completions pending so it will
103 * either be called again in a nested event loop or will be called after all
104 * events have been completed. When there are no events left to complete, the
105 * BH returns without rescheduling.
106 */
107 static void qemu_laio_completion_bh(void *opaque)
108 {
109 LinuxAioState *s = opaque;
110
111 /* Fetch more completion events when empty */
112 if (s->event_idx == s->event_max) {
113 do {
114 struct timespec ts = { 0 };
115 s->event_max = io_getevents(s->ctx, MAX_EVENTS, MAX_EVENTS,
116 s->events, &ts);
117 } while (s->event_max == -EINTR);
118
119 s->event_idx = 0;
120 if (s->event_max <= 0) {
121 s->event_max = 0;
122 return; /* no more events */
123 }
124 }
125
126 /* Reschedule so nested event loops see currently pending completions */
127 qemu_bh_schedule(s->completion_bh);
128
129 /* Process completion events */
130 while (s->event_idx < s->event_max) {
131 struct iocb *iocb = s->events[s->event_idx].obj;
132 struct qemu_laiocb *laiocb =
133 container_of(iocb, struct qemu_laiocb, iocb);
134
135 laiocb->ret = io_event_ret(&s->events[s->event_idx]);
136 s->event_idx++;
137
138 qemu_laio_process_completion(laiocb);
139 }
140
141 if (!s->io_q.plugged && !QSIMPLEQ_EMPTY(&s->io_q.pending)) {
142 ioq_submit(s);
143 }
144 }
145
146 static void qemu_laio_completion_cb(EventNotifier *e)
147 {
148 LinuxAioState *s = container_of(e, LinuxAioState, e);
149
150 if (event_notifier_test_and_clear(&s->e)) {
151 qemu_bh_schedule(s->completion_bh);
152 }
153 }
154
155 static void laio_cancel(BlockAIOCB *blockacb)
156 {
157 struct qemu_laiocb *laiocb = (struct qemu_laiocb *)blockacb;
158 struct io_event event;
159 int ret;
160
161 if (laiocb->ret != -EINPROGRESS) {
162 return;
163 }
164 ret = io_cancel(laiocb->ctx->ctx, &laiocb->iocb, &event);
165 laiocb->ret = -ECANCELED;
166 if (ret != 0) {
167 /* iocb is not cancelled, cb will be called by the event loop later */
168 return;
169 }
170
171 laiocb->common.cb(laiocb->common.opaque, laiocb->ret);
172 }
173
174 static const AIOCBInfo laio_aiocb_info = {
175 .aiocb_size = sizeof(struct qemu_laiocb),
176 .cancel_async = laio_cancel,
177 };
178
179 static void ioq_init(LaioQueue *io_q)
180 {
181 QSIMPLEQ_INIT(&io_q->pending);
182 io_q->plugged = 0;
183 io_q->n = 0;
184 io_q->blocked = false;
185 }
186
187 static void ioq_submit(LinuxAioState *s)
188 {
189 int ret, len;
190 struct qemu_laiocb *aiocb;
191 struct iocb *iocbs[MAX_QUEUED_IO];
192 QSIMPLEQ_HEAD(, qemu_laiocb) completed;
193
194 do {
195 len = 0;
196 QSIMPLEQ_FOREACH(aiocb, &s->io_q.pending, next) {
197 iocbs[len++] = &aiocb->iocb;
198 if (len == MAX_QUEUED_IO) {
199 break;
200 }
201 }
202
203 ret = io_submit(s->ctx, len, iocbs);
204 if (ret == -EAGAIN) {
205 break;
206 }
207 if (ret < 0) {
208 abort();
209 }
210
211 s->io_q.n -= ret;
212 aiocb = container_of(iocbs[ret - 1], struct qemu_laiocb, iocb);
213 QSIMPLEQ_SPLIT_AFTER(&s->io_q.pending, aiocb, next, &completed);
214 } while (ret == len && !QSIMPLEQ_EMPTY(&s->io_q.pending));
215 s->io_q.blocked = (s->io_q.n > 0);
216 }
217
218 void laio_io_plug(BlockDriverState *bs, LinuxAioState *s)
219 {
220 assert(!s->io_q.plugged);
221 s->io_q.plugged = 1;
222 }
223
224 void laio_io_unplug(BlockDriverState *bs, LinuxAioState *s)
225 {
226 assert(s->io_q.plugged);
227 s->io_q.plugged = 0;
228 if (!s->io_q.blocked && !QSIMPLEQ_EMPTY(&s->io_q.pending)) {
229 ioq_submit(s);
230 }
231 }
232
233 BlockAIOCB *laio_submit(BlockDriverState *bs, LinuxAioState *s, int fd,
234 int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
235 BlockCompletionFunc *cb, void *opaque, int type)
236 {
237 struct qemu_laiocb *laiocb;
238 struct iocb *iocbs;
239 off_t offset = sector_num * 512;
240
241 laiocb = qemu_aio_get(&laio_aiocb_info, bs, cb, opaque);
242 laiocb->nbytes = nb_sectors * 512;
243 laiocb->ctx = s;
244 laiocb->ret = -EINPROGRESS;
245 laiocb->is_read = (type == QEMU_AIO_READ);
246 laiocb->qiov = qiov;
247
248 iocbs = &laiocb->iocb;
249
250 switch (type) {
251 case QEMU_AIO_WRITE:
252 io_prep_pwritev(iocbs, fd, qiov->iov, qiov->niov, offset);
253 break;
254 case QEMU_AIO_READ:
255 io_prep_preadv(iocbs, fd, qiov->iov, qiov->niov, offset);
256 break;
257 /* Currently Linux kernel does not support other operations */
258 default:
259 fprintf(stderr, "%s: invalid AIO request type 0x%x.\n",
260 __func__, type);
261 goto out_free_aiocb;
262 }
263 io_set_eventfd(&laiocb->iocb, event_notifier_get_fd(&s->e));
264
265 QSIMPLEQ_INSERT_TAIL(&s->io_q.pending, laiocb, next);
266 s->io_q.n++;
267 if (!s->io_q.blocked &&
268 (!s->io_q.plugged || s->io_q.n >= MAX_QUEUED_IO)) {
269 ioq_submit(s);
270 }
271 return &laiocb->common;
272
273 out_free_aiocb:
274 qemu_aio_unref(laiocb);
275 return NULL;
276 }
277
278 void laio_detach_aio_context(LinuxAioState *s, AioContext *old_context)
279 {
280 aio_set_event_notifier(old_context, &s->e, false, NULL);
281 qemu_bh_delete(s->completion_bh);
282 }
283
284 void laio_attach_aio_context(LinuxAioState *s, AioContext *new_context)
285 {
286 s->completion_bh = aio_bh_new(new_context, qemu_laio_completion_bh, s);
287 aio_set_event_notifier(new_context, &s->e, false,
288 qemu_laio_completion_cb);
289 }
290
291 LinuxAioState *laio_init(void)
292 {
293 LinuxAioState *s;
294
295 s = g_malloc0(sizeof(*s));
296 if (event_notifier_init(&s->e, false) < 0) {
297 goto out_free_state;
298 }
299
300 if (io_setup(MAX_EVENTS, &s->ctx) != 0) {
301 goto out_close_efd;
302 }
303
304 ioq_init(&s->io_q);
305
306 return s;
307
308 out_close_efd:
309 event_notifier_cleanup(&s->e);
310 out_free_state:
311 g_free(s);
312 return NULL;
313 }
314
315 void laio_cleanup(LinuxAioState *s)
316 {
317 event_notifier_cleanup(&s->e);
318
319 if (io_destroy(s->ctx) != 0) {
320 fprintf(stderr, "%s: destroy AIO context %p failed\n",
321 __func__, &s->ctx);
322 }
323 g_free(s);
324 }