Merge remote-tracking branch 'remotes/alistair23/tags/pull-riscv-to-apply-20211022...
[qemu.git] / migration / migration.h
1 /*
2 * QEMU live migration
3 *
4 * Copyright IBM, Corp. 2008
5 *
6 * Authors:
7 * Anthony Liguori <aliguori@us.ibm.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 */
13
14 #ifndef QEMU_MIGRATION_H
15 #define QEMU_MIGRATION_H
16
17 #include "exec/cpu-common.h"
18 #include "hw/qdev-core.h"
19 #include "qapi/qapi-types-migration.h"
20 #include "qemu/thread.h"
21 #include "qemu/coroutine_int.h"
22 #include "io/channel.h"
23 #include "io/channel-buffer.h"
24 #include "net/announce.h"
25 #include "qom/object.h"
26
27 struct PostcopyBlocktimeContext;
28
29 #define MIGRATION_RESUME_ACK_VALUE (1)
30
31 /*
32 * 1<<6=64 pages -> 256K chunk when page size is 4K. This gives us
33 * the benefit that all the chunks are 64 pages aligned then the
34 * bitmaps are always aligned to LONG.
35 */
36 #define CLEAR_BITMAP_SHIFT_MIN 6
37 /*
38 * 1<<18=256K pages -> 1G chunk when page size is 4K. This is the
39 * default value to use if no one specified.
40 */
41 #define CLEAR_BITMAP_SHIFT_DEFAULT 18
42 /*
43 * 1<<31=2G pages -> 8T chunk when page size is 4K. This should be
44 * big enough and make sure we won't overflow easily.
45 */
46 #define CLEAR_BITMAP_SHIFT_MAX 31
47
48 /* State for the incoming migration */
49 struct MigrationIncomingState {
50 QEMUFile *from_src_file;
51
52 /* A hook to allow cleanup at the end of incoming migration */
53 void *transport_data;
54 void (*transport_cleanup)(void *data);
55
56 /*
57 * Free at the start of the main state load, set as the main thread finishes
58 * loading state.
59 */
60 QemuEvent main_thread_load_event;
61
62 /* For network announces */
63 AnnounceTimer announce_timer;
64
65 size_t largest_page_size;
66 bool have_fault_thread;
67 QemuThread fault_thread;
68 QemuSemaphore fault_thread_sem;
69 /* Set this when we want the fault thread to quit */
70 bool fault_thread_quit;
71
72 bool have_listen_thread;
73 QemuThread listen_thread;
74 QemuSemaphore listen_thread_sem;
75
76 /* For the kernel to send us notifications */
77 int userfault_fd;
78 /* To notify the fault_thread to wake, e.g., when need to quit */
79 int userfault_event_fd;
80 QEMUFile *to_src_file;
81 QemuMutex rp_mutex; /* We send replies from multiple threads */
82 /* RAMBlock of last request sent to source */
83 RAMBlock *last_rb;
84 void *postcopy_tmp_page;
85 void *postcopy_tmp_zero_page;
86 /* PostCopyFD's for external userfaultfds & handlers of shared memory */
87 GArray *postcopy_remote_fds;
88
89 QEMUBH *bh;
90
91 int state;
92
93 bool have_colo_incoming_thread;
94 QemuThread colo_incoming_thread;
95 /* The coroutine we should enter (back) after failover */
96 Coroutine *migration_incoming_co;
97 QemuSemaphore colo_incoming_sem;
98
99 /*
100 * PostcopyBlocktimeContext to keep information for postcopy
101 * live migration, to calculate vCPU block time
102 * */
103 struct PostcopyBlocktimeContext *blocktime_ctx;
104
105 /* notify PAUSED postcopy incoming migrations to try to continue */
106 bool postcopy_recover_triggered;
107 QemuSemaphore postcopy_pause_sem_dst;
108 QemuSemaphore postcopy_pause_sem_fault;
109
110 /* List of listening socket addresses */
111 SocketAddressList *socket_address_list;
112
113 /* A tree of pages that we requested to the source VM */
114 GTree *page_requested;
115 /* For debugging purpose only, but would be nice to keep */
116 int page_requested_count;
117 /*
118 * The mutex helps to maintain the requested pages that we sent to the
119 * source, IOW, to guarantee coherent between the page_requests tree and
120 * the per-ramblock receivedmap. Note! This does not guarantee consistency
121 * of the real page copy procedures (using UFFDIO_[ZERO]COPY). E.g., even
122 * if one bit in receivedmap is cleared, UFFDIO_COPY could have happened
123 * for that page already. This is intended so that the mutex won't
124 * serialize and blocked by slow operations like UFFDIO_* ioctls. However
125 * this should be enough to make sure the page_requested tree always
126 * contains valid information.
127 */
128 QemuMutex page_request_mutex;
129 };
130
131 MigrationIncomingState *migration_incoming_get_current(void);
132 void migration_incoming_state_destroy(void);
133 /*
134 * Functions to work with blocktime context
135 */
136 void fill_destination_postcopy_migration_info(MigrationInfo *info);
137
138 #define TYPE_MIGRATION "migration"
139
140 typedef struct MigrationClass MigrationClass;
141 DECLARE_OBJ_CHECKERS(MigrationState, MigrationClass,
142 MIGRATION_OBJ, TYPE_MIGRATION)
143
144 struct MigrationClass {
145 /*< private >*/
146 DeviceClass parent_class;
147 };
148
149 struct MigrationState {
150 /*< private >*/
151 DeviceState parent_obj;
152
153 /*< public >*/
154 QemuThread thread;
155 QEMUBH *vm_start_bh;
156 QEMUBH *cleanup_bh;
157 QEMUFile *to_dst_file;
158 QIOChannelBuffer *bioc;
159 /*
160 * Protects to_dst_file pointer. We need to make sure we won't
161 * yield or hang during the critical section, since this lock will
162 * be used in OOB command handler.
163 */
164 QemuMutex qemu_file_lock;
165
166 /*
167 * Used to allow urgent requests to override rate limiting.
168 */
169 QemuSemaphore rate_limit_sem;
170
171 /* pages already send at the beginning of current iteration */
172 uint64_t iteration_initial_pages;
173
174 /* pages transferred per second */
175 double pages_per_second;
176
177 /* bytes already send at the beginning of current iteration */
178 uint64_t iteration_initial_bytes;
179 /* time at the start of current iteration */
180 int64_t iteration_start_time;
181 /*
182 * The final stage happens when the remaining data is smaller than
183 * this threshold; it's calculated from the requested downtime and
184 * measured bandwidth
185 */
186 int64_t threshold_size;
187
188 /* params from 'migrate-set-parameters' */
189 MigrationParameters parameters;
190
191 int state;
192
193 /* State related to return path */
194 struct {
195 QEMUFile *from_dst_file;
196 QemuThread rp_thread;
197 bool error;
198 QemuSemaphore rp_sem;
199 } rp_state;
200
201 double mbps;
202 /* Timestamp when recent migration starts (ms) */
203 int64_t start_time;
204 /* Total time used by latest migration (ms) */
205 int64_t total_time;
206 /* Timestamp when VM is down (ms) to migrate the last stuff */
207 int64_t downtime_start;
208 int64_t downtime;
209 int64_t expected_downtime;
210 bool enabled_capabilities[MIGRATION_CAPABILITY__MAX];
211 int64_t setup_time;
212 /*
213 * Whether guest was running when we enter the completion stage.
214 * If migration is interrupted by any reason, we need to continue
215 * running the guest on source.
216 */
217 bool vm_was_running;
218
219 /* Flag set once the migration has been asked to enter postcopy */
220 bool start_postcopy;
221 /* Flag set after postcopy has sent the device state */
222 bool postcopy_after_devices;
223
224 /* Flag set once the migration thread is running (and needs joining) */
225 bool migration_thread_running;
226
227 /* Flag set once the migration thread called bdrv_inactivate_all */
228 bool block_inactive;
229
230 /* Migration is waiting for guest to unplug device */
231 QemuSemaphore wait_unplug_sem;
232
233 /* Migration is paused due to pause-before-switchover */
234 QemuSemaphore pause_sem;
235
236 /* The semaphore is used to notify COLO thread that failover is finished */
237 QemuSemaphore colo_exit_sem;
238
239 /* The event is used to notify COLO thread to do checkpoint */
240 QemuEvent colo_checkpoint_event;
241 int64_t colo_checkpoint_time;
242 QEMUTimer *colo_delay_timer;
243
244 /* The first error that has occurred.
245 We used the mutex to be able to return the 1st error message */
246 Error *error;
247 /* mutex to protect errp */
248 QemuMutex error_mutex;
249
250 /* Do we have to clean up -b/-i from old migrate parameters */
251 /* This feature is deprecated and will be removed */
252 bool must_remove_block_options;
253
254 /*
255 * Global switch on whether we need to store the global state
256 * during migration.
257 */
258 bool store_global_state;
259
260 /* Whether we send QEMU_VM_CONFIGURATION during migration */
261 bool send_configuration;
262 /* Whether we send section footer during migration */
263 bool send_section_footer;
264
265 /* Needed by postcopy-pause state */
266 QemuSemaphore postcopy_pause_sem;
267 QemuSemaphore postcopy_pause_rp_sem;
268 /*
269 * Whether we abort the migration if decompression errors are
270 * detected at the destination. It is left at false for qemu
271 * older than 3.0, since only newer qemu sends streams that
272 * do not trigger spurious decompression errors.
273 */
274 bool decompress_error_check;
275
276 /*
277 * This decides the size of guest memory chunk that will be used
278 * to track dirty bitmap clearing. The size of memory chunk will
279 * be GUEST_PAGE_SIZE << N. Say, N=0 means we will clear dirty
280 * bitmap for each page to send (1<<0=1); N=10 means we will clear
281 * dirty bitmap only once for 1<<10=1K continuous guest pages
282 * (which is in 4M chunk).
283 */
284 uint8_t clear_bitmap_shift;
285
286 /*
287 * This save hostname when out-going migration starts
288 */
289 char *hostname;
290 };
291
292 void migrate_set_state(int *state, int old_state, int new_state);
293
294 void migration_fd_process_incoming(QEMUFile *f, Error **errp);
295 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp);
296 void migration_incoming_process(void);
297
298 bool migration_has_all_channels(void);
299
300 uint64_t migrate_max_downtime(void);
301
302 void migrate_set_error(MigrationState *s, const Error *error);
303 void migrate_fd_error(MigrationState *s, const Error *error);
304
305 void migrate_fd_connect(MigrationState *s, Error *error_in);
306
307 bool migration_is_setup_or_active(int state);
308 bool migration_is_running(int state);
309
310 void migrate_init(MigrationState *s);
311 bool migration_is_blocked(Error **errp);
312 /* True if outgoing migration has entered postcopy phase */
313 bool migration_in_postcopy(void);
314 MigrationState *migrate_get_current(void);
315
316 bool migrate_postcopy(void);
317
318 bool migrate_release_ram(void);
319 bool migrate_postcopy_ram(void);
320 bool migrate_zero_blocks(void);
321 bool migrate_dirty_bitmaps(void);
322 bool migrate_ignore_shared(void);
323 bool migrate_validate_uuid(void);
324
325 bool migrate_auto_converge(void);
326 bool migrate_use_multifd(void);
327 bool migrate_pause_before_switchover(void);
328 int migrate_multifd_channels(void);
329 MultiFDCompression migrate_multifd_compression(void);
330 int migrate_multifd_zlib_level(void);
331 int migrate_multifd_zstd_level(void);
332
333 int migrate_use_xbzrle(void);
334 uint64_t migrate_xbzrle_cache_size(void);
335 bool migrate_colo_enabled(void);
336
337 bool migrate_use_block(void);
338 bool migrate_use_block_incremental(void);
339 int migrate_max_cpu_throttle(void);
340 bool migrate_use_return_path(void);
341
342 uint64_t ram_get_total_transferred_pages(void);
343
344 bool migrate_use_compression(void);
345 int migrate_compress_level(void);
346 int migrate_compress_threads(void);
347 int migrate_compress_wait_thread(void);
348 int migrate_decompress_threads(void);
349 bool migrate_use_events(void);
350 bool migrate_postcopy_blocktime(void);
351 bool migrate_background_snapshot(void);
352
353 /* Sending on the return path - generic and then for each message type */
354 void migrate_send_rp_shut(MigrationIncomingState *mis,
355 uint32_t value);
356 void migrate_send_rp_pong(MigrationIncomingState *mis,
357 uint32_t value);
358 int migrate_send_rp_req_pages(MigrationIncomingState *mis, RAMBlock *rb,
359 ram_addr_t start, uint64_t haddr);
360 int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
361 RAMBlock *rb, ram_addr_t start);
362 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
363 char *block_name);
364 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value);
365
366 void dirty_bitmap_mig_before_vm_start(void);
367 void dirty_bitmap_mig_cancel_outgoing(void);
368 void dirty_bitmap_mig_cancel_incoming(void);
369 bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm,
370 Error **errp);
371
372 void migrate_add_address(SocketAddress *address);
373
374 int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque);
375
376 #define qemu_ram_foreach_block \
377 #warning "Use foreach_not_ignored_block in migration code"
378
379 void migration_make_urgent_request(void);
380 void migration_consume_urgent_request(void);
381 bool migration_rate_limit(void);
382 void migration_cancel(void);
383
384 void populate_vfio_info(MigrationInfo *info);
385
386 #endif