From b39a1fe1656ee4be0d244466b474cad7ada100c8 Mon Sep 17 00:00:00 2001 From: Shivaprasad G Bhat Date: Thu, 8 Oct 2015 17:59:07 +0530 Subject: [PATCH] Close the source fd if the destination qemu exits during tunnelled migration Tunnelled migration can hang if the destination qemu exits despite all the ABI checks. This happens whenever the destination qemu exits before the complete transfer is noticed by source qemu. The savevm state checks at runtime can fail at destination and cause qemu to error out. The source qemu cant notice it as the EPIPE is not propogated to it. The qemuMigrationIOFunc() notices the stream being broken from virStreamSend() and it cleans up the stream alone. The qemuMigrationWaitForCompletion() would never get to 100% transfer completion. The qemuMigrationWaitForCompletion() never breaks out as well since the ssh connection to destination is healthy, and the source qemu also thinks the migration is ongoing as the Fd to which it transfers, is never closed or broken. So, the migration will hang forever. Even Ctrl-C on the virsh migrate wouldn't be honoured. Close the source side FD when there is an error in the stream. That way, the source qemu updates itself and qemuMigrationWaitForCompletion() notices the failure. Close the FD for all kinds of errors to be sure. The error message is not copied for EPIPE so that the destination error is copied instead later. Note: Reproducible with repeated migrations between Power hosts running in different subcores-per-core modes. Signed-off-by: Shivaprasad G Bhat --- src/qemu/qemu_migration.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index d0f1e464f9..b53491a3ea 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -4058,6 +4058,7 @@ static void qemuMigrationIOFunc(void *arg) if (virStreamFinish(data->st) < 0) goto error; + VIR_FORCE_CLOSE(data->sock); VIR_FREE(buffer); return; @@ -4075,7 +4076,11 @@ static void qemuMigrationIOFunc(void *arg) } error: - virCopyLastError(&data->err); + /* Let the source qemu know that the transfer cant continue anymore. + * Don't copy the error for EPIPE as destination has the actual error. */ + VIR_FORCE_CLOSE(data->sock); + if (!virLastErrorIsSystemErrno(EPIPE)) + virCopyLastError(&data->err); virResetLastError(); VIR_FREE(buffer); } @@ -4399,9 +4404,14 @@ qemuMigrationRun(virQEMUDriverPtr driver, } } - if (spec->fwdType != MIGRATION_FWD_DIRECT && - !(iothread = qemuMigrationStartTunnel(spec->fwd.stream, fd))) - goto cancel; + if (spec->fwdType != MIGRATION_FWD_DIRECT) { + if (!(iothread = qemuMigrationStartTunnel(spec->fwd.stream, fd))) + goto cancel; + /* If we've created a tunnel, then the 'fd' will be closed in the + * qemuMigrationIOFunc as data->sock. + */ + fd = -1; + } rc = qemuMigrationWaitForCompletion(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT,