[libvirt] [PATCH 0/2] Fix virnetsockettest on FreeBSD
The virnetsockettest is somewhat unreliable on FreeBSD. On a single CPU guest it will appear to run successfully, as you add more CPUs though it will increasingly get failures. Testing on a 8 CPU guest gives 100% failure rate. Daniel P. Berrangé (2): tests: merge code for UNIX and TCP socket testing tests: rewrite socket to do something sensible and reliable tests/virnetsockettest.c | 228 +++++++++++++++++++++++++++++++---------------- 1 file changed, 153 insertions(+), 75 deletions(-) -- 2.14.3
The test code for UNIX and TCP sockets will need to be rewritten and extended later, and will benefit from code sharing. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/virnetsockettest.c | 120 ++++++++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 69 deletions(-) diff --git a/tests/virnetsockettest.c b/tests/virnetsockettest.c index 9f9a243484..e463d432ff 100644 --- a/tests/virnetsockettest.c +++ b/tests/virnetsockettest.c @@ -116,38 +116,67 @@ checkProtocols(bool *hasIPv4, bool *hasIPv6, } -struct testTCPData { +struct testSocketData { const char *lnode; int port; const char *cnode; }; -static int testSocketTCPAccept(const void *opaque) +static int testSocketAccept(const void *opaque) { virNetSocketPtr *lsock = NULL; /* Listen socket */ size_t nlsock = 0, i; virNetSocketPtr ssock = NULL; /* Server socket */ virNetSocketPtr csock = NULL; /* Client socket */ - const struct testTCPData *data = opaque; + const struct testSocketData *data = opaque; int ret = -1; char portstr[100]; + char *tmpdir = NULL; + char *path = NULL; + char template[] = "/tmp/libvirt_XXXXXX"; - snprintf(portstr, sizeof(portstr), "%d", data->port); + if (!data) { + virNetSocketPtr usock; + tmpdir = mkdtemp(template); + if (tmpdir == NULL) { + VIR_WARN("Failed to create temporary directory"); + goto cleanup; + } + if (virAsprintf(&path, "%s/test.sock", tmpdir) < 0) + goto cleanup; - if (virNetSocketNewListenTCP(data->lnode, portstr, - AF_UNSPEC, - &lsock, &nlsock) < 0) - goto cleanup; + if (virNetSocketNewListenUNIX(path, 0700, -1, getegid(), &usock) < 0) + goto cleanup; + + if (VIR_ALLOC_N(lsock, 1) < 0) { + virObjectUnref(usock); + goto cleanup; + } + + lsock[0] = usock; + nlsock = 1; + } else { + snprintf(portstr, sizeof(portstr), "%d", data->port); + if (virNetSocketNewListenTCP(data->lnode, portstr, + AF_UNSPEC, + &lsock, &nlsock) < 0) + goto cleanup; + } for (i = 0; i < nlsock; i++) { if (virNetSocketListen(lsock[i], 0) < 0) goto cleanup; } - if (virNetSocketNewConnectTCP(data->cnode, portstr, - AF_UNSPEC, - &csock) < 0) - goto cleanup; + if (!data) { + if (virNetSocketNewConnectUNIX(path, false, NULL, &csock) < 0) + goto cleanup; + } else { + if (virNetSocketNewConnectTCP(data->cnode, portstr, + AF_UNSPEC, + &csock) < 0) + goto cleanup; + } virObjectUnref(csock); @@ -171,62 +200,15 @@ static int testSocketTCPAccept(const void *opaque) for (i = 0; i < nlsock; i++) virObjectUnref(lsock[i]); VIR_FREE(lsock); - return ret; -} -#endif - - -#ifndef WIN32 -static int testSocketUNIXAccept(const void *data ATTRIBUTE_UNUSED) -{ - virNetSocketPtr lsock = NULL; /* Listen socket */ - virNetSocketPtr ssock = NULL; /* Server socket */ - virNetSocketPtr csock = NULL; /* Client socket */ - int ret = -1; - - char *path = NULL; - char *tmpdir; - char template[] = "/tmp/libvirt_XXXXXX"; - - tmpdir = mkdtemp(template); - if (tmpdir == NULL) { - VIR_WARN("Failed to create temporary directory"); - goto cleanup; - } - if (virAsprintf(&path, "%s/test.sock", tmpdir) < 0) - goto cleanup; - - if (virNetSocketNewListenUNIX(path, 0700, -1, getegid(), &lsock) < 0) - goto cleanup; - - if (virNetSocketListen(lsock, 0) < 0) - goto cleanup; - - if (virNetSocketNewConnectUNIX(path, false, NULL, &csock) < 0) - goto cleanup; - - virObjectUnref(csock); - - if (virNetSocketAccept(lsock, &ssock) != -1) { - char c = 'a'; - if (virNetSocketWrite(ssock, &c, 1) != -1) { - VIR_DEBUG("Unexpected client socket present"); - goto cleanup; - } - } - - ret = 0; - - cleanup: VIR_FREE(path); - virObjectUnref(lsock); - virObjectUnref(ssock); if (tmpdir) rmdir(tmpdir); return ret; } +#endif +#ifndef WIN32 static int testSocketUNIXAddrs(const void *data ATTRIBUTE_UNUSED) { virNetSocketPtr lsock = NULL; /* Listen socket */ @@ -456,28 +438,28 @@ mymain(void) } if (hasIPv4) { - struct testTCPData tcpData = { "127.0.0.1", freePort, "127.0.0.1" }; - if (virTestRun("Socket TCP/IPv4 Accept", testSocketTCPAccept, &tcpData) < 0) + struct testSocketData tcpData = { "127.0.0.1", freePort, "127.0.0.1" }; + if (virTestRun("Socket TCP/IPv4 Accept", testSocketAccept, &tcpData) < 0) ret = -1; } if (hasIPv6) { - struct testTCPData tcpData = { "::1", freePort, "::1" }; - if (virTestRun("Socket TCP/IPv6 Accept", testSocketTCPAccept, &tcpData) < 0) + struct testSocketData tcpData = { "::1", freePort, "::1" }; + if (virTestRun("Socket TCP/IPv6 Accept", testSocketAccept, &tcpData) < 0) ret = -1; } if (hasIPv6 && hasIPv4) { - struct testTCPData tcpData = { NULL, freePort, "127.0.0.1" }; - if (virTestRun("Socket TCP/IPv4+IPv6 Accept", testSocketTCPAccept, &tcpData) < 0) + struct testSocketData tcpData = { NULL, freePort, "127.0.0.1" }; + if (virTestRun("Socket TCP/IPv4+IPv6 Accept", testSocketAccept, &tcpData) < 0) ret = -1; tcpData.cnode = "::1"; - if (virTestRun("Socket TCP/IPv4+IPv6 Accept", testSocketTCPAccept, &tcpData) < 0) + if (virTestRun("Socket TCP/IPv4+IPv6 Accept", testSocketAccept, &tcpData) < 0) ret = -1; } #endif #ifndef WIN32 - if (virTestRun("Socket UNIX Accept", testSocketUNIXAccept, NULL) < 0) + if (virTestRun("Socket UNIX Accept", testSocketAccept, NULL) < 0) ret = -1; if (virTestRun("Socket UNIX Addrs", testSocketUNIXAddrs, NULL) < 0) -- 2.14.3
On Tue, 2018-05-01 at 12:06 +0100, Daniel P. Berrangé wrote:
The test code for UNIX and TCP sockets will need to be rewritten and extended later, and will benefit from code sharing.
Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/virnetsockettest.c | 120 ++++++++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 69 deletions(-)
Reviewed-by: Andrea Bolognani <abologna@redhat.com> -- Andrea Bolognani / Red Hat / Virtualization
The current socket test is rather crazy in that it sets up a server listening for sockets and then runs a client connect call, relying on the fact that the kernel will accept this despite the application not having called accept() yet. It then closes the client socket and calls accept() on the server. On Linux accept() will always see that the client has gone and so skip the rest of the code. On FreeBSD, however, the accept sometimes succeeds, causing us to then go into code that attempts to read and write to the client which will fail aborting the test. The accept() never succeeds on FreeBSD guests with a single CPU, but as you add more CPUs, accept() becomes more and more likely to succeed, giving a 100% failure rate for the test when using 8 CPUs. This completely rewrites the test so that it is avoids this designed in race condition. We simply spawn a background thread to act as the client, which will read a byte from the server and write it back again. The main thread can now properly listen and accept the client in a synchronous manner avoiding any races. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/virnetsockettest.c | 138 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 117 insertions(+), 21 deletions(-) diff --git a/tests/virnetsockettest.c b/tests/virnetsockettest.c index e463d432ff..bdc9f5b366 100644 --- a/tests/virnetsockettest.c +++ b/tests/virnetsockettest.c @@ -115,6 +115,56 @@ checkProtocols(bool *hasIPv4, bool *hasIPv6, return ret; } +struct testClientData { + const char *path; + const char *cnode; + const char *portstr; +}; + +static void +testSocketClient(void *opaque) +{ + struct testClientData *data = opaque; + char c; + virNetSocketPtr csock = NULL; + + if (data->path) { + if (virNetSocketNewConnectUNIX(data->path, false, + NULL, &csock) < 0) + return; + } else { + if (virNetSocketNewConnectTCP(data->cnode, data->portstr, + AF_UNSPEC, + &csock) < 0) + return; + } + + virNetSocketSetBlocking(csock, true); + + if (virNetSocketRead(csock, &c, 1) != 1) { + VIR_DEBUG("Cannot read from server"); + goto done; + } + if (virNetSocketWrite(csock, &c, 1) != 1) { + VIR_DEBUG("Cannot write to server"); + goto done; + } + + done: + virObjectUnref(csock); +} + + +static void +testSocketIncoming(virNetSocketPtr sock, + int events ATTRIBUTE_UNUSED, + void *opaque) +{ + virNetSocketPtr *retsock = opaque; + VIR_DEBUG("Incoming sock=%p events=%d\n", sock, events); + *retsock = sock; +} + struct testSocketData { const char *lnode; @@ -122,18 +172,24 @@ struct testSocketData { const char *cnode; }; -static int testSocketAccept(const void *opaque) + +static int +testSocketAccept(const void *opaque) { virNetSocketPtr *lsock = NULL; /* Listen socket */ size_t nlsock = 0, i; virNetSocketPtr ssock = NULL; /* Server socket */ - virNetSocketPtr csock = NULL; /* Client socket */ + virNetSocketPtr rsock = NULL; /* Server from poll */ const struct testSocketData *data = opaque; int ret = -1; char portstr[100]; char *tmpdir = NULL; char *path = NULL; char template[] = "/tmp/libvirt_XXXXXX"; + virThread th; + struct testClientData cdata = { 0 }; + bool goodsock = false; + char a = 'a', b = '\0'; if (!data) { virNetSocketPtr usock; @@ -155,50 +211,88 @@ static int testSocketAccept(const void *opaque) lsock[0] = usock; nlsock = 1; + + cdata.path = path; } else { snprintf(portstr, sizeof(portstr), "%d", data->port); if (virNetSocketNewListenTCP(data->lnode, portstr, AF_UNSPEC, &lsock, &nlsock) < 0) goto cleanup; + + cdata.cnode = data->cnode; + cdata.portstr = portstr; } for (i = 0; i < nlsock; i++) { if (virNetSocketListen(lsock[i], 0) < 0) goto cleanup; - } - if (!data) { - if (virNetSocketNewConnectUNIX(path, false, NULL, &csock) < 0) - goto cleanup; - } else { - if (virNetSocketNewConnectTCP(data->cnode, portstr, - AF_UNSPEC, - &csock) < 0) + if (virNetSocketAddIOCallback(lsock[i], + VIR_EVENT_HANDLE_READABLE, + testSocketIncoming, + &rsock, + NULL) < 0) { goto cleanup; + } } - virObjectUnref(csock); + if (virThreadCreate(&th, true, + testSocketClient, + &cdata) < 0) + goto cleanup; + + while (rsock == NULL) + virEventRunDefaultImpl(); for (i = 0; i < nlsock; i++) { - if (virNetSocketAccept(lsock[i], &ssock) != -1 && ssock) { - char c = 'a'; - if (virNetSocketWrite(ssock, &c, 1) != -1 && - virNetSocketRead(ssock, &c, 1) != -1) { - VIR_DEBUG("Unexpected client socket present"); - goto cleanup; - } + if (lsock[i] == rsock) { + goodsock = true; + break; } - virObjectUnref(ssock); - ssock = NULL; } + if (!goodsock) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + "Unexpected server socket seen"); + goto join; + } + + if (virNetSocketAccept(rsock, &ssock) < 0) + goto join; + + if (!ssock) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + "Client went away unexpectedly"); + goto join; + } + + virNetSocketSetBlocking(ssock, true); + + if (virNetSocketWrite(ssock, &a, 1) < 0 || + virNetSocketRead(ssock, &b, 1) < 0) + goto join; + + if (a != b) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "Bad data received '%x' != '%x'", a, b); + goto join; + } + virObjectUnref(ssock); + ssock = NULL; + ret = 0; + join: + virThreadJoin(&th); + cleanup: virObjectUnref(ssock); - for (i = 0; i < nlsock; i++) + for (i = 0; i < nlsock; i++) { + virNetSocketRemoveIOCallback(lsock[i]); + virNetSocketClose(lsock[i]); virObjectUnref(lsock[i]); + } VIR_FREE(lsock); VIR_FREE(path); if (tmpdir) @@ -431,6 +525,8 @@ mymain(void) signal(SIGPIPE, SIG_IGN); + virEventRegisterDefaultImpl(); + #ifdef HAVE_IFADDRS_H if (checkProtocols(&hasIPv4, &hasIPv6, &freePort) < 0) { fprintf(stderr, "Cannot identify IPv4/6 availability\n"); -- 2.14.3
On Tue, 2018-05-01 at 12:06 +0100, Daniel P. Berrangé wrote:
The current socket test is rather crazy in that it sets up a server listening for sockets and then runs a client connect call, relying on the fact that the kernel will accept this despite the application not having called accept() yet. It then closes the client socket and calls accept() on the server. On Linux accept() will always see that the client has gone and so skip the rest of the code. On FreeBSD, however, the accept sometimes succeeds, causing us to then go into code that attempts to read and write to the client which will fail aborting the test. The accept() never succeeds on FreeBSD guests
Double space.
with a single CPU, but as you add more CPUs, accept() becomes more and more likely to succeed, giving a 100% failure rate for the test when using 8 CPUs.
This completely rewrites the test so that it is avoids this designed in race condition. We simply spawn a background thread to act as the client, which will read a byte from the server and write it back again.
Same. [...]
+static int +testSocketAccept(const void *opaque) { virNetSocketPtr *lsock = NULL; /* Listen socket */ size_t nlsock = 0, i; virNetSocketPtr ssock = NULL; /* Server socket */ - virNetSocketPtr csock = NULL; /* Client socket */ + virNetSocketPtr rsock = NULL; /* Server from poll */
This comment doesn't look right... [...]
+ virThread th; + struct testClientData cdata = { 0 }; + bool goodsock = false; + char a = 'a', b = '\0';
Please put each variable declaration on a separate line. [...]
+ if (virNetSocketWrite(ssock, &a, 1) < 0 || + virNetSocketRead(ssock, &b, 1) < 0) + goto join;
Curly braces are required around the body here.
+ if (a != b) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "Bad data received '%x' != '%x'", a, b); + goto join; + }
I'd leave an empty line here. Everything else looks sensible enough and I've verified that with these changes the issue no longer reproduces on my test environment, so with the nits fixed Reviewed-by: Andrea Bolognani <abologna@redhat.com> -- Andrea Bolognani / Red Hat / Virtualization
participants (2)
-
Andrea Bolognani -
Daniel P. Berrangé