[libvirt] [patch 0/3] Fixes related to the IP address learning thread

Proper resubmission -- sorry for the noise. The following set of patches are primarily related to the thread learning the IP address of a VM and deal with: - ebtables cleanup before applying basic filtering rules while the IP address is detected - shutting down all traffic in case the filtering rules could not be applied by the thread - serialization of the teardown of eb/ip/ip6tables rules to occurr after the IP address learning thread has terminated - no to apply or tear any eb/ip/ip6tables rules while the IP address learning thread is active Regards, Stefan

The functions invoked by the IP address learning thread that apply some basic filtering rules did not clean up any previous filtering rules that may still be there (due to a libvirt restart for example). With the patch below all the rules are cleaned up first. Also, I am introducing a function to drop all traffic in case the IP address learning thread could not apply the rules. Signed-off-by: Stefan Berger <stefanb@us.ibm.com> --- src/conf/nwfilter_conf.h | 3 src/nwfilter/nwfilter_ebiptables_driver.c | 104 +++++++++++++++++++++++++----- src/nwfilter/nwfilter_learnipaddr.c | 4 - src/nwfilter/nwfilter_learnipaddr.h | 2 4 files changed, 96 insertions(+), 17 deletions(-) Index: libvirt-acl/src/nwfilter/nwfilter_ebiptables_driver.c =================================================================== --- libvirt-acl.orig/src/nwfilter/nwfilter_ebiptables_driver.c +++ libvirt-acl/src/nwfilter/nwfilter_ebiptables_driver.c @@ -102,6 +102,7 @@ static const char *m_physdev_out_str = " static int ebtablesRemoveBasicRules(const char *ifname); static int ebiptablesDriverInit(void); static void ebiptablesDriverShutdown(void); +static int ebtablesCleanAll(const char *ifname); struct ushort_map { @@ -2679,12 +2680,7 @@ ebtablesApplyBasicRules(const char *ifna virFormatMacAddr(macaddr, macaddr_str); - ebtablesUnlinkTmpRootChain(&buf, 1, ifname); - ebtablesUnlinkTmpRootChain(&buf, 0, ifname); - ebtablesRemoveTmpSubChains(&buf, ifname); - ebtablesRemoveTmpRootChain(&buf, 1, ifname); - ebtablesRemoveTmpRootChain(&buf, 0, ifname); - ebiptablesExecCLI(&buf, &cli_status); + ebtablesCleanAll(ifname); ebtablesCreateTmpRootChain(&buf, 1, ifname, 1); @@ -2723,6 +2719,7 @@ ebtablesApplyBasicRules(const char *ifna CMD_STOPONERR(1)); ebtablesLinkTmpRootChain(&buf, 1, ifname, 1); + ebtablesRenameTmpRootChain(&buf, 1, ifname); if (ebiptablesExecCLI(&buf, &cli_status) || cli_status != 0) goto tear_down_tmpebchains; @@ -2730,7 +2727,7 @@ ebtablesApplyBasicRules(const char *ifna return 0; tear_down_tmpebchains: - ebtablesRemoveBasicRules(ifname); + ebtablesCleanAll(ifname); virNWFilterReportError(VIR_ERR_BUILD_FIREWALL, "%s", @@ -2782,12 +2779,7 @@ ebtablesApplyDHCPOnlyRules(const char *i virFormatMacAddr(macaddr, macaddr_str); - ebtablesUnlinkTmpRootChain(&buf, 1, ifname); - ebtablesUnlinkTmpRootChain(&buf, 0, ifname); - ebtablesRemoveTmpSubChains(&buf, ifname); - ebtablesRemoveTmpRootChain(&buf, 1, ifname); - ebtablesRemoveTmpRootChain(&buf, 0, ifname); - ebiptablesExecCLI(&buf, &cli_status); + ebtablesCleanAll(ifname); ebtablesCreateTmpRootChain(&buf, 1, ifname, 1); ebtablesCreateTmpRootChain(&buf, 0, ifname, 1); @@ -2842,6 +2834,8 @@ ebtablesApplyDHCPOnlyRules(const char *i ebtablesLinkTmpRootChain(&buf, 1, ifname, 1); ebtablesLinkTmpRootChain(&buf, 0, ifname, 1); + ebtablesRenameTmpRootChain(&buf, 1, ifname); + ebtablesRenameTmpRootChain(&buf, 0, ifname); if (ebiptablesExecCLI(&buf, &cli_status) || cli_status != 0) goto tear_down_tmpebchains; @@ -2851,7 +2845,7 @@ ebtablesApplyDHCPOnlyRules(const char *i return 0; tear_down_tmpebchains: - ebtablesRemoveBasicRules(ifname); + ebtablesCleanAll(ifname); virNWFilterReportError(VIR_ERR_BUILD_FIREWALL, "%s", @@ -2863,15 +2857,96 @@ tear_down_tmpebchains: } +/** + * ebtablesApplyDropAllRules + * + * @ifname: name of the backend-interface to which to apply the rules + * + * Returns 0 on success, 1 on failure with the rules removed + * + * Apply filtering rules so that the VM cannot receive or send traffic. + */ +static int +ebtablesApplyDropAllRules(const char *ifname) +{ + virBuffer buf = VIR_BUFFER_INITIALIZER; + int cli_status; + char chain_in [MAX_CHAINNAME_LENGTH], + chain_out[MAX_CHAINNAME_LENGTH]; + + if (!ebtables_cmd_path) { + virNWFilterReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("cannot create rules since ebtables tool is " + "missing.")); + return 1; + } + + ebtablesCleanAll(ifname); + + ebtablesCreateTmpRootChain(&buf, 1, ifname, 1); + ebtablesCreateTmpRootChain(&buf, 0, ifname, 1); + + PRINT_ROOT_CHAIN(chain_in , CHAINPREFIX_HOST_IN_TEMP , ifname); + PRINT_ROOT_CHAIN(chain_out, CHAINPREFIX_HOST_OUT_TEMP, ifname); + + virBufferVSprintf(&buf, + CMD_DEF("%s -t %s -A %s -j DROP") CMD_SEPARATOR + CMD_EXEC + "%s", + + ebtables_cmd_path, EBTABLES_DEFAULT_TABLE, chain_in, + CMD_STOPONERR(1)); + + virBufferVSprintf(&buf, + CMD_DEF("%s -t %s -A %s -j DROP") CMD_SEPARATOR + CMD_EXEC + "%s", + + ebtables_cmd_path, EBTABLES_DEFAULT_TABLE, chain_out, + CMD_STOPONERR(1)); + + ebtablesLinkTmpRootChain(&buf, 1, ifname, 1); + ebtablesLinkTmpRootChain(&buf, 0, ifname, 1); + ebtablesRenameTmpRootChain(&buf, 1, ifname); + ebtablesRenameTmpRootChain(&buf, 0, ifname); + + if (ebiptablesExecCLI(&buf, &cli_status) || cli_status != 0) + goto tear_down_tmpebchains; + + return 0; + +tear_down_tmpebchains: + ebtablesCleanAll(ifname); + + virNWFilterReportError(VIR_ERR_BUILD_FIREWALL, + "%s", + _("Some rules could not be created.")); + + return 1; +} + + static int ebtablesRemoveBasicRules(const char *ifname) { + return ebtablesCleanAll(ifname); +} + + +static int ebtablesCleanAll(const char *ifname) +{ virBuffer buf = VIR_BUFFER_INITIALIZER; int cli_status; if (!ebtables_cmd_path) return 0; + ebtablesUnlinkRootChain(&buf, 1, ifname); + ebtablesUnlinkRootChain(&buf, 0, ifname); + ebtablesRemoveSubChains(&buf, ifname); + ebtablesRemoveRootChain(&buf, 1, ifname); + ebtablesRemoveRootChain(&buf, 0, ifname); + ebtablesUnlinkTmpRootChain(&buf, 1, ifname); ebtablesUnlinkTmpRootChain(&buf, 0, ifname); ebtablesRemoveTmpSubChains(&buf, ifname); @@ -3265,6 +3340,7 @@ virNWFilterTechDriver ebiptables_driver .canApplyBasicRules = ebiptablesCanApplyBasicRules, .applyBasicRules = ebtablesApplyBasicRules, .applyDHCPOnlyRules = ebtablesApplyDHCPOnlyRules, + .applyDropAllRules = ebtablesApplyDropAllRules, .removeBasicRules = ebtablesRemoveBasicRules, }; Index: libvirt-acl/src/nwfilter/nwfilter_learnipaddr.c =================================================================== --- libvirt-acl.orig/src/nwfilter/nwfilter_learnipaddr.c +++ libvirt-acl/src/nwfilter/nwfilter_learnipaddr.c @@ -598,8 +598,6 @@ learnIPAddressThread(void *arg) if (handle) pcap_close(handle); - techdriver->removeBasicRules(req->ifname); - if (req->status == 0) { int ret; char inetaddr[INET_ADDRSTRLEN]; @@ -624,6 +622,8 @@ learnIPAddressThread(void *arg) _("encountered an error on interface %s " "index %d"), req->ifname, req->ifindex); + + techdriver->applyDropAllRules(req->ifname); } memset(&req->thread, 0x0, sizeof(req->thread)); Index: libvirt-acl/src/conf/nwfilter_conf.h =================================================================== --- libvirt-acl.orig/src/conf/nwfilter_conf.h +++ libvirt-acl/src/conf/nwfilter_conf.h @@ -521,6 +521,8 @@ typedef int (*virNWFilterApplyDHCPOnlyRu typedef int (*virNWFilterRemoveBasicRules)(const char *ifname); +typedef int (*virNWFilterDropAllRules)(const char *ifname); + enum techDrvFlags { TECHDRV_FLAG_INITIALIZED = (1 << 0), }; @@ -544,6 +546,7 @@ struct _virNWFilterTechDriver { virNWFilterCanApplyBasicRules canApplyBasicRules; virNWFilterApplyBasicRules applyBasicRules; virNWFilterApplyDHCPOnlyRules applyDHCPOnlyRules; + virNWFilterDropAllRules applyDropAllRules; virNWFilterRemoveBasicRules removeBasicRules; }; Index: libvirt-acl/src/nwfilter/nwfilter_learnipaddr.h =================================================================== --- libvirt-acl.orig/src/nwfilter/nwfilter_learnipaddr.h +++ libvirt-acl/src/nwfilter/nwfilter_learnipaddr.h @@ -60,7 +60,7 @@ int virNWFilterLearnIPAddress(virNWFilte enum howDetect howDetect); virNWFilterIPAddrLearnReqPtr virNWFilterLookupLearnReq(int ifindex); - +int virNWFilterTerminateLearnReq(const char *ifname); void virNWFilterDelIpAddrForIfname(const char *ifname); const char *virNWFilterGetIpAddrForIfname(const char *ifname);

On Thu, Apr 29, 2010 at 09:34:46PM -0400, Stefan Berger wrote:
The functions invoked by the IP address learning thread that apply some basic filtering rules did not clean up any previous filtering rules that may still be there (due to a libvirt restart for example). With the patch below all the rules are cleaned up first.
Also, I am introducing a function to drop all traffic in case the IP address learning thread could not apply the rules.
Signed-off-by: Stefan Berger <stefanb@us.ibm.com>
--- src/conf/nwfilter_conf.h | 3 src/nwfilter/nwfilter_ebiptables_driver.c | 104 +++++++++++++++++++++++++----- src/nwfilter/nwfilter_learnipaddr.c | 4 - src/nwfilter/nwfilter_learnipaddr.h | 2 4 files changed, 96 insertions(+), 17 deletions(-)
Okay, I had to double check that ebiptablesExecCLI() reall freed the passed buffer content in all case, looks fine, ACK, Daniel -- Daniel Veillard | libxml Gnome XML XSLT toolkit http://xmlsoft.org/ daniel@veillard.com | Rpmfind RPM search engine http://rpmfind.net/ http://veillard.com/ | virtualization library http://libvirt.org/

Daniel Veillard <veillard@redhat.com> wrote on 04/30/2010 07:59:39 AM:
Okay, I had to double check that ebiptablesExecCLI() reall freed the passed buffer content in all case, looks fine,
ACK,
Pushed. Stefan
Daniel
-- Daniel Veillard | libxml Gnome XML XSLT toolkit
daniel@veillard.com | Rpmfind RPM search engine http://rpmfind.net/ http://veillard.com/ | virtualization library http://libvirt.org/

Introduce a function to notify the IP address learning thread to terminate and thus release the lock on the interface. Notify the thread before grabbing the lock on the interface and tearing down the rules. This prevents a 'virsh destroy' to tear down the rules that the IP address learning thread has applied. Signed-off-by; Stefan Berger <stefanb@us.ibm.com> --- src/nwfilter/nwfilter_gentech_driver.c | 8 ++++++++ src/nwfilter/nwfilter_learnipaddr.c | 27 ++++++++++++++++++++++++++- src/nwfilter/nwfilter_learnipaddr.h | 1 + 3 files changed, 35 insertions(+), 1 deletion(-) Index: libvirt-acl/src/nwfilter/nwfilter_learnipaddr.c =================================================================== --- libvirt-acl.orig/src/nwfilter/nwfilter_learnipaddr.c +++ libvirt-acl/src/nwfilter/nwfilter_learnipaddr.c @@ -243,8 +243,33 @@ virNWFilterRegisterLearnReq(virNWFilterI return res; } + #endif +int +virNWFilterTerminateLearnReq(const char *ifname) { + int rc = 1; + int ifindex; + virNWFilterIPAddrLearnReqPtr req; + + if (ifaceGetIndex(false, ifname, &ifindex) == 0) { + + IFINDEX2STR(ifindex_str, ifindex); + + virMutexLock(&pendingLearnReqLock); + + req = virHashLookup(pendingLearnReq, ifindex_str); + if (req) { + rc = 0; + req->terminate = true; + } + + virMutexUnlock(&pendingLearnReqLock); + } + + return rc; +} + virNWFilterIPAddrLearnReqPtr virNWFilterLookupLearnReq(int ifindex) { @@ -472,7 +497,7 @@ learnIPAddressThread(void *arg) if (!packet) { - if (threadsTerminate) { + if (threadsTerminate || req->terminate) { req->status = ECANCELED; showError = false; break; Index: libvirt-acl/src/nwfilter/nwfilter_gentech_driver.c =================================================================== --- libvirt-acl.orig/src/nwfilter/nwfilter_gentech_driver.c +++ libvirt-acl/src/nwfilter/nwfilter_gentech_driver.c @@ -937,10 +937,18 @@ _virNWFilterTeardownFilter(const char *i drvname); return 1; } + + virNWFilterTerminateLearnReq(ifname); + + if (virNWFilterLockIface(ifname)) + return 1; + techdriver->allTeardown(ifname); virNWFilterDelIpAddrForIfname(ifname); + virNWFilterUnlockIface(ifname); + return 0; } Index: libvirt-acl/src/nwfilter/nwfilter_learnipaddr.h =================================================================== --- libvirt-acl.orig/src/nwfilter/nwfilter_learnipaddr.h +++ libvirt-acl/src/nwfilter/nwfilter_learnipaddr.h @@ -46,6 +46,7 @@ struct _virNWFilterIPAddrLearnReq { int status; pthread_t thread; + volatile bool terminate; }; int virNWFilterLearnIPAddress(virNWFilterTechDriverPtr techdriver,

On Thu, Apr 29, 2010 at 09:34:47PM -0400, Stefan Berger wrote:
Introduce a function to notify the IP address learning thread to terminate and thus release the lock on the interface. Notify the thread before grabbing the lock on the interface and tearing down the rules. This prevents a 'virsh destroy' to tear down the rules that the IP address learning thread has applied.
Signed-off-by; Stefan Berger <stefanb@us.ibm.com>
--- src/nwfilter/nwfilter_gentech_driver.c | 8 ++++++++ src/nwfilter/nwfilter_learnipaddr.c | 27 ++++++++++++++++++++++++++- src/nwfilter/nwfilter_learnipaddr.h | 1 + 3 files changed, 35 insertions(+), 1 deletion(-)
Index: libvirt-acl/src/nwfilter/nwfilter_learnipaddr.c =================================================================== --- libvirt-acl.orig/src/nwfilter/nwfilter_learnipaddr.c +++ libvirt-acl/src/nwfilter/nwfilter_learnipaddr.c @@ -243,8 +243,33 @@ virNWFilterRegisterLearnReq(virNWFilterI return res; }
+ #endif
+int +virNWFilterTerminateLearnReq(const char *ifname) { + int rc = 1; + int ifindex; + virNWFilterIPAddrLearnReqPtr req; + + if (ifaceGetIndex(false, ifname, &ifindex) == 0) { + + IFINDEX2STR(ifindex_str, ifindex); + + virMutexLock(&pendingLearnReqLock); + + req = virHashLookup(pendingLearnReq, ifindex_str); + if (req) { + rc = 0; + req->terminate = true; + } + + virMutexUnlock(&pendingLearnReqLock); + } + + return rc; +} +
virNWFilterIPAddrLearnReqPtr virNWFilterLookupLearnReq(int ifindex) { @@ -472,7 +497,7 @@ learnIPAddressThread(void *arg)
if (!packet) {
- if (threadsTerminate) { + if (threadsTerminate || req->terminate) { req->status = ECANCELED; showError = false; break; Index: libvirt-acl/src/nwfilter/nwfilter_gentech_driver.c =================================================================== --- libvirt-acl.orig/src/nwfilter/nwfilter_gentech_driver.c +++ libvirt-acl/src/nwfilter/nwfilter_gentech_driver.c @@ -937,10 +937,18 @@ _virNWFilterTeardownFilter(const char *i drvname); return 1; } + + virNWFilterTerminateLearnReq(ifname); + + if (virNWFilterLockIface(ifname)) + return 1; + techdriver->allTeardown(ifname);
virNWFilterDelIpAddrForIfname(ifname);
+ virNWFilterUnlockIface(ifname); + return 0; }
Index: libvirt-acl/src/nwfilter/nwfilter_learnipaddr.h =================================================================== --- libvirt-acl.orig/src/nwfilter/nwfilter_learnipaddr.h +++ libvirt-acl/src/nwfilter/nwfilter_learnipaddr.h @@ -46,6 +46,7 @@ struct _virNWFilterIPAddrLearnReq {
int status; pthread_t thread; + volatile bool terminate; };
int virNWFilterLearnIPAddress(virNWFilterTechDriverPtr techdriver,
ACK, Daniel -- Daniel Veillard | libxml Gnome XML XSLT toolkit http://xmlsoft.org/ daniel@veillard.com | Rpmfind RPM search engine http://rpmfind.net/ http://veillard.com/ | virtualization library http://libvirt.org/

Daniel Veillard <veillard@redhat.com> wrote on 04/30/2010 08:00:46 AM: [...]
};
int virNWFilterLearnIPAddress(virNWFilterTechDriverPtr techdriver,
ACK,
Pushed. Stefan
Daniel
-- Daniel Veillard | libxml Gnome XML XSLT toolkit
daniel@veillard.com | Rpmfind RPM search engine http://rpmfind.net/ http://veillard.com/ | virtualization library http://libvirt.org/

Prevent updating and tearing down of filter while the IP address learning thread is running and has its own filtering rules applied. Signed-off-by: Stefan Berger <stefanb@us.ibm.com> Index: libvirt-acl/src/nwfilter/nwfilter_gentech_driver.c =================================================================== --- libvirt-acl.orig/src/nwfilter/nwfilter_gentech_driver.c +++ libvirt-acl/src/nwfilter/nwfilter_gentech_driver.c @@ -610,6 +610,8 @@ virNWFilterInstantiate(virConnectPtr con } else if (virHashSize(missing_vars->hashTable) > 1) { rc = 1; goto err_exit; + } else if (virNWFilterLookupLearnReq(ifindex) == NULL) { + goto err_exit; } rc = _virNWFilterInstantiateRec(conn, @@ -890,7 +892,9 @@ int virNWFilterRollbackUpdateFilter(virC const virDomainNetDefPtr net) { const char *drvname = EBIPTABLES_DRIVER_ID; + int ifindex; virNWFilterTechDriverPtr techdriver; + techdriver = virNWFilterTechDriverForName(drvname); if (!techdriver) { virNWFilterReportError(VIR_ERR_INTERNAL_ERROR, @@ -900,6 +904,11 @@ int virNWFilterRollbackUpdateFilter(virC return 1; } + /* don't tear anything while the address is being learned */ + if (ifaceGetIndex(true, net->ifname, &ifindex) == 0 && + virNWFilterLookupLearnReq(ifindex) != NULL) + return 0; + return techdriver->tearNewRules(conn, net->ifname); } @@ -909,7 +918,9 @@ virNWFilterTearOldFilter(virConnectPtr c virDomainNetDefPtr net) { const char *drvname = EBIPTABLES_DRIVER_ID; + int ifindex; virNWFilterTechDriverPtr techdriver; + techdriver = virNWFilterTechDriverForName(drvname); if (!techdriver) { virNWFilterReportError(VIR_ERR_INTERNAL_ERROR, @@ -919,6 +930,11 @@ virNWFilterTearOldFilter(virConnectPtr c return 1; } + /* don't tear anything while the address is being learned */ + if (ifaceGetIndex(true, net->ifname, &ifindex) == 0 && + virNWFilterLookupLearnReq(ifindex) != NULL) + return 0; + return techdriver->tearOldRules(conn, net->ifname); }

On Thu, Apr 29, 2010 at 09:34:48PM -0400, Stefan Berger wrote:
Prevent updating and tearing down of filter while the IP address learning thread is running and has its own filtering rules applied.
Signed-off-by: Stefan Berger <stefanb@us.ibm.com>
Index: libvirt-acl/src/nwfilter/nwfilter_gentech_driver.c =================================================================== --- libvirt-acl.orig/src/nwfilter/nwfilter_gentech_driver.c +++ libvirt-acl/src/nwfilter/nwfilter_gentech_driver.c @@ -610,6 +610,8 @@ virNWFilterInstantiate(virConnectPtr con } else if (virHashSize(missing_vars->hashTable) > 1) { rc = 1; goto err_exit; + } else if (virNWFilterLookupLearnReq(ifindex) == NULL) { + goto err_exit; }
rc = _virNWFilterInstantiateRec(conn, @@ -890,7 +892,9 @@ int virNWFilterRollbackUpdateFilter(virC const virDomainNetDefPtr net) { const char *drvname = EBIPTABLES_DRIVER_ID; + int ifindex; virNWFilterTechDriverPtr techdriver; + techdriver = virNWFilterTechDriverForName(drvname); if (!techdriver) { virNWFilterReportError(VIR_ERR_INTERNAL_ERROR, @@ -900,6 +904,11 @@ int virNWFilterRollbackUpdateFilter(virC return 1; }
+ /* don't tear anything while the address is being learned */ + if (ifaceGetIndex(true, net->ifname, &ifindex) == 0 && + virNWFilterLookupLearnReq(ifindex) != NULL) + return 0; + return techdriver->tearNewRules(conn, net->ifname); }
@@ -909,7 +918,9 @@ virNWFilterTearOldFilter(virConnectPtr c virDomainNetDefPtr net) { const char *drvname = EBIPTABLES_DRIVER_ID; + int ifindex; virNWFilterTechDriverPtr techdriver; + techdriver = virNWFilterTechDriverForName(drvname); if (!techdriver) { virNWFilterReportError(VIR_ERR_INTERNAL_ERROR, @@ -919,6 +930,11 @@ virNWFilterTearOldFilter(virConnectPtr c return 1; }
+ /* don't tear anything while the address is being learned */ + if (ifaceGetIndex(true, net->ifname, &ifindex) == 0 && + virNWFilterLookupLearnReq(ifindex) != NULL) + return 0; + return techdriver->tearOldRules(conn, net->ifname); }
Looks fine, ACK, Daniel -- Daniel Veillard | libxml Gnome XML XSLT toolkit http://xmlsoft.org/ daniel@veillard.com | Rpmfind RPM search engine http://rpmfind.net/ http://veillard.com/ | virtualization library http://libvirt.org/

Daniel Veillard <veillard@redhat.com> wrote on 04/30/2010 08:01:50 AM:
Looks fine,
ACK,
Pushed. Thanks. Stefan
Daniel
-- Daniel Veillard | libxml Gnome XML XSLT toolkit
daniel@veillard.com | Rpmfind RPM search engine http://rpmfind.net/ http://veillard.com/ | virtualization library http://libvirt.org/
participants (2)
-
Daniel Veillard
-
Stefan Berger