This is so far a more or less experimental patch that adds code for communicating with the external lldpad daemon using the recently added netlink messages. It's communicating using multicast netlink messages in case of 802.1Qbg and unicast using 802.1Qbh. In the multicast case I am currently expecting an immediate response back from the (dummy) server to indicate the status of the port profile setup (IFLA_PORT_RESPONSE). A real server should probably then mimic the kernel driver and wait for a RTM_GETLINK to return the response... After adding netlink code for 802.1Qbg it wasn't difficult to add code for 802.1Qbh anymore, so some code for 802.1Qbh is already here. I am also adding detection code to configure.ac on whether the recently added netlink messages and the defines are available and set the WITH_VSI_VEPA #define if it is. Lots of debugging output is still in this patch. Signed-off-by: Stefan Berger <stefanb@linux.vnet.ibm.com> --- configure.ac | 16 + src/util/macvtap.c | 595 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 604 insertions(+), 7 deletions(-) Index: libvirt-acl/src/util/macvtap.c =================================================================== --- libvirt-acl.orig/src/util/macvtap.c +++ libvirt-acl/src/util/macvtap.c @@ -27,7 +27,7 @@ #include <config.h> -#if WITH_MACVTAP +#if WITH_MACVTAP || WITH_VSI_VEPA # include <stdio.h> # include <errno.h> @@ -41,6 +41,8 @@ # include <linux/rtnetlink.h> # include <linux/if_tun.h> +# include <netlink/msg.h> + # include "util.h" # include "memory.h" # include "logging.h" @@ -58,6 +60,10 @@ # define MACVTAP_NAME_PREFIX "macvtap" # define MACVTAP_NAME_PATTERN "macvtap%d" +# define MICROSEC_PER_SEC (1000 * 1000) + + +# define USE_IFLA_PORT_SELF static int associatePortProfileId(const char *macvtap_ifname, const virVSIProfileDefPtr vsi, @@ -67,6 +73,11 @@ static int associatePortProfileId(const static int disassociatePortProfileId(const char *macvtap_ifname, const virVSIProfileDefPtr vsi); +enum virVSIOp { + ASSOCIATE = 0x1, + DISASSOCIATE = 0x2, +}; + static int nlOpen(void) { @@ -159,6 +170,156 @@ err_exit: } +/** + * nlCommWaitSuccess: + * + * @nlmsg: pointer to netlink message + * @nl_grousp: the netlink multicast groups to send to + * @respbuf: pointer to pointer where response buffer will be allocated + * @respbuflen: pointer to integer holding the size of the response buffer + * on return of the function. + * @to_usecs: timeout in microseconds to wait for a success message + * to be returned + * + * Send the given message to the netlink multicast group and receive + * responses. Skip responses indicating an error and keep on receiving + * responses until a success response is returned. + * Returns 0 on success, -1 on error. In case of error, no response + * buffer will be returned. + */ +static int +nlCommWaitSuccess(struct nlmsghdr *nlmsg, int nl_groups, + char **respbuf, int *respbuflen, long to_usecs) +{ + int rc = 0; + struct sockaddr_nl nladdr = { + .nl_family = AF_NETLINK, + .nl_pid = 0, + .nl_groups = nl_groups, + }; + int rcvChunkSize = 1024; // expecting less than that + int rcvoffset = 0; + ssize_t nbytes; + int n; + struct timeval tv = { + .tv_sec = to_usecs / MICROSEC_PER_SEC, + .tv_usec = to_usecs % MICROSEC_PER_SEC, + }; + fd_set rfds; + bool gotvalid = false; + int fd = nlOpen(); + static uint32_t seq = 0x1234; + uint32_t myseq = seq++; + uint32_t mypid = getpid(); + + if (fd < 0) + return -1; + + nlmsg->nlmsg_pid = mypid; + nlmsg->nlmsg_seq = myseq; + nlmsg->nlmsg_flags |= NLM_F_ACK; + + nbytes = sendto(fd, (void *)nlmsg, nlmsg->nlmsg_len, 0, + (struct sockaddr *)&nladdr, sizeof(nladdr)); + if (nbytes < 0) { + virReportSystemError(errno, + "%s", _("cannot send to netlink socket")); + rc = -1; + goto err_exit; + } + fprintf(stderr,"sent %d bytes\n", (int)nbytes); + + while (!gotvalid) { + rcvoffset = 0; + while (1) { + socklen_t addrlen = sizeof(nladdr); + + if (VIR_REALLOC_N(*respbuf, rcvoffset+rcvChunkSize) < 0) { + virReportOOMError(); + rc = -1; + goto err_exit; + } + + FD_ZERO(&rfds); + FD_SET(fd, &rfds); + + n = select(fd + 1, &rfds, NULL, NULL, &tv); + if (n == 0) { + fprintf(stderr,"Select timed out.\n"); + rc = -1; + goto err_exit; + } + + nbytes = recvfrom(fd, &((*respbuf)[rcvoffset]), rcvChunkSize, 0, + (struct sockaddr *)&nladdr, &addrlen); + fprintf(stderr,"got something... %d bytes\n",(int)nbytes); + if (nbytes < 0) { + if (errno == EAGAIN || errno == EINTR) + continue; + virReportSystemError(errno, "%s", + _("error receiving from netlink socket")); + rc = -1; + goto err_exit; + } + rcvoffset += nbytes; + break; + } + *respbuflen = rcvoffset; + + /* check message for error */ + if (*respbuflen > NLMSG_LENGTH(0) && *respbuf != NULL) { + struct nlmsghdr *resp = (struct nlmsghdr *)*respbuf; + struct nlmsgerr *err; + + fprintf(stderr,"resp->nlmsg_type=%d\n", resp->nlmsg_type); + fprintf(stderr,"resp->nlmsg_flags=0x%x\n", resp->nlmsg_flags); + fprintf(stderr,"resp->nlmsg_seq=%d (%d)\n", resp->nlmsg_seq, myseq); + fprintf(stderr,"resp->nlmsg_pid=%d (%d)\n", resp->nlmsg_pid, mypid); + + if (resp->nlmsg_pid != mypid || + resp->nlmsg_seq != myseq) + continue; + + switch (resp->nlmsg_type) { + default: + case NLMSG_ERROR: + err = (struct nlmsgerr *)NLMSG_DATA(resp); + if (resp->nlmsg_len >= NLMSG_LENGTH(sizeof(*err))) { + fprintf(stderr,"error %d\n", -err->error); + if (-err->error != EOPNOTSUPP) { + /* assuming error msg from daemon */ + fprintf(stderr,"Got error %d, but assuming its response from lldpad: len = %d\n", + -err->error, + *respbuflen); + gotvalid = true; + break; + } + } + /* whatever this is, skip it */ + VIR_FREE(*respbuf); + *respbuf = NULL; + *respbuflen = 0; + fprintf(stderr,"Skipping error resp. msg.\n"); + break; + case NLMSG_DONE: + gotvalid = true; + break; + } + } + } + +err_exit: + if (rc == -1) { + VIR_FREE(*respbuf); + *respbuf = NULL; + *respbuflen = 0; + } + + nlClose(fd); + return rc; +} + + static struct rtattr * rtattrCreate(char *buffer, int bufsize, int type, const void *data, int datalen) @@ -204,6 +365,8 @@ nlAppend(struct nlmsghdr *nlm, int totle } +# if WITH_MACVTAP + static int link_add(const char *type, const unsigned char *macaddress, int macaddrsize, @@ -652,7 +815,7 @@ create_name: rc = associatePortProfileId(cr_ifname, &net->data.direct.vsiProfile, - -1, + PORT_SELF_VF, vmuuid); if (rc != 0) goto link_del_exit; @@ -711,8 +874,422 @@ delMacvtap(virDomainNetDefPtr net) } } -#endif +# endif + + +# ifdef IFLA_PORT_MAX + +static struct nla_policy ifla_policy[IFLA_MAX + 1] = +{ + [IFLA_VF_PORTS] = { .type = NLA_NESTED }, +}; + +static struct nla_policy ifla_vf_ports_policy[IFLA_VF_PORT_MAX + 1] = +{ + [IFLA_VF_PORT] = { .type = NLA_NESTED }, +}; + +static struct nla_policy ifla_port_policy[IFLA_PORT_MAX + 1] = +{ + [IFLA_PORT_RESPONSE] = { .type = NLA_U16 }, +}; + + +static int +getPortProfileStatus(struct nlmsghdr *nlm, uint16_t *status) +{ + int rc = 1; + const char *msg = NULL; + struct nlattr *tb[IFLA_MAX+1], + *tb2[IFLA_VF_PORT_MAX + 1], + *tb3[IFLA_PORT_MAX+1]; + + if (nlmsg_parse(nlm, sizeof(struct ifinfomsg), + (struct nlattr **)&tb, IFLA_MAX, ifla_policy)) { + msg = _("error parsing netlink response.\n"); + goto err_exit; + } + +# ifndef USE_IFLA_PORT_SELF + if (tb[IFLA_VF_PORTS]) { + if (nla_parse_nested(tb2, IFLA_VF_PORT_MAX, tb[IFLA_VF_PORTS], + ifla_vf_ports_policy)) { + msg = _("error parsing nested IFLA_VF_PORTS part"); + goto err_exit; + } + if (tb2[IFLA_VF_PORT]) { + if (nla_parse_nested(tb3, IFLA_PORT_MAX, tb2[IFLA_VF_PORT], + ifla_port_policy)) { + msg = _("error parsing nested IFLA_VF_PORT part"); + goto err_exit; + } +# else + (void)tb2; + (void)ifla_vf_ports_policy; + if (tb[IFLA_PORT_SELF]) { + if (1) { + if (nla_parse_nested(tb3, IFLA_PORT_MAX, tb[IFLA_PORT_SELF], + ifla_port_policy)) { + msg = _("error parsing nested IFLA_VF_PORT part"); + goto err_exit; + } +# endif + + if (tb3[IFLA_PORT_RESPONSE]) { + *status = *(uint16_t *)RTA_DATA(tb3[IFLA_PORT_RESPONSE]); + rc = 0; + } else { + msg = _("no IFLA_PORT_RESPONSE found in netlink message"); + goto err_exit; + } + } else { + msg = _("no IFLA_VF_PORT found in netlink message"); + goto err_exit; + } + } else { + msg = _("no IFLA_VF_PORTS found in netlink message"); + goto err_exit; + } + + err_exit: + if (msg) + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", msg); + + return rc; +} + + +static int +doPortProfileOpSetLink(bool multicast, + const char *ifname, int ifindex, + const char *profileId, + struct ifla_port_vsi *portVsi, + const unsigned char *instanceId, + const unsigned char *hostUUID, + int32_t vf, + uint8_t op) +{ + int rc = 0; + char nlmsgbuf[256]; + struct nlmsghdr *nlm = (struct nlmsghdr *)nlmsgbuf, *resp; + struct nlmsgerr *err; + char rtattbuf[64]; + struct rtattr *rta, *vfports, *vfport; + struct ifinfomsg ifinfo = { + .ifi_family = AF_UNSPEC, + .ifi_index = ifindex, + }; + char *recvbuf = NULL; + int recvbuflen = 0; + uint16_t status; + + memset(&nlmsgbuf, 0, sizeof(nlmsgbuf)); + + nlInit(nlm, NLM_F_REQUEST, RTM_SETLINK); + + if (!nlAppend(nlm, sizeof(nlmsgbuf), &ifinfo, sizeof(ifinfo))) + goto buffer_too_small; + +# ifndef USE_IFLA_PORT_SELF + + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_VF_PORTS, NULL, 0); + if (!rta) + goto buffer_too_small; + + if (!(vfports = nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len))) + goto buffer_too_small; + + /* beging nesting vfports */ + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_VF_PORT, NULL, 0); +# else + (void)vfports; + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_SELF, NULL, 0); +# endif + if (!rta) + goto buffer_too_small; + + if (!(vfport = nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len))) + goto buffer_too_small; + + /* begin nesting of vfport */ + if (profileId) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_PROFILE, + profileId, strlen(profileId) + 1); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + if (portVsi) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_VSI_TYPE, + portVsi, sizeof(*portVsi)); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + if (instanceId) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_INSTANCE_UUID, + instanceId, VIR_UUID_BUFLEN); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + if (hostUUID) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_HOST_UUID, + hostUUID, VIR_UUID_BUFLEN); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + if (vf >= 0) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_VF, + &vf, sizeof(vf)); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_REQUEST, + &op, sizeof(op)); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + + /* end nesting of vport */ + vfport->rta_len = (char *)nlm + nlm->nlmsg_len - (char *)vfport; + +# ifndef USE_IFLA_PORT_SELF + /* end nesting of vfports */ + vfports->rta_len = (char *)nlm + nlm->nlmsg_len - (char *)vfports; +# endif + + if (!multicast) { + if (nlComm(nlm, &recvbuf, &recvbuflen) < 0) + return -1; + } else { + if (nlCommWaitSuccess(nlm, RTMGRP_LINK, &recvbuf, &recvbuflen, + 5 * MICROSEC_PER_SEC) < 0) + return -1; + } + + if (recvbuflen < NLMSG_LENGTH(0) || recvbuf == NULL) + goto malformed_resp; + + resp = (struct nlmsghdr *)recvbuf; + + switch (resp->nlmsg_type) { + case NLMSG_ERROR: + err = (struct nlmsgerr *)NLMSG_DATA(resp); + if (resp->nlmsg_len < NLMSG_LENGTH(sizeof(*err))) + goto malformed_resp; + + switch (-err->error) { + case 0: + break; + + default: + virReportSystemError(-err->error, + _("error during VSI configuration of %s interface"), ifname); + rc = -1; + } + break; + + case NLMSG_DONE: + rc = getPortProfileStatus(resp, &status); + if (rc == 0 && status != PORT_VDP_RESPONSE_SUCCESS) { + macvtapError(VIR_ERR_INTERNAL_ERROR, + _("Response code from port profile setup: %d"), + status); + rc = -1; + } + break; + + default: + goto malformed_resp; + } + + VIR_FREE(recvbuf); + + return rc; + +malformed_resp: + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("malformed netlink response message")); + VIR_FREE(recvbuf); + return -1; + +buffer_too_small: + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("internal buffer is too small")); + return -1; +} + + +static int +doPortProfileOpCommon(bool multicast, + const char *ifname, int ifindex, + const char *profileId, + struct ifla_port_vsi *portVsi, + const unsigned char *instanceId, + const unsigned char *hostUUID, + int32_t vf, + uint8_t op) +{ + int rc; + + rc = doPortProfileOpSetLink(multicast, + ifname, ifindex, + profileId, + portVsi, + instanceId, + hostUUID, + vf, + op); + + if (rc != 0) { + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Sending of PortProfileRequest failed.\n")); + return rc; + } + + return rc; +} + +# endif /* IFLA_PORT_MAX */ + +static int +doPortProfileOp8021Qbg(const char *ifname, + const virVSIProfileDefPtr vsi, + int32_t vf, + enum virVSIOp vsiOp) +{ + int rc; + +# ifndef IFLA_VF_PORT_MAX + + (void)ifname; + (void)vsi; + (void)vf; + (void)vsiOp; + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Kernel VF Port support was missing at compile time.")); + rc = 1; + +# else /* IFLA_VF_PORT_MAX */ + + int op = PORT_REQUEST_ASSOCIATE; + struct ifla_port_vsi portVsi = { + .vsi_mgr_id = vsi->u.vsi8021Qbg.managerID, + .vsi_type_version = vsi->u.vsi8021Qbg.typeIDVersion, + }; + bool multicast = true; + int ifindex; + + if (ifaceGetIndex(true, ifname, &ifindex) != 0) { + rc = 1; + goto err_exit; + } + + portVsi.vsi_type_id[2] = vsi->u.vsi8021Qbg.typeID >> 16; + portVsi.vsi_type_id[1] = vsi->u.vsi8021Qbg.typeID >> 8; + portVsi.vsi_type_id[0] = vsi->u.vsi8021Qbg.typeID; + + switch (vsiOp) { + case ASSOCIATE: + op = PORT_REQUEST_ASSOCIATE; + break; + case DISASSOCIATE: + op = PORT_REQUEST_DISASSOCIATE; + break; + default: + macvtapError(VIR_ERR_INTERNAL_ERROR, + _("operation type %d not supported"), op); + rc = 1; + goto err_exit; + } + + rc = doPortProfileOpCommon(multicast, NULL, ifindex, + NULL, + &portVsi, + vsi->u.vsi8021Qbg.instanceID, + NULL, + vf, + op); + +err_exit: + +# endif /* IFLA_VF_PORT_MAX */ + + return rc; +} + + +static int +doPortProfileOp8021Qbh(const virVSIProfileDefPtr vsi, + const unsigned char *host_uuid, + const unsigned char *vm_uuid, + enum virVSIOp vsiOp) +{ + int rc; + +# ifndef IFLA_VF_PORT_MAX + + (void)vsi; + (void)host_uuid; + (void)vm_uuid; + (void)vsiOp; + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Kernel VF Port support was missing at compile time.")); + rc = 1; + +# else /* IFLA_VF_PORT_MAX */ + + int op = PORT_REQUEST_ASSOCIATE; + bool multicast = false; + + switch (vsiOp) { + case ASSOCIATE: + op = PORT_REQUEST_ASSOCIATE; + break; + case DISASSOCIATE: + op = PORT_REQUEST_DISASSOCIATE; + break; + default: + macvtapError(VIR_ERR_INTERNAL_ERROR, + _("operation type %d not supported"), op); + rc = 1; + goto err_exit; + } + + rc = doPortProfileOpCommon(multicast, NULL, -1, + vsi->u.vsi8021Qbh.profileID, + NULL, + host_uuid, + vm_uuid, + -1, + op); + +err_exit: + +# endif /* IFLA_VF_PORT_MAX */ + + return rc; +} /** * associatePortProfile @@ -749,11 +1326,13 @@ associatePortProfileId(const char *macvt break; case VIR_VSI_8021QBG: - + rc = doPortProfileOp8021Qbg(macvtap_ifname, vsi, vf, + ASSOCIATE); break; case VIR_VSI_8021QBH: - + rc = doPortProfileOp8021Qbh(vsi, vmuuid /* host_uuid */, vmuuid, + ASSOCIATE); break; } @@ -784,13 +1363,17 @@ disassociatePortProfileId(const char *ma break; case VIR_VSI_8021QBG: - + rc = doPortProfileOp8021Qbg(macvtap_ifname, vsi, -1, + DISASSOCIATE); break; case VIR_VSI_8021QBH: - + rc = doPortProfileOp8021Qbh(vsi, NULL, NULL, + DISASSOCIATE); break; } return rc; } + +#endif Index: libvirt-acl/configure.ac =================================================================== --- libvirt-acl.orig/configure.ac +++ libvirt-acl/configure.ac @@ -2005,13 +2005,26 @@ if test "$with_macvtap" != "no" ; then fi AM_CONDITIONAL([WITH_MACVTAP], [test "$with_macvtap" = "yes"]) +AC_TRY_COMPILE([ #include <sys/socket.h> + #include <linux/rtnetlink.h> ], + [ int x = IFLA_PORT_MAX; ], + [ with_vsi_vepa=yes ], + [ with_vsi_vepa=no ]) +if test "$with_vsi_vepa" = "yes"; then + val=1 +else + val=0 +fi +AC_DEFINE_UNQUOTED([WITH_VSI_VEPA], $val, [whether vsi vepa support is enabled]) +AM_CONDITIONAL([WITH_VSI_VEPA], [test "$with_vsi_vepa" = "yes"]) + dnl netlink library LIBNL_CFLAGS="" LIBNL_LIBS="" -if test "$with_macvtap" = "yes"; then +if test "$with_macvtap" = "yes" || "$with_vsi_vepa" = "yes"; then PKG_CHECK_MODULES([LIBNL], [libnl-1 >= $LIBNL_REQUIRED], [ ], [ AC_MSG_ERROR([libnl >= $LIBNL_REQUIRED is required for macvtap support]) @@ -2084,6 +2097,7 @@ AC_MSG_NOTICE([ Network: $with_network]) AC_MSG_NOTICE([Libvirtd: $with_libvirtd]) AC_MSG_NOTICE([ netcf: $with_netcf]) AC_MSG_NOTICE([ macvtap: $with_macvtap]) +AC_MSG_NOTICE([vsi-vepa: $with_vsi_vepa]) AC_MSG_NOTICE([]) AC_MSG_NOTICE([Storage Drivers]) AC_MSG_NOTICE([])