[libvirt] [v2 PATCH] add 802.1Qbh handling for port-profiles based on Stefan's previous patches
From: Scott Feldman <scofeldm@cisco.com> This patch builds on the work recently posted by Stefan Berger. It builds on top of Stefan's two posted patches: [PATCH v8] vepa: parsing for 802.1Qb{g|h} XML [RFC][PATCH 1/3] vepa+vsi: Introduce dependency on libnl Stefan's patches 2/3 and 3/3 are incorporated into my patch. Changes from v1 to v2: - Add Stefan's code for getPortProfileStatus - Poll for up to 2 secs for port-profile status, at 1/8 sec intervals: - if status indicates error, abort openMacvtapTap - if status indicates success, exit polling - if status is "in-progress" after 2 secs of polling, exit polling loop silently, without error My patch finishes out the 802.1Qbh parts, which Stefan had mostly complete. I've tested using the recent kernel updates for VF_PORT netlink msgs and enic for Cisco's 10G Ethernet NIC. I tested many VMs, each with several direct interfaces, each configured with a port-profile per the XML. VM-to-VM, and VM-to-external work as expected. VM-to-VM on same host (using same NIC) works same as VM-to-VM where VMs are on diff hosts. I'm able to change settings on the port-profile while the VM is running to change the virtual port behaviour. For example, adjusting a QoS setting like rate limit. All VMs with interfaces using that port-profile immediatly see the effect of the change to the port-profile. I don't have a SR-IOV device to test so source dev is a non-SR-IOV device, but most of the code paths include support for specifing the source dev and VF index. We'll need to complete this by discovering the PF given the VF linkdev. Once we have the PF, we'll also have the VF index. All this info- mation is available from sysfs. Signed-off-by: Scott Feldman <scofeldm@cisco.com> --- configure.ac | 16 + src/util/macvtap.c | 778 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 778 insertions(+), 16 deletions(-) diff --git a/configure.ac b/configure.ac index 36ba703..885b0ae 100644 --- a/configure.ac +++ b/configure.ac @@ -2005,13 +2005,26 @@ if test "$with_macvtap" != "no" ; then fi AM_CONDITIONAL([WITH_MACVTAP], [test "$with_macvtap" = "yes"]) +AC_TRY_COMPILE([ #include <sys/socket.h> + #include <linux/rtnetlink.h> ], + [ int x = IFLA_PORT_MAX; ], + [ with_virtualport=yes ], + [ with_virtualport=no ]) +if test "$with_virtualport" = "yes"; then + val=1 +else + val=0 +fi +AC_DEFINE_UNQUOTED([WITH_VIRTUALPORT], $val, [whether vsi vepa support is enabled]) +AM_CONDITIONAL([WITH_VIRTUALPORT], [test "$with_virtualport" = "yes"]) + dnl netlink library LIBNL_CFLAGS="" LIBNL_LIBS="" -if test "$with_macvtap" = "yes"; then +if test "$with_macvtap" = "yes" || "$with_virtualport" = "yes"; then PKG_CHECK_MODULES([LIBNL], [libnl-1 >= $LIBNL_REQUIRED], [ ], [ AC_MSG_ERROR([libnl >= $LIBNL_REQUIRED is required for macvtap support]) @@ -2084,6 +2097,7 @@ AC_MSG_NOTICE([ Network: $with_network]) AC_MSG_NOTICE([Libvirtd: $with_libvirtd]) AC_MSG_NOTICE([ netcf: $with_netcf]) AC_MSG_NOTICE([ macvtap: $with_macvtap]) +AC_MSG_NOTICE([virtport: $with_virtualport]) AC_MSG_NOTICE([]) AC_MSG_NOTICE([Storage Drivers]) AC_MSG_NOTICE([]) diff --git a/src/util/macvtap.c b/src/util/macvtap.c index da81447..9ecc7f1 100644 --- a/src/util/macvtap.c +++ b/src/util/macvtap.c @@ -27,7 +27,7 @@ #include <config.h> -#if WITH_MACVTAP +#if WITH_MACVTAP || WITH_VIRTUALPORT # include <stdio.h> # include <errno.h> @@ -41,6 +41,8 @@ # include <linux/rtnetlink.h> # include <linux/if_tun.h> +# include <netlink/msg.h> + # include "util.h" # include "memory.h" # include "logging.h" @@ -58,15 +60,24 @@ # define MACVTAP_NAME_PREFIX "macvtap" # define MACVTAP_NAME_PATTERN "macvtap%d" +# define MICROSEC_PER_SEC (1000 * 1000) + static int associatePortProfileId(const char *macvtap_ifname, + const char *linkdev, const virVirtualPortProfileDefPtr virtPort, - int vf, - const unsigned char *vmuuid); + const unsigned char *vmuuid, + const unsigned char *hostuuid); static int disassociatePortProfileId(const char *macvtap_ifname, + const char *linkdev, const virVirtualPortProfileDefPtr virtPort); +enum virVirtualPortOp { + ASSOCIATE = 0x1, + DISASSOCIATE = 0x2, +}; + static int nlOpen(void) { @@ -159,6 +170,169 @@ err_exit: } +# ifdef IFLA_VF_PORT_MAX + +/** + * nlCommWaitSuccess: + * + * @nlmsg: pointer to netlink message + * @nl_grousp: the netlink multicast groups to send to + * @respbuf: pointer to pointer where response buffer will be allocated + * @respbuflen: pointer to integer holding the size of the response buffer + * on return of the function. + * @to_usecs: timeout in microseconds to wait for a success message + * to be returned + * + * Send the given message to the netlink multicast group and receive + * responses. Skip responses indicating an error and keep on receiving + * responses until a success response is returned. + * Returns 0 on success, -1 on error. In case of error, no response + * buffer will be returned. + */ +static int +nlCommWaitSuccess(struct nlmsghdr *nlmsg, int nl_groups, + char **respbuf, int *respbuflen, long to_usecs) +{ + int rc = 0; + struct sockaddr_nl nladdr = { + .nl_family = AF_NETLINK, + .nl_pid = getpid(), + .nl_groups = nl_groups, + }; + int rcvChunkSize = 1024; // expecting less than that + int rcvoffset = 0; + ssize_t nbytes; + int n; + struct timeval tv = { + .tv_sec = to_usecs / MICROSEC_PER_SEC, + .tv_usec = to_usecs % MICROSEC_PER_SEC, + }; + fd_set rfds; + bool gotvalid = false; + int fd = nlOpen(); + static uint32_t seq = 0x1234; + uint32_t myseq = seq++; + uint32_t mypid = getpid(); + + if (fd < 0) + return -1; + + nlmsg->nlmsg_pid = mypid; + nlmsg->nlmsg_seq = myseq; + nlmsg->nlmsg_flags |= NLM_F_ACK; + + nbytes = sendto(fd, (void *)nlmsg, nlmsg->nlmsg_len, 0, + (struct sockaddr *)&nladdr, sizeof(nladdr)); + if (nbytes < 0) { + virReportSystemError(errno, + "%s", _("cannot send to netlink socket")); + rc = -1; + goto err_exit; + } + fprintf(stderr,"sent %d bytes\n", (int)nbytes); + + while (!gotvalid) { + rcvoffset = 0; + while (1) { + socklen_t addrlen = sizeof(nladdr); + + if (VIR_REALLOC_N(*respbuf, rcvoffset+rcvChunkSize) < 0) { + virReportOOMError(); + rc = -1; + goto err_exit; + } + + FD_ZERO(&rfds); + FD_SET(fd, &rfds); + + n = select(fd + 1, &rfds, NULL, NULL, &tv); + if (n == 0) { + fprintf(stderr,"Select timed out.\n"); + rc = -1; + goto err_exit; + } + + nbytes = recvfrom(fd, &((*respbuf)[rcvoffset]), rcvChunkSize, 0, + (struct sockaddr *)&nladdr, &addrlen); + fprintf(stderr,"got something... %d bytes\n",(int)nbytes); + if (nbytes < 0) { + if (errno == EAGAIN || errno == EINTR) + continue; + virReportSystemError(errno, "%s", + _("error receiving from netlink socket")); + rc = -1; + goto err_exit; + } + rcvoffset += nbytes; + break; + } + *respbuflen = rcvoffset; + + /* check message for error */ + if (*respbuflen > NLMSG_LENGTH(0) && *respbuf != NULL) { + struct nlmsghdr *resp = (struct nlmsghdr *)*respbuf; + struct nlmsgerr *err; + + fprintf(stderr,"resp->nlmsg_type=%d\n", resp->nlmsg_type); + fprintf(stderr,"resp->nlmsg_flags=0x%x\n", resp->nlmsg_flags); + fprintf(stderr,"resp->nlmsg_seq=%d (%d)\n", resp->nlmsg_seq, myseq); + fprintf(stderr,"resp->nlmsg_pid=%d (%d)\n", resp->nlmsg_pid, mypid); + + if (resp->nlmsg_pid != mypid || + resp->nlmsg_seq != myseq) + continue; + + /* skip reflected message */ + if (resp->nlmsg_type & 0x10) + continue; + + switch (resp->nlmsg_type) { + case NLMSG_ERROR: + err = (struct nlmsgerr *)NLMSG_DATA(resp); + if (resp->nlmsg_len >= NLMSG_LENGTH(sizeof(*err))) { + fprintf(stderr,"error %d\n", -err->error); + if (-err->error != EOPNOTSUPP) { + /* assuming error msg from daemon */ + fprintf(stderr,"Got error %d, but assuming its response from lldpad: len = %d\n", + -err->error, + *respbuflen); + gotvalid = true; + break; + } + } + /* whatever this is, skip it */ + VIR_FREE(*respbuf); + *respbuf = NULL; + *respbuflen = 0; + fprintf(stderr,"Skipping error resp. msg.\n"); + break; + + case NLMSG_DONE: + gotvalid = true; + break; + + default: + VIR_FREE(*respbuf); + *respbuf = NULL; + *respbuflen = 0; + break; + } + } + } + +err_exit: + if (rc == -1) { + VIR_FREE(*respbuf); + *respbuf = NULL; + *respbuflen = 0; + } + + nlClose(fd); + return rc; +} + +#endif + static struct rtattr * rtattrCreate(char *buffer, int bufsize, int type, const void *data, int datalen) @@ -204,6 +378,8 @@ nlAppend(struct nlmsghdr *nlm, int totlen, const void *data, int datalen) } +# if WITH_MACVTAP + static int link_add(const char *type, const unsigned char *macaddress, int macaddrsize, @@ -651,9 +827,10 @@ create_name: } if (associatePortProfileId(cr_ifname, + net->data.direct.linkdev, &net->data.direct.virtPortProfile, - -1, - vmuuid) != 0) { + vmuuid, + NULL /* hostuuid */) != 0) { rc = -1; goto link_del_exit; } @@ -685,6 +862,7 @@ create_name: disassociate_exit: disassociatePortProfileId(cr_ifname, + net->data.direct.linkdev, &net->data.direct.virtPortProfile); link_del_exit: @@ -707,21 +885,572 @@ delMacvtap(virDomainNetDefPtr net) { if (net->ifname) { disassociatePortProfileId(net->ifname, + net->data.direct.linkdev, &net->data.direct.virtPortProfile); link_del(net->ifname); } } -#endif +# endif + + +# ifdef IFLA_PORT_MAX + +static struct nla_policy ifla_policy[IFLA_MAX + 1] = +{ + [IFLA_VF_PORTS] = { .type = NLA_NESTED }, +}; + +static struct nla_policy ifla_vf_ports_policy[IFLA_VF_PORT_MAX + 1] = +{ + [IFLA_VF_PORT] = { .type = NLA_NESTED }, +}; + +static struct nla_policy ifla_port_policy[IFLA_PORT_MAX + 1] = +{ + [IFLA_PORT_RESPONSE] = { .type = NLA_U16 }, +}; + + +static int +link_dump(int ifindex, const char *ifname, struct nlattr **tb, + char **recvbuf) +{ + int rc = 0; + char nlmsgbuf[256] = { 0, }; + struct nlmsghdr *nlm = (struct nlmsghdr *)nlmsgbuf, *resp; + struct nlmsgerr *err; + char rtattbuf[64]; + struct rtattr *rta; + struct ifinfomsg i = { + .ifi_family = AF_UNSPEC, + .ifi_index = ifindex + }; + int recvbuflen; + + *recvbuf = NULL; + + nlInit(nlm, NLM_F_REQUEST, RTM_GETLINK); + + if (!nlAppend(nlm, sizeof(nlmsgbuf), &i, sizeof(i))) + goto buffer_too_small; + + if (ifindex < 0 && ifname != NULL) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_IFNAME, + ifname, strlen(ifname) + 1); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + if (nlComm(nlm, recvbuf, &recvbuflen) < 0) + return -1; + + if (recvbuflen < NLMSG_LENGTH(0) || *recvbuf == NULL) + goto malformed_resp; + + resp = (struct nlmsghdr *)*recvbuf; + + switch (resp->nlmsg_type) { + case NLMSG_ERROR: + err = (struct nlmsgerr *)NLMSG_DATA(resp); + if (resp->nlmsg_len < NLMSG_LENGTH(sizeof(*err))) + goto malformed_resp; + + switch (-err->error) { + case 0: + break; + + default: + virReportSystemError(-err->error, + _("error dumping %d interface"), + ifindex); + rc = -1; + } + break; + + case GENL_ID_CTRL: + case NLMSG_DONE: + if (nlmsg_parse(resp, sizeof(struct ifinfomsg), + tb, IFLA_MAX, ifla_policy)) { + goto malformed_resp; + } + break; + + default: + goto malformed_resp; + } + + if (rc != 0) + VIR_FREE(*recvbuf); + + return rc; + +malformed_resp: + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("malformed netlink response message")); + VIR_FREE(*recvbuf); + return -1; + +buffer_too_small: + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("internal buffer is too small")); + return -1; +} + + +static int +getPortProfileStatus(struct nlattr **tb, int32_t vf, uint16_t *status) +{ + int rc = 1; + const char *msg = NULL; + struct nlattr *tb2[IFLA_VF_PORT_MAX + 1], + *tb3[IFLA_PORT_MAX+1]; + + if (vf == PORT_SELF_VF) { + if (tb[IFLA_PORT_SELF]) { + if (nla_parse_nested(tb3, IFLA_PORT_MAX, tb[IFLA_PORT_SELF], + ifla_port_policy)) { + msg = _("error parsing nested IFLA_VF_PORT part"); + goto err_exit; + } + } + } else { + if (tb[IFLA_VF_PORTS]) { + if (nla_parse_nested(tb2, IFLA_VF_PORT_MAX, tb[IFLA_VF_PORTS], + ifla_vf_ports_policy)) { + msg = _("error parsing nested IFLA_VF_PORTS part"); + goto err_exit; + } + if (tb2[IFLA_VF_PORT]) { + if (nla_parse_nested(tb3, IFLA_PORT_MAX, tb2[IFLA_VF_PORT], + ifla_port_policy)) { + msg = _("error parsing nested IFLA_VF_PORT part"); + goto err_exit; + } + } + } + } + + if (tb3[IFLA_PORT_RESPONSE]) { + *status = *(uint16_t *)RTA_DATA(tb3[IFLA_PORT_RESPONSE]); + rc = 0; + } else { + msg = _("no IFLA_PORT_RESPONSE found in netlink message"); + goto err_exit; + } + +err_exit: + if (msg) + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", msg); + + return rc; +} + + +static int +doPortProfileOpSetLink(bool multicast, + const char *ifname, int ifindex, + const char *profileId, + struct ifla_port_vsi *portVsi, + const unsigned char *instanceId, + const unsigned char *hostUUID, + int32_t vf, + uint8_t op) +{ + int rc = 0; + char nlmsgbuf[256]; + struct nlmsghdr *nlm = (struct nlmsghdr *)nlmsgbuf, *resp; + struct nlmsgerr *err; + char rtattbuf[64]; + struct rtattr *rta, *vfports, *vfport; + struct ifinfomsg ifinfo = { + .ifi_family = AF_UNSPEC, + .ifi_index = ifindex, + }; + char *recvbuf = NULL; + int recvbuflen = 0; + + memset(&nlmsgbuf, 0, sizeof(nlmsgbuf)); + + nlInit(nlm, NLM_F_REQUEST, RTM_SETLINK); + + if (!nlAppend(nlm, sizeof(nlmsgbuf), &ifinfo, sizeof(ifinfo))) + goto buffer_too_small; + + if (vf == PORT_SELF_VF) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_SELF, NULL, 0); + } else { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_VF_PORTS, NULL, 0); + if (!rta) + goto buffer_too_small; + + if (!(vfports = nlAppend(nlm, sizeof(nlmsgbuf), + rtattbuf, rta->rta_len))) + goto buffer_too_small; + + /* beging nesting vfports */ + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_VF_PORT, NULL, 0); + } + + if (!rta) + goto buffer_too_small; + + if (!(vfport = nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len))) + goto buffer_too_small; + + if (profileId) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_PROFILE, + profileId, strlen(profileId) + 1); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + if (portVsi) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_VSI_TYPE, + portVsi, sizeof(*portVsi)); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + if (instanceId) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_INSTANCE_UUID, + instanceId, VIR_UUID_BUFLEN); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + if (hostUUID) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_HOST_UUID, + hostUUID, VIR_UUID_BUFLEN); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + if (vf != PORT_SELF_VF) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_VF, + &vf, sizeof(vf)); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_REQUEST, + &op, sizeof(op)); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + + /* end nesting of vport */ + vfport->rta_len = (char *)nlm + nlm->nlmsg_len - (char *)vfport; + + if (vf != PORT_SELF_VF) { + /* end nesting of vfports */ + vfports->rta_len = (char *)nlm + nlm->nlmsg_len - (char *)vfports; + } + + if (!multicast) { + if (nlComm(nlm, &recvbuf, &recvbuflen) < 0) + return -1; + } else { + if (nlCommWaitSuccess(nlm, RTMGRP_LINK, &recvbuf, &recvbuflen, + 5 * MICROSEC_PER_SEC) < 0) + return -1; + } + + if (recvbuflen < NLMSG_LENGTH(0) || recvbuf == NULL) + goto malformed_resp; + + resp = (struct nlmsghdr *)recvbuf; + + switch (resp->nlmsg_type) { + case NLMSG_ERROR: + err = (struct nlmsgerr *)NLMSG_DATA(resp); + if (resp->nlmsg_len < NLMSG_LENGTH(sizeof(*err))) + goto malformed_resp; + + switch (-err->error) { + case 0: + break; + + default: + virReportSystemError(-err->error, + _("error during virtual port configuration of %s interface"), ifname); + rc = -1; + } + break; + + case NLMSG_DONE: + break; + + default: + goto malformed_resp; + } + + VIR_FREE(recvbuf); + + return rc; + +malformed_resp: + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("malformed netlink response message")); + VIR_FREE(recvbuf); + return -1; + +buffer_too_small: + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("internal buffer is too small")); + return -1; +} + + +static int +doPortProfileOpCommon(bool multicast, + const char *ifname, int ifindex, + const char *profileId, + struct ifla_port_vsi *portVsi, + const unsigned char *instanceId, + const unsigned char *hostUUID, + int32_t vf, + uint8_t op) +{ + int rc; + char *recvbuf = NULL; + struct nlattr *tb[IFLA_MAX + 1]; + int repeats = 16; + uint16_t status; + + rc = doPortProfileOpSetLink(multicast, + ifname, ifindex, + profileId, + portVsi, + instanceId, + hostUUID, + vf, + op); + + if (rc != 0) { + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("sending of PortProfileRequest failed.\n")); + return rc; + } + + if (!multicast) { + while (--repeats) { + rc = link_dump(ifindex, ifname, tb, &recvbuf); + if (rc) + goto err_exit; + rc = getPortProfileStatus(tb, vf, &status); + if (rc == 0) { + if (status == PORT_PROFILE_RESPONSE_SUCCESS || + status == PORT_VDP_RESPONSE_SUCCESS) + break; + if (status != PORT_PROFILE_RESPONSE_INPROGRESS) + rc = 1; + } + usleep(125000); + + VIR_FREE(recvbuf); + } + } + +err_exit: + VIR_FREE(recvbuf); + + return rc; +} + +# endif /* IFLA_PORT_MAX */ + +static int +doPortProfileOp8021Qbg(const char *ifname, + const virVirtualPortProfileDefPtr virtPort, + int32_t vf, + enum virVirtualPortOp virtPortOp) +{ + int rc; + +# ifndef IFLA_VF_PORT_MAX + + (void)ifname; + (void)virtPort; + (void)vf; + (void)virtPortOp; + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Kernel VF Port support was missing at compile time.")); + rc = 1; +# else /* IFLA_VF_PORT_MAX */ + + int op = PORT_REQUEST_ASSOCIATE; + struct ifla_port_vsi portVsi = { + .vsi_mgr_id = virtPort->u.virtPort8021Qbg.managerID, + .vsi_type_version = virtPort->u.virtPort8021Qbg.typeIDVersion, + }; + bool multicast = true; + int ifindex; + + if (ifaceGetIndex(true, ifname, &ifindex) != 0) { + rc = 1; + goto err_exit; + } + + portVsi.vsi_type_id[2] = virtPort->u.virtPort8021Qbg.typeID >> 16; + portVsi.vsi_type_id[1] = virtPort->u.virtPort8021Qbg.typeID >> 8; + portVsi.vsi_type_id[0] = virtPort->u.virtPort8021Qbg.typeID; + + switch (virtPortOp) { + case ASSOCIATE: + op = PORT_REQUEST_ASSOCIATE; + break; + case DISASSOCIATE: + op = PORT_REQUEST_DISASSOCIATE; + break; + default: + macvtapError(VIR_ERR_INTERNAL_ERROR, + _("operation type %d not supported"), op); + rc = 1; + goto err_exit; + } + + rc = doPortProfileOpCommon(multicast, NULL, ifindex, + NULL, + &portVsi, + virtPort->u.virtPort8021Qbg.instanceID, + NULL, + vf, + op); + +err_exit: + +# endif /* IFLA_VF_PORT_MAX */ + + return rc; +} + + +static int +doPortProfileOp8021Qbh(const char *ifname, + const virVirtualPortProfileDefPtr virtPort, + int32_t vf, + const unsigned char *host_uuid, + const unsigned char *vm_uuid, + enum virVirtualPortOp virtPortOp) +{ + int rc; + +# ifndef IFLA_VF_PORT_MAX + + (void)ifname; + (void)virtPort; + (void)host_uuid; + (void)vm_uuid; + (void)virtPortOp; + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Kernel VF Port support was missing at compile time.")); + rc = 1; + +# else /* IFLA_VF_PORT_MAX */ + + int op = PORT_REQUEST_ASSOCIATE; + bool multicast = false; + int ifindex; + + if (ifaceGetIndex(true, ifname, &ifindex) != 0) { + rc = 1; + goto err_exit; + } + + switch (virtPortOp) { + case ASSOCIATE: + op = PORT_REQUEST_ASSOCIATE; + break; + case DISASSOCIATE: + op = PORT_REQUEST_DISASSOCIATE; + break; + default: + macvtapError(VIR_ERR_INTERNAL_ERROR, + _("operation type %d not supported"), op); + rc = 1; + goto err_exit; + } + + rc = doPortProfileOpCommon(multicast, ifname, ifindex, + virtPort->u.virtPort8021Qbh.profileID, + NULL, + vm_uuid, + host_uuid, + vf, + op); + + switch (virtPortOp) { + case ASSOCIATE: + ifaceUp(ifname); + break; + case DISASSOCIATE: + ifaceDown(ifname); + break; + } + +err_exit: + +# endif /* IFLA_VF_PORT_MAX */ + + return rc; +} + +static void +getPhysfn(const char *linkdev, + int32_t *vf, + char **physfndev) +{ + bool virtfn = false; + + if (virtfn) { + + // XXX: if linkdev is SR-IOV VF, then set vf = VF index + // XXX: and set linkdev = PF device + // XXX: need to use get_physical_function_linux() or + // XXX: something like that to get PF + // XXX: device and figure out VF index + + } else { + + /* Not SR-IOV VF: physfndev is linkdev and VF index + * refers to linkdev self + */ + + *vf = PORT_SELF_VF; + *physfndev = (char *)linkdev; + } +} /** * associatePortProfile * * @macvtap_ifname: The name of the macvtap device + * @linkdev: The link device in case of macvtap * @virtPort: pointer to the object holding port profile parameters - * @vf: virtual function number, -1 if to be ignored * @vmuuid : the UUID of the virtual machine + * @hostuuid : the UUID of the host machine * * Associate a port on a swtich with a profile. This function * may notify a kernel driver or an external daemon to run @@ -734,15 +1463,19 @@ delMacvtap(virDomainNetDefPtr net) */ static int associatePortProfileId(const char *macvtap_ifname, + const char *linkdev, const virVirtualPortProfileDefPtr virtPort, - int vf, - const unsigned char *vmuuid) + const unsigned char *vmuuid, + const unsigned char *hostuuid) { + char *physfndev; + int32_t vf; int rc = 0; + VIR_DEBUG("Associating port profile '%p' on link device '%s'", virtPort, macvtap_ifname); - (void)vf; - (void)vmuuid; + + getPhysfn(linkdev, &vf, &physfndev); switch (virtPort->virtPortType) { case VIR_VIRTUALPORT_NONE: @@ -750,11 +1483,14 @@ associatePortProfileId(const char *macvtap_ifname, break; case VIR_VIRTUALPORT_8021QBG: - + rc = doPortProfileOp8021Qbg(macvtap_ifname, virtPort, vf, + ASSOCIATE); break; case VIR_VIRTUALPORT_8021QBH: - + rc = doPortProfileOp8021Qbh(physfndev, virtPort, vf, + hostuuid, vmuuid, + ASSOCIATE); break; } @@ -766,6 +1502,7 @@ associatePortProfileId(const char *macvtap_ifname, * disassociatePortProfile * * @macvtap_ifname: The name of the macvtap device + * @linkdev: The link device in case of macvtap * @virtPort: point to object holding port profile parameters * * Returns 0 in case of success, != 0 otherwise with error @@ -773,25 +1510,36 @@ associatePortProfileId(const char *macvtap_ifname, */ static int disassociatePortProfileId(const char *macvtap_ifname, + const char *linkdev, const virVirtualPortProfileDefPtr virtPort) { + char *physfndev; + int32_t vf; int rc = 0; + VIR_DEBUG("Disassociating port profile id '%p' on link device '%s' ", virtPort, macvtap_ifname); + getPhysfn(linkdev, &vf, &physfndev); + switch (virtPort->virtPortType) { case VIR_VIRTUALPORT_NONE: case VIR_VIRTUALPORT_TYPE_LAST: break; case VIR_VIRTUALPORT_8021QBG: - + rc = doPortProfileOp8021Qbg(macvtap_ifname, virtPort, vf, + DISASSOCIATE); break; case VIR_VIRTUALPORT_8021QBH: - + rc = doPortProfileOp8021Qbh(physfndev, virtPort, vf, + NULL, NULL, + DISASSOCIATE); break; } return rc; } + +#endif
Scott, I got build errors (with upstream source, haven't tried RHEL yet) trying to build with the latest versions of Stefan's two patches plus yours. In the first case it looks like it's caused by IFLA_VF_PORT_MAX not being #defined (vf is not referenced inside that part of the #if). In the second case, well, PORT_SELF_VF isn't #defined. Is something missing from your patch? Or is there a missing build dependency? Or? CC libvirt_util_la-macvtap.lo cc1: warnings being treated as errors util/macvtap.c: In function 'doPortProfileOp8021Qbh': util/macvtap.c:1351: error: unused parameter 'vf' [-Wunused-parameter] util/macvtap.c: In function 'getPhysfn': util/macvtap.c:1439: error: 'PORT_SELF_VF' undeclared (first use in this function) util/macvtap.c:1439: error: (Each undeclared identifier is reported only once util/macvtap.c:1439: error: for each function it appears in.) make[3]: *** [libvirt_util_la-macvtap.lo] Error 1 On 05/24/2010 03:07 AM, Scott Feldman wrote:
From: Scott Feldman<scofeldm@cisco.com>
This patch builds on the work recently posted by Stefan Berger. It builds on top of Stefan's two posted patches:
[PATCH v8] vepa: parsing for 802.1Qb{g|h} XML [RFC][PATCH 1/3] vepa+vsi: Introduce dependency on libnl
Stefan's patches 2/3 and 3/3 are incorporated into my patch.
Changes from v1 to v2: - Add Stefan's code for getPortProfileStatus - Poll for up to 2 secs for port-profile status, at 1/8 sec intervals: - if status indicates error, abort openMacvtapTap - if status indicates success, exit polling - if status is "in-progress" after 2 secs of polling, exit polling loop silently, without error
My patch finishes out the 802.1Qbh parts, which Stefan had mostly complete. I've tested using the recent kernel updates for VF_PORT netlink msgs and enic for Cisco's 10G Ethernet NIC. I tested many VMs, each with several direct interfaces, each configured with a port-profile per the XML. VM-to-VM, and VM-to-external work as expected. VM-to-VM on same host (using same NIC) works same as VM-to-VM where VMs are on diff hosts. I'm able to change settings on the port-profile while the VM is running to change the virtual port behaviour. For example, adjusting a QoS setting like rate limit. All VMs with interfaces using that port-profile immediatly see the effect of the change to the port-profile.
I don't have a SR-IOV device to test so source dev is a non-SR-IOV device, but most of the code paths include support for specifing the source dev and VF index. We'll need to complete this by discovering the PF given the VF linkdev. Once we have the PF, we'll also have the VF index. All this info- mation is available from sysfs.
Signed-off-by: Scott Feldman<scofeldm@cisco.com> --- configure.ac | 16 + src/util/macvtap.c | 778 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 778 insertions(+), 16 deletions(-)
diff --git a/configure.ac b/configure.ac index 36ba703..885b0ae 100644 --- a/configure.ac +++ b/configure.ac @@ -2005,13 +2005,26 @@ if test "$with_macvtap" != "no" ; then fi AM_CONDITIONAL([WITH_MACVTAP], [test "$with_macvtap" = "yes"])
+AC_TRY_COMPILE([ #include<sys/socket.h> + #include<linux/rtnetlink.h> ], + [ int x = IFLA_PORT_MAX; ], + [ with_virtualport=yes ], + [ with_virtualport=no ]) +if test "$with_virtualport" = "yes"; then + val=1 +else + val=0 +fi +AC_DEFINE_UNQUOTED([WITH_VIRTUALPORT], $val, [whether vsi vepa support is enabled]) +AM_CONDITIONAL([WITH_VIRTUALPORT], [test "$with_virtualport" = "yes"]) +
dnl netlink library
LIBNL_CFLAGS="" LIBNL_LIBS=""
-if test "$with_macvtap" = "yes"; then +if test "$with_macvtap" = "yes" || "$with_virtualport" = "yes"; then PKG_CHECK_MODULES([LIBNL], [libnl-1>= $LIBNL_REQUIRED], [ ], [ AC_MSG_ERROR([libnl>= $LIBNL_REQUIRED is required for macvtap support]) @@ -2084,6 +2097,7 @@ AC_MSG_NOTICE([ Network: $with_network]) AC_MSG_NOTICE([Libvirtd: $with_libvirtd]) AC_MSG_NOTICE([ netcf: $with_netcf]) AC_MSG_NOTICE([ macvtap: $with_macvtap]) +AC_MSG_NOTICE([virtport: $with_virtualport]) AC_MSG_NOTICE([]) AC_MSG_NOTICE([Storage Drivers]) AC_MSG_NOTICE([]) diff --git a/src/util/macvtap.c b/src/util/macvtap.c index da81447..9ecc7f1 100644 --- a/src/util/macvtap.c +++ b/src/util/macvtap.c @@ -27,7 +27,7 @@
#include<config.h>
-#if WITH_MACVTAP +#if WITH_MACVTAP || WITH_VIRTUALPORT
# include<stdio.h> # include<errno.h> @@ -41,6 +41,8 @@ # include<linux/rtnetlink.h> # include<linux/if_tun.h>
+# include<netlink/msg.h> + # include "util.h" # include "memory.h" # include "logging.h" @@ -58,15 +60,24 @@ # define MACVTAP_NAME_PREFIX "macvtap" # define MACVTAP_NAME_PATTERN "macvtap%d"
+# define MICROSEC_PER_SEC (1000 * 1000) +
static int associatePortProfileId(const char *macvtap_ifname, + const char *linkdev, const virVirtualPortProfileDefPtr virtPort, - int vf, - const unsigned char *vmuuid); + const unsigned char *vmuuid, + const unsigned char *hostuuid);
static int disassociatePortProfileId(const char *macvtap_ifname, + const char *linkdev, const virVirtualPortProfileDefPtr virtPort);
+enum virVirtualPortOp { + ASSOCIATE = 0x1, + DISASSOCIATE = 0x2, +}; +
static int nlOpen(void) { @@ -159,6 +170,169 @@ err_exit: }
+# ifdef IFLA_VF_PORT_MAX + +/** + * nlCommWaitSuccess: + * + * @nlmsg: pointer to netlink message + * @nl_grousp: the netlink multicast groups to send to + * @respbuf: pointer to pointer where response buffer will be allocated + * @respbuflen: pointer to integer holding the size of the response buffer + * on return of the function. + * @to_usecs: timeout in microseconds to wait for a success message + * to be returned + * + * Send the given message to the netlink multicast group and receive + * responses. Skip responses indicating an error and keep on receiving + * responses until a success response is returned. + * Returns 0 on success, -1 on error. In case of error, no response + * buffer will be returned. + */ +static int +nlCommWaitSuccess(struct nlmsghdr *nlmsg, int nl_groups, + char **respbuf, int *respbuflen, long to_usecs) +{ + int rc = 0; + struct sockaddr_nl nladdr = { + .nl_family = AF_NETLINK, + .nl_pid = getpid(), + .nl_groups = nl_groups, + }; + int rcvChunkSize = 1024; // expecting less than that + int rcvoffset = 0; + ssize_t nbytes; + int n; + struct timeval tv = { + .tv_sec = to_usecs / MICROSEC_PER_SEC, + .tv_usec = to_usecs % MICROSEC_PER_SEC, + }; + fd_set rfds; + bool gotvalid = false; + int fd = nlOpen(); + static uint32_t seq = 0x1234; + uint32_t myseq = seq++; + uint32_t mypid = getpid(); + + if (fd< 0) + return -1; + + nlmsg->nlmsg_pid = mypid; + nlmsg->nlmsg_seq = myseq; + nlmsg->nlmsg_flags |= NLM_F_ACK; + + nbytes = sendto(fd, (void *)nlmsg, nlmsg->nlmsg_len, 0, + (struct sockaddr *)&nladdr, sizeof(nladdr)); + if (nbytes< 0) { + virReportSystemError(errno, + "%s", _("cannot send to netlink socket")); + rc = -1; + goto err_exit; + } + fprintf(stderr,"sent %d bytes\n", (int)nbytes); + + while (!gotvalid) { + rcvoffset = 0; + while (1) { + socklen_t addrlen = sizeof(nladdr); + + if (VIR_REALLOC_N(*respbuf, rcvoffset+rcvChunkSize)< 0) { + virReportOOMError(); + rc = -1; + goto err_exit; + } + + FD_ZERO(&rfds); + FD_SET(fd,&rfds); + + n = select(fd + 1,&rfds, NULL, NULL,&tv); + if (n == 0) { + fprintf(stderr,"Select timed out.\n"); + rc = -1; + goto err_exit; + } + + nbytes = recvfrom(fd,&((*respbuf)[rcvoffset]), rcvChunkSize, 0, + (struct sockaddr *)&nladdr,&addrlen); + fprintf(stderr,"got something... %d bytes\n",(int)nbytes); + if (nbytes< 0) { + if (errno == EAGAIN || errno == EINTR) + continue; + virReportSystemError(errno, "%s", + _("error receiving from netlink socket")); + rc = -1; + goto err_exit; + } + rcvoffset += nbytes; + break; + } + *respbuflen = rcvoffset; + + /* check message for error */ + if (*respbuflen> NLMSG_LENGTH(0)&& *respbuf != NULL) { + struct nlmsghdr *resp = (struct nlmsghdr *)*respbuf; + struct nlmsgerr *err; + + fprintf(stderr,"resp->nlmsg_type=%d\n", resp->nlmsg_type); + fprintf(stderr,"resp->nlmsg_flags=0x%x\n", resp->nlmsg_flags); + fprintf(stderr,"resp->nlmsg_seq=%d (%d)\n", resp->nlmsg_seq, myseq); + fprintf(stderr,"resp->nlmsg_pid=%d (%d)\n", resp->nlmsg_pid, mypid); + + if (resp->nlmsg_pid != mypid || + resp->nlmsg_seq != myseq) + continue; + + /* skip reflected message */ + if (resp->nlmsg_type& 0x10) + continue; + + switch (resp->nlmsg_type) { + case NLMSG_ERROR: + err = (struct nlmsgerr *)NLMSG_DATA(resp); + if (resp->nlmsg_len>= NLMSG_LENGTH(sizeof(*err))) { + fprintf(stderr,"error %d\n", -err->error); + if (-err->error != EOPNOTSUPP) { + /* assuming error msg from daemon */ + fprintf(stderr,"Got error %d, but assuming its response from lldpad: len = %d\n", + -err->error, + *respbuflen); + gotvalid = true; + break; + } + } + /* whatever this is, skip it */ + VIR_FREE(*respbuf); + *respbuf = NULL; + *respbuflen = 0; + fprintf(stderr,"Skipping error resp. msg.\n"); + break; + + case NLMSG_DONE: + gotvalid = true; + break; + + default: + VIR_FREE(*respbuf); + *respbuf = NULL; + *respbuflen = 0; + break; + } + } + } + +err_exit: + if (rc == -1) { + VIR_FREE(*respbuf); + *respbuf = NULL; + *respbuflen = 0; + } + + nlClose(fd); + return rc; +} + +#endif + static struct rtattr * rtattrCreate(char *buffer, int bufsize, int type, const void *data, int datalen) @@ -204,6 +378,8 @@ nlAppend(struct nlmsghdr *nlm, int totlen, const void *data, int datalen) }
+# if WITH_MACVTAP + static int link_add(const char *type, const unsigned char *macaddress, int macaddrsize, @@ -651,9 +827,10 @@ create_name: }
if (associatePortProfileId(cr_ifname, + net->data.direct.linkdev, &net->data.direct.virtPortProfile, - -1, - vmuuid) != 0) { + vmuuid, + NULL /* hostuuid */) != 0) { rc = -1; goto link_del_exit; } @@ -685,6 +862,7 @@ create_name:
disassociate_exit: disassociatePortProfileId(cr_ifname, + net->data.direct.linkdev, &net->data.direct.virtPortProfile);
link_del_exit: @@ -707,21 +885,572 @@ delMacvtap(virDomainNetDefPtr net) { if (net->ifname) { disassociatePortProfileId(net->ifname, + net->data.direct.linkdev, &net->data.direct.virtPortProfile); link_del(net->ifname); } }
-#endif +# endif + + +# ifdef IFLA_PORT_MAX + +static struct nla_policy ifla_policy[IFLA_MAX + 1] = +{ + [IFLA_VF_PORTS] = { .type = NLA_NESTED }, +}; + +static struct nla_policy ifla_vf_ports_policy[IFLA_VF_PORT_MAX + 1] = +{ + [IFLA_VF_PORT] = { .type = NLA_NESTED }, +}; + +static struct nla_policy ifla_port_policy[IFLA_PORT_MAX + 1] = +{ + [IFLA_PORT_RESPONSE] = { .type = NLA_U16 }, +}; + + +static int +link_dump(int ifindex, const char *ifname, struct nlattr **tb, + char **recvbuf) +{ + int rc = 0; + char nlmsgbuf[256] = { 0, }; + struct nlmsghdr *nlm = (struct nlmsghdr *)nlmsgbuf, *resp; + struct nlmsgerr *err; + char rtattbuf[64]; + struct rtattr *rta; + struct ifinfomsg i = { + .ifi_family = AF_UNSPEC, + .ifi_index = ifindex + }; + int recvbuflen; + + *recvbuf = NULL; + + nlInit(nlm, NLM_F_REQUEST, RTM_GETLINK); + + if (!nlAppend(nlm, sizeof(nlmsgbuf),&i, sizeof(i))) + goto buffer_too_small; + + if (ifindex< 0&& ifname != NULL) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_IFNAME, + ifname, strlen(ifname) + 1); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + if (nlComm(nlm, recvbuf,&recvbuflen)< 0) + return -1; + + if (recvbuflen< NLMSG_LENGTH(0) || *recvbuf == NULL) + goto malformed_resp; + + resp = (struct nlmsghdr *)*recvbuf; + + switch (resp->nlmsg_type) { + case NLMSG_ERROR: + err = (struct nlmsgerr *)NLMSG_DATA(resp); + if (resp->nlmsg_len< NLMSG_LENGTH(sizeof(*err))) + goto malformed_resp; + + switch (-err->error) { + case 0: + break; + + default: + virReportSystemError(-err->error, + _("error dumping %d interface"), + ifindex); + rc = -1; + } + break; + + case GENL_ID_CTRL: + case NLMSG_DONE: + if (nlmsg_parse(resp, sizeof(struct ifinfomsg), + tb, IFLA_MAX, ifla_policy)) { + goto malformed_resp; + } + break; + + default: + goto malformed_resp; + } + + if (rc != 0) + VIR_FREE(*recvbuf); + + return rc; + +malformed_resp: + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("malformed netlink response message")); + VIR_FREE(*recvbuf); + return -1; + +buffer_too_small: + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("internal buffer is too small")); + return -1; +} + + +static int +getPortProfileStatus(struct nlattr **tb, int32_t vf, uint16_t *status) +{ + int rc = 1; + const char *msg = NULL; + struct nlattr *tb2[IFLA_VF_PORT_MAX + 1], + *tb3[IFLA_PORT_MAX+1]; + + if (vf == PORT_SELF_VF) { + if (tb[IFLA_PORT_SELF]) { + if (nla_parse_nested(tb3, IFLA_PORT_MAX, tb[IFLA_PORT_SELF], + ifla_port_policy)) { + msg = _("error parsing nested IFLA_VF_PORT part"); + goto err_exit; + } + } + } else { + if (tb[IFLA_VF_PORTS]) { + if (nla_parse_nested(tb2, IFLA_VF_PORT_MAX, tb[IFLA_VF_PORTS], + ifla_vf_ports_policy)) { + msg = _("error parsing nested IFLA_VF_PORTS part"); + goto err_exit; + } + if (tb2[IFLA_VF_PORT]) { + if (nla_parse_nested(tb3, IFLA_PORT_MAX, tb2[IFLA_VF_PORT], + ifla_port_policy)) { + msg = _("error parsing nested IFLA_VF_PORT part"); + goto err_exit; + } + } + } + } + + if (tb3[IFLA_PORT_RESPONSE]) { + *status = *(uint16_t *)RTA_DATA(tb3[IFLA_PORT_RESPONSE]); + rc = 0; + } else { + msg = _("no IFLA_PORT_RESPONSE found in netlink message"); + goto err_exit; + } + +err_exit: + if (msg) + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", msg); + + return rc; +} + + +static int +doPortProfileOpSetLink(bool multicast, + const char *ifname, int ifindex, + const char *profileId, + struct ifla_port_vsi *portVsi, + const unsigned char *instanceId, + const unsigned char *hostUUID, + int32_t vf, + uint8_t op) +{ + int rc = 0; + char nlmsgbuf[256]; + struct nlmsghdr *nlm = (struct nlmsghdr *)nlmsgbuf, *resp; + struct nlmsgerr *err; + char rtattbuf[64]; + struct rtattr *rta, *vfports, *vfport; + struct ifinfomsg ifinfo = { + .ifi_family = AF_UNSPEC, + .ifi_index = ifindex, + }; + char *recvbuf = NULL; + int recvbuflen = 0; + + memset(&nlmsgbuf, 0, sizeof(nlmsgbuf)); + + nlInit(nlm, NLM_F_REQUEST, RTM_SETLINK); + + if (!nlAppend(nlm, sizeof(nlmsgbuf),&ifinfo, sizeof(ifinfo))) + goto buffer_too_small; + + if (vf == PORT_SELF_VF) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_SELF, NULL, 0); + } else { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_VF_PORTS, NULL, 0); + if (!rta) + goto buffer_too_small; + + if (!(vfports = nlAppend(nlm, sizeof(nlmsgbuf), + rtattbuf, rta->rta_len))) + goto buffer_too_small; + + /* beging nesting vfports */ + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_VF_PORT, NULL, 0); + } + + if (!rta) + goto buffer_too_small; + + if (!(vfport = nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len))) + goto buffer_too_small; + + if (profileId) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_PROFILE, + profileId, strlen(profileId) + 1); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + if (portVsi) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_VSI_TYPE, + portVsi, sizeof(*portVsi)); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + if (instanceId) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_INSTANCE_UUID, + instanceId, VIR_UUID_BUFLEN); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + if (hostUUID) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_HOST_UUID, + hostUUID, VIR_UUID_BUFLEN); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + if (vf != PORT_SELF_VF) { + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_VF, +&vf, sizeof(vf)); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + } + + rta = rtattrCreate(rtattbuf, sizeof(rtattbuf), IFLA_PORT_REQUEST, +&op, sizeof(op)); + if (!rta) + goto buffer_too_small; + + if (!nlAppend(nlm, sizeof(nlmsgbuf), rtattbuf, rta->rta_len)) + goto buffer_too_small; + + /* end nesting of vport */ + vfport->rta_len = (char *)nlm + nlm->nlmsg_len - (char *)vfport; + + if (vf != PORT_SELF_VF) { + /* end nesting of vfports */ + vfports->rta_len = (char *)nlm + nlm->nlmsg_len - (char *)vfports; + } + + if (!multicast) { + if (nlComm(nlm,&recvbuf,&recvbuflen)< 0) + return -1; + } else { + if (nlCommWaitSuccess(nlm, RTMGRP_LINK,&recvbuf,&recvbuflen, + 5 * MICROSEC_PER_SEC)< 0) + return -1; + } + + if (recvbuflen< NLMSG_LENGTH(0) || recvbuf == NULL) + goto malformed_resp; + + resp = (struct nlmsghdr *)recvbuf; + + switch (resp->nlmsg_type) { + case NLMSG_ERROR: + err = (struct nlmsgerr *)NLMSG_DATA(resp); + if (resp->nlmsg_len< NLMSG_LENGTH(sizeof(*err))) + goto malformed_resp; + + switch (-err->error) { + case 0: + break; + + default: + virReportSystemError(-err->error, + _("error during virtual port configuration of %s interface"), ifname); + rc = -1; + } + break; + + case NLMSG_DONE: + break; + + default: + goto malformed_resp; + } + + VIR_FREE(recvbuf); + + return rc; + +malformed_resp: + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("malformed netlink response message")); + VIR_FREE(recvbuf); + return -1; + +buffer_too_small: + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("internal buffer is too small")); + return -1; +} + + +static int +doPortProfileOpCommon(bool multicast, + const char *ifname, int ifindex, + const char *profileId, + struct ifla_port_vsi *portVsi, + const unsigned char *instanceId, + const unsigned char *hostUUID, + int32_t vf, + uint8_t op) +{ + int rc; + char *recvbuf = NULL; + struct nlattr *tb[IFLA_MAX + 1]; + int repeats = 16; + uint16_t status; + + rc = doPortProfileOpSetLink(multicast, + ifname, ifindex, + profileId, + portVsi, + instanceId, + hostUUID, + vf, + op); + + if (rc != 0) { + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("sending of PortProfileRequest failed.\n")); + return rc; + } + + if (!multicast) { + while (--repeats) { + rc = link_dump(ifindex, ifname, tb,&recvbuf); + if (rc) + goto err_exit; + rc = getPortProfileStatus(tb, vf,&status); + if (rc == 0) { + if (status == PORT_PROFILE_RESPONSE_SUCCESS || + status == PORT_VDP_RESPONSE_SUCCESS) + break; + if (status != PORT_PROFILE_RESPONSE_INPROGRESS) + rc = 1; + } + usleep(125000); + + VIR_FREE(recvbuf); + } + } + +err_exit: + VIR_FREE(recvbuf); + + return rc; +} + +# endif /* IFLA_PORT_MAX */ + +static int +doPortProfileOp8021Qbg(const char *ifname, + const virVirtualPortProfileDefPtr virtPort, + int32_t vf, + enum virVirtualPortOp virtPortOp) +{ + int rc; + +# ifndef IFLA_VF_PORT_MAX + + (void)ifname; + (void)virtPort; + (void)vf; + (void)virtPortOp; + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Kernel VF Port support was missing at compile time.")); + rc = 1;
+# else /* IFLA_VF_PORT_MAX */ + + int op = PORT_REQUEST_ASSOCIATE; + struct ifla_port_vsi portVsi = { + .vsi_mgr_id = virtPort->u.virtPort8021Qbg.managerID, + .vsi_type_version = virtPort->u.virtPort8021Qbg.typeIDVersion, + }; + bool multicast = true; + int ifindex; + + if (ifaceGetIndex(true, ifname,&ifindex) != 0) { + rc = 1; + goto err_exit; + } + + portVsi.vsi_type_id[2] = virtPort->u.virtPort8021Qbg.typeID>> 16; + portVsi.vsi_type_id[1] = virtPort->u.virtPort8021Qbg.typeID>> 8; + portVsi.vsi_type_id[0] = virtPort->u.virtPort8021Qbg.typeID; + + switch (virtPortOp) { + case ASSOCIATE: + op = PORT_REQUEST_ASSOCIATE; + break; + case DISASSOCIATE: + op = PORT_REQUEST_DISASSOCIATE; + break; + default: + macvtapError(VIR_ERR_INTERNAL_ERROR, + _("operation type %d not supported"), op); + rc = 1; + goto err_exit; + } + + rc = doPortProfileOpCommon(multicast, NULL, ifindex, + NULL, +&portVsi, + virtPort->u.virtPort8021Qbg.instanceID, + NULL, + vf, + op); + +err_exit: + +# endif /* IFLA_VF_PORT_MAX */ + + return rc; +} + + +static int +doPortProfileOp8021Qbh(const char *ifname, + const virVirtualPortProfileDefPtr virtPort, + int32_t vf, + const unsigned char *host_uuid, + const unsigned char *vm_uuid, + enum virVirtualPortOp virtPortOp) +{ + int rc; + +# ifndef IFLA_VF_PORT_MAX + + (void)ifname; + (void)virtPort; + (void)host_uuid; + (void)vm_uuid; + (void)virtPortOp; + macvtapError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Kernel VF Port support was missing at compile time.")); + rc = 1; + +# else /* IFLA_VF_PORT_MAX */ + + int op = PORT_REQUEST_ASSOCIATE; + bool multicast = false; + int ifindex; + + if (ifaceGetIndex(true, ifname,&ifindex) != 0) { + rc = 1; + goto err_exit; + } + + switch (virtPortOp) { + case ASSOCIATE: + op = PORT_REQUEST_ASSOCIATE; + break; + case DISASSOCIATE: + op = PORT_REQUEST_DISASSOCIATE; + break; + default: + macvtapError(VIR_ERR_INTERNAL_ERROR, + _("operation type %d not supported"), op); + rc = 1; + goto err_exit; + } + + rc = doPortProfileOpCommon(multicast, ifname, ifindex, + virtPort->u.virtPort8021Qbh.profileID, + NULL, + vm_uuid, + host_uuid, + vf, + op); + + switch (virtPortOp) { + case ASSOCIATE: + ifaceUp(ifname); + break; + case DISASSOCIATE: + ifaceDown(ifname); + break; + } + +err_exit: + +# endif /* IFLA_VF_PORT_MAX */ + + return rc; +} + +static void +getPhysfn(const char *linkdev, + int32_t *vf, + char **physfndev) +{ + bool virtfn = false; + + if (virtfn) { + + // XXX: if linkdev is SR-IOV VF, then set vf = VF index + // XXX: and set linkdev = PF device + // XXX: need to use get_physical_function_linux() or + // XXX: something like that to get PF + // XXX: device and figure out VF index + + } else { + + /* Not SR-IOV VF: physfndev is linkdev and VF index + * refers to linkdev self + */ + + *vf = PORT_SELF_VF; + *physfndev = (char *)linkdev; + } +}
/** * associatePortProfile * * @macvtap_ifname: The name of the macvtap device + * @linkdev: The link device in case of macvtap * @virtPort: pointer to the object holding port profile parameters - * @vf: virtual function number, -1 if to be ignored * @vmuuid : the UUID of the virtual machine + * @hostuuid : the UUID of the host machine * * Associate a port on a swtich with a profile. This function * may notify a kernel driver or an external daemon to run @@ -734,15 +1463,19 @@ delMacvtap(virDomainNetDefPtr net) */ static int associatePortProfileId(const char *macvtap_ifname, + const char *linkdev, const virVirtualPortProfileDefPtr virtPort, - int vf, - const unsigned char *vmuuid) + const unsigned char *vmuuid, + const unsigned char *hostuuid) { + char *physfndev; + int32_t vf; int rc = 0; + VIR_DEBUG("Associating port profile '%p' on link device '%s'", virtPort, macvtap_ifname); - (void)vf; - (void)vmuuid; + + getPhysfn(linkdev,&vf,&physfndev);
switch (virtPort->virtPortType) { case VIR_VIRTUALPORT_NONE: @@ -750,11 +1483,14 @@ associatePortProfileId(const char *macvtap_ifname, break;
case VIR_VIRTUALPORT_8021QBG: - + rc = doPortProfileOp8021Qbg(macvtap_ifname, virtPort, vf, + ASSOCIATE); break;
case VIR_VIRTUALPORT_8021QBH: - + rc = doPortProfileOp8021Qbh(physfndev, virtPort, vf, + hostuuid, vmuuid, + ASSOCIATE); break; }
@@ -766,6 +1502,7 @@ associatePortProfileId(const char *macvtap_ifname, * disassociatePortProfile * * @macvtap_ifname: The name of the macvtap device + * @linkdev: The link device in case of macvtap * @virtPort: point to object holding port profile parameters * * Returns 0 in case of success, != 0 otherwise with error @@ -773,25 +1510,36 @@ associatePortProfileId(const char *macvtap_ifname, */ static int disassociatePortProfileId(const char *macvtap_ifname, + const char *linkdev, const virVirtualPortProfileDefPtr virtPort) { + char *physfndev; + int32_t vf; int rc = 0; + VIR_DEBUG("Disassociating port profile id '%p' on link device '%s' ", virtPort, macvtap_ifname);
+ getPhysfn(linkdev,&vf,&physfndev); + switch (virtPort->virtPortType) { case VIR_VIRTUALPORT_NONE: case VIR_VIRTUALPORT_TYPE_LAST: break;
case VIR_VIRTUALPORT_8021QBG: - + rc = doPortProfileOp8021Qbg(macvtap_ifname, virtPort, vf, + DISASSOCIATE); break;
case VIR_VIRTUALPORT_8021QBH: - + rc = doPortProfileOp8021Qbh(physfndev, virtPort, vf, + NULL, NULL, + DISASSOCIATE); break; }
return rc; } + +#endif
-- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Mon, May 24, 2010 at 12:07:06AM -0700, Scott Feldman wrote:
+static int +nlCommWaitSuccess(struct nlmsghdr *nlmsg, int nl_groups, + char **respbuf, int *respbuflen, long to_usecs) +{ + int rc = 0; + struct sockaddr_nl nladdr = { + .nl_family = AF_NETLINK, + .nl_pid = getpid(), + .nl_groups = nl_groups, + }; + int rcvChunkSize = 1024; // expecting less than that + int rcvoffset = 0; + ssize_t nbytes; + int n; + struct timeval tv = { + .tv_sec = to_usecs / MICROSEC_PER_SEC, + .tv_usec = to_usecs % MICROSEC_PER_SEC, + }; + fd_set rfds; + bool gotvalid = false; + int fd = nlOpen(); + static uint32_t seq = 0x1234; + uint32_t myseq = seq++; + uint32_t mypid = getpid(); + + if (fd < 0) + return -1; + + nlmsg->nlmsg_pid = mypid; + nlmsg->nlmsg_seq = myseq; + nlmsg->nlmsg_flags |= NLM_F_ACK; + + nbytes = sendto(fd, (void *)nlmsg, nlmsg->nlmsg_len, 0, + (struct sockaddr *)&nladdr, sizeof(nladdr)); + if (nbytes < 0) { + virReportSystemError(errno, + "%s", _("cannot send to netlink socket")); + rc = -1; + goto err_exit; + } + fprintf(stderr,"sent %d bytes\n", (int)nbytes);
Can you change that to VIR_DEBUG("send %d bytes", (int)bytes), and likewise for all other places using fprintf() for debug output. NB VIR_DEBUG doesn't need a trailing \n, it adds one for you. If you want to view the debug messages from libvirtd then just run it using LIBVIRT_LOG_FILTERS=1:macvtap LIBVIRT_LOG_OUTPUTS=1:stderr /usr/sbin/libvirtd and it'll print all VIR_DEBUG() lines in macvtap.c Daniel -- |: Red Hat, Engineering, London -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://deltacloud.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|
participants (3)
-
Daniel P. Berrange -
Laine Stump -
Scott Feldman