[libvirt] [PATCH v2 0/3] vz: add statistics
Add vz statistics for network, cpu and memory. src/vz/vz_driver.c | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++- src/vz/vz_sdk.c | 69 ++++++++++++++++++++++++++- src/vz/vz_sdk.h | 4 ++ Changes from v1. subject prefix changed from 'parallels' to 'vz'
From: Nikolay Shirokovskiy <nshirokovskiy@parallels.com> Populate counters SDK currenly supports: rx_bytes rx_packets tx_bytes tx_packets Comments. 1. Use vzDomObjFromDomainRef/virDomainObjEndAPI pair to get domain object as we use prlsdkGetStatsParam that can release domain object lock and thus we need a reference in case domain is deleated meanwhile. 2. Introduce prlsdkGetAdapterIndex to convert from device name to SDK device index. We need this index as it is used in statistics data from SDK identify network device. Unfortunately we need to enumerate network devices to discover this mapping. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/vz/vz_driver.c | 44 +++++++++++++++++++++++++++++++++ src/vz/vz_sdk.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++- src/vz/vz_sdk.h | 4 +++ 3 files changed, 116 insertions(+), 1 deletions(-) diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c index cef3c77..deac572 100644 --- a/src/vz/vz_driver.c +++ b/src/vz/vz_driver.c @@ -1321,6 +1321,49 @@ vzDomainBlockStatsFlags(virDomainPtr domain, return ret; } +static int +vzDomainInterfaceStats(virDomainPtr domain, + const char *path, + virDomainInterfaceStatsPtr stats) +{ + virDomainObjPtr dom = NULL; + int ret = -1; + int net_index = -1; + char *name = NULL; + + if (!(dom = vzDomObjFromDomainRef(domain))) + return -1; + + net_index = prlsdkGetAdapterIndex(dom, path); + if (net_index < 0) + return -1; + +#define PARALLELS_GET_NET_COUNTER(VAL, NAME) \ + if (virAsprintf(&name, "net.nic%d.%s", net_index, NAME) < 0) \ + goto cleanup; \ + if (prlsdkGetStatsParam(dom, name, &stats->VAL) < 0) \ + goto cleanup; \ + VIR_FREE(name); + + PARALLELS_GET_NET_COUNTER(rx_bytes, "bytes_in") + PARALLELS_GET_NET_COUNTER(rx_packets, "pkts_in") + PARALLELS_GET_NET_COUNTER(tx_bytes, "bytes_out") + PARALLELS_GET_NET_COUNTER(tx_packets, "pkts_out") + stats->rx_errs = -1; + stats->rx_drop = -1; + stats->tx_errs = -1; + stats->tx_drop = -1; + +#undef PARALLELS_GET_NET_COUNTER + ret = 0; + + cleanup: + VIR_FREE(name); + if (dom) + virDomainObjEndAPI(&dom); + + return ret; +} static virHypervisorDriver vzDriver = { .name = "vz", @@ -1373,6 +1416,7 @@ static virHypervisorDriver vzDriver = { .domainGetMaxMemory = vzDomainGetMaxMemory, /* 1.2.15 */ .domainBlockStats = vzDomainBlockStats, /* 1.3.0 */ .domainBlockStatsFlags = vzDomainBlockStatsFlags, /* 1.3.0 */ + .domainInterfaceStats = vzDomainInterfaceStats, /* 1.3.0 */ }; static virConnectDriver vzConnectDriver = { diff --git a/src/vz/vz_sdk.c b/src/vz/vz_sdk.c index f9cde44..0956b58 100644 --- a/src/vz/vz_sdk.c +++ b/src/vz/vz_sdk.c @@ -3562,7 +3562,7 @@ prlsdkExtractStatsParam(PRL_HANDLE sdkstats, const char *name, long long *val) #define PARALLELS_STATISTICS_TIMEOUT (60 * 1000) -static int +int prlsdkGetStatsParam(virDomainObjPtr dom, const char *name, long long *val) { vzDomObjPtr privdom = dom->privateData; @@ -3658,3 +3658,70 @@ prlsdkGetBlockStats(virDomainObjPtr dom, virDomainDiskDefPtr disk, virDomainBloc VIR_FREE(name); return ret; } + +static PRL_HANDLE +prlsdkFindNetAdapter(virDomainObjPtr dom, const char *path) +{ + PRL_UINT32 count = 0; + vzDomObjPtr privdom = dom->privateData; + PRL_UINT32 buflen = 0; + PRL_RESULT pret; + size_t i; + char *name = NULL; + PRL_HANDLE net = PRL_INVALID_HANDLE; + + pret = PrlVmCfg_GetNetAdaptersCount(privdom->sdkdom, &count); + prlsdkCheckRetGoto(pret, error); + + for (i = 0; i < count; ++i) { + pret = PrlVmCfg_GetNetAdapter(privdom->sdkdom, i, &net); + prlsdkCheckRetGoto(pret, error); + + pret = PrlVmDevNet_GetHostInterfaceName(net, NULL, &buflen); + prlsdkCheckRetGoto(pret, error); + + if (VIR_ALLOC_N(name, buflen) < 0) + goto error; + + pret = PrlVmDevNet_GetHostInterfaceName(net, name, &buflen); + prlsdkCheckRetGoto(pret, error); + + if (STREQ(name, path)) + break; + + VIR_FREE(name); + PrlHandle_Free(net); + net = PRL_INVALID_HANDLE; + } + + if (net == PRL_INVALID_HANDLE) + virReportError(VIR_ERR_INVALID_ARG, + _("invalid path, '%s' is not a known interface"), path); + return net; + + error: + VIR_FREE(name); + PrlHandle_Free(net); + return PRL_INVALID_HANDLE; +} + +int +prlsdkGetAdapterIndex(virDomainObjPtr dom, const char *path) +{ + PRL_HANDLE net = PRL_INVALID_HANDLE; + PRL_UINT32 net_index = -1; + PRL_RESULT pret; + int ret = -1; + + net = prlsdkFindNetAdapter(dom, path); + if (net == PRL_INVALID_HANDLE) + return -1; + + pret = PrlVmDev_GetIndex(net, &net_index); + prlsdkCheckRetGoto(pret, cleanup); + + ret = net_index; + cleanup: + PrlHandle_Free(net); + return ret; +} diff --git a/src/vz/vz_sdk.h b/src/vz/vz_sdk.h index dd4fecf..8f72e24 100644 --- a/src/vz/vz_sdk.h +++ b/src/vz/vz_sdk.h @@ -66,3 +66,7 @@ int prlsdkDetachVolume(virDomainObjPtr dom, virDomainDiskDefPtr disk); int prlsdkGetBlockStats(virDomainObjPtr dom, virDomainDiskDefPtr disk, virDomainBlockStatsPtr stats); +int +prlsdkGetAdapterIndex(virDomainObjPtr dom, const char *path); +int +prlsdkGetStatsParam(virDomainObjPtr dom, const char *name, long long *val); -- 1.7.1
On 06/18/2015 12:28 PM, Nikolay Shirokovskiy wrote:
From: Nikolay Shirokovskiy <nshirokovskiy@parallels.com>
Populate counters SDK currenly supports: rx_bytes rx_packets tx_bytes tx_packets
Comments.
1. Use vzDomObjFromDomainRef/virDomainObjEndAPI pair to get domain object as we use prlsdkGetStatsParam that can release domain object lock and thus we need a reference in case domain is deleated meanwhile.
2. Introduce prlsdkGetAdapterIndex to convert from device name to SDK device index. We need this index as it is used in statistics data from SDK identify network device. Unfortunately we need to enumerate network devices to discover this mapping.
Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/vz/vz_driver.c | 44 +++++++++++++++++++++++++++++++++ src/vz/vz_sdk.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++- src/vz/vz_sdk.h | 4 +++ 3 files changed, 116 insertions(+), 1 deletions(-)
diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c index cef3c77..deac572 100644 --- a/src/vz/vz_driver.c +++ b/src/vz/vz_driver.c @@ -1321,6 +1321,49 @@ vzDomainBlockStatsFlags(virDomainPtr domain, return ret; }
+static int +vzDomainInterfaceStats(virDomainPtr domain, + const char *path, + virDomainInterfaceStatsPtr stats) +{ + virDomainObjPtr dom = NULL; + int ret = -1; + int net_index = -1; + char *name = NULL; + + if (!(dom = vzDomObjFromDomainRef(domain))) + return -1; + + net_index = prlsdkGetAdapterIndex(dom, path);
We have function prlsdkGetNetIndex, which looks up an adapter by given libvirt structure virDomainNetDef. Net and Adapter are sysnonyms, libvirt uses Net, and prlsdk uses 'Adapter' name, so I think we should rename these function, so the name will show, by which property adapter is looked up.
+ if (net_index < 0) In libvirt's code people usually combine the function call and a check in one line
if ((net_index = prlsdkGetAdapterIndex(dom, path)) < 0) ...
+ return -1; + +#define PARALLELS_GET_NET_COUNTER(VAL, NAME) \ + if (virAsprintf(&name, "net.nic%d.%s", net_index, NAME) < 0) \ + goto cleanup; \ + if (prlsdkGetStatsParam(dom, name, &stats->VAL) < 0) \ + goto cleanup; \ + VIR_FREE(name); + + PARALLELS_GET_NET_COUNTER(rx_bytes, "bytes_in") + PARALLELS_GET_NET_COUNTER(rx_packets, "pkts_in") + PARALLELS_GET_NET_COUNTER(tx_bytes, "bytes_out") + PARALLELS_GET_NET_COUNTER(tx_packets, "pkts_out") + stats->rx_errs = -1; + stats->rx_drop = -1; + stats->tx_errs = -1; + stats->tx_drop = -1; + +#undef PARALLELS_GET_NET_COUNTER + ret = 0; + + cleanup: + VIR_FREE(name); + if (dom) + virDomainObjEndAPI(&dom); + + return ret; +}
static virHypervisorDriver vzDriver = { .name = "vz", @@ -1373,6 +1416,7 @@ static virHypervisorDriver vzDriver = { .domainGetMaxMemory = vzDomainGetMaxMemory, /* 1.2.15 */ .domainBlockStats = vzDomainBlockStats, /* 1.3.0 */ .domainBlockStatsFlags = vzDomainBlockStatsFlags, /* 1.3.0 */ + .domainInterfaceStats = vzDomainInterfaceStats, /* 1.3.0 */ };
static virConnectDriver vzConnectDriver = { diff --git a/src/vz/vz_sdk.c b/src/vz/vz_sdk.c index f9cde44..0956b58 100644 --- a/src/vz/vz_sdk.c +++ b/src/vz/vz_sdk.c @@ -3562,7 +3562,7 @@ prlsdkExtractStatsParam(PRL_HANDLE sdkstats, const char *name, long long *val)
#define PARALLELS_STATISTICS_TIMEOUT (60 * 1000)
-static int +int prlsdkGetStatsParam(virDomainObjPtr dom, const char *name, long long *val) { vzDomObjPtr privdom = dom->privateData; @@ -3658,3 +3658,70 @@ prlsdkGetBlockStats(virDomainObjPtr dom, virDomainDiskDefPtr disk, virDomainBloc VIR_FREE(name); return ret; } + +static PRL_HANDLE +prlsdkFindNetAdapter(virDomainObjPtr dom, const char *path) +{ + PRL_UINT32 count = 0; + vzDomObjPtr privdom = dom->privateData; + PRL_UINT32 buflen = 0; + PRL_RESULT pret; + size_t i; + char *name = NULL; + PRL_HANDLE net = PRL_INVALID_HANDLE; + + pret = PrlVmCfg_GetNetAdaptersCount(privdom->sdkdom, &count); + prlsdkCheckRetGoto(pret, error); + + for (i = 0; i < count; ++i) { + pret = PrlVmCfg_GetNetAdapter(privdom->sdkdom, i, &net); + prlsdkCheckRetGoto(pret, error); + + pret = PrlVmDevNet_GetHostInterfaceName(net, NULL, &buflen); + prlsdkCheckRetGoto(pret, error); + + if (VIR_ALLOC_N(name, buflen) < 0) + goto error; + + pret = PrlVmDevNet_GetHostInterfaceName(net, name, &buflen); + prlsdkCheckRetGoto(pret, error); + + if (STREQ(name, path)) + break; + + VIR_FREE(name); + PrlHandle_Free(net); + net = PRL_INVALID_HANDLE; + } + + if (net == PRL_INVALID_HANDLE) + virReportError(VIR_ERR_INVALID_ARG, + _("invalid path, '%s' is not a known interface"), path); + return net; + + error: + VIR_FREE(name); + PrlHandle_Free(net); + return PRL_INVALID_HANDLE; +} +
I think you can just return 'i' as adapter index, you use prlsdkFindNetAdapter function only in prlsdkGetAdapterIndex, do you?
+int +prlsdkGetAdapterIndex(virDomainObjPtr dom, const char *path) +{ + PRL_HANDLE net = PRL_INVALID_HANDLE; + PRL_UINT32 net_index = -1; + PRL_RESULT pret; + int ret = -1; + + net = prlsdkFindNetAdapter(dom, path); + if (net == PRL_INVALID_HANDLE) + return -1; + + pret = PrlVmDev_GetIndex(net, &net_index); + prlsdkCheckRetGoto(pret, cleanup); + + ret = net_index; + cleanup: + PrlHandle_Free(net); + return ret; +} diff --git a/src/vz/vz_sdk.h b/src/vz/vz_sdk.h index dd4fecf..8f72e24 100644 --- a/src/vz/vz_sdk.h +++ b/src/vz/vz_sdk.h @@ -66,3 +66,7 @@ int prlsdkDetachVolume(virDomainObjPtr dom, virDomainDiskDefPtr disk); int prlsdkGetBlockStats(virDomainObjPtr dom, virDomainDiskDefPtr disk, virDomainBlockStatsPtr stats); +int +prlsdkGetAdapterIndex(virDomainObjPtr dom, const char *path); +int +prlsdkGetStatsParam(virDomainObjPtr dom, const char *name, long long *val);
-- Dmitry Guryanov
On 25.06.2015 17:51, Dmitry Guryanov wrote:
On 06/18/2015 12:28 PM, Nikolay Shirokovskiy wrote:
From: Nikolay Shirokovskiy <nshirokovskiy@parallels.com>
Populate counters SDK currenly supports: rx_bytes rx_packets tx_bytes tx_packets
Comments.
1. Use vzDomObjFromDomainRef/virDomainObjEndAPI pair to get domain object as we use prlsdkGetStatsParam that can release domain object lock and thus we need a reference in case domain is deleated meanwhile.
2. Introduce prlsdkGetAdapterIndex to convert from device name to SDK device index. We need this index as it is used in statistics data from SDK identify network device. Unfortunately we need to enumerate network devices to discover this mapping.
Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/vz/vz_driver.c | 44 +++++++++++++++++++++++++++++++++ src/vz/vz_sdk.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++- src/vz/vz_sdk.h | 4 +++ 3 files changed, 116 insertions(+), 1 deletions(-)
diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c index cef3c77..deac572 100644 --- a/src/vz/vz_driver.c +++ b/src/vz/vz_driver.c @@ -1321,6 +1321,49 @@ vzDomainBlockStatsFlags(virDomainPtr domain, return ret; } +static int +vzDomainInterfaceStats(virDomainPtr domain, + const char *path, + virDomainInterfaceStatsPtr stats) +{ + virDomainObjPtr dom = NULL; + int ret = -1; + int net_index = -1; + char *name = NULL; + + if (!(dom = vzDomObjFromDomainRef(domain))) + return -1; + + net_index = prlsdkGetAdapterIndex(dom, path);
We have function prlsdkGetNetIndex, which looks up an adapter by given libvirt structure virDomainNetDef. Net and Adapter are sysnonyms, libvirt uses Net, and prlsdk uses 'Adapter' name, so I think we should rename these function, so the name will show, by which property adapter is looked up.
+ if (net_index < 0) In libvirt's code people usually combine the function call and a check in one line
if ((net_index = prlsdkGetAdapterIndex(dom, path)) < 0) ...
+ return -1; + +#define PARALLELS_GET_NET_COUNTER(VAL, NAME) \ + if (virAsprintf(&name, "net.nic%d.%s", net_index, NAME) < 0) \ + goto cleanup; \ + if (prlsdkGetStatsParam(dom, name, &stats->VAL) < 0) \ + goto cleanup; \ + VIR_FREE(name); + + PARALLELS_GET_NET_COUNTER(rx_bytes, "bytes_in") + PARALLELS_GET_NET_COUNTER(rx_packets, "pkts_in") + PARALLELS_GET_NET_COUNTER(tx_bytes, "bytes_out") + PARALLELS_GET_NET_COUNTER(tx_packets, "pkts_out") + stats->rx_errs = -1; + stats->rx_drop = -1; + stats->tx_errs = -1; + stats->tx_drop = -1; + +#undef PARALLELS_GET_NET_COUNTER + ret = 0; + + cleanup: + VIR_FREE(name); + if (dom) + virDomainObjEndAPI(&dom); + + return ret; +} static virHypervisorDriver vzDriver = { .name = "vz", @@ -1373,6 +1416,7 @@ static virHypervisorDriver vzDriver = { .domainGetMaxMemory = vzDomainGetMaxMemory, /* 1.2.15 */ .domainBlockStats = vzDomainBlockStats, /* 1.3.0 */ .domainBlockStatsFlags = vzDomainBlockStatsFlags, /* 1.3.0 */ + .domainInterfaceStats = vzDomainInterfaceStats, /* 1.3.0 */ }; static virConnectDriver vzConnectDriver = { diff --git a/src/vz/vz_sdk.c b/src/vz/vz_sdk.c index f9cde44..0956b58 100644 --- a/src/vz/vz_sdk.c +++ b/src/vz/vz_sdk.c @@ -3562,7 +3562,7 @@ prlsdkExtractStatsParam(PRL_HANDLE sdkstats, const char *name, long long *val) #define PARALLELS_STATISTICS_TIMEOUT (60 * 1000) -static int +int prlsdkGetStatsParam(virDomainObjPtr dom, const char *name, long long *val) { vzDomObjPtr privdom = dom->privateData; @@ -3658,3 +3658,70 @@ prlsdkGetBlockStats(virDomainObjPtr dom, virDomainDiskDefPtr disk, virDomainBloc VIR_FREE(name); return ret; } + +static PRL_HANDLE +prlsdkFindNetAdapter(virDomainObjPtr dom, const char *path) +{ + PRL_UINT32 count = 0; + vzDomObjPtr privdom = dom->privateData; + PRL_UINT32 buflen = 0; + PRL_RESULT pret; + size_t i; + char *name = NULL; + PRL_HANDLE net = PRL_INVALID_HANDLE; + + pret = PrlVmCfg_GetNetAdaptersCount(privdom->sdkdom, &count); + prlsdkCheckRetGoto(pret, error); + + for (i = 0; i < count; ++i) { + pret = PrlVmCfg_GetNetAdapter(privdom->sdkdom, i, &net); + prlsdkCheckRetGoto(pret, error); + + pret = PrlVmDevNet_GetHostInterfaceName(net, NULL, &buflen); + prlsdkCheckRetGoto(pret, error); + + if (VIR_ALLOC_N(name, buflen) < 0) + goto error; + + pret = PrlVmDevNet_GetHostInterfaceName(net, name, &buflen); + prlsdkCheckRetGoto(pret, error); + + if (STREQ(name, path)) + break; + + VIR_FREE(name); + PrlHandle_Free(net); + net = PRL_INVALID_HANDLE; + } + + if (net == PRL_INVALID_HANDLE) + virReportError(VIR_ERR_INVALID_ARG, + _("invalid path, '%s' is not a known interface"), path); + return net; + + error: + VIR_FREE(name); + PrlHandle_Free(net); + return PRL_INVALID_HANDLE; +} +
I think you can just return 'i' as adapter index, you use prlsdkFindNetAdapter function only in prlsdkGetAdapterIndex, do you? No I can't. Enumeration index and device index are not the same. Say you can add 3 net devices, net0, net1, net2, then delete net1 and you get net0, net2 when you enumerate them, for net2 enumeration index will be 1, not 2.
I use prlsdkFindNetAdapter only in prlsdkFindNetAdapter, but I want them to be splitted as they have different functions, namely 'find by property' and 'get property'.
+int +prlsdkGetAdapterIndex(virDomainObjPtr dom, const char *path) +{ + PRL_HANDLE net = PRL_INVALID_HANDLE; + PRL_UINT32 net_index = -1; + PRL_RESULT pret; + int ret = -1; + + net = prlsdkFindNetAdapter(dom, path); + if (net == PRL_INVALID_HANDLE) + return -1; + + pret = PrlVmDev_GetIndex(net, &net_index); + prlsdkCheckRetGoto(pret, cleanup); + + ret = net_index; + cleanup: + PrlHandle_Free(net); + return ret; +} diff --git a/src/vz/vz_sdk.h b/src/vz/vz_sdk.h index dd4fecf..8f72e24 100644 --- a/src/vz/vz_sdk.h +++ b/src/vz/vz_sdk.h @@ -66,3 +66,7 @@ int prlsdkDetachVolume(virDomainObjPtr dom, virDomainDiskDefPtr disk); int prlsdkGetBlockStats(virDomainObjPtr dom, virDomainDiskDefPtr disk, virDomainBlockStatsPtr stats); +int +prlsdkGetAdapterIndex(virDomainObjPtr dom, const char *path); +int +prlsdkGetStatsParam(virDomainObjPtr dom, const char *name, long long *val);
On 06/18/2015 12:28 PM, Nikolay Shirokovskiy wrote:
From: Nikolay Shirokovskiy <nshirokovskiy@parallels.com>
Populate counters SDK currenly supports: rx_bytes rx_packets tx_bytes tx_packets
Comments.
1. Use vzDomObjFromDomainRef/virDomainObjEndAPI pair to get domain object as we use prlsdkGetStatsParam that can release domain object lock and thus we need a reference in case domain is deleated meanwhile.
2. Introduce prlsdkGetAdapterIndex to convert from device name to SDK device index. We need this index as it is used in statistics data from SDK identify network device. Unfortunately we need to enumerate network devices to discover this mapping.
Also this patch doesn't apply at this moment.
Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/vz/vz_driver.c | 44 +++++++++++++++++++++++++++++++++ src/vz/vz_sdk.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++- src/vz/vz_sdk.h | 4 +++ 3 files changed, 116 insertions(+), 1 deletions(-)
diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c index cef3c77..deac572 100644 --- a/src/vz/vz_driver.c +++ b/src/vz/vz_driver.c @@ -1321,6 +1321,49 @@ vzDomainBlockStatsFlags(virDomainPtr domain, return ret; }
+static int +vzDomainInterfaceStats(virDomainPtr domain, + const char *path, + virDomainInterfaceStatsPtr stats) +{ + virDomainObjPtr dom = NULL; + int ret = -1; + int net_index = -1; + char *name = NULL; + + if (!(dom = vzDomObjFromDomainRef(domain))) + return -1; + + net_index = prlsdkGetAdapterIndex(dom, path); + if (net_index < 0) + return -1; + +#define PARALLELS_GET_NET_COUNTER(VAL, NAME) \ + if (virAsprintf(&name, "net.nic%d.%s", net_index, NAME) < 0) \ + goto cleanup; \ + if (prlsdkGetStatsParam(dom, name, &stats->VAL) < 0) \ + goto cleanup; \ + VIR_FREE(name); + + PARALLELS_GET_NET_COUNTER(rx_bytes, "bytes_in") + PARALLELS_GET_NET_COUNTER(rx_packets, "pkts_in") + PARALLELS_GET_NET_COUNTER(tx_bytes, "bytes_out") + PARALLELS_GET_NET_COUNTER(tx_packets, "pkts_out") + stats->rx_errs = -1; + stats->rx_drop = -1; + stats->tx_errs = -1; + stats->tx_drop = -1; + +#undef PARALLELS_GET_NET_COUNTER + ret = 0; + + cleanup: + VIR_FREE(name); + if (dom) + virDomainObjEndAPI(&dom); + + return ret; +}
static virHypervisorDriver vzDriver = { .name = "vz", @@ -1373,6 +1416,7 @@ static virHypervisorDriver vzDriver = { .domainGetMaxMemory = vzDomainGetMaxMemory, /* 1.2.15 */ .domainBlockStats = vzDomainBlockStats, /* 1.3.0 */ .domainBlockStatsFlags = vzDomainBlockStatsFlags, /* 1.3.0 */ + .domainInterfaceStats = vzDomainInterfaceStats, /* 1.3.0 */ };
static virConnectDriver vzConnectDriver = { diff --git a/src/vz/vz_sdk.c b/src/vz/vz_sdk.c index f9cde44..0956b58 100644 --- a/src/vz/vz_sdk.c +++ b/src/vz/vz_sdk.c @@ -3562,7 +3562,7 @@ prlsdkExtractStatsParam(PRL_HANDLE sdkstats, const char *name, long long *val)
#define PARALLELS_STATISTICS_TIMEOUT (60 * 1000)
-static int +int prlsdkGetStatsParam(virDomainObjPtr dom, const char *name, long long *val) { vzDomObjPtr privdom = dom->privateData; @@ -3658,3 +3658,70 @@ prlsdkGetBlockStats(virDomainObjPtr dom, virDomainDiskDefPtr disk, virDomainBloc VIR_FREE(name); return ret; } + +static PRL_HANDLE +prlsdkFindNetAdapter(virDomainObjPtr dom, const char *path) +{ + PRL_UINT32 count = 0; + vzDomObjPtr privdom = dom->privateData; + PRL_UINT32 buflen = 0; + PRL_RESULT pret; + size_t i; + char *name = NULL; + PRL_HANDLE net = PRL_INVALID_HANDLE; + + pret = PrlVmCfg_GetNetAdaptersCount(privdom->sdkdom, &count); + prlsdkCheckRetGoto(pret, error); + + for (i = 0; i < count; ++i) { + pret = PrlVmCfg_GetNetAdapter(privdom->sdkdom, i, &net); + prlsdkCheckRetGoto(pret, error); + + pret = PrlVmDevNet_GetHostInterfaceName(net, NULL, &buflen); + prlsdkCheckRetGoto(pret, error); + + if (VIR_ALLOC_N(name, buflen) < 0) + goto error; + + pret = PrlVmDevNet_GetHostInterfaceName(net, name, &buflen); + prlsdkCheckRetGoto(pret, error); + + if (STREQ(name, path)) + break; + + VIR_FREE(name); + PrlHandle_Free(net); + net = PRL_INVALID_HANDLE; + } + + if (net == PRL_INVALID_HANDLE) + virReportError(VIR_ERR_INVALID_ARG, + _("invalid path, '%s' is not a known interface"), path); + return net; + + error: + VIR_FREE(name); + PrlHandle_Free(net); + return PRL_INVALID_HANDLE; +} + +int +prlsdkGetAdapterIndex(virDomainObjPtr dom, const char *path) +{ + PRL_HANDLE net = PRL_INVALID_HANDLE; + PRL_UINT32 net_index = -1; + PRL_RESULT pret; + int ret = -1; + + net = prlsdkFindNetAdapter(dom, path); + if (net == PRL_INVALID_HANDLE) + return -1; + + pret = PrlVmDev_GetIndex(net, &net_index); + prlsdkCheckRetGoto(pret, cleanup); + + ret = net_index; + cleanup: + PrlHandle_Free(net); + return ret; +} diff --git a/src/vz/vz_sdk.h b/src/vz/vz_sdk.h index dd4fecf..8f72e24 100644 --- a/src/vz/vz_sdk.h +++ b/src/vz/vz_sdk.h @@ -66,3 +66,7 @@ int prlsdkDetachVolume(virDomainObjPtr dom, virDomainDiskDefPtr disk); int prlsdkGetBlockStats(virDomainObjPtr dom, virDomainDiskDefPtr disk, virDomainBlockStatsPtr stats); +int +prlsdkGetAdapterIndex(virDomainObjPtr dom, const char *path); +int +prlsdkGetStatsParam(virDomainObjPtr dom, const char *name, long long *val);
-- Dmitry Guryanov
From: Nikolay Shirokovskiy <nshirokovskiy@parallels.com> Comments. Replace vzDomObjFromDomain/virObjectUnlock pair to vzDomObjFromDomainRef/virDomainObjEndAPI as we use prlsdkGetStatsParam. See previous statistics comments. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/vz/vz_driver.c | 16 ++++++++++++++-- 1 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c index deac572..4197569 100644 --- a/src/vz/vz_driver.c +++ b/src/vz/vz_driver.c @@ -825,8 +825,9 @@ vzDomainGetVcpus(virDomainPtr domain, size_t i; int v, maxcpu, hostcpus; int ret = -1; + char *name = NULL; - if (!(privdom = vzDomObjFromDomain(domain))) + if (!(privdom = vzDomObjFromDomainRef(domain))) goto cleanup; if (!virDomainObjIsActive(privdom)) { @@ -847,8 +848,18 @@ vzDomainGetVcpus(virDomainPtr domain, if (info != NULL) { memset(info, 0, sizeof(*info) * maxinfo); for (i = 0; i < maxinfo; i++) { + long long vcpuTime = 0; + info[i].number = i; info[i].state = VIR_VCPU_RUNNING; + + if (virAsprintf(&name, "guest.vcpu%u.time", (unsigned int)i) < 0) + goto cleanup; + if (prlsdkGetStatsParam(privdom, name, &vcpuTime) < 0) + goto cleanup; + if (vcpuTime != -1) + info[i].cpuTime = vcpuTime; + VIR_FREE(name); } } if (cpumaps != NULL) { @@ -871,7 +882,8 @@ vzDomainGetVcpus(virDomainPtr domain, cleanup: if (privdom) - virObjectUnlock(privdom); + virDomainObjEndAPI(&privdom); + VIR_FREE(name); return ret; } -- 1.7.1
On 06/18/2015 12:28 PM, Nikolay Shirokovskiy wrote:
From: Nikolay Shirokovskiy <nshirokovskiy@parallels.com>
Comments.
Replace vzDomObjFromDomain/virObjectUnlock pair to vzDomObjFromDomainRef/virDomainObjEndAPI as we use prlsdkGetStatsParam. See previous statistics comments. Looks good to me, ACK.
Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/vz/vz_driver.c | 16 ++++++++++++++-- 1 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c index deac572..4197569 100644 --- a/src/vz/vz_driver.c +++ b/src/vz/vz_driver.c @@ -825,8 +825,9 @@ vzDomainGetVcpus(virDomainPtr domain, size_t i; int v, maxcpu, hostcpus; int ret = -1; + char *name = NULL;
- if (!(privdom = vzDomObjFromDomain(domain))) + if (!(privdom = vzDomObjFromDomainRef(domain))) goto cleanup;
if (!virDomainObjIsActive(privdom)) { @@ -847,8 +848,18 @@ vzDomainGetVcpus(virDomainPtr domain, if (info != NULL) { memset(info, 0, sizeof(*info) * maxinfo); for (i = 0; i < maxinfo; i++) { + long long vcpuTime = 0; + info[i].number = i; info[i].state = VIR_VCPU_RUNNING; + + if (virAsprintf(&name, "guest.vcpu%u.time", (unsigned int)i) < 0) + goto cleanup; + if (prlsdkGetStatsParam(privdom, name, &vcpuTime) < 0) + goto cleanup; + if (vcpuTime != -1) + info[i].cpuTime = vcpuTime; + VIR_FREE(name); } } if (cpumaps != NULL) { @@ -871,7 +882,8 @@ vzDomainGetVcpus(virDomainPtr domain,
cleanup: if (privdom) - virObjectUnlock(privdom); + virDomainObjEndAPI(&privdom); + VIR_FREE(name); return ret; }
-- Dmitry Guryanov
From: Nikolay Shirokovskiy <nshirokovskiy@parallels.com> Implemented counters: VIR_DOMAIN_MEMORY_STAT_SWAP_IN VIR_DOMAIN_MEMORY_STAT_SWAP_OUT VIR_DOMAIN_MEMORY_STAT_MINOR_FAULT VIR_DOMAIN_MEMORY_STAT_MAJOR_FAULT VIR_DOMAIN_MEMORY_STAT_AVAILABLE VIR_DOMAIN_MEMORY_STAT_ACTUAL_BALLOON VIR_DOMAIN_MEMORY_STAT_UNUSED Comments. 1. Use vzDomObjFromDomainRef/virDomainObjEndAPI pair to get domain object as we use prlsdkGetStatsParam. See previous statistics comments. 2. Balloon statistics is not applicable to containers. Fault statistics for containers not provided in PCS6 yet. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/vz/vz_driver.c | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 73 insertions(+), 0 deletions(-) diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c index 4197569..8dae7c4 100644 --- a/src/vz/vz_driver.c +++ b/src/vz/vz_driver.c @@ -1377,6 +1377,78 @@ vzDomainInterfaceStats(virDomainPtr domain, return ret; } +static int +vzDomainMemoryStats(virDomainPtr domain, + virDomainMemoryStatPtr stats, + unsigned int nr_stats, + unsigned int flags) +{ + virDomainObjPtr dom = NULL; + int ret = -1; + long long v = 0, t = 0, u = 0; + size_t i = 0; + + virCheckFlags(0, -1); + if (!(dom = vzDomObjFromDomainRef(domain))) + return -1; + +#define PARALLELS_GET_COUNTER(NAME, VALUE) \ + if (prlsdkGetStatsParam(dom, NAME, &VALUE) < 0) \ + goto cleanup; \ + +#define PARALLELS_MEMORY_STAT_SET(TAG, VALUE) \ + if (i < nr_stats) { \ + stats[i].tag = (TAG); \ + stats[i].val = (VALUE); \ + i++; \ + } + +#define PARALLELS_COUNTER_PROTECT(VALUE) VALUE == -1 : ? + + i = 0; + + // count to kb + PARALLELS_GET_COUNTER("guest.ram.swap_in", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_SWAP_IN, v << 12) + + PARALLELS_GET_COUNTER("guest.ram.swap_out", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_SWAP_OUT, v << 12) + + PARALLELS_GET_COUNTER("guest.ram.minor_fault", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_MINOR_FAULT, v) + + PARALLELS_GET_COUNTER("guest.ram.major_fault", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_MAJOR_FAULT, v) + + PARALLELS_GET_COUNTER("guest.ram.total", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_AVAILABLE, v << 10) + + PARALLELS_GET_COUNTER("guest.ram.balloon_actual", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_ACTUAL_BALLOON, v << 10) + + PARALLELS_GET_COUNTER("guest.ram.usage", u) + PARALLELS_GET_COUNTER("guest.ram.total", t) + if (u != -1 && t != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_UNUSED, (t - u) << 10) + + +#undef PARALLELS_GET_COUNTER +#undef PARALLELS_MEMORY_STAT_SET + + ret = i; + cleanup: + if (dom) + virDomainObjEndAPI(&dom); + + return ret; +} + static virHypervisorDriver vzDriver = { .name = "vz", .connectOpen = vzConnectOpen, /* 0.10.0 */ @@ -1429,6 +1501,7 @@ static virHypervisorDriver vzDriver = { .domainBlockStats = vzDomainBlockStats, /* 1.3.0 */ .domainBlockStatsFlags = vzDomainBlockStatsFlags, /* 1.3.0 */ .domainInterfaceStats = vzDomainInterfaceStats, /* 1.3.0 */ + .domainMemoryStats = vzDomainMemoryStats, /* 1.3.0 */ }; static virConnectDriver vzConnectDriver = { -- 1.7.1
On 06/18/2015 12:28 PM, Nikolay Shirokovskiy wrote:
From: Nikolay Shirokovskiy <nshirokovskiy@parallels.com>
Implemented counters: VIR_DOMAIN_MEMORY_STAT_SWAP_IN VIR_DOMAIN_MEMORY_STAT_SWAP_OUT VIR_DOMAIN_MEMORY_STAT_MINOR_FAULT VIR_DOMAIN_MEMORY_STAT_MAJOR_FAULT VIR_DOMAIN_MEMORY_STAT_AVAILABLE VIR_DOMAIN_MEMORY_STAT_ACTUAL_BALLOON VIR_DOMAIN_MEMORY_STAT_UNUSED
Comments.
1. Use vzDomObjFromDomainRef/virDomainObjEndAPI pair to get domain object as we use prlsdkGetStatsParam. See previous statistics comments.
2. Balloon statistics is not applicable to containers. Fault statistics for containers not provided in PCS6 yet.
At some reason it returns -1 for all counters on my server, need to check, why is it...
Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/vz/vz_driver.c | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 73 insertions(+), 0 deletions(-)
diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c index 4197569..8dae7c4 100644 --- a/src/vz/vz_driver.c +++ b/src/vz/vz_driver.c @@ -1377,6 +1377,78 @@ vzDomainInterfaceStats(virDomainPtr domain, return ret; }
+static int +vzDomainMemoryStats(virDomainPtr domain, + virDomainMemoryStatPtr stats, + unsigned int nr_stats, + unsigned int flags) +{ + virDomainObjPtr dom = NULL; + int ret = -1; + long long v = 0, t = 0, u = 0; + size_t i = 0; + + virCheckFlags(0, -1); + if (!(dom = vzDomObjFromDomainRef(domain))) + return -1; + +#define PARALLELS_GET_COUNTER(NAME, VALUE) \ + if (prlsdkGetStatsParam(dom, NAME, &VALUE) < 0) \ + goto cleanup; \ + +#define PARALLELS_MEMORY_STAT_SET(TAG, VALUE) \ + if (i < nr_stats) { \ + stats[i].tag = (TAG); \ + stats[i].val = (VALUE); \ + i++; \ + } + +#define PARALLELS_COUNTER_PROTECT(VALUE) VALUE == -1 : ? + + i = 0; + + // count to kb + PARALLELS_GET_COUNTER("guest.ram.swap_in", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_SWAP_IN, v << 12) + + PARALLELS_GET_COUNTER("guest.ram.swap_out", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_SWAP_OUT, v << 12) + + PARALLELS_GET_COUNTER("guest.ram.minor_fault", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_MINOR_FAULT, v) + + PARALLELS_GET_COUNTER("guest.ram.major_fault", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_MAJOR_FAULT, v)
To be honest, I don't think macros here improve code readability, look, how it would be without them: if (prlsdkGetStatsParam(dom, "guest.ram.major_fault", &v) < 0) goto cleanup; if (v != -1 && i < nr_stats) { stats[i].tag = VIR_DOMAIN_MEMORY_STAT_MAJOR_FAULT; stats[i].val = v; i++; } Only "goto cleanup" and "i++" are repeating information.
+ + PARALLELS_GET_COUNTER("guest.ram.total", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_AVAILABLE, v << 10) + + PARALLELS_GET_COUNTER("guest.ram.balloon_actual", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_ACTUAL_BALLOON, v << 10) + + PARALLELS_GET_COUNTER("guest.ram.usage", u) + PARALLELS_GET_COUNTER("guest.ram.total", t) + if (u != -1 && t != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_UNUSED, (t - u) << 10) + + +#undef PARALLELS_GET_COUNTER +#undef PARALLELS_MEMORY_STAT_SET + + ret = i; + cleanup: + if (dom) + virDomainObjEndAPI(&dom); + + return ret; +} + static virHypervisorDriver vzDriver = { .name = "vz", .connectOpen = vzConnectOpen, /* 0.10.0 */ @@ -1429,6 +1501,7 @@ static virHypervisorDriver vzDriver = { .domainBlockStats = vzDomainBlockStats, /* 1.3.0 */ .domainBlockStatsFlags = vzDomainBlockStatsFlags, /* 1.3.0 */ .domainInterfaceStats = vzDomainInterfaceStats, /* 1.3.0 */ + .domainMemoryStats = vzDomainMemoryStats, /* 1.3.0 */ };
static virConnectDriver vzConnectDriver = {
-- Dmitry Guryanov
On 25.06.2015 20:36, Dmitry Guryanov wrote:
On 06/18/2015 12:28 PM, Nikolay Shirokovskiy wrote:
From: Nikolay Shirokovskiy <nshirokovskiy@parallels.com>
Implemented counters: VIR_DOMAIN_MEMORY_STAT_SWAP_IN VIR_DOMAIN_MEMORY_STAT_SWAP_OUT VIR_DOMAIN_MEMORY_STAT_MINOR_FAULT VIR_DOMAIN_MEMORY_STAT_MAJOR_FAULT VIR_DOMAIN_MEMORY_STAT_AVAILABLE VIR_DOMAIN_MEMORY_STAT_ACTUAL_BALLOON VIR_DOMAIN_MEMORY_STAT_UNUSED
Comments.
1. Use vzDomObjFromDomainRef/virDomainObjEndAPI pair to get domain object as we use prlsdkGetStatsParam. See previous statistics comments.
2. Balloon statistics is not applicable to containers. Fault statistics for containers not provided in PCS6 yet.
At some reason it returns -1 for all counters on my server, need to check, why is it... You need an recent build of PCS6 update10 and update guest tools for VMs too.
Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/vz/vz_driver.c | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 73 insertions(+), 0 deletions(-)
diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c index 4197569..8dae7c4 100644 --- a/src/vz/vz_driver.c +++ b/src/vz/vz_driver.c @@ -1377,6 +1377,78 @@ vzDomainInterfaceStats(virDomainPtr domain, return ret; } +static int +vzDomainMemoryStats(virDomainPtr domain, + virDomainMemoryStatPtr stats, + unsigned int nr_stats, + unsigned int flags) +{ + virDomainObjPtr dom = NULL; + int ret = -1; + long long v = 0, t = 0, u = 0; + size_t i = 0; + + virCheckFlags(0, -1); + if (!(dom = vzDomObjFromDomainRef(domain))) + return -1; + +#define PARALLELS_GET_COUNTER(NAME, VALUE) \ + if (prlsdkGetStatsParam(dom, NAME, &VALUE) < 0) \ + goto cleanup; \ + +#define PARALLELS_MEMORY_STAT_SET(TAG, VALUE) \ + if (i < nr_stats) { \ + stats[i].tag = (TAG); \ + stats[i].val = (VALUE); \ + i++; \ + } + +#define PARALLELS_COUNTER_PROTECT(VALUE) VALUE == -1 : ? + + i = 0; + + // count to kb + PARALLELS_GET_COUNTER("guest.ram.swap_in", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_SWAP_IN, v << 12) + + PARALLELS_GET_COUNTER("guest.ram.swap_out", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_SWAP_OUT, v << 12) + + PARALLELS_GET_COUNTER("guest.ram.minor_fault", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_MINOR_FAULT, v) + + PARALLELS_GET_COUNTER("guest.ram.major_fault", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_MAJOR_FAULT, v)
To be honest, I don't think macros here improve code readability, look, how it would be without them:
if (prlsdkGetStatsParam(dom, "guest.ram.major_fault", &v) < 0) goto cleanup;
if (v != -1 && i < nr_stats) { stats[i].tag = VIR_DOMAIN_MEMORY_STAT_MAJOR_FAULT; stats[i].val = v; i++; }
Only "goto cleanup" and "i++" are repeating information.
But with macros it is easy to see actual structure: 1. you get some counter 2. check if it is present (!= -1) 3. save to libvirt structure optionally converting to different units Without conversion this could be just 1 line. May be introduce conversion macros. Then we could get just: PARALLELS_GET_COUNTER("guest.ram.swap_out", VIR_DOMAIN_MEMORY_STAT_SWAP_OUT, BYTES_TO_PAGES) PARALLELS_GET_COUNTER("guest.ram.minor_fault", VIR_DOMAIN_MEMORY_STAT_MINOR_FAULT, NO_CONVERSION) Without macros it is entangled with counters vector capacity checks( i < nr_stats), incrementing counters. It looks like low-level but there is no wish to factor it out to type.
+ + PARALLELS_GET_COUNTER("guest.ram.total", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_AVAILABLE, v << 10) + + PARALLELS_GET_COUNTER("guest.ram.balloon_actual", v) + if (v != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_ACTUAL_BALLOON, v << 10) + + PARALLELS_GET_COUNTER("guest.ram.usage", u) + PARALLELS_GET_COUNTER("guest.ram.total", t) + if (u != -1 && t != -1) + PARALLELS_MEMORY_STAT_SET(VIR_DOMAIN_MEMORY_STAT_UNUSED, (t - u) << 10) + + +#undef PARALLELS_GET_COUNTER +#undef PARALLELS_MEMORY_STAT_SET + + ret = i; + cleanup: + if (dom) + virDomainObjEndAPI(&dom); + + return ret; +} + static virHypervisorDriver vzDriver = { .name = "vz", .connectOpen = vzConnectOpen, /* 0.10.0 */ @@ -1429,6 +1501,7 @@ static virHypervisorDriver vzDriver = { .domainBlockStats = vzDomainBlockStats, /* 1.3.0 */ .domainBlockStatsFlags = vzDomainBlockStatsFlags, /* 1.3.0 */ .domainInterfaceStats = vzDomainInterfaceStats, /* 1.3.0 */ + .domainMemoryStats = vzDomainMemoryStats, /* 1.3.0 */ }; static virConnectDriver vzConnectDriver = {
participants (3)
-
Dmitry Guryanov -
Nikolay Shirokovskiy -
Nikolay Shirokovskiy