8:47 a.m.
Mikhail Feoktistov (6):
vz: save vz version in connection structure
vz: add vzCapabilities to connection structure
vz: check supported disk format and bus
vz: report correct disk format in domainGetXMLDesc
vz: check supported controllers
vz: set default SCSI model
src/vz/vz_driver.c | 61 +++++------------
src/vz/vz_sdk.c | 80 +++++++++++-----------
src/vz/vz_sdk.h | 2 +-
src/vz/vz_utils.c | 191 +++++++++++++++++++++++++++++++++++++++++++++++++++++
src/vz/vz_utils.h | 24 +++++++
5 files changed, 274 insertions(+), 84 deletions(-)
--
1.8.3.1
8:47 a.m.
New subject: [libvirt] [PATCH 1/6] vz: save vz version in connection structure
Move code from connectGetVersion callback to vzInitVersion function
---
src/vz/vz_driver.c | 50 +++++++-------------------------------------------
src/vz/vz_utils.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++
src/vz/vz_utils.h | 3 +++
3 files changed, 57 insertions(+), 43 deletions(-)
diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c
index 6de7cb9..5d48d7e 100644
--- a/src/vz/vz_driver.c
+++ b/src/vz/vz_driver.c
@@ -62,7 +62,6 @@
VIR_LOG_INIT("parallels.parallels_driver");
#define PRLCTL "prlctl"
-#define PRLSRVCTL "prlsrvctl"
static int vzConnectClose(virConnectPtr conn);
@@ -234,6 +233,9 @@ vzOpenDefault(virConnectPtr conn)
if (prlsdkConnect(privconn) < 0)
goto err_free;
+ if (vzInitVersion(privconn) < 0)
+ goto error;
+
if (!(privconn->caps = vzBuildCapabilities()))
goto error;
@@ -345,49 +347,11 @@ vzConnectClose(virConnectPtr conn)
}
static int
-vzConnectGetVersion(virConnectPtr conn ATTRIBUTE_UNUSED, unsigned long *hvVer)
+vzConnectGetVersion(virConnectPtr conn, unsigned long *hvVer)
{
- char *output, *sVer, *tmp;
- const char *searchStr = "prlsrvctl version ";
- int ret = -1;
-
- output = vzGetOutput(PRLSRVCTL, "--help", NULL);
-
- if (!output) {
- vzParseError();
- goto cleanup;
- }
-
- if (!(sVer = strstr(output, searchStr))) {
- vzParseError();
- goto cleanup;
- }
-
- sVer = sVer + strlen(searchStr);
-
- /* parallels server has versions number like 6.0.17977.782218,
- * so libvirt can handle only first two numbers. */
- if (!(tmp = strchr(sVer, '.'))) {
- vzParseError();
- goto cleanup;
- }
-
- if (!(tmp = strchr(tmp + 1, '.'))) {
- vzParseError();
- goto cleanup;
- }
-
- tmp[0] = '\0';
- if (virParseVersionString(sVer, hvVer, true) < 0) {
- vzParseError();
- goto cleanup;
- }
-
- ret = 0;
-
- cleanup:
- VIR_FREE(output);
- return ret;
+ vzConnPtr privconn = conn->privateData;
+ *hvVer = privconn->vzVersion;
+ return 0;
}
diff --git a/src/vz/vz_utils.c b/src/vz/vz_utils.c
index a1ddad0..cf37597 100644
--- a/src/vz/vz_utils.c
+++ b/src/vz/vz_utils.c
@@ -34,6 +34,7 @@
#include "datatypes.h"
#define VIR_FROM_THIS VIR_FROM_PARALLELS
+#define PRLSRVCTL "prlsrvctl"
/**
* vzDomObjFromDomain:
@@ -178,3 +179,49 @@ vzNewDomain(vzConnPtr privconn, char *name, const unsigned char
*uuid)
VIR_FREE(pdom);
return NULL;
}
+
+int
+vzInitVersion(vzConnPtr privconn)
+{
+ char *output, *sVer, *tmp;
+ const char *searchStr = "prlsrvctl version ";
+ int ret = -1;
+
+ output = vzGetOutput(PRLSRVCTL, "--help", NULL);
+
+ if (!output) {
+ vzParseError();
+ goto cleanup;
+ }
+
+ if (!(sVer = strstr(output, searchStr))) {
+ vzParseError();
+ goto cleanup;
+ }
+
+ sVer = sVer + strlen(searchStr);
+
+ /* parallels server has versions number like 6.0.17977.782218 or 7.0.0,
+ * In libvirt we handle only first two numbers. */
+ if (!(tmp = strchr(sVer, '.'))) {
+ vzParseError();
+ goto cleanup;
+ }
+
+ if (!(tmp = strchr(tmp + 1, '.'))) {
+ vzParseError();
+ goto cleanup;
+ }
+
+ tmp[0] = '\0';
+ if (virParseVersionString(sVer, &(privconn->vzVersion), true) < 0) {
+ vzParseError();
+ goto cleanup;
+ }
+
+ ret = 0;
+
+ cleanup:
+ VIR_FREE(output);
+ return ret;
+}
diff --git a/src/vz/vz_utils.h b/src/vz/vz_utils.h
index ed100ac..fbade2e 100644
--- a/src/vz/vz_utils.h
+++ b/src/vz/vz_utils.h
@@ -62,6 +62,7 @@ struct _vzConn {
const char *drivername;
/* Immutable pointer, self-locking APIs */
virConnectCloseCallbackDataPtr closeCallback;
+ unsigned long vzVersion;
};
typedef struct _vzConn vzConn;
@@ -97,6 +98,8 @@ virDomainObjPtr
vzNewDomain(vzConnPtr privconn,
char *name,
const unsigned char *uuid);
+int
+vzInitVersion(vzConnPtr privconn);
# define PARALLELS_BLOCK_STATS_FOREACH(OP) \
OP(rd_req, VIR_DOMAIN_BLOCK_STATS_READ_REQ, "read_requests") \
--
1.8.3.1
8:47 a.m.
New subject: [libvirt] [PATCH 2/6] vz: add vzCapabilities to connection structure
Virtuozzo6 supports only ploop disks for containers and
virtual machines on IDE SCSI and SATA buses.
Virtuozzo7 supports only qcow2 for virtual machines on IDE and SCSI buses.
In this patch we add vzCapabilities structure to connection structure.
It contains disk formats and list of buses that we support for virtual machines and
containers.
In openConnection() callback we get virtuozzo version and select proper capabilities.
---
src/vz/vz_utils.c | 24 ++++++++++++++++++++++++
src/vz/vz_utils.h | 10 ++++++++++
2 files changed, 34 insertions(+)
diff --git a/src/vz/vz_utils.c b/src/vz/vz_utils.c
index cf37597..d8a95ac 100644
--- a/src/vz/vz_utils.c
+++ b/src/vz/vz_utils.c
@@ -36,6 +36,15 @@
#define VIR_FROM_THIS VIR_FROM_PARALLELS
#define PRLSRVCTL "prlsrvctl"
+static virDomainDiskBus vz6DiskBuses[] = {VIR_DOMAIN_DISK_BUS_IDE,
+ VIR_DOMAIN_DISK_BUS_SCSI,
+ VIR_DOMAIN_DISK_BUS_SATA,
+ VIR_DOMAIN_DISK_BUS_LAST};
+
+static virDomainDiskBus vz7DiskBuses[] = {VIR_DOMAIN_DISK_BUS_IDE,
+ VIR_DOMAIN_DISK_BUS_SCSI,
+ VIR_DOMAIN_DISK_BUS_LAST};
+
/**
* vzDomObjFromDomain:
* @domain: Domain pointer that has to be looked up
@@ -180,6 +189,20 @@ vzNewDomain(vzConnPtr privconn, char *name, const unsigned char
*uuid)
return NULL;
}
+static void
+vzInitCaps(unsigned long vzVersion, vzCapabilities *vzCaps)
+{
+ if (vzVersion < VIRTUOZZO_VER_7) {
+ vzCaps->ctDiskFormat = VIR_STORAGE_FILE_PLOOP;
+ vzCaps->vmDiskFormat = VIR_STORAGE_FILE_PLOOP;
+ vzCaps->diskBuses = vz6DiskBuses;
+ } else {
+ vzCaps->ctDiskFormat = VIR_STORAGE_FILE_PLOOP;
+ vzCaps->vmDiskFormat = VIR_STORAGE_FILE_QCOW2;
+ vzCaps->diskBuses = vz7DiskBuses;
+ }
+}
+
int
vzInitVersion(vzConnPtr privconn)
{
@@ -219,6 +242,7 @@ vzInitVersion(vzConnPtr privconn)
goto cleanup;
}
+ vzInitCaps(privconn->vzVersion, &privconn->vzCaps);
ret = 0;
cleanup:
diff --git a/src/vz/vz_utils.h b/src/vz/vz_utils.h
index fbade2e..e0b0105 100644
--- a/src/vz/vz_utils.h
+++ b/src/vz/vz_utils.h
@@ -48,6 +48,15 @@
# define PARALLELS_DOMAIN_ROUTED_NETWORK_NAME "Routed"
# define PARALLELS_DOMAIN_BRIDGED_NETWORK_NAME "Bridged"
+# define VIRTUOZZO_VER_7 ((unsigned long) 7000000)
+
+struct _vzCapabilities {
+ virStorageFileFormat vmDiskFormat;
+ virStorageFileFormat ctDiskFormat;
+ virDomainDiskBus *diskBuses;
+};
+typedef struct _vzCapabilities vzCapabilities;
+typedef struct _vzCapabilities *vzCapabilitiesPtr;
struct _vzConn {
virMutex lock;
@@ -63,6 +72,7 @@ struct _vzConn {
/* Immutable pointer, self-locking APIs */
virConnectCloseCallbackDataPtr closeCallback;
unsigned long vzVersion;
+ vzCapabilities vzCaps;
};
typedef struct _vzConn vzConn;
--
1.8.3.1
8:47 a.m.
New subject: [libvirt] [PATCH 3/6] vz: check supported disk format and bus
In DomainPostParse phase we check disk format and bus.
If disk format is not specified in XML (VIR_STORAGE_FILE_NONE) than we accept this disk.
Otherwise we try to find disk format in vzCapabilities struct.
Also we try to find disk bus in vzCapabilities->diskBuses.
It contains buses that we support.
---
src/vz/vz_driver.c | 6 +++++-
src/vz/vz_sdk.c | 24 ++----------------------
src/vz/vz_utils.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++
src/vz/vz_utils.h | 3 +++
4 files changed, 62 insertions(+), 23 deletions(-)
diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c
index 5d48d7e..4f52bc7 100644
--- a/src/vz/vz_driver.c
+++ b/src/vz/vz_driver.c
@@ -175,8 +175,11 @@ static int
vzDomainDefPostParse(virDomainDefPtr def ATTRIBUTE_UNUSED,
virCapsPtr caps ATTRIBUTE_UNUSED,
unsigned int parseFlags ATTRIBUTE_UNUSED,
- void *opaque ATTRIBUTE_UNUSED)
+ void *opaque)
{
+ if (vzCheckUnsupportedDisks(def, opaque) < 0)
+ return -1;
+
return 0;
}
@@ -239,6 +242,7 @@ vzOpenDefault(virConnectPtr conn)
if (!(privconn->caps = vzBuildCapabilities()))
goto error;
+ vzDomainDefParserConfig.priv = &privconn->vzCaps;
if (!(privconn->xmlopt = virDomainXMLOptionNew(&vzDomainDefParserConfig,
NULL, NULL)))
goto error;
diff --git a/src/vz/vz_sdk.c b/src/vz/vz_sdk.c
index de73c31..2a6e908 100644
--- a/src/vz/vz_sdk.c
+++ b/src/vz/vz_sdk.c
@@ -3143,30 +3143,10 @@ static int prlsdkAddDisk(PRL_HANDLE sdkdom,
pret = PrlVmDev_SetConnected(sdkdisk, 1);
prlsdkCheckRetGoto(pret, cleanup);
- if (disk->src->type == VIR_STORAGE_TYPE_FILE) {
- if (disk->device == VIR_DOMAIN_DISK_DEVICE_DISK &&
- virDomainDiskGetFormat(disk) != VIR_STORAGE_FILE_PLOOP) {
-
- virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("Invalid format of "
- "disk %s, vz driver
supports only "
- "images in ploop
format."), disk->src->path);
- goto cleanup;
- }
-
+ if (disk->src->type == VIR_STORAGE_TYPE_FILE)
emutype = PDT_USE_IMAGE_FILE;
- } else {
- if (disk->device == VIR_DOMAIN_DISK_DEVICE_DISK &&
- (virDomainDiskGetFormat(disk) != VIR_STORAGE_FILE_RAW &&
- virDomainDiskGetFormat(disk) != VIR_STORAGE_FILE_NONE &&
- virDomainDiskGetFormat(disk) != VIR_STORAGE_FILE_AUTO)) {
-
- virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("Invalid format "
- "of disk %s, it should be
either not set, or set "
- "to raw or auto."),
disk->src->path);
- goto cleanup;
- }
+ else
emutype = PDT_USE_REAL_DEVICE;
- }
pret = PrlVmDev_SetEmulatedType(sdkdisk, emutype);
prlsdkCheckRetGoto(pret, cleanup);
diff --git a/src/vz/vz_utils.c b/src/vz/vz_utils.c
index d8a95ac..bf62538 100644
--- a/src/vz/vz_utils.c
+++ b/src/vz/vz_utils.c
@@ -249,3 +249,55 @@ vzInitVersion(vzConnPtr privconn)
VIR_FREE(output);
return ret;
}
+
+int vzCheckUnsupportedDisks(virDomainDefPtr def,
+ vzCapabilitiesPtr vzCaps)
+{
+ size_t i, j;
+ virDomainDiskDefPtr disk;
+ virStorageFileFormat diskFormat;
+ bool supported;
+
+ for (i = 0; i < def->ndisks; i++) {
+ disk = def->disks[i];
+ diskFormat = virDomainDiskGetFormat(disk);
+ // disk format
+ supported = true;
+ if (disk->src->type == VIR_STORAGE_TYPE_FILE) {
+ if (disk->device == VIR_DOMAIN_DISK_DEVICE_DISK &&
+ diskFormat != VIR_STORAGE_FILE_NONE) {
+
+ if (IS_CT(def))
+ supported = vzCaps->ctDiskFormat == diskFormat;
+ else
+ supported = vzCaps->vmDiskFormat == diskFormat;
+ }
+ } else {
+ if (disk->device == VIR_DOMAIN_DISK_DEVICE_DISK) {
+ supported = diskFormat == VIR_STORAGE_FILE_RAW ||
+ diskFormat == VIR_STORAGE_FILE_NONE ||
+ diskFormat == VIR_STORAGE_FILE_AUTO;
+ }
+ }
+
+ if (!supported) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Unsupported format of disk %s"),
+ disk->src->path);
+ return -1;
+ }
+ //disk bus
+ for (j = 0; vzCaps->diskBuses[j] != VIR_DOMAIN_DISK_BUS_LAST; j++) {
+ if (disk->bus == vzCaps->diskBuses[j])
+ break;
+ }
+
+ if (vzCaps->diskBuses[j] == VIR_DOMAIN_DISK_BUS_LAST) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Unsupported disk bus type %s"),
+ virDomainDiskBusTypeToString(disk->bus));
+ return -1;
+ }
+ }
+ return 0;
+}
diff --git a/src/vz/vz_utils.h b/src/vz/vz_utils.h
index e0b0105..851322f 100644
--- a/src/vz/vz_utils.h
+++ b/src/vz/vz_utils.h
@@ -110,6 +110,9 @@ vzNewDomain(vzConnPtr privconn,
const unsigned char *uuid);
int
vzInitVersion(vzConnPtr privconn);
+int
+vzCheckUnsupportedDisks(virDomainDefPtr def,
+ vzCapabilitiesPtr vzCaps);
# define PARALLELS_BLOCK_STATS_FOREACH(OP) \
OP(rd_req, VIR_DOMAIN_BLOCK_STATS_READ_REQ, "read_requests") \
--
1.8.3.1
8:53 a.m.
New subject: [libvirt] [PATCH 3/6] vz: check supported disk format and bus
15.03.2016 10:47, Mikhail Feoktistov пишет:
In DomainPostParse phase we check disk format and bus.
If disk format is not specified in XML (VIR_STORAGE_FILE_NONE) than we accept this disk.
Otherwise we try to find disk format in vzCapabilities struct.
Also we try to find disk bus in vzCapabilities->diskBuses.
It contains buses that we support.
---
src/vz/vz_driver.c | 6 +++++-
src/vz/vz_sdk.c | 24 ++----------------------
src/vz/vz_utils.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++
src/vz/vz_utils.h | 3 +++
4 files changed, 62 insertions(+), 23 deletions(-)
diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c
index 5d48d7e..4f52bc7 100644
--- a/src/vz/vz_driver.c
+++ b/src/vz/vz_driver.c
@@ -175,8 +175,11 @@ static int
vzDomainDefPostParse(virDomainDefPtr def ATTRIBUTE_UNUSED,
virCapsPtr caps ATTRIBUTE_UNUSED,
unsigned int parseFlags ATTRIBUTE_UNUSED,
- void *opaque ATTRIBUTE_UNUSED)
+ void *opaque)
{
+ if (vzCheckUnsupportedDisks(def, opaque) < 0)
+ return -1;
+
return 0;
}
@@ -239,6 +242,7 @@ vzOpenDefault(virConnectPtr conn)
if (!(privconn->caps = vzBuildCapabilities()))
goto error;
+ vzDomainDefParserConfig.priv = &privconn->vzCaps;
if (!(privconn->xmlopt = virDomainXMLOptionNew(&vzDomainDefParserConfig,
NULL, NULL)))
goto error;
diff --git a/src/vz/vz_sdk.c b/src/vz/vz_sdk.c
index de73c31..2a6e908 100644
--- a/src/vz/vz_sdk.c
+++ b/src/vz/vz_sdk.c
@@ -3143,30 +3143,10 @@ static int prlsdkAddDisk(PRL_HANDLE sdkdom,
pret = PrlVmDev_SetConnected(sdkdisk, 1);
prlsdkCheckRetGoto(pret, cleanup);
- if (disk->src->type == VIR_STORAGE_TYPE_FILE) {
- if (disk->device == VIR_DOMAIN_DISK_DEVICE_DISK &&
- virDomainDiskGetFormat(disk) != VIR_STORAGE_FILE_PLOOP) {
-
- virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("Invalid format of "
- "disk %s, vz driver
supports only "
- "images in ploop
format."), disk->src->path);
- goto cleanup;
- }
-
+ if (disk->src->type == VIR_STORAGE_TYPE_FILE)
emutype = PDT_USE_IMAGE_FILE;
- } else {
- if (disk->device == VIR_DOMAIN_DISK_DEVICE_DISK &&
- (virDomainDiskGetFormat(disk) != VIR_STORAGE_FILE_RAW &&
- virDomainDiskGetFormat(disk) != VIR_STORAGE_FILE_NONE &&
- virDomainDiskGetFormat(disk) != VIR_STORAGE_FILE_AUTO)) {
-
- virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("Invalid format "
- "of disk %s, it should be
either not set, or set "
- "to raw or auto."),
disk->src->path);
- goto cleanup;
- }
+ else
emutype = PDT_USE_REAL_DEVICE;
I know you keep the logic we currently have but this is not true because
"not file" doesn't mean "real device"
and else if (disk->src->type == VIR_STORAGE_TYPE_BLOCK) would make sence
here.
- }
pret = PrlVmDev_SetEmulatedType(sdkdisk, emutype);
prlsdkCheckRetGoto(pret, cleanup);
diff --git a/src/vz/vz_utils.c b/src/vz/vz_utils.c
index d8a95ac..bf62538 100644
--- a/src/vz/vz_utils.c
+++ b/src/vz/vz_utils.c
@@ -249,3 +249,55 @@ vzInitVersion(vzConnPtr privconn)
VIR_FREE(output);
return ret;
}
+
+int vzCheckUnsupportedDisks(virDomainDefPtr def,
+ vzCapabilitiesPtr vzCaps)
+{
+ size_t i, j;
+ virDomainDiskDefPtr disk;
+ virStorageFileFormat diskFormat;
+ bool supported;
+
+ for (i = 0; i < def->ndisks; i++) {
+ disk = def->disks[i];
+ diskFormat = virDomainDiskGetFormat(disk);
+ // disk format
unnecessary commentary
+ supported = true;
+ if (disk->src->type == VIR_STORAGE_TYPE_FILE) {
+ if (disk->device == VIR_DOMAIN_DISK_DEVICE_DISK &&
+ diskFormat != VIR_STORAGE_FILE_NONE) {
+
+ if (IS_CT(def))
+ supported = vzCaps->ctDiskFormat == diskFormat;
+ else
+ supported = vzCaps->vmDiskFormat == diskFormat;
+ }
+ } else {
again, else if (disk->src->type == VIR_STORAGE_TYPE_BLOCK) would make
sence here
+ if (disk->device == VIR_DOMAIN_DISK_DEVICE_DISK) {
+ supported = diskFormat == VIR_STORAGE_FILE_RAW ||
+ diskFormat == VIR_STORAGE_FILE_NONE ||
+ diskFormat == VIR_STORAGE_FILE_AUTO;
+ }
+ }
else block with supported = false is required because there are
VIR_STORAGE_TYPE_NETWORK
VIR_STORAGE_TYPE_DIR and VIR_STORAGE_TYPE_VOLUME
+
+ if (!supported) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Unsupported format of disk %s"),
swearing about unsupported format isn't completely correct here if we do
proposed above correction
+ disk->src->path);
+ return -1;
+ }
+ //disk bus
+ for (j = 0; vzCaps->diskBuses[j] != VIR_DOMAIN_DISK_BUS_LAST; j++) {
+ if (disk->bus == vzCaps->diskBuses[j])
+ break;
+ }
+
+ if (vzCaps->diskBuses[j] == VIR_DOMAIN_DISK_BUS_LAST) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Unsupported disk bus type %s"),
+ virDomainDiskBusTypeToString(disk->bus));
+ return -1;
+ }
+ }
+ return 0;
+}
diff --git a/src/vz/vz_utils.h b/src/vz/vz_utils.h
index e0b0105..851322f 100644
--- a/src/vz/vz_utils.h
+++ b/src/vz/vz_utils.h
@@ -110,6 +110,9 @@ vzNewDomain(vzConnPtr privconn,
const unsigned char *uuid);
int
vzInitVersion(vzConnPtr privconn);
+int
+vzCheckUnsupportedDisks(virDomainDefPtr def,
+ vzCapabilitiesPtr vzCaps);
# define PARALLELS_BLOCK_STATS_FOREACH(OP) \
OP(rd_req, VIR_DOMAIN_BLOCK_STATS_READ_REQ, "read_requests") \
8:47 a.m.
New subject: [libvirt] [PATCH 4/6] vz: report correct disk format in domainGetXMLDesc
We should report correct disk format, because it depends on
virtuozzo version and domain type.
Now we support only one disk format for each domain type.
So we can take it from vzCapabilities structure.
---
src/vz/vz_sdk.c | 29 +++++++++++++++++------------
1 file changed, 17 insertions(+), 12 deletions(-)
diff --git a/src/vz/vz_sdk.c b/src/vz/vz_sdk.c
index 2a6e908..c45ca09 100644
--- a/src/vz/vz_sdk.c
+++ b/src/vz/vz_sdk.c
@@ -458,7 +458,8 @@ prlsdkAddDomainVideoInfo(PRL_HANDLE sdkdom, virDomainDefPtr def)
}
static int
-prlsdkGetDiskInfo(PRL_HANDLE prldisk,
+prlsdkGetDiskInfo(vzConnPtr privconn,
+ PRL_HANDLE prldisk,
virDomainDiskDefPtr disk,
bool isCdrom,
bool isCt)
@@ -476,10 +477,14 @@ prlsdkGetDiskInfo(PRL_HANDLE prldisk,
prlsdkCheckRetGoto(pret, cleanup);
if (emulatedType == PDT_USE_IMAGE_FILE) {
virDomainDiskSetType(disk, VIR_STORAGE_TYPE_FILE);
- if (isCdrom)
+ if (isCdrom) {
virDomainDiskSetFormat(disk, VIR_STORAGE_FILE_RAW);
- else
- virDomainDiskSetFormat(disk, VIR_STORAGE_FILE_PLOOP);
+ } else {
+ if (isCt)
+ virDomainDiskSetFormat(disk, privconn->vzCaps.ctDiskFormat);
+ else
+ virDomainDiskSetFormat(disk, privconn->vzCaps.vmDiskFormat);
+ }
} else {
virDomainDiskSetType(disk, VIR_STORAGE_TYPE_BLOCK);
virDomainDiskSetFormat(disk, VIR_STORAGE_FILE_RAW);
@@ -607,7 +612,7 @@ prlsdkGetFSInfo(PRL_HANDLE prldisk,
}
static int
-prlsdkAddDomainHardDisksInfo(PRL_HANDLE sdkdom, virDomainDefPtr def)
+prlsdkAddDomainHardDisksInfo(vzConnPtr privconn, PRL_HANDLE sdkdom, virDomainDefPtr def)
{
PRL_RESULT pret;
PRL_UINT32 hddCount;
@@ -647,7 +652,7 @@ prlsdkAddDomainHardDisksInfo(PRL_HANDLE sdkdom, virDomainDefPtr def)
if (!(disk = virDomainDiskDefNew(NULL)))
goto error;
- if (prlsdkGetDiskInfo(hdd, disk, false, IS_CT(def)) < 0)
+ if (prlsdkGetDiskInfo(privconn, hdd, disk, false, IS_CT(def)) < 0)
goto error;
if (VIR_APPEND_ELEMENT(def->disks, def->ndisks, disk) < 0)
@@ -669,7 +674,7 @@ prlsdkAddDomainHardDisksInfo(PRL_HANDLE sdkdom, virDomainDefPtr def)
}
static int
-prlsdkAddDomainOpticalDisksInfo(PRL_HANDLE sdkdom, virDomainDefPtr def)
+prlsdkAddDomainOpticalDisksInfo(vzConnPtr privconn, PRL_HANDLE sdkdom, virDomainDefPtr
def)
{
PRL_RESULT pret;
PRL_UINT32 cdromsCount;
@@ -687,7 +692,7 @@ prlsdkAddDomainOpticalDisksInfo(PRL_HANDLE sdkdom, virDomainDefPtr
def)
if (!(disk = virDomainDiskDefNew(NULL)))
goto error;
- if (prlsdkGetDiskInfo(cdrom, disk, true, IS_CT(def)) < 0)
+ if (prlsdkGetDiskInfo(privconn, cdrom, disk, true, IS_CT(def)) < 0)
goto error;
PrlHandle_Free(cdrom);
@@ -950,16 +955,16 @@ prlsdkAddSerialInfo(PRL_HANDLE sdkdom,
static int
-prlsdkAddDomainHardware(PRL_HANDLE sdkdom, virDomainDefPtr def)
+prlsdkAddDomainHardware(vzConnPtr privconn, PRL_HANDLE sdkdom, virDomainDefPtr def)
{
if (!IS_CT(def))
if (prlsdkAddDomainVideoInfo(sdkdom, def) < 0)
goto error;
- if (prlsdkAddDomainHardDisksInfo(sdkdom, def) < 0)
+ if (prlsdkAddDomainHardDisksInfo(privconn, sdkdom, def) < 0)
goto error;
- if (prlsdkAddDomainOpticalDisksInfo(sdkdom, def) < 0)
+ if (prlsdkAddDomainOpticalDisksInfo(privconn, sdkdom, def) < 0)
goto error;
if (prlsdkAddDomainNetInfo(sdkdom, def) < 0)
@@ -1317,7 +1322,7 @@ prlsdkLoadDomain(vzConnPtr privconn, virDomainObjPtr dom)
if (prlsdkConvertDomainType(sdkdom, def) < 0)
goto error;
- if (prlsdkAddDomainHardware(sdkdom, def) < 0)
+ if (prlsdkAddDomainHardware(privconn, sdkdom, def) < 0)
goto error;
if (prlsdkAddVNCInfo(sdkdom, def) < 0)
--
1.8.3.1
8:47 a.m.
New subject: [libvirt] [PATCH 5/6] vz: check supported controllers
Virtuozzo6 supports only SCSI(BUSLOGIC) IDE and SATA controllers.
Virtuozzo7 supports only SCSI(VIRTIO_SCSI) and IDE.
In this patch we add list of supported controllers and scsi models to vzCapabilities
structure.
In openConnection() callback we get virtuozzo version and select proper capabilities
values.
In XMLPostParse phase we check controller type and SCSI model.
---
src/vz/vz_driver.c | 3 +++
src/vz/vz_utils.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
src/vz/vz_utils.h | 5 +++++
3 files changed, 56 insertions(+)
diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c
index 4f52bc7..74e1f5d 100644
--- a/src/vz/vz_driver.c
+++ b/src/vz/vz_driver.c
@@ -180,6 +180,9 @@ vzDomainDefPostParse(virDomainDefPtr def ATTRIBUTE_UNUSED,
if (vzCheckUnsupportedDisks(def, opaque) < 0)
return -1;
+ if (vzCheckUnsupportedControllers(def, opaque) < 0)
+ return -1;
+
return 0;
}
diff --git a/src/vz/vz_utils.c b/src/vz/vz_utils.c
index bf62538..58014ec 100644
--- a/src/vz/vz_utils.c
+++ b/src/vz/vz_utils.c
@@ -45,6 +45,15 @@ static virDomainDiskBus vz7DiskBuses[] = {VIR_DOMAIN_DISK_BUS_IDE,
VIR_DOMAIN_DISK_BUS_SCSI,
VIR_DOMAIN_DISK_BUS_LAST};
+static virDomainControllerType vz6ControllerTypes[] = {VIR_DOMAIN_CONTROLLER_TYPE_SCSI,
+ VIR_DOMAIN_CONTROLLER_TYPE_IDE,
+ VIR_DOMAIN_CONTROLLER_TYPE_SATA,
+ VIR_DOMAIN_CONTROLLER_TYPE_LAST};
+
+static virDomainControllerType vz7ControllerTypes[] = {VIR_DOMAIN_CONTROLLER_TYPE_SCSI,
+ VIR_DOMAIN_CONTROLLER_TYPE_IDE,
+ VIR_DOMAIN_CONTROLLER_TYPE_LAST};
+
/**
* vzDomObjFromDomain:
* @domain: Domain pointer that has to be looked up
@@ -196,10 +205,14 @@ vzInitCaps(unsigned long vzVersion, vzCapabilities *vzCaps)
vzCaps->ctDiskFormat = VIR_STORAGE_FILE_PLOOP;
vzCaps->vmDiskFormat = VIR_STORAGE_FILE_PLOOP;
vzCaps->diskBuses = vz6DiskBuses;
+ vzCaps->controllerTypes = vz6ControllerTypes;
+ vzCaps->scsiControllerModel = VIR_DOMAIN_CONTROLLER_MODEL_SCSI_BUSLOGIC;
} else {
vzCaps->ctDiskFormat = VIR_STORAGE_FILE_PLOOP;
vzCaps->vmDiskFormat = VIR_STORAGE_FILE_QCOW2;
vzCaps->diskBuses = vz7DiskBuses;
+ vzCaps->controllerTypes = vz7ControllerTypes;
+ vzCaps->scsiControllerModel = VIR_DOMAIN_CONTROLLER_MODEL_SCSI_VIRTIO_SCSI;
}
}
@@ -301,3 +314,38 @@ int vzCheckUnsupportedDisks(virDomainDefPtr def,
}
return 0;
}
+
+int vzCheckUnsupportedControllers(virDomainDefPtr def,
+ vzCapabilitiesPtr vzCaps)
+{
+ size_t i, j;
+ virDomainControllerDefPtr controller;
+
+ for (i = 0; i < def->ncontrollers; i++) {
+ controller = def->controllers[i];
+
+ for (j = 0; vzCaps->controllerTypes[j] != VIR_DOMAIN_CONTROLLER_TYPE_LAST;
j++) {
+ if (controller->type == vzCaps->controllerTypes[j])
+ break;
+ }
+
+ if (vzCaps->controllerTypes[j] == VIR_DOMAIN_CONTROLLER_TYPE_LAST) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Unsupported controller type %s"),
+ virDomainControllerTypeToString(controller->type));
+ return -1;
+ }
+
+ if (controller->type == VIR_DOMAIN_CONTROLLER_TYPE_SCSI &&
+ controller->model != -1 &&
+ controller->model != VIR_DOMAIN_CONTROLLER_MODEL_SCSI_AUTO &&
+ controller->model != vzCaps->scsiControllerModel) {
+
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Unsupported SCSI controller model %s"),
+
virDomainControllerModelSCSITypeToString(controller->model));
+ return -1;
+ }
+ }
+ return 0;
+}
diff --git a/src/vz/vz_utils.h b/src/vz/vz_utils.h
index 851322f..88e74fc 100644
--- a/src/vz/vz_utils.h
+++ b/src/vz/vz_utils.h
@@ -54,6 +54,8 @@ struct _vzCapabilities {
virStorageFileFormat vmDiskFormat;
virStorageFileFormat ctDiskFormat;
virDomainDiskBus *diskBuses;
+ virDomainControllerType *controllerTypes;
+ virDomainControllerModelSCSI scsiControllerModel;
};
typedef struct _vzCapabilities vzCapabilities;
typedef struct _vzCapabilities *vzCapabilitiesPtr;
@@ -113,6 +115,9 @@ vzInitVersion(vzConnPtr privconn);
int
vzCheckUnsupportedDisks(virDomainDefPtr def,
vzCapabilitiesPtr vzCaps);
+int
+vzCheckUnsupportedControllers(virDomainDefPtr def,
+ vzCapabilitiesPtr vzCaps);
# define PARALLELS_BLOCK_STATS_FOREACH(OP) \
OP(rd_req, VIR_DOMAIN_BLOCK_STATS_READ_REQ, "read_requests") \
--
1.8.3.1
9:04 a.m.
New subject: [libvirt] [PATCH 5/6] vz: check supported controllers
15.03.2016 10:47, Mikhail Feoktistov пишет:
Virtuozzo6 supports only SCSI(BUSLOGIC) IDE and SATA controllers.
Virtuozzo7 supports only SCSI(VIRTIO_SCSI) and IDE.
In this patch we add list of supported controllers and scsi models to vzCapabilities
structure.
In openConnection() callback we get virtuozzo version and select proper capabilities
values.
In XMLPostParse phase we check controller type and SCSI model.
---
src/vz/vz_driver.c | 3 +++
src/vz/vz_utils.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
src/vz/vz_utils.h | 5 +++++
3 files changed, 56 insertions(+)
diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c
index 4f52bc7..74e1f5d 100644
--- a/src/vz/vz_driver.c
+++ b/src/vz/vz_driver.c
@@ -180,6 +180,9 @@ vzDomainDefPostParse(virDomainDefPtr def ATTRIBUTE_UNUSED,
if (vzCheckUnsupportedDisks(def, opaque) < 0)
return -1;
+ if (vzCheckUnsupportedControllers(def, opaque) < 0)
+ return -1;
+
return 0;
}
diff --git a/src/vz/vz_utils.c b/src/vz/vz_utils.c
index bf62538..58014ec 100644
--- a/src/vz/vz_utils.c
+++ b/src/vz/vz_utils.c
@@ -45,6 +45,15 @@ static virDomainDiskBus vz7DiskBuses[] = {VIR_DOMAIN_DISK_BUS_IDE,
VIR_DOMAIN_DISK_BUS_SCSI,
VIR_DOMAIN_DISK_BUS_LAST};
+static virDomainControllerType vz6ControllerTypes[] = {VIR_DOMAIN_CONTROLLER_TYPE_SCSI,
+ VIR_DOMAIN_CONTROLLER_TYPE_IDE,
+ VIR_DOMAIN_CONTROLLER_TYPE_SATA,
+
VIR_DOMAIN_CONTROLLER_TYPE_LAST};
+
+static virDomainControllerType vz7ControllerTypes[] = {VIR_DOMAIN_CONTROLLER_TYPE_SCSI,
+ VIR_DOMAIN_CONTROLLER_TYPE_IDE,
+
VIR_DOMAIN_CONTROLLER_TYPE_LAST};
+
/**
* vzDomObjFromDomain:
* @domain: Domain pointer that has to be looked up
@@ -196,10 +205,14 @@ vzInitCaps(unsigned long vzVersion, vzCapabilities *vzCaps)
vzCaps->ctDiskFormat = VIR_STORAGE_FILE_PLOOP;
vzCaps->vmDiskFormat = VIR_STORAGE_FILE_PLOOP;
vzCaps->diskBuses = vz6DiskBuses;
+ vzCaps->controllerTypes = vz6ControllerTypes;
+ vzCaps->scsiControllerModel = VIR_DOMAIN_CONTROLLER_MODEL_SCSI_BUSLOGIC;
} else {
vzCaps->ctDiskFormat = VIR_STORAGE_FILE_PLOOP;
vzCaps->vmDiskFormat = VIR_STORAGE_FILE_QCOW2;
vzCaps->diskBuses = vz7DiskBuses;
+ vzCaps->controllerTypes = vz7ControllerTypes;
+ vzCaps->scsiControllerModel = VIR_DOMAIN_CONTROLLER_MODEL_SCSI_VIRTIO_SCSI;
}
}
@@ -301,3 +314,38 @@ int vzCheckUnsupportedDisks(virDomainDefPtr def,
}
return 0;
}
+
+int vzCheckUnsupportedControllers(virDomainDefPtr def,
+ vzCapabilitiesPtr vzCaps)
int
vzCheckUnsupportedControllers(virDomainDefPtr def, vzCapabilitiesPtr vzCaps)
declaration would comply with both line lenght and other function
declaration style
+{
+ size_t i, j;
+ virDomainControllerDefPtr controller;
+
+ for (i = 0; i < def->ncontrollers; i++) {
+ controller = def->controllers[i];
+
+ for (j = 0; vzCaps->controllerTypes[j] != VIR_DOMAIN_CONTROLLER_TYPE_LAST;
j++) {
+ if (controller->type == vzCaps->controllerTypes[j])
+ break;
+ }
+
+ if (vzCaps->controllerTypes[j] == VIR_DOMAIN_CONTROLLER_TYPE_LAST) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Unsupported controller type %s"),
+ virDomainControllerTypeToString(controller->type));
+ return -1;
+ }
+
+ if (controller->type == VIR_DOMAIN_CONTROLLER_TYPE_SCSI &&
+ controller->model != -1 &&
+ controller->model != VIR_DOMAIN_CONTROLLER_MODEL_SCSI_AUTO &&
+ controller->model != vzCaps->scsiControllerModel) {
+
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Unsupported SCSI controller model %s"),
+
virDomainControllerModelSCSITypeToString(controller->model));
+ return -1;
+ }
+ }
+ return 0;
+}
diff --git a/src/vz/vz_utils.h b/src/vz/vz_utils.h
index 851322f..88e74fc 100644
--- a/src/vz/vz_utils.h
+++ b/src/vz/vz_utils.h
@@ -54,6 +54,8 @@ struct _vzCapabilities {
virStorageFileFormat vmDiskFormat;
virStorageFileFormat ctDiskFormat;
virDomainDiskBus *diskBuses;
+ virDomainControllerType *controllerTypes;
+ virDomainControllerModelSCSI scsiControllerModel;
};
typedef struct _vzCapabilities vzCapabilities;
typedef struct _vzCapabilities *vzCapabilitiesPtr;
@@ -113,6 +115,9 @@ vzInitVersion(vzConnPtr privconn);
int
vzCheckUnsupportedDisks(virDomainDefPtr def,
vzCapabilitiesPtr vzCaps);
+int
+vzCheckUnsupportedControllers(virDomainDefPtr def,
+ vzCapabilitiesPtr vzCaps);
# define PARALLELS_BLOCK_STATS_FOREACH(OP) \
OP(rd_req, VIR_DOMAIN_BLOCK_STATS_READ_REQ, "read_requests") \
8:47 a.m.
New subject: [libvirt] [PATCH 6/6] vz: set default SCSI model
Each version of virtuozzo supports only one type of SCSI controller
So if we add disk on SCSI bus, we should set SCSI controller model.
We can take it from vzCapabilities structure.
---
src/vz/vz_driver.c | 2 +-
src/vz/vz_sdk.c | 27 +++++++++++++++++++++++----
src/vz/vz_sdk.h | 2 +-
src/vz/vz_utils.c | 20 ++++++++++++++++++++
src/vz/vz_utils.h | 3 +++
5 files changed, 48 insertions(+), 6 deletions(-)
diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c
index 74e1f5d..b02e59f 100644
--- a/src/vz/vz_driver.c
+++ b/src/vz/vz_driver.c
@@ -1080,7 +1080,7 @@ static int vzDomainAttachDeviceFlags(virDomainPtr dom, const char
*xml,
switch (dev->type) {
case VIR_DOMAIN_DEVICE_DISK:
- ret = prlsdkAttachVolume(privdom, dev->data.disk);
+ ret = prlsdkAttachVolume(privconn, privdom, dev->data.disk);
if (ret) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("disk attach failed"));
diff --git a/src/vz/vz_sdk.c b/src/vz/vz_sdk.c
index c45ca09..8baad8e 100644
--- a/src/vz/vz_sdk.c
+++ b/src/vz/vz_sdk.c
@@ -3115,7 +3115,8 @@ static int prlsdkDelDisk(PRL_HANDLE sdkdom, int idx)
return ret;
}
-static int prlsdkAddDisk(PRL_HANDLE sdkdom,
+static int prlsdkAddDisk(vzConnPtr privconn,
+ PRL_HANDLE sdkdom,
virDomainDiskDefPtr disk,
bool bootDisk,
bool isCt)
@@ -3129,6 +3130,7 @@ static int prlsdkAddDisk(PRL_HANDLE sdkdom,
virDomainDeviceDriveAddressPtr drive;
PRL_UINT32 devIndex;
PRL_DEVICE_TYPE devType;
+ PRL_CLUSTERED_DEVICE_SUBTYPE scsiModel;
char *dst = NULL;
if (prlsdkCheckDiskUnsupportedParams(disk) < 0)
@@ -3229,6 +3231,13 @@ static int prlsdkAddDisk(PRL_HANDLE sdkdom,
goto cleanup;
}
+ if (disk->bus == VIR_DOMAIN_DISK_BUS_SCSI) {
+ if (vzGetDefaultSCSIModel(privconn, &scsiModel) < 0)
+ goto cleanup;
+ pret = PrlVmDev_SetSubType(sdkdisk, scsiModel);
+ prlsdkCheckRetGoto(pret, cleanup);
+ }
+
pret = PrlVmDev_SetIfaceType(sdkdisk, sdkbus);
prlsdkCheckRetGoto(pret, cleanup);
@@ -3276,7 +3285,9 @@ static int prlsdkAddDisk(PRL_HANDLE sdkdom,
}
int
-prlsdkAttachVolume(virDomainObjPtr dom, virDomainDiskDefPtr disk)
+prlsdkAttachVolume(vzConnPtr privconn,
+ virDomainObjPtr dom,
+ virDomainDiskDefPtr disk)
{
int ret = -1;
vzDomObjPtr privdom = dom->privateData;
@@ -3286,7 +3297,11 @@ prlsdkAttachVolume(virDomainObjPtr dom, virDomainDiskDefPtr disk)
if (PRL_FAILED(waitJob(job)))
goto cleanup;
- ret = prlsdkAddDisk(privdom->sdkdom, disk, false, IS_CT(dom->def));
+ ret = prlsdkAddDisk(privconn,
+ privdom->sdkdom,
+ disk,
+ false,
+ IS_CT(dom->def));
if (ret == 0) {
job = PrlVm_CommitEx(privdom->sdkdom, PVCF_DETACH_HDD_BUNDLE);
if (PRL_FAILED(waitJob(job))) {
@@ -3517,7 +3532,11 @@ prlsdkDoApplyConfig(virConnectPtr conn,
needBoot = false;
bootDisk = true;
}
- if (prlsdkAddDisk(sdkdom, def->disks[i], bootDisk, IS_CT(def)) < 0)
+ if (prlsdkAddDisk(conn->privateData,
+ sdkdom,
+ def->disks[i],
+ bootDisk,
+ IS_CT(def)) < 0)
goto error;
}
diff --git a/src/vz/vz_sdk.h b/src/vz/vz_sdk.h
index c2d9cb4..8a38797 100644
--- a/src/vz/vz_sdk.h
+++ b/src/vz/vz_sdk.h
@@ -63,7 +63,7 @@ prlsdkUnregisterDomain(vzConnPtr privconn, virDomainObjPtr dom, unsigned
int fla
int
prlsdkDomainManagedSaveRemove(virDomainObjPtr dom);
int
-prlsdkAttachVolume(virDomainObjPtr dom, virDomainDiskDefPtr disk);
+prlsdkAttachVolume(vzConnPtr privconn, virDomainObjPtr dom, virDomainDiskDefPtr disk);
int
prlsdkDetachVolume(virDomainObjPtr dom, virDomainDiskDefPtr disk);
int
diff --git a/src/vz/vz_utils.c b/src/vz/vz_utils.c
index 58014ec..ebde0f5 100644
--- a/src/vz/vz_utils.c
+++ b/src/vz/vz_utils.c
@@ -349,3 +349,23 @@ int vzCheckUnsupportedControllers(virDomainDefPtr def,
}
return 0;
}
+
+int vzGetDefaultSCSIModel(vzConnPtr privconn,
+ PRL_CLUSTERED_DEVICE_SUBTYPE *scsiModel)
+{
+ switch (privconn->vzCaps.scsiControllerModel) {
+ case VIR_DOMAIN_CONTROLLER_MODEL_SCSI_VIRTIO_SCSI:
+ *scsiModel = PCD_VIRTIO_SCSI;
+ break;
+ case VIR_DOMAIN_CONTROLLER_MODEL_SCSI_BUSLOGIC:
+ *scsiModel = PCD_BUSLOGIC;
+ break;
+ default:
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("Unknown SCSI controller model %s"),
+ virDomainControllerModelSCSITypeToString(
+ privconn->vzCaps.scsiControllerModel));
+ return -1;
+ }
+ return 0;
+}
diff --git a/src/vz/vz_utils.h b/src/vz/vz_utils.h
index 88e74fc..f373850 100644
--- a/src/vz/vz_utils.h
+++ b/src/vz/vz_utils.h
@@ -118,6 +118,9 @@ vzCheckUnsupportedDisks(virDomainDefPtr def,
int
vzCheckUnsupportedControllers(virDomainDefPtr def,
vzCapabilitiesPtr vzCaps);
+int
+vzGetDefaultSCSIModel(vzConnPtr privconn,
+ PRL_CLUSTERED_DEVICE_SUBTYPE *scsiModel);
# define PARALLELS_BLOCK_STATS_FOREACH(OP) \
OP(rd_req, VIR_DOMAIN_BLOCK_STATS_READ_REQ, "read_requests") \
--
1.8.3.1
3:24 p.m.
15.03.2016 10:47, Mikhail Feoktistov пишет:
Mikhail Feoktistov (6):
vz: save vz version in connection structure
vz: add vzCapabilities to connection structure
vz: check supported disk format and bus
vz: report correct disk format in domainGetXMLDesc
vz: check supported controllers
vz: set default SCSI model
src/vz/vz_driver.c | 61 +++++------------
src/vz/vz_sdk.c | 80 +++++++++++-----------
src/vz/vz_sdk.h | 2 +-
src/vz/vz_utils.c | 191 +++++++++++++++++++++++++++++++++++++++++++++++++++++
src/vz/vz_utils.h | 24 +++++++
5 files changed, 274 insertions(+), 84 deletions(-)
I've just sent a new version of the series with all found issues fixed.
Please take a look.
Best,
Maxim
3109
days inactive
3110
days old
9 comments
2 participants
participants (2)
-
Maxim Nestratov -
Mikhail Feoktistov