4:44 p.m.
I started the download part and the stream driver quite a while ago
but stopped and put it aside as I realized that the storage volume
API up- and download functions assume one file per volume. This is a
problem with ESX as a VMDK can consist of two files. To solve this
now I added two new flags: METADATA and CONTENT. With this flags the
user can specify which part of such a volume to transfer. See patch
3/3 for more details.
Matthias
4:44 p.m.
New subject: [libvirt] [PATCH 1/3] esx: Wrap libcurl multi handle
---
src/esx/esx_vi.c | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/esx/esx_vi.h | 18 +++++++++
2 files changed, 129 insertions(+), 0 deletions(-)
diff --git a/src/esx/esx_vi.c b/src/esx/esx_vi.c
index 5b5ab69..48718b6 100644
--- a/src/esx/esx_vi.c
+++ b/src/esx/esx_vi.c
@@ -86,6 +86,7 @@ ESX_VI__TEMPLATE__ALLOC(CURL)
ESX_VI__TEMPLATE__FREE(CURL,
{
esxVI_SharedCURL *shared = item->shared;
+ esxVI_MultiCURL *multi = item->multi;
if (shared != NULL) {
esxVI_SharedCURL_Remove(shared, item);
@@ -95,6 +96,14 @@ ESX_VI__TEMPLATE__FREE(CURL,
}
}
+ if (multi != NULL) {
+ esxVI_MultiCURL_Remove(multi, item);
+
+ if (multi->count == 0) {
+ esxVI_MultiCURL_Free(&multi);
+ }
+ }
+
if (item->handle != NULL) {
curl_easy_cleanup(item->handle);
}
@@ -555,11 +564,15 @@ esxVI_SharedCURL_Add(esxVI_SharedCURL *shared, esxVI_CURL *curl)
}
}
+ virMutexLock(&curl->lock);
+
curl_easy_setopt(curl->handle, CURLOPT_SHARE, shared->handle);
curl->shared = shared;
++shared->count;
+ virMutexUnlock(&curl->lock);
+
return 0;
}
@@ -583,11 +596,109 @@ esxVI_SharedCURL_Remove(esxVI_SharedCURL *shared, esxVI_CURL
*curl)
return -1;
}
+ virMutexLock(&curl->lock);
+
curl_easy_setopt(curl->handle, CURLOPT_SHARE, NULL);
curl->shared = NULL;
--shared->count;
+ virMutexUnlock(&curl->lock);
+
+ return 0;
+}
+
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * MultiCURL
+ */
+
+/* esxVI_MultiCURL_Alloc */
+ESX_VI__TEMPLATE__ALLOC(MultiCURL)
+
+/* esxVI_MultiCURL_Free */
+ESX_VI__TEMPLATE__FREE(MultiCURL,
+{
+ if (item->count > 0) {
+ /* Better leak than crash */
+ VIR_ERROR(_("Trying to free MultiCURL object that is still in use"));
+ return;
+ }
+
+ if (item->handle != NULL) {
+ curl_multi_cleanup(item->handle);
+ }
+})
+
+int
+esxVI_MultiCURL_Add(esxVI_MultiCURL *multi, esxVI_CURL *curl)
+{
+ if (curl->handle == NULL) {
+ ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot add uninitialized CURL handle to a multi
handle"));
+ return -1;
+ }
+
+ if (curl->multi != NULL) {
+ ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot add CURL handle to a multi handle twice"));
+ return -1;
+ }
+
+ if (multi->handle == NULL) {
+ multi->handle = curl_multi_init();
+
+ if (multi->handle == NULL) {
+ ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Could not initialize CURL (multi)"));
+ return -1;
+ }
+ }
+
+ virMutexLock(&curl->lock);
+
+ curl_multi_add_handle(multi->handle, curl->handle);
+
+ curl->multi = multi;
+ ++multi->count;
+
+ virMutexUnlock(&curl->lock);
+
+ return 0;
+}
+
+int
+esxVI_MultiCURL_Remove(esxVI_MultiCURL *multi, esxVI_CURL *curl)
+{
+ if (curl->handle == NULL) {
+ ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot remove uninitialized CURL handle from a "
+ "multi handle"));
+ return -1;
+ }
+
+ if (curl->multi == NULL) {
+ ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot remove CURL handle from a multi handle when it
"
+ "wasn't added before"));
+ return -1;
+ }
+
+ if (curl->multi != multi) {
+ ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s", _("CURL (multi)
mismatch"));
+ return -1;
+ }
+
+ virMutexLock(&curl->lock);
+
+ curl_multi_remove_handle(multi->handle, curl->handle);
+
+ curl->multi = NULL;
+ --multi->count;
+
+ virMutexUnlock(&curl->lock);
+
return 0;
}
diff --git a/src/esx/esx_vi.h b/src/esx/esx_vi.h
index 78d3986..9560bd2 100644
--- a/src/esx/esx_vi.h
+++ b/src/esx/esx_vi.h
@@ -85,6 +85,7 @@ typedef enum _esxVI_Occurrence esxVI_Occurrence;
typedef struct _esxVI_ParsedHostCpuIdInfo esxVI_ParsedHostCpuIdInfo;
typedef struct _esxVI_CURL esxVI_CURL;
typedef struct _esxVI_SharedCURL esxVI_SharedCURL;
+typedef struct _esxVI_MultiCURL esxVI_MultiCURL;
typedef struct _esxVI_Context esxVI_Context;
typedef struct _esxVI_Response esxVI_Response;
typedef struct _esxVI_Enumeration esxVI_Enumeration;
@@ -160,6 +161,7 @@ struct _esxVI_CURL {
struct curl_slist *headers;
char error[CURL_ERROR_SIZE];
esxVI_SharedCURL *shared;
+ esxVI_MultiCURL *multi;
};
int esxVI_CURL_Alloc(esxVI_CURL **curl);
@@ -188,6 +190,22 @@ int esxVI_SharedCURL_Remove(esxVI_SharedCURL *shared, esxVI_CURL
*curl);
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * MultiCURL
+ */
+
+struct _esxVI_MultiCURL {
+ CURLM *handle;
+ size_t count;
+};
+
+int esxVI_MultiCURL_Alloc(esxVI_MultiCURL **multi);
+void esxVI_MultiCURL_Free(esxVI_MultiCURL **multi);
+int esxVI_MultiCURL_Add(esxVI_MultiCURL *multi, esxVI_CURL *curl);
+int esxVI_MultiCURL_Remove(esxVI_MultiCURL *multi, esxVI_CURL *curl);
+
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Context
*/
--
1.7.4.1
5:18 p.m.
New subject: [libvirt] [PATCH 1/3] esx: Wrap libcurl multi handle
On Mon, Jul 2, 2012 at 4:44 PM, Matthias Bolte
<matthias.bolte(a)googlemail.com> wrote:
---
src/esx/esx_vi.c | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/esx/esx_vi.h | 18 +++++++++
2 files changed, 129 insertions(+), 0 deletions(-)
diff --git a/src/esx/esx_vi.c b/src/esx/esx_vi.c
index 5b5ab69..48718b6 100644
--- a/src/esx/esx_vi.c
+++ b/src/esx/esx_vi.c
@@ -86,6 +86,7 @@ ESX_VI__TEMPLATE__ALLOC(CURL)
ESX_VI__TEMPLATE__FREE(CURL,
{
esxVI_SharedCURL *shared = item->shared;
+ esxVI_MultiCURL *multi = item->multi;
if (shared != NULL) {
esxVI_SharedCURL_Remove(shared, item);
@@ -95,6 +96,14 @@ ESX_VI__TEMPLATE__FREE(CURL,
}
}
+ if (multi != NULL) {
+ esxVI_MultiCURL_Remove(multi, item);
+
+ if (multi->count == 0) {
+ esxVI_MultiCURL_Free(&multi);
+ }
+ }
+
if (item->handle != NULL) {
curl_easy_cleanup(item->handle);
}
@@ -555,11 +564,15 @@ esxVI_SharedCURL_Add(esxVI_SharedCURL *shared, esxVI_CURL *curl)
}
}
+ virMutexLock(&curl->lock);
+
curl_easy_setopt(curl->handle, CURLOPT_SHARE, shared->handle);
curl->shared = shared;
++shared->count;
+ virMutexUnlock(&curl->lock);
+
return 0;
}
@@ -583,11 +596,109 @@ esxVI_SharedCURL_Remove(esxVI_SharedCURL *shared, esxVI_CURL
*curl)
return -1;
}
+ virMutexLock(&curl->lock);
+
curl_easy_setopt(curl->handle, CURLOPT_SHARE, NULL);
curl->shared = NULL;
--shared->count;
+ virMutexUnlock(&curl->lock);
+
+ return 0;
+}
+
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * MultiCURL
+ */
+
+/* esxVI_MultiCURL_Alloc */
+ESX_VI__TEMPLATE__ALLOC(MultiCURL)
+
+/* esxVI_MultiCURL_Free */
+ESX_VI__TEMPLATE__FREE(MultiCURL,
+{
+ if (item->count > 0) {
+ /* Better leak than crash */
+ VIR_ERROR(_("Trying to free MultiCURL object that is still in use"));
+ return;
+ }
+
+ if (item->handle != NULL) {
+ curl_multi_cleanup(item->handle);
+ }
Since its a double pointer maybe setting the passed in value to NULL
to prevent a double free situations?
+})
+
+int
+esxVI_MultiCURL_Add(esxVI_MultiCURL *multi, esxVI_CURL *curl)
+{
+ if (curl->handle == NULL) {
+ ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot add uninitialized CURL handle to a multi
handle"));
+ return -1;
+ }
+
+ if (curl->multi != NULL) {
+ ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot add CURL handle to a multi handle twice"));
+ return -1;
+ }
+
+ if (multi->handle == NULL) {
+ multi->handle = curl_multi_init();
+
+ if (multi->handle == NULL) {
+ ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Could not initialize CURL (multi)"));
+ return -1;
+ }
+ }
+
+ virMutexLock(&curl->lock);
+
+ curl_multi_add_handle(multi->handle, curl->handle);
+
+ curl->multi = multi;
+ ++multi->count;
+
+ virMutexUnlock(&curl->lock);
+
+ return 0;
+}
+
+int
+esxVI_MultiCURL_Remove(esxVI_MultiCURL *multi, esxVI_CURL *curl)
+{
+ if (curl->handle == NULL) {
+ ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot remove uninitialized CURL handle from a "
+ "multi handle"));
+ return -1;
+ }
+
+ if (curl->multi == NULL) {
+ ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot remove CURL handle from a multi handle when it
"
+ "wasn't added before"));
+ return -1;
+ }
+
+ if (curl->multi != multi) {
+ ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s", _("CURL (multi)
mismatch"));
+ return -1;
+ }
+
+ virMutexLock(&curl->lock);
+
+ curl_multi_remove_handle(multi->handle, curl->handle);
+
+ curl->multi = NULL;
+ --multi->count;
+
+ virMutexUnlock(&curl->lock);
Maybe add your free code here when count is 0? That way you wouldn't
have to contend with a potential memory leak case when the free is
called when its still ref'd.
+
return 0;
}
diff --git a/src/esx/esx_vi.h b/src/esx/esx_vi.h
index 78d3986..9560bd2 100644
--- a/src/esx/esx_vi.h
+++ b/src/esx/esx_vi.h
@@ -85,6 +85,7 @@ typedef enum _esxVI_Occurrence esxVI_Occurrence;
typedef struct _esxVI_ParsedHostCpuIdInfo esxVI_ParsedHostCpuIdInfo;
typedef struct _esxVI_CURL esxVI_CURL;
typedef struct _esxVI_SharedCURL esxVI_SharedCURL;
+typedef struct _esxVI_MultiCURL esxVI_MultiCURL;
typedef struct _esxVI_Context esxVI_Context;
typedef struct _esxVI_Response esxVI_Response;
typedef struct _esxVI_Enumeration esxVI_Enumeration;
@@ -160,6 +161,7 @@ struct _esxVI_CURL {
struct curl_slist *headers;
char error[CURL_ERROR_SIZE];
esxVI_SharedCURL *shared;
+ esxVI_MultiCURL *multi;
};
int esxVI_CURL_Alloc(esxVI_CURL **curl);
@@ -188,6 +190,22 @@ int esxVI_SharedCURL_Remove(esxVI_SharedCURL *shared, esxVI_CURL
*curl);
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * MultiCURL
+ */
+
+struct _esxVI_MultiCURL {
+ CURLM *handle;
+ size_t count;
+};
+
+int esxVI_MultiCURL_Alloc(esxVI_MultiCURL **multi);
+void esxVI_MultiCURL_Free(esxVI_MultiCURL **multi);
+int esxVI_MultiCURL_Add(esxVI_MultiCURL *multi, esxVI_CURL *curl);
+int esxVI_MultiCURL_Remove(esxVI_MultiCURL *multi, esxVI_CURL *curl);
+
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Context
*/
--
1.7.4.1
--
libvir-list mailing list
libvir-list(a)redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list
--
Doug Goldstein
3:17 p.m.
New subject: [libvirt] [PATCH 1/3] esx: Wrap libcurl multi handle
2012/7/3 Doug Goldstein <cardoe(a)gentoo.org>:
On Mon, Jul 2, 2012 at 4:44 PM, Matthias Bolte
<matthias.bolte(a)googlemail.com> wrote:
> ---
> src/esx/esx_vi.c | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
> src/esx/esx_vi.h | 18 +++++++++
> 2 files changed, 129 insertions(+), 0 deletions(-)
> +/* esxVI_MultiCURL_Alloc */
> +ESX_VI__TEMPLATE__ALLOC(MultiCURL)
> +
> +/* esxVI_MultiCURL_Free */
> +ESX_VI__TEMPLATE__FREE(MultiCURL,
> +{
> + if (item->count > 0) {
> + /* Better leak than crash */
> + VIR_ERROR(_("Trying to free MultiCURL object that is still in
use"));
> + return;
> + }
> +
> + if (item->handle != NULL) {
> + curl_multi_cleanup(item->handle);
> + }
Since its a double pointer maybe setting the passed in value to NULL
to prevent a double free situations?
Actually that's what already happens here but hidden inside the
ESX_VI__TEMPLATE__FREE macro. Expanded esxVI_MultiCURL_Free looks like
this
void esxVI_MultiCURL_Free(esxVI_MultiCURL **multi)
{
esxVI_MultiCURL *item;
if (multi == NULL || *multi == NULL) {
return;
}
item = *multi;
if (item->count > 0) {
/* Better leak than crash */
VIR_ERROR(_("Trying to free MultiCURL object that is still in use"));
return;
}
if (item->handle != NULL) {
curl_multi_cleanup(item->handle);
}
VIR_FREE(*multi);
}
As VIR_FREE sets its argument to NULL after freeing it a double free
is not possible here.
> +int
> +esxVI_MultiCURL_Remove(esxVI_MultiCURL *multi, esxVI_CURL *curl)
> +{
> + if (curl->handle == NULL) {
> + ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
> + _("Cannot remove uninitialized CURL handle from a "
> + "multi handle"));
> + return -1;
> + }
> +
> + if (curl->multi == NULL) {
> + ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
> + _("Cannot remove CURL handle from a multi handle when it
"
> + "wasn't added before"));
> + return -1;
> + }
> +
> + if (curl->multi != multi) {
> + ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s", _("CURL (multi)
mismatch"));
> + return -1;
> + }
> +
> + virMutexLock(&curl->lock);
> +
> + curl_multi_remove_handle(multi->handle, curl->handle);
> +
> + curl->multi = NULL;
> + --multi->count;
> +
> + virMutexUnlock(&curl->lock);
Maybe add your free code here when count is 0? That way you wouldn't
have to contend with a potential memory leak case when the free is
called when its still ref'd.
count is not meant as a refcount here. It's sole purpose is to avoid
freeing a multi handle that still has easy handles attached to it.
Also calling free one count == 0 here would not allow a call sequence
such as add, remove, add, remove.
--
Matthias Bolte
http://photron.blogspot.com
4:22 p.m.
New subject: [libvirt] [PATCH 1/3] esx: Wrap libcurl multi handle
On Tue, Jul 3, 2012 at 3:17 PM, Matthias Bolte
<matthias.bolte(a)googlemail.com> wrote:
2012/7/3 Doug Goldstein <cardoe(a)gentoo.org>:
> On Mon, Jul 2, 2012 at 4:44 PM, Matthias Bolte
> <matthias.bolte(a)googlemail.com> wrote:
>> ---
>> src/esx/esx_vi.c | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
>> src/esx/esx_vi.h | 18 +++++++++
>> 2 files changed, 129 insertions(+), 0 deletions(-)
>> +/* esxVI_MultiCURL_Alloc */
>> +ESX_VI__TEMPLATE__ALLOC(MultiCURL)
>> +
>> +/* esxVI_MultiCURL_Free */
>> +ESX_VI__TEMPLATE__FREE(MultiCURL,
>> +{
>> + if (item->count > 0) {
>> + /* Better leak than crash */
>> + VIR_ERROR(_("Trying to free MultiCURL object that is still in
use"));
>> + return;
>> + }
>> +
>> + if (item->handle != NULL) {
>> + curl_multi_cleanup(item->handle);
>> + }
>
> Since its a double pointer maybe setting the passed in value to NULL
> to prevent a double free situations?
Actually that's what already happens here but hidden inside the
ESX_VI__TEMPLATE__FREE macro. Expanded esxVI_MultiCURL_Free looks like
this
void esxVI_MultiCURL_Free(esxVI_MultiCURL **multi)
{
esxVI_MultiCURL *item;
if (multi == NULL || *multi == NULL) {
return;
}
item = *multi;
if (item->count > 0) {
/* Better leak than crash */
VIR_ERROR(_("Trying to free MultiCURL object that is still in use"));
return;
}
if (item->handle != NULL) {
curl_multi_cleanup(item->handle);
}
VIR_FREE(*multi);
}
As VIR_FREE sets its argument to NULL after freeing it a double free
is not possible here.
>> +int
>> +esxVI_MultiCURL_Remove(esxVI_MultiCURL *multi, esxVI_CURL *curl)
>> +{
>> + if (curl->handle == NULL) {
>> + ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
>> + _("Cannot remove uninitialized CURL handle from a
"
>> + "multi handle"));
>> + return -1;
>> + }
>> +
>> + if (curl->multi == NULL) {
>> + ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
>> + _("Cannot remove CURL handle from a multi handle when
it "
>> + "wasn't added before"));
>> + return -1;
>> + }
>> +
>> + if (curl->multi != multi) {
>> + ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s", _("CURL
(multi) mismatch"));
>> + return -1;
>> + }
>> +
>> + virMutexLock(&curl->lock);
>> +
>> + curl_multi_remove_handle(multi->handle, curl->handle);
>> +
>> + curl->multi = NULL;
>> + --multi->count;
>> +
>> + virMutexUnlock(&curl->lock);
>
> Maybe add your free code here when count is 0? That way you wouldn't
> have to contend with a potential memory leak case when the free is
> called when its still ref'd.
count is not meant as a refcount here. It's sole purpose is to avoid
freeing a multi handle that still has easy handles attached to it.
Also calling free one count == 0 here would not allow a call sequence
such as add, remove, add, remove.
--
Matthias Bolte
http://photron.blogspot.com
Ok good. Just was double checking. Then ACK from me on this patch (its
not affected by the multi-CONTENT issue mentioned in the other
e-mail).
--
Doug Goldstein
4:30 a.m.
New subject: [libvirt] [PATCH 1/3] esx: Wrap libcurl multi handle
2012/7/3 Doug Goldstein <cardoe(a)gentoo.org>:
On Tue, Jul 3, 2012 at 3:17 PM, Matthias Bolte
<matthias.bolte(a)googlemail.com> wrote:
> 2012/7/3 Doug Goldstein <cardoe(a)gentoo.org>:
>> On Mon, Jul 2, 2012 at 4:44 PM, Matthias Bolte
>> <matthias.bolte(a)googlemail.com> wrote:
>>> ---
>>> src/esx/esx_vi.c | 111
++++++++++++++++++++++++++++++++++++++++++++++++++++++
>>> src/esx/esx_vi.h | 18 +++++++++
>>> 2 files changed, 129 insertions(+), 0 deletions(-)
>
>>> +/* esxVI_MultiCURL_Alloc */
>>> +ESX_VI__TEMPLATE__ALLOC(MultiCURL)
>>> +
>>> +/* esxVI_MultiCURL_Free */
>>> +ESX_VI__TEMPLATE__FREE(MultiCURL,
>>> +{
>>> + if (item->count > 0) {
>>> + /* Better leak than crash */
>>> + VIR_ERROR(_("Trying to free MultiCURL object that is still in
use"));
>>> + return;
>>> + }
>>> +
>>> + if (item->handle != NULL) {
>>> + curl_multi_cleanup(item->handle);
>>> + }
>>
>> Since its a double pointer maybe setting the passed in value to NULL
>> to prevent a double free situations?
>
> Actually that's what already happens here but hidden inside the
> ESX_VI__TEMPLATE__FREE macro. Expanded esxVI_MultiCURL_Free looks like
> this
>
> void esxVI_MultiCURL_Free(esxVI_MultiCURL **multi)
> {
> esxVI_MultiCURL *item;
>
> if (multi == NULL || *multi == NULL) {
> return;
> }
>
> item = *multi;
>
> if (item->count > 0) {
> /* Better leak than crash */
> VIR_ERROR(_("Trying to free MultiCURL object that is still in
use"));
> return;
> }
>
> if (item->handle != NULL) {
> curl_multi_cleanup(item->handle);
> }
>
> VIR_FREE(*multi);
> }
>
> As VIR_FREE sets its argument to NULL after freeing it a double free
> is not possible here.
>
>>> +int
>>> +esxVI_MultiCURL_Remove(esxVI_MultiCURL *multi, esxVI_CURL *curl)
>>> +{
>>> + if (curl->handle == NULL) {
>>> + ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
>>> + _("Cannot remove uninitialized CURL handle from a
"
>>> + "multi handle"));
>>> + return -1;
>>> + }
>>> +
>>> + if (curl->multi == NULL) {
>>> + ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
>>> + _("Cannot remove CURL handle from a multi handle
when it "
>>> + "wasn't added before"));
>>> + return -1;
>>> + }
>>> +
>>> + if (curl->multi != multi) {
>>> + ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR, "%s", _("CURL
(multi) mismatch"));
>>> + return -1;
>>> + }
>>> +
>>> + virMutexLock(&curl->lock);
>>> +
>>> + curl_multi_remove_handle(multi->handle, curl->handle);
>>> +
>>> + curl->multi = NULL;
>>> + --multi->count;
>>> +
>>> + virMutexUnlock(&curl->lock);
>>
>> Maybe add your free code here when count is 0? That way you wouldn't
>> have to contend with a potential memory leak case when the free is
>> called when its still ref'd.
>
> count is not meant as a refcount here. It's sole purpose is to avoid
> freeing a multi handle that still has easy handles attached to it.
> Also calling free one count == 0 here would not allow a call sequence
> such as add, remove, add, remove.
>
> --
> Matthias Bolte
> http://photron.blogspot.com
Ok good. Just was double checking. Then ACK from me on this patch (its
not affected by the multi-CONTENT issue mentioned in the other
e-mail).
Thanks, I pushed this one.
--
Matthias Bolte
http://photron.blogspot.com
4:44 p.m.
New subject: [libvirt] [PATCH 2/3] esx: Extend esxVI_CURL_Download for partial downloads
---
src/esx/esx_driver.c | 2 +-
src/esx/esx_vi.c | 27 +++++++++++++++++++++++++--
src/esx/esx_vi.h | 3 ++-
3 files changed, 28 insertions(+), 4 deletions(-)
diff --git a/src/esx/esx_driver.c b/src/esx/esx_driver.c
index db2144c..95b9286 100644
--- a/src/esx/esx_driver.c
+++ b/src/esx/esx_driver.c
@@ -2802,7 +2802,7 @@ esxDomainGetXMLDesc(virDomainPtr domain, unsigned int flags)
url = virBufferContentAndReset(&buffer);
- if (esxVI_CURL_Download(priv->primary->curl, url, &vmx) < 0) {
+ if (esxVI_CURL_Download(priv->primary->curl, url, &vmx, 0, NULL) < 0) {
goto cleanup;
}
diff --git a/src/esx/esx_vi.c b/src/esx/esx_vi.c
index 48718b6..3f8d745 100644
--- a/src/esx/esx_vi.c
+++ b/src/esx/esx_vi.c
@@ -355,8 +355,10 @@ esxVI_CURL_Connect(esxVI_CURL *curl, esxUtil_ParsedUri *parsedUri)
}
int
-esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char **content)
+esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char **content,
+ unsigned long long offset, unsigned long long *length)
{
+ char *range = NULL;
virBuffer buffer = VIR_BUFFER_INITIALIZER;
int responseCode = 0;
@@ -365,9 +367,22 @@ esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char
**content)
return -1;
}
+ if (length != NULL && *length > 0) {
+ if (virAsprintf(&range, "%llu-%llu", offset, offset + *length - 1)
< 0) {
+ virReportOOMError();
+ goto cleanup;
+ }
+ } else if (offset > 0) {
+ if (virAsprintf(&range, "%llu-", offset) < 0) {
+ virReportOOMError();
+ goto cleanup;
+ }
+ }
+
virMutexLock(&curl->lock);
curl_easy_setopt(curl->handle, CURLOPT_URL, url);
+ curl_easy_setopt(curl->handle, CURLOPT_RANGE, range);
curl_easy_setopt(curl->handle, CURLOPT_WRITEDATA, &buffer);
curl_easy_setopt(curl->handle, CURLOPT_UPLOAD, 0);
curl_easy_setopt(curl->handle, CURLOPT_HTTPGET, 1);
@@ -378,7 +393,7 @@ esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char
**content)
if (responseCode < 0) {
goto cleanup;
- } else if (responseCode != 200) {
+ } else if (responseCode != 200 && responseCode != 206) {
ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR,
_("HTTP response code %d for download from
'%s'"),
responseCode, url);
@@ -390,9 +405,15 @@ esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char
**content)
goto cleanup;
}
+ if (length != NULL) {
+ *length = virBufferUse(&buffer);
+ }
+
*content = virBufferContentAndReset(&buffer);
cleanup:
+ VIR_FREE(range);
+
if (*content == NULL) {
virBufferFreeAndReset(&buffer);
return -1;
@@ -414,6 +435,7 @@ esxVI_CURL_Upload(esxVI_CURL *curl, const char *url, const char
*content)
virMutexLock(&curl->lock);
curl_easy_setopt(curl->handle, CURLOPT_URL, url);
+ curl_easy_setopt(curl->handle, CURLOPT_RANGE, NULL);
curl_easy_setopt(curl->handle, CURLOPT_READDATA, &content);
curl_easy_setopt(curl->handle, CURLOPT_UPLOAD, 1);
curl_easy_setopt(curl->handle, CURLOPT_INFILESIZE, strlen(content));
@@ -1231,6 +1253,7 @@ esxVI_Context_Execute(esxVI_Context *ctx, const char *methodName,
virMutexLock(&ctx->curl->lock);
curl_easy_setopt(ctx->curl->handle, CURLOPT_URL, ctx->url);
+ curl_easy_setopt(ctx->curl->handle, CURLOPT_RANGE, NULL);
curl_easy_setopt(ctx->curl->handle, CURLOPT_WRITEDATA, &buffer);
curl_easy_setopt(ctx->curl->handle, CURLOPT_UPLOAD, 0);
curl_easy_setopt(ctx->curl->handle, CURLOPT_POSTFIELDS, request);
diff --git a/src/esx/esx_vi.h b/src/esx/esx_vi.h
index 9560bd2..49b7ca2 100644
--- a/src/esx/esx_vi.h
+++ b/src/esx/esx_vi.h
@@ -167,7 +167,8 @@ struct _esxVI_CURL {
int esxVI_CURL_Alloc(esxVI_CURL **curl);
void esxVI_CURL_Free(esxVI_CURL **curl);
int esxVI_CURL_Connect(esxVI_CURL *curl, esxUtil_ParsedUri *parsedUri);
-int esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char **content);
+int esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char **content,
+ unsigned long long offset, unsigned long long *length);
int esxVI_CURL_Upload(esxVI_CURL *curl, const char *url, const char *content);
--
1.7.4.1
5:32 p.m.
New subject: [libvirt] [PATCH 2/3] esx: Extend esxVI_CURL_Download for partial downloads
On Mon, Jul 2, 2012 at 4:44 PM, Matthias Bolte
<matthias.bolte(a)googlemail.com> wrote:
---
src/esx/esx_driver.c | 2 +-
src/esx/esx_vi.c | 27 +++++++++++++++++++++++++--
src/esx/esx_vi.h | 3 ++-
3 files changed, 28 insertions(+), 4 deletions(-)
diff --git a/src/esx/esx_driver.c b/src/esx/esx_driver.c
index db2144c..95b9286 100644
--- a/src/esx/esx_driver.c
+++ b/src/esx/esx_driver.c
@@ -2802,7 +2802,7 @@ esxDomainGetXMLDesc(virDomainPtr domain, unsigned int flags)
url = virBufferContentAndReset(&buffer);
- if (esxVI_CURL_Download(priv->primary->curl, url, &vmx) < 0) {
+ if (esxVI_CURL_Download(priv->primary->curl, url, &vmx, 0, NULL) < 0)
{
goto cleanup;
}
diff --git a/src/esx/esx_vi.c b/src/esx/esx_vi.c
index 48718b6..3f8d745 100644
--- a/src/esx/esx_vi.c
+++ b/src/esx/esx_vi.c
@@ -355,8 +355,10 @@ esxVI_CURL_Connect(esxVI_CURL *curl, esxUtil_ParsedUri *parsedUri)
}
int
-esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char **content)
+esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char **content,
+ unsigned long long offset, unsigned long long *length)
{
+ char *range = NULL;
virBuffer buffer = VIR_BUFFER_INITIALIZER;
int responseCode = 0;
@@ -365,9 +367,22 @@ esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char
**content)
return -1;
}
+ if (length != NULL && *length > 0) {
+ if (virAsprintf(&range, "%llu-%llu", offset, offset + *length - 1)
< 0) {
+ virReportOOMError();
+ goto cleanup;
+ }
+ } else if (offset > 0) {
+ if (virAsprintf(&range, "%llu-", offset) < 0) {
+ virReportOOMError();
+ goto cleanup;
+ }
+ }
+
virMutexLock(&curl->lock);
curl_easy_setopt(curl->handle, CURLOPT_URL, url);
+ curl_easy_setopt(curl->handle, CURLOPT_RANGE, range);
curl_easy_setopt(curl->handle, CURLOPT_WRITEDATA, &buffer);
curl_easy_setopt(curl->handle, CURLOPT_UPLOAD, 0);
curl_easy_setopt(curl->handle, CURLOPT_HTTPGET, 1);
@@ -378,7 +393,7 @@ esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char
**content)
if (responseCode < 0) {
goto cleanup;
- } else if (responseCode != 200) {
+ } else if (responseCode != 200 && responseCode != 206) {
ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR,
_("HTTP response code %d for download from
'%s'"),
responseCode, url);
@@ -390,9 +405,15 @@ esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char
**content)
goto cleanup;
}
+ if (length != NULL) {
+ *length = virBufferUse(&buffer);
+ }
+
*content = virBufferContentAndReset(&buffer);
cleanup:
+ VIR_FREE(range);
+
if (*content == NULL) {
virBufferFreeAndReset(&buffer);
return -1;
@@ -414,6 +435,7 @@ esxVI_CURL_Upload(esxVI_CURL *curl, const char *url, const char
*content)
virMutexLock(&curl->lock);
curl_easy_setopt(curl->handle, CURLOPT_URL, url);
+ curl_easy_setopt(curl->handle, CURLOPT_RANGE, NULL);
curl_easy_setopt(curl->handle, CURLOPT_READDATA, &content);
curl_easy_setopt(curl->handle, CURLOPT_UPLOAD, 1);
curl_easy_setopt(curl->handle, CURLOPT_INFILESIZE, strlen(content));
@@ -1231,6 +1253,7 @@ esxVI_Context_Execute(esxVI_Context *ctx, const char *methodName,
virMutexLock(&ctx->curl->lock);
curl_easy_setopt(ctx->curl->handle, CURLOPT_URL, ctx->url);
+ curl_easy_setopt(ctx->curl->handle, CURLOPT_RANGE, NULL);
curl_easy_setopt(ctx->curl->handle, CURLOPT_WRITEDATA, &buffer);
curl_easy_setopt(ctx->curl->handle, CURLOPT_UPLOAD, 0);
curl_easy_setopt(ctx->curl->handle, CURLOPT_POSTFIELDS, request);
diff --git a/src/esx/esx_vi.h b/src/esx/esx_vi.h
index 9560bd2..49b7ca2 100644
--- a/src/esx/esx_vi.h
+++ b/src/esx/esx_vi.h
@@ -167,7 +167,8 @@ struct _esxVI_CURL {
int esxVI_CURL_Alloc(esxVI_CURL **curl);
void esxVI_CURL_Free(esxVI_CURL **curl);
int esxVI_CURL_Connect(esxVI_CURL *curl, esxUtil_ParsedUri *parsedUri);
-int esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char **content);
+int esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char **content,
+ unsigned long long offset, unsigned long long *length);
int esxVI_CURL_Upload(esxVI_CURL *curl, const char *url, const char *content);
--
1.7.4.1
This patch technically allows you to operate with ranges larger than
an unsigned int due to the use of unsigned long long. But virBuffer
can only work with unsigned ints. So there's a very real possibility
to overflow the type here when performing a download over the 4GB
mark. Since the content can only be in 1 file, this is a very real
possibility.
--
Doug Goldstein
3:25 p.m.
New subject: [libvirt] [PATCH 2/3] esx: Extend esxVI_CURL_Download for partial downloads
2012/7/3 Doug Goldstein <cardoe(a)gentoo.org>:
On Mon, Jul 2, 2012 at 4:44 PM, Matthias Bolte
<matthias.bolte(a)googlemail.com> wrote:
> ---
> src/esx/esx_driver.c | 2 +-
> src/esx/esx_vi.c | 27 +++++++++++++++++++++++++--
> src/esx/esx_vi.h | 3 ++-
> 3 files changed, 28 insertions(+), 4 deletions(-)
>
> diff --git a/src/esx/esx_driver.c b/src/esx/esx_driver.c
> index db2144c..95b9286 100644
> --- a/src/esx/esx_driver.c
> +++ b/src/esx/esx_driver.c
> @@ -2802,7 +2802,7 @@ esxDomainGetXMLDesc(virDomainPtr domain, unsigned int flags)
>
> url = virBufferContentAndReset(&buffer);
>
> - if (esxVI_CURL_Download(priv->primary->curl, url, &vmx) < 0) {
> + if (esxVI_CURL_Download(priv->primary->curl, url, &vmx, 0, NULL) <
0) {
> goto cleanup;
> }
>
> diff --git a/src/esx/esx_vi.c b/src/esx/esx_vi.c
> index 48718b6..3f8d745 100644
> --- a/src/esx/esx_vi.c
> +++ b/src/esx/esx_vi.c
> @@ -355,8 +355,10 @@ esxVI_CURL_Connect(esxVI_CURL *curl, esxUtil_ParsedUri
*parsedUri)
> }
>
> int
> -esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char **content)
> +esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char **content,
> + unsigned long long offset, unsigned long long *length)
> {
> + char *range = NULL;
> virBuffer buffer = VIR_BUFFER_INITIALIZER;
> int responseCode = 0;
>
> @@ -365,9 +367,22 @@ esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char
**content)
> return -1;
> }
>
> + if (length != NULL && *length > 0) {
> + if (virAsprintf(&range, "%llu-%llu", offset, offset + *length
- 1) < 0) {
> + virReportOOMError();
> + goto cleanup;
> + }
> + } else if (offset > 0) {
> + if (virAsprintf(&range, "%llu-", offset) < 0) {
> + virReportOOMError();
> + goto cleanup;
> + }
> + }
> +
> virMutexLock(&curl->lock);
>
> curl_easy_setopt(curl->handle, CURLOPT_URL, url);
> + curl_easy_setopt(curl->handle, CURLOPT_RANGE, range);
> curl_easy_setopt(curl->handle, CURLOPT_WRITEDATA, &buffer);
> curl_easy_setopt(curl->handle, CURLOPT_UPLOAD, 0);
> curl_easy_setopt(curl->handle, CURLOPT_HTTPGET, 1);
> @@ -378,7 +393,7 @@ esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char
**content)
>
> if (responseCode < 0) {
> goto cleanup;
> - } else if (responseCode != 200) {
> + } else if (responseCode != 200 && responseCode != 206) {
> ESX_VI_ERROR(VIR_ERR_INTERNAL_ERROR,
> _("HTTP response code %d for download from
'%s'"),
> responseCode, url);
> @@ -390,9 +405,15 @@ esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char
**content)
> goto cleanup;
> }
>
> + if (length != NULL) {
> + *length = virBufferUse(&buffer);
> + }
> +
> *content = virBufferContentAndReset(&buffer);
>
> cleanup:
> + VIR_FREE(range);
> +
> if (*content == NULL) {
> virBufferFreeAndReset(&buffer);
> return -1;
> @@ -414,6 +435,7 @@ esxVI_CURL_Upload(esxVI_CURL *curl, const char *url, const char
*content)
> virMutexLock(&curl->lock);
>
> curl_easy_setopt(curl->handle, CURLOPT_URL, url);
> + curl_easy_setopt(curl->handle, CURLOPT_RANGE, NULL);
> curl_easy_setopt(curl->handle, CURLOPT_READDATA, &content);
> curl_easy_setopt(curl->handle, CURLOPT_UPLOAD, 1);
> curl_easy_setopt(curl->handle, CURLOPT_INFILESIZE, strlen(content));
> @@ -1231,6 +1253,7 @@ esxVI_Context_Execute(esxVI_Context *ctx, const char
*methodName,
> virMutexLock(&ctx->curl->lock);
>
> curl_easy_setopt(ctx->curl->handle, CURLOPT_URL, ctx->url);
> + curl_easy_setopt(ctx->curl->handle, CURLOPT_RANGE, NULL);
> curl_easy_setopt(ctx->curl->handle, CURLOPT_WRITEDATA, &buffer);
> curl_easy_setopt(ctx->curl->handle, CURLOPT_UPLOAD, 0);
> curl_easy_setopt(ctx->curl->handle, CURLOPT_POSTFIELDS, request);
> diff --git a/src/esx/esx_vi.h b/src/esx/esx_vi.h
> index 9560bd2..49b7ca2 100644
> --- a/src/esx/esx_vi.h
> +++ b/src/esx/esx_vi.h
> @@ -167,7 +167,8 @@ struct _esxVI_CURL {
> int esxVI_CURL_Alloc(esxVI_CURL **curl);
> void esxVI_CURL_Free(esxVI_CURL **curl);
> int esxVI_CURL_Connect(esxVI_CURL *curl, esxUtil_ParsedUri *parsedUri);
> -int esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char **content);
> +int esxVI_CURL_Download(esxVI_CURL *curl, const char *url, char **content,
> + unsigned long long offset, unsigned long long *length);
> int esxVI_CURL_Upload(esxVI_CURL *curl, const char *url, const char *content);
>
>
> --
> 1.7.4.1
This patch technically allows you to operate with ranges larger than
an unsigned int due to the use of unsigned long long. But virBuffer
can only work with unsigned ints. So there's a very real possibility
to overflow the type here when performing a download over the 4GB
mark. Since the content can only be in 1 file, this is a very real
possibility.
You're right. But the problem was already there before this patch.
I'll address it in a v2.
--
Matthias Bolte
http://photron.blogspot.com
4:44 p.m.
New subject: [libvirt] [PATCH 3/3] esx: Add volume upload and download to the storage driver
This requires new flags for the public API functions, because a
VMDK volume can have metadata and content in separate files. This
is not yet supported by the libvirt storage API as it currently
assumes that a single file represents a volume. At least it does
so in case of the volume upload and download functions.
For example, the default VMDK structure created by an ESX server
is a volume.vmdk file (containing metadata in plaintext key/value
form) and a volume-flat.vmdk (containing the actual content of the
volume). The volume.vmdk contains the name of its content file.
The volume.vmdk is the one that represents the volume to the ESX
server and is the one that is specified in datastore paths. The
volume-flat.vmdk is basically an implementation detail here.
There are also one-file VMDK volumes, as created by qemu-img when
targeting the VMDK format. Such VMDK volumes starts with a 'KDMV'
magic and contain metadata and content.
The new METADATA and CONTENT flags allow a caller to tell the ESX
storage driver which part of a volume should be up- or downloaded
in case of a two-file VMDK. In case of a one-file VMDK both or no
flags must be given. In case of non-VMDK volumes such as ISO or
floppy images no flags must be specified.
If the CONTENT flag is given the the ESX driver reads the metadata
file in the datastore to figure out the name of the content file.
This means that for two-file VMDKs the matadata file has to be
uploaded first.
The libcurl based stream driver cannot use a libcurl easy handle
alone because curl_easy_perform would do the whole transfer before
it returns. But there is no place in the stream handling concept
that would allow for such a call to be made. The stream is driven
by esxStream(Send|Recv) which is probably called multiple times to
send/receive the stream in chunks. Therefore, a libcurl multi handle
is used that allows to perform the data transfer in chunks and also
allows to support non-blocking operations. Although, the stream
callbacks for non-blocking operations are not implemented yet.
---
include/libvirt/libvirt.h.in | 14 +
src/Makefile.am | 1 +
src/esx/esx_storage_driver.c | 268 ++++++++++++++++++
src/esx/esx_stream.c | 610 ++++++++++++++++++++++++++++++++++++++++++
src/esx/esx_stream.h | 33 +++
src/esx/esx_util.c | 82 ++++++
src/esx/esx_util.h | 2 +
src/esx/esx_vi.c | 29 ++
src/esx/esx_vi.h | 2 +
tools/virsh.c | 22 ++-
10 files changed, 1061 insertions(+), 2 deletions(-)
create mode 100644 src/esx/esx_stream.c
create mode 100644 src/esx/esx_stream.h
diff --git a/include/libvirt/libvirt.h.in b/include/libvirt/libvirt.h.in
index 6e8d5dd..d11ed68 100644
--- a/include/libvirt/libvirt.h.in
+++ b/include/libvirt/libvirt.h.in
@@ -2562,11 +2562,25 @@ virStorageVolPtr virStorageVolCreateXMLFrom
(virStoragePoolPtr pool,
const char *xmldesc,
virStorageVolPtr clonevol,
unsigned int flags);
+
+typedef enum {
+ VIR_STORAGE_VOL_DOWNLOAD_DEFAULT = 0, /* default behavior */
+ VIR_STORAGE_VOL_DOWNLOAD_METADATA = 1 << 0, /* download metadata only */
+ VIR_STORAGE_VOL_DOWNLOAD_CONTENT = 1 << 1, /* download content only */
+} virStorageVolDownloadFlags;
+
int virStorageVolDownload (virStorageVolPtr vol,
virStreamPtr stream,
unsigned long long offset,
unsigned long long length,
unsigned int flags);
+
+typedef enum {
+ VIR_STORAGE_VOL_UPLOAD_DEFAULT = 0, /* default behavior */
+ VIR_STORAGE_VOL_UPLOAD_METADATA = 1 << 0, /* upload metadata only */
+ VIR_STORAGE_VOL_UPLOAD_CONTENT = 1 << 1, /* upload content only */
+} virStorageVolUploadFlags;
+
int virStorageVolUpload (virStorageVolPtr vol,
virStreamPtr stream,
unsigned long long offset,
diff --git a/src/Makefile.am b/src/Makefile.am
index 2309984..db8fb91 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -430,6 +430,7 @@ ESX_DRIVER_SOURCES = \
esx/esx_secret_driver.c esx/esx_secret_driver.h \
esx/esx_nwfilter_driver.c esx/esx_nwfilter_driver.h \
esx/esx_util.c esx/esx_util.h \
+ esx/esx_stream.c esx/esx_stream.h \
esx/esx_vi.c esx/esx_vi.h \
esx/esx_vi_methods.c esx/esx_vi_methods.h \
esx/esx_vi_types.c esx/esx_vi_types.h
diff --git a/src/esx/esx_storage_driver.c b/src/esx/esx_storage_driver.c
index 9b64891..40f9471 100644
--- a/src/esx/esx_storage_driver.c
+++ b/src/esx/esx_storage_driver.c
@@ -38,6 +38,7 @@
#include "esx_vi.h"
#include "esx_vi_methods.h"
#include "esx_util.h"
+#include "esx_stream.h"
#define VIR_FROM_THIS VIR_FROM_ESX
@@ -1381,6 +1382,271 @@ esxStorageVolumeCreateXMLFrom(virStoragePoolPtr pool, const char
*xmldesc,
static int
+esxStorageVolumeDownload(virStorageVolPtr volume, virStreamPtr stream,
+ unsigned long long offset,
+ unsigned long long length,
+ unsigned int flags)
+{
+ int result = -1;
+ esxPrivate *priv = volume->conn->storagePrivateData;
+ virBuffer buffer = VIR_BUFFER_INITIALIZER;
+ char *url = NULL;
+ char *magic = NULL;
+ unsigned long long magic_length = 4;
+ char *metadata = NULL;
+ unsigned long long metadata_length = 0;
+ char *contentName = NULL;
+ char *directoryName = NULL;
+
+ virCheckFlags(VIR_STORAGE_VOL_DOWNLOAD_METADATA |
+ VIR_STORAGE_VOL_DOWNLOAD_CONTENT, -1);
+
+ /* Build URL */
+ virBufferAsprintf(&buffer, "%s://%s:%d/folder/",
priv->parsedUri->transport,
+ volume->conn->uri->server,
volume->conn->uri->port);
+ virBufferURIEncodeString(&buffer, volume->name);
+ virBufferAddLit(&buffer, "?dcPath=");
+ virBufferURIEncodeString(&buffer, priv->primary->datacenterPath);
+ virBufferAddLit(&buffer, "&dsName=");
+ virBufferURIEncodeString(&buffer, volume->pool);
+
+ if (virBufferError(&buffer)) {
+ virReportOOMError();
+ goto cleanup;
+ }
+
+ url = virBufferContentAndReset(&buffer);
+
+ if (virFileHasSuffix(volume->name, ".vmdk")) {
+ /* Download VMDK magic to check VMDK type */
+ if (esxVI_CURL_Download(priv->primary->curl, url, &magic, 0,
+ &magic_length) < 0) {
+ goto cleanup;
+ }
+
+ if (magic_length < 4) {
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR, "%s", _("Volume is too
small"));
+ goto cleanup;
+ }
+
+ if (STREQLEN(magic, "KDMV", 4)) {
+ /* It's a one-file VMDK with metadata and content */
+ if (flags != VIR_STORAGE_VOL_DOWNLOAD_DEFAULT &&
+ flags != (VIR_STORAGE_VOL_DOWNLOAD_METADATA |
+ VIR_STORAGE_VOL_DOWNLOAD_CONTENT)) {
+ ESX_ERROR(VIR_ERR_INVALID_ARG, "%s",
+ _("Non or both of metadata and content flag is required
for this volume"));
+ goto cleanup;
+ }
+ } else {
+ /* It's a two-file VMDK, this is the metadata file */
+ if (flags != VIR_STORAGE_VOL_DOWNLOAD_METADATA &&
+ flags != VIR_STORAGE_VOL_DOWNLOAD_CONTENT) {
+ ESX_ERROR(VIR_ERR_INVALID_ARG, "%s",
+ _("Either metadata or content flag is required for this
volume"));
+ goto cleanup;
+ }
+
+ if (flags == VIR_STORAGE_VOL_DOWNLOAD_CONTENT) {
+ /* Download full metadata */
+ metadata_length = 0;
+
+ if (esxVI_CURL_Download(priv->primary->curl, url, &metadata,
0,
+ &metadata_length) < 0) {
+ goto cleanup;
+ }
+
+ contentName = esxUtil_ParseVMDKMetadata(metadata, metadata_length);
+
+ if (contentName == NULL) {
+ goto cleanup;
+ }
+
+ /* Build content URL */
+ virBufferAsprintf(&buffer, "%s://%s:%d/folder/",
+ priv->parsedUri->transport,
+ volume->conn->uri->server,
+ volume->conn->uri->port);
+
+ directoryName = strrchr(volume->name, '/');
+
+ if (directoryName != NULL) {
+ directoryName = strndup(volume->name,
+ directoryName - volume->name + 1);
+
+ if (directoryName == NULL) {
+ virReportOOMError();
+ goto cleanup;
+ }
+
+ virBufferURIEncodeString(&buffer, directoryName);
+ }
+
+ virBufferURIEncodeString(&buffer, contentName);
+ virBufferAddLit(&buffer, "?dcPath=");
+ virBufferURIEncodeString(&buffer,
priv->primary->datacenterPath);
+ virBufferAddLit(&buffer, "&dsName=");
+ virBufferURIEncodeString(&buffer, volume->pool);
+
+ if (virBufferError(&buffer)) {
+ virReportOOMError();
+ goto cleanup;
+ }
+
+ VIR_FREE(url);
+ url = virBufferContentAndReset(&buffer);
+ }
+ }
+ } else {
+ if (flags != VIR_STORAGE_VOL_DOWNLOAD_DEFAULT) {
+ ESX_ERROR(VIR_ERR_INVALID_ARG, "%s",
+ _("Invalid flag combination for this volume"));
+ goto cleanup;
+ }
+ }
+
+ if (esxStreamOpenDownload(stream, priv, url, offset, length) < 0) {
+ goto cleanup;
+ }
+
+ result = 0;
+
+ cleanup:
+ if (url == NULL) {
+ virBufferFreeAndReset(&buffer);
+ }
+
+ VIR_FREE(url);
+ VIR_FREE(magic);
+ VIR_FREE(metadata);
+ VIR_FREE(contentName);
+ VIR_FREE(directoryName);
+
+ return result;
+}
+
+
+
+static int
+esxStorageVolumeUpload(virStorageVolPtr volume, virStreamPtr stream,
+ unsigned long long offset,
+ unsigned long long length,
+ unsigned int flags)
+{
+ int result = -1;
+ esxPrivate *priv = volume->conn->storagePrivateData;
+ virBuffer buffer = VIR_BUFFER_INITIALIZER;
+ char *url = NULL;
+ char *metadata = NULL;
+ unsigned long long metadata_length = 0;
+ char *contentName = NULL;
+ char *directoryName = NULL;
+
+ virCheckFlags(VIR_STORAGE_VOL_UPLOAD_METADATA |
+ VIR_STORAGE_VOL_UPLOAD_CONTENT, -1);
+
+ if (offset != 0 || length != 0) {
+ ESX_ERROR(VIR_ERR_INVALID_ARG, "%s",
+ _("Partial volume upload is not supported"));
+ return -1;
+ }
+
+ /* Build URL */
+ virBufferAsprintf(&buffer, "%s://%s:%d/folder/",
priv->parsedUri->transport,
+ volume->conn->uri->server,
volume->conn->uri->port);
+ virBufferURIEncodeString(&buffer, volume->name);
+ virBufferAddLit(&buffer, "?dcPath=");
+ virBufferURIEncodeString(&buffer, priv->primary->datacenterPath);
+ virBufferAddLit(&buffer, "&dsName=");
+ virBufferURIEncodeString(&buffer, volume->pool);
+
+ if (virBufferError(&buffer)) {
+ virReportOOMError();
+ goto cleanup;
+ }
+
+ url = virBufferContentAndReset(&buffer);
+
+ if (virFileHasSuffix(volume->name, ".vmdk")) {
+ if (flags == VIR_STORAGE_VOL_UPLOAD_CONTENT) {
+ /* Download metadata */
+ metadata_length = 0;
+
+ if (esxVI_CURL_Download(priv->primary->curl, url, &metadata, 0,
+ &metadata_length) < 0) {
+ goto cleanup;
+ }
+
+ contentName = esxUtil_ParseVMDKMetadata(metadata, metadata_length);
+
+ if (contentName == NULL) {
+ goto cleanup;
+ }
+
+ /* Build content URL */
+ virBufferAsprintf(&buffer, "%s://%s:%d/folder/",
+ priv->parsedUri->transport,
+ volume->conn->uri->server,
+ volume->conn->uri->port);
+
+ directoryName = strrchr(volume->name, '/');
+
+ if (directoryName != NULL) {
+ directoryName = strndup(volume->name,
+ directoryName - volume->name + 1);
+
+ if (directoryName == NULL) {
+ virReportOOMError();
+ goto cleanup;
+ }
+
+ virBufferURIEncodeString(&buffer, directoryName);
+ }
+
+ virBufferURIEncodeString(&buffer, contentName);
+ virBufferAddLit(&buffer, "?dcPath=");
+ virBufferURIEncodeString(&buffer, priv->primary->datacenterPath);
+ virBufferAddLit(&buffer, "&dsName=");
+ virBufferURIEncodeString(&buffer, volume->pool);
+
+ if (virBufferError(&buffer)) {
+ virReportOOMError();
+ goto cleanup;
+ }
+
+ VIR_FREE(url);
+ url = virBufferContentAndReset(&buffer);
+ }
+ } else {
+ if (flags != VIR_STORAGE_VOL_UPLOAD_DEFAULT) {
+ ESX_ERROR(VIR_ERR_INVALID_ARG, "%s",
+ _("Invalid flag combination for this volume"));
+ goto cleanup;
+ }
+ }
+
+ if (esxStreamOpenUpload(stream, priv, url) < 0) {
+ goto cleanup;
+ }
+
+ result = 0;
+
+ cleanup:
+ if (url == NULL) {
+ virBufferFreeAndReset(&buffer);
+ }
+
+ VIR_FREE(url);
+ VIR_FREE(metadata);
+ VIR_FREE(contentName);
+ VIR_FREE(directoryName);
+
+ return result;
+}
+
+
+
+static int
esxStorageVolumeDelete(virStorageVolPtr volume, unsigned int flags)
{
int result = -1;
@@ -1668,6 +1934,8 @@ static virStorageDriver esxStorageDriver = {
.volLookupByPath = esxStorageVolumeLookupByPath, /* 0.8.4 */
.volCreateXML = esxStorageVolumeCreateXML, /* 0.8.4 */
.volCreateXMLFrom = esxStorageVolumeCreateXMLFrom, /* 0.8.7 */
+ .volDownload = esxStorageVolumeDownload, /* 0.9.14 */
+ .volUpload = esxStorageVolumeUpload, /* 0.9.14 */
.volDelete = esxStorageVolumeDelete, /* 0.8.7 */
.volWipe = esxStorageVolumeWipe, /* 0.8.7 */
.volGetInfo = esxStorageVolumeGetInfo, /* 0.8.4 */
diff --git a/src/esx/esx_stream.c b/src/esx/esx_stream.c
new file mode 100644
index 0000000..fd84d14
--- /dev/null
+++ b/src/esx/esx_stream.c
@@ -0,0 +1,610 @@
+
+/*
+ * esx_stream.c: libcurl based stream driver
+ *
+ * Copyright (C) 2012 Matthias Bolte <matthias.bolte(a)googlemail.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ */
+
+#include <config.h>
+
+#include "internal.h"
+#include "datatypes.h"
+#include "util.h"
+#include "memory.h"
+#include "logging.h"
+#include "esx_stream.h"
+
+#define VIR_FROM_THIS VIR_FROM_ESX
+
+enum _esxStreamMode {
+ ESX_STREAM_MODE_UPLOAD = 1,
+ ESX_STREAM_MODE_DOWNLOAD = 2
+};
+
+typedef struct _esxStreamPrivate esxStreamPrivate;
+typedef enum _esxStreamMode esxStreamMode;
+
+struct _esxStreamPrivate {
+ esxVI_CURL *curl;
+ int mode;
+
+ /* Backlog of downloaded data that has not been esxStreamRecv'ed yet */
+ char *backlog;
+ size_t backlog_size;
+ size_t backlog_used;
+
+ /* Buffer given to esxStream(Send|Recv) to (read|write) data (from|to) */
+ char *buffer;
+ size_t buffer_size;
+ size_t buffer_used;
+};
+
+static size_t
+esxVI_CURL_ReadStream(char *output, size_t size, size_t nmemb, void *userdata)
+{
+ esxStreamPrivate *priv = userdata;
+ size_t output_size = size * nmemb;
+ size_t output_used = 0;
+
+ if (output_size > priv->buffer_used) {
+ output_used = priv->buffer_used;
+ } else {
+ output_used = output_size;
+ }
+
+ memcpy(output, priv->buffer + priv->buffer_size - priv->buffer_used,
+ output_used);
+
+ priv->buffer_used -= output_used;
+
+ return output_used;
+}
+
+static size_t
+esxVI_CURL_WriteStream(char *input, size_t size, size_t nmemb, void *userdata)
+{
+ esxStreamPrivate *priv = userdata;
+ size_t input_size = size * nmemb;
+ size_t input_used = priv->buffer_size - priv->buffer_used;
+
+ if (input_used > input_size) {
+ input_used = input_size;
+ }
+
+ /* Fill buffer */
+ memcpy(priv->buffer + priv->buffer_used, input, input_used);
+ priv->buffer_used += input_used;
+
+ /* Move rest to backlog */
+ if (input_size > input_used) {
+ size_t input_remaining = input_size - input_used;
+ size_t backlog_remaining = priv->backlog_size - priv->backlog_used;
+
+ if (priv->backlog == NULL) {
+ priv->backlog_size = input_remaining;
+ priv->backlog_used = 0;
+
+ if (VIR_ALLOC_N(priv->backlog, priv->backlog_size) < 0) {
+ virReportOOMError();
+ return 0;
+ }
+ } else if (input_remaining > backlog_remaining) {
+ priv->backlog_size += input_remaining - backlog_remaining;
+
+ if (VIR_REALLOC_N(priv->backlog, priv->backlog_size) < 0) {
+ virReportOOMError();
+ return 0;
+ }
+ }
+
+ memcpy(priv->backlog + priv->backlog_used, input + input_used,
+ input_remaining);
+
+ priv->backlog_used += input_remaining;
+ }
+
+ return input_size;
+}
+
+/* Return -1 on error and 1 when it's done */
+static int
+esxStreamTransferNonBlocking(esxStreamPrivate *priv)
+{
+ int runningHandles = 0;
+ CURLMcode multiErrorCode;
+ CURLcode errorCode;
+
+ /* Perform transfer */
+ do {
+ multiErrorCode = curl_multi_perform(priv->curl->multi->handle,
+ &runningHandles);
+ } while (multiErrorCode == CURLM_CALL_MULTI_PERFORM);
+
+ if (multiErrorCode != CURLM_OK) {
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR,
+ _("Could not transfer data: %s (%d)"),
+ curl_multi_strerror(multiErrorCode), multiErrorCode);
+ return -1;
+ }
+
+ /* Check status */
+ if (runningHandles == 0) {
+ long responseCode = 0;
+ int status = esxVI_MultiCURL_CheckFirstMessage(priv->curl->multi,
+ &responseCode,
+ &errorCode);
+
+ if (status == 0) {
+ return 1;
+ }
+
+ if (status < 0) {
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR,
+ _("Could not complete transfer: %s (%d)"),
+ curl_easy_strerror(errorCode), errorCode);
+ return -1;
+ }
+
+ if (responseCode != 200 && responseCode != 201) {
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR,
+ _("Unexpected HTTP response code %lu"),
+ responseCode);
+ return -1;
+ }
+ }
+
+ return 1;
+}
+
+/* Return -1 on error, 0 when it needs to be called again, and 1 when it 's done */
+static int
+esxStreamTransferBlocking(esxStreamPrivate *priv)
+{
+ int runningHandles = 0;
+ CURLMcode multiErrorCode;
+ fd_set fdread;
+ fd_set fdwrite;
+ fd_set fdexcep;
+ int maxfd = 0;
+ long timeout = -1;
+ struct timeval tv_timeout;
+ int rc;
+ CURLcode errorCode;
+
+ /* Select */
+ curl_multi_timeout(priv->curl->multi->handle, &timeout);
+
+ if (timeout < 0) {
+ timeout = 1000; /* default to 1 sec timeout */
+ }
+
+ tv_timeout.tv_sec = 0;
+ tv_timeout.tv_usec = timeout * 1000;
+
+ while (tv_timeout.tv_usec >= 1000000) {
+ tv_timeout.tv_sec += 1;
+ tv_timeout.tv_usec -= 1000000;
+ }
+
+ do {
+ rc = 0;
+
+ FD_ZERO(&fdread);
+ FD_ZERO(&fdwrite);
+ FD_ZERO(&fdexcep);
+
+ curl_multi_fdset(priv->curl->multi->handle,
+ &fdread, &fdwrite, &fdexcep, &maxfd);
+
+ if (maxfd >= 0) {
+ rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep,
&tv_timeout);
+ }
+ } while (rc < 0 && errno == EINTR);
+
+ if (rc < 0) {
+ virReportSystemError(errno, "%s",
+ _("Could not block on transfer"));
+ return -1;
+ }
+
+ /* Perform transfer */
+ do {
+ multiErrorCode = curl_multi_perform(priv->curl->multi->handle,
+ &runningHandles);
+ } while (multiErrorCode == CURLM_CALL_MULTI_PERFORM);
+
+ if (multiErrorCode != CURLM_OK) {
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR,
+ _("Could not transfer data: %s (%d)"),
+ curl_multi_strerror(multiErrorCode), multiErrorCode);
+ return -1;
+ }
+
+ /* Check status */
+ if (runningHandles == 0) {
+ long responseCode = 0;
+ int status = esxVI_MultiCURL_CheckFirstMessage(priv->curl->multi,
+ &responseCode,
+ &errorCode);
+
+ if (status == 0) {
+ return 1;
+ }
+
+ if (status < 0) {
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR,
+ _("Could not complete transfer: %s (%d)"),
+ curl_easy_strerror(errorCode), errorCode);
+ return -1;
+ }
+
+ if (responseCode != 200 && responseCode != 206) {
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR,
+ _("Unexpected HTTP response code %lu"),
+ responseCode);
+ return -1;
+ }
+
+ return 1;
+ }
+
+ return 0;
+}
+
+static int
+esxStreamSend(virStreamPtr stream, const char *data, size_t nbytes)
+{
+ int result = -1;
+ esxStreamPrivate *priv = stream->privateData;
+ int runningHandles = 0;
+ CURLMcode multiErrorCode;
+
+ if (nbytes < 1) {
+ return 0;
+ }
+
+ if (priv == NULL) {
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR, "%s", _("Stream is not
open"));
+ return -1;
+ }
+
+ if (priv->mode != ESX_STREAM_MODE_UPLOAD) {
+ ESX_ERROR(VIR_ERR_INVALID_ARG, "%s", _("Not an upload
stream"));
+ return -1;
+ }
+
+ virMutexLock(&priv->curl->lock);
+
+ priv->buffer = (char *)data;
+ priv->buffer_size = nbytes;
+ priv->buffer_used = nbytes;
+
+ if (stream->flags & VIR_STREAM_NONBLOCK) {
+ if (esxStreamTransferNonBlocking(priv) < 0) {
+ goto cleanup;
+ }
+
+ if (priv->buffer_used < priv->buffer_size) {
+ result = priv->buffer_size - priv->buffer_used;
+ } else {
+ result = -2;
+ }
+ } else /* blocking */ {
+ do {
+ multiErrorCode = curl_multi_perform(priv->curl->multi->handle,
+ &runningHandles);
+ } while (multiErrorCode == CURLM_CALL_MULTI_PERFORM);
+
+ if (multiErrorCode != CURLM_OK) {
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR,
+ _("Could not transfer data: %s (%d)"),
+ curl_multi_strerror(multiErrorCode), multiErrorCode);
+ goto cleanup;
+ }
+
+ do {
+ int status = esxStreamTransferBlocking(priv);
+
+ if (status < 0) {
+ goto cleanup;
+ }
+
+ if (status > 0) {
+ break;
+ }
+ } while (priv->buffer_used > 0);
+
+ result = priv->buffer_size - priv->buffer_used;
+ }
+
+ cleanup:
+ virMutexUnlock(&priv->curl->lock);
+
+ return result;
+}
+
+static int
+esxStreamRecv(virStreamPtr stream, char *data, size_t nbytes)
+{
+ int result = -1;
+ esxStreamPrivate *priv = stream->privateData;
+ int runningHandles = 0;
+ CURLMcode multiErrorCode;
+
+ if (nbytes < 1) {
+ return 0;
+ }
+
+ if (priv == NULL) {
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR, "%s", _("Stream is not
open"));
+ return -1;
+ }
+
+ if (priv->mode != ESX_STREAM_MODE_DOWNLOAD) {
+ ESX_ERROR(VIR_ERR_INVALID_ARG, "%s", _("Not a download
stream"));
+ return -1;
+ }
+
+ virMutexLock(&priv->curl->lock);
+
+ priv->buffer = data;
+ priv->buffer_size = nbytes;
+ priv->buffer_used = 0;
+
+ if (priv->backlog_used > 0) {
+ if (priv->buffer_size > priv->backlog_used) {
+ priv->buffer_used = priv->backlog_used;
+ } else {
+ priv->buffer_used = priv->buffer_size;
+ }
+
+ memcpy(priv->buffer, priv->backlog, priv->buffer_used);
+ memmove(priv->backlog, priv->backlog + priv->buffer_used,
+ priv->backlog_used - priv->buffer_used);
+
+ priv->backlog_used -= priv->buffer_used;
+
+ result = priv->buffer_used;
+ } else if (stream->flags & VIR_STREAM_NONBLOCK) {
+ if (esxStreamTransferNonBlocking(priv) < 0) {
+ goto cleanup;
+ }
+
+ if (priv->buffer_used > 0) {
+ result = priv->buffer_used;
+ } else {
+ result = -2;
+ }
+ } else /* blocking */ {
+ do {
+ multiErrorCode = curl_multi_perform(priv->curl->multi->handle,
+ &runningHandles);
+ } while (multiErrorCode == CURLM_CALL_MULTI_PERFORM);
+
+ if (multiErrorCode != CURLM_OK) {
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR,
+ _("curl_multi_perform() returned an error: %s (%d)"),
+ curl_multi_strerror(multiErrorCode), multiErrorCode);
+ goto cleanup;
+ }
+
+ do {
+ int status = esxStreamTransferBlocking(priv);
+
+ if (status < 0) {
+ goto cleanup;
+ }
+
+ if (status > 0) {
+ break;
+ }
+ } while (priv->buffer_used < priv->buffer_size);
+
+ result = priv->buffer_used;
+ }
+
+ cleanup:
+ virMutexUnlock(&priv->curl->lock);
+
+ return result;
+}
+
+static void
+esxFreeStreamPrivate(esxStreamPrivate **priv)
+{
+ if (priv == NULL || *priv == NULL) {
+ return;
+ }
+
+ esxVI_CURL_Free(&(*priv)->curl);
+ VIR_FREE((*priv)->backlog);
+ VIR_FREE(*priv);
+}
+
+static int
+esxStreamClose(virStreamPtr stream, bool finish)
+{
+ int result = 0;
+ esxStreamPrivate *priv = stream->privateData;
+
+ if (priv == NULL) {
+ return 0;
+ }
+
+ virMutexLock(&priv->curl->lock);
+
+ if (finish && priv->backlog_used > 0) {
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Stream has untransferred data left"));
+ result = -1;
+ }
+
+ stream->privateData = NULL;
+
+ virMutexUnlock(&priv->curl->lock);
+
+ esxFreeStreamPrivate(&priv);
+
+ return result;
+}
+
+static int
+esxStreamFinish(virStreamPtr stream)
+{
+ return esxStreamClose(stream, true);
+}
+
+static int
+esxStreamAbort(virStreamPtr stream)
+{
+ return esxStreamClose(stream, false);
+}
+
+virStreamDriver esxStreamDriver = {
+ .streamSend = esxStreamSend,
+ .streamRecv = esxStreamRecv,
+ /* FIXME: streamAddCallback missing */
+ /* FIXME: streamUpdateCallback missing */
+ /* FIXME: streamRemoveCallback missing */
+ .streamFinish = esxStreamFinish,
+ .streamAbort = esxStreamAbort,
+};
+
+static int
+esxStreamOpen(virStreamPtr stream, esxPrivate *priv, const char *url,
+ unsigned long long offset, unsigned long long length)
+{
+ int result = -1;
+ esxStreamPrivate *streamPriv;
+ char *range = NULL;
+ char *userpwd = NULL;
+ esxVI_MultiCURL *multi = NULL;
+
+ /* Create stream private data */
+ if (VIR_ALLOC(streamPriv) < 0) {
+ virReportOOMError();
+ return -1;
+ }
+
+ if (length > 0) {
+ if (virAsprintf(&range, "%llu-%llu", offset, offset + length - 1)
< 0) {
+ virReportOOMError();
+ goto cleanup;
+ }
+ } else if (offset > 0) {
+ if (virAsprintf(&range, "%llu-", offset) < 0) {
+ virReportOOMError();
+ goto cleanup;
+ }
+ }
+
+ /*
+ * Initialize CURL handle. We cannot use an easy handle alone here because
+ * curl_easy_perform whould do the whole transfer before it returns. But
+ * there is no place in the stream handling that would allow for such a
+ * call. The stream is driven by esxStream(Send|Recv) that is probably
+ * called multiple times to send/receive the stream in chunks. Therefore,
+ * we use the multi handle here that allows to perform the (up|down)load in
+ * chunks and also allows to support non-blocking operations.
+ */
+ if (esxVI_CURL_Alloc(&streamPriv->curl) < 0 ||
+ esxVI_CURL_Connect(streamPriv->curl, priv->parsedUri) < 0 ||
+ esxVI_MultiCURL_Alloc(&multi) < 0 ||
+ esxVI_MultiCURL_Add(multi, streamPriv->curl) < 0) {
+ goto cleanup;
+ }
+
+ curl_easy_setopt(streamPriv->curl->handle, CURLOPT_URL, url);
+ curl_easy_setopt(streamPriv->curl->handle, CURLOPT_RANGE, range);
+
+#if LIBCURL_VERSION_NUM >= 0x071301 /* 7.19.1 */
+ curl_easy_setopt(streamPriv->curl->handle, CURLOPT_USERNAME,
+ priv->primary->username);
+ curl_easy_setopt(streamPriv->curl->handle, CURLOPT_PASSWORD,
+ priv->primary->password);
+#else
+ if (virAsprintf(&userpwd, "%s:%s", priv->primary->username,
+ priv->primary->password) < 0) {
+ virReportOOMError();
+ goto cleanup;
+ }
+
+ curl_easy_setopt(streamPriv->curl->handle, CURLOPT_USERPWD, userpwd);
+#endif
+
+ stream->driver = &esxStreamDriver;
+ stream->privateData = streamPriv;
+
+ result = 0;
+
+ cleanup:
+ if (result < 0) {
+ if (streamPriv->curl != NULL && multi !=
streamPriv->curl->multi) {
+ esxVI_MultiCURL_Free(&multi);
+ }
+
+ esxFreeStreamPrivate(&streamPriv);
+ }
+
+ VIR_FREE(range);
+ VIR_FREE(userpwd);
+
+ return result;
+}
+
+int
+esxStreamOpenUpload(virStreamPtr stream, esxPrivate *priv, const char *url)
+{
+ esxStreamPrivate *streamPriv;
+
+ if (esxStreamOpen(stream, priv, url, 0, 0) < 0) {
+ return -1;
+ }
+
+ streamPriv = stream->privateData;
+ streamPriv->mode = ESX_STREAM_MODE_UPLOAD;
+
+ curl_easy_setopt(streamPriv->curl->handle, CURLOPT_UPLOAD, 1);
+ curl_easy_setopt(streamPriv->curl->handle, CURLOPT_READDATA, streamPriv);
+ curl_easy_setopt(streamPriv->curl->handle, CURLOPT_READFUNCTION,
+ esxVI_CURL_ReadStream);
+
+ return 0;
+}
+
+int
+esxStreamOpenDownload(virStreamPtr stream, esxPrivate *priv, const char *url,
+ unsigned long long offset, unsigned long long length)
+{
+ esxStreamPrivate *streamPriv;
+
+ if (esxStreamOpen(stream, priv, url, offset, length) < 0) {
+ return -1;
+ }
+
+ streamPriv = stream->privateData;
+ streamPriv->mode = ESX_STREAM_MODE_DOWNLOAD;
+
+ curl_easy_setopt(streamPriv->curl->handle, CURLOPT_UPLOAD, 0);
+ curl_easy_setopt(streamPriv->curl->handle, CURLOPT_HTTPGET, 1);
+ curl_easy_setopt(streamPriv->curl->handle, CURLOPT_WRITEDATA, streamPriv);
+ curl_easy_setopt(streamPriv->curl->handle, CURLOPT_WRITEFUNCTION,
+ esxVI_CURL_WriteStream);
+
+ return 0;
+}
diff --git a/src/esx/esx_stream.h b/src/esx/esx_stream.h
new file mode 100644
index 0000000..ef4e2a8
--- /dev/null
+++ b/src/esx/esx_stream.h
@@ -0,0 +1,33 @@
+
+/*
+ * esx_stream.h: libcurl based stream driver
+ *
+ * Copyright (C) 2012 Matthias Bolte <matthias.bolte(a)googlemail.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ */
+
+#ifndef __ESX_STREAM_H__
+# define __ESX_STREAM_H__
+
+# include "internal.h"
+# include "esx_private.h"
+
+int esxStreamOpenUpload(virStreamPtr stream, esxPrivate *priv, const char *url);
+int esxStreamOpenDownload(virStreamPtr stream, esxPrivate *priv, const char *url,
+ unsigned long long offset, unsigned long long length);
+
+#endif /* __ESX_STREAM_H__ */
diff --git a/src/esx/esx_util.c b/src/esx/esx_util.c
index 478f60b..f4c37c5 100644
--- a/src/esx/esx_util.c
+++ b/src/esx/esx_util.c
@@ -331,6 +331,88 @@ esxUtil_ParseDatastorePath(const char *datastorePath, char
**datastoreName,
+char *
+esxUtil_ParseVMDKMetadata(const char *metadata, /* might not be NUL-terminated */
+ unsigned long long metadata_length)
+{
+ /* Parse 'test-flat.vmdk' from 'RW 4194304 VMFS
"test-flat.vmdk"' */
+ const char *p = metadata;
+ const char *end = metadata + metadata_length;
+ const char *s = NULL;
+ char *contentName = NULL;
+
+ while (p < end) {
+ /* Skip leading whitespace */
+ while (p < end && (*p == ' ' || *p == '\t')) {
+ ++p;
+ }
+
+ if (p + 3 < end && STRPREFIX(p, "RW ")) {
+ /* Skip until first '"' */
+ while (p < end && *p != '"') {
+ if (*p == '\n') {
+ goto nextline;
+ }
+
+ ++p;
+ }
+
+ if (p < end) {
+ ++p;
+ }
+
+ s = p;
+
+ /* Read until second '"' */
+ while (p < end && *p != '"') {
+ if (*p == '\n') {
+ goto nextline;
+ }
+
+ ++p;
+ }
+
+ if (p < end && *p == '"') {
+ contentName = strndup(s, p - s);
+
+ if (contentName == NULL) {
+ virReportOOMError();
+ return NULL;
+ }
+
+ break;
+ }
+ }
+
+ /* Skip rest of line */
+ while (p < end && *p != '\n') {
+ ++p;
+ }
+
+ nextline:
+ while (p < end && *p == '\n') {
+ ++p;
+ }
+ }
+
+ if (contentName == NULL) {
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Metadata does not contain a content file name"));
+ return NULL;
+ }
+
+ if (strchr(contentName, '/') != NULL) {
+ VIR_FREE(contentName);
+ ESX_ERROR(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Metadata contains invalid content file name"));
+ return NULL;
+ }
+
+ return contentName;
+}
+
+
+
int
esxUtil_ResolveHostname(const char *hostname,
char *ipAddress, size_t ipAddress_length)
diff --git a/src/esx/esx_util.h b/src/esx/esx_util.h
index a69b3f4..8cd0e4b 100644
--- a/src/esx/esx_util.h
+++ b/src/esx/esx_util.h
@@ -49,6 +49,8 @@ int esxUtil_ParseVirtualMachineIDString(const char *id_string, int
*id);
int esxUtil_ParseDatastorePath(const char *datastorePath, char **datastoreName,
char **directoryName, char **directoryAndFileName);
+char *esxUtil_ParseVMDKMetadata(const char *metadata, unsigned long long
metadata_length);
+
int esxUtil_ResolveHostname(const char *hostname,
char *ipAddress, size_t ipAddress_length);
diff --git a/src/esx/esx_vi.c b/src/esx/esx_vi.c
index 3f8d745..ebb3ed9 100644
--- a/src/esx/esx_vi.c
+++ b/src/esx/esx_vi.c
@@ -724,6 +724,35 @@ esxVI_MultiCURL_Remove(esxVI_MultiCURL *multi, esxVI_CURL *curl)
return 0;
}
+/* Returns -1 on error, 0 if there is no message, 1 if there is a message */
+int
+esxVI_MultiCURL_CheckFirstMessage(esxVI_MultiCURL *multi, long *responseCode,
+ CURLcode *errorCode)
+{
+ int messagesInQueue;
+ CURLMsg* msg = curl_multi_info_read(multi->handle, &messagesInQueue);
+
+ *responseCode = 0;
+
+ if (msg == NULL) {
+ return 0;
+ }
+
+ if (msg->msg != CURLMSG_DONE) {
+ return 0;
+ }
+
+ *errorCode = msg->data.result;
+
+ if (*errorCode != CURLE_OK) {
+ return -1;
+ }
+
+ curl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, responseCode);
+
+ return 1;
+}
+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
diff --git a/src/esx/esx_vi.h b/src/esx/esx_vi.h
index 49b7ca2..c464725 100644
--- a/src/esx/esx_vi.h
+++ b/src/esx/esx_vi.h
@@ -203,6 +203,8 @@ int esxVI_MultiCURL_Alloc(esxVI_MultiCURL **multi);
void esxVI_MultiCURL_Free(esxVI_MultiCURL **multi);
int esxVI_MultiCURL_Add(esxVI_MultiCURL *multi, esxVI_CURL *curl);
int esxVI_MultiCURL_Remove(esxVI_MultiCURL *multi, esxVI_CURL *curl);
+int esxVI_MultiCURL_CheckFirstMessage(esxVI_MultiCURL *multi, long *responseCode,
+ CURLcode *errorCode);
diff --git a/tools/virsh.c b/tools/virsh.c
index 53d1825..1fca8c8 100644
--- a/tools/virsh.c
+++ b/tools/virsh.c
@@ -12044,6 +12044,8 @@ static const vshCmdOptDef opts_vol_upload[] = {
{"pool", VSH_OT_STRING, 0, N_("pool name or uuid")},
{"offset", VSH_OT_INT, 0, N_("volume offset to upload to") },
{"length", VSH_OT_INT, 0, N_("amount of data to upload") },
+ {"metadata", VSH_OT_BOOL, 0, N_("upload metadata only") },
+ {"content", VSH_OT_BOOL, 0, N_("upload content only") },
{NULL, 0, 0, NULL}
};
@@ -12066,6 +12068,7 @@ cmdVolUpload (vshControl *ctl, const vshCmd *cmd)
virStreamPtr st = NULL;
const char *name = NULL;
unsigned long long offset = 0, length = 0;
+ unsigned int flags = 0;
if (!vshConnectionUsability(ctl, ctl->conn))
goto cleanup;
@@ -12080,6 +12083,12 @@ cmdVolUpload (vshControl *ctl, const vshCmd *cmd)
return false;
}
+ if (vshCommandOptBool(cmd, "metadata"))
+ flags |= VIR_STORAGE_VOL_UPLOAD_METADATA;
+
+ if (vshCommandOptBool(cmd, "content"))
+ flags |= VIR_STORAGE_VOL_UPLOAD_CONTENT;
+
if (!(vol = vshCommandOptVol(ctl, cmd, "vol", "pool",
&name))) {
return false;
}
@@ -12095,7 +12104,7 @@ cmdVolUpload (vshControl *ctl, const vshCmd *cmd)
}
st = virStreamNew(ctl->conn, 0);
- if (virStorageVolUpload(vol, st, offset, length, 0) < 0) {
+ if (virStorageVolUpload(vol, st, offset, length, flags) < 0) {
vshError(ctl, _("cannot upload to volume %s"), name);
goto cleanup;
}
@@ -12144,6 +12153,8 @@ static const vshCmdOptDef opts_vol_download[] = {
{"pool", VSH_OT_STRING, 0, N_("pool name or uuid")},
{"offset", VSH_OT_INT, 0, N_("volume offset to download from")
},
{"length", VSH_OT_INT, 0, N_("amount of data to download") },
+ {"metadata", VSH_OT_BOOL, 0, N_("download metadata only") },
+ {"content", VSH_OT_BOOL, 0, N_("download content only") },
{NULL, 0, 0, NULL}
};
@@ -12157,6 +12168,7 @@ cmdVolDownload (vshControl *ctl, const vshCmd *cmd)
virStreamPtr st = NULL;
const char *name = NULL;
unsigned long long offset = 0, length = 0;
+ unsigned int flags = 0;
bool created = false;
if (!vshConnectionUsability(ctl, ctl->conn))
@@ -12172,6 +12184,12 @@ cmdVolDownload (vshControl *ctl, const vshCmd *cmd)
return false;
}
+ if (vshCommandOptBool(cmd, "metadata"))
+ flags |= VIR_STORAGE_VOL_DOWNLOAD_METADATA;
+
+ if (vshCommandOptBool(cmd, "content"))
+ flags |= VIR_STORAGE_VOL_DOWNLOAD_CONTENT;
+
if (!(vol = vshCommandOptVol(ctl, cmd, "vol", "pool",
&name)))
return false;
@@ -12191,7 +12209,7 @@ cmdVolDownload (vshControl *ctl, const vshCmd *cmd)
}
st = virStreamNew(ctl->conn, 0);
- if (virStorageVolDownload(vol, st, offset, length, 0) < 0) {
+ if (virStorageVolDownload(vol, st, offset, length, flags) < 0) {
vshError(ctl, _("cannot download from volume %s"), name);
goto cleanup;
}
--
1.7.4.1
5:12 p.m.
New subject: [libvirt] [PATCH 3/3] esx: Add volume upload and download to the storage driver
On 07/02/2012 03:44 PM, Matthias Bolte wrote:
The new METADATA and CONTENT flags allow a caller to tell the ESX
storage driver which part of a volume should be up- or downloaded
in case of a two-file VMDK. In case of a one-file VMDK both or no
flags must be given. In case of non-VMDK volumes such as ISO or
floppy images no flags must be specified.
These flags would also be useful for other drivers; I've complained in
the past that qcow2 images under the qemu driver should be able to
differentiate between metadata (let download give me the qcow2 headers)
and content (return the data as seen by the guest, effectively
converting from qcow2 to raw in the download).
---
include/libvirt/libvirt.h.in | 14 +
Before we can accept the new flags, we need to also document their
effect on the functions in src/libvirt.c. I'd suggest splitting this
commit into two parts - add the new flags and expose them in virsh in
part 1, then implement the flags for esx in part 2. This matters in
case someone else implements the flags for qemu, then wants to backport
the new flags and the qemu implementation but not the esx implementation.
src/Makefile.am | 1 +
src/esx/esx_storage_driver.c | 268 ++++++++++++++++++
src/esx/esx_stream.c | 610 ++++++++++++++++++++++++++++++++++++++++++
src/esx/esx_stream.h | 33 +++
src/esx/esx_util.c | 82 ++++++
src/esx/esx_util.h | 2 +
src/esx/esx_vi.c | 29 ++
src/esx/esx_vi.h | 2 +
tools/virsh.c | 22 ++-
Also, needs documentation in tools/virsh.pod for the new options in virsh.c.
+++ b/include/libvirt/libvirt.h.in
@@ -2562,11 +2562,25 @@ virStorageVolPtr virStorageVolCreateXMLFrom
(virStoragePoolPtr pool,
const char *xmldesc,
virStorageVolPtr clonevol,
unsigned int flags);
+
+typedef enum {
+ VIR_STORAGE_VOL_DOWNLOAD_DEFAULT = 0, /* default behavior */
I'm not sure whether it makes sense to define a name for '0'; in some
cases we have done it, in others we have not. I'm not opposed to the
idea, but we aren't very consistent.
I do, however, like the idea of using the flags argument to support the
needs of ESX in representing things through two files.
I have not closely reviewed this patch for content, so much as offering
this as my first impressions high-level review.
+
+ if (STREQLEN(magic, "KDMV", 4)) {
+ /* It's a one-file VMDK with metadata and content */
+ if (flags != VIR_STORAGE_VOL_DOWNLOAD_DEFAULT &&
+ flags != (VIR_STORAGE_VOL_DOWNLOAD_METADATA |
+ VIR_STORAGE_VOL_DOWNLOAD_CONTENT)) {
+ ESX_ERROR(VIR_ERR_INVALID_ARG, "%s",
+ _("Non or both of metadata and content flag is required
for this volume"));
s/Non/None/
@@ -1668,6 +1934,8 @@ static virStorageDriver esxStorageDriver = {
.volLookupByPath = esxStorageVolumeLookupByPath, /* 0.8.4 */
.volCreateXML = esxStorageVolumeCreateXML, /* 0.8.4 */
.volCreateXMLFrom = esxStorageVolumeCreateXMLFrom, /* 0.8.7 */
+ .volDownload = esxStorageVolumeDownload, /* 0.9.14 */
+ .volUpload = esxStorageVolumeUpload, /* 0.9.14 */
I think consensus was that the next version will be numberd 0.10.0,
thanks to the addition of the parallels driver.
--
Eric Blake eblake(a)redhat.com +1-919-301-3266
Libvirt virtualization library http://libvirt.org
5:12 p.m.
On Mon, Jul 2, 2012 at 4:44 PM, Matthias Bolte
<matthias.bolte(a)googlemail.com> wrote:
I started the download part and the stream driver quite a while ago
but stopped and put it aside as I realized that the storage volume
API up- and download functions assume one file per volume. This is a
problem with ESX as a VMDK can consist of two files. To solve this
now I added two new flags: METADATA and CONTENT. With this flags the
user can specify which part of such a volume to transfer. See patch
3/3 for more details.
Well technically VMDKs can have more than 2 files per disk. They have
the .vmdk which is just the metadata but then the content pieces can
be divided into 2G chunks. Would this be something you'd want to
account for in this patch series or a future update?
--
Doug Goldstein
2:51 p.m.
2012/7/3 Doug Goldstein <cardoe(a)gentoo.org>:
On Mon, Jul 2, 2012 at 4:44 PM, Matthias Bolte
<matthias.bolte(a)googlemail.com> wrote:
> I started the download part and the stream driver quite a while ago
> but stopped and put it aside as I realized that the storage volume
> API up- and download functions assume one file per volume. This is a
> problem with ESX as a VMDK can consist of two files. To solve this
> now I added two new flags: METADATA and CONTENT. With this flags the
> user can specify which part of such a volume to transfer. See patch
> 3/3 for more details.
>
Well technically VMDKs can have more than 2 files per disk. They have
the .vmdk which is just the metadata but then the content pieces can
be divided into 2G chunks. Would this be something you'd want to
account for in this patch series or a future update?
You're right. I forgot about the possibility for having the content
split into 2GB chunks. Well, I have no idea how to represent this in
the libvirt storage API except by adding CONTENT1, CONTENT2, CONTENT3,
... flags.
So, I'm open for suggestions here. As with most things in the ESX
driver findings a suitable mapping between ESX API and libvirt API is
the hard part, implementing it is the easy part :)
Matthias
3:40 p.m.
On Tue, Jul 03, 2012 at 09:51:52PM +0200, Matthias Bolte wrote:
2012/7/3 Doug Goldstein <cardoe(a)gentoo.org>:
> On Mon, Jul 2, 2012 at 4:44 PM, Matthias Bolte
> <matthias.bolte(a)googlemail.com> wrote:
>> I started the download part and the stream driver quite a while ago
>> but stopped and put it aside as I realized that the storage volume
>> API up- and download functions assume one file per volume. This is a
>> problem with ESX as a VMDK can consist of two files. To solve this
>> now I added two new flags: METADATA and CONTENT. With this flags the
>> user can specify which part of such a volume to transfer. See patch
>> 3/3 for more details.
>>
>
> Well technically VMDKs can have more than 2 files per disk. They have
> the .vmdk which is just the metadata but then the content pieces can
> be divided into 2G chunks. Would this be something you'd want to
> account for in this patch series or a future update?
You're right. I forgot about the possibility for having the content
split into 2GB chunks. Well, I have no idea how to represent this in
the libvirt storage API except by adding CONTENT1, CONTENT2, CONTENT3,
... flags.
In the case of a split VMDK file, I'd expect that the list of
virStorageVolPtr objects corresponds to the .vmdk metadata files.
Do not expose each individual data file as a volume. We could
perhaps extend the XML format for virStorageVolPtr to list the
names of the data files. The upload/download streaming APIs would
transparently write to the appropriate data file depending on what
offset you currently access.
Daniel
--
|: http://berrange.com -o- http://www.flickr.com/photos/dberrange/ :|
|: http://libvirt.org -o- http://virt-manager.org :|
|: http://autobuild.org -o- http://search.cpan.org/~danberr/ :|
|: http://entangle-photo.org -o- http://live.gnome.org/gtk-vnc :|
3:53 p.m.
2012/7/3 Daniel P. Berrange <berrange(a)redhat.com>:
On Tue, Jul 03, 2012 at 09:51:52PM +0200, Matthias Bolte wrote:
> 2012/7/3 Doug Goldstein <cardoe(a)gentoo.org>:
> > On Mon, Jul 2, 2012 at 4:44 PM, Matthias Bolte
> > <matthias.bolte(a)googlemail.com> wrote:
> >> I started the download part and the stream driver quite a while ago
> >> but stopped and put it aside as I realized that the storage volume
> >> API up- and download functions assume one file per volume. This is a
> >> problem with ESX as a VMDK can consist of two files. To solve this
> >> now I added two new flags: METADATA and CONTENT. With this flags the
> >> user can specify which part of such a volume to transfer. See patch
> >> 3/3 for more details.
> >>
> >
> > Well technically VMDKs can have more than 2 files per disk. They have
> > the .vmdk which is just the metadata but then the content pieces can
> > be divided into 2G chunks. Would this be something you'd want to
> > account for in this patch series or a future update?
>
> You're right. I forgot about the possibility for having the content
> split into 2GB chunks. Well, I have no idea how to represent this in
> the libvirt storage API except by adding CONTENT1, CONTENT2, CONTENT3,
> ... flags.
In the case of a split VMDK file, I'd expect that the list of
virStorageVolPtr objects corresponds to the .vmdk metadata files.
That's how its implemented also also how vSphere API sees VMDKs. The
metadata .vmdk represents the volume. Where the content is is an
implementation detail.
Do not expose each individual data file as a volume. We could
perhaps extend the XML format for virStorageVolPtr to list the
names of the data files.
The individual datafiles are currently not listed as volumes and will
never be. Also I see no gain in exposing them in the volume XML.
The upload/download streaming APIs would
transparently write to the appropriate data file depending on what
offset you currently access.
I think that's good idea and will work but require a more complex
implementation especially in cases when a virStreamSend/Recv call
crosses a 2GB boundary.
--
Matthias Bolte
http://photron.blogspot.com
4523
days inactive
4529
days old
14 comments
4 participants
participants (4)
-
Daniel P. Berrange -
Doug Goldstein -
Eric Blake -
Matthias Bolte