[libvirt] [PATCH sandbox 20/24] docker: introduce a DockerRegistry class

Introduce a class to handle HTTP requests with a docker registry server, and associated auth credentials. Signed-off-by: Daniel P. Berrange <berrange(a)redhat.com&gt; --- libvirt-sandbox/image/sources/docker.py | 264 ++++++++++++++++---------------- 1 file changed, 136 insertions(+), 128 deletions(-) diff --git a/libvirt-sandbox/image/sources/docker.py b/libvirt-sandbox/image/sources/docker.py index a54f563..b2706b1 100644 --- a/libvirt-sandbox/image/sources/docker.py +++ b/libvirt-sandbox/image/sources/docker.py @@ -31,6 +31,7 @@ import shutil import urlparse import hashlib from abc import ABCMeta, abstractmethod +import copy from . import base @@ -152,14 +153,132 @@ class DockerAuthToken(DockerAuth): return False -class DockerSource(base.Source): +class DockerRegistry(): - def __init__(self): + def __init__(self, uri_base): + + self.uri_base = list(urlparse.urlparse(uri_base)) self.auth_handler = DockerAuthNop() def set_auth_handler(self, auth_handler): self.auth_handler = auth_handler + def set_server(self, server): + self.uri_base[1] = server + + @classmethod + def from_template(cls, template): + protocol = template.protocol + hostname = template.hostname + port = template.port + + if protocol is None: + protocol = "https" + if hostname is None: + hostname = "index.docker.io" + + if port is None: + server = hostname + else: + server = "%s:%s" % (hostname, port) + + url = urlparse.urlunparse((protocol, server, "", None, None, None)) + + return cls(url) + + def get_url(self, path, headers=None): + url_bits = copy.copy(self.uri_base) + url_bits[2] = path + url = urlparse.urlunparse(url_bits) + debug("Fetching %s..." % url) + + req = urllib2.Request(url=url) + + if headers is not None: + for h in headers.keys(): + req.add_header(h, headers[h]) + + self.auth_handler.prepare_req(req) + + try: + res = urllib2.urlopen(req) + self.auth_handler.process_res(res) + return res + except urllib2.HTTPError as e: + if e.code == 401: + retry = self.auth_handler.process_err(e) + if retry: + debug("Re-Fetching %s..." % url) + self.auth_handler.prepare_req(req) + res = urllib2.urlopen(req) + self.auth_handler.process_res(res) + return res + else: + debug("Not re-fetching") + raise + else: + raise + + def save_data(self, path, dest, checksum=None): + try: + res = self.get_url(path) + + datalen = res.info().getheader("Content-Length") + if datalen is not None: + datalen = int(datalen) + + csum = None + if checksum is not None: + csum = hashlib.sha256() + + pattern = [".", "o", "O", "o"] + patternIndex = 0 + donelen = 0 + + with open(dest, "w") as f: + while 1: + buf = res.read(1024*64) + if not buf: + break + if csum is not None: + csum.update(buf) + f.write(buf) + + if datalen is not None: + donelen = donelen + len(buf) + debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % ( + pattern[patternIndex], (donelen/1024), (datalen/1024) + )) + patternIndex = (patternIndex + 1) % 4 + + debug("\x1b[K") + if csum is not None: + csumstr = "sha256:" + csum.hexdigest() + if csumstr != checksum: + debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum)) + os.remove(dest) + raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum)) + debug("OK\n") + return res + except Exception, e: + debug("FAIL %s\n" % str(e)) + raise + + def get_json(self, path): + try: + headers = {} + headers["Accept"] = "application/json" + res = self.get_url(path, headers) + data = json.loads(res.read()) + debug("OK\n") + return (data, res) + except Exception, e: + debug("FAIL %s\n" % str(e)) + raise + + +class DockerSource(base.Source): + def _check_cert_validate(self): major = sys.version_info.major SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher" @@ -189,38 +308,33 @@ class DockerSource(base.Source): def download_template(self, image, template, templatedir): self._check_cert_validate() + registry = DockerRegistry.from_template(template) basicauth = DockerAuthBasic(template.username, template.password) - self.set_auth_handler(basicauth) + registry.set_auth_handler(basicauth) try: - (data, res) = self._get_json(template, - None, - "/v1/repositories/%s/%s/images" % ( - image.repo, image.name, - )) + (data, res) = registry.get_json("/v1/repositories/%s/%s/images" % ( + image.repo, image.name, + )) except urllib2.HTTPError, e: raise ValueError(["Image '%s' does not exist" % template]) registryendpoint = res.info().getheader('X-Docker-Endpoints') if basicauth.token is not None: - self.set_auth_handler(DockerAuthToken(basicauth.token)) + registry.set_auth_handler(DockerAuthToken(basicauth.token)) else: - self.set_auth_handler(DockerAuthNop()) + registry.set_auth_handler(DockerAuthNop()) - (data, res) = self._get_json(template, - registryendpoint, - "/v1/repositories/%s/%s/tags" %( - image.repo, image.name - )) + (data, res) = registry.get_json("/v1/repositories/%s/%s/tags" %( + image.repo, image.name + )) if image.tag not in data: raise ValueError(["Tag '%s' does not exist for image '%s'" % (image.tag, template)]) imagetagid = data[image.tag] - (data, res) = self._get_json(template, - registryendpoint, - "/v1/images/" + imagetagid + "/ancestry") + (data, res) = registry.get_json("/v1/images/" + imagetagid + "/ancestry") if data[0] != imagetagid: raise ValueError(["Expected first layer id '%s' to match image id '%s'", @@ -240,16 +354,12 @@ class DockerSource(base.Source): datafile = layerdir + "/template.tar.gz" if not os.path.exists(jsonfile) or not os.path.exists(datafile): - res = self._save_data(template, - registryendpoint, - "/v1/images/" + layerid + "/json", - jsonfile) + res = registry.save_data("/v1/images/" + layerid + "/json", + jsonfile) createdFiles.append(jsonfile) - self._save_data(template, - registryendpoint, - "/v1/images/" + layerid + "/layer", - datafile) + registry.save_data("/v1/images/" + layerid + "/layer", + datafile) createdFiles.append(datafile) index = { @@ -275,108 +385,6 @@ class DockerSource(base.Source): except: pass - def _save_data(self, template, server, path, - dest, checksum=None): - try: - res = self._get_url(template, server, path) - - datalen = res.info().getheader("Content-Length") - if datalen is not None: - datalen = int(datalen) - - csum = None - if checksum is not None: - csum = hashlib.sha256() - - pattern = [".", "o", "O", "o"] - patternIndex = 0 - donelen = 0 - - with open(dest, "w") as f: - while 1: - buf = res.read(1024*64) - if not buf: - break - if csum is not None: - csum.update(buf) - f.write(buf) - - if datalen is not None: - donelen = donelen + len(buf) - debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % ( - pattern[patternIndex], (donelen/1024), (datalen/1024) - )) - patternIndex = (patternIndex + 1) % 4 - - debug("\x1b[K") - if csum is not None: - csumstr = "sha256:" + csum.hexdigest() - if csumstr != checksum: - debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum)) - os.remove(dest) - raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum)) - debug("OK\n") - return res - except Exception, e: - debug("FAIL %s\n" % str(e)) - raise - - def _get_url(self, template, server, path, headers=None): - if template.protocol is None: - protocol = "https" - else: - protocol = template.protocol - - if server is None: - if template.hostname is None: - server = "index.docker.io" - else: - if template.port is not None: - server = "%s:%d" % (template.hostname, template.port) - else: - server = template.hostname - - url = urlparse.urlunparse((protocol, server, path, None, None, None)) - debug("Fetching %s..." % url) - - req = urllib2.Request(url=url) - if headers is not None: - for h in headers.keys(): - req.add_header(h, headers[h]) - - self.auth_handler.prepare_req(req) - - try: - res = urllib2.urlopen(req) - self.auth_handler.process_res(res) - return res - except urllib2.HTTPError as e: - if e.code == 401: - retry = self.auth_handler.process_err(e) - if retry: - debug("Re-Fetching %s..." % url) - self.auth_handler.prepare_req(req) - res = urllib2.urlopen(req) - self.auth_handler.process_res(res) - return res - else: - debug("Not re-fetching") - raise - else: - raise - - def _get_json(self, template, server, path): - try: - headers = {} - headers["Accept"] = "application/json") - res = self._get_url(template, server, path, headers) - data = json.loads(res.read()) - debug("OK\n") - return (data, res) - except Exception, e: - debug("FAIL %s\n" % str(e)) - raise - def create_template(self, template, templatedir, connect=None): image = DockerImage.from_template(template) -- 2.7.4