Commit 4667f5c2 authored by Furkan Mustafa's avatar Furkan Mustafa

Merge branch '301-ensure-diffid-is-mapped' into '201903-small-improvements'

Ensure that diff id has been mapped

See merge request !40
parents 551d5de5 89143c7d
......@@ -398,6 +398,31 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
diffid)
return layers
async def ensure_diffid_mapped(self, diffid: str):
"""
Ensures that given diff id has been mapped
Args:
diffid (string)
"""
# self.diffid_mapping = keys: diffid, values: chain-id
# self.layerdb_mapping = keys: diffid, values: digest
if diffid in self.diffid_mapping and diffid in self.layerdb_mapping:
return
if diffid not in self.diffid_mapping:
await self.get_diffid_mappings()
if diffid not in self.layerdb_mapping:
await self.get_layerdb_mappings()
# if (now) diffid is mapped, return.
if diffid in self.diffid_mapping and diffid in self.layerdb_mapping:
return
# otherwise fail
raise Exception("Could not map diffid: {}".format(diffid))
async def get_layer_by_diffid(self, diffid: str, idx: int, image_id: str) -> DockerLayer:
"""
Makes an DockerLayer objects using diffid of layer
......@@ -409,6 +434,7 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
Returns:
(DockerLayer): `layer` object
"""
await self.ensure_diffid_mapped(diffid)
layerdb_path = self.storage + "/image/overlay2/layerdb"
if diffid not in self.diffid_mapping:
......@@ -437,12 +463,8 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
if idx == 0:
layer.chain_id = diffid
else:
if diffid not in self.layerdb_mapping:
await self.get_layerdb_mappings()
layer.chain_id = self.layerdb_mapping[diffid]
# print("layerdb: ", layer.chain_id)
# try:
layer_meta_folder = layerdb_path + '/' + layer.chain_id.replace(':', '/')
async with aiofiles.open(layer_meta_folder + '/size', mode='r') as layer_size_file:
size_str = await layer_size_file.read()
......@@ -627,6 +649,7 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
self.get_additional_time_downlaod(layer_size),
retry=self.RETRY,
queue=self.queues[jobid][digest]['queue'])
self.logger.debug("downloaded layer %s to %s", digest, save_path)
self.queues[jobid][digest]['status'] = self.DL_FINISH
return resp
......@@ -648,6 +671,7 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
digest(str): digest of layer
"""
# beiran cache directory
diff_id = self.get_diffid_by_digest(digest)
tar_layer_path = self.get_layer_tar_file(diff_id)
gz_layer_path = self.get_layer_gz_file(digest)
......@@ -680,7 +704,7 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
for node_id in layer.available_at:
if node_id == self.local_node.uuid.hex: # type: ignore
continue
self.logger.debug("Download from node that has tarball in own cache directory.")
node = Node.get(Node.uuid == node_id)
resp = await self.download_layer_from_node(
node.url_without_uuid, layer.digest, jobid)
......@@ -695,6 +719,7 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
# TODO: Wait for finish if another beiran is currently downloading it
# TODO: -- or ask for simultaneous streaming download
self.logger.debug("Download layer from origin.")
await self.download_layer_from_origin(ref, digest, jobid, **kwargs)
return 'cache-gz', gz_layer_path
......@@ -1164,6 +1189,7 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
"""
Assemble layer tarball from Docker's storage. Now we need 'tar-split'.
"""
await self.ensure_diffid_mapped(diff_id)
input_file = os.path.join(
self.layerdb_path, del_idpref(self.layerdb_mapping[diff_id]), "tar-split.json.gz")
if not os.path.exists(input_file):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment