Commit 81365034 authored by Hilal Ozdemir's avatar Hilal Ozdemir Committed by Hilal Ozdemir

refactor and pylint fixes

parent d67b0013
......@@ -92,7 +92,8 @@ async def async_write_file_stream(url: str, save_path: str, queue=None, # pylint
if resp.status >= 400:
raise Exception("Failed to write stream to file. code: %d"%resp.status)
if not resp.content:
raise Exception("Failed to write stream to file. Response does not have a body.")
raise Exception(
"Failed to write stream to file. Response does not have a body.")
with open(save_path, 'wb')as file:
async for chunk in input_reader(resp.content):
file.write(chunk)
......
......@@ -335,19 +335,20 @@ class ImageList(RPCEndpoint):
)
def when_done(task):
if task.exception():
Services.logger.error("create_or_download_config error: {}".format(task.exception()))
Services.logger.error(
"create_or_download_config error: {}".format(task.exception()))
first_layer_start_task.set_exception(task.exception())
else:
Services.logger.debug("create_or_download_config done.")
config_future.add_done_callback(when_done)
try:
await first_layer_start_task
except Exception as e:
except Services.docker_util.ManifestError as err:
if show_progress:
rpc_endpoint.write('{"progress":[{"error":"%s"}]}'%str(e))
rpc_endpoint.write('{"progress":[{"error":"%s"}]}'%str(err))
rpc_endpoint.finish()
else:
rpc_endpoint.write('{"resp_info":[{"error":"%s"}]}'%str(e))
rpc_endpoint.write('{"resp_info":[{"error":"%s"}]}'%str(err))
rpc_endpoint.finish()
return
......@@ -446,7 +447,6 @@ class ImageList(RPCEndpoint):
online_availables = [n for n in available_nodes if n in online_nodes]
if online_availables:
node_identifier = random.choice(online_availables)
if not node_identifier:
raise HTTPError(status_code=404, log_message='Image is not available in cluster')
......
......@@ -26,13 +26,12 @@ import asyncio
# import progressbar
import click
from tabulate import tabulate
import json
from beiran_package_docker.util import DockerUtil
from beiran.util import json_streamer
from beiran.util import sizeof_fmt
from beiran.multiple_progressbar import MultipleProgressBar
from beiran.cli import pass_context
from beiran_package_docker.util import DockerUtil
@click.group()
......@@ -53,44 +52,19 @@ def image():
"""
pass
@image.command('pull')
@click.option('--from', 'node', default=None,
help='Pull from spesific node')
@click.option('--wait', 'wait', default=False, is_flag=True,
help='Wait the result of pulling image')
@click.option('--force', 'force', default=False, is_flag=True,
help='Forces download of image even if the node is not recognised')
@click.option('--no-progress', 'noprogress', default=False, is_flag=True,
help='Disable image transfer progress display')
@click.option('--whole-image-only', 'whole_image_only', default=False, is_flag=True,
help='Pull an image from other node (not each layer)')
@click.argument('imagename')
@click.pass_obj
@pass_context
# pylint: disable-msg=too-many-arguments
def image_pull(ctx, node: str, wait: bool, force: bool, noprogress: bool,
whole_image_only: bool, imagename: str):
"""Pull a container image from cluster or repository"""
click.echo(
'Pulling image %s from %s!' % (imagename, node or "available nodes"))
if not noprogress:
if whole_image_only:
async def _pull_with_progress(ctx, imagename, node, force):
"""Pull image with async client (whole image)"""
progbar = MultipleProgressBar(desc=imagename)
async def _pull_with_progress():
"""Pull image with async client"""
resp = await ctx.async_beiran_client.pull_image(
imagename,
node=node,
wait=True,
force=force,
whole_image_only=whole_image_only,
whole_image_only=True,
progress=True,
raise_error=True
)
isFinished = False
is_finished = False
async for update in json_streamer(resp.content, '$.progress[::]'):
err = update.get('error')
if err:
......@@ -98,20 +72,16 @@ def image_pull(ctx, node: str, wait: bool, force: bool, noprogress: bool,
return
if update.get('finished'):
click.echo("Image pulling process finished.")
isFinished = True
is_finished = True
break
progbar.update(update['progress'])
progbar.finish()
if not isFinished:
if not is_finished:
click.echo('An error occured!')
loop = asyncio.get_event_loop()
loop.run_until_complete(_pull_with_progress())
else:
async def _pull_with_progress():
"""Pull image with async client"""
async def _pull_with_progress_distributed(ctx, imagename, node, force):
"""Pull image with async client (distributed)"""
progbars = {}
try:
resp = await ctx.async_beiran_client.pull_image(
......@@ -119,13 +89,13 @@ def image_pull(ctx, node: str, wait: bool, force: bool, noprogress: bool,
node=node,
wait=True,
force=force,
whole_image_only=whole_image_only,
whole_image_only=False,
progress=True,
raise_error=True
)
except Exception as e:
except Exception as err: # pylint: disable=broad-except
click.echo('An exception is catched while requesting pull image!')
click.echo(str(e))
click.echo(str(err))
return
click.echo('Downloading layers...')
lastbar = None
......@@ -161,10 +131,7 @@ def image_pull(ctx, node: str, wait: bool, force: bool, noprogress: bool,
lastbar.seek_last_line()
click.echo('An error occured!')
loop = asyncio.get_event_loop()
loop.run_until_complete(_pull_with_progress())
else:
async def _pull_without_progress():
async def _pull_without_progress(ctx, imagename, node, wait, force, whole_image_only): # pylint: disable=too-many-arguments
"""Pull image with async client"""
try:
click.echo("Requesting image pull...")
......@@ -177,9 +144,9 @@ def image_pull(ctx, node: str, wait: bool, force: bool, noprogress: bool,
progress=False,
raise_error=True
)
except Exception as e:
except Exception as err: # pylint: disable=broad-except
click.echo('An exception is catched while requesting pull image!')
click.echo(str(e))
click.echo(str(err))
return
async for data in json_streamer(resp.content, '$.resp_info[::]'):
......@@ -195,8 +162,38 @@ def image_pull(ctx, node: str, wait: bool, force: bool, noprogress: bool,
click.echo("Image pulling process finished.")
return
click.echo('An error occured while pulling image!')
@image.command('pull')
@click.option('--from', 'node', default=None,
help='Pull from spesific node')
@click.option('--wait', 'wait', default=False, is_flag=True,
help='Wait the result of pulling image')
@click.option('--force', 'force', default=False, is_flag=True,
help='Forces download of image even if the node is not recognised')
@click.option('--no-progress', 'noprogress', default=False, is_flag=True,
help='Disable image transfer progress display')
@click.option('--whole-image-only', 'whole_image_only', default=False, is_flag=True,
help='Pull an image from other node (not each layer)')
@click.argument('imagename')
@click.pass_obj
@pass_context
# pylint: disable-msg=too-many-arguments
def image_pull(ctx, node: str, wait: bool, force: bool, noprogress: bool,
whole_image_only: bool, imagename: str):
"""Pull a container image from cluster or repository"""
click.echo(
'Pulling image %s from %s!' % (imagename, node or "available nodes"))
loop = asyncio.get_event_loop()
loop.run_until_complete(_pull_without_progress())
if not noprogress:
if whole_image_only:
loop.run_until_complete(_pull_with_progress(ctx, imagename, node, force))
else:
loop.run_until_complete(_pull_with_progress_distributed(ctx, imagename, node, force))
else:
loop.run_until_complete(
_pull_without_progress(ctx, imagename, node, wait, force, whole_image_only))
# pylint: enable-msg=too-many-arguments
......
......@@ -29,11 +29,6 @@ from aiodocker import Docker
from aiodocker.exceptions import DockerError
from peewee import SQL
from beiran.config import config
from beiran.plugin import BasePackagePlugin, History
from beiran.models import Node
from beiran.daemon.peer import Peer
from beiran_package_docker.image_ref import del_idpref
from beiran_package_docker.models import DockerImage, DockerLayer
from beiran_package_docker.models import MODEL_LIST
......@@ -41,6 +36,10 @@ from beiran_package_docker.util import DockerUtil
from beiran_package_docker.api import ROUTES
from beiran_package_docker.api import Services as ApiDependencies
from beiran.config import config
from beiran.plugin import BasePackagePlugin, History
from beiran.models import Node
from beiran.daemon.peer import Peer
PLUGIN_NAME = 'docker'
PLUGIN_TYPE = 'package'
......
......@@ -298,7 +298,7 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
async def get_digest_by_diffid(self, diffid: str)-> Optional[str]:
"""Return digest of a layer by diff id."""
try:
with open(os.path.join(self.v2metadata_path, del_idpref(diffid)))as file:
with open(os.path.join(self.v2metadata_path, del_idpref(diffid))) as file:
content = json.load(file)
return content[0]['Digest']
except FileNotFoundError:
......@@ -608,11 +608,15 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
resp, _ = await async_req(url=url, return_json=False, timeout=self.TIMEOUT,
retry=self.RETRY, method='HEAD', Authorization=requirements)
if resp.status >= 400:
raise DockerUtil.LayerDownloadFailed("Failed to download layer from %s. code: %d"%(url, resp.status))
raise DockerUtil.LayerDownloadFailed(
"Failed to download layer from %s. code: %d"%(url, resp.status))
if not resp.content:
raise DockerUtil.LayerDownloadFailed("Failed to download layer from %s. Response does not have a body."%url)
raise DockerUtil.LayerDownloadFailed(
"Failed to download layer from %s. Response does not have a body."%url)
if not resp.headers.get('content-length'):
raise DockerUtil.LayerDownloadFailed("Failed to download layer from %s. Response headers do not have content-length."%url)
raise DockerUtil.LayerDownloadFailed(
"Failed to download layer from %s. Response headers do not have content-length."
%url)
layer_size = int(resp.headers.get('content-length'))
......@@ -647,11 +651,15 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
resp, _ = await async_req(url=url, return_json=False, timeout=self.TIMEOUT,
retry=self.RETRY, method='HEAD')
if resp.status >= 400:
raise DockerUtil.LayerDownloadFailed("Failed to download layer from %s. code: %d"%(url, resp.status))
raise DockerUtil.LayerDownloadFailed(
"Failed to download layer from %s. code: %d"%(url, resp.status))
if not resp.content:
raise DockerUtil.LayerDownloadFailed("Failed to download layer from %s. Response does not have a body."%url)
raise DockerUtil.LayerDownloadFailed(
"Failed to download layer from %s. Response does not have a body."%url)
if not resp.headers.get('content-length'):
raise DockerUtil.LayerDownloadFailed("Failed to download layer from %s. Response headers do not have content-length."%url)
raise DockerUtil.LayerDownloadFailed(
"Failed to download layer from %s. " +
"Response headers do not have content-length."%url)
layer_size = int(resp.headers.get('content-length'))
......@@ -954,18 +962,16 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
try:
manifest = await self.fetch_docker_image_manifest(ref['domain'], ref['repo'],
manifest_digest)
except DockerUtil.FetchManifestFailed as e:
raise DockerUtil.ManifestError(str(e))
except DockerUtil.FetchManifestFailed as err:
raise DockerUtil.ManifestError(str(err))
schema_v = manifest['schemaVersion']
if schema_v == 1:
if manifest['schemaVersion'] == 1:
# pull layers and create config from version 1 manifest
config_json_str, config_digest, _ = await self.fetch_config_schema_v1(
ref, manifest, jobid
)
elif schema_v == 2:
elif manifest['schemaVersion'] == 2:
if manifest['mediaType'] == 'application/vnd.docker.distribution.manifest.v2+json':
# pull layers using version 2 manifest
config_json_str, config_digest, _ = await self.fetch_config_schema_v2(
......@@ -974,7 +980,7 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
else:
raise DockerUtil.ManifestError('Invalid media type: %d' % manifest['mediaType'])
else:
raise DockerUtil.ManifestError('Invalid schema version: %d' % schema_v)
raise DockerUtil.ManifestError('Invalid schema version: %d' % manifest['schemaVersion'])
manifestlist_str = json.dumps(manifestlist, indent=3)
repo_digest = add_idpref(hashlib.sha256(manifestlist_str.encode('utf-8')).hexdigest())
......@@ -1076,8 +1082,8 @@ class DockerUtil: # pylint: disable=too-many-instance-attributes
try:
manifest = await self.fetch_docker_image_manifest(
ref['domain'], ref['repo'], ref['suffix'])
except DockerUtil.FetchManifestFailed as e:
raise DockerUtil.ManifestError(str(e))
except DockerUtil.FetchManifestFailed as err:
raise DockerUtil.ManifestError(str(err))
schema_v = manifest['schemaVersion']
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment