Commit ef409283 authored by Furkan Mustafa's avatar Furkan Mustafa

Merge branch '264-assemble-tarball-docker' into 'dev-0.0.9'

create tarball of layer from docker storage using 'tar-split'

See merge request !33
parents 223f2a8d fb171ab9
......@@ -11,3 +11,4 @@ env
.mypy_cache
**/.pytest_cache/
doc/build
tar-split
......@@ -94,8 +94,9 @@ publish:oci-image-layout:
tags:
- docker
script:
- echo http://dl-cdn.alpinelinux.org/alpine/edge/community/ >> /etc/apk/repositories
- apk update
- apk add go=1.9.4-r0
- apk add go=1.11.5-r0
- apk add musl-dev
- go get github.com/coolljt0725/docker2oci
- docker save $CI_REGISTRY_IMAGE:ci-${CI_PIPELINE_ID} | $HOME/go/bin/docker2oci image-layout-${CI_COMMIT_TAG}
......@@ -180,6 +181,19 @@ start_test_services:
# allow beiran to start up (might not be enough)
- sleep 5
# build and copy tar-split
- echo http://dl-cdn.alpinelinux.org/alpine/edge/community/ >> /etc/apk/repositories
- apk update
- apk add go=1.11.5-r0
- apk add musl-dev
- go get -d github.com/vbatts/tar-split/cmd/tar-split
- go build -o $HOME/tar-split -a -ldflags '-extldflags "-static"' $HOME/go/src/github.com/vbatts/tar-split/cmd/tar-split
- |
for node in beiran1 beiran2 beiran3 beiran4
do
docker cp $HOME/tar-split $(docker-compose ps -q $node):/opt/beiran/plugins/beiran_package_docker/tar-split
done
test:unit:
stage: test
image: $CI_REGISTRY_IMAGE:ci-${CI_PIPELINE_ID}
......
......@@ -16,7 +16,7 @@
- [zeroconf](https://pypi.python.org/pypi/zeroconf) (for local node discovery)
- [click](https://pypi.python.org/pypi/click) (for cli options, commands)
## Docker Setteings
## Docker Settings
### Install Docker
......@@ -57,6 +57,23 @@ Storage Driver: overlay2
...
```
### Install tar-split
beiran's docker plugin needs [tar-split](https://github.com/vbatts/tar-split) now. Please create the binary file and set `tar_split_path`;
```
$ go get github.com/vbatts/tar-split
$ export BEIRAN_PACKAGE_DOCKER_CONFIG='tar_split_path=path/to/tar-split'
```
Or, copy the binary file to the plugin directory (deafult `tar-split_path` is there)
```
$ go get github.com/vbatts/tar-split
$ cp path/to/tar-split plugins/beiran_package_docker/
```
## Virtualenv
### - Setup
......
......@@ -368,13 +368,11 @@ async def wait_event(emitter, event_name, timeout=None):
emitter.once(event_name, _handler)
return await asyncio.wait_for(future, timeout)
def gunzip(path: str) -> None:
def gunzip(input_path: str, output_path: str) -> None:
"""Decompress .gz file"""
with gzip.open(path, 'rb') as gzfile:
with gzip.open(input_path, 'rb') as gzfile:
data = gzfile.read()
path = path.rstrip('.gz')
with open(path, "wb") as tarf:
with open(output_path, "wb") as tarf:
tarf.write(data)
def clean_keys(dict_: dict, keys: list) -> None:
......
......@@ -73,6 +73,14 @@ export BEIRAN_DB_FILE=${DIR}/beiran.db
export BEIRAN_LISTEN_ADDRESS=0.0.0.0
export BEIRAN_CONFIG_DIR=${DIR}
# make binary of 'tar-split' with Docker
tarsplit_dir=${DIR}/plugins/beiran_package_docker/
if [ ! -e ${tarsplit_dir}/tar-split ]; then
docker build -t tarsplit -f $tarsplit_dir/DockerfileTarsplit $tarsplit_dir
docker create --name tarsplit tarsplit && docker cp tarsplit:/tar-split ${tarsplit_dir}/tar-split && docker rm tarsplit && docker rmi tarsplit
fi
function ps1_context {
# For any of these bits of context that exist, display them and append
# a space.
......
FROM golang:1.11.5-alpine3.9
LABEL maintainer="info@beiran.io"
RUN apk add git && go get -d github.com/vbatts/tar-split/cmd/tar-split && \
CGO_ENABLED=0 go build -o /tar-split /go/src/github.com/vbatts/tar-split/cmd/tar-split
......@@ -163,7 +163,7 @@ class LayerDownload(web.RequestHandler):
self.set_header("cache-control", "max-age=31536000")
@staticmethod
def prepare_tar_archive(layer_id: str) -> str:
async def prepare_tar_archive(layer_id: str) -> str:
"""
Finds docker layer path and prepare a tar archive for `layer_id`.
......@@ -182,35 +182,39 @@ class LayerDownload(web.RequestHandler):
except DockerLayer.DoesNotExist:
raise HTTPError(status_code=404, log_message="Layer Not Found")
if not layer.cache_path:
if not layer.cache_path and not layer.cache_gz_path and not layer.docker_path:
raise HTTPError(status_code=404, log_message="Layer Not Found")
# Do not create tarball from storage of docker!!!
# The digest of the tarball varies depending on the mtime of the file to be added
# layer.cache_path = Services.docker_util.layer_storage_path(layer_id).split('.gz')[0] # type: ignore # pylint: disable=line-too-long
# if not os.path.isfile(layer.cache_path):
# create_tar_archive(layer.docker_path, layer.cache_path)
# layer.save()
# not deal with .tar.gz in cache directory now
if not layer.cache_path:
if layer.cache_gz_path:
_, layer.cache_path = \
await Services.docker_util.decompress_gz_layer(layer.cache_gz_path) # type: ignore # pylint: disable=line-too-long
elif layer.docker_path:
layer.cache_path = \
await Services.docker_util.assemble_layer_tar(layer.diff_id) # type: ignore
layer.save()
return layer.cache_path
# pylint: disable=arguments-differ
def head(self, layer_id: str):
async def head(self, layer_id: str):
"""Head response with actual Content-Lenght of layer"""
self._set_headers(layer_id)
tar_path = self.prepare_tar_archive(layer_id)
tar_path = await self.prepare_tar_archive(layer_id)
self.set_header("Content-Length", str(os.path.getsize(tar_path)))
self.finish()
# pylint: enable=arguments-differ
# pylint: disable=arguments-differ
def get(self, layer_id):
async def get(self, layer_id):
"""
Get layer info by given layer_id
"""
self._set_headers(layer_id)
tar_path = self.prepare_tar_archive(layer_id)
tar_path = await self.prepare_tar_archive(layer_id)
with open(tar_path, 'rb') as file:
while True:
......
......@@ -188,6 +188,7 @@ class DockerLayer(BaseModel, CommonDockerObjectFunctions):
local_image_refs = JSONStringField(default=list)
cache_path = CharField(null=True, default=None) # .tar file in cache dir
cache_gz_path = CharField(null=True, default=None) # .tar.gz file in cache dir
docker_path = CharField(null=True) # layer's directory under /var/lib/docker
def set_local_image_refs(self, image_id: str):
......
......@@ -59,12 +59,14 @@ class DockerPackaging(BasePackagePlugin): # pylint: disable=too-many-instance-a
def set_dynamic_defaults(self):
"""Set dynamic configuration value like using ``run_dir``"""
self.config.setdefault('cache_dir', config.cache_dir + '/docker')
self.config.setdefault('tar_split_path', os.path.dirname(__file__) + '/tar-split')
async def init(self):
self.aiodocker = Docker()
self.util = DockerUtil(cache_dir=self.config["cache_dir"],
storage=self.config["storage"], aiodocker=self.aiodocker,
logger=self.log, local_node=self.node)
logger=self.log, local_node=self.node,
tar_split_path=self.config['tar_split_path'])
self.docker = docker.from_env()
self.docker_lc = docker.APIClient()
self.probe_task = None
......@@ -131,7 +133,7 @@ class DockerPackaging(BasePackagePlugin): # pylint: disable=too-many-instance-a
self.log.debug("new layer from remote %s", str(layer))
def save_local_paths(self, layer: DockerLayer):
"""Update 'cache_path' and 'docker_path' with paths of local node"""
"""Update 'cache_path' and 'cache_gz_path' and 'docker_path' with paths of local node"""
try:
docker_path = self.util.layerdir_path.format(
layer_dir_name=self.util.get_cache_id_from_chain_id(layer.chain_id))
......@@ -143,12 +145,19 @@ class DockerPackaging(BasePackagePlugin): # pylint: disable=too-many-instance-a
except FileNotFoundError:
layer.docker_path = None
cache_path = self.util.get_layer_tar_file(layer.diff_id)
if os.path.exists(cache_path):
layer.cache_path = cache_path
else:
layer.cache_path = None
if layer.digest:
cache_path = os.path.join(self.util.layer_cache_path, del_idpref(layer.digest) + '.tar')
if os.path.exists(cache_path):
layer.cache_path = cache_path
cache_gz_path = self.util.get_layer_gz_file(layer.digest)
if os.path.exists(cache_gz_path):
layer.cache_gz_path = cache_gz_path
else:
layer.cache_path = None
layer.cache_gz_path = None
async def fetch_images_from_peer(self, peer: Peer):
"""fetch image list from the node and update local db"""
......@@ -389,10 +398,7 @@ class DockerPackaging(BasePackagePlugin): # pylint: disable=too-many-instance-a
# skip verbose updates of records
if not skip_updating_layer:
for layer in layers:
# 'available_at' field is set if the node has the layer in cache directory
# fix this code if we can create traball from docker storage
if layer.cache_path:
layer.set_available_at(self.node.uuid.hex)
layer.set_available_at(self.node.uuid.hex)
layer.save()
self.log.debug("image layers updated, record updated.. %s \n\n", layer.to_dict())
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment