Commit 2f0f1a5c authored by Hilal Ozdemir's avatar Hilal Ozdemir

Merge branch 'dev-0.0.9' into 319-add-init-files

parents b94fd216 3c54f6f1
......@@ -122,7 +122,7 @@ checks:lint-plugins:
tags:
- docker
script:
- pylint plugins
- pylint plugins/beiran_package_docker/beiran_package_docker plugins/beiran_interface_k8s/beiran_interface_k8s
checks:mypy-beiran:
stage: checks
......@@ -138,7 +138,8 @@ checks:mypy-plugins:
tags:
- docker
script:
- helpers/mypy --config-file mypy.ini plugins
- helpers/mypy --config-file mypy.ini plugins/beiran_package_docker
- helpers/mypy --config-file mypy.ini plugins/beiran_interface_k8s
checks:bandit-beiran:
stage: checks
......@@ -155,7 +156,7 @@ checks:bandit-plugins:
tags:
- docker
script:
- bandit -r plugins
- bandit -r plugins/beiran_package_docker/beiran_package_docker plugins/beiran_interface_k8s/beiran_interface_k8s
allow_failure: true
start_test_services:
......@@ -191,7 +192,7 @@ start_test_services:
- |
for node in beiran1 beiran2 beiran3 beiran4 beiran5 beiran6 beiran7
do
docker cp $HOME/tar-split $(docker-compose ps -q $node):/opt/beiran/plugins/beiran_package_docker/tar-split
docker cp $HOME/tar-split $(docker-compose ps -q $node):/opt/beiran_package_docker/beiran_package_docker/tar-split
done
test:unit:
......
......@@ -15,7 +15,7 @@ ignore-patterns=
# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
#init-hook=
# init-hook=
# Use multiple processes to speed up Pylint.
jobs=1
......@@ -133,7 +133,8 @@ disable=print-statement,
assignment-from-no-return,
too-many-public-methods,
too-many-instance-attributes,
unnecessary-pass
unnecessary-pass,
import-error
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
......
......@@ -8,29 +8,25 @@ RUN apt-get update && apt-get -y install \
--no-install-recommends \
python3-pip git curl make libsqlite3-dev libyajl-dev libyajl2
RUN mkdir -p /opt/beiran/beiran
WORKDIR /opt
COPY beiran /opt/beiran/beiran
COPY README.md /opt/beiran/README.md
COPY setup.py /opt/beiran/setup.py
COPY plugins/beiran_package_docker /opt/beiran_package_docker/
COPY --from=tar-split-builder /tar-split /opt/beiran_package_docker/beiran_package_docker/tar-split
COPY plugins/beiran_interface_k8s /opt/beiran_interface_k8s/
ADD beiran/requirements.txt /opt/beiran/r-lib.txt
RUN pip install -r /opt/beiran/r-lib.txt
WORKDIR /opt/beiran
RUN python setup.py install
ADD plugins/beiran_package_docker/requirements.txt /opt/beiran/r-docker.txt
RUN pip install -r /opt/beiran/r-docker.txt
WORKDIR /opt/beiran_package_docker
RUN python setup.py install
ADD plugins/beiran_interface_k8s/requirements.txt /opt/beiran/r-k8s.txt
RUN pip install -r /opt/beiran/r-k8s.txt
# ADD plugins/beiran_package_npm/requirements.txt /opt/beiran/r-npm.txt
# RUN pip install -r /opt/beiran/r-npm.txt
COPY --from=tar-split-builder /tar-split /opt/beiran/plugins/beiran_package_docker/tar-split
WORKDIR /opt/beiran_interface_k8s
RUN python setup.py install
WORKDIR /opt/beiran
RUN echo 'alias beiran="python -m beiran.cli"\nalias beirand="python -m beiran.daemon"\n' > /root/.bashrc
ADD [ "beiran", "/opt/beiran/beiran" ]
ADD [ "plugins", "/opt/beiran/plugins" ]
ENV PYTHONPATH=/opt/beiran:/opt/beiran/plugins
VOLUME /var/lib/beiran
......
......@@ -18,6 +18,12 @@ enabled = true
cache_dir = '/var/cache/beiran/docker'
module_path = 'beiran_package_docker'
# k8s interface settings
[interface.k8s]
enabled = true
cache_dir = '/var/cache/beiran/k8s'
module_path 'beiran_interface_k8s'
# dns discovery settings
[discovery.dns]
enabled = true
......
......@@ -157,7 +157,7 @@ class BeiranDaemon(EventEmitter):
plugin_name = plugin['name']
plugin_package = plugin['package']
try:
config['logger'] = build_logger()
config['logger'] = build_logger('beiran.plugin.' + plugin_name)
config['node'] = self.nodes.local_node
config['daemon'] = self
config['events'] = EVENTS
......@@ -285,7 +285,6 @@ class BeiranDaemon(EventEmitter):
**type_specific_config,
**config.get_plugin_config(plugin['type'], plugin['name'])
})
# Only one discovery plugin at a time is supported (for now)
if plugin['type'] == 'discovery' and plugin['name'] == config.discovery_method:
Services.plugins['discovery'] = _plugin_obj
......
......@@ -27,7 +27,8 @@ from typing import List, Union # pylint: disable=unused-import
from beiran.config import config
def build_logger(filename: str = config.log_file,
def build_logger(name: str = __package__,
filename: str = config.log_file,
log_level: str = logging.getLevelName(config.log_level)) -> logging.Logger:
""" Build logger class for module """
stdout_handler = logging.StreamHandler(sys.stdout)
......@@ -41,4 +42,4 @@ def build_logger(filename: str = config.log_file,
format='[%(asctime)s] [%(name)s] %(levelname)s - %(message)s',
handlers=handlers
)
return logging.getLogger(__package__)
return logging.getLogger(name)
......@@ -11,5 +11,7 @@ aiofiles==0.3.2
netifaces==0.10.9
progressbar2==3.39.2
pylint==2.3.1
mypy==0.720
aiodns==2.0.0
aiozeroconf==0.1.3
astroid==2.2.5
\ No newline at end of file
......@@ -4,7 +4,7 @@ DIR="$( cd "$(dirname "$0")" ; pwd -P )"
NAME=$(basename $DIR)
export VIRTUAL_ENV_DIR="${DIR}/env"
unset PYTHON_HOME
export PYTHONPATH=${DIR}:${DIR}/plugins
export PYTHONPATH=${DIR}:${DIR}/plugins/beiran_package_docker:${DIR}/plugins/beiran_interface_k8s
export PYTHON_BINARY=python3.6
export PKG_DIR=${VIRTUAL_ENV_DIR}/lib/${PYTHON_BINARY}/site-packages
export VIRTUALENV=$(which virtualenv3 || which virtualenv || echo 'virtualenv')
......@@ -29,9 +29,26 @@ else
export PIP=$(which pip3 || which pip || echo 'pip')
fi
STAMP=$(date +%s)
INSTALLED=0
LAST_INSTALL=$(date -r ${VIRTUAL_ENV_DIR}/.last_install +%s 2>/dev/null || echo "0")
DEV_SH_MODIFIED=$(date -r ${DIR}/dev.sh +%s)
if [ $DEV_SH_MODIFIED -gt $LAST_INSTALL ]; then
#install prerequisite package `yajl`
apt-get install libyajl2
apt-get install libyajl-dev
# make binary of 'tar-split' with Docker
tarsplit_dir=${DIR}/plugins/beiran_package_docker/
if [ ! -e ${tarsplit_dir}/tar-split ]; then
docker build -t tarsplit -f $tarsplit_dir/DockerfileTarsplit $tarsplit_dir
docker create --name tarsplit tarsplit && docker cp tarsplit:/tar-split ${tarsplit_dir}/tar-split && docker rm tarsplit && docker rmi tarsplit
fi
fi
packages="beiran plugins/*"
for package in $packages; do
package_name=$(basename $package)
......@@ -75,14 +92,8 @@ export BEIRAN_URL=http://localhost:${BEIRAN_PORT}
export BEIRAN_DB_FILE=${DIR}/beiran.db
export BEIRAN_LISTEN_ADDRESS=0.0.0.0
export BEIRAN_CONFIG_DIR=${DIR}
# make binary of 'tar-split' with Docker
tarsplit_dir=${DIR}/plugins/beiran_package_docker/
if [ ! -e ${tarsplit_dir}/tar-split ]; then
docker build -t tarsplit -f $tarsplit_dir/DockerfileTarsplit $tarsplit_dir
docker create --name tarsplit tarsplit && docker cp tarsplit:/tar-split ${tarsplit_dir}/tar-split && docker rm tarsplit && docker rmi tarsplit
fi
export BEIRAN_DISCOVERY_METHOD=zeroconf
export BEIRAN_PLUGINS=beiran.plugins.discovery.zeroconf,package.docker,interface.k8s
function ps1_context {
# For any of these bits of context that exist, display them and append
......
......@@ -9,13 +9,18 @@ That's very simple. Give the path of Beiran's cri endpoint to kubelet's command
kubelet --image-service-endpoint=unix:///var/run/beiran-cri.sock
For example, if you are running kubelet with systemd, please edit the configuration file like as follows::
If you are running kubelet with systemd, please edit the configuration file `/etc/systemd/system/kubelet.service.d/10-kubeadm.conf` as follows::
[Service]
...
Environment="KUBELET_CERTIFICATE_ARGS=--rotate-certificates=true --cert-dir=/var/lib/kubelet/pki"
Environment="KUBELET_EXTRA_ARGS=--image-service-endpoint=unix:///var/run/beiran-cri.sock"
**Environment="KUBELET_EXTRA_ARGS=--image-service-endpoint=unix:///var/run/beiran-cri.sock"**
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_NETWORK_ARGS $KUBELET_DNS_ARGS $KUBELET_AUTHZ_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CERTIFICATE_ARGS $KUBELET_EXTRA_ARGS
Then to restart kubelet, run::
systemctl daemon-reload && systemctl restart kubelet
Note that `/var/run` is the default location for Beiran unix sockets. If `BEIRAN_RUN_DIR` environment variable or `run_dir` in Beiran config file is set to something else, unix sockets will be there. If you have difficulty determining the path of Beiran's cri endpoint, you can check Beiran daemon's log messages starting with `[beiran.plugin.k8s]`.
......@@ -56,10 +56,8 @@ ignore_missing_imports = True
ignore_errors = True
# disable import errors such as from plugins.beiran_package_docker.models import DockerImage
[mypy-plugins.*]
[mypy-beiran_package_docker.*]
ignore_missing_imports = True
[mypy-click.*]
ignore_missing_imports = True
This diff is collapsed.
Developer Certificate of Origin
Version 1.1
Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
1 Letterman Drive
Suite D4700
San Francisco, CA, 94129
Everyone is permitted to copy and distribute verbatim copies of this
license document, but changing it is not allowed.
Developer's Certificate of Origin 1.1
By making a contribution to this project, I certify that:
(a) The contribution was created in whole or in part by me and I
have the right to submit it under the open source license
indicated in the file; or
(b) The contribution is based upon previous work that, to the best
of my knowledge, is covered under an appropriate open source
license and I have the right under that license to submit that
work with modifications, whether created in whole or in part
by me, under the same open source license (unless I am
permitted to submit under a different license), as indicated
in the file; or
(c) The contribution was provided directly to me by some other
person who certified (a), (b) or (c) and I have not modified
it.
(d) I understand and agree that this project and the contribution
are public and that a record of the contribution (including all
personal information I submit with it, including my sign-off) is
maintained indefinitely and may be redistributed consistent with
this project or the open source license(s) involved.
# Beiran Kubernetes Interface Plugin
......@@ -28,10 +28,11 @@ import json
import asyncio
import grpc
from beiran.util import run_in_loop
from beiran_package_docker.models import DockerImage
from beiran_package_docker.api import ImageList
from beiran.util import run_in_loop
from .api_pb2_grpc import ImageServiceServicer, ImageServiceStub
from .api_pb2 import ImageStatusResponse
from .api_pb2 import ListImagesResponse, Image
......@@ -83,7 +84,7 @@ class K8SImageServicer(ImageServiceServicer):
images = []
for image in DockerImage.select():
uid, username = get_username_or_uid(image.config["User"])
uid, username = get_username_or_uid(json.loads(image.config)["config"]["User"])
images.append(Image(
id=image.hash_id,
......@@ -119,7 +120,7 @@ class K8SImageServicer(ImageServiceServicer):
if request.verbose:
info = {'config': json.dumps(image.config)} # tentatively return config...
uid, username = get_username_or_uid(image.config["User"])
uid, username = get_username_or_uid(json.loads(image.config)["config"]["User"])
response = ImageStatusResponse(image=Image(
id=image.hash_id,
......
......@@ -60,6 +60,7 @@ class K8SInterface(BaseInterfacePlugin):
self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
add_ImageServiceServicer_to_server(self.servicer, self.server)
self.server.add_insecure_port(self.unix_socket_path)
self.log.info("Listening on unix socket: %s", self.unix_socket_path[7:])
async def start(self):
"""Start gRPC server"""
......
# Beiran P2P Package Distribution Layer
# Copyright (C) 2019 Rainlab Inc & Creationline, Inc & Beiran Contributors
#
# Rainlab Inc. https://rainlab.co.jp
# Creationline, Inc. https://creationline.com">
# Beiran Contributors https://docs.beiran.io/contributors.html
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""setup.py for packaging beiran"""
from setuptools import setup, find_packages
REQUIREMENTS_PATH = 'requirements.txt'
def read_requirements(path):
"""Read requirements.txt and return a list of requirements."""
with open(path, 'r') as file:
reqs = file.read().splitlines()
return reqs
def read_long_description():
"""Read a file written about long description of the package."""
with open("README.md", "r") as file:
long_description = file.read()
return long_description
def find_beiran_packages():
"""Find beiran package."""
return find_packages(
# exclude=["plugins", "doc", "helpers",]
)
def get_version():
"""
Reads VERSION file
Returns:
(str): plugin version
"""
with open('VERSION', 'r') as file:
version = file.read()
return version
setup(
name="beiran_interface_k8s",
version=get_version(),
author="RainLab",
author_email="info@rainlab.co.jp",
description="Beiran Kubernetes Plugin",
keywords=['beiran', 'kubernetes', 'k8s', 'p2p package manager', 'system tools'],
long_description=read_long_description(),
long_description_content_type="text/markdown",
url="https://beiran.io",
classifiers=[
"Development Status :: 3 - Alpha",
"Intended Audience :: System Administrators",
"Topic :: System :: Software Distribution"
"Topic :: System :: Systems Administration",
"Programming Language :: Python :: 3",
"License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", # ???
"Operating System :: OS Independent",
],
install_requires=read_requirements(REQUIREMENTS_PATH),
python_requires='~=3.6',
packages=find_beiran_packages(),
)
This diff is collapsed.
Developer Certificate of Origin
Version 1.1
Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
1 Letterman Drive
Suite D4700
San Francisco, CA, 94129
Everyone is permitted to copy and distribute verbatim copies of this
license document, but changing it is not allowed.
Developer's Certificate of Origin 1.1
By making a contribution to this project, I certify that:
(a) The contribution was created in whole or in part by me and I
have the right to submit it under the open source license
indicated in the file; or
(b) The contribution is based upon previous work that, to the best
of my knowledge, is covered under an appropriate open source
license and I have the right under that license to submit that
work with modifications, whether created in whole or in part
by me, under the same open source license (unless I am
permitted to submit under a different license), as indicated
in the file; or
(c) The contribution was provided directly to me by some other
person who certified (a), (b) or (c) and I have not modified
it.
(d) I understand and agree that this project and the contribution
are public and that a record of the contribution (including all
personal information I submit with it, including my sign-off) is
maintained indefinitely and may be redistributed consistent with
this project or the open source license(s) involved.
0.0.8
\ No newline at end of file
......@@ -28,11 +28,11 @@ import sys
import click
from tabulate import tabulate
from beiran_package_docker.util import DockerUtil
from beiran.util import json_streamer
from beiran.util import sizeof_fmt
from beiran.multiple_progressbar import MultipleProgressBar
from beiran.cli import pass_context
from .util import DockerUtil
@click.group()
......
......@@ -29,18 +29,18 @@ from aiodocker import Docker
from aiodocker.exceptions import DockerError
from peewee import SQL
from beiran_package_docker.image_ref import del_idpref
from beiran_package_docker.models import DockerImage, DockerLayer
from beiran_package_docker.models import MODEL_LIST
from beiran_package_docker.util import DockerUtil
from beiran_package_docker.api import ROUTES
from beiran_package_docker.api import Services as ApiDependencies
from beiran.config import config
from beiran.plugin import BasePackagePlugin, History
from beiran.models import Node
from beiran.daemon.peer import Peer
from .image_ref import del_idpref
from .models import DockerImage, DockerLayer
from .models import MODEL_LIST
from .util import DockerUtil
from .api import ROUTES
from .api import Services as ApiDependencies
PLUGIN_NAME = 'docker'
PLUGIN_TYPE = 'package'
......
# Beiran P2P Package Distribution Layer
# Copyright (C) 2019 Rainlab Inc & Creationline, Inc & Beiran Contributors
#
# Rainlab Inc. https://rainlab.co.jp
# Creationline, Inc. https://creationline.com">
# Beiran Contributors https://docs.beiran.io/contributors.html
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""setup.py for packaging beiran"""
from setuptools import setup, find_packages
REQUIREMENTS_PATH = 'requirements.txt'
def read_requirements(path):
"""Read requirements.txt and return a list of requirements."""
with open(path, 'r') as file:
reqs = file.read().splitlines()
return reqs
def read_long_description():
"""Read a file written about long description of the package."""
with open("README.md", "r") as file:
long_description = file.read()
return long_description
def find_beiran_packages():
"""Find beiran package."""
return find_packages(
# exclude=["plugins", "doc", "helpers",]
)
def get_version():
"""
Reads VERSION file
Returns:
(str): plugin version
"""
with open('VERSION', 'r') as file:
version = file.read()
return version
setup(
name="beiran_package_docker",
version=get_version(),
author="RainLab",
author_email="info@rainlab.co.jp",
description="Beiran Docker Plugin",
keywords=['beiran', 'docker', 'p2p package manager', 'system tools'],
long_description=read_long_description(),
long_description_content_type="text/markdown",
url="https://beiran.io",
classifiers=[
"Development Status :: 3 - Alpha",
"Intended Audience :: System Administrators",
"Topic :: System :: Software Distribution"
"Topic :: System :: Systems Administration",
"Programming Language :: Python :: 3",
"License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", # ???
"Operating System :: OS Independent",
],
install_requires=read_requirements(REQUIREMENTS_PATH),
python_requires='~=3.6',
packages=find_beiran_packages(),
data_files=[
('beiran_package_docker', ['beiran_package_docker/tar-split'])
],
)
......@@ -35,11 +35,13 @@ RUN /pyinstaller/pyinstaller.sh --onefile --name beiran \
--clean --log-level DEBUG --noconfirm \
--hidden-import beiran.cli_node \
--hidden-import beiran_package_docker.cli_docker \
--hidden-import beiran_interface_k8s \
--hidden-import beiran.plugins.discovery_dns \
--hidden-import beiran.plugins.discovery_zeroconf \
--paths /opt/beiran/pyinstaller \
--paths ../beiran \
--paths ../plugins/beiran_package_docker \
--paths ../plugins/beiran_interface_k8s \
--exclude-module pycrypto \
--exclude-module PyInstaller \
--additional-hooks-dir ./hooks/ \
......
......@@ -8,7 +8,7 @@
- variable_binds: {test_docker_layer: 'sha256:d1725b59e92d6462c6d688ef028979cc6bb150762db99d18dddc7fa54b82b0ce',
test_docker_image: 'hello-world',
test_docker_image_id: 'sha256:fce289e99eb9bca977dae136fbe2a82b6b7d4c372474c9235adc1741675f587e',
test_docker_repo_digest: 'sha256:a1b41357d6597b6e5344e57d7134624f9920d7bae99adb88eb29d16e35949567'}
test_docker_repo_digest: 'sha256:247806bbd7131f883838f880835fcd634e845bdce69207b8316fdad03e74ed22'}
- test:
- name: "Version from root handler"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment