.gitlab-ci.yml 11 KB
Newer Older
1 2
image: dkr.rlab.io/system/thedocker:latest

Furkan Mustafa's avatar
Furkan Mustafa committed
3
stages:
4 5
 - pre_build
 - checks
Furkan Mustafa's avatar
Furkan Mustafa committed
6
 - prepare
Furkan Mustafa's avatar
Furkan Mustafa committed
7
 - test
8
 - post-test
9
 - publish
10 11 12 13 14 15 16 17
 - cleanup

pre_build:daemon:
  stage: pre_build
  tags:
   - docker
  script:
   - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" $CI_REGISTRY
18 19 20
   - docker pull $CI_REGISTRY_IMAGE:latest || true
   - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME || true
   - docker build --cache-from $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME,$CI_REGISTRY_IMAGE:latest -t $CI_REGISTRY_IMAGE:ci-${CI_PIPELINE_ID} -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME .
21
   - docker push $CI_REGISTRY_IMAGE:ci-${CI_PIPELINE_ID}
22
   - docker push $CI_REGISTRY_IMAGE:${CI_COMMIT_REF_NAME}
23 24 25 26 27 28 29

pre_build:checker:
  stage: pre_build
  tags:
   - docker
  script:
   - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" $CI_REGISTRY
30 31
   - docker pull $CI_REGISTRY_IMAGE/checks:latest || true
   - docker build --cache-from $CI_REGISTRY_IMAGE/checks:latest -t $CI_REGISTRY_IMAGE/checks:ci-${CI_PIPELINE_ID} -t $CI_REGISTRY_IMAGE/checks:latest -f helpers/docker/checks/Dockerfile .
32
   - docker push $CI_REGISTRY_IMAGE/checks:ci-${CI_PIPELINE_ID}
33
   - docker push $CI_REGISTRY_IMAGE/checks:latest
34

35
package:docs:
Furkan Mustafa's avatar
Furkan Mustafa committed
36
  stage: prepare
37 38 39 40 41 42 43 44 45
  tags:
  - docker
  image: $CI_REGISTRY_IMAGE:ci-${CI_PIPELINE_ID}
  script:
  - cd doc/
  - pip3 install -r requirements.txt
  - make html
  cache:
    key: doc-build-cache
Furkan Mustafa's avatar
Furkan Mustafa committed
46
    policy: push
47 48 49
    paths:
    - doc/

Furkan Mustafa's avatar
Furkan Mustafa committed
50 51 52 53 54 55 56 57 58 59
publish:dev-docs:
  stage: publish
  tags:
  - docker
  script:
  - cd doc/build/html
  - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" $CI_REGISTRY
  - echo -e "FROM nginx:latest\nADD . /usr/share/nginx/html/\n" > Dockerfile
  - docker build -t ${CI_REGISTRY_IMAGE}/docs:${CI_COMMIT_REF_SLUG} -t ${CI_REGISTRY_IMAGE}/docs:dev .
  # temporarily pushing all docs for now
60 61 62
  - docker push ${CI_REGISTRY_IMAGE}/docs:${CI_COMMIT_REF_SLUG}
  # TODO: Only if latest dev version?
  - docker push ${CI_REGISTRY_IMAGE}/docs:dev
63
  - curl --insecure https://gitlab.beiran.io:16434
Furkan Mustafa's avatar
Furkan Mustafa committed
64 65 66 67 68 69
  cache:
    key: doc-build-cache
    policy: pull
    paths:
    - doc/
  only:
Furkan Mustafa's avatar
Furkan Mustafa committed
70
    - /^dev-.*/
Furkan Mustafa's avatar
Furkan Mustafa committed
71

72 73 74 75 76 77 78 79
publish:docs:
  stage: publish
  tags:
  - docker
  script:
  - cd doc/build/html
  - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" $CI_REGISTRY
  - echo -e "FROM nginx:latest\nADD . /usr/share/nginx/html/\n" > Dockerfile
Furkan Mustafa's avatar
Furkan Mustafa committed
80
  - docker build -t $CI_REGISTRY_IMAGE/docs:${CI_COMMIT_TAG} .
81
  # temporarily pushing all docs for now
Furkan Mustafa's avatar
Furkan Mustafa committed
82
  - docker push $CI_REGISTRY_IMAGE/docs:${CI_COMMIT_TAG}
83
  - curl --insecure https://gitlab.beiran.io:16434
84 85 86 87 88
  cache:
    key: doc-build-cache
    policy: pull
    paths:
    - doc/
Furkan Mustafa's avatar
Furkan Mustafa committed
89 90
  only:
    - tags
91

92 93 94 95 96
publish:oci-image-layout:
  stage: publish
  tags:
  - docker
  script:
97 98 99 100
    - echo http://dl-cdn.alpinelinux.org/alpine/edge/community/ >> /etc/apk/repositories
    - apk update
    - apk add go=1.11.5-r0
    - apk add musl-dev
101
    - go get github.com/coolljt0725/docker2oci
102
    - docker save $CI_REGISTRY_IMAGE:ci-${CI_PIPELINE_ID} | $HOME/go/bin/docker2oci image-layout-${CI_COMMIT_TAG}
103
    # upload image_layer to minio
104
    - tar czf image-layout-${CI_COMMIT_TAG}.tar.gz image-layout-${CI_COMMIT_TAG}
105
    - go get github.com/minio/mc
Shota Nakamura's avatar
Shota Nakamura committed
106
    - $HOME/go/bin/mc config host add beiran https://minio.beiran.io $MINIO_ACCESS_KEY $MINIO_SECRET_KEY
107 108 109
    - $HOME/go/bin/mc cp image-layout-${CI_COMMIT_TAG}.tar.gz beiran/releases
  only:
    - tags
110

111
checks:lint-beiran:
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
  stage: checks
  image: $CI_REGISTRY_IMAGE/checks:ci-${CI_PIPELINE_ID}
  tags:
   - docker
  script:
   - pylint beiran

checks:lint-plugins:
  stage: checks
  image: $CI_REGISTRY_IMAGE/checks:ci-${CI_PIPELINE_ID}
  tags:
   - docker
  script:
   - pylint plugins

127
checks:mypy-beiran:
128 129 130 131 132
  stage: checks
  image: $CI_REGISTRY_IMAGE/checks:ci-${CI_PIPELINE_ID}
  tags:
   - docker
  script:
133
   - helpers/mypy --config-file mypy.ini beiran
134 135 136 137 138 139 140

checks:mypy-plugins:
  stage: checks
  image: $CI_REGISTRY_IMAGE/checks:ci-${CI_PIPELINE_ID}
  tags:
   - docker
  script:
141
   - helpers/mypy --config-file mypy.ini plugins
142

143
checks:bandit-beiran:
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
  stage: checks
  image: $CI_REGISTRY_IMAGE/checks:ci-${CI_PIPELINE_ID}
  tags:
   - docker
  script:
   - bandit -r beiran
  allow_failure: true

checks:bandit-plugins:
  stage: checks
  image: $CI_REGISTRY_IMAGE/checks:ci-${CI_PIPELINE_ID}
  tags:
   - docker
  script:
   - bandit -r plugins
  allow_failure: true

161
start_test_services:
Furkan Mustafa's avatar
Furkan Mustafa committed
162
  stage: prepare
163 164 165 166
  tags:
   - docker
  script:
   - SRC_DIR=$(pwd)
167 168
   - mkdir -p /srv/builds/tmp/beirantest-${CI_PIPELINE_ID}
   - cd /srv/builds/tmp/beirantest-${CI_PIPELINE_ID}
169
   - cp $SRC_DIR/docker-compose.multipod.yml ./docker-compose.yml
170
   - sed -i "s#dkr\.beiran\.io\/beiran\/beiran:dev#${CI_REGISTRY_IMAGE}:ci-${CI_PIPELINE_ID}#g" docker-compose.yml
171
   - docker-compose up -d pod1 docker1 pod2 docker2 pod3 docker3 pod4 docker4
172 173
   - sleep 5
   # seed docker daemon (docker1 only)
174
   - docker-compose exec -T docker1 docker pull hello-world@sha256:0add3ace90ecb4adbf7777e9aacf18357296e799f81cabc9fde470971e499788
175
   - docker-compose exec -T docker1 docker tag hello-world@sha256:0add3ace90ecb4adbf7777e9aacf18357296e799f81cabc9fde470971e499788 hello-world:latest
176
   - docker-compose exec -T docker1 docker pull alpine@sha256:621c2f39f8133acb8e64023a94dbdf0d5ca81896102b9e57c0dc184cadaf5528
177
   - docker-compose exec -T docker1 docker tag alpine@sha256:621c2f39f8133acb8e64023a94dbdf0d5ca81896102b9e57c0dc184cadaf5528 alpine:latest
178 179
   - docker-compose exec -T docker1 docker images
   # starting beiran
180
   - docker-compose up -d beiran1 beiran2 beiran3 beiran4
181 182
   # allow beiran to start up (might not be enough)
   - sleep 5
183

184 185 186 187 188 189 190
   # build and copy tar-split
   - echo http://dl-cdn.alpinelinux.org/alpine/edge/community/ >> /etc/apk/repositories
   - apk update
   - apk add go=1.11.5-r0
   - apk add musl-dev
   - go get -d github.com/vbatts/tar-split/cmd/tar-split
   - go build -o $HOME/tar-split -a -ldflags '-extldflags "-static"' $HOME/go/src/github.com/vbatts/tar-split/cmd/tar-split
191 192 193 194 195 196
   - |
     for node in beiran1 beiran2 beiran3 beiran4
     do
        docker cp $HOME/tar-split $(docker-compose ps -q $node):/opt/beiran/plugins/beiran_package_docker/tar-split
     done

197 198 199 200 201 202 203 204
test:unit:
  stage: test
  image: $CI_REGISTRY_IMAGE:ci-${CI_PIPELINE_ID}
  tags:
   - docker
  script:
   - pytest

205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
test:basic-behavior:
  stage: test
  tags:
   - docker
  script:
   - SRC_DIR=$(pwd)
   - cd /srv/builds/tmp/beirantest-${CI_PIPELINE_ID}
   - docker-compose ps
   - docker-compose logs beiran1
   - |
     echo "=========================================="
     echo "waiting until beirand is accessible"
     times=0
     until $(docker-compose exec -T beiran1 curl --output /dev/null --silent --fail http://localhost:8888); do
         printf '.'
         sleep 3
         times=$(( $times + 1 ))
         if [ $times -eq 10 ]; then
           2>&1 echo "timed out waiting for beirand"
           exit 1
         fi
     done
   - CONTAINER=$(docker-compose ps -q beiran1)
228
   - docker exec ${CONTAINER} python -m beiran node info
229 230 231 232 233 234
   - docker exec ${CONTAINER} python -m beiran node list
   - docker exec ${CONTAINER} python -m beiran docker image list --all
   # TODO: Check if output has expected images
   - docker exec ${CONTAINER} python -m beiran docker layer list --all
   # TODO: Check if output has expected layers!?

235 236 237 238 239
test:api:
  stage: test
  tags:
   - docker
  script:
240
   - SRC_DIR=$(pwd)
241
   - cd /srv/builds/tmp/beirantest-${CI_PIPELINE_ID}
242 243
   - docker-compose ps
   - docker-compose logs beiran1
244
   - |
245
     echo "=========================================="
246 247
     echo "waiting until beirand is accessible"
     times=0
248
     until $(docker-compose exec -T beiran1 curl --output /dev/null --silent --fail http://localhost:8888); do
249 250 251 252 253 254 255 256
         printf '.'
         sleep 3
         times=$(( $times + 1 ))
         if [ $times -eq 10 ]; then
           2>&1 echo "timed out waiting for beirand"
           exit 1
         fi
     done
257
   - CONTAINER=$(docker-compose ps -q beiran1)
258 259 260 261
   - docker exec ${CONTAINER} pip install pyresttest==1.7.1
   # send test files to the container
   - cd ${SRC_DIR}
   - tar c $(find test/rest_api -name "*.yaml") | docker exec -i ${CONTAINER} tar xv -C /opt
Furkan Mustafa's avatar
Furkan Mustafa committed
262 263
   # we make beiran download an image from origin, and cache the layers in
   # beiran's own cache folder, so we can test it in api
264
   - docker exec ${CONTAINER} python -m beiran docker image pull --wait hello-world@sha256:0add3ace90ecb4adbf7777e9aacf18357296e799f81cabc9fde470971e499788
265
   - docker exec ${CONTAINER} pyresttest --url http://localhost:8888 --test /opt/test/rest_api/beirand.yaml
266

267 268 269 270 271
test:bats:
  stage: test
  tags:
   - docker
  script:
272 273
   - |
     SRC_DIR=$(pwd)
274 275 276
     mkdir ${SRC_DIR}/temp
     git clone https://github.com/sstephenson/bats.git
     bats/install.sh ${SRC_DIR}/temp
277
   - cd /srv/builds/tmp/beirantest-${CI_PIPELINE_ID}
278
   - docker-compose ps
279
   - docker-compose logs beiran2 beiran3 beiran4
280 281 282
   - |
     echo "=========================================="
     echo "waiting until beirand is accessible"
283
     for node in beiran1 beiran2 beiran3 beiran4
284 285
     do
       times=0
286
       docker-compose exec -T $node apt-get update
287 288
       docker-compose exec -T $node apt-get install sqlite3 -y
       until $(docker-compose -f docker-compose.yml exec -T $node curl --output /dev/null --silent --fail http://localhost:8888); do
289 290 291 292 293 294 295 296
           printf '.'
           sleep 3
           times=$(( $times + 1 ))
           if [ $times -eq 10 ]; then
             2>&1 echo "timed out waiting for beirand"
             exit 1
           fi
       done
297
     done
298
   - |
299 300 301 302
     for node in beiran2 beiran3 beiran4
     do
       CONTAINER=$(docker-compose ps -q $node)
       docker cp ${SRC_DIR}/test/bats/beiran-cli-$node.bats ${CONTAINER}:/opt/beiran-cli.bats
303
       docker cp ${SRC_DIR}/test/bats/beiran-node-discovery.bats ${CONTAINER}:/opt/beiran-node-discovery.bats
304 305 306
       docker cp ${SRC_DIR}/temp ${CONTAINER}:/usr/local/
       echo "============== start bats test ($node) =============="
       docker exec ${CONTAINER} /usr/local/temp/bin/bats /opt/beiran-cli.bats
307
       docker exec ${CONTAINER} /usr/local/temp/bin/bats /opt/beiran-node-discovery.bats
308
     done
309

310 311 312 313 314
post-test:error-logs:
  stage: post-test
  tags:
   - docker
  script: |
Furkan Mustafa's avatar
Furkan Mustafa committed
315
   set -x
316
   cd /srv/builds/tmp/beirantest-${CI_PIPELINE_ID}
317
   BEIRAN_ERROR_COUNT=$(docker-compose logs | grep '[beiran] ERROR' || true | wc -l)
318 319 320 321 322
   if [ $BEIRAN_ERROR_COUNT -gt 0 ]; then
     echo "Errors in beiran logs, please check"
     exit 1
   fi

323 324 325 326 327 328
cleanup:services:
  stage: cleanup
  tags:
   - docker
  when: always
  script:
329
   - cd /srv/builds/tmp/beirantest-${CI_PIPELINE_ID}
330 331
   # dump logs for debugging
   - docker-compose logs
332
   - docker-compose kill || true
333
   - docker-compose rm -f -v
334
   - docker network rm beirantest-${CI_PIPELINE_ID}_test
335
   - cd /
336
   - rm -rf /srv/builds/tmp/beirantest-${CI_PIPELINE_ID}
337

338 339 340 341
# cleanup:
#   image: alpine:3.7
#   FIXME: volumes is not supported by gitlab-ci yet
#   volumes:
342
#    - /cache/beiran-build-vols:/cache/beiran-build-vols
343 344
#   commands:
#    - echo "Cleaning up docker storage"
345
#    - export B_DIR=/cache/beiran-build-vols/dockerd1-${CI_PIPELINE_ID} 
346 347 348 349
#    - if [ ! -d $B_DIR ]; then exit 0; fi
#    - du -ch -d1 $B_DIR
#    - rm -rf $B_DIR
#   when: always