Browse Source

CI: make build jobs much faster

All jobs were running as docker image builds inside of KVMs. Now the
first build job runs as a regular job inside the docker image generated
in the dependencies stage. This reduces the initial build jobs to around
1 minute, down from 10~ minutes.

The jobs building the docker images have been moved into their own
seperate stage which only runs if the first quick builds succeed.
master
Devan Carpenter 3 weeks ago
parent
commit
482bf39497
6 changed files with 136 additions and 36 deletions
  1. +86
    -6
      .gitlab-ci.yml
  2. +13
    -10
      scripts/centos8/Makefile
  3. +3
    -0
      scripts/centos8/deps.pEpEngine.centos8.Dockerfile
  4. +2
    -1
      scripts/common/build.pEpEngine.Dockerfile
  5. +24
    -5
      scripts/debian10/Makefile
  6. +8
    -14
      scripts/debian10/deps.pEpEngine.debian10.Dockerfile

+ 86
- 6
.gitlab-ci.yml View File

@ -4,14 +4,33 @@ include:
stages:
- deps
- build
- build-docker
- packages
# Debian
debian10:build:
debian10:deps:
extends: .make_in_docker
stage: deps
variables:
MAKE_TARGET: "deps"
CI_DISTRO_TARGET: "debian10"
DEBIAN_FRONTEND: "noninteractive"
rules:
- changes:
- DEPENDENCIES
- scripts/debian10/deps.pEpEngine.debian10.Dockerfile
- scripts/common/build_pEpEngine_deps.sh
debian10:build:
tags:
- linux
stage: build
image: ${DOCKER_REGISTRY_HOST}/pep-debian10-engine-deps:latest
script:
- cd scripts/${CI_DISTRO_TARGET}
- make ${MAKE_TARGET}
variables:
MAKE_TARGET: "build"
CI_DISTRO_TARGET: "debian10"
@ -20,8 +39,13 @@ debian10:build:
- if: '$CI_COMMIT_TAG !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
debian10:tagged-build:
extends: .make_in_docker
tags:
- linux
stage: build
image: ${DOCKER_REGISTRY_HOST}/pep-debian10-engine-deps:latest
script:
- cd scripts/${CI_DISTRO_TARGET}
- make ${MAKE_TARGET}
variables:
MAKE_TARGET: "build"
CI_DISTRO_TARGET: "debian10"
@ -30,6 +54,29 @@ debian10:tagged-build:
rules:
- if: '$CI_COMMIT_TAG =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
debian10:build-docker:
extends: .make_in_docker
stage: build-docker
needs: ["debian10:build"]
variables:
MAKE_TARGET: "build-docker"
CI_DISTRO_TARGET: "debian10"
DEBIAN_FRONTEND: "noninteractive"
rules:
- if: '$CI_COMMIT_TAG !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
debian10:tagged-build-docker:
extends: .make_in_docker
stage: build-docker
needs: ["debian10:tagged-build"]
variables:
MAKE_TARGET: "build-docker"
CI_DISTRO_TARGET: "debian10"
DEBIAN_FRONTEND: "noninteractive"
TAGGED_BUILD: "true"
rules:
- if: '$CI_COMMIT_TAG =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
# CentOS
centos8:deps:
@ -41,11 +88,18 @@ centos8:deps:
rules:
- changes:
- DEPENDENCIES
- scripts/centos8/deps.pEpEngine.centos8.Dockerfile
- scripts/common/build_pEpEngine_deps.sh
centos8:build:
extends: .make_in_docker
tags:
- linux
stage: build
image: ${DOCKER_REGISTRY_HOST}/pep-centos8-engine-deps:latest
script:
- cd scripts/${CI_DISTRO_TARGET}
- make ${MAKE_TARGET}
variables:
MAKE_TARGET: "build"
CI_DISTRO_TARGET: "centos8"
@ -53,8 +107,13 @@ centos8:build:
- if: '$CI_COMMIT_TAG !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
centos8:tagged-build:
extends: .make_in_docker
tags:
- linux
stage: build
image: ${DOCKER_REGISTRY_HOST}/pep-centos8-engine-deps:latest
script:
- cd scripts/${CI_DISTRO_TARGET}
- make ${MAKE_TARGET}
variables:
MAKE_TARGET: "build"
CI_DISTRO_TARGET: "centos8"
@ -62,10 +121,31 @@ centos8:tagged-build:
rules:
- if: '$CI_COMMIT_TAG =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
centos8:build-docker:
extends: .make_in_docker
stage: build-docker
needs: ["centos8:build"]
variables:
MAKE_TARGET: "build-docker"
CI_DISTRO_TARGET: "centos8"
rules:
- if: '$CI_COMMIT_TAG !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
centos8:tagged-build-docker:
extends: .make_in_docker
stage: build-docker
needs: ["centos8:tagged-build"]
variables:
MAKE_TARGET: "build-docker"
CI_DISTRO_TARGET: "centos8"
TAGGED_BUILD: "true"
rules:
- if: '$CI_COMMIT_TAG =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
centos8:rpm:
extends: .make_in_docker
stage: packages
needs: ["centos8:build"]
needs: ["centos8:build-docker"]
variables:
MAKE_TARGET: "rpm"
CI_DISTRO_TARGET: "centos8"
@ -76,7 +156,7 @@ centos8:rpm:
centos8:rpm:tagged-build:
extends: .upload_pkg
stage: packages
needs: ["centos8:tagged-build"]
needs: ["centos8:tagged-build-docker"]
variables:
MAKE_TARGET: "rpm"
CI_DISTRO_TARGET: "centos8"


+ 13
- 10
scripts/centos8/Makefile View File

@ -22,7 +22,7 @@ endif
all: deps build
deps:
-docker pull $(IMAGE_NAME)-deps:latest
-docker pull --quiet $(IMAGE_NAME)-deps:latest
cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
--build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
--build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
@ -33,11 +33,14 @@ deps:
--tag=$(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION) \
--tag=$(IMAGE_NAME)-deps:latest \
-f scripts/${CURRENT_DISTRO}/deps.$(DOCKERFILE) .
docker push $(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION)
docker push $(IMAGE_NAME)-deps:latest
docker push --quiet $(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION)
docker push --quiet $(IMAGE_NAME)-deps:latest
build:
-docker pull $(IMAGE_NAME):latest
cd ../../ && /bin/sh ./scripts/common/build_pEpEngine.sh
build-docker:
-docker pull --quiet $(IMAGE_NAME):latest
cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
--build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
--build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
@ -47,12 +50,12 @@ build:
--cache-from $(IMAGE_NAME):latest \
--tag=$(IMAGE_NAME):$(PEPENGINE_VERSION) \
--tag=$(IMAGE_NAME):latest \
-f scripts/${CURRENT_DISTRO}/$(DOCKERFILE) .
docker push $(IMAGE_NAME):$(PEPENGINE_VERSION)
docker push $(IMAGE_NAME):latest
-f scripts/common/build.pEpEngine.Dockerfile .
docker push --quiet $(IMAGE_NAME):$(PEPENGINE_VERSION)
docker push --quiet $(IMAGE_NAME):latest
rpm:
-docker pull $(PKG_BUILD_IMAGE)-engine:latest
-docker pull --quiet $(PKG_BUILD_IMAGE)-engine:latest
@docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
--build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
--build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
@ -62,8 +65,8 @@ rpm:
--tag=$(PKG_BUILD_IMAGE)-engine:$(PEPENGINE_VERSION) \
--tag=$(PKG_BUILD_IMAGE)-engine:latest \
packages/rpm
@docker push $(PKG_BUILD_IMAGE)-engine:$(PEPENGINE_VERSION)
@docker push $(PKG_BUILD_IMAGE)-engine:latest
@docker push --quiet $(PKG_BUILD_IMAGE)-engine:$(PEPENGINE_VERSION)
@docker push --quiet $(PKG_BUILD_IMAGE)-engine:latest
@docker run -e PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
-e PEP_MACHINE_DIR=$(PEP_MACHINE_DIR) \
-e PKG_VERSION=$(PEPENGINE_VERSION) \


+ 3
- 0
scripts/centos8/deps.pEpEngine.centos8.Dockerfile View File

@ -17,6 +17,9 @@ USER root
RUN yum install -y python3 python3-lxml binutils && yum clean all
### Setup PEP_MACHINE_DIR
RUN mkdir -p ${PEP_MACHINE_DIR}
RUN chown -R pep-builder:pep-builder ${BUILDROOT}/pEpEngine
WORKDIR ${BUILDROOT}/pEpEngine


scripts/centos8/pEpEngine.centos8.Dockerfile → scripts/common/build.pEpEngine.Dockerfile View File


+ 24
- 5
scripts/debian10/Makefile View File

@ -6,6 +6,7 @@ SEQUOIA_VERSION=$(shell echo ${sequoia} | sed 's/\//-/')
CURRENT_DISTRO=$(shell basename $(shell pwd))
IMAGE_NAME=${DOCKER_REGISTRY_HOST}/pep-$(CURRENT_DISTRO)-engine
DOCKERFILE=pEpEngine.$(CURRENT_DISTRO).Dockerfile
PKG_INSTALL_PATH=/opt/pEp
IS_TAGGED=${TAGGED_BUILD}
ifeq ($(IS_TAGGED), true)
# $CI_COMMIT_TAG is a predefined environment variable from Gitlab
@ -13,10 +14,28 @@ ifeq ($(IS_TAGGED), true)
else
PEPENGINE_VERSION=$(shell git rev-parse --short=8 HEAD)
endif
all: build
all: deps build
deps:
-docker pull --quiet $(IMAGE_NAME)-deps:latest
cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
--build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
--build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
--build-arg SEQUOIA_VERSION=$(SEQUOIA_VERSION) \
--build-arg YML2_VERSION=$(YML2_VERSION) \
--build-arg PEP_MACHINE_DIR=$(PEP_MACHINE_DIR) \
--cache-from $(IMAGE_NAME):latest \
--tag=$(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION) \
--tag=$(IMAGE_NAME)-deps:latest \
-f scripts/${CURRENT_DISTRO}/deps.$(DOCKERFILE) .
docker push --quiet $(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION)
docker push --quiet $(IMAGE_NAME)-deps:latest
build:
-docker pull $(IMAGE_NAME):latest
cd ../../ && /bin/sh ./scripts/common/build_pEpEngine.sh
build-docker:
-docker pull --quiet $(IMAGE_NAME):latest
cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
--build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
--build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
@ -26,6 +45,6 @@ build:
--cache-from $(IMAGE_NAME):latest \
--tag=$(IMAGE_NAME):$(PEPENGINE_VERSION) \
--tag=$(IMAGE_NAME):latest \
-f scripts/${CURRENT_DISTRO}/$(DOCKERFILE) .
docker push $(IMAGE_NAME):$(PEPENGINE_VERSION)
docker push $(IMAGE_NAME):latest
-f scripts/common/build.pEpEngine.Dockerfile .
docker push --quiet $(IMAGE_NAME):$(PEPENGINE_VERSION)
docker push --quiet $(IMAGE_NAME):latest

scripts/debian10/pEpEngine.debian10.Dockerfile → scripts/debian10/deps.pEpEngine.debian10.Dockerfile View File


Loading…
Cancel
Save