CI: make build jobs much faster

All jobs were running as docker image builds inside of KVMs. Now the first build job runs as a regular job inside the docker image generated in the dependencies stage. This reduces the initial build jobs to around 1 minute, down from 10~ minutes. The jobs building the docker images have been moved into their own seperate stage which only runs if the first quick builds succeed.
2 years ago · 482bf39497
parent 9e7a1109cd
commit 482bf39497
6 changed files with 136 additions and 36 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -4,14 +4,33 @@ include:
 stages:
  - deps
  - build
+  - build-docker
  - packages


 # Debian

-debian10:build:
+debian10:deps:
  extends: .make_in_docker
+  stage: deps
+  variables:
+    MAKE_TARGET: "deps"
+    CI_DISTRO_TARGET: "debian10"
+    DEBIAN_FRONTEND: "noninteractive"
+  rules:
+    - changes:
+        - DEPENDENCIES
+        - scripts/debian10/deps.pEpEngine.debian10.Dockerfile
+        - scripts/common/build_pEpEngine_deps.sh
+
+debian10:build:
+  tags:
+    - linux
  stage: build
+  image: ${DOCKER_REGISTRY_HOST}/pep-debian10-engine-deps:latest
+  script:
+    - cd scripts/${CI_DISTRO_TARGET}
+    - make ${MAKE_TARGET}
  variables:
    MAKE_TARGET: "build"
    CI_DISTRO_TARGET: "debian10"
@ -20,8 +39,13 @@ debian10:build:
    - if: '$CI_COMMIT_TAG  !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'

 debian10:tagged-build:
-  extends: .make_in_docker
+  tags:
+    - linux
  stage: build
+  image: ${DOCKER_REGISTRY_HOST}/pep-debian10-engine-deps:latest
+  script:
+    - cd scripts/${CI_DISTRO_TARGET}
+    - make ${MAKE_TARGET}
  variables:
    MAKE_TARGET: "build"
    CI_DISTRO_TARGET: "debian10"
@ -30,6 +54,29 @@ debian10:tagged-build:
  rules:
    - if: '$CI_COMMIT_TAG  =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'

+debian10:build-docker:
+  extends: .make_in_docker
+  stage: build-docker
+  needs: ["debian10:build"]
+  variables:
+    MAKE_TARGET: "build-docker"
+    CI_DISTRO_TARGET: "debian10"
+    DEBIAN_FRONTEND: "noninteractive"
+  rules:
+    - if: '$CI_COMMIT_TAG  !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
+
+debian10:tagged-build-docker:
+  extends: .make_in_docker
+  stage: build-docker
+  needs: ["debian10:tagged-build"]
+  variables:
+    MAKE_TARGET: "build-docker"
+    CI_DISTRO_TARGET: "debian10"
+    DEBIAN_FRONTEND: "noninteractive"
+    TAGGED_BUILD: "true"
+  rules:
+    - if: '$CI_COMMIT_TAG  =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
+
 # CentOS

 centos8:deps:
@ -41,11 +88,18 @@ centos8:deps:
  rules:
    - changes:
        - DEPENDENCIES
+        - scripts/centos8/deps.pEpEngine.centos8.Dockerfile
+        - scripts/common/build_pEpEngine_deps.sh


 centos8:build:
-  extends: .make_in_docker
+  tags:
+    - linux
  stage: build
+  image: ${DOCKER_REGISTRY_HOST}/pep-centos8-engine-deps:latest
+  script:
+    - cd scripts/${CI_DISTRO_TARGET}
+    - make ${MAKE_TARGET}
  variables:
    MAKE_TARGET: "build"
    CI_DISTRO_TARGET: "centos8"
@ -53,8 +107,13 @@ centos8:build:
    - if: '$CI_COMMIT_TAG  !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'

 centos8:tagged-build:
-  extends: .make_in_docker
+  tags:
+    - linux
  stage: build
+  image: ${DOCKER_REGISTRY_HOST}/pep-centos8-engine-deps:latest
+  script:
+    - cd scripts/${CI_DISTRO_TARGET}
+    - make ${MAKE_TARGET}
  variables:
    MAKE_TARGET: "build"
    CI_DISTRO_TARGET: "centos8"
@ -62,10 +121,31 @@ centos8:tagged-build:
  rules:
    - if: '$CI_COMMIT_TAG  =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'

+centos8:build-docker:
+  extends: .make_in_docker
+  stage: build-docker
+  needs: ["centos8:build"]
+  variables:
+    MAKE_TARGET: "build-docker"
+    CI_DISTRO_TARGET: "centos8"
+  rules:
+    - if: '$CI_COMMIT_TAG  !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
+
+centos8:tagged-build-docker:
+  extends: .make_in_docker
+  stage: build-docker
+  needs: ["centos8:tagged-build"]
+  variables:
+    MAKE_TARGET: "build-docker"
+    CI_DISTRO_TARGET: "centos8"
+    TAGGED_BUILD: "true"
+  rules:
+    - if: '$CI_COMMIT_TAG  =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
+
 centos8:rpm:
  extends: .make_in_docker
  stage: packages
-  needs: ["centos8:build"]
+  needs: ["centos8:build-docker"]
  variables:
    MAKE_TARGET: "rpm"
    CI_DISTRO_TARGET: "centos8"
@ -76,7 +156,7 @@ centos8:rpm:
 centos8:rpm:tagged-build:
  extends: .upload_pkg
  stage: packages
-  needs: ["centos8:tagged-build"]
+  needs: ["centos8:tagged-build-docker"]
  variables:
    MAKE_TARGET: "rpm"
    CI_DISTRO_TARGET: "centos8"
--- a/scripts/centos8/Makefile
+++ b/scripts/centos8/Makefile
@ -22,7 +22,7 @@ endif
 all: deps build

 deps:
-	-docker pull $(IMAGE_NAME)-deps:latest
+	-docker pull --quiet $(IMAGE_NAME)-deps:latest
 	cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
 		      --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
 		      --build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
@ -33,11 +33,14 @@ deps:
 		      --tag=$(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION) \
 		      --tag=$(IMAGE_NAME)-deps:latest \
 		      -f scripts/${CURRENT_DISTRO}/deps.$(DOCKERFILE) .
-	docker push $(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION)
-	docker push $(IMAGE_NAME)-deps:latest
+	docker push --quiet $(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION)
+	docker push --quiet $(IMAGE_NAME)-deps:latest

 build:
-	-docker pull $(IMAGE_NAME):latest
+	cd ../../ && /bin/sh ./scripts/common/build_pEpEngine.sh
+
+build-docker:
+	-docker pull --quiet $(IMAGE_NAME):latest
 	cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
 		      --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
 		      --build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
@ -47,12 +50,12 @@ build:
 		      --cache-from $(IMAGE_NAME):latest \
 		      --tag=$(IMAGE_NAME):$(PEPENGINE_VERSION) \
 		      --tag=$(IMAGE_NAME):latest \
-		      -f scripts/${CURRENT_DISTRO}/$(DOCKERFILE) .
-	docker push $(IMAGE_NAME):$(PEPENGINE_VERSION)
-	docker push $(IMAGE_NAME):latest
+		      -f scripts/common/build.pEpEngine.Dockerfile .
+	docker push --quiet $(IMAGE_NAME):$(PEPENGINE_VERSION)
+	docker push --quiet $(IMAGE_NAME):latest

 rpm:
-	-docker pull $(PKG_BUILD_IMAGE)-engine:latest
+	-docker pull --quiet $(PKG_BUILD_IMAGE)-engine:latest
 	@docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
 		--build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
 		--build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
@ -62,8 +65,8 @@ rpm:
 		--tag=$(PKG_BUILD_IMAGE)-engine:$(PEPENGINE_VERSION) \
 		--tag=$(PKG_BUILD_IMAGE)-engine:latest \
 		packages/rpm
-	@docker push $(PKG_BUILD_IMAGE)-engine:$(PEPENGINE_VERSION)
-	@docker push $(PKG_BUILD_IMAGE)-engine:latest
+	@docker push --quiet $(PKG_BUILD_IMAGE)-engine:$(PEPENGINE_VERSION)
+	@docker push --quiet $(PKG_BUILD_IMAGE)-engine:latest
 	@docker run -e PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
 		-e PEP_MACHINE_DIR=$(PEP_MACHINE_DIR) \
 		-e PKG_VERSION=$(PEPENGINE_VERSION) \
--- a/scripts/centos8/deps.pEpEngine.centos8.Dockerfile
+++ b/scripts/centos8/deps.pEpEngine.centos8.Dockerfile
@ -17,6 +17,9 @@ USER root

 RUN yum install -y python3 python3-lxml binutils && yum clean all

+### Setup PEP_MACHINE_DIR
+RUN mkdir -p ${PEP_MACHINE_DIR}
+
 RUN chown -R pep-builder:pep-builder ${BUILDROOT}/pEpEngine
 WORKDIR ${BUILDROOT}/pEpEngine

--- a/scripts/centos8/pEpEngine.centos8.Dockerfile
+++ b/scripts/centos8/pEpEngine.centos8.Dockerfile
@ -12,7 +12,8 @@ ARG PEP_MACHINE_DIR

 ### Setup working directory
 USER root
-RUN mkdir -p ${BUILDROOT}/pEpEngine
+RUN rm -rf ${BUILDROOT}/pEpEngine && \
+    mkdir -p ${BUILDROOT}/pEpEngine
 COPY . ${BUILDROOT}/pEpEngine

 RUN chown -R pep-builder:pep-builder ${BUILDROOT}/pEpEngine
--- a/scripts/debian10/Makefile
+++ b/scripts/debian10/Makefile
@ -6,6 +6,7 @@ SEQUOIA_VERSION=$(shell echo ${sequoia} | sed 's/\//-/')
 CURRENT_DISTRO=$(shell basename $(shell pwd))
 IMAGE_NAME=${DOCKER_REGISTRY_HOST}/pep-$(CURRENT_DISTRO)-engine
 DOCKERFILE=pEpEngine.$(CURRENT_DISTRO).Dockerfile
+PKG_INSTALL_PATH=/opt/pEp
 IS_TAGGED=${TAGGED_BUILD}
 ifeq ($(IS_TAGGED), true)
 # $CI_COMMIT_TAG is a predefined environment variable from Gitlab
@ -13,10 +14,28 @@ ifeq ($(IS_TAGGED), true)
 else
 	PEPENGINE_VERSION=$(shell git rev-parse --short=8 HEAD)
 endif
-all: build
+all: deps build
+
+deps:
+	-docker pull --quiet $(IMAGE_NAME)-deps:latest
+	cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
+		      --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
+		      --build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
+		      --build-arg SEQUOIA_VERSION=$(SEQUOIA_VERSION) \
+		      --build-arg YML2_VERSION=$(YML2_VERSION) \
+		      --build-arg PEP_MACHINE_DIR=$(PEP_MACHINE_DIR) \
+		      --cache-from $(IMAGE_NAME):latest \
+		      --tag=$(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION) \
+		      --tag=$(IMAGE_NAME)-deps:latest \
+		      -f scripts/${CURRENT_DISTRO}/deps.$(DOCKERFILE) .
+	docker push --quiet $(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION)
+	docker push --quiet $(IMAGE_NAME)-deps:latest

 build:
-	-docker pull $(IMAGE_NAME):latest
+	cd ../../ && /bin/sh ./scripts/common/build_pEpEngine.sh
+
+build-docker:
+	-docker pull --quiet $(IMAGE_NAME):latest
 	cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
 		      --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
 		      --build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
@ -26,6 +45,6 @@ build:
 		      --cache-from $(IMAGE_NAME):latest \
 		      --tag=$(IMAGE_NAME):$(PEPENGINE_VERSION) \
 		      --tag=$(IMAGE_NAME):latest \
-		      -f scripts/${CURRENT_DISTRO}/$(DOCKERFILE) .
-	docker push $(IMAGE_NAME):$(PEPENGINE_VERSION)
-	docker push $(IMAGE_NAME):latest
+		      -f scripts/common/build.pEpEngine.Dockerfile .
+	docker push --quiet $(IMAGE_NAME):$(PEPENGINE_VERSION)
+	docker push --quiet $(IMAGE_NAME):latest
--- a/scripts/debian10/deps.pEpEngine.debian10.Dockerfile
+++ b/scripts/debian10/deps.pEpEngine.debian10.Dockerfile
@ -11,10 +11,16 @@ ARG PEP_MACHINE_DIR

 ### Setup working directory
 RUN mkdir ${BUILDROOT}/pEpEngine
-COPY . ${BUILDROOT}/pEpEngine
+COPY ./scripts/common/build_pEpEngine_deps.sh ${BUILDROOT}/pEpEngine

 USER root

+RUN apt-get update && apt-get install -y bzip2 && \
+     rm -rf /var/lib/apt/lists/*
+
+### Setup PEP_MACHINE_DIR
+RUN mkdir -p ${PEP_MACHINE_DIR}
+
 RUN chown -R pep-builder:pep-builder ${BUILDROOT}/pEpEngine
 WORKDIR ${BUILDROOT}/pEpEngine

@ -22,19 +28,7 @@ ARG YML2_VERSION
 ARG ENGINE_VERSION
 ARG CURRENT_DISTRO

-RUN  apt-get update && apt-get install -y bzip2 && \
-     rm -rf /var/lib/apt/lists/*
-
 ### Build pEpEngine dependencies
 USER pep-builder

-RUN sh ./scripts/common/build_pEpEngine_deps.sh
-
-### Build pEpEngine
-RUN sh ./scripts/common/build_pEpEngine.sh
-
-### Install Systemdb
-USER root
-
-RUN sh ./scripts/common/install_pEpEngine_systemdb.sh && \
-    rm -rf ${BUILDROOT}/*
+RUN sh ./build_pEpEngine_deps.sh