CI: build CentOS/RedHat 8 RPM packages

This commit cleans up the existing CI scripts a bit while adding jobs to
build RPM packages. The directory structure allows for easily adding more
package targets.

RPM outputs will only be uploaded on tagged commits.

.gitlab-ci-files/common-prepare.yml

@@ -8,15 +8,44 @@
   - 'which rsync || ( sudo apt-get update -y && sudo apt-get install rsync -y )'
   - 'which make || ( sudo apt-get update -y && sudo apt-get install make -y )'
 
+.add_ssh_keys: &add_ssh_keys
+  # Add the SSH key (stored in the SSH_PRIVATE_KEY variable) to the agent.
+  - eval $(ssh-agent -s)
+  - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+
+.verify_ssh_host_keys: &verify_ssh_host_keys
+  # Verify SSH host keys
+  - mkdir -p ~/.ssh
+  - chmod 700 ~/.ssh
+  - echo "${SSH_KNOWN_HOSTS}" >> ~/.ssh/known_hosts
+  - chmod 644 ~/.ssh/known_hosts
+
 .standard_job:
   tags: [kvm]
   before_script:
     - *ensure_docker
-    - *ensure_rsync
 
 .make_in_docker:
   extends: .standard_job
   script:
     - docker login -u ${DOCKER_REGISTRY_USER} -p ${DOCKER_REGISTRY_PASS} ${DOCKER_REGISTRY_HOST}
-    - cd scripts/${CI_DISTRO_TARGET}
-    - make
+    - cd scripts
+    - make ${MAKE_TARGET}
+
+.upload_pkg:
+  extends: .standard_job
+  before_script:
+    - *ensure_rsync
+    - *add_ssh_keys
+    - *verify_ssh_host_keys
+  script:
+    - docker login -u ${DOCKER_REGISTRY_USER} -p ${DOCKER_REGISTRY_PASS} ${DOCKER_REGISTRY_HOST}
+    - cd scripts
+    - make ${MAKE_TARGET}
+    - pwd
+    - cd out
+    - time rsync -avP -e "ssh -p ${PKG_HOST_SSH_PORT}" depot@${PKG_HOST}:files/pkgs/RHEL/8/libpEpAdapter/SHA256SUMS || true
+    - if sha256sum --ignore-missing --check SHA256SUMS ; then echo "Package already exists... Exiting..." && exit ; else true ; fi
+    - sha256sum ./*.rpm | tee --append SHA256SUMS
+    - cat ./SHA256SUMS
+    - time rsync -azvP -e "ssh -p ${PKG_HOST_SSH_PORT}" --rsync-path="mkdir -p /home/depot/files/pkgs/RHEL/8/libpEpAdapter && rsync" ./*.rpm SHA256SUMS depot@${PKG_HOST}:files/pkgs/RHEL/8/libpEpAdapter/

.gitlab-ci.yml

@@ -3,6 +3,7 @@ include:
 
 stages:
   - build
+  - packages
 
 
 # Debian
@@ -11,6 +12,7 @@ debian10:build:
   extends: .make_in_docker
   stage: build
   variables:
+    MAKE_TARGET: "build"
     CI_DISTRO_TARGET: "debian10"
     DEBIAN_FRONTEND: "noninteractive"
   rules:
@@ -20,8 +22,52 @@ debian10:tagged-build:
   extends: .make_in_docker
   stage: build
   variables:
+    MAKE_TARGET: "build"
     CI_DISTRO_TARGET: "debian10"
     DEBIAN_FRONTEND: "noninteractive"
     TAGGED_BUILD: "true"
   rules:
     - if: '$CI_COMMIT_TAG  =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
+
+# CentOS
+
+centos8:build:
+  extends: .make_in_docker
+  stage: build
+  variables:
+    MAKE_TARGET: "build"
+    CI_DISTRO_TARGET: "centos8"
+  rules:
+    - if: '$CI_COMMIT_TAG  !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
+
+centos8:tagged-build:
+  extends: .make_in_docker
+  stage: build
+  variables:
+    MAKE_TARGET: "build"
+    CI_DISTRO_TARGET: "centos8"
+    TAGGED_BUILD: "true"
+  rules:
+    - if: '$CI_COMMIT_TAG  =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
+
+centos8:rpm:
+  extends: .make_in_docker
+  stage: packages
+  needs: ["centos8:build"]
+  variables:
+    MAKE_TARGET: "rpm"
+    CI_DISTRO_TARGET: "centos8"
+  rules:
+    - if: '$CI_COMMIT_TAG  !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
+
+
+centos8:rpm:tagged-build:
+  extends: .upload_pkg
+  stage: packages
+  needs: ["centos8:tagged-build"]
+  variables:
+    MAKE_TARGET: "rpm"
+    CI_DISTRO_TARGET: "centos8"
+    TAGGED_BUILD: "true"
+  rules:
+    - if: '$CI_COMMIT_TAG  =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'

scripts/Makefile

@@ -0,0 +1,56 @@
+include ../DEPENDENCIES
+export
+CURRENT_DISTRO=${CI_DISTRO_TARGET}
+IMAGE_NAME=${DOCKER_REGISTRY_HOST}/pep-$(CI_DISTRO_TARGET)-libpepadapter
+DOCKERFILE=libpEpAdapter.linux.Dockerfile
+PKG_BUILD_IMAGE=${DOCKER_REGISTRY_HOST}/fpm-$(CURRENT_DISTRO)
+PKG_INSTALL_PATH=/opt/pEp
+PKG_NAME=libpEpAdapter
+PKG_DESCRIPTION="p≡p libpEpAdapter Binary RPM Package"
+PKG_DEPENDS=pEpEngine
+PKG_INSTALL_PATH_STRING="/ /package/lib=${PKG_INSTALL_PATH} /package/include/pEp=${PKG_INSTALL_PATH}/include"
+IS_TAGGED=${TAGGED_BUILD}
+ifeq ($(IS_TAGGED), true)
+# $CI_COMMIT_TAG is a predefined environment variable from Gitlab
+	LIBPEPADAPTER_VERSION=${CI_COMMIT_TAG}
+else
+	LIBPEPADAPTER_VERSION=$(shell git rev-parse --short=8 HEAD)
+endif
+all: build
+
+build:
+	-docker pull $(IMAGE_NAME):latest
+	cd ../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
+		  --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
+		  --build-arg LIBPEPADAPTER_VERSION=$(LIBPEPADAPTER_VERSION) \
+		  --cache-from $(IMAGE_NAME):latest \
+		  --tag=$(IMAGE_NAME):$(LIBPEPADAPTER_VERSION) \
+		  --tag=$(IMAGE_NAME):latest \
+		  -f scripts/common/$(DOCKERFILE) .
+	docker push $(IMAGE_NAME):${LIBPEPADAPTER_VERSION}
+	docker push $(IMAGE_NAME):latest
+
+rpm:
+	-docker pull $(PKG_BUILD_IMAGE)-libpepadapter:latest
+	@docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
+		--build-arg LIBPEPADAPTER_VERSION=$(LIBPEPADAPTER_VERSION) \
+		--build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
+		--build-arg PKG_INSTALL_PATH=$(PKG_INSTALL_PATH) \
+		--cache-from $(PKG_BUILD_IMAGE)-engine:latest \
+		--tag=$(PKG_BUILD_IMAGE)-libpepadapter:$(LIBPEPADAPTER_VERSION) \
+		--tag=$(PKG_BUILD_IMAGE)-libpepadapter:latest \
+		centos8/packages/rpm
+	@docker push $(PKG_BUILD_IMAGE)-libpepadapter:$(LIBPEPADAPTER_VERSION)
+	@docker push $(PKG_BUILD_IMAGE)-libpepadapter:latest
+	@docker run -e LIBPEPADAPTER_VERSION=$(LIBPEPADAPTER_VERSION) \
+		-e PEP_MACHINE_DIR=$(PEP_MACHINE_DIR) \
+		-e PKG_VERSION=$(LIBPEPADAPTER_VERSION) \
+		-e PKG_INSTALL_PATH=$(PKG_INSTALL_PATH) \
+		-e PKG_NAME=$(PKG_NAME) \
+		-e PKG_DESCRIPTION=$(PKG_DESCRIPTION) \
+		-e PKG_DEPENDS=$(PKG_DEPENDS) \
+		-e PKG_INSTALL_PATH_STRING=$(PKG_INSTALL_PATH_STRING) \
+		--rm -v $(shell pwd)/centos8/packages/rpm/create-libpepadapter-rpm.sh:/usr/bin/create-rpm.sh:ro  \
+		-v $(shell pwd)/out:/out \
+		-w / $(PKG_BUILD_IMAGE)-libpepadapter:${LIBPEPADAPTER_VERSION} \
+		/usr/bin/create-rpm.sh

scripts/centos8/Makefile

@@ -1,15 +0,0 @@
-CURRENT_DISTRO=$(shell basename $(shell pwd))
-LIBPEPADAPTER_VERSION=$(shell git rev-parse --short=8 HEAD)
-IMAGE_NAME=${DOCKER_REGISTRY_HOST}/pep-$(CURRENT_DISTRO)-libpepadapter
-DOCKERFILE=libpEpAdapter.$(CURRENT_DISTRO).Dockerfile
-all:
-	-docker pull $(IMAGE_NAME):latest
-	cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
-		      --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
-		      --build-arg LIBPEPADAPTER_VERSION=$(LIBPEPADAPTER_VERSION) \
-		      --cache-from $(IMAGE_NAME):latest \
-		      --tag=$(IMAGE_NAME):$(LIBPEPADAPTER_VERSION) \
-		      --tag=$(IMAGE_NAME):latest \
-		      -f scripts/${CURRENT_DISTRO}/$(DOCKERFILE) .
-	docker push $(IMAGE_NAME):${LIBPEPADAPTER_VERSION}
-	docker push $(IMAGE_NAME):latest

scripts/centos8/libpEpAdapter.centos8.Dockerfile

@@ -1,23 +0,0 @@
-ARG DOCKER_REGISTRY_HOST
-ARG CURRENT_DISTRO
-ARG PEPENGINE_VERSION=latest
-FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-engine:${PEPENGINE_VERSION}
-
-ENV BUILDROOT /build
-ENV INSTPREFIX /install
-ENV OUTDIR /out
-
-### Setup working directory
-RUN mkdir ${BUILDROOT}/libpEpAdapter
-COPY . ${BUILDROOT}/libpEpAdapter
-USER root
-RUN chown -R pep-builder:pep-builder ${BUILDROOT}/libpEpAdapter
-USER pep-builder
-WORKDIR ${BUILDROOT}/libpEpAdapter
-
-ARG LIBPEPADAPTER_VERSION
-ARG CURRENT_DISTRO
-
-### Build libpEpAdapter
-RUN sh ./scripts/${CURRENT_DISTRO}/build_libpEpAdapter.sh && \
-    rm -rf ${BUILDROOT}/*

scripts/centos8/packages/rpm/Dockerfile

@@ -0,0 +1,18 @@
+ARG DOCKER_REGISTRY_HOST
+ARG CURRENT_DISTRO
+ARG LIBPEPADAPTER_VERSION
+FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-libpepadapter:${LIBPEPADAPTER_VERSION} AS pEpBuild
+
+FROM ${DOCKER_REGISTRY_HOST}/fpm-${CURRENT_DISTRO}:latest
+# whatever is required for building should be installed in this image; just like BuildRequires: for RPM specs
+RUN yum -y install readline-devel epel-release && \
+    yum -y install patchelf chrpath
+
+ARG PEP_MACHINE_DIR
+COPY --from=pEpBuild /install /source
+
+COPY install.sh /usr/local/bin/install.sh
+
+ENV INSTPREFIX /source
+
+RUN /usr/local/bin/install.sh

scripts/centos8/packages/rpm/create-libpepadapter-rpm.sh

@@ -0,0 +1,27 @@
+#!/bin/bash -ex
+# we should always set proper ownership before exiting, otherwise
+# the created packages will have root:root ownership and we'll be unable
+# to delete them from our host.
+trap 'chown -R --reference /usr/bin/create-rpm.sh /out/' EXIT
+
+# the source directory is mounted read-only to prevent issues where the build
+# could alter the source; we should copy it somewhere inside the container
+cd /source/out
+ls -alh
+tree
+INSTALL_TOP=/package
+mkdir -p ${INSTALL_TOP}/lib
+mkdir -p ${INSTALL_TOP}/include/pEp
+cp -ar lib/* ${INSTALL_TOP}/lib/.
+cp -ar include/pEp/* ${INSTALL_TOP}/include/pEp/.
+
+
+cd /out
+
+#this would be the no-signature command line
+fpm -t rpm -s dir \
+	-n ${PKG_NAME} \
+	--version ${PKG_VERSION} \
+	--description "${PKG_DESCRIPTION}" \
+	--depends ${PKG_DEPENDS} \
+	-C ${PKG_INSTALL_PATH_STRING}

scripts/centos8/packages/rpm/install.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -exuo pipefail
+
+# ===========================
+# Distro
+# ===========================
+
+echo 7 >"${INSTPREFIX}/D_REVISION"
+
+D_REV=$(cat ${INSTPREFIX}/D_REVISION)
+D=""
+
+D=${INSTPREFIX}/out
+
+mkdir -p ${INSTPREFIX}/out
+rm -rf ${INSTPREFIX}/out/*
+mkdir -p "$D"/{bin,ld,lib/pEp,share/pEp,include/pEp}
+
+# libpEpAdapter
+cp -a ${INSTPREFIX}/lib/libpEpAdapter.a "$D"/lib
+cp -av ${INSTPREFIX}/include/pEp/{status_to_string.hh,message_cache.hh,call_with_lock.hh,Adapter.hh,callback_dispatcher.hh,locked_queue.hh,slurp.hh,constant_time_algo.hh,Semaphore.hh,pc_container.hh,passphrase_cache.hh,pEpLog.hh,Adapter.hxx,passphrase_cache.hxx} "$D"/include/pEp
+
+
+# versions
+cp -a ${INSTPREFIX}/*.ver "$D"
+
+find "$D"/lib -maxdepth 1 -type f -print -exec patchelf --set-rpath '$ORIGIN/pEp:$ORIGIN' {} \;
+find "$D"/lib/pEp         -type f -print -exec patchelf --set-rpath '$ORIGIN' {} \;
+find "$D"/bin -type f -print -exec patchelf --set-rpath '$ORIGIN/../lib/pEp:$ORIGIN/../lib' {} \;
+
+ls -lh "$D"/*
+du -sch "$D"

scripts/debian10/Makefile

@@ -1,27 +0,0 @@
-include ../../DEPENDENCIES
-export
-PEPENGINE_VERSION=${pEpEngine}
-CURRENT_DISTRO=$(shell basename $(shell pwd))
-IMAGE_NAME=${DOCKER_REGISTRY_HOST}/pep-$(CURRENT_DISTRO)-libpepadapter
-DOCKERFILE=libpEpAdapter.$(CURRENT_DISTRO).Dockerfile
-IS_TAGGED=${TAGGED_BUILD}
-ifeq ($(IS_TAGGED), true)
-# $CI_COMMIT_TAG is a predefined environment variable from Gitlab
-	LIBPEPADAPTER_VERSION=${CI_COMMIT_TAG}
-else
-	LIBPEPADAPTER_VERSION=$(shell git rev-parse --short=8 HEAD)
-endif
-all:
-	-docker pull $(IMAGE_NAME):latest
-	cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
-		      --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
-		      --build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
-		      --build-arg LIBPEPADAPTER_VERSION=$(LIBPEPADAPTER_VERSION) \
-		      --cache-from $(IMAGE_NAME):latest \
-		      --tag=$(IMAGE_NAME):$(LIBPEPADAPTER_VERSION) \
-		      --tag=$(IMAGE_NAME):${LIBPEPADAPTER_VERSION}_engine-${PEPENGINE_VERSION} \
-		      --tag=$(IMAGE_NAME):latest \
-		      -f scripts/${CURRENT_DISTRO}/$(DOCKERFILE) .
-	docker push $(IMAGE_NAME):${LIBPEPADAPTER_VERSION}
-	docker push $(IMAGE_NAME):${LIBPEPADAPTER_VERSION}_engine-${PEPENGINE_VERSION}
-	docker push $(IMAGE_NAME):latest

scripts/debian10/build_libpEpAdapter.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/env sh
-set -exo
-
-echo "ENGINE_LIB_PATH=${INSTPREFIX}/lib" >> local.conf
-echo "ENGINE_INC_PATH=${INSTPREFIX}/include" >> local.conf
-
-make
-make install PREFIX="${INSTPREFIX}"