buff
/
pEpEngine
forked from pEp.foundation/pEpEngine
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

CI: build CentOS/RedHat 8 RPM packages 

This commit cleans up the existing CI scripts a bit while adding jobs to
build RPM packages. The directory structure allows for easily adding more
package targets.

RPM outputs will only be uploaded on tagged commits.
IPS-2
Devan Carpenter 2021-07-08 15:33:58 -07:00
parent 56cd005c02
commit 9999900575
9 changed files with 255 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
.gitlab-ci-files/common-prepare.yml
View File

@ -8,15 +8,44 @@
- 'which rsync || ( sudo apt-get update -y && sudo apt-get install rsync -y )'
- 'which make || ( sudo apt-get update -y && sudo apt-get install make -y )'
.add_ssh_keys: &add_ssh_keys
# Add the SSH key (stored in the SSH_PRIVATE_KEY variable) to the agent.
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
.verify_ssh_host_keys: &verify_ssh_host_keys
# Verify SSH host keys
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo "${SSH_KNOWN_HOSTS}" >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
.standard_job:
tags: [kvm]
before_script:
- *ensure_docker
- *ensure_rsync
.make_in_docker:
extends: .standard_job
script:
- docker login -u ${DOCKER_REGISTRY_USER} -p ${DOCKER_REGISTRY_PASS} ${DOCKER_REGISTRY_HOST}
- cd scripts/${CI_DISTRO_TARGET}
- make
- make ${MAKE_TARGET}
.upload_pkg:
extends: .standard_job
before_script:
- *ensure_rsync
- *add_ssh_keys
- *verify_ssh_host_keys
script:
- docker login -u ${DOCKER_REGISTRY_USER} -p ${DOCKER_REGISTRY_PASS} ${DOCKER_REGISTRY_HOST}
- cd scripts/${CI_DISTRO_TARGET}
- make ${MAKE_TARGET}
- pwd
- cd out
- time rsync -avP -e "ssh -p ${PKG_HOST_SSH_PORT}" depot@${PKG_HOST}:files/pkgs/RHEL/8/pEpEngine/SHA256SUMS || true
- if sha256sum --ignore-missing --check SHA256SUMS ; then echo "Package already exists... Exiting..." && exit ; else true ; fi
- sha256sum ./*.rpm | tee --append SHA256SUMS
- cat ./SHA256SUMS
- time rsync -azvP -e "ssh -p ${PKG_HOST_SSH_PORT}" --rsync-path="mkdir -p /home/depot/files/pkgs/RHEL/8/pEpEngine && rsync" ./*.rpm SHA256SUMS depot@${PKG_HOST}:files/pkgs/RHEL/8/pEpEngine/

39
.gitlab-ci.yml
View File

@ -2,7 +2,9 @@ include:
- '.gitlab-ci-files/common-prepare.yml'
stages:
- deps
- build
- packages
# Debian
@ -11,6 +13,7 @@ debian10:build:
extends: .make_in_docker
stage: build
variables:
MAKE_TARGET: "build"
CI_DISTRO_TARGET: "debian10"
DEBIAN_FRONTEND: "noninteractive"
rules:
@ -20,6 +23,7 @@ debian10:tagged-build:
extends: .make_in_docker
stage: build
variables:
MAKE_TARGET: "build"
CI_DISTRO_TARGET: "debian10"
DEBIAN_FRONTEND: "noninteractive"
TAGGED_BUILD: "true"
@ -28,10 +32,22 @@ debian10:tagged-build:
# CentOS
centos8:deps:
extends: .make_in_docker
stage: deps
variables:
MAKE_TARGET: "deps"
CI_DISTRO_TARGET: "centos8"
rules:
- changes:
- DEPENDENCIES
centos8:build:
extends: .make_in_docker
stage: build
variables:
MAKE_TARGET: "build"
CI_DISTRO_TARGET: "centos8"
rules:
- if: '$CI_COMMIT_TAG !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
@ -40,6 +56,29 @@ centos8:tagged-build:
extends: .make_in_docker
stage: build
variables:
MAKE_TARGET: "build"
CI_DISTRO_TARGET: "centos8"
TAGGED_BUILD: "true"
rules:
- if: '$CI_COMMIT_TAG =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
centos8:rpm:
extends: .make_in_docker
stage: packages
needs: ["centos8:build"]
variables:
MAKE_TARGET: "rpm"
CI_DISTRO_TARGET: "centos8"
rules:
- if: '$CI_COMMIT_TAG !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
centos8:rpm:tagged-build:
extends: .upload_pkg
stage: packages
needs: ["centos8:tagged-build"]
variables:
MAKE_TARGET: "rpm"
CI_DISTRO_TARGET: "centos8"
TAGGED_BUILD: "true"
rules:

51
scripts/centos8/Makefile
View File

@ -6,6 +6,12 @@ SEQUOIA_VERSION=$(shell echo ${sequoia} | sed 's/\//-/')
CURRENT_DISTRO=$(shell basename $(shell pwd))
IMAGE_NAME=${DOCKER_REGISTRY_HOST}/pep-$(CURRENT_DISTRO)-engine
DOCKERFILE=pEpEngine.$(CURRENT_DISTRO).Dockerfile
PKG_BUILD_IMAGE=${DOCKER_REGISTRY_HOST}/fpm-$(CURRENT_DISTRO)
PKG_INSTALL_PATH=/opt/pEp
PKG_NAME=pEpEngine
PKG_DESCRIPTION="p≡p Engine Binary RPM Package"
PKG_DEPENDS=sequoia-openpgp
PKG_INSTALL_PATH_STRING="/ /package/lib=${PKG_INSTALL_PATH} /package/include/pEp=${PKG_INSTALL_PATH}/include /package/share=${PKG_INSTALL_PATH}"
IS_TAGGED=${TAGGED_BUILD}
ifeq ($(IS_TAGGED), true)
# $CI_COMMIT_TAG is a predefined environment variable from Gitlab
@ -13,7 +19,24 @@ ifeq ($(IS_TAGGED), true)
else
PEPENGINE_VERSION=$(shell git rev-parse --short=8 HEAD)
endif
all:
all: deps build
deps:
-docker pull $(IMAGE_NAME)-deps:latest
cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
--build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
--build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
--build-arg SEQUOIA_VERSION=$(SEQUOIA_VERSION) \
--build-arg YML2_VERSION=$(YML2_VERSION) \
--build-arg PEP_MACHINE_DIR=$(PEP_MACHINE_DIR) \
--cache-from $(IMAGE_NAME):latest \
--tag=$(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION) \
--tag=$(IMAGE_NAME)-deps:latest \
-f scripts/${CURRENT_DISTRO}/deps.$(DOCKERFILE) .
docker push $(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION)
docker push $(IMAGE_NAME)-deps:latest
build:
-docker pull $(IMAGE_NAME):latest
cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
--build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
@ -27,3 +50,29 @@ all:
-f scripts/${CURRENT_DISTRO}/$(DOCKERFILE) .
docker push $(IMAGE_NAME):$(PEPENGINE_VERSION)
docker push $(IMAGE_NAME):latest
rpm:
-docker pull $(PKG_BUILD_IMAGE)-engine:latest
@docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
--build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
--build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
--build-arg PEP_MACHINE_DIR=$(PEP_MACHINE_DIR) \
--build-arg PKG_INSTALL_PATH=$(PKG_INSTALL_PATH) \
--cache-from $(PKG_BUILD_IMAGE)-engine:latest \
--tag=$(PKG_BUILD_IMAGE)-engine:$(PEPENGINE_VERSION) \
--tag=$(PKG_BUILD_IMAGE)-engine:latest \
packages/rpm
@docker push $(PKG_BUILD_IMAGE)-engine:$(PEPENGINE_VERSION)
@docker push $(PKG_BUILD_IMAGE)-engine:latest
@docker run -e PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
-e PEP_MACHINE_DIR=$(PEP_MACHINE_DIR) \
-e PKG_VERSION=$(PEPENGINE_VERSION) \
-e PKG_INSTALL_PATH=$(PKG_INSTALL_PATH) \
-e PKG_NAME=$(PKG_NAME) \
-e PKG_DESCRIPTION=$(PKG_DESCRIPTION) \
-e PKG_DEPENDS=$(PKG_DEPENDS) \
-e PKG_INSTALL_PATH_STRING=$(PKG_INSTALL_PATH_STRING) \
--rm -v $(shell pwd)/packages/rpm/create-engine-rpm.sh:/usr/bin/create-rpm.sh:ro \
-v $(shell pwd)/out:/out \
-w / $(PKG_BUILD_IMAGE)-engine:latest \
/usr/bin/create-rpm.sh

30
scripts/centos8/deps.pEpEngine.centos8.Dockerfile Normal file
View File

@ -0,0 +1,30 @@
ARG DOCKER_REGISTRY_HOST
ARG CURRENT_DISTRO
ARG PEPENGINE_VERSION
ARG SEQUOIA_VERSION
FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-sequoia:${SEQUOIA_VERSION}
ENV BUILDROOT /build
ENV INSTPREFIX /install
ENV OUTDIR /out
ARG PEP_MACHINE_DIR
### Setup working directory
RUN mkdir ${BUILDROOT}/pEpEngine
COPY ./scripts/common/build_pEpEngine_deps.sh ${BUILDROOT}/pEpEngine
USER root
RUN yum install -y python3 python3-lxml binutils && yum clean all
RUN chown -R pep-builder:pep-builder ${BUILDROOT}/pEpEngine
WORKDIR ${BUILDROOT}/pEpEngine
ARG YML2_VERSION
ARG ENGINE_VERSION
ARG CURRENT_DISTRO
### Build pEpEngine dependencies
USER pep-builder
RUN sh ./build_pEpEngine_deps.sh

16
scripts/centos8/pEpEngine.centos8.Dockerfile
View File

@ -2,7 +2,8 @@ ARG DOCKER_REGISTRY_HOST
ARG CURRENT_DISTRO
ARG PEPENGINE_VERSION
ARG SEQUOIA_VERSION
FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-sequoia:${SEQUOIA_VERSION}
ARG YML2_VERSION
FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-engine-deps:${SEQUOIA_VERSION}-${YML2_VERSION}
ENV BUILDROOT /build
ENV INSTPREFIX /install
@ -10,25 +11,18 @@ ENV OUTDIR /out
ARG PEP_MACHINE_DIR
### Setup working directory
RUN mkdir ${BUILDROOT}/pEpEngine
COPY . ${BUILDROOT}/pEpEngine
USER root
RUN yum install -y python3 python3-lxml binutils && yum clean all
RUN mkdir -p ${BUILDROOT}/pEpEngine
COPY . ${BUILDROOT}/pEpEngine
RUN chown -R pep-builder:pep-builder ${BUILDROOT}/pEpEngine
WORKDIR ${BUILDROOT}/pEpEngine
USER pep-builder
ARG YML2_VERSION
ARG ENGINE_VERSION
ARG CURRENT_DISTRO
### Build pEpEngine dependencies
USER pep-builder
RUN sh ./scripts/common/build_pEpEngine_deps.sh
### Build pEpEngine
RUN sh ./scripts/common/build_pEpEngine.sh

20
scripts/centos8/packages/rpm/Dockerfile Normal file
View File

@ -0,0 +1,20 @@
ARG DOCKER_REGISTRY_HOST
ARG CURRENT_DISTRO
ARG PEP_MACHINE_DIR
ARG PEPENGINE_VERSION
FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-engine:${PEPENGINE_VERSION} AS pEpBuild
FROM ${DOCKER_REGISTRY_HOST}/fpm-${CURRENT_DISTRO}:latest
# whatever is required for building should be installed in this image; just like BuildRequires: for RPM specs
RUN yum -y install readline-devel epel-release && \
yum -y install patchelf chrpath
ARG PEP_MACHINE_DIR
COPY --from=pEpBuild /install /source
COPY --from=pEpBuild ${PEP_MACHINE_DIR}/system.db ${PEP_MACHINE_DIR}/system.db
COPY install.sh /usr/local/bin/install.sh
ENV INSTPREFIX /source
RUN /usr/local/bin/install.sh

29
scripts/centos8/packages/rpm/create-engine-rpm.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash -ex
# we should always set proper ownership before exiting, otherwise
# the created packages will have root:root ownership and we'll be unable
# to delete them from our host.
trap 'chown -R --reference /usr/bin/create-rpm.sh /out/' EXIT
# the source directory is mounted read-only to prevent issues where the build
# could alter the source; we should copy it somewhere inside the container
cd /source/out
ls -alh
tree
INSTALL_TOP=/package
mkdir -p ${INSTALL_TOP}/lib/pEp
mkdir -p ${INSTALL_TOP}/include/pEp
mkdir -p ${INSTALL_TOP}/share
cp -ar lib/libpEpEngine.so ${INSTALL_TOP}/lib/.
cp -ar lib/pEp/libetpan* ${INSTALL_TOP}/lib/pEp/..
cp -ar include/pEp/* ${INSTALL_TOP}/include/pEp/.
cp -ar share/* ${INSTALL_TOP}/share/.
cd /out
#this would be the no-signature command line
fpm -t rpm -s dir \
-n ${PKG_NAME} \
--version ${PKG_VERSION} \
--description "${PKG_DESCRIPTION}" \
--depends ${PKG_DEPENDS} \
-C ${PKG_INSTALL_PATH_STRING}

48
scripts/centos8/packages/rpm/install.sh Executable file
View File

@ -0,0 +1,48 @@
#!/bin/bash
set -exuo pipefail
# ===========================
# Distro
# ===========================
echo 7 >"${INSTPREFIX}/D_REVISION"
D_REV=$(cat ${INSTPREFIX}/D_REVISION)
D=""
D=${INSTPREFIX}/out
mkdir -p ${INSTPREFIX}/out
rm -rf ${INSTPREFIX}/out/*
# pep asn1c capnp cmake curl gmp llvm nettle ninja sequoia
# bin include lib lib64 libexec share
mkdir -p "$D"/{bin,ld,lib/pEp,share/pEp,include/pEp}
# Engine and below, and libpEpAdapter
cp -a ${INSTPREFIX}/lib/libpEpEngine.so "$D"/lib
cp -ar ${INSTPREFIX}/libetpan/lib/libetpan.so* "$D"/lib/pEp
cp -arv ${INSTPREFIX}/include/pEp/. "$D"/include/pEp
cp -arv ${PEP_MACHINE_DIR}/system.db "$D"/share/pEp
# Sequoia cmdline (optional above)
if [ -f ${INSTPREFIX}/bin/sq ] ; then
cp -a ${INSTPREFIX}/lib/libsequoia_*.so* "$D"/lib/pEp
cp -a ${INSTPREFIX}/bin/sq "$D"/bin
cp -a ${INSTPREFIX}/bin/sqv "$D"/bin
cp -arv ${INSTPREFIX}/lib/sequoia "$D"/lib/pEp/.
else
cp -a ${INSTPREFIX}/lib/libsequoia_openpgp_ffi.* "$D"/lib/pEp
cp -arv ${INSTPREFIX}/lib/sequoia "$D"/lib/pEp/.
fi
# versions
cp -a ${INSTPREFIX}/*.ver "$D"
find "$D"/lib -maxdepth 1 -type f -print -exec patchelf --set-rpath '$ORIGIN/pEp:$ORIGIN' {} \;
find "$D"/lib/pEp -type f -print -exec patchelf --set-rpath '$ORIGIN' {} \;
find "$D"/bin -type f -print -exec patchelf --set-rpath '$ORIGIN/../lib/pEp:$ORIGIN/../lib' {} \;
ls -lh "$D"/*
du -sch "$D"

4
scripts/debian10/Makefile
View File

@ -13,7 +13,9 @@ ifeq ($(IS_TAGGED), true)
else
PEPENGINE_VERSION=$(shell git rev-parse --short=8 HEAD)
endif
all:
all: build
build:
-docker pull $(IMAGE_NAME):latest
cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
--build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \