diff --git a/.gitlab-ci-files/common-prepare.yml b/.gitlab-ci-files/common-prepare.yml index 8053794f..831c0316 100644 --- a/.gitlab-ci-files/common-prepare.yml +++ b/.gitlab-ci-files/common-prepare.yml @@ -8,15 +8,44 @@ - 'which rsync || ( sudo apt-get update -y && sudo apt-get install rsync -y )' - 'which make || ( sudo apt-get update -y && sudo apt-get install make -y )' +.add_ssh_keys: &add_ssh_keys + # Add the SSH key (stored in the SSH_PRIVATE_KEY variable) to the agent. + - eval $(ssh-agent -s) + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - + +.verify_ssh_host_keys: &verify_ssh_host_keys + # Verify SSH host keys + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - echo "${SSH_KNOWN_HOSTS}" >> ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts + .standard_job: tags: [kvm] before_script: - *ensure_docker - - *ensure_rsync .make_in_docker: extends: .standard_job script: - docker login -u ${DOCKER_REGISTRY_USER} -p ${DOCKER_REGISTRY_PASS} ${DOCKER_REGISTRY_HOST} - cd scripts/${CI_DISTRO_TARGET} - - make + - make ${MAKE_TARGET} + +.upload_pkg: + extends: .standard_job + before_script: + - *ensure_rsync + - *add_ssh_keys + - *verify_ssh_host_keys + script: + - docker login -u ${DOCKER_REGISTRY_USER} -p ${DOCKER_REGISTRY_PASS} ${DOCKER_REGISTRY_HOST} + - cd scripts/${CI_DISTRO_TARGET} + - make ${MAKE_TARGET} + - pwd + - cd out + - time rsync -avP -e "ssh -p ${PKG_HOST_SSH_PORT}" depot@${PKG_HOST}:files/pkgs/RHEL/8/pEpEngine/SHA256SUMS || true + - if sha256sum --ignore-missing --check SHA256SUMS ; then echo "Package already exists... Exiting..." && exit ; else true ; fi + - sha256sum ./*.rpm | tee --append SHA256SUMS + - cat ./SHA256SUMS + - time rsync -azvP -e "ssh -p ${PKG_HOST_SSH_PORT}" --rsync-path="mkdir -p /home/depot/files/pkgs/RHEL/8/pEpEngine && rsync" ./*.rpm SHA256SUMS depot@${PKG_HOST}:files/pkgs/RHEL/8/pEpEngine/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f5d1a16..5508fa96 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,9 @@ include: - '.gitlab-ci-files/common-prepare.yml' stages: + - deps - build + - packages # Debian @@ -11,6 +13,7 @@ debian10:build: extends: .make_in_docker stage: build variables: + MAKE_TARGET: "build" CI_DISTRO_TARGET: "debian10" DEBIAN_FRONTEND: "noninteractive" rules: @@ -20,6 +23,7 @@ debian10:tagged-build: extends: .make_in_docker stage: build variables: + MAKE_TARGET: "build" CI_DISTRO_TARGET: "debian10" DEBIAN_FRONTEND: "noninteractive" TAGGED_BUILD: "true" @@ -28,10 +32,22 @@ debian10:tagged-build: # CentOS +centos8:deps: + extends: .make_in_docker + stage: deps + variables: + MAKE_TARGET: "deps" + CI_DISTRO_TARGET: "centos8" + rules: + - changes: + - DEPENDENCIES + + centos8:build: extends: .make_in_docker stage: build variables: + MAKE_TARGET: "build" CI_DISTRO_TARGET: "centos8" rules: - if: '$CI_COMMIT_TAG !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/' @@ -40,6 +56,29 @@ centos8:tagged-build: extends: .make_in_docker stage: build variables: + MAKE_TARGET: "build" + CI_DISTRO_TARGET: "centos8" + TAGGED_BUILD: "true" + rules: + - if: '$CI_COMMIT_TAG =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/' + +centos8:rpm: + extends: .make_in_docker + stage: packages + needs: ["centos8:build"] + variables: + MAKE_TARGET: "rpm" + CI_DISTRO_TARGET: "centos8" + rules: + - if: '$CI_COMMIT_TAG !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/' + + +centos8:rpm:tagged-build: + extends: .upload_pkg + stage: packages + needs: ["centos8:tagged-build"] + variables: + MAKE_TARGET: "rpm" CI_DISTRO_TARGET: "centos8" TAGGED_BUILD: "true" rules: diff --git a/scripts/centos8/Makefile b/scripts/centos8/Makefile index 1f935592..ab60889d 100644 --- a/scripts/centos8/Makefile +++ b/scripts/centos8/Makefile @@ -6,6 +6,12 @@ SEQUOIA_VERSION=$(shell echo ${sequoia} | sed 's/\//-/') CURRENT_DISTRO=$(shell basename $(shell pwd)) IMAGE_NAME=${DOCKER_REGISTRY_HOST}/pep-$(CURRENT_DISTRO)-engine DOCKERFILE=pEpEngine.$(CURRENT_DISTRO).Dockerfile +PKG_BUILD_IMAGE=${DOCKER_REGISTRY_HOST}/fpm-$(CURRENT_DISTRO) +PKG_INSTALL_PATH=/opt/pEp +PKG_NAME=pEpEngine +PKG_DESCRIPTION="p≡p Engine Binary RPM Package" +PKG_DEPENDS=sequoia-openpgp +PKG_INSTALL_PATH_STRING="/ /package/lib=${PKG_INSTALL_PATH} /package/include/pEp=${PKG_INSTALL_PATH}/include /package/share=${PKG_INSTALL_PATH}" IS_TAGGED=${TAGGED_BUILD} ifeq ($(IS_TAGGED), true) # $CI_COMMIT_TAG is a predefined environment variable from Gitlab @@ -13,7 +19,24 @@ ifeq ($(IS_TAGGED), true) else PEPENGINE_VERSION=$(shell git rev-parse --short=8 HEAD) endif -all: +all: deps build + +deps: + -docker pull $(IMAGE_NAME)-deps:latest + cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \ + --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \ + --build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \ + --build-arg SEQUOIA_VERSION=$(SEQUOIA_VERSION) \ + --build-arg YML2_VERSION=$(YML2_VERSION) \ + --build-arg PEP_MACHINE_DIR=$(PEP_MACHINE_DIR) \ + --cache-from $(IMAGE_NAME):latest \ + --tag=$(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION) \ + --tag=$(IMAGE_NAME)-deps:latest \ + -f scripts/${CURRENT_DISTRO}/deps.$(DOCKERFILE) . + docker push $(IMAGE_NAME)-deps:$(SEQUOIA_VERSION)-$(YML2_VERSION) + docker push $(IMAGE_NAME)-deps:latest + +build: -docker pull $(IMAGE_NAME):latest cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \ --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \ @@ -27,3 +50,29 @@ all: -f scripts/${CURRENT_DISTRO}/$(DOCKERFILE) . docker push $(IMAGE_NAME):$(PEPENGINE_VERSION) docker push $(IMAGE_NAME):latest + +rpm: + -docker pull $(PKG_BUILD_IMAGE)-engine:latest + @docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \ + --build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \ + --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \ + --build-arg PEP_MACHINE_DIR=$(PEP_MACHINE_DIR) \ + --build-arg PKG_INSTALL_PATH=$(PKG_INSTALL_PATH) \ + --cache-from $(PKG_BUILD_IMAGE)-engine:latest \ + --tag=$(PKG_BUILD_IMAGE)-engine:$(PEPENGINE_VERSION) \ + --tag=$(PKG_BUILD_IMAGE)-engine:latest \ + packages/rpm + @docker push $(PKG_BUILD_IMAGE)-engine:$(PEPENGINE_VERSION) + @docker push $(PKG_BUILD_IMAGE)-engine:latest + @docker run -e PEPENGINE_VERSION=$(PEPENGINE_VERSION) \ + -e PEP_MACHINE_DIR=$(PEP_MACHINE_DIR) \ + -e PKG_VERSION=$(PEPENGINE_VERSION) \ + -e PKG_INSTALL_PATH=$(PKG_INSTALL_PATH) \ + -e PKG_NAME=$(PKG_NAME) \ + -e PKG_DESCRIPTION=$(PKG_DESCRIPTION) \ + -e PKG_DEPENDS=$(PKG_DEPENDS) \ + -e PKG_INSTALL_PATH_STRING=$(PKG_INSTALL_PATH_STRING) \ + --rm -v $(shell pwd)/packages/rpm/create-engine-rpm.sh:/usr/bin/create-rpm.sh:ro \ + -v $(shell pwd)/out:/out \ + -w / $(PKG_BUILD_IMAGE)-engine:latest \ + /usr/bin/create-rpm.sh diff --git a/scripts/centos8/deps.pEpEngine.centos8.Dockerfile b/scripts/centos8/deps.pEpEngine.centos8.Dockerfile new file mode 100644 index 00000000..94ca52b6 --- /dev/null +++ b/scripts/centos8/deps.pEpEngine.centos8.Dockerfile @@ -0,0 +1,30 @@ +ARG DOCKER_REGISTRY_HOST +ARG CURRENT_DISTRO +ARG PEPENGINE_VERSION +ARG SEQUOIA_VERSION +FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-sequoia:${SEQUOIA_VERSION} + +ENV BUILDROOT /build +ENV INSTPREFIX /install +ENV OUTDIR /out +ARG PEP_MACHINE_DIR + +### Setup working directory +RUN mkdir ${BUILDROOT}/pEpEngine +COPY ./scripts/common/build_pEpEngine_deps.sh ${BUILDROOT}/pEpEngine + +USER root + +RUN yum install -y python3 python3-lxml binutils && yum clean all + +RUN chown -R pep-builder:pep-builder ${BUILDROOT}/pEpEngine +WORKDIR ${BUILDROOT}/pEpEngine + +ARG YML2_VERSION +ARG ENGINE_VERSION +ARG CURRENT_DISTRO + +### Build pEpEngine dependencies +USER pep-builder + +RUN sh ./build_pEpEngine_deps.sh diff --git a/scripts/centos8/pEpEngine.centos8.Dockerfile b/scripts/centos8/pEpEngine.centos8.Dockerfile index 856e04d9..635799ee 100644 --- a/scripts/centos8/pEpEngine.centos8.Dockerfile +++ b/scripts/centos8/pEpEngine.centos8.Dockerfile @@ -2,7 +2,8 @@ ARG DOCKER_REGISTRY_HOST ARG CURRENT_DISTRO ARG PEPENGINE_VERSION ARG SEQUOIA_VERSION -FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-sequoia:${SEQUOIA_VERSION} +ARG YML2_VERSION +FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-engine-deps:${SEQUOIA_VERSION}-${YML2_VERSION} ENV BUILDROOT /build ENV INSTPREFIX /install @@ -10,25 +11,18 @@ ENV OUTDIR /out ARG PEP_MACHINE_DIR ### Setup working directory -RUN mkdir ${BUILDROOT}/pEpEngine -COPY . ${BUILDROOT}/pEpEngine - USER root - -RUN yum install -y python3 python3-lxml binutils && yum clean all +RUN mkdir -p ${BUILDROOT}/pEpEngine +COPY . ${BUILDROOT}/pEpEngine RUN chown -R pep-builder:pep-builder ${BUILDROOT}/pEpEngine WORKDIR ${BUILDROOT}/pEpEngine +USER pep-builder ARG YML2_VERSION ARG ENGINE_VERSION ARG CURRENT_DISTRO -### Build pEpEngine dependencies -USER pep-builder - -RUN sh ./scripts/common/build_pEpEngine_deps.sh - ### Build pEpEngine RUN sh ./scripts/common/build_pEpEngine.sh diff --git a/scripts/centos8/packages/rpm/Dockerfile b/scripts/centos8/packages/rpm/Dockerfile new file mode 100644 index 00000000..86da958f --- /dev/null +++ b/scripts/centos8/packages/rpm/Dockerfile @@ -0,0 +1,20 @@ +ARG DOCKER_REGISTRY_HOST +ARG CURRENT_DISTRO +ARG PEP_MACHINE_DIR +ARG PEPENGINE_VERSION +FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-engine:${PEPENGINE_VERSION} AS pEpBuild + +FROM ${DOCKER_REGISTRY_HOST}/fpm-${CURRENT_DISTRO}:latest +# whatever is required for building should be installed in this image; just like BuildRequires: for RPM specs +RUN yum -y install readline-devel epel-release && \ + yum -y install patchelf chrpath + +ARG PEP_MACHINE_DIR +COPY --from=pEpBuild /install /source +COPY --from=pEpBuild ${PEP_MACHINE_DIR}/system.db ${PEP_MACHINE_DIR}/system.db + +COPY install.sh /usr/local/bin/install.sh + +ENV INSTPREFIX /source + +RUN /usr/local/bin/install.sh diff --git a/scripts/centos8/packages/rpm/create-engine-rpm.sh b/scripts/centos8/packages/rpm/create-engine-rpm.sh new file mode 100755 index 00000000..f1021ad8 --- /dev/null +++ b/scripts/centos8/packages/rpm/create-engine-rpm.sh @@ -0,0 +1,29 @@ +#!/bin/bash -ex +# we should always set proper ownership before exiting, otherwise +# the created packages will have root:root ownership and we'll be unable +# to delete them from our host. +trap 'chown -R --reference /usr/bin/create-rpm.sh /out/' EXIT + +# the source directory is mounted read-only to prevent issues where the build +# could alter the source; we should copy it somewhere inside the container +cd /source/out +ls -alh +tree +INSTALL_TOP=/package +mkdir -p ${INSTALL_TOP}/lib/pEp +mkdir -p ${INSTALL_TOP}/include/pEp +mkdir -p ${INSTALL_TOP}/share +cp -ar lib/libpEpEngine.so ${INSTALL_TOP}/lib/. +cp -ar lib/pEp/libetpan* ${INSTALL_TOP}/lib/pEp/.. +cp -ar include/pEp/* ${INSTALL_TOP}/include/pEp/. +cp -ar share/* ${INSTALL_TOP}/share/. + +cd /out + +#this would be the no-signature command line +fpm -t rpm -s dir \ + -n ${PKG_NAME} \ + --version ${PKG_VERSION} \ + --description "${PKG_DESCRIPTION}" \ + --depends ${PKG_DEPENDS} \ + -C ${PKG_INSTALL_PATH_STRING} diff --git a/scripts/centos8/packages/rpm/install.sh b/scripts/centos8/packages/rpm/install.sh new file mode 100755 index 00000000..42535280 --- /dev/null +++ b/scripts/centos8/packages/rpm/install.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -exuo pipefail + +# =========================== +# Distro +# =========================== + +echo 7 >"${INSTPREFIX}/D_REVISION" + +D_REV=$(cat ${INSTPREFIX}/D_REVISION) +D="" + +D=${INSTPREFIX}/out + +mkdir -p ${INSTPREFIX}/out +rm -rf ${INSTPREFIX}/out/* +# pep asn1c capnp cmake curl gmp llvm nettle ninja sequoia +# bin include lib lib64 libexec share +mkdir -p "$D"/{bin,ld,lib/pEp,share/pEp,include/pEp} + +# Engine and below, and libpEpAdapter +cp -a ${INSTPREFIX}/lib/libpEpEngine.so "$D"/lib +cp -ar ${INSTPREFIX}/libetpan/lib/libetpan.so* "$D"/lib/pEp + +cp -arv ${INSTPREFIX}/include/pEp/. "$D"/include/pEp + +cp -arv ${PEP_MACHINE_DIR}/system.db "$D"/share/pEp + +# Sequoia cmdline (optional above) +if [ -f ${INSTPREFIX}/bin/sq ] ; then + cp -a ${INSTPREFIX}/lib/libsequoia_*.so* "$D"/lib/pEp + cp -a ${INSTPREFIX}/bin/sq "$D"/bin + cp -a ${INSTPREFIX}/bin/sqv "$D"/bin + cp -arv ${INSTPREFIX}/lib/sequoia "$D"/lib/pEp/. +else + cp -a ${INSTPREFIX}/lib/libsequoia_openpgp_ffi.* "$D"/lib/pEp + cp -arv ${INSTPREFIX}/lib/sequoia "$D"/lib/pEp/. +fi + +# versions +cp -a ${INSTPREFIX}/*.ver "$D" + +find "$D"/lib -maxdepth 1 -type f -print -exec patchelf --set-rpath '$ORIGIN/pEp:$ORIGIN' {} \; +find "$D"/lib/pEp -type f -print -exec patchelf --set-rpath '$ORIGIN' {} \; +find "$D"/bin -type f -print -exec patchelf --set-rpath '$ORIGIN/../lib/pEp:$ORIGIN/../lib' {} \; + +ls -lh "$D"/* +du -sch "$D" diff --git a/scripts/debian10/Makefile b/scripts/debian10/Makefile index 1f935592..da5399cf 100644 --- a/scripts/debian10/Makefile +++ b/scripts/debian10/Makefile @@ -13,7 +13,9 @@ ifeq ($(IS_TAGGED), true) else PEPENGINE_VERSION=$(shell git rev-parse --short=8 HEAD) endif -all: +all: build + +build: -docker pull $(IMAGE_NAME):latest cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \ --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \