Add build_rpm script and rpm spec (#1831)

Also, link stdlib++ and libgcc statically.

Signed-off-by: Roman Gershman <roman@dragonflydb.io>

.devcontainer/fedora/devcontainer.json

@@ -0,0 +1,14 @@
+{
+  "name": "helio",
+  "image": "ghcr.io/romange/fedora:30",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-vscode.cpptools",
+        "ms-vscode.cmake-tools",
+        "ms-vscode.cpptools-themes",
+        "twxs.cmake"
+      ]
+    }
+  }
+}

.github/actions/regression-tests/action.yml

@@ -29,7 +29,7 @@ runs:
         ls -l ${GITHUB_WORKSPACE}/
         cd ${GITHUB_WORKSPACE}/tests
         echo "Current commit is ${{github.sha}}"
-        pip install -r dragonfly/requirements.txt
+        pip3 install -r dragonfly/requirements.txt
         # used by PyTests
         export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/${{inputs.build-folder-name}}/${{inputs.dfly-executable}}"
         export UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 # to crash on errors

.github/workflows/release.yml

@@ -54,13 +54,14 @@ jobs:
         # Mount the artifacts directory as /artifacts in the container
         dockerRunArgs: |
           --volume "${{ github.workspace }}:/src"
+
         # The shell to run commands with in the container
         shell: /bin/bash
         install: |
           export DEBIAN_FRONTEND=noninteractive
           apt update && apt install -q -y autoconf-archive cmake curl git libssl-dev \
-              libunwind-dev ninja-build libtool gcc-9 g++-9 libboost-fiber-dev \
+              libunwind-dev ninja-build libtool gcc-9 g++-9 libboost-context-dev \
-              libxml2-dev zip libzstd-dev debhelper moreutils bison
+              zip libzstd-dev debhelper moreutils bison zlib1g-dev
           update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 40  \
                               --slave /usr/bin/g++ g++ /usr/bin/g++-9
         run: |
@@ -96,15 +97,25 @@ jobs:
   build-native:
     runs-on: ubuntu-latest
     needs: create-release
+    strategy:
+      matrix:
+        include:
+          # Build with these flags
+          - name: debian
+            container: ubuntu-dev:20
+          - name: rpm
+            container: fedora:30
     container:
-      image: ghcr.io/romange/ubuntu-dev:20
+      image: ghcr.io/romange/${{ matrix.container }}
     steps:
     - uses: actions/checkout@v3
       with:
         submodules: true
     - name: Configure
       run: |
-          apt update && apt install -y debhelper moreutils pip jq
+          if [ -f /etc/redhat-release ]; then
+            dnf install -y rpm-build libstdc++-static
+          fi
     - name: Build artifacts
       run: |
           # Work around https://github.com/actions/checkout/issues/766
@@ -112,8 +123,15 @@ jobs:
           git describe --always --tags ${{ github.sha }}
           ./tools/release.sh
           # once the build is over, we want to generate a Debian package
+          if [ -f /etc/debian_version ]; then
             ./tools/packaging/generate_debian_package.sh build-opt/dragonfly-x86_64
+          else
+            echo "TODO: build rpm"
+          fi
+
     - name: Run regression tests
+      # Fedora 30 has older python packages, so this fails during requirements installation.
+      if  : matrix.container != 'fedora:30'
       uses: ./.github/actions/regression-tests
       with:
         dfly-executable: dragonfly-x86_64
@@ -121,12 +139,19 @@ jobs:
         run-only-on-ubuntu-latest: false
         build-folder-name: build-opt
     - name: Save artifacts
+      if  : matrix.container != 'fedora:30' # TODO
       run: |
           # place all artifacts at the same location
           mkdir -p results-artifacts
+          if [ -f /etc/debian_version ]; then
             mv build-opt/dragonfly-*tar.gz results-artifacts
             mv dragonfly_*.deb results-artifacts
+          else
+            echo "TODO: upload rpm"
+            ls *.rpm
+          fi
     - name: Upload
+      if  : matrix.container != 'fedora:30' # TODO
       uses: actions/upload-artifact@v3
       with:
         name: dragonfly-amd64

Makefile

@@ -2,18 +2,30 @@ BUILD_ARCH := $(shell uname -m)
 RELEASE_NAME := "dragonfly-${BUILD_ARCH}"
 
 HELIO_RELEASE := $(if $(HELIO_RELEASE),y,)
-HELIO_RELEASE_FLAGS = -DHELIO_RELEASE_FLAGS="-flto -g1 -gz"
+
+HELIO_RELEASE_FLAGS = -DHELIO_RELEASE_FLAGS="-g1 -gz"
 HELIO_USE_STATIC_LIBS = ON
 HELIO_OPENSSL_USE_STATIC_LIBS = ON
 HELIO_ENABLE_GIT_VERSION = ON
 HELIO_WITH_UNWIND = OFF
 
+# Some distributions (old fedora) have incorrect dependencies for crypto
+# so we add -lz for them.
+LINKER_FLAGS=-lz
+
 # equivalent to: if $(uname_m) == x86_64 || $(uname_m) == amd64
 ifneq (, $(filter $(BUILD_ARCH),x86_64 amd64))
 HELIO_MARCH_OPT := -march=core2 -msse4.1 -mpopcnt -mtune=skylake
 endif
 
+# For release builds we link statically libstdc++ and libgcc. Currently,
+# all the release builds are performed by gcc.
+ifeq ($(HELIO_RELEASE),y)
+	LINKER_FLAGS += -static-libstdc++ -static-libgcc
+endif
+
 HELIO_FLAGS = $(if $(HELIO_RELEASE),-release $(HELIO_RELEASE_FLAGS),) \
+			  -DCMAKE_EXE_LINKER_FLAGS="$(LINKER_FLAGS)" \
               -DBoost_USE_STATIC_LIBS=$(HELIO_USE_STATIC_LIBS) \
               -DOPENSSL_USE_STATIC_LIBS=$(HELIO_OPENSSL_USE_STATIC_LIBS) \
               -DENABLE_GIT_VERSION=$(HELIO_ENABLE_GIT_VERSION) \
@@ -26,13 +38,11 @@ configure:
 
 build:
 	cd build-opt; \
-	ninja dragonfly; \
+	ninja dragonfly && ldd dragonfly
-	ldd dragonfly
 
 build-debug:
 	cd build-dbg; \
-	ninja dragonfly; \
+	ninja dragonfly && ldd dragonfly
-	ldd dragonfly
 
 package:
 	cd build-opt; \

tests/README.md

@@ -38,7 +38,7 @@ source <virtual env name>/bin/activate
 ```
 Then install all the required dependencies for the tests:
 ```
-pip install -r dragonfly/requirements.txt
+pip3 install -r dragonfly/requirements.txt
 ```
 
 ### Running the tests

tools/packaging/Dockerfile.alpine-dev

@@ -5,7 +5,7 @@ FROM alpine:3 as builder
 
 # "openssl-libs-static" fixes "Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the"
 RUN apk add autoconf-archive automake bash bison boost-dev cmake coreutils \
-        curl ccache git gcc gdb g++ libunwind-dev libtool libxml2-dev make ninja \
+        curl ccache git gcc gdb g++ libunwind-dev libtool make ninja \
         openssl-dev openssl-libs-static patch zip zstd-static
 
 # This is required to make static linking work
@@ -30,7 +30,7 @@ COPY tools/docker/healthcheck.sh /usr/local/bin/healthcheck.sh
 COPY --from=builder /build/build-opt/dragonfly /usr/local/bin/
 
 RUN apk --no-cache add libgcc libstdc++  \
-     su-exec netcat-openbsd libxml2 icu boost-context && ldd /usr/local/bin/dragonfly
+     su-exec netcat-openbsd boost-context && ldd /usr/local/bin/dragonfly
 
 RUN addgroup -S -g 1000 dfly && adduser -S -G dfly -u 999 dfly
 RUN mkdir /data && chown dfly:dfly /data

tools/packaging/Dockerfile.alpine-prod.wip

@@ -14,7 +14,7 @@ RUN ninja dragonfly
 FROM alpine:latest
 
 RUN addgroup -S -g 1000 dfly && adduser -S -G dfly -u 999 dfly
-RUN apk --no-cache add libgcc libstdc++ libunwind boost1.77-fiber libxml2 \
+RUN apk --no-cache add libgcc libstdc++ libunwind boost-context \
     'su-exec>=0.2' netcat-openbsd openssl
 
 RUN mkdir /data && chown dfly:dfly /data

tools/packaging/Dockerfile.ubuntu-dev

@@ -11,8 +11,8 @@ RUN \
   export DEBIAN_FRONTEND=noninteractive && \
   apt update && \
   apt install -q -y autoconf-archive cmake curl git libssl-dev \
-  libunwind-dev ninja-build libtool gcc-9 g++-9 libboost-fiber-dev \
+  libunwind-dev ninja-build libtool gcc-9 g++-9 libboost-context-dev \
-  libxml2-dev zip libzstd-dev bison
+  zip libzstd-dev bison
 
 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 40  \
   --slave   /usr/bin/g++ g++ /usr/bin/g++-9

tools/packaging/Dockerfile.ubuntu-prod

@@ -20,7 +20,7 @@ FROM ubuntu:20.04
 ARG QEMU_CPU
 ARG DEBIAN_FRONTEND=noninteractive
 
-RUN apt clean && apt update && apt -y install netcat-openbsd ca-certificates redis-tools libxml2
+RUN apt clean && apt update && apt -y install netcat-openbsd ca-certificates redis-tools
 
 RUN groupadd -r -g 999 dfly && useradd -r -g dfly -u 999 dfly
 RUN mkdir /data && chown dfly:dfly /data

tools/packaging/rpm/build_rpm.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+
+# Get the full path of the binary
+ARCHIVE=$(realpath "$1")
+VERSION="$2"
+echo "Preparing $ARCHIVE"
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+
+# Setup RPM build environment in a unique subdirectory under /tmp
+RPM_ROOT=$(mktemp -d /tmp/rpmbuild_XXXXXX)
+echo "Working dir is $RPM_ROOT"
+mkdir -p $RPM_ROOT/{BUILD,RPMS,SOURCES,SPECS}
+
+# Put the archive and configuration files to the SOURCES directory
+ln -s "$ARCHIVE" -t "$RPM_ROOT/SOURCES/"
+cp $SCRIPT_DIR/dragonfly.service $RPM_ROOT/SOURCES/
+cp $SCRIPT_DIR/dragonfly.conf $RPM_ROOT/SOURCES/
+
+cp $SCRIPT_DIR/dragonfly.spec $RPM_ROOT/SPECS/
+
+rpmbuild --define "_topdir $RPM_ROOT" --define "version $VERSION" -bb "$RPM_ROOT/SPECS/dragonfly.spec"
+mv $RPM_ROOT/RPMS/*.rpm ./

tools/packaging/rpm/dragonfly.conf

@@ -0,0 +1 @@
+../debian/dragonfly.conf

tools/packaging/rpm/dragonfly.service

@@ -0,0 +1 @@
+../debian/dragonfly.service

tools/packaging/rpm/dragonfly.spec

@@ -0,0 +1,53 @@
+%define     pkg_name dragonfly
+%define     archive dragonfly-%{_arch}.tar.gz
+
+# How the package name looks like
+%define     _build_name_fmt  %%{NAME}.%%{ARCH}.rpm
+
+Name:       %{pkg_name}
+Version:    %{version}
+Release:    1%{?dist}
+Summary:    DragonflyDB memory store
+License:    BUSL-1.1
+URL:        https://www.dragonflydb.io
+Source0:    %{archive}
+Source1:    dragonfly.service
+Source2:    dragonfly.conf
+Group:      Applications/System
+Provides:   user(dfly)
+Provides:   group(dfly)
+
+%description
+DragonflyDB is a vertically scalable and memory efficient in-memory store
+that is compatible with Redis OSS and Memcached.
+
+%pre
+
+getent group dfly >/dev/null || groupadd -r dfly
+getent passwd dfly >/dev/null || useradd -r -g dfly -M -s /sbin/nologin -c "User for DragonflyDB service" dfly
+
+%prep
+
+%build
+tar xvfz %{SOURCE0}
+mv ./dragonfly-%{_arch} ./dragonfly
+
+%install
+mkdir -p %{buildroot}/usr/local/bin
+mkdir -p %{buildroot}/etc/dragonfly
+
+install -m 755 ./dragonfly %{buildroot}/usr/local/bin/
+mkdir -p %{buildroot}/usr/lib/systemd/system
+cp %{SOURCE1} %{buildroot}/usr/lib/systemd/system/
+cp %{SOURCE2} %{buildroot}/etc/dragonfly/
+
+%clean
+rm -rf %{buildroot}
+rm -rf %{_builddir}/*
+
+%files
+%attr(-,dfly,dfly) /usr/local/bin/dragonfly
+%attr(-,dfly,dfly) /usr/lib/systemd/system/dragonfly.service
+%attr(-,dfly,dfly) /etc/dragonfly/dragonfly.conf
+
+%changelog