Updates: rmbuilder, clang, hugo, noroot-docker (#15)

rafie · web-flow · commit fdea3a1a6d35 · 2022-05-17T10:04:31.000+03:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -62,16 +62,18 @@ jobs:
           command: |
             sudo ./deps/readies/bin/getpy3
             sudo ./deps/readies/bin/getdocker --just-enable-exp
+            sudo usermod -aG docker $USER
             docker version
             docker login --username redisfab --password "$DOCKER_REDISFAB_PWD"
       - run:
           name: Build and publish
+          shell: /bin/bash -l -eo pipefail
           command: |
             cd rmbuilder
             for nick in <<parameters.platforms>>; do
-                bash -l -c "make build X64=1 OSNICK=$nick"
+                make build X64=1 OSNICK=$nick
                 if [[ $CIRCLE_BRANCH == master ]]; then
-                    bash -l -c "make publish X64=1 OSNICK=$nick"
+                    make publish X64=1 OSNICK=$nick
                 fi
             done
           no_output_timeout: 30m
@@ -213,6 +215,7 @@ workflows:
           <<: *not-on-integ-branch
           platforms: "bullseye"
           context: common
+          force: "1"
       - platform-vm-builder:
           name: integ-vm-builder
           <<: *on-integ-branch
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+/[0-9]*/
diff --git a/hugo/Dockerfile b/hugo/Dockerfile
@@ -0,0 +1,16 @@
+
+FROM debian:bullseye
+
+ENV HUGO_VER=0.88.1
+
+RUN apt-get update -qq
+RUN apt-get install -y ca-certificates wget curl rsync
+
+RUN apt-get install -y python3-pip python3-yaml
+RUN python3 -m pip install --upgrade pip
+
+RUN apt-get install -y nodejs npm
+RUN npm install -g npm
+
+RUN wget -q -O /tmp/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VER}/hugo_extended_${HUGO_VER}_Linux-64bit.deb
+RUN dpkg -i /tmp/hugo.deb
diff --git a/hugo/Dockerfile.site b/hugo/Dockerfile.site
@@ -0,0 +1,10 @@
+
+FROM redisfab/hugo:latest
+
+ADD . /build
+WORKDIR /build
+RUN cd website; npm install
+
+RUN make init stack
+RUN make serve IP=0.0.0.0
+
diff --git a/hugo/Makefile b/hugo/Makefile
@@ -0,0 +1,8 @@
+
+IMAGE=redisfab/hugo:latest
+
+build:
+	docker build -t $(IMAGE) .
+
+publish:
+	docker push $(IMAGE)
diff --git a/nonroot-docker/Dockerfile b/nonroot-docker/Dockerfile
@@ -0,0 +1,22 @@
+# docker run -it --user 1000:1000 IMAGE bash
+
+ARG USERID=1000
+ARG GROUPID=1000
+
+FROM ubuntu:focal
+
+ARG USERID
+ARG GROUPID
+
+RUN apt-get -qq update
+RUN apt install -y sudo
+
+RUN echo "CREATE_HOME yes" >> /etc/login.defs
+RUN echo "user    ALL=(ALL:ALL) ALL" >> /etc/sudoers
+
+RUN groupadd -g ${USERID} user
+RUN useradd -d /home/user -g ${GROUPID} -u ${USERID} user
+RUN bash -c 'echo -e "user\nuser" | passwd user'
+
+WORKDIR /home/user
+USER user
diff --git a/rmbuilder/Dockerfile b/rmbuilder/Dockerfile
@@ -1,12 +1,8 @@
 # BUILD redisfab/rmbuilder:${REDIS_VER}-${ARCH}-${OSNICK}
 
-ARG REDIS_VER=6.2.5
-
-# OSNICK=bionic|stretch|buster
-ARG OSNICK=buster
-
-# OS=debian:buster-slim|debian:stretch-slim|ubuntu:bionic
-ARG OS=debian:buster-slim
+ARG REDIS_VER=6.2.7
+ARG OSNICK=bullseye
+ARG OS=debian:bullseye-slim
 
 # ARCH=x64|arm64v8|arm32v7
 ARG ARCH=x64
@@ -31,7 +27,6 @@ RUN python3 ./system-setup.py
 RUN python2 ./gears/system-setup.py
 RUN python3 ./graph/system-setup.py
 RUN python2 ./search/system-setup.py
-RUN python2 ./rscoordinator/system-setup.py
 RUN python3 ./timeseries/system-setup.py
 RUN python3 ./ai/system-setup.py
 RUN python3 ./redisjson/system-setup.py
diff --git a/rmbuilder/Makefile b/rmbuilder/Makefile
@@ -5,7 +5,7 @@ OSNICK ?= buster
 
 # OS ?= debian:buster-slim
 
-REDIS_VER ?= 6.2.5
+REDIS_VER ?= 6.2.7
 
 #----------------------------------------------------------------------------------------------
 
diff --git a/rmbuilder/modules/ai/system-setup.py b/rmbuilder/modules/ai/system-setup.py
@@ -20,20 +20,16 @@ def common_first(self):
         self.install_downloaders()
         self.pip_install("wheel")
 
-        self.install("git unzip") # patchelf
+        self.install("git unzip")
         if self.osnick != 'centos8':
             self.install("coreutils") # for realpath
 
     def debian_compat(self):
         self.run("%s/bin/enable-utf8" % READIES)
         self.run("%s/bin/getgcc" % READIES)
-        self.install("gawk")
-        self.install("libssl-dev")
-        self.install("python3-regex")
-        self.install("python3-networkx python3-numpy")
+        self.install("gawk libssl-dev python3-regex python3-networkx libmpich-dev libopenblas-dev")
         if self.platform.is_arm():
             self.install("python3-dev") # python3-skimage
-        self.install("libmpich-dev libopenblas-dev") # for libtorch
         self.install_git_lfs_on_linux()
 
     def redhat_compat(self):
@@ -71,7 +67,7 @@ def macos(self):
     def common_last(self):
         self.run("%s/bin/getclang --format" % READIES)
         if self.platform == "arm":
-            self.run("%s/bin/getcmake" % READIES)
+            self.run("%s/bin/getcmake --from-repo" % READIES)
         else:
             self.run("%s/bin/getcmake" % READIES)
 
diff --git a/rmbuilder/modules/gears/build/cpython/system-setup.py b/rmbuilder/modules/gears/build/cpython/system-setup.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 import sys
 import os
@@ -31,7 +31,6 @@ def debian_compat(self):
         if sh("apt-cache search libgdbm-compat-dev") != "":
             self.install("libgdbm-compat-dev")
         self.install("libgdbm-dev")
-        self.install("tcl-dev tix-dev tk-dev")
 
         self.install("lsb-release")
 
@@ -42,7 +41,6 @@ def redhat_compat(self):
 
         self.install("bzip2-devel expat-devel gdbm-devel glibc-devel gmp-devel libffi-devel libuuid-devel ncurses-devel "
             "openssl-devel readline-devel sqlite-devel xz-devel zlib-devel")
-        self.install("tcl-devel tix-devel tk-devel")
 
         self.install("redhat-lsb-core")
         self.install("libatomic file")
@@ -55,7 +53,6 @@ def fedora(self):
         self.install("autoconf automake libtool")
         self.install("bzip2-devel expat-devel gdbm-devel glibc-devel gmp-devel libffi-devel libnsl2-devel libuuid-devel ncurses-devel "
             "openssl-devel readline-devel sqlite-devel xz-devel zlib-devel")
-        self.install("tcl-devel tix-devel tk-devel")
 
         self.install("libatomic file")
 
diff --git a/rmbuilder/modules/gears/system-setup.py b/rmbuilder/modules/gears/system-setup.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 import sys
 import os
@@ -32,12 +32,13 @@ def debian_compat(self):
         self.install("lsb-release")
         self.install("zip unzip gawk")
         self.install("python-dev")
+        self.install("locales-all")
 
         # pip cannot build gevent on ARM
         if self.platform.is_arm() and self.dist == 'ubuntu' and self.os_version[0] < 20:
             self.install("python-gevent")
         else:
-            self.pip_install("gevent~=1.2.0")
+            self.pip_install("gevent")
 
     def redhat_compat(self):
         self.run("%s/bin/getgcc --modern" % READIES)
@@ -53,8 +54,8 @@ def redhat_compat(self):
         if self.arch == 'x64':
             self.install_linux_gnu_tar()
 
-        if self.platform.is_arm() or self.dist == 'centos' and self.os_version[0] == 8:
-            self.install("python3-gevent python3-ujson")
+        if self.platform.is_arm():
+            self.install("python-gevent python-ujson")
         else:
             self.pip_install("gevent ujson")
 
diff --git a/rmbuilder/modules/graph/system-setup.py b/rmbuilder/modules/graph/system-setup.py
@@ -18,18 +18,25 @@ def __init__(self, nop=False):
 
     def common_first(self):
         self.install_downloaders()
-        self.pip_install("wheel virtualenv")
-        self.pip_install("setuptools --upgrade")
 
-        self.install("git automake libtool autoconf astyle")
+        self.run("%s/bin/enable-utf8" % READIES, sudo=self.os != 'macos')
+        self.install("git automake libtool autoconf")
 
     def debian_compat(self):
         self.install("locales")
-        self.run("%s/bin/getgcc" % READIES)
+        if self.platform.is_arm():
+            self.run("%s/bin/getgcc --modern" % READIES)
+        else:
+            self.run("%s/bin/getgcc" % READIES)
         self.install("peg")
 
     def redhat_compat(self):
         self.install("redhat-lsb-core")
+        if not self.platform.is_arm():
+            self.install_linux_gnu_tar()
+        if self.osnick == 'ol8':
+            self.install("which") # for automake
+        self.run("%s/bin/getepel" % READIES, sudo=True)
         self.run("%s/bin/getgcc --modern" % READIES)
         self.install("m4 libgomp")
         self.install_peg()
@@ -40,29 +47,44 @@ def fedora(self):
 
     def macos(self):
         self.install_gnu_utils()
-        self.run("%s/bin/getgcc --modern" % READIES)
+        # self.run("%s/bin/getgcc --modern" % READIES)
+        self.run("brew install libomp")
         self.install("redis")
         self.install_peg()
 
+    def alpine(self):
+        self.install("automake make autoconf libtool m4")
+        self.run("%s/bin/getgcc" % READIES)
+        self.install_peg()
+
     def linux_last(self):
         self.install("valgrind")
 
     def common_last(self):
-        self.run("%s/bin/getcmake" % READIES)
-        # self.run("{PYTHON} {READIES}/bin/getrmpytools".format(PYTHON=self.python, READIES=READIES))
-
+        self.install("astyle", _try=True) # fails for centos7
+        self.run("{PYTHON} {READIES}/bin/getcmake --usr".format(PYTHON=self.python, READIES=READIES),
+                 sudo=self.os != 'macos')
+        if self.dist != "arch":
+            self.install("lcov")
+        else:
+            self.install("lcov-git", aur=True)
+
+        # self.run("{PYTHON} {READIES}/bin/getrmpytools --reinstall --modern".format(PYTHON=self.python, READIES=READIES))
+        self.pip_install("awscli")
         # self.pip_install("-r tests/requirements.txt")
+        # self.pip_install("-r tests/fuzz/requirements.txt")
+        self.run("%s/bin/getpy2" % READIES) # for RediSearch build
 
     def install_peg(self):
         self.run(r"""
             cd /tmp
             build_dir=$(mktemp -d)
             cd $build_dir
-            wget https://www.piumarta.com/software/peg/peg-0.1.18.tar.gz
-            tar xzf peg-0.1.18.tar.gz
+            wget -q -O peg.tar.gz https://github.com/gpakosz/peg/archive/0.1.18.tar.gz
+            tar xzf peg.tar.gz
             cd peg-0.1.18
             make
-            make install
+            $(command -v sudo) make install MANDIR=.
             cd /tmp
             rm -rf $build_dir
             """)
diff --git a/rmbuilder/modules/redisjson/system-setup.py b/rmbuilder/modules/redisjson/system-setup.py
@@ -13,20 +13,20 @@
 #----------------------------------------------------------------------------------------------
 
 class RedisJSONSetup(paella.Setup):
-    def __init__(self, nop=False):
-        paella.Setup.__init__(self, nop)
+    def __init__(self, args):
+        paella.Setup.__init__(self, args.nop)
 
     def common_first(self):
         self.install_downloaders()
-        self.pip_install("wheel")
-        self.pip_install("setuptools --upgrade")
-
-        self.install("git")
+        self.run("%s/bin/enable-utf8" % READIES, sudo=self.os != 'macos')
+        self.install("git unzip rsync")
 
+        if self.osnick == 'ol8':
+            self.install("tar")
         self.run("%s/bin/getclang --modern" % READIES)
         if not self.has_command("rustc"):
             self.run("%s/bin/getrust" % READIES)
-        self.run("%s/bin/getcmake" % READIES)
+        self.run("%s/bin/getcmake --usr" % READIES)
 
     def debian_compat(self):
         self.run("%s/bin/getgcc" % READIES)
@@ -39,18 +39,21 @@ def fedora(self):
         self.run("%s/bin/getgcc" % READIES)
 
     def macos(self):
+        self.install_gnu_utils()
+        self.install("binutils")
         self.run("%s/bin/getgcc" % READIES)
 
     def common_last(self):
-        # self.run("python3 %s/bin/getrmpytools" % READIES)
+        # self.run("{PYTHON} {READIES}/bin/getrmpytools".format(PYTHON=self.python, READIES=READIES))
         # self.pip_install("-r %s/tests/pytest/requirements.txt" % ROOT)
         self.pip_install("toml")
-        self.pip_install("awscli")
+        self.pip_install("pudb awscli")
+        self.pip_install("gevent")
 
 #----------------------------------------------------------------------------------------------
 
 parser = argparse.ArgumentParser(description='Set up system for build.')
 parser.add_argument('-n', '--nop', action="store_true", help='no operation')
 args = parser.parse_args()
 
-RedisJSONSetup(nop = args.nop).setup()
+RedisJSONSetup(args).setup()
diff --git a/rmbuilder/modules/rscoordinator/system-setup.py b/rmbuilder/modules/rscoordinator/system-setup.py
diff --git a/rmbuilder/modules/search/system-setup.py b/rmbuilder/modules/search/system-setup.py
diff --git a/rmbuilder/modules/timeseries/system-setup.py b/rmbuilder/modules/timeseries/system-setup.py
diff --git a/rmbuilder/system-setup.py b/rmbuilder/system-setup.py
