splitgraph · gruuya · Dec 27, 2021 · Dec 15, 2021 · Dec 16, 2021 · Dec 16, 2021
diff --git a/.github/workflows/build_and_test_and_release.yml b/.github/workflows/build_and_test_and_release.yml
@@ -226,10 +226,10 @@ jobs:
         with:
           path: artifacts
       - name: Rename artifacts
-        run:
-          - mv artifacts/sgr-windows/sgr.exe artifacts/sgr-windows-x86_64.exe
-          - mv artifacts/sgr-linux/sgr artifacts/sgr-linux-x86_64
-          - mv artifacts/sgr-linux/osx artifacts/sgr-osx-x86_64
+        run: |
+          mv artifacts/sgr-windows/sgr.exe artifacts/sgr-windows-x86_64.exe
+          mv artifacts/sgr-linux/sgr artifacts/sgr-linux-x86_64
+          mv artifacts/sgr-linux/osx artifacts/sgr-osx-x86_64
       - name: Release artifacts
         uses: softprops/action-gh-release@v1
         with:

diff --git a/engine/Dockerfile b/engine/Dockerfile
@@ -197,7 +197,7 @@ ENV PYTHONPATH "${PYTHONPATH}:/splitgraph:/pg_es_fdw"
 ARG with_postgis
 RUN test -z "${with_postgis}" || (\
     export POSTGIS_MAJOR=3 && \
-    export POSTGIS_VERSION=3.1.4+dfsg-3.pgdg100+1 && \
+    export POSTGIS_VERSION=3.2.0+dfsg-1.pgdg100+1 && \
     apt-get update \
       && apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \
       && apt-get install -y --no-install-recommends \

diff --git a/engine/Dockerfile.debug b/engine/Dockerfile.debug
@@ -2,6 +2,9 @@
 
 # Multistage Dockerfile to build the Splitgraph engine with debugging capabilities.
 # For now does not include MySQL and Mongo FDWs
+# The underlying splitgraph/pg_debug:development image is built via
+# Dockerfile.pg_debug, and the reason for this separation is so that we don't
+# need to rebuild PG on every change in Multicorn/FDW instances.
 
 
 #####
@@ -28,14 +31,9 @@ FROM splitgraph/pg_debug:development
 
 RUN apt-get update -qq && \
     apt-get install -y \
-        curl \
         libprotobuf-c1 \
         libpython3.7 \
-        python3.7 \
-        python3-setuptools \
-        postgresql-plpython3-12 \
-        git \
-        wget && \
+        postgresql-plpython3-12 && \
     curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
     python3.7 get-pip.py && \
     rm get-pip.py && \
@@ -100,7 +98,6 @@ COPY ./engine/init_scripts /docker-entrypoint-initdb.d/
 # We don't install Mongo/MySQL extensions in the debug image
 RUN sed -i '/\(mongo_fdw\|mysql_fdw\)/d' /docker-entrypoint-initdb.d/000_create_extensions.sql
 
-
 # Copy the actual Splitgraph code over at this point.
 COPY ./splitgraph /splitgraph/splitgraph
 COPY ./bin /splitgraph/bin

diff --git a/engine/Dockerfile.pg_debug b/engine/Dockerfile.pg_debug
@@ -1,4 +1,4 @@
-# syntax = docker/dockerfile:experimental
+# syntax = docker/dockerfile:1.3-labs
 
 # Base PostgreSQL debug image, where we compile and install it from source with
 # full debugging capabilities. For more details see:
@@ -15,12 +15,16 @@
 FROM postgres:12.3
 
 # Compile and install PG in debug mode
+
+ARG use_valgrind
 RUN rm -rf /usr/lib/postgresql/12/* && \
     apt-get update -qq && \
     apt-get install -y --allow-downgrades \
         build-essential \
+        curl \
         wget \
         git \
+        vim \
         libssl-dev \
         libsasl2-dev \
         pkgconf \
@@ -51,12 +55,55 @@ RUN rm -rf /usr/lib/postgresql/12/* && \
         libxml2-dbg \
         zlib1g-dbg \
         zlib1g \
-        zlib1g-dev && \
+        zlib1g-dev \
+        xsltproc \
+        libxml2-utils && \
     rm -rf /var/lib/apt/lists/* && \
-    git clone https://github.com/postgres/postgres.git && \
+    # pgprint command for GDB
+    git clone https://github.com/tvesely/gdbpg.git && \
+    # Download PG source
+    git clone -b REL_12_STABLE https://github.com/postgres/postgres.git && \
+    # Download Valgrind source, compile and install
+    test -z "${use_valgrind}" || (\
+        # Make a switcheroo in the image entrypoint, so that instead of starting plain
+        # postgres we start it under Valgrind
+        sed -i -e 's/"$BASH_SOURCE"/"$BASH_SOURCE" valgrind --leak-check=full --show-leak-kinds=definite,indirect \
+--num-callers=25 --log-file=\/pg-valgrind\/valgrind-%p.log --trace-children=yes \
+--gen-suppressions=all --suppressions=postgres\/src\/tools\/valgrind.supp \
+--suppressions=valgrind-python.supp --verbose --time-stamp=yes \
+--error-markers=VALGRINDERROR-BEGIN,VALGRINDERROR-END/' /usr/local/bin/docker-entrypoint.sh && \
+        # Configure PG with Valgrind support
+        sed -i -e 's=/\*\s#define\sUSE_VALGRIND\s\*/=#define USE_VALGRIND=' postgres/src/include/pg_config_manual.h && \
+        # Download Python Valgrind suppressions and make folder for output files
+        curl https://svn.python.org/projects/python/trunk/Misc/valgrind-python.supp --output valgrind-python.supp && \
+        mkdir -p /pg-valgrind && sudo chmod a+w /pg-valgrind && \
+        # Build Valgrind
+        curl https://sourceware.org/pub/valgrind/valgrind-3.18.1.tar.bz2 --output valgrind-3.18.1.tar.bz2 && \
+        tar xf valgrind-3.18.1.tar.bz2 && \
+        cd valgrind-3.18.1 && \
+        ./configure && \
+        make && \
+        make install && \
+        cd ..) && \
+    # Compile and install PG from source
     cd /postgres && \
-    git checkout REL_12_STABLE && \
     mkdir -p /usr/lib/postgresql/12 && \
-    ./configure --enable-cassert --enable-debug CFLAGS="-ggdb -Og -g3 -fno-omit-frame-pointer" -prefix=/usr/lib/postgresql/12 && \
+    ./configure \
+        --enable-cassert \
+        --enable-debug \
+        CFLAGS="-ggdb -Og -g3 -fno-omit-frame-pointer" \
+        -prefix=/usr/lib/postgresql/12 && \
     make && \
     make install
+    # Extensions are not installed automatically; install extensions so that
+    # we can step through them in GDB (but otherwise they work fine without this).
+    # cd contrib && \
+    # make all && \
+    # make install
+
+# Configure GDB: enable pgprint, disable confirmation and dynamic load prompt
+COPY <<EOF /etc/gdb/gdbinit
+source /gdbpg/gdbpg.py
+set confirm off
+set breakpoint pending on
+EOF
diff --git a/engine/Makefile b/engine/Makefile
@@ -15,11 +15,13 @@ SHELL=/bin/bash
 .DEFAULT_GOAL := build
 
 with_postgis ?= ""
+use_valgrind ?= ""
 
 
 build.pg_debug:
 	cd .. && docker build \
 	--build-arg BUILDKIT_INLINE_CACHE=1 \
+	--build-arg use_valgrind=$(use_valgrind) \
 	-t splitgraph/pg_debug:development \
 	-f engine/Dockerfile.pg_debug .
 

diff --git a/engine/etc/postgresql/postgresql_debug.conf b/engine/etc/postgresql/postgresql_debug.conf
@@ -19,3 +19,4 @@ log_checkpoints = on
 log_connections = on
 log_disconnections = on
 log_duration = on
+log_error_verbosity = 'VERBOSE'
diff --git a/engine/src/Multicorn b/engine/src/Multicorn
diff --git a/engine/src/postgres-elasticsearch-fdw b/engine/src/postgres-elasticsearch-fdw
diff --git a/test/architecture/docker-compose.ci.yml b/test/architecture/docker-compose.ci.yml
@@ -6,3 +6,6 @@ services:
   ci_mongoorigin:
     image: splitgraphci/mongoorigin
     build: ./src/mongoorigin
+  ci_esorigin:
+    image: splitgraphci/esorigin
+    build: ./src/esorigin
diff --git a/test/architecture/docker-compose.dev.yml b/test/architecture/docker-compose.dev.yml
@@ -74,5 +74,6 @@ services:
       - pgorigin
       - mongoorigin
       - mysqlorigin
+      - esorigin
 volumes:
   venvs:
diff --git a/test/architecture/docker-compose.mounting.yml b/test/architecture/docker-compose.mounting.yml
@@ -38,3 +38,7 @@ services:
       - 3306
     volumes:
       - ./data/mysqlorigin/setup.sql:/docker-entrypoint-initdb.d/setup.sql:ro
+  esorigin:
+    image: splitgraphci/esorigin
+    expose:
+      - 9200
diff --git a/test/architecture/src/esorigin/Dockerfile b/test/architecture/src/esorigin/Dockerfile
@@ -0,0 +1,21 @@
+##
+# esorigin
+# Build seed data into our image for tests; appropriated from:
+# https://stackoverflow.com/questions/35526532/how-to-add-an-elasticsearch-index-during-docker-build
+##
+FROM elasticsearch:7.16.1
+
+RUN mkdir /data && \
+    chown -R elasticsearch:elasticsearch /data && \
+    echo 'path.data: /data' >> config/elasticsearch.yml && \
+    echo 'discovery.type: "single-node"' >> config/elasticsearch.yml && \
+    echo "xpack.security.enabled: false" >> config/elasticsearch.yml && \
+    echo 'cluster.routing.allocation.disk.watermark.flood_stage: "99%"' >> config/elasticsearch.yml && \
+    echo 'cluster.routing.allocation.disk.watermark.high: "99%"' >> config/elasticsearch.yml
+
+ADD https://raw.githubusercontent.com/vishnubob/wait-for-it/e1f115e4ca285c3c24e847c4dd4be955e0ed51c2/wait-for-it.sh /utils/wait-for-it.sh
+
+COPY accounts.json /accounts.json
+COPY init-data.sh /init-data.sh
+RUN chmod a+x /init-data.sh
+RUN /usr/local/bin/docker-entrypoint.sh elasticsearch -p /tmp/epid & /bin/bash /utils/wait-for-it.sh -t 0 localhost:9200 -- /init-data.sh; kill $(cat /tmp/epid) && wait $(cat /tmp/epid); exit 0;
+2 −2		Makefile
+35 −2		python/multicorn/__init__.py
+603 −0		src/deparse.c
+551 −29		src/multicorn.c
+119 −4		src/multicorn.h
+136 −2		src/python.c
+13 −3		src/query.c
+78 −17		pg_es_fdw/__init__.py
+45 −4		pg_es_fdw/_es_query.py