Skip to content

Commit f41f924

Browse files
gruuyagruuya
authored
Merge pull request #581 from splitgraph/engine-aggregation-pushdown-cu-1x57q56
Enable aggregation and grouping pushdown in the engine
2 parents 5a4a7ac + bf30141 commit f41f924

21 files changed

Lines changed: 2929 additions & 23 deletions

File tree

‎.github/workflows/build_and_test_and_release.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -226,10 +226,10 @@ jobs:
226226
with:
227227
path: artifacts
228228
- name: Rename artifacts
229-
run:
230-
- mv artifacts/sgr-windows/sgr.exe artifacts/sgr-windows-x86_64.exe
231-
- mv artifacts/sgr-linux/sgr artifacts/sgr-linux-x86_64
232-
- mv artifacts/sgr-linux/osx artifacts/sgr-osx-x86_64
229+
run: |
230+
mv artifacts/sgr-windows/sgr.exe artifacts/sgr-windows-x86_64.exe
231+
mv artifacts/sgr-linux/sgr artifacts/sgr-linux-x86_64
232+
mv artifacts/sgr-linux/osx artifacts/sgr-osx-x86_64
233233
- name: Release artifacts
234234
uses: softprops/action-gh-release@v1
235235
with:

‎engine/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ ENV PYTHONPATH "${PYTHONPATH}:/splitgraph:/pg_es_fdw"
197197
ARG with_postgis
198198
RUN test -z "${with_postgis}" || (\
199199
export POSTGIS_MAJOR=3 && \
200-
export POSTGIS_VERSION=3.1.4+dfsg-3.pgdg100+1 && \
200+
export POSTGIS_VERSION=3.2.0+dfsg-1.pgdg100+1 && \
201201
apt-get update \
202202
&& apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \
203203
&& apt-get install -y --no-install-recommends \

‎engine/Dockerfile.debug‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
# Multistage Dockerfile to build the Splitgraph engine with debugging capabilities.
44
# For now does not include MySQL and Mongo FDWs
5+
# The underlying splitgraph/pg_debug:development image is built via
6+
# Dockerfile.pg_debug, and the reason for this separation is so that we don't
7+
# need to rebuild PG on every change in Multicorn/FDW instances.
58

69

710
#####
@@ -28,14 +31,9 @@ FROM splitgraph/pg_debug:development
2831

2932
RUN apt-get update -qq && \
3033
apt-get install -y \
31-
curl \
3234
libprotobuf-c1 \
3335
libpython3.7 \
34-
python3.7 \
35-
python3-setuptools \
36-
postgresql-plpython3-12 \
37-
git \
38-
wget && \
36+
postgresql-plpython3-12 && \
3937
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
4038
python3.7 get-pip.py && \
4139
rm get-pip.py && \
@@ -100,7 +98,6 @@ COPY ./engine/init_scripts /docker-entrypoint-initdb.d/
10098
# We don't install Mongo/MySQL extensions in the debug image
10199
RUN sed -i '/\(mongo_fdw\|mysql_fdw\)/d' /docker-entrypoint-initdb.d/000_create_extensions.sql
102100

103-
104101
# Copy the actual Splitgraph code over at this point.
105102
COPY ./splitgraph /splitgraph/splitgraph
106103
COPY ./bin /splitgraph/bin

‎engine/Dockerfile.pg_debug‎

Lines changed: 52 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# syntax = docker/dockerfile:experimental
1+
# syntax = docker/dockerfile:1.3-labs
22

33
# Base PostgreSQL debug image, where we compile and install it from source with
44
# full debugging capabilities. For more details see:
@@ -15,12 +15,16 @@
1515
FROM postgres:12.3
1616

1717
# Compile and install PG in debug mode
18+
19+
ARG use_valgrind
1820
RUN rm -rf /usr/lib/postgresql/12/* && \
1921
apt-get update -qq && \
2022
apt-get install -y --allow-downgrades \
2123
build-essential \
24+
curl \
2225
wget \
2326
git \
27+
vim \
2428
libssl-dev \
2529
libsasl2-dev \
2630
pkgconf \
@@ -51,12 +55,55 @@ RUN rm -rf /usr/lib/postgresql/12/* && \
5155
libxml2-dbg \
5256
zlib1g-dbg \
5357
zlib1g \
54-
zlib1g-dev && \
58+
zlib1g-dev \
59+
xsltproc \
60+
libxml2-utils && \
5561
rm -rf /var/lib/apt/lists/* && \
56-
git clone https://github.com/postgres/postgres.git && \
62+
# pgprint command for GDB
63+
git clone https://github.com/tvesely/gdbpg.git && \
64+
# Download PG source
65+
git clone -b REL_12_STABLE https://github.com/postgres/postgres.git && \
66+
# Download Valgrind source, compile and install
67+
test -z "${use_valgrind}" || (\
68+
# Make a switcheroo in the image entrypoint, so that instead of starting plain
69+
# postgres we start it under Valgrind
70+
sed -i -e 's/"$BASH_SOURCE"/"$BASH_SOURCE" valgrind --leak-check=full --show-leak-kinds=definite,indirect \
71+
--num-callers=25 --log-file=\/pg-valgrind\/valgrind-%p.log --trace-children=yes \
72+
--gen-suppressions=all --suppressions=postgres\/src\/tools\/valgrind.supp \
73+
--suppressions=valgrind-python.supp --verbose --time-stamp=yes \
74+
--error-markers=VALGRINDERROR-BEGIN,VALGRINDERROR-END/' /usr/local/bin/docker-entrypoint.sh && \
75+
# Configure PG with Valgrind support
76+
sed -i -e 's=/\*\s#define\sUSE_VALGRIND\s\*/=#define USE_VALGRIND=' postgres/src/include/pg_config_manual.h && \
77+
# Download Python Valgrind suppressions and make folder for output files
78+
curl https://svn.python.org/projects/python/trunk/Misc/valgrind-python.supp --output valgrind-python.supp && \
79+
mkdir -p /pg-valgrind && sudo chmod a+w /pg-valgrind && \
80+
# Build Valgrind
81+
curl https://sourceware.org/pub/valgrind/valgrind-3.18.1.tar.bz2 --output valgrind-3.18.1.tar.bz2 && \
82+
tar xf valgrind-3.18.1.tar.bz2 && \
83+
cd valgrind-3.18.1 && \
84+
./configure && \
85+
make && \
86+
make install && \
87+
cd ..) && \
88+
# Compile and install PG from source
5789
cd /postgres && \
58-
git checkout REL_12_STABLE && \
5990
mkdir -p /usr/lib/postgresql/12 && \
60-
./configure --enable-cassert --enable-debug CFLAGS="-ggdb -Og -g3 -fno-omit-frame-pointer" -prefix=/usr/lib/postgresql/12 && \
91+
./configure \
92+
--enable-cassert \
93+
--enable-debug \
94+
CFLAGS="-ggdb -Og -g3 -fno-omit-frame-pointer" \
95+
-prefix=/usr/lib/postgresql/12 && \
6196
make && \
6297
make install
98+
# Extensions are not installed automatically; install extensions so that
99+
# we can step through them in GDB (but otherwise they work fine without this).
100+
# cd contrib && \
101+
# make all && \
102+
# make install
103+
104+
# Configure GDB: enable pgprint, disable confirmation and dynamic load prompt
105+
COPY <<EOF /etc/gdb/gdbinit
106+
source /gdbpg/gdbpg.py
107+
set confirm off
108+
set breakpoint pending on
109+
EOF

‎engine/Makefile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,13 @@ SHELL=/bin/bash
1515
.DEFAULT_GOAL := build
1616

1717
with_postgis ?= ""
18+
use_valgrind ?= ""
1819

1920

2021
build.pg_debug:
2122
cd .. && docker build \
2223
--build-arg BUILDKIT_INLINE_CACHE=1 \
24+
--build-arg use_valgrind=$(use_valgrind) \
2325
-t splitgraph/pg_debug:development \
2426
-f engine/Dockerfile.pg_debug .
2527

‎engine/etc/postgresql/postgresql_debug.conf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,4 @@ log_checkpoints = on
1919
log_connections = on
2020
log_disconnections = on
2121
log_duration = on
22+
log_error_verbosity = 'VERBOSE'

‎engine/src/Multicorn‎

‎engine/src/postgres-elasticsearch-fdw‎

‎test/architecture/docker-compose.ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,6 @@ services:
66
ci_mongoorigin:
77
image: splitgraphci/mongoorigin
88
build: ./src/mongoorigin
9+
ci_esorigin:
10+
image: splitgraphci/esorigin
11+
build: ./src/esorigin

‎test/architecture/docker-compose.dev.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,5 +74,6 @@ services:
7474
- pgorigin
7575
- mongoorigin
7676
- mysqlorigin
77+
- esorigin
7778
volumes:
7879
venvs:

0 commit comments

Comments
 (0)