Issue #8: Add MonetDB support and tests

Signed-off-by: Lucas Fernandes de Oliveira's avatarLucas Fernandes de Oliveira <lfoliveira@inf.ufpr.br>
parent 0597b88f
Pipeline #20974 passed with stage
in 1 minute and 4 seconds
image: debian:stretch
image: debian:buster
services:
- "postgres:10"
-
name: "dockerregistry.c3sl.ufpr.br:5000/c3sl/monetdb-ci-service:latest"
alias: "monet"
variables:
POSTGRES_DB: 'blendb_fixture'
POSTGRES_DB: 'usql_fixture'
POSTGRES_USER: 'runner'
POSTGRES_HOST: 'postgres'
PGPASSWORD: 'pass'
MONETDB_NAME: 'usql_fixture'
stages:
- test
cache:
paths:
- /var/cache/apt
- /var/cache/apt/archives
run_test:
postgres_test:
stage: test
variables:
DB_NAME: 'usql_fixture'
DB_USER: 'runner'
DB_HOST: 'postgres'
DB_PASSWORD: 'pass'
script:
- apt-get update -q -y
- apt-get install bzip2 wget gnupg -y
- echo "deb http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main" > /etc/apt/sources.list.d/pgdg.list
- echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list
- wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
- apt-get update -q -y
- apt-get install -y postgresql-client-10
- ./test/test.sh
- ./test/psql.sh
tags:
- debian
- postgres
monet_test:
stage: test
variables:
DB_NAME: 'usql_fixture'
DB_USER: 'monetdb'
DB_HOST: 'monet'
DB_PASSWORD: 'monetdb'
script:
- apt-get update -q -y
- apt-get install bzip2 wget gnupg -y
- echo "deb https://dev.monetdb.org/downloads/deb/ buster monetdb" > /etc/apt/sources.list.d/monetdb.list
- wget --quiet -O - https://www.monetdb.org/downloads/MonetDB-GPG-KEY | apt-key add -
- apt-get update -q -y
- apt-get install -y monetdb-client
- ./test/monet.sh
tags:
- debian
- monet
......@@ -3,7 +3,8 @@
# Script used as interface to manage a PostgreSQL database
if [[ $# -ne 1 && $# -ne 2 ]]; then
echo "Correct use: $0 <task> [directory]"
echo "Correct use: $0 <dbms> <task> [directory]"
echo "The allowed dbms's are: psql and monet"
echo "The allowed tasks are: create, load, fixture, drop and clean"
echo "The directory structure should respect the example struct used in:"
echo "./data directory"
......@@ -12,8 +13,9 @@ if [[ $# -ne 1 && $# -ne 2 ]]; then
fi
# Rename arguments
task=$1
workspace=${2:-"./data"}
DBMS=$1
task=$2
workspace=${3:-"./data"}
# transform relative path into absolute path,
# only when the path is indicated
......@@ -46,14 +48,17 @@ fi
correctEnv || exit 1
if [ ${TASKS[$task]+_} ]; then
initDBMS $workspace
${TASKS[$task]} $workspace
if [[ $? -ne 0 ]]; then
echo "Some error occured on task <$task>"
echo "Check your connection parameters and schema files"
echo "Aborting..."
releaseDBMS $workspace
exit 1
else
echo "Task <$task> performed with success"
releaseDBMS $workspace
exit 0
fi
else
......
......@@ -5,10 +5,7 @@ cleanDB() {
# Get all deletable tables fron DB
tables=$(
unset -v $(locale | cut -f1 -d'=');
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-c "\d" |\
execStatement "\d" |\
tail -n+4 |\
head -n-2 |\
grep "^.*|.*| *table *|.*$" |\
......@@ -18,19 +15,16 @@ cleanDB() {
tmpWS=$(createTmpWorkspace)
# Create transactions files
echo "BEGIN;" > "$tmpWS/trans.sql"
beginTrans > "$tmpWS/trans.sql"
i=0
for t in $tables; do
echo "DELETE FROM $t;">> "$tmpWS/trans.sql"
done
echo "COMMIT;" >> "$tmpWS/trans.sql"
commitTrans >> "$tmpWS/trans.sql"
# Performs schema creation. As is in a transaction if fails it will
# NOT leave the database inconsistent.
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-f $tmpWS/trans.sql 2> $tmpWS/error.out
execFile $tmpWS/trans.sql $tmpWS/error.out
# If a error occurs will be sent to error.out
# If error.out is not empty, should return a error
......
......@@ -19,9 +19,10 @@ createDB() {
tmpWS=$(createTmpWorkspace)
# Create transactions files
echo "BEGIN;" > "$tmpWS/begin.sql"
echo "COMMIT;" > "$tmpWS/commit.sql"
beginTrans > "$tmpWS/begin.sql"
commitTrans > "$tmpWS/commit.sql"
mkfifo $tmpWS/trans.fifo
touch $tmpWS/error.out
# Concats the transaction and schema files
cat "$tmpWS/begin.sql"\
......@@ -31,10 +32,7 @@ createDB() {
# Performs schema creation. As is in a transaction if fails it will
# NOT leave the database inconsistent.
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-f $tmpWS/trans.fifo 2> $tmpWS/error.out
execFile $tmpWS/trans.fifo $tmpWS/error.out
# If a error occurs will be sent to error.out
# If error.out is not empty, should return a error
......
#! /bin/bash
# Executes the SQL file in the target DBMS
execFile() {
file=$1
error=${2:-"/tmp/error.out"}
if [[ ${DBMS} == "psql" ]]; then
PGPASSFILE=$PGPASSFILE psql -d ${DB_NAME} -U ${DB_USER} -h ${DB_HOST} -f $file 2> $error
elif [[ ${DBMS} == "monet" ]]; then
DOTMONETDBFILE=$DOTMONETDBFILE mclient < $file 2> $error
fi
}
# Executes the SQL file in the target DBMS
execStatement() {
statement=$1
if [[ ${DBMS} == "psql" ]]; then
PGPASSFILE=$PGPASSFILE psql -d ${DB_NAME} -U ${DB_USER} -h ${DB_HOST} -c "$statement"
elif [[ ${DBMS} == "monet" ]]; then
echo "$statement" | DOTMONETDBFILE=$DOTMONETDBFILE mclient -i
fi
}
# Begin Transaction
beginTrans() {
if [[ $DBMS == "psql" ]]; then
echo "BEGIN;"
elif [[ $DBMS == "monet" ]]; then
echo "START TRANSACTION;"
fi
}
# Commit Transaction
commitTrans() {
if [[ $DBMS == "psql" ]]; then
echo "COMMIT;"
elif [[ $DBMS == "monet" ]]; then
echo "COMMIT;"
fi
}
# Init DBMS configuration
initDBMS() {
ws=${1:-"./data"}
if [[ $DBMS == "psql" ]]; then
PGPASSFILE="$ws/.pgpass"
echo "${DB_HOST}:5432:${DB_NAME}:${DB_USER}:${DB_PASSWORD}" > $PGPASSFILE
chmod 0600 $PGPASSFILE
elif [[ $DBMS == "monet" ]]; then
DOTMONETDBFILE="$ws/.monetdb"
echo "user=${DB_USER}" > $DOTMONETDBFILE
echo "password=${DB_PASSWORD}" >> $DOTMONETDBFILE
echo "database=mapi:monetdb://${DB_HOST}:50000/${DB_NAME}" >> $DOTMONETDBFILE
chmod 0600 $DOTMONETDBFILE
fi
}
# Releases DBMS configuration
releaseDBMS() {
ws=${1:-"./data"}
if [[ $DBMS == "psql" ]]; then
rm $PGPASSFILE
return 0
elif [[ $DBMS == "monet" ]]; then
rm $DOTMONETDBFILE
fi
}
# Get the list of tables in database
getTables() {
if [[ $DBMS == "psql" ]]; then
unset -v $(locale | cut -f1 -d'=');
execStatement "\d" |\
grep "^.*|.*| *table *|.*$" |\
cut -d'|' -f2
elif [[ $DBMS == "monet" ]]; then
unset -v $(locale | cut -f1 -d'=');
execStatement "\d" |\
grep "^TABLE" |\
cut -d'.' -f2
fi
}
# Get the list of sequences in database
getSequences() {
#FIXME: Unexpected result if the table contains more than one sequence
if [[ $DBMS == "psql" ]]; then
tables=$(getTables)
for t in $tables; do
seq=$(\
execStatement "\d $t" |\
grep "nextval(" |\
sed -e "s/nextval('\|'::regclass)//g"
)
attr=$(echo "$seq" | cut -d'|' -f1)
seq_name=$(echo "$seq" | cut -d'|' -f5)
echo "$seq_name,$attr,$t" | sed -e 's/ //g'
done
elif [[ $DBMS == "monet" ]]; then
tables=$(getTables)
for t in $tables; do
seq=$(\
execStatement "\d $t" |\
grep "next value" |\
tr -s ' ' |\
sed -e 's/\t//g' |\
sed -e 's/,//g'\
)
attr=$(echo "$seq" | cut -d' ' -f1)
seq_name=$(echo "$seq" | cut -d'.' -f2)
echo "$seq_name,$attr,$t" | sed -e 's/ //g'
done
fi
}
# Created a query that updates a sequence
sequenceComand() {
seq_name=$1
attr=$2
table=$3
if [[ $DBMS == "psql" ]]; then
echo "SELECT setval('$seq_name', COALESCE((SELECT MAX($attr) FROM $table)+1,1),false);"
elif [[ $DBMS == "monet" ]]; then
echo "ALTER SEQUENCE "$seq_name" RESTART WITH (SELECT MAX($attr) + 1 FROM $table);"
fi
}
# Creates the COPY command used to insert data
cpyComand() {
table=${1:-""}
file=${2:-""}
if [[ $DBMS == "psql" ]]; then
echo "\\copy $table FROM '$file' WITH CSV HEADER ENCODING 'UTF8' DELIMITER ';'"
elif [[ $DBMS == "monet" ]]; then
echo "COPY OFFSET 2 INTO $table FROM '$file' ON CLIENT USING DELIMITERS ';','\\n','\"' NULL AS '';"
fi
}
......@@ -2,33 +2,22 @@
# Drop a database. Removes all tables and data
dropDB() {
tables=$(
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-c "\d" |\
tail -n+4 |\
head -n-2 |\
grep "^.*|.*|.*table.*|.*$" |\
cut -d '|' -f2\
)
tables=$(getTables)
# Create temporary workspace (utility function)
tmpWS=$(createTmpWorkspace)
# Create transactions files
echo "BEGIN;" > "$tmpWS/trans.sql"
beginTrans > "$tmpWS/trans.sql"
i=0
for t in $tables; do
dropComand $t >> "$tmpWS/trans.sql"
done
echo "COMMIT;" >> "$tmpWS/trans.sql"
commitTrans >> "$tmpWS/trans.sql"
# Performs schema creation. As is in a transaction if fails it will
# NOT leave the database inconsistent.
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-f $tmpWS/trans.sql 2> $tmpWS/error.out
execFile $tmpWS/trans.sql $tmpWS/error.out
# If a error occurs will be sent to error.out
# If error.out is not empty, should return a error
......
......@@ -14,9 +14,8 @@ fixtureDB() {
# Create temporary workspace (utility function)
tmpWS=$(createTmpWorkspace)
# Create transactions files
echo "BEGIN;" > "$tmpWS/trans.sql"
beginTrans > "$tmpWS/trans.sql"
i=0
tables=$(ls -1 $workspace | grep ".bz2" | rev | cut -f4- -d\. | rev)
for t in $tables; do
mkfifo $tmpWS/$t.csv
......@@ -27,16 +26,11 @@ fixtureDB() {
for t in $tables; do
cpyComand $t $workspace/$t.csv >> "$tmpWS/trans.sql"
done
echo "COMMIT;" >> "$tmpWS/trans.sql"
commitTrans >> "$tmpWS/trans.sql"
# Performs schema creation. As is in a transaction if fails it will
# NOT leave the database inconsistent.
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-f $tmpWS/trans.sql 2> $tmpWS/error.out
execFile $tmpWS/trans.sql $tmpWS/error.out
# If a error occurs will be sent to error.out
# If error.out is not empty, should return a error
......
......@@ -9,9 +9,7 @@ loadDB() {
# Create temporary workspace (utility function)
tmpWS=$(createTmpWorkspace)
# Create transactions files
echo "BEGIN;" > "$tmpWS/trans.sql"
beginTrans > "$tmpWS/trans.sql"
tables=$(ls -1 $workspace | grep ".bz2" | rev | cut -f4- -d\. | rev)
for t in $tables; do
mkfifo $tmpWs/$t.csv
......@@ -21,16 +19,13 @@ loadDB() {
tables=$(ls -1 $workspace | grep -v ".bz2" | rev | cut -f2- -d\. | rev)
for t in $tables; do
cpyComand $t $workspace/$t.csv >> "$tmpWS/trans.sql"
done
done
echo "COMMIT;" >> "$tmpWS/trans.sql"
commitTrans >> "$tmpWS/trans.sql"
# Performs schema creation. As is in a transaction if fails it will
# NOT leave the database inconsistent.
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-f $tmpWS/trans.sql 2> $tmpWS/error.out
execFile $tmpWS/trans.sql $tmpWS/error.out
# If a error occurs will be sent to error.out
# If error.out is not empty, should return a error
......
......@@ -9,7 +9,7 @@ createTmpWorkspace() {
# Check if the environment vars are set
correctEnv() {
vars="POSTGRES_HOST POSTGRES_DB POSTGRES_USER PGPASSWORD"
vars="DB_HOST DB_NAME DB_USER DB_PASSWORD"
error=0
for var in $vars; do
if [[ -z ${!var} ]]; then
......@@ -20,57 +20,24 @@ correctEnv() {
return $error
}
# Creates the COPY command used to insert data
cpyComand() {
table=${1:-""}
file=${2:-""}
echo "\\copy $table FROM '$file' WITH CSV HEADER ENCODING 'UTF8' DELIMITER ';'"
}
sequence() {
unset -v $(locale | cut -f1 -d'=');
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-c "\d $1" | grep "^.*|.*|.*|.*| *nextval('.*_seq'::regclass) *$"
}
# Updates the sequence tables. Used to fix serial data after copy
updateSequences() {
# Get sequences to update
tables=$(
unset -v $(locale | cut -f1 -d'=');
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-c "\d" |\
grep "^.*|.*| *table *|.*$" |\
cut -d'|' -f2)
# Create temporary workspace (utility function)
tmpWS=$(createTmpWorkspace)
# Create transactions files
echo "BEGIN;" > "$tmpWS/seq-trans.sql"
for t in $tables; do
results=$(sequence $t)
readarray -td')' arr <<< "$results"
unset 'arr[-1]'
for result in "${arr[@]}";do
id=$(echo $result| cut -d'|' -f1 | tr -d '[:space:]')
id_seq=$(echo $result | cut -d"'" -f2)
query="SELECT setval('$id_seq', COALESCE((SELECT MAX($id) FROM $t)+1,1),false);"
echo $query >> "$tmpWS/seq-trans.sql"
done
beginTrans > "$tmpWS/seq-trans.sql"
for i in $(getSequences); do
seq_name=$(echo $i | cut -d',' -f1)
attr=$(echo $i | cut -d',' -f2)
table=$(echo $i | cut -d',' -f3)
sequenceComand "$seq_name" "$attr" "$table" >> "$tmpWS/seq-trans.sql"
done
echo "COMMIT;" >> "$tmpWS/seq-trans.sql"
commitTrans >> "$tmpWS/seq-trans.sql"
# Performs sequence update. As is in a transaction if fails it will
# NOT leave the database inconsistent.
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-f $tmpWS/seq-trans.sql 2> $tmpWS/error.out
execFile $tmpWS/seq-trans.sql $tmpWS/error.out
# If a error occurs will be sent to error.out
# If error.out is not empty, should return a error
......
#! /bin/bash
error=0
basePath=$(echo $BASH_SOURCE | rev | cut -c 9- | rev)
cd $basePath/..
DOTMONETDBFILE="/tmp/.monetdb"
echo "user=${DB_USER}" > $DOTMONETDBFILE
echo "password=${DB_PASSWORD}" >> $DOTMONETDBFILE
echo "database=mapi:monetdb://${DB_HOST}:50000/${DB_NAME}" >> $DOTMONETDBFILE
chmod 0600 $DOTMONETDBFILE
#Before all tests
./manager.sh monet drop &> /dev/null
echo "Should create a database"
./manager.sh monet create &> /dev/null
if [[ $? -eq 0 ]]; then
echo -e "\tSucess"
else
error=$(($error +1))
echo -e "\tFail"
fi
echo "Should not create a database, when already exists"
./manager.sh monet create &> /dev/null
if [[ $? -ne 0 ]]; then
echo -e "\tSucess"
else
error=$(($error +1))
echo -e "\tFail"
fi
echo "Should load the database"
./manager.sh monet load &> /dev/null
if [[ $? -eq 0 ]]; then
echo -e "\tSucess"
else
error=$(($error +1))
echo -e "\tFail"
fi
echo "Should insert a tuple, with the right sequence"
DOTMONETDBFILE=$DOTMONETDBFILE mclient \
-s "INSERT INTO worker(login, password) VALUES('worker4', 'pass4')"\
&> /dev/null
value=$(DOTMONETDBFILE=$DOTMONETDBFILE mclient -f csv \
-s "SELECT MAX(id) FROM worker")
if [[ $value -eq 4 ]]; then
echo -e "\tSucess"
else
error=$(($error +1))
echo -e "\tFail"
fi
echo "Should insert a tuple, with the right sequence, on compactedfile"
DOTMONETDBFILE=$DOTMONETDBFILE mclient \
-s "INSERT INTO workercompact(login, password) VALUES('worker4', 'pass4')"\
&> /dev/null
value=$(DOTMONETDBFILE=$DOTMONETDBFILE mclient -f csv \
-s "SELECT MAX(id) FROM workercompact")
if [[ $value -eq 4 ]]; then
echo -e "\tSucess"
else
error=$(($error +1))
echo -e "\tFail"
fi
echo "Should create dummie data in the database"
./manager.sh monet fixture &> /dev/null
if [[ $? -eq 0 ]]; then
echo -e "\tSucess"
else
error=$(($error +1))
echo -e "\tFail"
fi
echo "Should clean the database"
./manager.sh monet clean &> /dev/null
if [[ $? -eq 0 ]]; then
echo -e "\tSucess"
else
error=$(($error +1))
echo -e "\tFail"
fi
echo "Should drop the database"
./manager.sh monet drop &> /dev/null
if [[ $? -eq 0 ]]; then
echo -e "\tSucess"
else
error=$(($error +1))
echo -e "\tFail"
fi
# For some reason does not work on gitlab-ci
# echo "Should not create database with wrong credentials"
# PGPASSWORD="wrongPass" ./manager.sh create &> /dev/null
# if [[ $? -ne 0 ]]; then
# echo -e "\tSucess"
# else
# error=$(($error +1))
# echo -e "\tFail"
# fi
# echo "Should not load database with wrong credentials"
# PGPASSWORD="wrongPass" ./manager.sh load &> /dev/null
# if [[ $? -ne 0 ]]; then
# echo -e "\tSucess"
# else
# error=$(($error +1))
# echo -e "\tFail"
# fi
# echo "Should not drop database with wrong credentials"
# PGPASSWORD="wrongPass" ./manager.sh drop &> /dev/null
# if [[ $? -ne 0 ]]; then
# echo -e "\tSucess"
# else
# error=$(($error +1))
# echo -e "\tFail"
# fi
rm $DOTMONETDBFILE
echo "$error error(s) occured."
if [[ $error -gt 0 ]]; then
exit 1
else
exit 0
fi
......@@ -5,10 +5,10 @@ basePath=$(echo $BASH_SOURCE | rev | cut -c 8- | rev)
cd $basePath/..
#Before all tests
./manager.sh drop &> /dev/null
./manager.sh psql drop &> /dev/null
echo "Should create a database"
./manager.sh create &> /dev/null
./manager.sh psql create &> /dev/null
if [[ $? -eq 0 ]]; then
echo -e "\tSucess"
......@@ -18,7 +18,7 @@ else
fi
echo "Should not create a database, when already exists"
./manager.sh create &> /dev/null
./manager.sh psql create &> /dev/null
if [[ $? -ne 0 ]]; then
echo -e "\tSucess"
......@@ -28,7 +28,7 @@ else
fi
echo "Should load the database"
./manager.sh load &> /dev/null
./manager.sh psql load &> /dev/null
if [[ $? -eq 0 ]]; then
echo -e "\tSucess"
......@@ -38,15 +38,16 @@ else
fi
echo "Should insert a tuple, with the right sequence"
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
PGPASSWORD=${DB_PASSWORD} psql -h ${DB_HOST}\
-d ${DB_NAME}\
-U ${DB_USER}\
-c "INSERT INTO worker(login, password) VALUES('worker4', 'pass4')"\
&> /dev/null
value=$(psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
value=$(PGPASSWORD=${DB_PASSWORD} psql -h ${DB_HOST}\
-d ${DB_NAME}\
-U ${DB_USER}\
-c "SELECT MAX(id) FROM worker" |\
head -n3 | tail -n1)
......@@ -58,17 +59,17 @@ else
fi
echo "Should insert a tuple, with the right sequence, on compactedfile"
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
PGPASSWORD=${DB_PASSWORD} psql -h ${DB_HOST}\
-d ${DB_NAME}\
-U ${DB_USER}\
-c "INSERT INTO workercompact(login, password) VALUES('worker4', 'pass4')"\
> /dev/null
&> /dev/null
value=$(psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
value=$(PGPASSWORD=${DB_PASSWORD} psql -h ${DB_HOST}\
-d ${DB_NAME}\
-U ${DB_USER}\
-c "SELECT MAX(id) FROM workercompact" |\
head -n3 | tail -n1)
head -n3 | tail -n1)
if [[ $value -eq 4 ]]; then
echo -e "\tSucess"
......@@ -78,7 +79,7 @@ else
fi
echo "Should create dummie data in the database"
./manager.sh fixture &> /dev/null
./manager.sh psql fixture &> /dev/null
if [[ $? -eq 0 ]]; then
echo -e "\tSucess"
......@@ -88,7 +89,7 @@ else
fi
echo "Should clean the database"
./manager.sh clean &> /dev/null
./manager.sh psql clean &> /dev/null
if [[ $? -eq 0 ]]; then
echo -e "\tSucess"
......@@ -98,7 +99,7 @@ else
fi
echo "Should drop the database"
./manager.sh drop &> /dev/null
./manager.sh psql drop &> /dev/null
if [[ $? -eq 0 ]]; then
echo -e "\tSucess"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment