Commit f79c571b authored by Lucas Fernandes de Oliveira's avatar Lucas Fernandes de Oliveira

Merge branch 'issue/1' into 'master'

Issue #1: Add basic manager

See merge request c3sl/psql-manager!1
parents 696b7e75 67e4b974
Pipeline #19324 passed with stage
in 50 seconds
*.in
*.out
*.swp
*.swo
*.tmp
image: debian:stretch
services:
- "postgres:10"
variables:
POSTGRES_DB: 'blendb_fixture'
POSTGRES_USER: 'runner'
POSTGRES_HOST: 'postgres'
PGPASSWORD: 'pass'
stages:
- test
cache:
paths:
- /var/cache/apt
run_test:
stage: test
script:
- apt-get update -q -y
- apt-get install wget gnupg -y
- echo "deb http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main" > /etc/apt/sources.list.d/pgdg.list
- wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
- apt-get update -q -y
- apt-get install -y postgresql-client-10
- ./test/test.sh
tags:
- debian
- postgres
CREATE TABLE worker(
id INTEGER
, login TEXT
, password TEXT
);
CREATE TABLE task(
workerId INTEGER
, description TEXT
);
1;worker1 task1
1;worker2 task1
2;worker2 task2
3;worker3 task1
3;worker3 task2
3;worker3 task3
1;worker1;pass1
2;worker2;pass2
3;worker3;pass3
#! /bin/bash
# Script used as interface to manage a PostgreSQL database
if [[ $# -ne 1 && $# -ne 2 ]]; then
echo "Correct use: $0 <task> [directory]"
echo "The allowed tasks are: create"
echo "The directory structure should respect the example struct used in:"
echo "./data directory"
exit 1
fi
# Rename arguments
task=$1
workspace=${2:-"./data"}
# Make independent of path
basePath=$(echo $BASH_SOURCE | rev | cut -c 11- | rev)
cd $basePath
# Dictonary with allowed tasks, mapped with executable funcion
declare -A TASKS=(
[create]=createDB
[load]=loadDB
[drop]=dropDB
)
# Source script files
SCRIPTS=$(ls "./src")
if [[ -n "$SCRIPTS" ]]; then
for script in $SCRIPTS; do
source "./src/$script"
done
fi
# Check env vars (utility)
correctEnv || exit 1
if [ ${TASKS[$task]+_} ]; then
${TASKS[$task]} $workspace
if [[ $? -ne 0 ]]; then
echo "Some error occured on task <$task>"
echo "Check your connection parameters and schema files"
echo "Aborting..."
exit 1
else
echo "Task <$task> performed with success"
exit 0
fi
else
echo "Task <$task> not found."
exit 1
fi
#! /bin/bash
# 'Create' a database. This function assumes that the database is
# acessible. In other words, it does not create the user and the 'database'
# itself, only creates the database schema (tables, views, etc.).
createDB() {
# Reads workspace
ws=${1:-"./data"}
workspace="$ws/create"
# Get schema files, and sort it
files=$(ls $workspace | sort)
schemaFiles=""
for f in $files; do
schemaFiles="$schemaFiles $workspace/$f"
done
# Create temporary workspace (utility function)
tmpWS=$(createTmpWorkspace)
# Create transactions files
echo "BEGIN;" > "$tmpWS/begin.sql"
echo "COMMIT;" > "$tmpWS/commit.sql"
mkfifo $tmpWS/trans.fifo
# Concats the transaction and schema files
cat "$tmpWS/begin.sql"\
$schemaFiles\
"$tmpWS/commit.sql"\
> "$tmpWS/trans.fifo" &
# Performs schema creation. As is in a transaction if fails it will
# NOT leave the database inconsistent.
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-f $tmpWS/trans.fifo 2> $tmpWS/error.out
# If a error occurs will be sent to error.out
# If error.out is not empty, should return a error
egrep . $tmpWS/error.out
if [[ $? -eq 0 ]]; then
error=1
else
error=0
fi
# Utility function that removes the temporary workspace
rm -rf $tmpWS
return $error
}
#! /bin/bash
# Drop a database. Removes all tables and data
dropDB() {
tables=$(
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-c "\d" |\
tail -n+4 |\
head -n-2 |\
cut -d '|' -f2\
)
# Create temporary workspace (utility function)
tmpWS=$(createTmpWorkspace)
# Create transactions files
echo "BEGIN;" > "$tmpWS/trans.sql"
i=0
for t in $tables; do
dropComand $t >> "$tmpWS/trans.sql"
done
echo "COMMIT;" >> "$tmpWS/trans.sql"
# Performs schema creation. As is in a transaction if fails it will
# NOT leave the database inconsistent.
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-f $tmpWS/trans.sql 2> $tmpWS/error.out
# If a error occurs will be sent to error.out
# If error.out is not empty, should return a error
egrep . $tmpWS/error.out
if [[ $? -eq 0 ]]; then
error=1
else
error=0
fi
# Utility function that removes the temporary workspace
rm -fr $tmpWs
return $error
}
# Creates the DROP TABLE command for each table
dropComand() {
table=${1:-""}
echo "DROP TABLE $table;"
}
#! /bin/bash
# Load csv files in the database. The name of each file MUST be <table name>.csv
loadDB() {
# Reads workspace
ws=${1:-"./data"}
workspace="$ws/load"
# Get load files, and sort it
tables=$(ls $workspace | sort | cut -d'.' -f1)
# Create temporary workspace (utility function)
tmpWS=$(createTmpWorkspace)
# Create transactions files
echo "BEGIN;" > "$tmpWS/trans.sql"
i=0
for t in $tables; do
cpyComand $t $workspace/$t.csv >> "$tmpWS/trans.sql"
done
echo "COMMIT;" >> "$tmpWS/trans.sql"
# Performs schema creation. As is in a transaction if fails it will
# NOT leave the database inconsistent.
psql -h ${POSTGRES_HOST}\
-d ${POSTGRES_DB}\
-U ${POSTGRES_USER}\
-f $tmpWS/trans.sql 2> $tmpWS/error.out
# If a error occurs will be sent to error.out
# If error.out is not empty, should return a error
egrep . $tmpWS/error.out
if [[ $? -eq 0 ]]; then
error=1
else
error=0
fi
# Utility function that removes the temporary workspace
rm -fr $tmpWs
return $error
}
# Creates the COPY command used to insert data
cpyComand() {
table=${1:-""}
file=${2:-""}
echo "\\copy $table FROM '$file' WITH CSV HEADER ENCODING 'UTF8' DELIMITER ';'"
}
#! /bin/bash
# Creates a temporary workspace directory
createTmpWorkspace() {
tmpWS='/tmp/psql-manager'
mkdir -p $tmpWS
echo $tmpWS
}
# Check if the environment vars are set
correctEnv() {
vars="POSTGRES_HOST POSTGRES_DB POSTGRES_USER PGPASSWORD"
error=0
for var in $vars; do
if [[ -z ${!var} ]]; then
echo "Env var $var is unset. Set it to use the script"
error=1
fi
done
return $error
}
#! /bin/bash
error=0
basePath=$(echo $BASH_SOURCE | rev | cut -c 8- | rev)
cd $basePath/..
#Before all tests
./manager.sh drop &> /dev/null
echo "Should create a database"
./manager.sh create &> /dev/null
if [[ $? -eq 0 ]]; then
echo -e "\tSucess"
else
error=$(($error +1))
echo -e "\tFail"
fi
echo "Should not create a database, when already exists"
./manager.sh create &> /dev/null
if [[ $? -ne 0 ]]; then
echo -e "\tSucess"
else
error=$(($error +1))
echo -e "\tFail"
fi
echo "Should load the database"
./manager.sh load &> /dev/null
if [[ $? -eq 0 ]]; then
echo -e "\tSucess"
else
error=$(($error +1))
echo -e "\tFail"
fi
echo "Should drop the database"
./manager.sh load &> /dev/null
if [[ $? -eq 0 ]]; then
echo -e "\tSucess"
else
error=$(($error +1))
echo -e "\tFail"
fi
# For some reason does not work on gitlab-ci
# echo "Should not create database with wrong credentials"
# PGPASSWORD="wrongPass" ./manager.sh create &> /dev/null
# if [[ $? -ne 0 ]]; then
# echo -e "\tSucess"
# else
# error=$(($error +1))
# echo -e "\tFail"
# fi
# echo "Should not load database with wrong credentials"
# PGPASSWORD="wrongPass" ./manager.sh load &> /dev/null
# if [[ $? -ne 0 ]]; then
# echo -e "\tSucess"
# else
# error=$(($error +1))
# echo -e "\tFail"
# fi
# echo "Should not drop database with wrong credentials"
# PGPASSWORD="wrongPass" ./manager.sh drop &> /dev/null
# if [[ $? -ne 0 ]]; then
# echo -e "\tSucess"
# else
# error=$(($error +1))
# echo -e "\tFail"
# fi
echo "$error error(s) occured."
if [[ $error -gt 0 ]]; then
exit 1
else
exit 0
fi
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment