Commit 9b0e0c11 authored by Cristian Weiland's avatar Cristian Weiland

Issue #80: Propaga parametros para outros scripts (nao usar ES, manter CSV, etc.)

Signed-off-by: Cristian Weiland's avatarCristian Weiland <cw14@inf.ufpr.br>
parent f9029e9b
......@@ -37,8 +37,8 @@ Para adicionar um caminho ao PATH você deve digitar no seu terminal:
`PATH=$PATH:~/caminho/ate/o/logstash`
Por exemplo, caso o logstash tenha sido instalado a partir de seu pacote debian (.deb):
`PATH=$PATH:/usr/share/logstash/bin`
Por exemplo:
`PATH=$PATH:/home/user/logstash/bin`
**3. Clone este repositorio localmente**
......
......@@ -6,23 +6,31 @@
# Input: year, month and day, ElasticSearch's username and password.
import sys, csv, json, math, subprocess
import sys, csv, json, math, subprocess, argparse
from pathlib import Path
from subprocess import call
if len(sys.argv) != 10:
print("Usage: " + sys.argv[0] + " <year (2016)> <month (01)> <day (31)> <index> <host> <entity> <username> <password> <path>")
sys.exit()
parser = argparse.ArgumentParser()
parser.add_argument("ano", help="Ano dos dados a serem inseridos")
parser.add_argument("mes", help="Mês dos dados a serem inseridos")
parser.add_argument("dia", help="Dia dos dados a serem inseridos")
parser.add_argument("index", help="Indice do ElasticSearch no qual serão inseridos os dados (exemplo: 'despesas-pagamentos'")
parser.add_argument("entity", help="Nome da entidade que vai ser inserida (exemplo: 'Ministério da Educação')")
parser.add_argument("caminho", help="Caminho relativo para os arquivos.")
parser.add_argument("-H", "--host", help="Hostname do ElasticSearch (default: localhost)", dest = 'host', default = 'localhost')
parser.add_argument("-U", "--usuario", help="Usuario do ElasticSearch (default: em branco)", dest = 'usuario', default = '')
parser.add_argument("-S", "--senha", help="Senha do ElasticSearch (default: sem senha)", dest = 'senha', default = '')
args = parser.parse_args()
year = sys.argv[1]
month = sys.argv[2]
day = sys.argv[3]
index = sys.argv[4]
host = sys.argv[5]
entity = sys.argv[6]
username = sys.argv[7]
passwd = sys.argv[8]
path = sys.argv[9]
year = args.ano
month = args.mes
day = args.dia
index = args.index
entity = args.entity
path = args.caminho
host = args.host
username = args.usuario
passwd = args.senha
# Generate JSON CNPJ merge config file:
data = {
......@@ -88,7 +96,7 @@ with open(path + '/config-natjur-' + year + '-' + month + '.json', 'w') as outfi
json.dump(data, outfile, indent=4, sort_keys=True)
# Generate logstash config file:
with open('logstash_config.example') as infile:
with open('logstash_config.example', encoding='utf-8') as infile:
example = infile.read()
output = example % { "timestamp": day + '/' + month + '/' + year + ' 00:00:00'
......@@ -99,5 +107,5 @@ output = example % { "timestamp": day + '/' + month + '/' + year + ' 00:00:00'
, "password": passwd }
date = year + '-' + month
with open(path + '/config-' + date, 'w') as outfile:
with open(path + '/config-' + date, 'w', encoding='utf-8') as outfile:
outfile.write(output)
This diff is collapsed.
......@@ -117,7 +117,7 @@ for row2 in csv_2:
result.append(newRow)
errors += 1
with open(params['outputFile'], 'w', newline='') as csvfile:
with open(params['outputFile'], 'w', newline='', encoding='utf-8') as csvfile:
writer = csv.writer(csvfile, delimiter='\t')
writer.writerow(getDataFromRows(title1, title2))
writer.writerows(result)
......@@ -6,23 +6,31 @@
# Input: year, month and day, ElasticSearch's username and password.
import sys, csv, json, math, subprocess
import sys, csv, json, math, subprocess, argparse
from pathlib import Path
from subprocess import call
if len(sys.argv) != 10:
print("Usage: " + sys.argv[0] + " <year (2016)> <month (01)> <day (31)> <index> <host> <entity> <username> <password> <path>")
sys.exit()
year = sys.argv[1]
month = sys.argv[2]
day = sys.argv[3]
index = sys.argv[4]
host = sys.argv[5]
entity = sys.argv[6]
username = sys.argv[7]
passwd = sys.argv[8]
path = sys.argv[9]
parser = argparse.ArgumentParser()
parser.add_argument("ano", help="Ano dos dados a serem inseridos")
parser.add_argument("mes", help="Mês dos dados a serem inseridos")
parser.add_argument("dia", help="Dia dos dados a serem inseridos")
parser.add_argument("index", help="Indice do ElasticSearch no qual serão inseridos os dados (exemplo: 'despesas-pagamentos'")
parser.add_argument("entity", help="Nome da entidade que vai ser inserida (exemplo: 'Ministério da Educação')")
parser.add_argument("caminho", help="Caminho relativo para os arquivos.")
parser.add_argument("-H", "--host", help="Hostname do ElasticSearch (default: localhost)", dest = 'host', default = 'localhost')
parser.add_argument("-U", "--usuario", help="Usuario do ElasticSearch (default: em branco)", dest = 'usuario', default = '')
parser.add_argument("-S", "--senha", help="Senha do ElasticSearch (default: sem senha)", dest = 'senha', default = '')
args = parser.parse_args()
year = args.ano
month = args.mes
day = args.dia
index = args.index
entity = args.entity
path = args.caminho
host = args.host
username = args.usuario
passwd = args.senha
with open('logstash_config.example') as infile:
example = infile.read()
......
......@@ -18,34 +18,128 @@ function inputError(){
return 0
}
if [ "$#" -ne 4 ]; then
echo "Usage: $0 <year> <month> <user> <password>"
echo "Example: $0 2016 12 myuser mypass"
exit
function usage() {
echo "Uso: $0 <--ano YYYY--> <--mes MM---> [OPÇÃO]"
echo ""
echo "Conexão com o ElasticSearch"
echo -e " -E, --sem-elastic\tNão utiliza o ElasticStack para armazenar os dados, implica em --csv"
echo -e " -u, --usuario\t\tUsuário do ElasticSearch (default: em branco)"
echo -e " -s, --senha\t\tSenha do ElasticSearch (default: sem senha)"
echo -e " -H, --hostname\tHostname do ElasticSearch (default: localhost)"
echo ""
echo "Definição de período de tempo para inserção"
echo -e " -a, --ano\t\tAno dos dados, no formato YYYY"
echo -e " -m, --mes\t\tMês dos dados, no formato MM"
echo ""
echo "Outras opções"
echo -e " --mantem-csv\tNão remove os arquivos CSV utilizados para inserção no banco"
echo ""
}
function echoerr() {
echo "$@" 1>&2;
}
function echoerr_and_exit() {
echoerr "$@"
usage
exit 1
}
function empty_arg_sanity_check() {
local arg_to_check=$1
if [[ "$arg_to_check" = -* || "$arg_to_check" = "" ]]; then
return 1
else
echo $arg_to_check
return 0
fi
}
if [[ $# -eq 0 ]]; then
usage
exit 1
fi
year=$1
month=$2
user=$3
passwd=$4
if [[ $? -ne 0 ]]; then
echo "Error while reading command line arguments, exiting."
exit 1
fi
KEEP_CSV=false
ELASTICSTACK=true
ELASTIC_USERNAME=
ELASTIC_PASSWORD=
ELASTIC_HOSTNAME="localhost"
while true; do
case "$1" in
-E | --sem-elastic ) ELASTICSTACK=false; shift ;;
--mantem-csv ) KEEP_CSV=true ; shift ;;
-s | --senha )
ELASTIC_PASSWORD=$(empty_arg_sanity_check "$2")
if test -z "$ELASTIC_PASSWORD"; then
echoerr_and_exit "Você não forneceu o valor para '$1'"
fi
ELASTIC_PASSWORD_PARAM="-S $ELASTIC_PASSWORD"
shift 2 ;;
-H | --hostname )
ELASTIC_HOSTNAME=$(empty_arg_sanity_check "$2")
if test -z "$ELASTIC_HOSTNAME"; then
echoerr_and_exit "Você não forneceu o valor para '$1'"
fi
ELASTIC_HOSTNAME_PARAM="-H $ELASTIC_HOSTNAME"
shift 2 ;;
-u | --usuario )
ELASTIC_USERNAME=$(empty_arg_sanity_check "$2")
if test -z "$ELASTIC_USERNAME"; then
echoerr_and_exit "Você não forneceu o valor para '$1'"
fi
ELASTIC_USERNAME_PARAM="-U $ELASTIC_USERNAME"
shift 2 ;;
-a | --ano )
YEAR=$(empty_arg_sanity_check "$2")
if test -z "$YEAR"; then
echoerr_and_exit "Data inválida em '$1': '$2"
fi
shift 2 ;;
-m | --mes )
MONTH=$(empty_arg_sanity_check "$2")
if test -z "$MONTH"; then
echoerr_and_exit "Data inválida em '$1': '$2"
fi
shift 2 ;;
-h | --help )
usage
exit 0 ;;
-- )
shift
break ;;
* )
break
;;
esac
done
# Treating the cases where the year input is invalid
if [[ "${#year}" != "4" ]]; then
if [[ "${#YEAR}" != "4" ]]; then
echo " Year is invalid. Please enter a 4 digit year."
echo " Example: $0 2016 12 myuser mypass all"
exit
fi
# Treating the case where month is too big
if [[ "${#month}" -gt "2" ]]; then
if [[ "${#MONTH}" -gt "2" ]]; then
echo " Month is invalid. Please enter a 2 digit month."
echo " Example: $0 2016 12 myuser mypass all"
exit
fi
# Treating the case where the user uses only 1 digit to the month value
if [[ "${#month}" -eq "1" ]]; then
month="0$month"
if [[ "${#MONTH}" -eq "1" ]]; then
MONTH="0$MONTH"
fi
if [ -e config.sh ]; then
......@@ -56,7 +150,7 @@ else
fi
# Check if all variables in config file are set:
configFile='scripts/travel_allowance/config.sh'
configFile='scripts/expenses/config.sh'
if [ -z "${index}" ]; then
inputError "index" $configFile
exit;
......@@ -78,13 +172,13 @@ fi
# Getting the Last day of this month (Using date 2016-05-15 as example):
# First, get next month (201606).
nxtMonth=$(date +%Y%m -d "$(date +${year}${month}15) next month")
nxtMonth=$(date +%Y%m -d "$(date +${YEAR}${MONTH}15) next month")
# Append day 01 (20160601).
tempDate=$(date -d "${nxtMonth}01")
# Remove 1 day: 20160531, get only day: 31.
day=$(date -d "$tempDate - 1 day" "+%d")
ym=$year-$month
ym=$YEAR-$MONTH
path="./tmp_$ym"
# Step 1:
......@@ -92,28 +186,36 @@ path="./tmp_$ym"
mkdir -p "$path"
# Download files
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${year}'&m='${month}'&consulta=Diarias'
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${YEAR}'&m='${MONTH}'&consulta=Diarias'
curl $request --compressed > $path/${year}${month}_Diarias.zip
curl $request --compressed > $path/${YEAR}${MONTH}_Diarias.zip
# Unzip them
unzip -o $path/${year}${month}_Diarias.zip -d $path/
unzip -o $path/${YEAR}${MONTH}_Diarias.zip -d $path/
# Remove zip file
rm $path/${year}${month}_Diarias.zip
rm $path/${YEAR}${MONTH}_Diarias.zip
for key in "${!filter[@]}"
do
# Step 2:
./create_travel_allowance_config.py $year $month "$day" "$index" "$host" "$key" $user $passwd "${path}"
# Step 3:
./resume_travel_allowance.sh "$path" ${year}-${month} "${filter[$key]}" "${columnName}"
# Step 4:
logstash -f ${path}/config-${year}-${month} < ${path}/${year}${month}.csv
# Step 2: Create config file
./create_travel_allowance_config.py $YEAR $MONTH "$day" "$index" "$key" "${path}" $ELASTIC_HOSTNAME_PARAM $ELASTIC_USERNAME_PARAM $ELASTIC_PASSWORD_PARAM
# Step 3: Filter data
./resume_travel_allowance.sh "$path" ${YEAR}-${MONTH} "${filter[$key]}" "${columnName}"
# Step 4: Insert data in ElasticSearch
if [ "$ELASTICSTACK" = true ]; then
logstash -f ${path}/config-${YEAR}-${MONTH} < ${path}/${YEAR}${MONTH}.csv
fi
# Remove processed file
rm ${path}/${year}${month}.csv
rm ${path}/config-${year}-${month}
if [ "$KEEP_CSV" = false ]; then
rm ${path}/${YEAR}${MONTH}.csv
else
mkdir -p csv/
mv ${path}/${YEAR}${MONTH}.csv csv/travel_allowances_${YEAR}-${MONTH}.csv
fi
rm ${path}/config-${YEAR}-${MONTH}
done
rm $path/${year}${month}_Diarias.csv
rm $path/${YEAR}${MONTH}_Diarias.csv
rmdir $path
......@@ -8,23 +8,31 @@
# - JSON: This config will be used for script merge_files.py, and will be stored in transparencia/configs/workers/JSON, with its name being config-year-month.
# - Logstash: This config will be used by logstash to insert the resulting CSV from merge_files.py into ElasticSearch.
import sys, csv, json, math, subprocess
import sys, csv, json, math, subprocess, argparse
from pathlib import Path
from subprocess import call
if len(sys.argv) != 10:
print("Usage: " + sys.argv[0] + " <year (2016)> <month (01)> <day (31)> <index> <host> <entity> <username> <password> <path>")
sys.exit()
year = sys.argv[1]
month = sys.argv[2]
day = sys.argv[3]
index = sys.argv[4]
host = sys.argv[5]
entity = sys.argv[6]
username = sys.argv[7]
passwd = sys.argv[8]
path = sys.argv[9]
parser = argparse.ArgumentParser()
parser.add_argument("ano", help="Ano dos dados a serem inseridos")
parser.add_argument("mes", help="Mês dos dados a serem inseridos")
parser.add_argument("dia", help="Dia dos dados a serem inseridos")
parser.add_argument("index", help="Indice do ElasticSearch no qual serão inseridos os dados (exemplo: 'despesas-pagamentos'")
parser.add_argument("entity", help="Nome da entidade que vai ser inserida (exemplo: 'Ministério da Educação')")
parser.add_argument("caminho", help="Caminho relativo para os arquivos.")
parser.add_argument("-H", "--host", help="Hostname do ElasticSearch (default: localhost)", dest = 'host', default = 'localhost')
parser.add_argument("-U", "--usuario", help="Usuario do ElasticSearch (default: em branco)", dest = 'usuario', default = '')
parser.add_argument("-S", "--senha", help="Senha do ElasticSearch (default: sem senha)", dest = 'senha', default = '')
args = parser.parse_args()
year = args.ano
month = args.mes
day = args.dia
index = args.index
entity = args.entity
path = args.caminho
host = args.host
username = args.usuario
passwd = args.senha
data = {
"path": path
......
......@@ -22,34 +22,128 @@ function inputError(){
return 0
}
if [ "$#" -ne 4 ]; then
echo "Usage: $0 <year> <month> <user> <password>"
echo "Example: $0 2016 12 myuser mypassword"
exit
function usage() {
echo "Uso: $0 <--ano YYYY--> <--mes MM---> [OPÇÃO]"
echo ""
echo "Conexão com o ElasticSearch"
echo -e " -E, --sem-elastic\tNão utiliza o ElasticStack para armazenar os dados, implica em --csv"
echo -e " -u, --usuario\t\tUsuário do ElasticSearch (default: em branco)"
echo -e " -s, --senha\t\tSenha do ElasticSearch (default: sem senha)"
echo -e " -H, --hostname\tHostname do ElasticSearch (default: localhost)"
echo ""
echo "Definição de período de tempo para inserção"
echo -e " -a, --ano\t\tAno dos dados, no formato YYYY"
echo -e " -m, --mes\t\tMês dos dados, no formato MM"
echo ""
echo "Outras opções"
echo -e " --mantem-csv\tNão remove os arquivos CSV utilizados para inserção no banco"
echo ""
}
function echoerr() {
echo "$@" 1>&2;
}
function echoerr_and_exit() {
echoerr "$@"
usage
exit 1
}
function empty_arg_sanity_check() {
local arg_to_check=$1
if [[ "$arg_to_check" = -* || "$arg_to_check" = "" ]]; then
return 1
else
echo $arg_to_check
return 0
fi
}
if [[ $# -eq 0 ]]; then
usage
exit 1
fi
year=$1
month=$2
user=$3
passwd=$4
if [[ $? -ne 0 ]]; then
echo "Error while reading command line arguments, exiting."
exit 1
fi
KEEP_CSV=false
ELASTICSTACK=true
ELASTIC_USERNAME=
ELASTIC_PASSWORD=
ELASTIC_HOSTNAME="localhost"
while true; do
case "$1" in
-E | --sem-elastic ) ELASTICSTACK=false; shift ;;
--mantem-csv ) KEEP_CSV=true ; shift ;;
-s | --senha )
ELASTIC_PASSWORD=$(empty_arg_sanity_check "$2")
if test -z "$ELASTIC_PASSWORD"; then
echoerr_and_exit "Você não forneceu o valor para '$1'"
fi
ELASTIC_PASSWORD_PARAM="-S $ELASTIC_PASSWORD"
shift 2 ;;
-H | --hostname )
ELASTIC_HOSTNAME=$(empty_arg_sanity_check "$2")
if test -z "$ELASTIC_HOSTNAME"; then
echoerr_and_exit "Você não forneceu o valor para '$1'"
fi
ELASTIC_HOSTNAME_PARAM="-H $ELASTIC_HOSTNAME"
shift 2 ;;
-u | --usuario )
ELASTIC_USERNAME=$(empty_arg_sanity_check "$2")
if test -z "$ELASTIC_USERNAME"; then
echoerr_and_exit "Você não forneceu o valor para '$1'"
fi
ELASTIC_USERNAME_PARAM="-U $ELASTIC_USERNAME"
shift 2 ;;
-a | --ano )
YEAR=$(empty_arg_sanity_check "$2")
if test -z "$YEAR"; then
echoerr_and_exit "Data inválida em '$1': '$2"
fi
shift 2 ;;
-m | --mes )
MONTH=$(empty_arg_sanity_check "$2")
if test -z "$MONTH"; then
echoerr_and_exit "Data inválida em '$1': '$2"
fi
shift 2 ;;
-h | --help )
usage
exit 0 ;;
-- )
shift
break ;;
* )
break
;;
esac
done
# Treating the cases where the year input is invalid
if [[ "${#year}" != "4" ]]; then
if [[ "${#YEAR}" != "4" ]]; then
echo " Year is invalid. Please enter a 4 digit year."
echo " Example: $0 2016 12 myuser mypass all"
exit
fi
# Treating the case where month is too big
if [[ "${#month}" -gt "2" ]]; then
if [[ "${#MONTH}" -gt "2" ]]; then
echo " Month is invalid. Please enter a 2 digit month."
echo " Example: $0 2016 12 myuser mypass all"
exit
fi
# Treating the case where the user uses only 1 digit to the month value
if [[ "${#month}" -eq "1" ]]; then
month="0$month"
if [[ "${#MONTH}" -eq "1" ]]; then
month="0$MONTH"
fi
if [ -e config.sh ]; then
......@@ -59,7 +153,8 @@ else
exit
fi
configFile='scripts/workers/config.sh'
# Check if all variables in config file are set:
configFile='scripts/expenses/config.sh'
if [ -z "${index}" ]; then
inputError "index" $configFile
exit;
......@@ -79,7 +174,7 @@ if [ "$size" -lt 1 ]; then
exit;
fi
ym=$year-$month
ym=$YEAR-$MONTH
path="./tmp_$ym"
# Step 1:
......@@ -87,45 +182,51 @@ path="./tmp_$ym"
mkdir -p "$path"
# Download files
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${year}'&m='${month}'&d=C&consulta=Servidores'
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${YEAR}'&m='${MONTH}'&d=C&consulta=Servidores'
curl $request --compressed > $path/${year}${month}_Servidores.zip
curl $request --compressed > $path/${YEAR}${MONTH}_Servidores.zip
# Unzip them
unzip -o $path/${year}${month}_Servidores.zip -d $path/
unzip -o $path/${YEAR}${MONTH}_Servidores.zip -d $path/
# Remove zip file
rm $path/${year}${month}_Servidores.zip
rm $path/${YEAR}${MONTH}_Servidores.zip
# Get day
day=$(ls $path | grep -m 1 $year$month | cut -c 7,8)
DAY=$(ls $path | grep -m 1 $YEAR$MONTH | cut -c 7,8)
for key in "${!filter[@]}"
do
# Step 2:
# Create config files
./create_config.py $year $month "$day" "$index" "$host" "$key" $user $passwd "${path}"
# Step 2: Create config files
./create_workers_config.py $YEAR $MONTH "$DAY" "$index" "$key" "${path}" $ELASTIC_HOSTNAME_PARAM $ELASTIC_USERNAME_PARAM $ELASTIC_PASSWORD_PARAM
# Step 3:
# Start processing
# Step 3: Start processing
strReplacement=$( echo "${filter[$key]}" | sed 's/ /\\ /g' )
./merge_files.py $path/config-${year}-${month}.json "$strReplacement" "${columnName}"
rm $path/${year}${month}${day}_Cadastro_Unique.csv
./merge_files.py $path/config-${YEAR}-${MONTH}.json "$strReplacement" "${columnName}"
rm $path/${YEAR}${MONTH}${DAY}_Cadastro_Unique.csv
# Step 4:
# Insert data in ElasticSearch
logstash -f $path/config-${year}-${month} < $path/${year}${month}${day}.csv
# Step 4: Insert data in ElasticSearch
if [ "$ELASTICSTACK" = true ]; then
logstash -f $path/config-${YEAR}-${MONTH} < $path/${YEAR}${MONTH}${DAY}.csv
fi
# Remove data
rm -f $path/config-${year}-${month}
rm -f $path/config-${year}-${month}.json
rm -f $path/${year}${month}${day}.csv
rm -f $path/config-${YEAR}-${MONTH}
rm -f $path/config-${YEAR}-${MONTH}.json
if [ "$KEEP_CSV" = false ]; then
rm -f $path/${YEAR}${MONTH}${DAY}.csv
else
mkdir -p csv/
mv ${path}/${YEAR}${MONTH}${DAY}.csv csv/workers_${YEAR}-${MONTH}-${DAY}.csv
fi
done
rm -f $path/${year}${month}${day}_Afastamentos.csv
rm -f $path/${year}${month}${day}_Cadastro.csv
rm -f $path/${year}${month}${day}_Honorarios\(Jetons\).csv
rm -f $path/${year}${month}${day}_Jetom.csv
rm -f $path/${year}${month}${day}_Observacoes.csv
rm -f $path/${year}${month}${day}_Remuneracao.csv
rmdir $path
rm -f $path/${YEAR}${MONTH}${DAY}_Afastamentos.csv
rm -f $path/${YEAR}${MONTH}${DAY}_Cadastro.csv
rm -f $path/${YEAR}${MONTH}${DAY}_Honorarios\(Jetons\).csv
rm -f $path/${YEAR}${MONTH}${DAY}_Jetom.csv
rm -f $path/${YEAR}${MONTH}${DAY}_Observacoes.csv
rm -f $path/${YEAR}${MONTH}${DAY}_Remuneracao.csv
if [ "$KEEP_CSV" = false ]; then
rmdir $path
fi
......@@ -70,7 +70,6 @@ title1 = csv_1.pop(0)
file_exists = Path(file2)
if not file_exists.is_file():
print("File2 does not exist. Calling script resume_register to create it...")
call(["./resume_register.sh " + params['path'] + " " + params['date'] + " " + filterReport + " " + columnId], shell=True)
with open(file2, newline='', encoding='Windows-1252') as f:
......
#!/bin/bash
# This script will call every script needed to insert data.
# Parameters are the year and month from the data that will be inserted, and an elasticsearch username and password.
# Also, scripts called by insert_data.sh use a config file, located in every subfolder and called 'config.sh'.
# Those config files have some variables that have to be set:
# - Index: The index prefix to be saved on ElasticSearch.
# - Host: The hostname of the machine runnning ElasticSearch.
# - Filter: An array of n values, that will create n indexes in ElasticSearch, each one filtering data from Portal Transparencia using its corresponding string. Ex: "UNIVERSIDADE FEDERAL DO PARANA"
function usage() {
echo "Uso: $0 [OPÇÃO] <--inicio YYYY-MM> <--fim YYYY-MM>"
echo ""
echo "Conexão com o ElasticSearch"
echo -e " -E, --sem-elastic\tNão utiliza o ElasticStack para armazenar os dados, implica em --mantem-csv"
echo -e " -u, --usuario\t\tUsuário do ElasticSearch (default: em branco)"
echo -e " -s, --senha\t\tSenha do ElasticSearch (default: sem senha)"
echo -e " -H, --hostname\tHostname do ElasticSearch (default: localhost)"
echo ""
echo "Definição de período de tempo para inserção"
echo -e " -i, --inicio\t\tAno e mês de início dos dados, no formato YYYY-MM (default: current year-month)"
echo -e " -f, --fim\t\tAno e mês de fim dos dados, no formato YYYY-MM (default: current year-month)"
echo ""
echo "Definição das bases a serem inseridas"
echo -e " --todas-as-bases\tInsere todas as bases (despesas, diarias e servidores)"
echo -e " --despesas\tInsere dados de despesas gerais das união"
echo -e " --diarias\t\tInsere dados de despesas com diárias da união"
echo -e " --servidores\tInsere dados de remuneração de servidores"
echo ""
echo "Outras opções"
echo -e " --mantem-csv\tNão remove os arquivos CSV utilizados para inserção no banco"
echo ""
}
function echoerr() {
echo "$@" 1>&2;
}
function echoerr_and_exit() {
echoerr "$@"
usage
exit 1
}
function empty_arg_sanity_check() {
local arg_to_check=$1
if [[ "$arg_to_check" = -* || "$arg_to_check" = "" ]]; then
return 1
else
echo $arg_to_check
return 0
fi
}
function date_sanity_check() {
local date_to_check=$1
local current_epoch=$(date -d "$(date +%Y-%m-01)" "+%s")
local given_year=
local given_month=
local given_epoch=
if empty_arg_sanity_check "$date_to_check" | egrep -q '^[0-9]{4}-[0-9]{2}$'; then
# check if user is trying to insert data from the future
given_year=$(cut -d- -f1 <<< "$date_to_check")
given_month=$(cut -d- -f2 <<