Commit 2c861934 authored by Cristian Weiland's avatar Cristian Weiland

Merge branch 'issue/63' into 'master'

Issue #63: Created variables for long and repetitive strings

See merge request !33
parents 02271923 085b47bc
# WARNING: This is only an example file. You should copy it and rename to 'config.sh' in the same directory as this file, and then apply your modifications.
# This file only contains some config variables:
# This variable sets the host in which the database is running
dbHostname="localhost:9200/"
......@@ -6,8 +6,10 @@ if [ "$#" -ne 3 ]; then
exit
fi
source ./config.sh
# Copy old index to new index...
curl -u $1 -XPOST 'http://node1.c3sl.ufpr.br:9200/_reindex?pretty' -H 'Content-Type: application/json' -d'
curl -XPOST -u $1 "${dbHostname}_reindex?pretty" -H 'Content-Type: application/json' -d'
{
"source": {
"index": "'$2'"
......@@ -19,4 +21,4 @@ curl -u $1 -XPOST 'http://node1.c3sl.ufpr.br:9200/_reindex?pretty' -H 'Content-T
'
# Delete old index...
curl -u $1 -XDELETE "http://node1.c3sl.ufpr.br:9200/$2?pretty"
curl -XDELETE -u $1 "${dbHostname}$2?pretty"
......@@ -12,6 +12,11 @@
# 5- Insert data in ElasticSearch via logstash, using the config file created and the CSV created by resume_expenses.sh.
# Output: The commands/scripts outputs.
function inputError(){
echo "Var ${1} is unset. Set in file '${2}'."
return 0
}
if [ "$#" -ne 4 ]; then
echo "Usage: $0 <year> <month> <user> <password>"
echo "Example: $0 2016 12 myuser mypass"
......@@ -21,22 +26,23 @@ fi
source ./config.sh
# Check if all variables in config file are set:
configFile='scripts/expenses/config.sh'
if [ -z "${index}" ]; then
echo "Var 'index' is unset. Set it in file 'scripts/expenses/config.sh'.";
inputError "index" $configFile
exit;
fi
if [ -z "${host}" ]; then
echo "Var 'host' is unset. Set it in file 'scripts/expenses/config.sh'.";
inputError "host" $configFile
exit;
fi
if [ -z "${columnName}" ]; then
echo "Var 'columnName' is unset. Set it in file 'scripts/expenses/config.sh'.";
inputError "columnName" $configFile
exit;
fi
size=${#filter[@]}
if [ "$size" -lt 1 ]; then
echo "Var 'filter' is unset. Set it in file 'scripts/expenses/config.sh'.";
inputError "filter" $configFile
exit;
fi
......@@ -56,12 +62,15 @@ path="./tmp_$ym"
mkdir -p "$path"
# Download files
downloadLink='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='
# Download expenses file:
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${1}'&m='${2}'&consulta=GastosDiretos'
curl -o $path/${1}${2}_GastosDiretos.zip $request -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: en-US,en;q=0.8' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Referer: http://transparencia.gov.br/downloads/mensal.asp?c=GastosDiretos' -H 'Cookie: ASPSESSIONIDAQRABSAD=OJDLNBCANLIDINCHJHELHHFB; ASPSESSIONIDAQSDCQAD=BOKBKPNCDKOBJKGAMMEKADFL; _ga=GA1.3.1927288562.1481545643; ASPSESSIONIDSCSBBTCD=IGJLJBBCEEJBGLOOJKGNMHBH' -H 'Connection: keep-alive' --compressed
request="${downloadLink}${1}&m=${2}&consulta=GastosDiretos"
curl -o $path/${1}${2}_GastosDiretos.zip $request --compressed
# Download file with information about company:
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${1}'&m='${2}'&consulta=FavorecidosGastosDiretos'
curl -o $path/${1}${2}_Favorecidos.zip $request -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: en-US,en;q=0.8,pt;q=0.6' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/56.0.2924.76 Chrome/56.0.2924.76 Safari/537.36' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Referer: http://www.portaltransparencia.gov.br/downloads/mensal.asp?c=FavorecidosGastosDiretos' -H 'Cookie: ASPSESSIONIDSCBRBBTT=KPBDKGCAENJIEFBMMPOACBHJ' -H 'Connection: keep-alive' --compressed
request="${downloadLink}${1}&m=${2}&consulta=FavorecidosGastosDiretos"
curl -o $path/${1}${2}_Favorecidos.zip $request --compressed
# Unzip them
unzip -o $path/${1}${2}_GastosDiretos.zip -d $path/
......
......@@ -11,6 +11,11 @@
# 4- Insert data in ElasticSearch via logstash, using the config file created and the CSV created by resume_travel_allowance.sh.
# Output: The commands/scripts outputs.
function inputError(){
echo "Var ${1} is unset. Set in file '${2}'."
return 0
}
if [ "$#" -ne 4 ]; then
echo "Usage: $0 <year> <month> <user> <password>"
echo "Example: $0 2016 12 myuser mypass"
......@@ -20,22 +25,23 @@ fi
source ./config.sh
# Check if all variables in config file are set:
configFile='scripts/travel_allowance/config.sh'
if [ -z "${index}" ]; then
echo "Var 'index' is unset. Set it in file 'scripts/travel_allowance/config.sh'.";
inputError "index" $configFile
exit;
fi
if [ -z "${host}" ]; then
echo "Var 'host' is unset. Set it in file 'scripts/travel_allowance/config.sh'.";
inputError "host" $configFile
exit;
fi
if [ -z "${columnName}" ]; then
echo "Var 'host' is unset. Set it in file 'scripts/travel_allowance/config.sh'.";
inputError "columnName" $configFile
exit;
fi
size=${#filter[@]}
if [ "$size" -lt 1 ]; then
echo "Var 'filter' is unset. Set it in file 'scripts/expenses/config.sh'.";
inputError "filter" $configFile
exit;
fi
......@@ -56,7 +62,8 @@ mkdir -p "$path"
# Download files
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${1}'&m='${2}'&consulta=Diarias'
curl $request -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: en-US,en;q=0.8' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Referer: http://transparencia.gov.br/downloads/mensal.asp?c=GastosDiretos' -H 'Cookie: ASPSESSIONIDAQRABSAD=OJDLNBCANLIDINCHJHELHHFB; ASPSESSIONIDAQSDCQAD=BOKBKPNCDKOBJKGAMMEKADFL; _ga=GA1.3.1927288562.1481545643; ASPSESSIONIDSCSBBTCD=IGJLJBBCEEJBGLOOJKGNMHBH' -H 'Connection: keep-alive' --compressed > $path/${1}${2}_Diarias.zip
curl $request --compressed > $path/${1}${2}_Diarias.zip
# Unzip them
unzip -o $path/${1}${2}_Diarias.zip -d $path/
......
......@@ -15,6 +15,12 @@
# WARNING: We get the day from the CSV file by using cut in characters 7 and 8. This means we assume they will write something like 01 as day 1. If they change it to 1, this script will not work!
function inputError(){
echo "Var ${1} is unset. Set in file '${2}'."
return 0
}
if [ "$#" -ne 4 ]; then
echo "Usage: $0 <year> <month> <user> <password>"
echo "Example: $0 2016 12 myuser mypassword"
......@@ -22,23 +28,23 @@ if [ "$#" -ne 4 ]; then
fi
source ./config.sh
configFile='scripts/workers/config.sh'
if [ -z "${index}" ]; then
echo "Var 'index' is unset. Set it in file 'scripts/workers/config.sh'.";
inputError "index" $configFile
exit;
fi
if [ -z "${host}" ]; then
echo "Var 'host' is unset. Set it in file 'scripts/workers/config.sh'.";
inputError "host" $configFile
exit;
fi
if [ -z "${columnName}" ]; then
echo "Var 'columnName' is unset. Set it in file 'scripts/workers/config.sh'.";
inputError "columnName" $configFile
exit;
fi
size=${#filter[@]}
if [ "$size" -lt 1 ]; then
echo "Var 'filter' is unset. Set it in file 'scripts/expenses/config.sh'.";
inputError "filter" $configFile
exit;
fi
......@@ -51,7 +57,8 @@ mkdir -p "$path"
# Download files
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${1}'&m='${2}'&d=C&consulta=Servidores'
curl $request -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: en-US,en;q=0.8' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_ 64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Referer: http://www.portaldatranspar encia.gov.br/downloads/servidores.asp' -H 'Cookie: ASPSESSIONIDAQRABSAD=OJDLNBCANLIDINCHJHELHHFB; ASPSESSIONIDAQSDCQAD=BOKBKPNCDKOBJKGAMMEKADFL; _ga=GA1.3.1927288562.1481545643; ASPSESSIONIDSCSBBTCD=IGJLJBBC EEJBGLOOJKGNMHBH' -H 'Connection: keep-alive' --compressed > $path/${1}${2}_Servidores.zip
curl $request --compressed > $path/${1}${2}_Servidores.zip
# Unzip them
unzip -o $path/${1}${2}_Servidores.zip -d $path/
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment