Commit 3851a7c7 authored by Luiza Wille's avatar Luiza Wille

Issue #63: Created variables for long and repetitive strings

Signed-off-by: Luiza Wille's avatarLuiza Wille <lmwc14@inf.ufpr.br>
parent bedd948e
......@@ -6,8 +6,10 @@ if [ "$#" -ne 3 ]; then
exit
fi
node1c3sl='http://node1.c3sl.ufpr.br:9200/'
# Copy old index to new index...
curl -u $1 -XPOST 'http://node1.c3sl.ufpr.br:9200/_reindex?pretty' -H 'Content-Type: application/json' -d'
curl -u $1 -XPOST "${node1c3sl}_reindex?pretty" -H 'Content-Type: application/json' -d'
{
"source": {
"index": "'$2'"
......@@ -19,4 +21,4 @@ curl -u $1 -XPOST 'http://node1.c3sl.ufpr.br:9200/_reindex?pretty' -H 'Content-T
'
# Delete old index...
curl -u $1 -XDELETE "http://node1.c3sl.ufpr.br:9200/$2?pretty"
curl -u $1 -XDELETE "${node1c3sl}$2?pretty"
......@@ -21,22 +21,23 @@ fi
source ./config.sh
# Check if all variables in config file are set:
setInFile="Set it in file 'scripts/expenses/config.sh'."
if [ -z "${index}" ]; then
echo "Var 'index' is unset. Set it in file 'scripts/expenses/config.sh'.";
echo "Var 'index' is unset. ${setInFile}";
exit;
fi
if [ -z "${host}" ]; then
echo "Var 'host' is unset. Set it in file 'scripts/expenses/config.sh'.";
echo "Var 'host' is unset. ${setInFile}";
exit;
fi
if [ -z "${columnName}" ]; then
echo "Var 'columnName' is unset. Set it in file 'scripts/expenses/config.sh'.";
echo "Var 'columnName' is unset. ${setInFile}";
exit;
fi
size=${#filter[@]}
if [ "$size" -lt 1 ]; then
echo "Var 'filter' is unset. Set it in file 'scripts/expenses/config.sh'.";
echo "Var 'filter' is unset. ${setInFile}";
exit;
fi
......@@ -56,12 +57,22 @@ path="./tmp_$ym"
mkdir -p "$path"
# Download files
downloadLink='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='
acceptedEncoding='Accept-Encoding: gzip, deflate, sdch'
userAgent='User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)'
acceptedFormat='Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
connection='Connection: keep-alive'
expensesCookie='Cookie: ASPSESSIONIDAQRABSAD=OJDLNBCANLIDINCHJHELHHFB; ASPSESSIONIDAQSDCQAD=BOKBKPNCDKOBJKGAMMEKADFL; _ga=GA1.3.1927288562.1481545643; ASPSESSIONIDSCSBBTCD=IGJLJBBCEEJBGLOOJKGNMHBH'
expensesReferer='Referer: http://transparencia.gov.br/downloads/mensal.asp?c=GastosDiretos'
companyCookie='Cookie: ASPSESSIONIDSCBRBBTT=KPBDKGCAENJIEFBMMPOACBHJ'
companyReferer='Referer: http://www.portaltransparencia.gov.br/downloads/mensal.asp?c=FavorecidosGastosDiretos'
# Download expenses file:
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${1}'&m='${2}'&consulta=GastosDiretos'
curl -o $path/${1}${2}_GastosDiretos.zip $request -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: en-US,en;q=0.8' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Referer: http://transparencia.gov.br/downloads/mensal.asp?c=GastosDiretos' -H 'Cookie: ASPSESSIONIDAQRABSAD=OJDLNBCANLIDINCHJHELHHFB; ASPSESSIONIDAQSDCQAD=BOKBKPNCDKOBJKGAMMEKADFL; _ga=GA1.3.1927288562.1481545643; ASPSESSIONIDSCSBBTCD=IGJLJBBCEEJBGLOOJKGNMHBH' -H 'Connection: keep-alive' --compressed
request="${downloadLink}${1}&m=${2}&consulta=GastosDiretos"
curl -o $path/${1}${2}_GastosDiretos.zip $request -H "${acceptedEncoding}" -H 'Accept-Language: en-US,en;q=0.8' -H 'Upgrade-Insecure-Requests: 1' -H "${userAgent} Chrome/53.0.2785.143 Safari/537.36" -H "${acceptedFormat}" -H "${expensesReferer}" -H "${expensesCookie}" -H "${connection}" --compressed
# Download file with information about company:
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${1}'&m='${2}'&consulta=FavorecidosGastosDiretos'
curl -o $path/${1}${2}_Favorecidos.zip $request -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: en-US,en;q=0.8,pt;q=0.6' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/56.0.2924.76 Chrome/56.0.2924.76 Safari/537.36' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Referer: http://www.portaltransparencia.gov.br/downloads/mensal.asp?c=FavorecidosGastosDiretos' -H 'Cookie: ASPSESSIONIDSCBRBBTT=KPBDKGCAENJIEFBMMPOACBHJ' -H 'Connection: keep-alive' --compressed
request="${downloadLink}${1}&m=${2}&consulta=FavorecidosGastosDiretos"
curl -o $path/${1}${2}_Favorecidos.zip $request -H "${acceptedEncoding}" -H 'Accept-Language: en-US,en;q=0.8,pt;q=0.6' -H 'Upgrade-Insecure-Requests: 1' -H "${userAgent} Ubuntu Chromium/56.0.2924.76 Chrome/56.0.2924.76 Safari/537.36" -H "${acceptedFormat}" -H "${companyReferer}" -H "${companyCookie}" -H "${connection}" --compressed
# Unzip them
unzip -o $path/${1}${2}_GastosDiretos.zip -d $path/
......
......@@ -55,8 +55,16 @@ path="./tmp_$ym"
mkdir -p "$path"
# Download files
acceptedEncoding='Accept-Encoding: gzip, deflate, sdch'
acceptedLanguage='Accept-Language: en-US,en;q=0.8'
userAgent='User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36'
acceptedFormat='Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
referer='Referer: http://transparencia.gov.br/downloads/mensal.asp?c=GastosDiretos'
cookie='Cookie: ASPSESSIONIDAQRABSAD=OJDLNBCANLIDINCHJHELHHFB; ASPSESSIONIDAQSDCQAD=BOKBKPNCDKOBJKGAMMEKADFL; _ga=GA1.3.1927288562.1481545643; ASPSESSIONIDSCSBBTCD=IGJLJBBCEEJBGLOOJKGNMHBH'
connection='Connection: keep-alive'
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${1}'&m='${2}'&consulta=Diarias'
curl $request -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: en-US,en;q=0.8' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Referer: http://transparencia.gov.br/downloads/mensal.asp?c=GastosDiretos' -H 'Cookie: ASPSESSIONIDAQRABSAD=OJDLNBCANLIDINCHJHELHHFB; ASPSESSIONIDAQSDCQAD=BOKBKPNCDKOBJKGAMMEKADFL; _ga=GA1.3.1927288562.1481545643; ASPSESSIONIDSCSBBTCD=IGJLJBBCEEJBGLOOJKGNMHBH' -H 'Connection: keep-alive' --compressed > $path/${1}${2}_Diarias.zip
curl $request -H "${acceptedEncoding}" -H "${acceptedLanguage}" -H 'Upgrade-Insecure-Requests: 1' -H "${userAgent}" -H "${acceptedFormat}" -H "${referer}" -H "${cookie}" -H "${connection}" --compressed > $path/${1}${2}_Diarias.zip
# Unzip them
unzip -o $path/${1}${2}_Diarias.zip -d $path/
......
......@@ -50,8 +50,16 @@ path="./tmp_$ym"
mkdir -p "$path"
# Download files
acceptedEncoding='Accept-Encoding: gzip, deflate, sdch'
acceptedLanguage='Accept-Language: en-US,en;q=0.8'
userAgent='User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36'
acceptedFormat='Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
referer='Referer: http://www.portaldatransparencia.gov.br/downloads/servidores.asp'
cookie='Cookie: ASPSESSIONIDAQRABSAD=OJDLNBCANLIDINCHJHELHHFB; ASPSESSIONIDAQSDCQAD=BOKBKPNCDKOBJKGAMMEKADFL; _ga=GA1.3.1927288562.1481545643; ASPSESSIONIDSCSBBTCD=IGJLJBBCEEJBGLOOJKGNMHBH'
connection='Connection: keep-alive'
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${1}'&m='${2}'&d=C&consulta=Servidores'
curl $request -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: en-US,en;q=0.8' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_ 64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Referer: http://www.portaldatranspar encia.gov.br/downloads/servidores.asp' -H 'Cookie: ASPSESSIONIDAQRABSAD=OJDLNBCANLIDINCHJHELHHFB; ASPSESSIONIDAQSDCQAD=BOKBKPNCDKOBJKGAMMEKADFL; _ga=GA1.3.1927288562.1481545643; ASPSESSIONIDSCSBBTCD=IGJLJBBC EEJBGLOOJKGNMHBH' -H 'Connection: keep-alive' --compressed > $path/${1}${2}_Servidores.zip
curl $request -H "${acceptedEncoding}" -H "${acceptedLanguage}" -H 'Upgrade-Insecure-Requests: 1' -H "${userAgent}" -H "${acceptedFormat}" -H "${referer}" -H "${cookie}" -H "${connection}" --compressed > $path/${1}${2}_Servidores.zip
# Unzip them
unzip -o $path/${1}${2}_Servidores.zip -d $path/
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment