Commit d35f4d15 authored by Cristian Weiland's avatar Cristian Weiland

Issue #80: Fix small bugs and change program output

Signed-off-by: Cristian Weiland's avatarCristian Weiland <cw14@inf.ufpr.br>
parent 9b0e0c11
......@@ -189,17 +189,18 @@ mkdir -p "$path"
# Download files
downloadLink='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='
echo "Efetuando download da base de dados..."
# Download expenses file:
request="${downloadLink}${YEAR}&m=${MONTH}&consulta=GastosDiretos"
curl -o $path/${YEAR}${MONTH}_GastosDiretos.zip $request --compressed
curl --silent -o $path/${YEAR}${MONTH}_GastosDiretos.zip $request --compressed
# Download file with information about company:
request="${downloadLink}${YEAR}&m=${MONTH}&consulta=FavorecidosGastosDiretos"
curl -o $path/${YEAR}${MONTH}_Favorecidos.zip $request --compressed
curl --silent -o $path/${YEAR}${MONTH}_Favorecidos.zip $request --compressed
# Unzip them
unzip -o $path/${YEAR}${MONTH}_GastosDiretos.zip -d $path/
unzip -o $path/${YEAR}${MONTH}_Favorecidos.zip -d $path/
unzip -qq -o $path/${YEAR}${MONTH}_GastosDiretos.zip -d $path/
unzip -qq -o $path/${YEAR}${MONTH}_Favorecidos.zip -d $path/
# Remove zip file
rm $path/${YEAR}${MONTH}_GastosDiretos.zip
......@@ -211,6 +212,8 @@ cat $path/${YEAR}${MONTH}_NaturezaJuridica.csv | tr -d '\000' > $path/${YEAR}${M
mv $path/${YEAR}${MONTH}_CNPJ_NotNull.csv $path/${YEAR}${MONTH}_CNPJ.csv
mv $path/${YEAR}${MONTH}_NatJur_NotNull.csv $path/${YEAR}${MONTH}_NaturezaJuridica.csv
echo "Processando os arquivos obtidos..."
for key in "${!filter[@]}"
do
# Step 2:
......@@ -223,7 +226,7 @@ for key in "${!filter[@]}"
./merge_files.py $path/config-natjur-${YEAR}-${MONTH}.json
# Step 4:
if [ "$ELASTICSTACK" = true ]; then
logstash -f ${path}/config-${YEAR}-${MONTH} < ${path}/${YEAR}${MONTH}.csv
logstash -f ${path}/config-${YEAR}-${MONTH} < ${path}/${YEAR}${MONTH}.csv > output.log
fi
# Data inserted, we can now remove it.
if [ "$KEEP_CSV" = false ]; then
......@@ -240,6 +243,8 @@ for key in "${!filter[@]}"
rm ${path}/config-natjur-${YEAR}-${MONTH}.json
done
echo "Finalizando..."
# Remove downloaded csvs.
rm -f $path/${YEAR}${MONTH}_GastosDiretos.csv
rm -f $path/${YEAR}${MONTH}_GastosDiretosFiltered.csv
......
......@@ -32,7 +32,7 @@ host = args.host
username = args.usuario
passwd = args.senha
with open('logstash_config.example') as infile:
with open('logstash_config.example', encoding='utf-8') as infile:
example = infile.read()
output = example % { "timestamp": day + '/' + month + '/' + year + ' 00:00:00'
......@@ -43,5 +43,5 @@ output = example % { "timestamp": day + '/' + month + '/' + year + ' 00:00:00'
, "password": passwd }
date = year + '-' + month
with open(path + '/config-' + date, 'w') as outfile:
with open(path + '/config-' + date, 'w', encoding='utf-8') as outfile:
outfile.write(output)
......@@ -185,17 +185,20 @@ path="./tmp_$ym"
# Create directory to store files
mkdir -p "$path"
echo "Efetuando download da base de dados..."
# Download files
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${YEAR}'&m='${MONTH}'&consulta=Diarias'
curl $request --compressed > $path/${YEAR}${MONTH}_Diarias.zip
curl --silent $request --compressed > $path/${YEAR}${MONTH}_Diarias.zip
# Unzip them
unzip -o $path/${YEAR}${MONTH}_Diarias.zip -d $path/
unzip -qq -o $path/${YEAR}${MONTH}_Diarias.zip -d $path/
# Remove zip file
rm $path/${YEAR}${MONTH}_Diarias.zip
echo "Processando os arquivos obtidos..."
for key in "${!filter[@]}"
do
# Step 2: Create config file
......@@ -204,7 +207,7 @@ do
./resume_travel_allowance.sh "$path" ${YEAR}-${MONTH} "${filter[$key]}" "${columnName}"
# Step 4: Insert data in ElasticSearch
if [ "$ELASTICSTACK" = true ]; then
logstash -f ${path}/config-${YEAR}-${MONTH} < ${path}/${YEAR}${MONTH}.csv
logstash -f ${path}/config-${YEAR}-${MONTH} < ${path}/${YEAR}${MONTH}.csv > output.log
fi
# Remove processed file
if [ "$KEEP_CSV" = false ]; then
......@@ -217,5 +220,7 @@ do
rm ${path}/config-${YEAR}-${MONTH}
done
echo "Finalizando..."
rm $path/${YEAR}${MONTH}_Diarias.csv
rmdir $path
......@@ -47,14 +47,14 @@ data = {
, "outputFile": path + '/' + year + month + day + ".csv"
}
with open(path + '/config-' + year + '-' + month + '.json', 'w') as outfile:
with open(path + '/config-' + year + '-' + month + '.json', 'w', encoding='utf-8') as outfile:
json.dump(data, outfile, indent=4, sort_keys=True)
if int(year) <= 2014 or (int(year) == 2015 and int(month) <= 3):
with open('previous_logstash_config.example') as infile:
with open('previous_logstash_config.example', encoding='utf-8') as infile:
example = infile.read()
else:
with open('logstash_config.example') as infile:
with open('logstash_config.example', encoding='utf-8') as infile:
example = infile.read()
output = example % { "timestamp": day + '/' + month + '/' + year + ' 00:00:00'
......@@ -64,5 +64,5 @@ output = example % { "timestamp": day + '/' + month + '/' + year + ' 00:00:00'
, "user": username
, "password": passwd }
with open(path + '/config-' + year + '-' + month, 'w') as outfile:
with open(path + '/config-' + year + '-' + month, 'w', encoding='utf-8') as outfile:
outfile.write(output)
......@@ -181,13 +181,14 @@ path="./tmp_$ym"
# Create directory to store files
mkdir -p "$path"
echo "Efetuando download da base de dados..."
# Download files
request='http://arquivos.portaldatransparencia.gov.br/downloads.asp?a='${YEAR}'&m='${MONTH}'&d=C&consulta=Servidores'
curl $request --compressed > $path/${YEAR}${MONTH}_Servidores.zip
curl --silent $request --compressed > $path/${YEAR}${MONTH}_Servidores.zip
# Unzip them
unzip -o $path/${YEAR}${MONTH}_Servidores.zip -d $path/
unzip -qq -o $path/${YEAR}${MONTH}_Servidores.zip -d $path/
# Remove zip file
rm $path/${YEAR}${MONTH}_Servidores.zip
......@@ -195,6 +196,8 @@ rm $path/${YEAR}${MONTH}_Servidores.zip
# Get day
DAY=$(ls $path | grep -m 1 $YEAR$MONTH | cut -c 7,8)
echo "Processando os arquivos obtidos..."
for key in "${!filter[@]}"
do
# Step 2: Create config files
......@@ -207,7 +210,7 @@ do
# Step 4: Insert data in ElasticSearch
if [ "$ELASTICSTACK" = true ]; then
logstash -f $path/config-${YEAR}-${MONTH} < $path/${YEAR}${MONTH}${DAY}.csv
logstash -f $path/config-${YEAR}-${MONTH} < $path/${YEAR}${MONTH}${DAY}.csv > output.log
fi
# Remove data
......@@ -221,6 +224,8 @@ do
fi
done
echo "Finalizando..."
rm -f $path/${YEAR}${MONTH}${DAY}_Afastamentos.csv
rm -f $path/${YEAR}${MONTH}${DAY}_Cadastro.csv
rm -f $path/${YEAR}${MONTH}${DAY}_Honorarios\(Jetons\).csv
......
......@@ -104,7 +104,7 @@ for row2 in csv_2:
result.append(newRow)
errors += 1
with open(params['outputFile'], 'w', newline='') as csvfile:
with open(params['outputFile'], 'w', newline='', encoding='utf-8') as csvfile:
writer = csv.writer(csvfile, delimiter='\t')
writer.writerow(getDataFromRows(title1, title2))
writer.writerows(result)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment