Add script to insert expenses

Signed-off-by: Cristian Weiland <cw14@inf.ufpr.br>

add_registers.sh

 #!/bin/bash
 
-./insert_resgiter_payment.sh 2016 11 30
-./insert_resgiter_payment.sh 2016 10 31
-./insert_resgiter_payment.sh 2016 09 30
-./insert_resgiter_payment.sh 2016 08 31
-./insert_resgiter_payment.sh 2016 07 31
-./insert_resgiter_payment.sh 2016 06 30
-./insert_resgiter_payment.sh 2016 05 31
-./insert_resgiter_payment.sh 2016 04 30
-./insert_resgiter_payment.sh 2016 03 31
-./insert_resgiter_payment.sh 2016 02 29
-./insert_resgiter_payment.sh 2016 01 31
+echo "Usage: $0 <user> <password>"
 
-./insert_resgiter_payment.sh 2015 12 31
-./insert_resgiter_payment.sh 2015 11 30
-./insert_resgiter_payment.sh 2015 10 31
-./insert_resgiter_payment.sh 2015 09 30
-./insert_resgiter_payment.sh 2015 08 31
-./insert_resgiter_payment.sh 2015 07 31
-./insert_resgiter_payment.sh 2015 06 30
-./insert_resgiter_payment.sh 2015 05 31
-./insert_resgiter_payment.sh 2015 04 30
-./insert_resgiter_payment.sh 2015 03 31
-./insert_resgiter_payment.sh 2015 02 28
-./insert_resgiter_payment.sh 2015 01 31
+./insert_register_payment.sh 2016 11 30 $1 $2
+./insert_register_payment.sh 2016 10 31 $1 $2
+./insert_register_payment.sh 2016 09 30 $1 $2
+./insert_register_payment.sh 2016 08 31 $1 $2
+./insert_register_payment.sh 2016 07 31 $1 $2
+./insert_register_payment.sh 2016 06 30 $1 $2
+./insert_register_payment.sh 2016 05 31 $1 $2
+./insert_register_payment.sh 2016 04 30 $1 $2
+./insert_register_payment.sh 2016 03 31 $1 $2
+./insert_register_payment.sh 2016 02 29 $1 $2
+./insert_register_payment.sh 2016 01 31 $1 $2
 
-./insert_resgiter_payment.sh 2014 12 31
-./insert_resgiter_payment.sh 2014 11 30
-./insert_resgiter_payment.sh 2014 10 31
-./insert_resgiter_payment.sh 2014 09 30
-./insert_resgiter_payment.sh 2014 08 31
-./insert_resgiter_payment.sh 2014 07 31
-./insert_resgiter_payment.sh 2014 06 30
-./insert_resgiter_payment.sh 2014 05 31
-./insert_resgiter_payment.sh 2014 04 30
-./insert_resgiter_payment.sh 2014 03 31
-./insert_resgiter_payment.sh 2014 02 28
-./insert_resgiter_payment.sh 2014 01 31
+./insert_register_payment.sh 2015 12 31 $1 $2
+./insert_register_payment.sh 2015 11 30 $1 $2
+./insert_register_payment.sh 2015 10 31 $1 $2
+./insert_register_payment.sh 2015 09 30 $1 $2
+./insert_register_payment.sh 2015 08 31 $1 $2
+./insert_register_payment.sh 2015 07 31 $1 $2
+./insert_register_payment.sh 2015 06 30 $1 $2
+./insert_register_payment.sh 2015 05 31 $1 $2
+./insert_register_payment.sh 2015 04 30 $1 $2
+./insert_register_payment.sh 2015 03 31 $1 $2
+./insert_register_payment.sh 2015 02 28 $1 $2
+./insert_register_payment.sh 2015 01 31 $1 $2
 
-./insert_resgiter_payment.sh 2013 12 31
-./insert_resgiter_payment.sh 2013 11 30
-./insert_resgiter_payment.sh 2013 10 31
-./insert_resgiter_payment.sh 2013 09 30
-./insert_resgiter_payment.sh 2013 08 31
-./insert_resgiter_payment.sh 2013 07 31
-./insert_resgiter_payment.sh 2013 06 30
-./insert_resgiter_payment.sh 2013 05 31
-./insert_resgiter_payment.sh 2013 04 30
-./insert_resgiter_payment.sh 2013 03 31
-./insert_resgiter_payment.sh 2013 02 28
-./insert_resgiter_payment.sh 2013 01 31
+./insert_register_payment.sh 2014 12 31 $1 $2
+./insert_register_payment.sh 2014 11 30 $1 $2
+./insert_register_payment.sh 2014 10 31 $1 $2
+./insert_register_payment.sh 2014 09 30 $1 $2
+./insert_register_payment.sh 2014 08 31 $1 $2
+./insert_register_payment.sh 2014 07 31 $1 $2
+./insert_register_payment.sh 2014 06 30 $1 $2
+./insert_register_payment.sh 2014 05 31 $1 $2
+./insert_register_payment.sh 2014 04 30 $1 $2
+./insert_register_payment.sh 2014 03 31 $1 $2
+./insert_register_payment.sh 2014 02 28 $1 $2
+./insert_register_payment.sh 2014 01 31 $1 $2
+
+./insert_register_payment.sh 2013 12 31 $1 $2
+./insert_register_payment.sh 2013 11 30 $1 $2
+./insert_register_payment.sh 2013 10 31 $1 $2
+./insert_register_payment.sh 2013 09 30 $1 $2
+./insert_register_payment.sh 2013 08 31 $1 $2
+./insert_register_payment.sh 2013 07 31 $1 $2
+./insert_register_payment.sh 2013 06 30 $1 $2
+./insert_register_payment.sh 2013 05 31 $1 $2
+./insert_register_payment.sh 2013 04 30 $1 $2
+./insert_register_payment.sh 2013 03 31 $1 $2
+./insert_register_payment.sh 2013 02 28 $1 $2
+./insert_register_payment.sh 2013 01 31 $1 $2

create_config.py

@@ -4,8 +4,8 @@ import sys, csv, json, math, subprocess
 from pathlib import Path
 from subprocess import call
 
-if len(sys.argv) != 4:
-    print("Usage: " + sys.argv[0] + " <year (2016)> <month (01)> <day (31)>")
+if len(sys.argv) != 6:
+    print("Usage: " + sys.argv[0] + " <year (2016)> <month (01)> <day (31)> <username> <password>")
     sys.exit()
 
 data = {
@@ -28,7 +28,9 @@ with open('logstash_config.example') as infile:
 	example = infile.read()
 
 output = example % { "timestamp": sys.argv[3] + '/' + sys.argv[2] + '/' + sys.argv[1] + ' 00:00:00'
-					 , "date": sys.argv[1] + '-' + sys.argv[2] }
+					 , "date": sys.argv[1] + '-' + sys.argv[2]
+					 , "user": sys.argv[4]
+					 , "password": sys.argv[5] }
 
 with open('logstash_configs/config-' + sys.argv[1] + '-' + sys.argv[2], 'w') as outfile:
 	outfile.write(output)

insert_resgiter_payment.sh → insert_register_payment.sh

 #!/bin/bash
 
-if [ "$#" -ne 3 ]; then
-	echo "Usage: $0 <year> <month> <day>"
+if [ "$#" -ne 5 ]; then
+	echo "Usage: $0 <year> <month> <day> <user> <password>"
 	echo "Example: $0 2016 12 01"
 	exit
 fi
 
-./create_config.py $1 $2 $3
+./create_config.py $1 $2 $3 $4 $5
 ./merge_files_es.py configs/config-${1}-${2}.json
 logstash -f logstash_configs/config-${1}-${2} < ~/transparencia/Dados_Servidores/Processados/${1}${2}.csv

logstash_config.example

@@ -56,8 +56,8 @@ filter {
 output {
 	elasticsearch {
 		action => "index"
-		user => "cw14"
-		password => "123mudar"
+		user => "%(user)s"
+		password => "%(password)s"
 		hosts => "http://node1.c3sl.ufpr.br:9200"
 		index => "ufpr-csv-%(date)s"
 		workers => 1

scripts/expenses/add_expenses.sh

+#!/bin/bash
+
+if [ "$#" -ne 2 ]; then
+	echo "Usage: $0 <user> <password>"
+	echo "Example: $0 myuser mypass"
+	exit
+fi
+
+./insert_expenses.sh 2016 11 30 $1 $2
+./insert_expenses.sh 2016 10 31 $1 $2
+./insert_expenses.sh 2016 09 30 $1 $2
+./insert_expenses.sh 2016 08 31 $1 $2
+./insert_expenses.sh 2016 07 31 $1 $2
+./insert_expenses.sh 2016 06 30 $1 $2
+./insert_expenses.sh 2016 05 31 $1 $2
+./insert_expenses.sh 2016 04 30 $1 $2
+./insert_expenses.sh 2016 03 31 $1 $2
+./insert_expenses.sh 2016 02 29 $1 $2
+./insert_expenses.sh 2016 01 31 $1 $2
+
+./insert_expenses.sh 2015 12 31 $1 $2
+./insert_expenses.sh 2015 11 30 $1 $2
+./insert_expenses.sh 2015 10 31 $1 $2
+./insert_expenses.sh 2015 09 30 $1 $2
+./insert_expenses.sh 2015 08 31 $1 $2
+./insert_expenses.sh 2015 07 31 $1 $2
+./insert_expenses.sh 2015 06 30 $1 $2
+./insert_expenses.sh 2015 05 31 $1 $2
+./insert_expenses.sh 2015 04 30 $1 $2
+./insert_expenses.sh 2015 03 31 $1 $2
+./insert_expenses.sh 2015 02 28 $1 $2
+./insert_expenses.sh 2015 01 31 $1 $2
+
+./insert_expenses.sh 2014 12 31 $1 $2
+./insert_expenses.sh 2014 11 30 $1 $2
+./insert_expenses.sh 2014 10 31 $1 $2
+./insert_expenses.sh 2014 09 30 $1 $2
+./insert_expenses.sh 2014 08 31 $1 $2
+./insert_expenses.sh 2014 07 31 $1 $2
+./insert_expenses.sh 2014 06 30 $1 $2
+./insert_expenses.sh 2014 05 31 $1 $2
+./insert_expenses.sh 2014 04 30 $1 $2
+./insert_expenses.sh 2014 03 31 $1 $2
+./insert_expenses.sh 2014 02 28 $1 $2
+./insert_expenses.sh 2014 01 31 $1 $2
+
+./insert_expenses.sh 2013 12 31 $1 $2
+./insert_expenses.sh 2013 11 30 $1 $2
+./insert_expenses.sh 2013 10 31 $1 $2
+./insert_expenses.sh 2013 09 30 $1 $2
+./insert_expenses.sh 2013 08 31 $1 $2
+./insert_expenses.sh 2013 07 31 $1 $2
+./insert_expenses.sh 2013 06 30 $1 $2
+./insert_expenses.sh 2013 05 31 $1 $2
+./insert_expenses.sh 2013 04 30 $1 $2
+./insert_expenses.sh 2013 03 31 $1 $2
+./insert_expenses.sh 2013 02 28 $1 $2
+./insert_expenses.sh 2013 01 31 $1 $2

scripts/expenses/create_expenses_config.py

+#!/usr/bin/env python3
+
+import sys, csv, json, math, subprocess
+from pathlib import Path
+from subprocess import call
+
+if len(sys.argv) != 6:
+    print("Usage: " + sys.argv[0] + " <year (2016)> <month (01)> <day (31)> <username> <password>")
+    sys.exit()
+
+with open('logstash_config.example') as infile:
+	example = infile.read()
+
+output = example % { "timestamp": sys.argv[3] + '/' + sys.argv[2] + '/' + sys.argv[1] + ' 00:00:00'
+					 , "date": sys.argv[1] + '-' + sys.argv[2]
+					 , "user": sys.argv[4]
+					 , "password": sys.argv[5] }
+
+with open('logstash_configs/config-' + sys.argv[1] + '-' + sys.argv[2], 'w') as outfile:
+	outfile.write(output)

scripts/expenses/insert_expenses.sh

+#!/bin/bash
+
+if [ "$#" -ne 5 ]; then
+	echo "Usage: $0 <year> <month> <day> <user> <password>"
+	echo "Example: $0 2016 12 31 myuser mypass"
+	exit
+fi
+
+./create_expenses_config.py $1 $2 $3 $4 $5
+logstash -f logstash_configs/config-${1}-${2} < ~/transparencia/Favorecidos/Processados/${1}${2}.csv

scripts/expenses/logstash_config.example

+input {
+	stdin {
+		codec => plain {
+			charset => "Windows-1252"
+		}
+	}
+}
+
+filter {
+	csv {
+		columns => ["Código Órgão Superior","Nome Órgão Superior","Código Órgão","Nome Órgao","Código Unidade Gestora","Nome Unidade Gestora","Código Grupo Despesa","Nome Grupo Despesa","Código Elemento Despesa","Nome Elemento Despesa","Código Função","Nome Função","Código Subfunção","Nome Subfunção","Código Programa","Nome Programa","Código Ação","Nome Ação","Linguagem Cidadã","Código Favorecido","Nome Favorecido","Número Documento","Gestão Pagamento","Data Pagamento","Valor"]
+		separator => "	"
+		add_field => { "timestamp" => "%(timestamp)s" }
+	}
+	mutate {
+		convert => { "Código Órgão Superior" => "integer" }
+		convert => { "Código Órgão" => "integer" }
+		convert => { "Código Unidade Gestora" => "integer" }
+		convert => { "Código Grupo Despesa" => "integer" }
+		convert => { "Código Elemento Despesa" => "integer" }
+		convert => { "Código Função" => "integer" }
+		convert => { "Código Subfunção" => "integer" }
+		convert => { "Código Programa" => "integer" }
+		convert => { "Código Ação" => "integer" }
+		convert => { "Código Favorecido" => "integer" }
+		convert => { "Gestão Pagamento" => "integer" }
+		convert => { "Valor" => "float" }
+	}
+	date {
+		match => [ "timestamp", "dd/MM/YYYY HH:mm:ss", "ISO8601" ]
+		target => [ "@timestamp" ]
+	}
+}
+
+output {
+	elasticsearch {
+		action => "index"
+		user => "%(user)s"
+		password => "%(password)s"
+		hosts => "http://node1.c3sl.ufpr.br:9200"
+		index => "ufpr-gastos-%(date)s"
+		workers => 1
+	}
+	stdout {}
+}