Commit 1d347d73 authored by Cristian Weiland's avatar Cristian Weiland

Issue #20: Create master script to insert data and remove day parameter from insert_expenses.sh

Signed-off-by: Cristian Weiland's avatarCristian Weiland <cw14@inf.ufpr.br>
parent 5d2b38e9
if [ "$#" -ne 1 ]; then
echo "Usage: $0 <index-name>"
exit
fi
curl -XDELETE "localhost:9200/$1?pretty"
......@@ -3,7 +3,7 @@
# This script is the one that should be called to insert data from one month.
# Input: Year, month and day from the data to be inserted, ElasticSearch's user and password. The day should be the last day of the month.
# Example: ./insert_expenses.sh 2016 10 31 myuser mypass
# Example: ./insert_expenses.sh 2016 10 myuser mypass
# It has 4 steps:
# 1- Download files and put them in the right location.
# 2- Generate logstash config file via create_expenses_config.py.
......@@ -11,12 +11,19 @@
# 4- Insert data in ElasticSearch via logstash, using the config file created and the CSV created by resume_expenses.sh.
# Output: The commands/scripts outputs.
if [ "$#" -ne 5 ]; then
echo "Usage: $0 <year> <month> <day> <user> <password>"
echo "Example: $0 2016 12 31 myuser mypass"
if [ "$#" -ne 4 ]; then
echo "Usage: $0 <year> <month> <user> <password>"
echo "Example: $0 2016 12 myuser mypass"
exit
fi
# Getting the Last day of this month (Using date 2016-05-15 as example):
# First, get next month (201606).
aux=$(date +%Y%m -d "$(date +%Y%m15) next month")
# Append day 01 (20160601).
temp=$(date -d "${aux}01")
# Remove 1 day: 20160531, get only day: 31.
day=$(date -d "$temp - 1 day" "+%d")
ym=$1-$2
dataPath="../../data/"
......@@ -44,7 +51,7 @@ unzip $path$ym/${1}${2}_GastosDiretos.zip -d $path$ym/
rm $path$ym/${1}${2}_GastosDiretos.zip
# Step 2:
./create_expenses_config.py $1 $2 $3 $4 $5
./create_expenses_config.py $1 $2 $day $3 $4
# Step 3:
./resume_expenses.sh ../../data/expenses/ ${1}-${2}
# Step 4:
......
......@@ -30,6 +30,10 @@ filter {
match => [ "timestamp", "dd/MM/YYYY HH:mm:ss", "ISO8601" ]
target => [ "@timestamp" ]
}
date {
match => [ "Data Pagamento", "dd/MM/YYYY" ]
target => [ "Data Pagamento Timestamp" ]
}
}
output {
......@@ -37,8 +41,8 @@ output {
action => "index"
user => "%(user)s"
password => "%(password)s"
hosts => "http://node1.c3sl.ufpr.br:9200"
index => "ufpr-gastos-%(date)s"
hosts => "localhost:9200"
index => "ufpr-gastos-pagamentos-%(date)s"
workers => 1
}
stdout {}
......
......@@ -12,53 +12,53 @@ if [ "$#" -ne 2 ]; then
exit
fi
./insert_expenses.sh 2016 11 30 $1 $2
./insert_expenses.sh 2016 10 31 $1 $2
./insert_expenses.sh 2016 09 30 $1 $2
./insert_expenses.sh 2016 08 31 $1 $2
./insert_expenses.sh 2016 07 31 $1 $2
./insert_expenses.sh 2016 06 30 $1 $2
./insert_expenses.sh 2016 05 31 $1 $2
./insert_expenses.sh 2016 04 30 $1 $2
./insert_expenses.sh 2016 03 31 $1 $2
./insert_expenses.sh 2016 02 29 $1 $2
./insert_expenses.sh 2016 01 31 $1 $2
./insert_expenses.sh 2016 11 $1 $2
./insert_expenses.sh 2016 10 $1 $2
./insert_expenses.sh 2016 09 $1 $2
./insert_expenses.sh 2016 08 $1 $2
./insert_expenses.sh 2016 07 $1 $2
./insert_expenses.sh 2016 06 $1 $2
./insert_expenses.sh 2016 05 $1 $2
./insert_expenses.sh 2016 04 $1 $2
./insert_expenses.sh 2016 03 $1 $2
./insert_expenses.sh 2016 02 $1 $2
./insert_expenses.sh 2016 01 $1 $2
./insert_expenses.sh 2015 12 31 $1 $2
./insert_expenses.sh 2015 11 30 $1 $2
./insert_expenses.sh 2015 10 31 $1 $2
./insert_expenses.sh 2015 09 30 $1 $2
./insert_expenses.sh 2015 08 31 $1 $2
./insert_expenses.sh 2015 07 31 $1 $2
./insert_expenses.sh 2015 06 30 $1 $2
./insert_expenses.sh 2015 05 31 $1 $2
./insert_expenses.sh 2015 04 30 $1 $2
./insert_expenses.sh 2015 03 31 $1 $2
./insert_expenses.sh 2015 02 28 $1 $2
./insert_expenses.sh 2015 01 31 $1 $2
./insert_expenses.sh 2015 12 $1 $2
./insert_expenses.sh 2015 11 $1 $2
./insert_expenses.sh 2015 10 $1 $2
./insert_expenses.sh 2015 09 $1 $2
./insert_expenses.sh 2015 08 $1 $2
./insert_expenses.sh 2015 07 $1 $2
./insert_expenses.sh 2015 06 $1 $2
./insert_expenses.sh 2015 05 $1 $2
./insert_expenses.sh 2015 04 $1 $2
./insert_expenses.sh 2015 03 $1 $2
./insert_expenses.sh 2015 02 $1 $2
./insert_expenses.sh 2015 01 $1 $2
./insert_expenses.sh 2014 12 31 $1 $2
./insert_expenses.sh 2014 11 30 $1 $2
./insert_expenses.sh 2014 10 31 $1 $2
./insert_expenses.sh 2014 09 30 $1 $2
./insert_expenses.sh 2014 08 31 $1 $2
./insert_expenses.sh 2014 07 31 $1 $2
./insert_expenses.sh 2014 06 30 $1 $2
./insert_expenses.sh 2014 05 31 $1 $2
./insert_expenses.sh 2014 04 30 $1 $2
./insert_expenses.sh 2014 03 31 $1 $2
./insert_expenses.sh 2014 02 28 $1 $2
./insert_expenses.sh 2014 01 31 $1 $2
./insert_expenses.sh 2014 12 $1 $2
./insert_expenses.sh 2014 11 $1 $2
./insert_expenses.sh 2014 10 $1 $2
./insert_expenses.sh 2014 09 $1 $2
./insert_expenses.sh 2014 08 $1 $2
./insert_expenses.sh 2014 07 $1 $2
./insert_expenses.sh 2014 06 $1 $2
./insert_expenses.sh 2014 05 $1 $2
./insert_expenses.sh 2014 04 $1 $2
./insert_expenses.sh 2014 03 $1 $2
./insert_expenses.sh 2014 02 $1 $2
./insert_expenses.sh 2014 01 $1 $2
./insert_expenses.sh 2013 12 31 $1 $2
./insert_expenses.sh 2013 11 30 $1 $2
./insert_expenses.sh 2013 10 31 $1 $2
./insert_expenses.sh 2013 09 30 $1 $2
./insert_expenses.sh 2013 08 31 $1 $2
./insert_expenses.sh 2013 07 31 $1 $2
./insert_expenses.sh 2013 06 30 $1 $2
./insert_expenses.sh 2013 05 31 $1 $2
./insert_expenses.sh 2013 04 30 $1 $2
./insert_expenses.sh 2013 03 31 $1 $2
./insert_expenses.sh 2013 02 28 $1 $2
./insert_expenses.sh 2013 01 31 $1 $2
./insert_expenses.sh 2013 12 $1 $2
./insert_expenses.sh 2013 11 $1 $2
./insert_expenses.sh 2013 10 $1 $2
./insert_expenses.sh 2013 09 $1 $2
./insert_expenses.sh 2013 08 $1 $2
./insert_expenses.sh 2013 07 $1 $2
./insert_expenses.sh 2013 06 $1 $2
./insert_expenses.sh 2013 05 $1 $2
./insert_expenses.sh 2013 04 $1 $2
./insert_expenses.sh 2013 03 $1 $2
./insert_expenses.sh 2013 02 $1 $2
./insert_expenses.sh 2013 01 $1 $2
# This script will call every script needed to insert data.
if [ "$#" -ne 4 ]; then
echo "Usage: $0 <year> <month> <user> <password>"
echo "Example: $0 2016 12 myuser mypass"
exit
fi
# First, insert Expenses data.
(cd expenses && ./insert_expenses.sh $1 $2 $3 $4)
# Now, insert Workers data.
(cd workers && ./insert_register_payment.sh $1 $2 $3 $4)
# We should now insert Travel allowance data.
(cd travel_allowances && ./insert_travel_allowances.sh $1 $2 $3 $4)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment