Skip to content
Snippets Groups Projects
Commit 98624a6d authored by Cristian Weiland's avatar Cristian Weiland
Browse files

Merge branch 'issue/14' into 'master' 

Issue #20: Create master script to insert data and remove day parameter from insert_expenses.sh

See merge request !3
parents 8c27487e 1d347d73
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
scripts/curl/delete_index.sh 0 → 100755
+ 6
− 0
View file @ 98624a6d
if [ "$#" -ne 1 ]; then
echo "Usage: $0 <index-name>"
exit
fi
curl -XDELETE "localhost:9200/$1?pretty"
scripts/expenses/insert_expenses.sh
+ 12
− 5
View file @ 98624a6d
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
# This script is the one that should be called to insert data from one month. # This script is the one that should be called to insert data from one month.
# Input: Year, month and day from the data to be inserted, ElasticSearch's user and password. The day should be the last day of the month. # Input: Year, month and day from the data to be inserted, ElasticSearch's user and password. The day should be the last day of the month.
# Example: ./insert_expenses.sh 2016 10 31 myuser mypass # Example: ./insert_expenses.sh 2016 10 myuser mypass
# It has 4 steps: # It has 4 steps:
# 1- Download files and put them in the right location. # 1- Download files and put them in the right location.
# 2- Generate logstash config file via create_expenses_config.py. # 2- Generate logstash config file via create_expenses_config.py.
...@@ -11,12 +11,19 @@ ...@@ -11,12 +11,19 @@
# 4- Insert data in ElasticSearch via logstash, using the config file created and the CSV created by resume_expenses.sh. # 4- Insert data in ElasticSearch via logstash, using the config file created and the CSV created by resume_expenses.sh.
# Output: The commands/scripts outputs. # Output: The commands/scripts outputs.
if [ "$#" -ne 5 ]; then if [ "$#" -ne 4 ]; then
echo "Usage: $0 <year> <month> <day> <user> <password>" echo "Usage: $0 <year> <month> <user> <password>"
echo "Example: $0 2016 12 31 myuser mypass" echo "Example: $0 2016 12 myuser mypass"
exit exit
fi fi
# Getting the Last day of this month (Using date 2016-05-15 as example):
# First, get next month (201606).
aux=$(date +%Y%m -d "$(date +%Y%m15) next month")
# Append day 01 (20160601).
temp=$(date -d "${aux}01")
# Remove 1 day: 20160531, get only day: 31.
day=$(date -d "$temp - 1 day" "+%d")
ym=$1-$2 ym=$1-$2
dataPath="../../data/" dataPath="../../data/"
...@@ -44,7 +51,7 @@ unzip $path$ym/${1}${2}_GastosDiretos.zip -d $path$ym/ ...@@ -44,7 +51,7 @@ unzip $path$ym/${1}${2}_GastosDiretos.zip -d $path$ym/
rm $path$ym/${1}${2}_GastosDiretos.zip rm $path$ym/${1}${2}_GastosDiretos.zip
# Step 2: # Step 2:
./create_expenses_config.py $1 $2 $3 $4 $5 ./create_expenses_config.py $1 $2 $day $3 $4
# Step 3: # Step 3:
./resume_expenses.sh ../../data/expenses/ ${1}-${2} ./resume_expenses.sh ../../data/expenses/ ${1}-${2}
# Step 4: # Step 4:
......
scripts/expenses/logstash_config.example
+ 6
− 2
View file @ 98624a6d
...@@ -30,6 +30,10 @@ filter { ...@@ -30,6 +30,10 @@ filter {
match => [ "timestamp", "dd/MM/YYYY HH:mm:ss", "ISO8601" ] match => [ "timestamp", "dd/MM/YYYY HH:mm:ss", "ISO8601" ]
target => [ "@timestamp" ] target => [ "@timestamp" ]
} }
date {
match => [ "Data Pagamento", "dd/MM/YYYY" ]
target => [ "Data Pagamento Timestamp" ]
}
} }
output { output {
...@@ -37,8 +41,8 @@ output { ...@@ -37,8 +41,8 @@ output {
action => "index" action => "index"
user => "%(user)s" user => "%(user)s"
password => "%(password)s" password => "%(password)s"
hosts => "http://node1.c3sl.ufpr.br:9200" hosts => "localhost:9200"
index => "ufpr-gastos-%(date)s" index => "ufpr-gastos-pagamentos-%(date)s"
workers => 1 workers => 1
} }
stdout {} stdout {}
......
scripts/expenses/process_expenses.sh
+ 47
− 47
View file @ 98624a6d
...@@ -12,53 +12,53 @@ if [ "$#" -ne 2 ]; then ...@@ -12,53 +12,53 @@ if [ "$#" -ne 2 ]; then
exit exit
fi fi
./insert_expenses.sh 2016 11 30 $1 $2 ./insert_expenses.sh 2016 11 $1 $2
./insert_expenses.sh 2016 10 31 $1 $2 ./insert_expenses.sh 2016 10 $1 $2
./insert_expenses.sh 2016 09 30 $1 $2 ./insert_expenses.sh 2016 09 $1 $2
./insert_expenses.sh 2016 08 31 $1 $2 ./insert_expenses.sh 2016 08 $1 $2
./insert_expenses.sh 2016 07 31 $1 $2 ./insert_expenses.sh 2016 07 $1 $2
./insert_expenses.sh 2016 06 30 $1 $2 ./insert_expenses.sh 2016 06 $1 $2
./insert_expenses.sh 2016 05 31 $1 $2 ./insert_expenses.sh 2016 05 $1 $2
./insert_expenses.sh 2016 04 30 $1 $2 ./insert_expenses.sh 2016 04 $1 $2
./insert_expenses.sh 2016 03 31 $1 $2 ./insert_expenses.sh 2016 03 $1 $2
./insert_expenses.sh 2016 02 29 $1 $2 ./insert_expenses.sh 2016 02 $1 $2
./insert_expenses.sh 2016 01 31 $1 $2 ./insert_expenses.sh 2016 01 $1 $2
./insert_expenses.sh 2015 12 31 $1 $2 ./insert_expenses.sh 2015 12 $1 $2
./insert_expenses.sh 2015 11 30 $1 $2 ./insert_expenses.sh 2015 11 $1 $2
./insert_expenses.sh 2015 10 31 $1 $2 ./insert_expenses.sh 2015 10 $1 $2
./insert_expenses.sh 2015 09 30 $1 $2 ./insert_expenses.sh 2015 09 $1 $2
./insert_expenses.sh 2015 08 31 $1 $2 ./insert_expenses.sh 2015 08 $1 $2
./insert_expenses.sh 2015 07 31 $1 $2 ./insert_expenses.sh 2015 07 $1 $2
./insert_expenses.sh 2015 06 30 $1 $2 ./insert_expenses.sh 2015 06 $1 $2
./insert_expenses.sh 2015 05 31 $1 $2 ./insert_expenses.sh 2015 05 $1 $2
./insert_expenses.sh 2015 04 30 $1 $2 ./insert_expenses.sh 2015 04 $1 $2
./insert_expenses.sh 2015 03 31 $1 $2 ./insert_expenses.sh 2015 03 $1 $2
./insert_expenses.sh 2015 02 28 $1 $2 ./insert_expenses.sh 2015 02 $1 $2
./insert_expenses.sh 2015 01 31 $1 $2 ./insert_expenses.sh 2015 01 $1 $2
./insert_expenses.sh 2014 12 31 $1 $2 ./insert_expenses.sh 2014 12 $1 $2
./insert_expenses.sh 2014 11 30 $1 $2 ./insert_expenses.sh 2014 11 $1 $2
./insert_expenses.sh 2014 10 31 $1 $2 ./insert_expenses.sh 2014 10 $1 $2
./insert_expenses.sh 2014 09 30 $1 $2 ./insert_expenses.sh 2014 09 $1 $2
./insert_expenses.sh 2014 08 31 $1 $2 ./insert_expenses.sh 2014 08 $1 $2
./insert_expenses.sh 2014 07 31 $1 $2 ./insert_expenses.sh 2014 07 $1 $2
./insert_expenses.sh 2014 06 30 $1 $2 ./insert_expenses.sh 2014 06 $1 $2
./insert_expenses.sh 2014 05 31 $1 $2 ./insert_expenses.sh 2014 05 $1 $2
./insert_expenses.sh 2014 04 30 $1 $2 ./insert_expenses.sh 2014 04 $1 $2
./insert_expenses.sh 2014 03 31 $1 $2 ./insert_expenses.sh 2014 03 $1 $2
./insert_expenses.sh 2014 02 28 $1 $2 ./insert_expenses.sh 2014 02 $1 $2
./insert_expenses.sh 2014 01 31 $1 $2 ./insert_expenses.sh 2014 01 $1 $2
./insert_expenses.sh 2013 12 31 $1 $2 ./insert_expenses.sh 2013 12 $1 $2
./insert_expenses.sh 2013 11 30 $1 $2 ./insert_expenses.sh 2013 11 $1 $2
./insert_expenses.sh 2013 10 31 $1 $2 ./insert_expenses.sh 2013 10 $1 $2
./insert_expenses.sh 2013 09 30 $1 $2 ./insert_expenses.sh 2013 09 $1 $2
./insert_expenses.sh 2013 08 31 $1 $2 ./insert_expenses.sh 2013 08 $1 $2
./insert_expenses.sh 2013 07 31 $1 $2 ./insert_expenses.sh 2013 07 $1 $2
./insert_expenses.sh 2013 06 30 $1 $2 ./insert_expenses.sh 2013 06 $1 $2
./insert_expenses.sh 2013 05 31 $1 $2 ./insert_expenses.sh 2013 05 $1 $2
./insert_expenses.sh 2013 04 30 $1 $2 ./insert_expenses.sh 2013 04 $1 $2
./insert_expenses.sh 2013 03 31 $1 $2 ./insert_expenses.sh 2013 03 $1 $2
./insert_expenses.sh 2013 02 28 $1 $2 ./insert_expenses.sh 2013 02 $1 $2
./insert_expenses.sh 2013 01 31 $1 $2 ./insert_expenses.sh 2013 01 $1 $2
scripts/insert_data.sh 0 → 100755
+ 16
− 0
View file @ 98624a6d
# This script will call every script needed to insert data.
if [ "$#" -ne 4 ]; then
echo "Usage: $0 <year> <month> <user> <password>"
echo "Example: $0 2016 12 myuser mypass"
exit
fi
# First, insert Expenses data.
(cd expenses && ./insert_expenses.sh $1 $2 $3 $4)
# Now, insert Workers data.
(cd workers && ./insert_register_payment.sh $1 $2 $3 $4)
# We should now insert Travel allowance data.
(cd travel_allowances && ./insert_travel_allowances.sh $1 $2 $3 $4)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment