Commit 3afadcab authored by Rafael Ravedutti's avatar Rafael Ravedutti

Adjust citizen uploads for also parsing the files

Signed-off-by: Rafael Ravedutti's avatarRafael Ravedutti <rrlmachado@inf.ufpr.br>
parent 2f737b52
Pipeline #16425 passed with stages
in 3 minutes and 17 seconds
module Api::V1
class CitizenUploadsController < ApplicationController
include Authenticable
include HasPolicies
require 'csv'
before_action :set_citizen_upload, only: [:show, :destroy]
# GET /citizen_uploads
def index
# Current citizen id
citizen_id = current_user[0][:id]
# Permission
permission = Professional.get_permission(current_user[1])
@resource_booking = policy_scope(ResourceBooking.filter(
params[:q], params[:page], params[:permission]))
# Find uploads for current citizen
@uploads = policy_scope(CitizenUpload.filter(
params[:q], params[:page], permission))
# Check if uploads are not null for current citizen
if @uploads.nil?
render json: {
errors: ["You don't have the permission to view citizen uploads."]
}, status: 403
return
end
# Sort uploads by date in descending order
@uploads = @uploads.order("created_at DESC")
# Create response object
response = Hash.new
response[:num_entries] = @uploads.total_count
response[:entries] = @uploads.as_json
# Render uploads in JSON format
render json: response.to_json
end
# GET /citizen_uploads/1
def show
if @upload_id.nil?
render json: {
errors: ["Upload task #{params[:upload_id]} does not exist."]
}, status: 404
else
begin
authorize @upload_id, :show?
rescue
render json: {
errors: ["You're not allowed to view this citizen upload log."]
}, status: 403
return
end
# Upload log path
path = @upload_id.log.path
# If log not found, displays not found message
if path.nil?
render json: {
errors: ["Log not found for current task."]
}, status: 404
# Otherwise, send file
else
send_file path,
type: @upload_id.log_content_type,
disposition: 'inline'
end
end
end
# POST /citizen_uploads
def create
# Current citizen id
citizen = current_user[0]
# Current citizen id
citizen_id = citizen[:id]
# City for current permission
city_id = citizen.professional.professionals_service_places.find(
current_user[1]).service_place.city_id
# Data must be defined in the parameters
if params.has_key?(:file) and params[:file].present?
# File content
content = params[:file].read
# Create upload object
upload_object = CitizenUpload.new({
citizen_id: citizen_id,
status: 0, # ready to start
amount: 0,
progress: 0.0
})
begin
authorize upload_object, :create?
rescue
render json: {
errors: ["You're not allowed to perform citizen uploads."]
}, status: 403
return
end
# Save upload object in the database
upload_object.save()
# Create sidekiq job for uploading the citizens
CitizenUploadWorker.perform_async(upload_object.id, content, city_id)
render json: {
errors: ["Citizens scheduled to be imported!"]
}, status: 201
else
render json: {
errors: ["Undefined citizens to import."]
}, status: 404
end
end
# DELETE /citizen_uploads/1
def destroy
end
private
# Use callbacks to share common setup or constraints between actions.
def set_citizen_upload
begin
@upload_id = CitizenUpload.find(params[:id])
rescue
@upload_id = nil
end
end
# Only allow a trusted parameter "white list" through.
def citizen_upload_params
params.permit(:file)
end
end
end
......@@ -164,94 +164,6 @@ module Api::V1
end
end
# GET /citizens/upload_log/1
def get_upload_log
# Find uploads for current citizen
@upload = CitizenUpload.find(params[:upload_id])
if @upload.nil?
render json: {
errors: ["Upload task #{params[:upload_id]} does not exist."]
}, status: 404
else
# Upload log path
path = @upload.log.path
# If log not found, displays not found message
if path.nil?
render json: {
errors: ["Log not found for current task."]
}, status: 404
# Otherwise, send file
else
send_file path,
type: @upload.log_content_type,
disposition: 'inline'
end
end
end
# GET /citizens/upload
def get_uploads
# Current citizen id
citizen_id = current_user[0][:id]
# Permission
permission = Professional.get_permission(current_user[1])
# Find uploads for current citizen
@uploads = CitizenUpload.where(citizen_id: citizen_id)
.order("created_at DESC")
.filter(params[:q], params[:page], permission)
# Create response object
response = Hash.new
response[:num_entries] = @uploads.total_count
response[:entries] = @uploads.as_json
# Render uploads in JSON format
render json: response.to_json
end
# POST /citizens/upload
def upload
# Current citizen id
citizen_id = current_user[0][:id]
# Data must be defined in the parameters
if params.has_key?(:data) and params[:data].present?
# Number of citizens to upload
upload_size = params[:data].size
# Create upload object
upload_object = CitizenUpload.new({
citizen_id: citizen_id,
status: 0, # ready to start
amount: upload_size,
progress: 0.0
})
# Save upload object in the database
upload_object.save()
# Create sidekiq job for uploading the citizens
CitizenUploadWorker.perform_async(
upload_object.id, upload_size, params[:data])
render json: {
errors: ["Citizens scheduled to be imported!"]
}, status: 201
else
render json: {
errors: ["Undefined citizens to import."]
}, status: 404
end
end
# PATCH/PUT /citizens/1
def update
if @citizen.nil?
......
class CitizenUploadPolicy < ApplicationPolicy
class Scope < Scope
def resolve
citizen = user[0]
permission = Professional.get_permission(user[1])
return case permission
when "adm_c3sl"
scope.all
when "adm_prefeitura"
scope.where(citizen_id: citizen.id)
else
nil
end
end
end
def show?
citizen = user[0]
permission = Professional.get_permission(user[1])
return case
when permission == "adm_c3sl"
return true
when permission == "adm_prefeitura"
return true
else
return false
end
end
def create?
citizen = user[0]
permission = Professional.get_permission(user[1])
return case
when permission == "adm_c3sl"
return true
when permission == "adm_prefeitura"
return (record.citizen_id == citizen.id)
else
return false
end
end
end
class CitizenUploadWorker
include Sidekiq::Worker
require 'csv'
sidekiq_options :queue => :citizens_upload
def perform(upload_id, upload_size, citizens)
def perform(upload_id, content, city_id)
# Batch size for upload
batch_size = 100
......@@ -22,6 +23,21 @@ class CitizenUploadWorker
# Columns for accounts
account_columns = [:uid, :provider, :encrypted_password]
# Update task status to in progress
CitizenUpload.update(
upload_id,
status: 1 # parsing content
)
# Parse citizens from the CSV data
citizens = CSV.parse(content).map { |row| Hash[columns.zip(row)] }
# Remove headers
citizens = citizens.drop(1)
# Number of citizens to be uploaded
upload_size = citizens.length
# Line number starts with one
line_number = 1
# Hash with errors
......@@ -34,15 +50,16 @@ class CitizenUploadWorker
# Update task status to in progress
CitizenUpload.update(
upload_id,
status: 1 # in progress
status: 2 # in progress
)
# Go through each citizen in the list
citizens.each do |c|
# Parameters for current line
upload_params = Hash[columns.zip(c)]
citizens.each do |citizen_params|
# Set active parameter
citizen_params[:active] = true
# Create citizen object with defined parameters
citizen = Citizen.new(upload_params)
citizen = Citizen.new(citizen_params)
# Create account object with defined parameters
account = Account.new({
......@@ -55,37 +72,43 @@ class CitizenUploadWorker
# Citizen remaining info is added when .valid? method is called
if citizen.valid? and account.valid?
# Add valid citizen with complete info to to_create array
inst = [
citizen.name,
citizen.cpf,
citizen.rg,
citizen.birth_date,
citizen.cep,
citizen.address_street,
citizen.address_number,
citizen.neighborhood,
citizen.address_complement,
citizen.city_id,
citizen.phone1,
citizen.phone2,
citizen.email,
citizen.pcd,
citizen.note,
citizen.active
]
# Add valid account with complete info to to_create array
acc_inst = [
account.uid,
account.provider,
account.encrypted_password
]
# Insert current citizen data to buffer of citizens to create
to_create.append(inst)
# Insert current account data to buffer of accounts to create
account_to_create.append(acc_inst)
if citizen.city_id != city_id
# If there was a permission error, store it in the errors hash
errors[line_number.to_s] = "Permission denied for this city!"
else
# Add valid citizen with complete info to to_create array
inst = [
citizen.name,
citizen.cpf,
citizen.rg,
citizen.birth_date,
citizen.cep,
citizen.address_street,
citizen.address_number,
citizen.neighborhood,
citizen.address_complement,
citizen.city_id,
citizen.phone1,
citizen.phone2,
citizen.email,
citizen.pcd,
citizen.note,
citizen.active
]
# Add valid account with complete info to to_create array
acc_inst = [
account.uid,
account.provider,
account.encrypted_password
]
# Insert current citizen data to buffer of citizens to create
to_create.append(inst)
# Insert current account data to buffer of accounts to create
account_to_create.append(acc_inst)
end
else
# If there was an error, store it in the errors hash
errors[line_number.to_s] = citizen.errors.to_hash
......@@ -138,11 +161,11 @@ class CitizenUploadWorker
end
# New status to update
new_status = 2 # completed with no errors
new_status = 3 # completed with no errors
# If there were errors, change status to completed with errors
if errors.size > 0
new_status = 3 # completed with errors
new_status = 4 # completed with errors
end
# Update upload object progress
......
......@@ -14,9 +14,6 @@ Rails.application.routes.draw do
get "accounts/self" => "accounts#index"
get "citizens/schedule_options" => "citizens#schedule_options"
get "citizens/upload" => "citizens#get_uploads"
get "citizens/upload_log/:upload_id" => "citizens#get_upload_log"
post "citizens/upload" => "citizens#upload"
resources :citizens do
resources :dependants
......@@ -39,6 +36,8 @@ Rails.application.routes.draw do
end
end
resources :citizen_uploads
resources :occupations
resources :professionals
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment