Commit 93c9c2c5 authored by Bruno Meyer's avatar Bruno Meyer
Browse files

sistema de upload funcionando (para xls pelo menos)

parent 76267383
cache/*
*.json
*.xls
*.csv
script/base/*.xls
script/base/*.csv
import numpy as np
from script.utils.situations import *
ANO_ATUAL = 2017
SEMESTRE_ATUAL = 2
def listagem_turma_ingresso(df):
#~ print(df.groupby(["ANO_INGRESSO", "SEMESTRE_INGRESSO"]).groups)
grupos = df.groupby(["ANO_INGRESSO", "SEMESTRE_INGRESSO"]).groups
for t in grupos:
print(t)
print("\n\n")
print(df["FORMA_INGRESSO"][grupos[t]].drop_duplicates())
def listagem_alunos_ativos(df):
return list(df["MATR_ALUNO"][df["FORMA_EVASAO"] == EvasionForm.EF_ATIVO].drop_duplicates())
def posicao_turmaIngresso_semestral(df):
iras = ira_semestral(df)
iraMax = {}
for matr in iras:
for semestreAno in iras[matr]:
if not (semestreAno in iraMax):
iraMax[semestreAno] = iras[matr][semestreAno]
else:
if (iras[matr][semestreAno] > iraMax[semestreAno]):
iraMax[semestreAno] = iras[matr][semestreAno]
for matr in iras:
for semestreAno in iras[matr]:
iras[matr][semestreAno] /= iraMax[semestreAno]
return iras
def periodo_real(df):
aux = df.groupby(["MATR_ALUNO"])
students = {}
for x in aux:
students[x[0]] = None
return students
def periodo_pretendido(df):
aux = df.groupby(["MATR_ALUNO", "ANO_INGRESSO", "SEMESTRE_INGRESSO"])
students = {}
for x in aux:
students[x[0][0]] = (ANO_ATUAL - int(x[0][1])) * 2 + SEMESTRE_ATUAL - int(x[0][2]) + 1
return students
def ira_semestral(df):
aux = ira_por_quantidade_disciplinas(df)
for matr in aux:
for periodo in aux[matr]:
aux[matr][periodo] = aux[matr][periodo][0]
return aux
def ira_por_quantidade_disciplinas(df):
students = {}
df = df.dropna(subset=["MEDIA_FINAL"])
total_students = len(df["MATR_ALUNO"])
for i in range(total_students):
matr = (df["MATR_ALUNO"][i])
if (not (matr in students)):
students[matr] = {}
ano = str(int(df["ANO"][i]))
semestre = str(df["PERIODO"][i])
situacao = int(df["SITUACAO"][i])
nota = float(df["MEDIA_FINAL"][i])
media_credito = int(df["MEDIA_CREDITO"][i])
if (situacao in Situation.SITUATION_AFFECT_IRA and media_credito != 0):
if not (ano + "/" + semestre in students[matr]):
students[matr][ano + "/" + semestre] = [0, 0]
students[matr][ano + "/" + semestre][0] += nota
students[matr][ano + "/" + semestre][1] += 1
for matr in students:
for periodo in students[matr]:
if (students[matr][periodo][1] != 0):
students[matr][periodo][0] /= students[matr][periodo][1] * 100
return (students)
def indice_aprovacao_semestral(df):
students = {}
df = df.dropna(subset=['MEDIA_FINAL'])
total_students = len(df["MATR_ALUNO"])
for i in range(total_students):
matr = (df["MATR_ALUNO"][i])
if (not (matr in students)):
students[matr] = {}
ano = str(int(df["ANO"][i]))
semestre = str(df["PERIODO"][i])
situacao = int(df["SITUACAO"][i])
if not (ano + "/" + semestre in students[matr]):
students[matr][ano + "/" + semestre] = [0, 0]
if situacao in Situation.SITUATION_PASS:
students[matr][ano + "/" + semestre][0] += 1
students[matr][ano + "/" + semestre][1] += 1
if situacao in Situation.SITUATION_FAIL:
students[matr][ano + "/" + semestre][1] += 1
return (students)
def aluno_turmas(df):
students = {}
df = df.dropna(subset=['MEDIA_FINAL'])
situations = dict(Situation.SITUATIONS)
for matr, hist in df.groupby('MATR_ALUNO'):
students[matr] = []
for _, row in hist.iterrows():
data = {
'ano': str(int(row["ANO"])),
'codigo': row["COD_ATIV_CURRIC"],
'nome': row["NOME_ATIV_CURRIC"],
'nota': row["MEDIA_FINAL"],
'semestre': row["PERIODO"],
'situacao': situations.get(row["SITUACAO"], Situation.SIT_OUTROS)
}
students[matr].append(data)
return students
import pandas as pd
import math
from script.utils.situations import Situation, EvasionForm
def average_graduation(df):
total_student = df['MATR_ALUNO'].drop_duplicates().shape[0]
total_graduate = df[df.FORMA_EVASAO == EvasionForm.EF_FORMATURA].shape[0]
return total_graduate / total_student
def general_failure(df):
affect_ira = df[df.SITUACAO.isin(Situation.SITUATION_AFFECT_IRA)]
failures = affect_ira[affect_ira.SITUACAO.isin(Situation.SITUATION_FAIL)]
average = failures.shape[0] / affect_ira.shape[0]
student_courses = affect_ira.groupby(['MATR_ALUNO'], as_index=False)\
.aggregate({'SITUACAO': 'count'})
student_failures = failures.groupby(['MATR_ALUNO'], as_index=False)\
.aggregate({'SITUACAO': 'count'})
merged = pd.merge(student_courses, student_failures, on=['MATR_ALUNO'])
merged.columns = ['MART_ALUNO', 'FEITAS', 'REPROVADO']
variance = merged['REPROVADO'].div(merged['FEITAS']).sub(average)\
.pow(2).sum() / merged.shape[0]
standard_deviation = math.sqrt(variance)
return (average, standard_deviation)
def general_ira(df):
fixed = df[df.SITUACAO.isin(Situation.SITUATION_AFFECT_IRA)]
fixed = fixed[fixed.MEDIA_FINAL <= 100]
return (fixed.MEDIA_FINAL.mean(), fixed.MEDIA_FINAL.std())
def total_evasion_rate(df):
students = df['MATR_ALUNO'].drop_duplicates()
total_student = students.shape[0]
total_evasion = students.loc[(df.FORMA_EVASAO != EvasionForm.EF_ATIVO) & (df.FORMA_EVASAO != EvasionForm.EF_FORMATURA) & (df.FORMA_EVASAO != EvasionForm.EF_REINTEGRACAO)].shape[0]
return total_evasion / total_student
def average_graduation_time(df):
graduates = df.loc[(df.FORMA_EVASAO == EvasionForm.EF_FORMATURA)]
total_graduate = graduates.shape[0]
average_time = 0
year_end = int(df['ANO'].max())
semester_end = graduates['PERIODO'].max()
for index, row in graduates.iterrows():
if pd.notnull(row['ANO_EVASAO']):
year_end = int(row['ANO_EVASAO'])
try:
semester_end = int(row['SEMESTRE_EVASAO'])
except ValueError:
semester_end = graduates['PERIODO'].max()
year = int(row['ANO_INGRESSO'])
semester = int(row['SEMESTRE_INGRESSO'])
difference = 2 * (year_end - year) + (semester_end - semester) + 1
average_time += difference
average_time /= total_graduate
average_time /= 2
return average_time
import numpy as np
from script.utils.situations import *
ANO_ATUAL = 2017
SEMESTRE_ATUAL = 2
def listagem_alunos_ativos(df):
return list(df["MATR_ALUNO"][df["FORMA_EVASAO"] == EvasionForm.EF_ATIVO].drop_duplicates())
def posicao_turmaIngresso_semestral(df):
iras = ira_semestral(df)
iraMax = {}
for matr in iras:
for semestreAno in iras[matr]:
if not (semestreAno in iraMax):
iraMax[semestreAno] = iras[matr][semestreAno]
else:
if (iras[matr][semestreAno] > iraMax[semestreAno]):
iraMax[semestreAno] = iras[matr][semestreAno]
for matr in iras:
for semestreAno in iras[matr]:
iras[matr][semestreAno] /= iraMax[semestreAno]
return iras
def periodo_real(df):
aux = df.groupby(["MATR_ALUNO"])
students = {}
for x in aux:
students[x[0]] = None
return students
def periodo_pretendido(df):
aux = df.groupby(["MATR_ALUNO", "ANO_INGRESSO", "SEMESTRE_INGRESSO"])
students = {}
for x in aux:
students[x[0][0]] = (ANO_ATUAL - int(x[0][1])) * 2 + SEMESTRE_ATUAL - int(x[0][2]) + 1
return students
def ira_semestral(df):
aux = ira_por_quantidade_disciplinas(df)
for matr in aux:
for periodo in aux[matr]:
aux[matr][periodo] = aux[matr][periodo][0]
return aux
def ira_por_quantidade_disciplinas(df):
students = {}
df = df.dropna(subset=["MEDIA_FINAL"])
total_students = len(df["MATR_ALUNO"])
for i in range(total_students):
matr = (df["MATR_ALUNO"][i])
if (not (matr in students)):
students[matr] = {}
ano = str(int(df["ANO"][i]))
semestre = str(df["PERIODO"][i])
situacao = int(df["SITUACAO"][i])
nota = float(df["MEDIA_FINAL"][i])
media_credito = int(df["MEDIA_CREDITO"][i])
if (situacao in Situation.SITUATION_AFFECT_IRA and media_credito != 0):
if not (ano + "/" + semestre in students[matr]):
students[matr][ano + "/" + semestre] = [0, 0]
students[matr][ano + "/" + semestre][0] += nota
students[matr][ano + "/" + semestre][1] += 1
for matr in students:
for periodo in students[matr]:
if (students[matr][periodo][1] != 0):
students[matr][periodo][0] /= students[matr][periodo][1] * 100
return (students)
def indice_aprovacao_semestral(df):
students = {}
df = df.dropna(subset=['MEDIA_FINAL'])
total_students = len(df["MATR_ALUNO"])
for i in range(total_students):
matr = (df["MATR_ALUNO"][i])
if (not (matr in students)):
students[matr] = {}
ano = str(int(df["ANO"][i]))
semestre = str(df["PERIODO"][i])
situacao = int(df["SITUACAO"][i])
if not (ano + "/" + semestre in students[matr]):
students[matr][ano + "/" + semestre] = [0, 0]
if situacao in Situation.SITUATION_PASS:
students[matr][ano + "/" + semestre][0] += 1
students[matr][ano + "/" + semestre][1] += 1
if situacao in Situation.SITUATION_FAIL:
students[matr][ano + "/" + semestre][1] += 1
return (students)
def aluno_turmas(df):
students = {}
df = df.dropna(subset=['MEDIA_FINAL'])
situations = dict(Situation.SITUATIONS)
for matr, hist in df.groupby('MATR_ALUNO'):
students[matr] = []
for _, row in hist.iterrows():
data = {
'ano': str(int(row["ANO"])),
'codigo': row["COD_ATIV_CURRIC"],
'nome': row["NOME_ATIV_CURRIC"],
'nota': row["MEDIA_FINAL"],
'semestre': row["PERIODO"],
'situacao': situations.get(row["SITUACAO"], Situation.SIT_OUTROS)
}
students[matr].append(data)
return students
import re
import os
import sys
import pandas as pd
import numpy as np
from glob import glob
from json import load as json_load
from script.utils.situations import *
class DataframeHolder:
def __init__(self, dataframe):
self.students = dataframe.groupby('MATR_ALUNO')
self.courses = dataframe.groupby('COD_ATIV_CURRIC')
self.admission = dataframe.groupby(['ANO_INGRESSO', 'SEMESTRE_INGRESSO'])
def load_dataframes(cwd='.'):
dataframes = []
for path, dirs, files in os.walk(cwd):
for f in files:
file_path = path + '/' + f
dh = {'name': f, 'dataframe': None}
if 'csv' in f:
dh['dataframe'] = read_csv(file_path)
if 'xls' in f:
dh['dataframe'] = read_excel(file_path)
if dh['dataframe'] is not None:
dataframes.append(dh)
dataframe = fix_dataframes(dataframes)
dh = DataframeHolder(dataframe)
#~ dh.students.aggregate(teste)
# print(dh.students['MEDIA_FINAL'].aggregate(teste))
return dataframe
def read_excel(path, planilha='Planilha1'):
return pd.read_excel(path)
def read_csv(path):
return pd.read_csv(path)
def fix_dataframes(dataframes):
for df in dataframes:
if df['name'] == 'historico.xls':
history = df['dataframe']
if df['name'] == 'matricula.xls':
register = df['dataframe']
clean_history(history)
clean_register(register)
merged = pd.merge(history, register, how='right', on=['MATR_ALUNO'])
#~ print(merged)
fix_situation(merged)
# fix_admission(merged)
fix_evasion(merged)
return merged
def clean_history(df):
df.drop(['ID_NOTA', 'CONCEITO', 'ID_LOCAL_DISPENSA', 'SITUACAO_CURRICULO',
'ID_CURSO_ALUNO', 'ID_VERSAO_CURSO', 'ID_CURRIC_ALUNO',
'ID_ATIV_CURRIC', 'SITUACAO_ITEM', 'ID_ESTRUTURA_CUR'
], axis=1, inplace=True)
df['PERIODO'] = df['PERIODO'].str.split('o').str[0]
def clean_register(df):
df_split = df['PERIODO_INGRESSO'].str.split('/')
df['ANO_INGRESSO'] = df_split.str[0]
df['SEMESTRE_INGRESSO'] = df_split.str[1].str.split('o').str[0]
df_split = df['PERIODO_EVASAO'].str.split('/')
df['ANO_EVASAO'] = df_split.str[0]
df['SEMESTRE_EVASAO'] = df_split.str[1].str.split('o').str[0]
df.drop(['ID_PESSOA', 'NOME_PESSOA', 'DT_NASCIMENTO', 'NOME_UNIDADE',
'COD_CURSO', 'NUM_VERSAO', 'PERIODO_INGRESSO', 'PERIODO_EVASAO',
],axis=1, inplace=True)
def fix_situation(df):
for situation in Situation.SITUATIONS:
df.loc[df.SITUACAO == situation[1], 'SITUACAO'] = situation[0]
def fix_admission(df):
for adm in AdmissionType.ADMISSION_FORM:
df.loc[df.FORMA_INGRESSO == adm[1], 'FORMA_INGRESSO'] = adm[0]
def fix_evasion(df):
evasionForms = [x[1] for x in EvasionForm.EVASION_FORM]
df.loc[~df.FORMA_EVASAO.isin(evasionForms), 'FORMA_EVASAO'] = 100
for evasion in EvasionForm.EVASION_FORM:
#~ df.loc[df.FORMA_EVASAO.str.contains(evasion[1]).fillna(1.0), 'FORMA_EVASAO'] = evasion[0]
df.loc[df.FORMA_EVASAO == evasion[1], 'FORMA_EVASAO'] = evasion[0]
#~ if(evasion[0] == 100):
#~ for x in df.FORMA_EVASAO.str.contains(evasion[1]).fillna(False):
#~ if(x != 0.0):
#~ print(x)
#~ print(df.FORMA_EVASAO.str.contains(evasion[1]).fillna(5))
#~ print(df[['MATR_ALUNO','FORMA_EVASAO']])
from script.utils.utils import *
from script.utils.situations import *
from script.analysis.degree_analysis import *
from script.analysis.student_analysis import *
from script.analysis.admission_analysis import *
try:
to_unicode = unicode
except NameError:
to_unicode = str
def build_cache(dataframe):
# os.chdir("../src")
path = 'cache/curso'
ensure_path_exists(path)
for cod, df in dataframe.groupby('COD_CURSO'):
generate_degree_data(path+'/'+cod+'/', df)
generate_student_data(path+'/'+cod+'/students/',df)
#~ generate_admission_data(path+'/'+cod+'/admission/',df)
#generate_degree_data(path, dataframe)
#generate_student_data(path, dataframe)
#generate_student_list(path)
#generate_admission_data(path)
#generate_admission_list(path)
#generate_course_data(path)
#generate_course_general_data(path)
def generate_degree_data(path, dataframe):
ensure_path_exists(path)
ensure_path_exists(path+'students')
students = dataframe[['MATR_ALUNO', 'FORMA_EVASAO']].drop_duplicates()
data = {
'average_graduation': average_graduation(dataframe),
'general_failure': general_failure(dataframe),
'general_ira': general_ira(dataframe),
'active_students': students[students.FORMA_EVASAO == EvasionForm.EF_ATIVO].shape[0],
'graduated_students': students[students.FORMA_EVASAO == EvasionForm.EF_FORMATURA].shape[0],
}
save_json(path+'/degree.json', data)
#~ for ind, hist in dataframe.groupby('MATR_ALUNO'):
#~ generate_student_data_old(path+'students/{}.json'.format(ind), dataframe)
def historico(dataframe):
res = []
for _, row in dataframe.iterrows():
res.append(dict(row[['ANO', 'MEDIA_FINAL', 'PERIODO', 'SITUACAO', 'COD_ATIV_CURRIC', 'NOME_ATIV_CURRIC',
'CREDITOS', 'CH_TOTAL', 'DESCR_ESTRUTURA', 'FREQUENCIA']]))
return res
def process_semestre(per, df):
ira = df[df.SITUACAO.isin(Situation.SITUATION_AFFECT_IRA)].MEDIA_FINAL.mean()
completas = df[df.SITUACAO.isin(Situation.SITUATION_PASS)].shape[0]
tentativas = df[df.SITUACAO.isin(Situation.SITUATION_COURSED)].shape[0]
return {
'semestre': per,
'ira': ira,
'completas': completas,
'tentativas': tentativas,
'aprovacao': completas/tentativas if tentativas else 0,
'ira_por_quantidade_disciplinas': ira/tentativas if tentativas else 0
}
def generate_student_data(path, dataframe):
student_data = dict()
all_grrs = list(dataframe["MATR_ALUNO"].drop_duplicates())
for x in all_grrs:
student_data[x] = dict()
analises = [
# tupla que contem no primeiro elemento a funcao que retorna um dicionario com {"GRR": valor}
# e na segunda posicao o nome que esta analise tera no json
(posicao_turmaIngresso_semestral(dataframe),
"posicao_turmaIngresso_semestral"),
(periodo_real(dataframe),
"periodo_real"),
(periodo_pretendido(dataframe),
"periodo_pretendido"),
(ira_semestral(dataframe),
"ira_semestral"),
(ira_por_quantidade_disciplinas(dataframe),
"ira_por_quantidade_disciplinas"),
(indice_aprovacao_semestral(dataframe),
"indice_aprovacao_semestral"),
(aluno_turmas(dataframe),
"aluno_turmas"),
]
for x in student_data:
for a in analises: # Usar para fazer a verificacao de
student_data[x][a[1]] = a[0][x] # analises nulas para um GRR
save_json(path+x+".json", student_data[x])
#Falta verificar se alguem nao recebeu algumas analises
def generate_student_list(path):
pass
def generate_admission_data(path,df):
listagem_turma_ingresso(df)
pass
def generate_admission_list(path):
pass
def generate_course_data(path):
pass
def generate_course_general_data(path):
pass