Commit d9984361 authored by Odair M.'s avatar Odair M. Committed by Bruno Meyer

Closes #157: renomeia uploads para submission e move pasta script para submission/analysis

parent 0733528c
Pipeline #19212 passed with stage
in 15 seconds
......@@ -28,3 +28,94 @@ static/*
**/__pycache__
*.ipynb
# Created by https://www.gitignore.io
### OSX ###
.DS_Store
.AppleDouble
.LSOverride
# Icon must end with two \r
Icon
# Thumbnails
._*
# Files that might appear on external disk
.Spotlight-V100
.Trashes
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.cache
nosetests.xml
coverage.xml
# Translations
*.mo
*.pot
# Sphinx documentation
docs/_build/
# PyBuilder
target/
### Django ###
*.log
*.pot
*.pyc
__pycache__/
local_settings.py
.env
db.sqlite3
......@@ -60,7 +60,7 @@ Enquanto o `sudo make docker-up` estiver sendo executado, as alterações feitas
Assim como é possível realizar qualquer comando como seria feito no com o manage.py, também é possível por meio do comando `sudo make docker-manage`. Por exemplo:
```bash
$ sudo make docker-manage makemigrations uploads
$ sudo make docker-manage makemigrations submission
$ sudo make docker-manage migrate
$ sudo make docker-manage createsuperuser
```
......
......@@ -2,6 +2,6 @@
# If ANY of this commands fails (return != 0) the container will be down
bash ./docker_scripts/wait_for_postgres.sh
cd src
python manage.py makemigrations degree admission educator uploads course
python manage.py makemigrations degree admission educator submission course
python manage.py migrate
python manage.py runserver 0.0.0.0:8000
......@@ -2,7 +2,7 @@
# If ANY of this commands fails (return != 0) the container will be down
bash ./docker_scripts/wait_for_postgres.sh
cd src
python manage.py makemigrations degree admission educator uploads course
python manage.py makemigrations degree admission educator submission course
python manage.py migrate
python manage.py collectstatic --noinput
......
from django.core.management.base import BaseCommand
from uploads.models import Submission
from submission.models import Submission
from script.main import analyze
from submission.analysis.main import analyze
class Command(BaseCommand):
......
......@@ -52,7 +52,7 @@ INSTALLED_APPS = [
'course',
'student',
'report_api',
'uploads'
'submission'
]
MIDDLEWARE_CLASSES = [
......@@ -170,7 +170,7 @@ else:
STATIC_ROOT = os.path.join(PROJECT_DIR, 'static')
MEDIA_URL = '/script/base/'
MEDIA_ROOT = os.path.join(PROJECT_DIR, 'uploads')
MEDIA_ROOT = os.path.join(PROJECT_DIR, 'submission')
# MEDIA_URL = None # a gente não quer ninguem fazendo download disso
# MEDIA_ROOT = os.path.join(PROJECT_DIR, 'uploads')
......@@ -10,8 +10,8 @@
submetidos e processados</b> referentes aos cursos registrados no sistema. Caso não exista nenhum
relatório processado relacionado aos cursos, o sistema acusará um erro.
<a href="{% url 'uploads:SubmissionCreateView' %}">Submeter relatório</a></a>,
que podem ser gerenciados na página <a href="{% url 'uploads:SubmissionListView' %}"> Gerenciar relatórios</a>.
<a href="{% url 'submission:SubmissionCreateView' %}">Submeter relatório</a></a>,
que podem ser gerenciados na página <a href="{% url 'submission:SubmissionListView' %}"> Gerenciar relatórios</a>.
<br><br>
......
......@@ -30,7 +30,7 @@
<a href="{% url 'dashboard' %}">Início</a>
</span>
<span class="navbar-text">
<a href="{% url 'uploads:SubmissionListView' %}">Relatórios</a>
<a href="{% url 'submission:SubmissionListView' %}">Relatórios</a>
</span>
&nbsp;&nbsp;&nbsp;
{% comment %}
......
<div class="sidebar left hidden-sm hidden-xs">
<li><a class="btn btn-primary text-left" href="{% url 'uploads:SubmissionListView' %}">Gerenciar relatórios</a></li>
<li><a class="btn btn-primary text-left" href="{% url 'submission:SubmissionListView' %}">Gerenciar relatórios</a></li>
<li><a class="btn btn-primary text-left" href="{% url 'degree:index' submission_id=submission.id%}">Informações gerais do curso</a></li>
<li><a class="btn btn-primary text-left" href="{% url 'student:index' submission_id=submission.id%}">Alunos</a></li>
<li style="display: block;">
......
......@@ -15,7 +15,7 @@ urlpatterns = [
url(r'^adega/course/(?P<submission_id>\w*)/', include('course.urls', namespace='course')),
url(r'^adega/submission/', include('uploads.urls', namespace='uploads')),
url(r'^adega/submission/', include('submission.urls', namespace='submission')),
url(r'^adega/student/(?P<submission_id>\w*)/', include('student.urls', namespace='student')),
......
from django.shortcuts import render, redirect
from django.contrib.auth.decorators import login_required
from django.contrib.auth import logout as process_logout
from uploads.models import Submission
from submission.models import Submission
......@@ -19,9 +19,10 @@ def dashboard(request):
"code": degree.code,
"last_submission": last_submission
})
return render(request, 'adega/dashboard.html', {'title': 'Dashboard',
"degrees_last_submissions":degrees_last_submissions, "hide_navbar": True
})
return render(request, 'adega/dashboard.html', {"title": "Dashboard",
"degrees_last_submissions":degrees_last_submissions,
"hide_navbar": True
})
@login_required
......
......@@ -5,7 +5,7 @@ from django.contrib import messages
from degree.models import Degree
from report_api.views import get_list_admission, get_admission_detail
from uploads.models import Submission
from submission.models import Submission
def detail(request, submission_id, ano, semestre):
......
......@@ -5,7 +5,7 @@ from django.contrib import messages
from degree.models import Degree
from report_api.views import get_list_courses, get_course_detail
from uploads.models import Submission
from submission.models import Submission
def detail(request, submission_id, codigo_disciplina):
......
......@@ -2,5 +2,5 @@ from django.conf.urls import url
from degree import views
urlpatterns = [
url(r'^$',views.index, name='index')
url(r'^$', views.index, name='index')
]
......@@ -4,7 +4,7 @@ from django.contrib.auth.decorators import login_required
from django.contrib.auth import logout as process_logout
from report_api.views import get_degree_information
from degree.models import Degree
from uploads.models import Submission
from submission.models import Submission
import json
......
from degree.models import Degree
from uploads.models import Submission
from submission.models import Submission
import json
def get_data(session, degree, data_name, submission_id=None):
......
......@@ -8,7 +8,7 @@ from report_api.views import get_list_students, get_student_detail
import json
from uploads.models import Submission
from submission.models import Submission
def detail(request, submission_id, grr):
submission_id = int(submission_id)
......
from django.contrib import admin
from .models import Submission
from submission.models import Submission
from script.main import analyze
from submission.analysis.main import analyze
from traceback import print_exc
......
import numpy as np
from script.utils.situations import Situation as sit
from script.utils.situations import EvasionForm as ef
from script.utils.situations import *
from script.utils.situations import Situation as sit
from script.utils.situations import EvasionForm as ef
from script.analysis.student_analysis import StudentAnalysis
from submission.analysis.utils.situations import Situation as sit
from submission.analysis.utils.situations import EvasionForm as ef
from submission.analysis.utils.situations import *
from submission.analysis.utils.situations import Situation as sit
from submission.analysis.utils.situations import EvasionForm as ef
from submission.analysis.analysis.student_analysis import StudentAnalysis
from collections import defaultdict
import numpy as np
......
# -*- coding: utf-8 -*-
import numpy as np
from script.utils.situations import Situation as sit
from submission.analysis.utils.situations import Situation as sit
from collections import namedtuple
"""
Rate é uma tupla nomeada(deixa mais legivel fazer rate.name do que rate[0]) o
......
......@@ -3,7 +3,7 @@ their academic record canceled."""
import numpy as np
import pandas as pd
import pprint
from script.utils.situations import *
from submission.analysis.utils.situations import *
fails_by_course = [10] # used at: student_three_fails_course
......
# -*- coding: utf-8 -*-
from script.utils.situations import Situation as sit
from script.analysis.analysis import Analysis, rate, mean
from submission.analysis.utils.situations import Situation as sit
from submission.analysis.analysis.analysis import Analysis, rate, mean
import numpy as np
......
import pandas as pd
import math
import json
from script.utils.situations import Situation, EvasionForm
import pandas as pd
from submission.analysis.utils.situations import Situation, EvasionForm
def average_graduation(df):
......
......@@ -4,8 +4,8 @@ import numpy as np
#~ FAZER CACHE DE TUDO
#~ AO CHAMAR A FUNCAO VERIFICAR SE TEM ALGO NA CACHE
from script.utils.situations import *
from script.utils.utils import memoize
from submission.analysis.utils.situations import *
from submission.analysis.utils.utils import memoize
import pandas as pd
from collections import defaultdict
......
import os
import pandas as pd
import numpy as np
from script.utils.situations import *
from script.utils.utils import invert_dict
from submission.analysis.utils.situations import *
from submission.analysis.utils.utils import invert_dict
class DataframeHolder:
......@@ -29,19 +28,15 @@ def load_dataframes(cwd='.'):
dataframe = fix_dataframes(dataframes)
dh = DataframeHolder(dataframe)
#~ dh.students.aggregate(teste)
# print(dh.students['MEDIA_FINAL'].aggregate(teste))
return dataframe
def read_excel(path, planilha='Planilha1'):
return pd.read_excel(path)
def read_csv(path):
return pd.read_csv(path)
def fix_dataframes(dataframes):
for df in dataframes:
if df['name'] == 'historico.xls' or df['name'] == 'historico.csv':
......@@ -59,7 +54,11 @@ def fix_dataframes(dataframes):
# inner = exste nos dois relatórios, é o que a gente quer
# o que fazer com quem não está em um dos dois é um questão em aberto
merged = pd.merge(history, register, how='inner', on=['MATR_ALUNO'])
merged = merged.rename(index=str, columns={"ANO_INGRESSO_x": "ANO_INGRESSO", "SEMESTRE_INGRESSO_x": "SEMESTRE_INGRESSO", "FORMA_INGRESSO_x": "FORMA_INGRESSO"})
merged = merged.rename(index=str, columns={
"ANO_INGRESSO_x": "ANO_INGRESSO",
"SEMESTRE_INGRESSO_x": "SEMESTRE_INGRESSO",
"FORMA_INGRESSO_x": "FORMA_INGRESSO"
})
fix_situation(merged)
fix_admission(merged)
......@@ -73,9 +72,9 @@ def clean_history(df):
print(df.columns)
drop_columns = ['ID_NOTA', 'CONCEITO', 'ID_LOCAL_DISPENSA', 'SITUACAO_CURRICULO',
'ID_CURSO_ALUNO', 'ID_VERSAO_CURSO', 'ID_CURRIC_ALUNO',
'ID_ATIV_CURRIC', 'SITUACAO_ITEM', 'ID_ESTRUTURA_CUR'
]
'ID_CURSO_ALUNO', 'ID_VERSAO_CURSO', 'ID_CURRIC_ALUNO',
'ID_ATIV_CURRIC', 'SITUACAO_ITEM', 'ID_ESTRUTURA_CUR'
]
drop_columns = [x for x in drop_columns if x in df.columns]
......@@ -92,7 +91,7 @@ def clean_register(df):
df['ANO_EVASAO'] = df_split.str[0]
df['SEMESTRE_EVASAO'] = df_split.str[1].str.split('o').str[0]
drop_columns = ['ID_PESSOA', 'NOME_PESSOA', 'DT_NASCIMENTO', 'NOME_UNIDADE','COD_CURSO',
drop_columns = ['ID_PESSOA', 'NOME_PESSOA', 'DT_NASCIMENTO', 'NOME_UNIDADE', 'COD_CURSO',
'PERIODO_INGRESSO', 'PERIODO_EVASAO']
drop_columns = [x for x in drop_columns if x in df.columns]
......@@ -101,6 +100,7 @@ def clean_register(df):
def get_situation(d, default):
def getter(x):
return invert_dict(d).get(x, default)
return getter
......
from script.utils.utils import save_json, ensure_path_exists
from script.utils.situations import *
from script.analysis.degree_analysis import *
from script.analysis.student_analysis import *
from script.analysis.course_analysis import Course
from submission.analysis.utils.utils import save_json, ensure_path_exists
from submission.analysis.utils.situations import *
from submission.analysis.analysis.degree_analysis import *
from submission.analysis.analysis.student_analysis import *
from submission.analysis.analysis.course_analysis import Course
from script.analysis.admission_analysis import Admission
from script.analysis.admission_analysis import media_ira_turma_ingresso
from script.analysis.admission_analysis import desvio_padrao_turma_ingresso
from script.analysis.admission_analysis import students_per_semester
from script.analysis.admission_analysis import admission_class_ira_per_semester
from script.analysis.admission_analysis import evasion_per_semester
from submission.analysis.analysis.admission_analysis import Admission
from submission.analysis.analysis.admission_analysis import media_ira_turma_ingresso
from submission.analysis.analysis.admission_analysis import desvio_padrao_turma_ingresso
from submission.analysis.analysis.admission_analysis import students_per_semester
from submission.analysis.analysis.admission_analysis import admission_class_ira_per_semester
from submission.analysis.analysis.admission_analysis import evasion_per_semester
from script.analysis.cepe9615_analysis import *
from submission.analysis.analysis.cepe9615_analysis import *
from collections import defaultdict
......@@ -60,7 +60,8 @@ def historico(dataframe):
res = []
for _, row in dataframe.iterrows():
res.append(dict(row[['ANO', 'MEDIA_FINAL', 'PERIODO', 'SITUACAO', 'COD_ATIV_CURRIC', 'NOME_ATIV_CURRIC',
res.append(dict(row[['ANO', 'MEDIA_FINAL', 'PERIODO', 'SITUACAO',
'COD_ATIV_CURRIC', 'NOME_ATIV_CURRIC',
'CREDITOS', 'CH_TOTAL', 'DESCR_ESTRUTURA', 'FREQUENCIA']]))
return res
......
import time
from script.base.dataframe_base import load_dataframes
from script.build_cache import build_cache
from submission.analysis.base.dataframe_base import load_dataframes
from submission.analysis.build_cache import build_cache
from datetime import timedelta
......@@ -29,11 +28,9 @@ def analyze(submission, debug=True):
except Exception as e:
if(debug):
print("Error on submission analysis:",e)
submission.set_fail(round(time.clock() - start_time), error_message=str(e))
print("Error on submission analysis:", e)
submission.set_fail(round(time.clock() - start_time), error_message=str(e))
def main():
print("Não pra você estar fazendo isso")
......
from django import forms
from datetime import datetime
from uploads.models import Submission
from submission.models import Submission
class SubmissionForm(forms.ModelForm):
......@@ -13,10 +13,10 @@ class SubmissionForm(forms.ModelForm):
class Meta:
model = Submission
fields = [
'historico',
'matricula',
'semester_status',
'relative_year',
'relative_semester',
'degree'
]
'historico',
'matricula',
'semester_status',
'relative_year',
'relative_semester',
'degree'
]
......@@ -5,12 +5,12 @@ from django.utils import timezone
from os import path
from django.conf import settings
from degree.models import Degree
from degree.models import Degree
from django.dispatch import receiver
from script import main as submission_analysis
from submission.analysis import main as submission_analysis
def get_path(instance, filename):
......@@ -107,4 +107,4 @@ class Submission(models.Model):
# if created:
# print(sender,instance,created)
# # submission_analysis.analyze(instance, debug=False)
# submission_analysis.analyze(sender)
\ No newline at end of file
# submission_analysis.analyze(sender)
......@@ -14,7 +14,7 @@
É possível, caso o processamento demore muito tempo, que a página perca o
status de ociosa, perdendo a indicação de que o processamento está sendo realizado.
Caso isso aconteça, verifique a página de <a href="{% url 'uploads:SubmissionListView' %}">Gerenciamento de relatórios</a>
Caso isso aconteça, verifique a página de <a href="{% url 'submission:SubmissionListView' %}">Gerenciamento de relatórios</a>
e visualize o status do processamento da submissão dos relatórios,
que pode ser "Terminado", "Executando" ou "Falha".<br>
Em caso de falha, verifique se as planilhas de Matrícula e Histórico
......
......@@ -6,5 +6,5 @@
<p>Status: {{ submission.processed }}</p>
<p>Autor: {{ submission.author }}</p>
<p>Status do semestre: {{ submission.semester_status }}</p>
<a href="{% url 'uploads:SubmissionUpdateView' submission.pk %}">Editar</a>
<a href="{% url 'submission:SubmissionUpdateView' submission.pk %}">Editar</a>
{% endblock %}
......@@ -7,7 +7,7 @@
<h2>Relatórios Submetidos</h2>
</div>
<div class="col-offset-8">
<a href="{% url 'uploads:SubmissionCreateView' %}"> <button type="button" class="btn btn-primary">Submeter relatório</button></a>
<a href="{% url 'submission:SubmissionCreateView' %}"> <button type="button" class="btn btn-primary">Submeter relatório</button></a>
</div>
</div>
......@@ -18,7 +18,8 @@
As linhas em vermelho indicam as submissões que não puderam ser processadas.
Caso este problema aconteça, verifique novamente na etapa de que pode ser causado
pela incompatibilidade das planilhas submetidas em <a href="{% url 'uploads:SubmissionCreateView' %}">Submeter relatório</a>
pela incompatibilidade das planilhas submetidas em <a href="{% url
'submission:SubmissionCreateView' %}">Submeter relatório</a>
(colunas faltando, ordem errada da submissão de matrícula e histórico).<br><br>
Também, ao submeter, verifique o Status da submissão que caso esteja em
......@@ -51,9 +52,9 @@
<td>{{sub.get_analysis_status_display}}</td>
<td>{{sub.timestamp}}</td>
<td class="table-primary bg-primary bg-white">
<!-- <a href="{% url 'uploads:SubmissionDetailView' sub.pk %}">Detalhes </a><br> -->
<a href="{% url 'uploads:SubmissionUpdateView' sub.pk %}">Editar </a><br>
<a href="{% url 'uploads:SubmissionDeleteView' sub.pk %}">Remover </a><br>
<!-- <a href="{% url 'submission:SubmissionDetailView' sub.pk %}">Detalhes </a><br> -->
<a href="{% url 'submission:SubmissionUpdateView' sub.pk %}">Editar </a><br>
<a href="{% url 'submission:SubmissionDeleteView' sub.pk %}">Remover </a><br>
{% if sub.get_analysis_status_display == 'Terminado' %}
<a href="{% url 'degree:index' submission_id=sub.id %}"> Visualizar relatório (gráficos e taxas) </a>
{% endif %}
......@@ -99,4 +100,4 @@
</script>
{% endblock js-foot %}
\ No newline at end of file
{% endblock js-foot %}
......@@ -2,10 +2,10 @@ from django.contrib.messages.views import SuccessMessageMixin
from django.views.generic import CreateView, UpdateView, DeleteView
from django.views.generic import ListView, DetailView
from django.urls import reverse_lazy
from uploads.models import Submission
from uploads.forms import SubmissionForm
from submission.models import Submission
from submission.forms import SubmissionForm
from degree.models import Degree
from script import main as submission_analysis
from submission.analysis import main as submission_analysis
class SubmissionCreate(SuccessMessageMixin, CreateView):
model = Submission
......@@ -13,12 +13,12 @@ class SubmissionCreate(SuccessMessageMixin, CreateView):
template_name = 'submission_create.html'
success_url = reverse_lazy('dashboard')
success_message = "Relatório enviado com suceso"
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
context["hide_navbar"] = True
return context
def form_valid(self, form):
# muda nomes dos arquivos
form.instance.historico.name = "historico.xls"
......@@ -29,11 +29,11 @@ class SubmissionCreate(SuccessMessageMixin, CreateView):
form.instance.degree = degree