Closes adega#157: renomeia uploads para submission e move pasta script para submission/analysis

.gitignore

@@ -28,3 +28,94 @@ static/*
 **/__pycache__
 
 *.ipynb
+
+
+# Created by https://www.gitignore.io
+
+### OSX ###
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear on external disk
+.Spotlight-V100
+.Trashes
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+
+### Django ###
+*.log
+*.pot
+*.pyc
+__pycache__/
+local_settings.py
+
+.env
+db.sqlite3

README.md

@@ -60,7 +60,7 @@ Enquanto o `sudo make docker-up` estiver sendo executado, as alterações feitas
 
 Assim como é possível realizar qualquer comando como seria feito no com o manage.py, também é possível por meio do comando `sudo make docker-manage`. Por exemplo:
 ```bash
-$ sudo make docker-manage makemigrations uploads
+$ sudo make docker-manage makemigrations submission
 $ sudo make docker-manage migrate
 $ sudo make docker-manage createsuperuser
 ```

docker_scripts/on_docker_init.sh

@@ -2,6 +2,6 @@
 # If ANY of this commands fails (return != 0) the container will be down
 bash ./docker_scripts/wait_for_postgres.sh
 cd src
-python manage.py makemigrations degree admission educator uploads course
+python manage.py makemigrations degree admission educator submission course
 python manage.py migrate
 python manage.py runserver 0.0.0.0:8000

docker_scripts/on_docker_init_production.sh

@@ -2,7 +2,7 @@
 # If ANY of this commands fails (return != 0) the container will be down
 bash ./docker_scripts/wait_for_postgres.sh
 cd src
-python manage.py makemigrations degree admission educator uploads course
+python manage.py makemigrations degree admission educator submission course
 python manage.py migrate
 
 python manage.py collectstatic --noinput

src/adega/management/commands/analyze.py

 from django.core.management.base import BaseCommand
-from uploads.models import Submission
+from submission.models import Submission
 
-from script.main import analyze
+from submission.analysis.main import analyze
 
 
 class Command(BaseCommand):

src/adega/settings.py

@@ -52,7 +52,7 @@ INSTALLED_APPS = [
     'course',
     'student',
     'report_api',
-    'uploads'
+    'submission'
 ]
 
 MIDDLEWARE_CLASSES = [
@@ -170,7 +170,7 @@ else:
 STATIC_ROOT = os.path.join(PROJECT_DIR, 'static')
 
 MEDIA_URL = '/script/base/'
-MEDIA_ROOT = os.path.join(PROJECT_DIR, 'uploads')
+MEDIA_ROOT = os.path.join(PROJECT_DIR, 'submission')
 
 # MEDIA_URL = None # a gente não quer ninguem fazendo download disso
 # MEDIA_ROOT = os.path.join(PROJECT_DIR, 'uploads')

src/adega/templates/adega/dashboard.html

@@ -10,8 +10,8 @@
     submetidos e processados</b> referentes aos cursos registrados no sistema. Caso não exista nenhum
     relatório processado relacionado aos cursos, o sistema acusará um erro.
 
-    <a href="{% url 'uploads:SubmissionCreateView' %}">Submeter relatório</a></a>,
-    que podem ser gerenciados na página <a href="{% url 'uploads:SubmissionListView' %}"> Gerenciar relatórios</a>.
+    <a href="{% url 'submission:SubmissionCreateView' %}">Submeter relatório</a></a>,
+    que podem ser gerenciados na página <a href="{% url 'submission:SubmissionListView' %}"> Gerenciar relatórios</a>.
     <br><br>
 
 

src/adega/templates/adega/navbar.html

@@ -30,7 +30,7 @@
             <a href="{% url 'dashboard' %}">Início</a>
         </span>
     <span class="navbar-text">
-        <a href="{% url 'uploads:SubmissionListView' %}">Relatórios</a>
+        <a href="{% url 'submission:SubmissionListView' %}">Relatórios</a>
     </span>
     &nbsp;&nbsp;&nbsp;
     {% comment %}

src/adega/templates/adega/sidebar.html

 <div class="sidebar left hidden-sm hidden-xs">
-    <li><a class="btn btn-primary text-left" href="{% url 'uploads:SubmissionListView' %}">Gerenciar relatórios</a></li>
+    <li><a class="btn btn-primary text-left" href="{% url 'submission:SubmissionListView' %}">Gerenciar relatórios</a></li>
     <li><a class="btn btn-primary text-left" href="{% url 'degree:index' submission_id=submission.id%}">Informações gerais do curso</a></li>
     <li><a class="btn btn-primary text-left" href="{% url 'student:index' submission_id=submission.id%}">Alunos</a></li>
     <li style="display: block;">

src/adega/urls.py

@@ -15,7 +15,7 @@ urlpatterns = [
 
     url(r'^adega/course/(?P<submission_id>\w*)/', include('course.urls', namespace='course')),
 
-    url(r'^adega/submission/', include('uploads.urls', namespace='uploads')),
+    url(r'^adega/submission/', include('submission.urls', namespace='submission')),
 
     url(r'^adega/student/(?P<submission_id>\w*)/', include('student.urls', namespace='student')),
 

src/adega/views.py

 from django.shortcuts import render, redirect
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth import logout as process_logout
-from uploads.models import Submission
+from submission.models import Submission
 
 
 
@@ -19,9 +19,10 @@ def dashboard(request):
             "code": degree.code,
             "last_submission": last_submission
         })
-    return render(request, 'adega/dashboard.html', {'title': 'Dashboard',
-        "degrees_last_submissions":degrees_last_submissions, "hide_navbar": True
-        })
+    return render(request, 'adega/dashboard.html', {"title": "Dashboard",
+                                                    "degrees_last_submissions":degrees_last_submissions,
+                                                    "hide_navbar": True
+                                                    })
 
 
 @login_required

src/admission/views.py

@@ -5,7 +5,7 @@ from django.contrib import messages
 
 from degree.models import Degree
 from report_api.views import get_list_admission, get_admission_detail
-from uploads.models import Submission
+from submission.models import Submission
 
 
 def detail(request, submission_id, ano, semestre):

src/course/views.py

@@ -5,7 +5,7 @@ from django.contrib import messages
 
 from degree.models import Degree
 from report_api.views import get_list_courses, get_course_detail
-from uploads.models import Submission
+from submission.models import Submission
 
 
 def detail(request, submission_id, codigo_disciplina):

src/degree/urls.py

@@ -2,5 +2,5 @@ from django.conf.urls import url
 from degree import views
 
 urlpatterns = [
-        url(r'^$',views.index, name='index')
+        url(r'^$', views.index, name='index')
 ]

src/degree/views.py

@@ -4,7 +4,7 @@ from django.contrib.auth.decorators import login_required
 from django.contrib.auth import logout as process_logout
 from report_api.views import get_degree_information
 from degree.models import Degree
-from uploads.models import Submission
+from submission.models import Submission
 import json
 
 

src/report_api/views.py

 from degree.models import Degree
-from uploads.models import Submission
+from submission.models import Submission
 import json
 
 def get_data(session, degree, data_name, submission_id=None):

src/student/views.py

@@ -8,7 +8,7 @@ from report_api.views import get_list_students, get_student_detail
 
 import json
 
-from uploads.models import Submission
+from submission.models import Submission
 
 def detail(request, submission_id, grr):
     submission_id = int(submission_id)

src/uploads/admin.py → src/submission/admin.py

-
 from django.contrib import admin
-from .models import Submission
+from submission.models import Submission
 
-from script.main import analyze
+from submission.analysis.main import analyze
 
 from traceback import print_exc
 

src/script/analysis/admission_analysis.py → src/submission/analysis/analysis/admission_analysis.py

-import numpy as np
-
-from script.utils.situations import Situation as sit
-from script.utils.situations import EvasionForm as ef
-from script.utils.situations import *
-from script.utils.situations import Situation as sit
-from script.utils.situations import EvasionForm as ef
-from script.analysis.student_analysis import StudentAnalysis
+from submission.analysis.utils.situations import Situation as sit
+from submission.analysis.utils.situations import EvasionForm as ef
+from submission.analysis.utils.situations import *
+from submission.analysis.utils.situations import Situation as sit
+from submission.analysis.utils.situations import EvasionForm as ef
+from submission.analysis.analysis.student_analysis import StudentAnalysis
 from collections import defaultdict
 
 import numpy as np

src/script/analysis/analysis.py → src/submission/analysis/analysis/analysis.py

 # -*- coding: utf-8 -*-
 import numpy as np
-from script.utils.situations import Situation as sit
+from submission.analysis.utils.situations import Situation as sit
 from collections import namedtuple
 """
 Rate é uma tupla nomeada(deixa mais legivel fazer rate.name do que rate[0]) o

src/script/analysis/cepe9615_analysis.py → src/submission/analysis/analysis/cepe9615_analysis.py

@@ -3,7 +3,7 @@ their academic record canceled."""
 import numpy as np
 import pandas as pd
 import pprint
-from script.utils.situations import *
+from submission.analysis.utils.situations import *
 
 
 fails_by_course = [10] # used at: student_three_fails_course

src/script/analysis/course_analysis.py → src/submission/analysis/analysis/course_analysis.py

 # -*- coding: utf-8 -*-
-from script.utils.situations import Situation as sit
-from script.analysis.analysis import Analysis, rate, mean
+from submission.analysis.utils.situations import Situation as sit
+from submission.analysis.analysis.analysis import Analysis, rate, mean
 import numpy as np
 
 

src/script/analysis/degree_analysis.py → src/submission/analysis/analysis/degree_analysis.py

-import pandas as pd
 import math
 import json
-from script.utils.situations import Situation, EvasionForm
+import pandas as pd
+from submission.analysis.utils.situations import Situation, EvasionForm
 
 
 def average_graduation(df):

src/script/analysis/student_analysis.py → src/submission/analysis/analysis/student_analysis.py

@@ -4,8 +4,8 @@ import numpy as np
 #~ FAZER CACHE DE TUDO
 #~ AO CHAMAR A FUNCAO VERIFICAR SE TEM ALGO NA CACHE
 
-from script.utils.situations import *
-from script.utils.utils import memoize
+from submission.analysis.utils.situations import *
+from submission.analysis.utils.utils import memoize
 import pandas as pd
 from collections import defaultdict
 

src/script/base/dataframe_base.py → src/submission/analysis/base/dataframe_base.py

-
 import os
 import pandas as pd
 import numpy as np
-from script.utils.situations import *
-from script.utils.utils import invert_dict
+from submission.analysis.utils.situations import *
+from submission.analysis.utils.utils import invert_dict
 
 
 class DataframeHolder:
@@ -29,19 +28,15 @@ def load_dataframes(cwd='.'):
 
     dataframe = fix_dataframes(dataframes)
     dh = DataframeHolder(dataframe)
-    #~ dh.students.aggregate(teste)
-#       print(dh.students['MEDIA_FINAL'].aggregate(teste))
     return dataframe
 
 
 def read_excel(path, planilha='Planilha1'):
     return pd.read_excel(path)
 
-
 def read_csv(path):
     return pd.read_csv(path)
 
-
 def fix_dataframes(dataframes):
     for df in dataframes:
         if df['name'] == 'historico.xls' or df['name'] == 'historico.csv':
@@ -59,7 +54,11 @@ def fix_dataframes(dataframes):
     # inner = exste nos dois relatórios, é o que a gente quer
     # o que fazer com quem não está em um dos dois é um questão em aberto
     merged = pd.merge(history, register, how='inner', on=['MATR_ALUNO'])
-    merged = merged.rename(index=str, columns={"ANO_INGRESSO_x": "ANO_INGRESSO", "SEMESTRE_INGRESSO_x": "SEMESTRE_INGRESSO", "FORMA_INGRESSO_x": "FORMA_INGRESSO"})
+    merged = merged.rename(index=str, columns={
+        "ANO_INGRESSO_x": "ANO_INGRESSO",
+        "SEMESTRE_INGRESSO_x": "SEMESTRE_INGRESSO",
+        "FORMA_INGRESSO_x": "FORMA_INGRESSO"
+        })
 
     fix_situation(merged)
     fix_admission(merged)
@@ -73,9 +72,9 @@ def clean_history(df):
     print(df.columns)
 
     drop_columns = ['ID_NOTA', 'CONCEITO', 'ID_LOCAL_DISPENSA', 'SITUACAO_CURRICULO',
-        'ID_CURSO_ALUNO', 'ID_VERSAO_CURSO', 'ID_CURRIC_ALUNO',
-        'ID_ATIV_CURRIC', 'SITUACAO_ITEM', 'ID_ESTRUTURA_CUR'
-    ]
+                    'ID_CURSO_ALUNO', 'ID_VERSAO_CURSO', 'ID_CURRIC_ALUNO',
+                    'ID_ATIV_CURRIC', 'SITUACAO_ITEM', 'ID_ESTRUTURA_CUR'
+                    ]
 
     drop_columns = [x for x in drop_columns if x in df.columns]
 
@@ -92,7 +91,7 @@ def clean_register(df):
     df['ANO_EVASAO'] = df_split.str[0]
     df['SEMESTRE_EVASAO'] = df_split.str[1].str.split('o').str[0]
 
-    drop_columns = ['ID_PESSOA', 'NOME_PESSOA', 'DT_NASCIMENTO', 'NOME_UNIDADE','COD_CURSO',
+    drop_columns = ['ID_PESSOA', 'NOME_PESSOA', 'DT_NASCIMENTO', 'NOME_UNIDADE', 'COD_CURSO',
                     'PERIODO_INGRESSO', 'PERIODO_EVASAO']
 
     drop_columns = [x for x in drop_columns if x in df.columns]
@@ -101,6 +100,7 @@ def clean_register(df):
 
 
 def get_situation(d, default):
+
     def getter(x):
         return invert_dict(d).get(x, default)
     return getter

src/script/build_cache.py → src/submission/analysis/build_cache.py

-from script.utils.utils import save_json, ensure_path_exists
-from script.utils.situations import *
-from script.analysis.degree_analysis import *
-from script.analysis.student_analysis import *
-from script.analysis.course_analysis import Course
+from submission.analysis.utils.utils import save_json, ensure_path_exists
+from submission.analysis.utils.situations import *
+from submission.analysis.analysis.degree_analysis import *
+from submission.analysis.analysis.student_analysis import *
+from submission.analysis.analysis.course_analysis import Course
 
-from script.analysis.admission_analysis import Admission
-from script.analysis.admission_analysis import media_ira_turma_ingresso
-from script.analysis.admission_analysis import desvio_padrao_turma_ingresso
-from script.analysis.admission_analysis import students_per_semester
-from script.analysis.admission_analysis import admission_class_ira_per_semester
-from script.analysis.admission_analysis import evasion_per_semester
+from submission.analysis.analysis.admission_analysis import Admission
+from submission.analysis.analysis.admission_analysis import media_ira_turma_ingresso
+from submission.analysis.analysis.admission_analysis import desvio_padrao_turma_ingresso
+from submission.analysis.analysis.admission_analysis import students_per_semester
+from submission.analysis.analysis.admission_analysis import admission_class_ira_per_semester
+from submission.analysis.analysis.admission_analysis import evasion_per_semester
 
 
-from script.analysis.cepe9615_analysis import *
+from submission.analysis.analysis.cepe9615_analysis import *
 
 from collections import defaultdict
 
@@ -60,7 +60,8 @@ def historico(dataframe):
     res = []
 
     for _, row in dataframe.iterrows():
-        res.append(dict(row[['ANO', 'MEDIA_FINAL', 'PERIODO', 'SITUACAO', 'COD_ATIV_CURRIC', 'NOME_ATIV_CURRIC',
+        res.append(dict(row[['ANO', 'MEDIA_FINAL', 'PERIODO', 'SITUACAO',
+                             'COD_ATIV_CURRIC', 'NOME_ATIV_CURRIC',
                              'CREDITOS', 'CH_TOTAL', 'DESCR_ESTRUTURA', 'FREQUENCIA']]))
 
     return res

src/script/main.py → src/submission/analysis/main.py

-
 import time
-from script.base.dataframe_base import load_dataframes
-from script.build_cache import build_cache
+from submission.analysis.base.dataframe_base import load_dataframes
+from submission.analysis.build_cache import build_cache
 
 from datetime import timedelta
 
@@ -29,11 +28,9 @@ def analyze(submission, debug=True):
 
     except Exception as e:
         if(debug):
-            print("Error on submission analysis:",e)
-        
-        submission.set_fail(round(time.clock() - start_time), error_message=str(e))
-    
+            print("Error on submission analysis:", e)
 
+        submission.set_fail(round(time.clock() - start_time), error_message=str(e))
 
 def main():
     print("Não pra você estar fazendo isso")

src/uploads/forms.py → src/submission/forms.py

 from django import forms
 from datetime import datetime
-from uploads.models import Submission
+from submission.models import Submission
 
 
 class SubmissionForm(forms.ModelForm):
@@ -13,10 +13,10 @@ class SubmissionForm(forms.ModelForm):
     class Meta:
         model = Submission
         fields = [
-                'historico',
-                'matricula',
-                'semester_status',
-                'relative_year',
-                'relative_semester',
-                'degree'
-                ]
+            'historico',
+            'matricula',
+            'semester_status',
+            'relative_year',
+            'relative_semester',
+            'degree'
+            ]

src/uploads/models.py → src/submission/models.py

@@ -5,12 +5,12 @@ from django.utils import timezone
 from os import path
 from django.conf import settings
 
-from degree.models import Degree 
+from degree.models import Degree
 
 from django.dispatch import receiver
 
 
-from script import main as submission_analysis
+from submission.analysis import main as submission_analysis
 
 
 def get_path(instance, filename):
@@ -107,4 +107,4 @@ class Submission(models.Model):
 #     if created:
 #         print(sender,instance,created)
 #         # submission_analysis.analyze(instance, debug=False)
-#         submission_analysis.analyze(sender)
\ No newline at end of file
+#         submission_analysis.analyze(sender)

src/uploads/templates/submission_create.html → src/submission/templates/submission_create.html

@@ -14,7 +14,7 @@
 
 	É possível, caso o processamento demore muito tempo, que a página perca o 
 	status de ociosa, perdendo a indicação de que o processamento está sendo realizado.
-	Caso isso aconteça, verifique a página de <a href="{% url 'uploads:SubmissionListView' %}">Gerenciamento de relatórios</a>
+	Caso isso aconteça, verifique a página de <a href="{% url 'submission:SubmissionListView' %}">Gerenciamento de relatórios</a>
 	e visualize o status do processamento da submissão dos relatórios,
 	que pode ser "Terminado", "Executando" ou "Falha".<br>
 	Em caso de falha, verifique se as planilhas de Matrícula e Histórico

src/uploads/templates/submission_detail.html → src/submission/templates/submission_detail.html

@@ -6,5 +6,5 @@
 <p>Status: {{ submission.processed }}</p>
 <p>Autor: {{ submission.author }}</p>
 <p>Status do semestre: {{ submission.semester_status }}</p>
-<a href="{% url 'uploads:SubmissionUpdateView' submission.pk %}">Editar</a>
+<a href="{% url 'submission:SubmissionUpdateView' submission.pk %}">Editar</a>
 {% endblock %}

src/uploads/templates/submission_list.html → src/submission/templates/submission_list.html

@@ -7,7 +7,7 @@
 		<h2>Relatórios Submetidos</h2>
 	</div>
 	<div class="col-offset-8">
-		<a href="{% url 'uploads:SubmissionCreateView' %}"> <button type="button" class="btn btn-primary">Submeter relatório</button></a>
+		<a href="{% url 'submission:SubmissionCreateView' %}"> <button type="button" class="btn btn-primary">Submeter relatório</button></a>
 	</div>
 	
 </div>
@@ -18,7 +18,8 @@
 	
 	As linhas em vermelho indicam as submissões que não puderam ser processadas.
 	Caso este problema aconteça, verifique novamente na etapa de que pode ser causado
-	pela incompatibilidade das planilhas submetidas em <a href="{% url 'uploads:SubmissionCreateView' %}">Submeter relatório</a>
+	pela incompatibilidade das planilhas submetidas em <a href="{% url
+        'submission:SubmissionCreateView' %}">Submeter relatório</a>
 	(colunas faltando, ordem errada da submissão de matrícula e histórico).<br><br>
 
 	Também, ao submeter, verifique o Status da submissão que caso esteja em
@@ -51,9 +52,9 @@
 							<td>{{sub.get_analysis_status_display}}</td>
 							<td>{{sub.timestamp}}</td>
 							<td class="table-primary bg-primary bg-white">
-								<!-- <a href="{% url 'uploads:SubmissionDetailView' sub.pk %}">Detalhes </a><br> -->
-								<a href="{% url 'uploads:SubmissionUpdateView' sub.pk %}">Editar </a><br>
-								<a href="{% url 'uploads:SubmissionDeleteView' sub.pk %}">Remover </a><br>
+								<!-- <a href="{% url 'submission:SubmissionDetailView' sub.pk %}">Detalhes </a><br> -->
+								<a href="{% url 'submission:SubmissionUpdateView' sub.pk %}">Editar </a><br>
+								<a href="{% url 'submission:SubmissionDeleteView' sub.pk %}">Remover </a><br>
 								{% if sub.get_analysis_status_display == 'Terminado' %}
 									<a href="{% url 'degree:index' submission_id=sub.id %}"> Visualizar relatório (gráficos e taxas) </a>
 								{% endif %}
@@ -99,4 +100,4 @@
 
 	
 </script>
-{% endblock js-foot %}
\ No newline at end of file
+{% endblock js-foot %}