Teste com models utilizando mongo

makefile

@@ -29,8 +29,11 @@ clean-docs:
 	@rm -rf docs
 
 install:
+	apt-get update
 	apt-get install -y python3-dev
 	apt-get install -y python3-pip
+	apt-get install -y libpq-dev
+	apt-get install -y postgresql postgresql-contrib
 	pip3 install -U pip setuptools
 	pip3 install -r requirements.txt
 

requirements.txt

 django==1.8
 django-widget-tweaks
 pandas==0.18.1
-django-extensions
+django-extension
+psycopg2

src/adega/settings.py

@@ -11,7 +11,7 @@ https://docs.djangoproject.com/en/1.11/ref/settings/
 """
 
 import os
-
+from mongoengine import connect
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -88,7 +88,14 @@ DATABASES = {
     }
 }
 
-
+#DATABASES = {
+#    'default': {
+#        'NAME': 'teste',
+#        'ENGINE': 'django_mongodb_engine',
+#    },
+#}
+MONGO_DATABASE_NAME = 'teste'
+connect(MONGO_DATABASE_NAME)
 # Password validation
 # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
 

src/degree/models.py

 from django.db import models
 from django.db.models import Max
 from course.models import Course
+from mongoengine import Document, EmbeddedDocument, fields
+
 # Create your models here.
 class Degree(models.Model):
     name = models.CharField(max_length = 50)
@@ -23,3 +25,14 @@ class CourseCurriculum(models.Model):
     course = models.ForeignKey(Course)
     period = models.PositiveIntegerField(null = True)
     type_course = models.CharField(max_length = 255)
+
+class CourseCurriculumMongo(Document):
+    start_year = fields.IntField()
+
+class CurriculumMongo(Document):
+    courses = fields.ListField(fields.EmbeddedDocumentField('CourseCurriculumMongo'))
+
+
+class DegreeMongo(Document):
+    name = fields.StringField(max_length=50)
+    curriculum = fields.ListField(fields.EmbeddedDocumentField('CurriculumMongo'))

src/test.py

+# -*- coding: utf-8 -*-
+import sys
+import os
+import django
+import time
+import json
+import io
+import math
+
+from datetime import timedelta
+from pathlib import Path
+
+sys.path.append(os.getcwd())
+os.environ["DJANGO_SETTINGS_MODULE"] = "adega.settings"
+django.setup()
+
+from django.db import models
+from student.models import *
+from course.models import *
+from degree.models import *
+from admission.models import *
+
+from student.analysis import *
+from course.analysis import *
+from degree.analysis import *
+from admission.analysis import *
+from utils.data import *
+try:
+    to_unicode = unicode
+except NameError:
+    to_unicode = str
+
+def main():
+    start_time = time.clock()
+    start_time_exec = time.time()
+
+#    generate_data()
+#    degree = DegreeMongo()
+#    degree.name = "teste"
+#    degree.save()
+#    course = CourseCurriculumMongo()
+#    print(course)
+#    course.start = 1923
+#    print(course)
+#    course.save()
+#    print("Saved")
+#    course = CourseCurriculumMongo()
+#    course.start = 1933
+#    print(course)
+#    course.save()
+
+#    curriculum = CurriculumMongo()
+#    curriculum.courses = CourseCurriculumMongo.objects   
+#    curriculum.save()
+#
+#    degree = DegreeMongo()
+#    degree.name = "Novo Teste"
+#    degree.curriculum = CurriculumMongo.objects
+#    degree.save()
+    new_degree = DegreeMongo.objects()
+    for nd in new_degree:
+        print(nd.name)
+        for curriculum in nd.curriculum:
+            for c in curriculum.courses:
+                print(CourseCurriculumMongo.objects(start=1933))
+    print(new_degree)
+    print("--- Tempo de cpu: {} ---".format(timedelta(seconds=round(time.clock() - start_time))))
+    print("--- Duração real: {} ---".format(timedelta(seconds=round(time.time() - start_time_exec))))
+
+
+def generate_data():
+    path = 'cache'
+    if not os.path.exists(path):
+        os.mkdir(path)
+
+    path = 'cache/curso'
+    if not os.path.exists(path):
+        os.mkdir(path)
+
+    degrees = Degree.objects.all()
+
+    for degree in degrees:
+        path = 'cache/curso/' + degree.code
+        if not os.path.exists(path):
+            os.mkdir(path)
+        generate_degree_data(degree, path)
+        generate_student_data(degree, path)
+        generate_student_list_data(degree, path)
+        generate_admission_data(degree, path)
+        generate_admission_list_data(degree, path)    
+        generate_course_data(degree, path)
+        generate_course_general_data(degree, path)
+        generate_cepe9615_data(degree, path)
+
+def generate_degree_data(degree, path):
+    print("Fazendo analises do Curso - {}".format(degree.name))
+    average_grad = average_graduation(degree) # media_formandos
+
+    dic = merge_dicts(
+         ['average_ira', 'semester_evasion', 'graduation'],
+         graph_average_ira(degree),
+         graph_average_ira_evasion_semester(degree),
+         graph_average_ira_graduation(degree)
+    )
+    
+    degree_data = {
+        'time_graduation': average_time_graduation_degree(degree),
+        'graduation_rate': average_grad[0],
+        'student_amount': average_grad[1],
+        'failure_rate': average_general_failure_standard_deviation(degree),
+        'failure_actives': average_actives_failure_standard_deviation(degree),
+        'ira_average': calculate_average_general_ira_standard_deviation(degree),
+        'ira_actives': calculate_average_actives_ira_standard_deviation(degree),
+        'evasion_rate': calculate_general_evasion(degree),
+        'average_ira_graph': json.dumps(sorted(dic.items())),
+        'evasion_graph': json.dumps(sorted(graph_evasion(degree).items())),
+        'retirement': student_retirement(degree),
+        'amount_student_actives': amount_student_actives(degree),
+        'amount_locking': student_lock(degree),
+        'gradueted': student_gradueted(degree)
+    }
+
+    with io.open(path + '/degree.json', 'w', encoding='utf8') as output:
+        str_ = json.dumps(degree_data, indent = 4, sort_keys = True,
+                   separators=(',',': '), ensure_ascii = False)
+        output.write(to_unicode(str_))
+
+def generate_student_data(degree, path):
+    print("\t- Fazendo analises dos alunos")
+    students = Student.objects.filter(admission__degree = degree)
+    students_amount = students.count()
+    counter = 0
+    student_path = path + '/student'
+    if not os.path.exists(student_path):
+        os.mkdir(student_path)
+
+    for student in students:
+        student_klasses = StudentKlass.objects.filter(student=student)
+        amount_courses_semester = get_amount_courses_completed(student)
+        failures_semester = semester_pass_rate(student)
+        failures_amount_courses_semester = merge_dicts(
+            ['reprovacoes', 'cursadas'],
+            failures_semester,
+            amount_courses_semester,
+        )
+
+        ira_courses = sorted(ira_amount_courses(student).items())
+        pass_rate = calculate_pass_rate(student_klasses)
+        pass_rate_semester = sorted(failures_amount_courses_semester.items())
+        position = sorted(get_student_position(student).items())
+        real_period = get_real_period(student)
+        intended_period = get_intended_period(student)
+
+        dict_ira_semester = {}
+        dict_ira_amount_courses = {}
+        dict_position = {}
+        dict_pass = {}
+
+        for item, course_pass, pos in zip(ira_courses,pass_rate_semester, position):
+            ca = list(course_pass)
+            i = list(item)
+            p = list(pos)
+            d_pass, d_done = ca[1]
+            date = ca[0].split('/')
+
+            semester_data = {}
+            data = '{}/{}'.format(date[0], date[1])
+            
+            dict_ira_semester[data] = i[1][0]
+            dict_ira_amount_courses[data] = [i[1][0], d_done]
+            dict_position[data] = pos[1]
+            dict_pass[data] = [d_pass, d_done]
+        student_klasses = StudentKlass.objects.filter(student=student)
+        student_klass = []
+        for sk in student_klasses:
+            sk_dict = {
+                'grade': sk.grade,
+                'name': sk.klass.course.name,
+                'code': sk.klass.course.code,
+                'situation': sk.situation,
+                'year': sk.klass.year,
+                'semester': sk.klass.semester
+            }
+            student_klass.append(sk_dict)
+         
+        student_data = {
+            'ira_semester': dict_ira_semester,
+            'semester_pass_rate': dict_pass,
+            'position': dict_position,
+            'ira_amount_courses': dict_ira_amount_courses,
+            'pass_rate': pass_rate,
+            'intended_period': intended_period,
+            'real_period': real_period,
+            'student_klass': student_klass
+        }
+
+        counter += 1
+        with io.open(student_path + '/' + student.grr + '.json', 'w', encoding = 'utf8') as output:
+            str_ = json.dumps(student_data, indent = 3, sort_keys = True,
+                separators=(',', ': '), ensure_ascii = False)
+            output.write(to_unicode(str_))
+
+        if counter % 100 == 0:
+            print("\t\t- %d alunos processados de %d" % (counter, students_amount))
+    return 
+
+def generate_student_list_data(degree, path):
+    print("\t- Criando lista de alunos")
+    return
+
+def generate_admission_data(degree, path):
+    return
+
+def generate_admission_list_data(degree, path):
+    return
+
+def generate_course_data(degree, path):
+    print("\t- Fazendo analises das disciplinas")
+    return
+
+def generate_course_general_data(degree, path):
+    print("\t- Fazendo analise geral das disciplinas")
+    return
+
+def generate_cepe9615_data(degree, path):
+    return
+
+if __name__ == '__main__':
+    main()