Merge branch 'fix_orientdb_scripts' into 'master'

Fix database scripts

Added author, grade_level and published_at to LearningObject OrientDB schema and unique indexes for Attribute and Subject.
Fixed tasks dspace:import and orientdb:create_learning_object_relations.

See merge request !76

app/builders/learning_object_builder.rb

@@ -23,18 +23,21 @@ class LearningObjectBuilder
     unless args.nil?
       # cache object when build
       lo = Rails.cache.fetch(cache_key(args['@rid'], args['last_modified']), expires_in: 12.hours) do
-        LearningObject.new(
+        obj = LearningObject.new(
           id: args['@rid'],
           name: args['name'],
           description: args['description'],
           thumbnail: args['thumbnail'],
-          created_at: args['created_at'],
           id_dspace: args['id_dspace'],
           type: args['type'],
           bitstreams: args['bitstreams'],
           last_modified: args['last_modified'],
           metadata: JSON.parse(args['metadata'])
         )
+        obj.created_at = DateTime.strptime(args['created_at'], "%Y-%m-%d %H:%M:%S") unless args['created_at'].nil?
+        obj.last_modified = DateTime.strptime(args['last_modified'], "%Y-%m-%d %H:%M:%S") unless args['last_modified'].nil?
+        obj.published_at = DateTime.strptime(args['published_at'], "%Y-%m-%d %H:%M:%S") unless args['published_at'].nil?
+        obj
       end
       lo.likes = args.key?('in_Likes') ? args['in_Likes'].size : 0
       lo.views = args.key?('in_Views') ? args['in_Views'].size : 0

app/models/learning_object.rb

@@ -4,7 +4,7 @@ class LearningObject
 
   attr_accessor :id, :id_dspace, :rid, :name, :author, :description,
                 :published_at, :thumbnail, :created_at, :last_modified,
-                :type, :bitstreams, :metadata, :likes, :views,
+                :type, :grade_level, :bitstreams, :metadata, :likes, :views,
                 :downloads, :subjects, :attributes
 
   validates_presence_of :name, :created_at, :type, :likes, :views, :downloads
@@ -47,6 +47,7 @@ class LearningObject
   def get_metadata_values_of key
     values = []
     @metadata.each do |m|
+      m = m.with_indifferent_access
       values << m["value"] if m["key"] == key
     end
     values
@@ -81,8 +82,9 @@ class LearningObject
     {
         likes: 0,
         views: 0,
-        downloads: 0,
-        created_at: DateTime.now.strftime("%Y-%m-%d %H:%M:%S")
+        downloads: 0
+        # Setting this default value wont allow created_at to be set trough the constructor
+        # created_at: DateTime.new
     }
   end
 

app/repositories/orient_db/attribute_repository.rb

@@ -2,22 +2,24 @@ module OrientDb
   class AttributeRepository < Base
     
     def find_by_key_and_value(key, value)
-        result = connection.query sprintf("SELECT FROM %s WHERE key = '%s' AND value = '%s'", odb_class, key, value)
-        build_objects(result)
+      # Escape double quotes for OrientDB
+      v = value.gsub('"','\\"')
+      result = connection.query("SELECT EXPAND(rid) FROM INDEX:attr_unique WHERE key = [\"#{key}\", \"#{v}\"]")
+      build_object(result.first)
     end
 
     def build_object(args={})
-      subject = nil
+      attribute = nil
       unless args.nil?
-        subject = Attribute.new(:id => args["@rid"])
+        attribute = Attribute.new(:id => args["@rid"])
         args.each do |var, val|
           var_name = "@"+var
-          if subject.respond_to?(var)
-            subject.instance_variable_set(var_name, val)
+          if attribute.respond_to?(var)
+            attribute.instance_variable_set(var_name, val)
           end
         end
       end
-      subject
+      attribute
     end
     
     private

app/repositories/orient_db/base.rb

@@ -45,6 +45,8 @@ class OrientDb::Base
     objects
   end
 
+  # Take the object and make a hash in the OrientDB format. 
+  # Used to create a document.
   def build_hash(object)
     hash = {'@class' => odb_class}
     object.instance_variables.each do |var|
@@ -52,6 +54,9 @@ class OrientDb::Base
       hash[var_name] = sanitize_orientdb_values(object.instance_variable_get(var))
     end
     hash.delete('id')
+    # TODO: find a better way to ignore variables from ActiveModel
+    hash.delete("validation_context")
+    hash.delete("errors")
     hash
   end
 
@@ -60,6 +65,9 @@ class OrientDb::Base
   end
 
   def sanitize_orientdb_values(val)
+    if val.respond_to? "strftime"
+      val = val.strftime("%Y-%m-%d %H:%M:%S")
+    end
     val
   end
 

app/repositories/orient_db/learning_object_repository.rb

@@ -186,6 +186,8 @@ module OrientDb
       hash.delete("downloads")
       hash.delete("subjects")
       hash.delete("attributes")
+
+      hash["metadata"] = hash["metadata"].to_json
       hash
     end
 

lib/orient_db/migration.rb

@@ -38,8 +38,8 @@ module OrientDb
     def add_index(klass, properties, type, name=nil, engine=nil, metadata=nil)
       engine = engine.nil? ? "" : " ENGINE #{engine}"
       metadata = metadata.nil? ? "" : " METADATA #{metadata.to_s}"
-      props = stringify(properties, ", ")
-      name = name.nil? ? "#{klass}."+stringify(properties, "_") : name;
+      props = properties.join(',')
+      name = name.nil? ? "#{klass}."+properties.join("_") : name;
 
       begin
         index = "CREATE INDEX #{name} ON #{klass} (#{props}) #{type}#{engine}#{metadata}"
@@ -53,16 +53,6 @@ module OrientDb
         end
       end
     end
-
-    private
-
-    def stringify(array, separator)
-      str = ""
-      array.each do |value|
-        str += value.to_s+separator
-      end
-      str[0..-(separator.size+1)]
-    end
-
+    
   end
 end
\ No newline at end of file

lib/orient_db/migrations.rb

@@ -13,7 +13,7 @@ class OrientDb::Migrations
     # Vertices inheriting from Object
     @migrations << CreateInstitution.new(client)
     @migrations << CreateLearningObject.new(client)
-    # Vertices inheriting from LearningObject
+    # Vertices dependent on LearningObject
     @migrations << CreateCollection.new(client)
     @migrations << CreateSubject.new(client)
     @migrations << CreateMainPage.new(client)
@@ -27,6 +27,7 @@ class OrientDb::Migrations
     @migrations << CreateComments.new(client)
     @migrations << CreateCommentedBy.new(client)
     @migrations << CreateComplaint.new(client)
+    @migrations << CreateSubtopic.new(client)
   end
 
   ##

lib/orient_db/migrations/create_attribute.rb

@@ -5,7 +5,7 @@ class OrientDb::Migrations::CreateAttribute < OrientDb::Migration
         c.property 'key', :string
         c.property 'value', :string
     end
-    add_index 'Attribute', ['key'], "NOTUNIQUE_HASH_INDEX"
+    add_index 'Attribute', ['key','value'], "UNIQUE_HASH_INDEX", "attr_unique"
     metadata = {:analyzer => "org.apache.lucene.analysis.br.BrazilianAnalyzer"}
     add_index 'Attribute', ['value'], "FULLTEXT", nil, "LUCENE", metadata.to_json
   end

lib/orient_db/migrations/create_collection.rb

@@ -3,6 +3,7 @@ class OrientDb::Migrations::CreateCollection < OrientDb::Migration
   def up
     create_class 'Collection', 'Object' do |c|
         c.property 'privacy', :string
+        c.property 'description', :string
         c.link     'learning_objects',  :linkset, 'LearningObject'
     end
   end

lib/orient_db/migrations/create_comment.rb

 class OrientDb::Migrations::CreateComment < OrientDb::Migration
 
   def up
-    create_class 'Comment', 'Object' do |c|
+    create_class 'Comment', 'V' do |c|
       c.property 'text', :string
+      c.property 'created_at', :datetime
     end
   end
 

lib/orient_db/migrations/create_learning_object.rb

@@ -6,6 +6,9 @@ class OrientDb::Migrations::CreateLearningObject < OrientDb::Migration
         c.property 'description', :string
         c.property 'thumbnail', :string
         c.property 'type', :string
+        c.property 'author', :string
+        c.property 'grade_level', :string
+        c.property 'published_at', :datetime
         c.property 'metadata', :string
     end
     add_index 'LearningObject', ['id_dspace'], "UNIQUE_HASH_INDEX"

lib/orient_db/migrations/create_subject.rb

@@ -4,6 +4,7 @@ class OrientDb::Migrations::CreateSubject < OrientDb::Migration
     create_class 'Subject', 'Object' do |c|
         c.link 'highlights',  :linkset, 'LearningObject'
     end
+    add_index 'Subject', ['name'], "UNIQUE_HASH_INDEX"
   end
 
   def down

lib/orient_db/migrations/create_subtopic.rb

+class OrientDb::Migrations::CreateSubtopic < OrientDb::Migration
+
+  def up
+    create_class 'Subtopic', 'E'
+  end
+
+  def down
+    drop_class 'Subtopic'
+  end
+end
\ No newline at end of file

lib/tasks/dspace.rake

@@ -41,10 +41,9 @@ namespace :dspace do
           result = learning_object_repository.get_by_dspace_id item.id
           if result.nil?
             learning_object = initialize_learning_object item
-            learning_object_repository.save learning_object
+            learning_object_repository.create learning_object
           end
         end
-
       end
 
     end
@@ -67,21 +66,23 @@ namespace :dspace do
   def initialize_learning_object item
     metadata = build_array_of(item.metadata)
     bitstreams = build_array_of(item.bit_streams)
-    current_date = Time.new
+    current_date = Time.now
 
-    LearningObject.new(
+    lo = LearningObject.new(
       :name => item.name,
-      :author => select_value_of(metadata, "dc.contributor.author"),
-      :description => select_value_of(metadata, "dc.description"),
-      :published_at => select_value_of(metadata, "dc.date.issued"),
-      :created_at => current_date,
       :last_modified => current_date,
+      :created_at => current_date,
       :id_dspace => item.id,
       :type => item.type,
       :thumbnail => nil,
       :bitstreams => bitstreams,
       :metadata => metadata
     )
+    lo.author = lo.get_metadata_values_of("dc.contributor.author").join(', ')
+    lo.description = lo.get_metadata_value_of("dc.description")
+    date = lo.get_metadata_value_of("dc.date.issued")
+    lo.published_at = Time.iso8601(date) unless date.nil?
+    lo
   end
 
   def build_array_of(item_content=[])
@@ -93,11 +94,4 @@ namespace :dspace do
     return content_array
   end
 
-  def select_value_of(array, key)
-    descriptions = array.select { |a|  a[:key] == key }
-    unless descriptions.empty?
-      descriptions.first[:value]
-    end
-  end
-
 end

lib/tasks/orientdb.rake

@@ -26,12 +26,14 @@ namespace :orientdb do
     offset = 0
 
     loop do
-      puts " --> importing LearningObjects from #{offset} to #{offset+limit}"
+      puts " --> creating LearningObjects realations from #{offset} to #{offset+limit}"
 
       begin
         # Get LearningObjects from OrientDB (from offset to offset+limit)
         learning_objects = learning_object_repository.all_from_offset_to_limit(offset,limit)
-      rescue
+      rescue => e
+        puts e
+        puts e.backtrace
         # Sleeps for a while to wait database's recovery
         sleep(30.seconds)
         # Goes to next iteration to retry
@@ -41,31 +43,61 @@ namespace :orientdb do
         break if learning_objects.empty?
 
         learning_objects.each do |lo|
+          puts "LO ID ==== "+lo.id.to_s
+          # Get metadata as a hash with metadata name as key
           metadata = get_unique_matadata_keys(lo)
+          # Get subjects
           subjects = metadata["dc.subject.category"]
+          # If subjects is nil, set as empty array
           subjects ||= []
+          # Remove subjects from the metadata hash, so they wont be inserted as Attributes
           metadata.delete("dc.subject.category")
-
-          subjects_array=[]
-          attributes_array=[]
-
-          subjects.each do |subject_name|
-            subjects_array << {:name => subject_name}
+          # Set LearningObject relations attributes so it wont perform a search in the database when accessed
+          lo.subjects=[]
+          lo.attributes=[]
+          # Remove nil values from subjects array
+          subjects.compact!
+          subjects.each do |name|
+            # Look for subject with the same name
+            subject = subject_repository.find_by_name(name)
+            if subject.nil?
+              # Create a Subject with this name if it doesnt exist
+              subject = Subject.new(:name => name)
+              subject_repository.create(subject)
+            end
+            # Add subject on the LearningObject list
+            lo.subjects << subject
           end
 
-          metadata.each do |key, value|
-            attributes_array << {:key => key, :value => value}
+          metadata.each do |key, values|
+            # Remove nil values from values array
+            values.compact!
+            values.each do |v|
+              # Remove characters that might throw exceptions in OrientDB
+              v = v.delete('\\').delete('[').delete(']')
+              # Look for attribute with the same key and value
+              attribute = attribute_repository.find_by_key_and_value(key,v)
+              if attribute.nil?
+                # Create an Attribute with this key and value if it doesnt exist
+                attribute = Attribute.new(:key => key, :value => v)
+                attribute_repository.create(attribute)
+              end
+              # Add attribute on the LearningObject list
+              lo.attributes << attribute
+            end
           end
-
-          CreateRelationsWorker.perform_async(lo.id, subjects_array, attributes_array)
+          
+          # Create LearningObject relations that were added to its attrs
+          learning_object_repository.create_relations(lo)
         end
-
+        # Increase offset for next loop
         offset += limit
       end
     end
 
   end
 
+  # For each unique key on LearningObject metadata, get all its values and add on a new hash
   def get_unique_matadata_keys(lo)
     hash = {}
     lo.metadata.each do |m|