Merge branch 'rake_import' into 'master'

Rake import



See merge request !159

app/services/institutions_importer.rb

+class InstitutionsImporter
+
+
+  def initialize items
+    @items = items
+    @@institutions = {}
+  end
+
+
+  def import
+
+
+    @items.each do |learning_object|
+      institution_name = learning_object.get_metadata_values_of("dc.creator").first
+      next if institution_exist? institution_name ||  institution_name.nil?
+      @@institutions[institution_name] = Institution.where(name: institution_name).new
+    end
+
+    Institution.import @@institutions.values
+
+  end
+
+  private
+
+  def institution_exist? institution_name
+    Institution.exists? name: institution_name  || !@@institutions[institution_name].nil?
+  end
+
+end

app/services/learning_object_relations.rb

-class LearningObjectRelations
-
-  def parse_topics values
-    topics = []
-    values.compact!
-    values.each do |value|
-      topics = value.split('::')
-    end
-    topics
-  end
-
-
-end

app/services/topics_importer.rb

+class TopicsImporter
+
+
+  def initialize items
+    @items = items
+    @@topics = {}
+    @@topics_relations = []
+  end
+
+  def import
+
+    relations = []
+
+    @items.each do |learning_object|
+
+      lo_topics = parse_topics ( learning_object.get_metadata_values_of "dc.subject.category" )
+
+      lo_topics.each_with_index do |topic_name, i|
+
+        next if topic_exists? topic_name
+
+        topic = Topic.where(name: topic_name).new
+        topic.learning_objects << learning_object
+        create_relations(lo_topics[i - 1], topic_name) unless i == 0
+        @@topics[topic_name] = topic
+
+      end
+      separate_relations()
+    end
+
+    Topic.import @@topics.values
+    import_topics_relations()
+
+  end
+
+
+  private
+
+  def create_relations parent_name, child_name
+    @@topics_relations.push(parent_name)
+    @@topics_relations.push(child_name)
+  end
+
+  def separate_relations
+    @@topics_relations.push nil
+  end
+
+  def import_topics_relations
+
+    relations = []
+
+    @@topics_relations.each_with_index do |relation, i|
+      unless  relation.nil? || @@topics_relations[i - 1].nil?
+        parent_id =  Topic.find_by_name(@@topics_relations[i - 1]).id
+        child_id = Topic.find_by_name(relation).id
+        unless TopicRelationship.exists?(parent_id: parent_id, child_id: child_id) || i == 0
+          relations.push TopicRelationship.where(parent_id: parent_id, child_id: child_id ).new
+        end
+      end
+    end
+
+    TopicRelationship.import relations
+
+  end
+
+  def parse_topics values
+    topics = []
+    values.compact!
+    values.each do |value|
+      topics = value.split('::')
+    end
+    topics.map{|topic| topic.strip}
+  end
+
+  def topic_exists? (topic_name)
+    Topic.exists? name: topic_name || !@@topics[topic_name].nil?
+  end
+
+end

lib/tasks/create_relations.rake → lib/tasks/import.rake

-namespace :create_relations do
-  desc "Create relations"
+namespace :import do
+  desc "Import Topics and Institutions"
 
-  task :learning_object, [:log] => [:environment] do |t, args|
+  task :topics, [:log] => [:environment] do |t, args|
     desc "Create LearningObject relations based on its metadata"
     include Log::Logging
 
@@ -10,71 +10,75 @@ namespace :create_relations do
     Log::Logging.logger = Log::DatabaseLogger.new(args.log)
     logger.level = Log::DatabaseLogger::CREATE
 
-    new_topics = {}
-    new_institutions = {}
-    relations = []
-    relations_order = []
 
+    # Quantity of items fetched on each iteration
     limit = 500
+    # Start point from where items will be fetched
     offset = 0
-    helper = LearningObjectRelations.new
 
     loop do
       logger.info " --> Creating LearningObjects relations from #{offset} to #{offset+limit}"
 
       begin
-          learning_objects = LearningObject.limit(limit).offset(offset)
+        # Get learning_objects (from offset to offset+limit)
+        learning_objects = LearningObject.limit(limit).offset(offset)
 
         rescue => e
           logger.warn "Database error, going to sleep"
           logger.error e
+          # Sleeps for a while to wait database's recovery
           sleep(10.seconds)
+          # Goes to next iteration to retry
           next
         else
+          # Terminate loop if there are no more learning_objects to import
           break if learning_objects.empty?
 
-          learning_objects.each do |learning_object|
-              lo_topics = helper.parse_topics ( learning_object.get_metadata_values_of "dc.subject.category" )
-              institution_name = learning_object.get_metadata_values_of("dc.creator").first
+          importer = TopicsImporter.new (learning_objects)
+          importer.import
+
+        end
+        offset += limit
+    end
+    logger.close
+  end
+
+
+  task :institutions, [:log] => [:environment] do |t, args|
+    desc "Create LearningObject relations based on its metadata"
+    include Log::Logging
+
+    args.with_defaults(:log => STDOUT)
+
+    Log::Logging.logger = Log::DatabaseLogger.new(args.log)
+    logger.level = Log::DatabaseLogger::CREATE
+
+    limit = 500
+    offset = 0
+
+    loop do
+      logger.info " --> Creating LearningObjects relations from #{offset} to #{offset+limit}"
 
-              lo_topics.each do |topic_name|
-                unless Topic.exists?(name: topic_name)
-                  new_topics[topic_name] ||= Topic.where(name: topic_name).new
-                  new_topics[topic_name].learning_objects << learning_object
-                  relations_order.push(topic_name)
-                end
-                end
+      begin
+          learning_objects = LearningObject.limit(limit).offset(offset)
 
-              unless institution_name.nil? || Institution.exists?(name:institution_name)
-                new_institutions[institution_name] ||= Institution.where(name: institution_name).new
-              end
+        rescue => e
+          logger.warn "Database error, going to sleep"
+          logger.error e
+          sleep(10.seconds)
+          next
+        else
+          break if learning_objects.empty?
 
-              relations_order.push nil
+          importer = InstitutionsImporter.new (learning_objects)
+          importer.import
 
-          end
         end
         offset += limit
       end
 
-    Institution.import new_institutions.values
     logger.info " --> Institutions imported"
-    Topic.import new_topics.values
-    logger.info " --> Topics imported"
-
-    relations_order.each_with_index do |relation, i|
-      unless  relation.nil? || relations_order[i - 1].nil?
-        parent_id =  Topic.find_by_name(relations_order[i - 1]).id
-        child_id = Topic.find_by_name(relation).id
-        unless TopicRelationship.exists?(parent_id: parent_id, child_id: child_id) || i == 0
-          relations.push TopicRelationship.where(parent_id: parent_id, child_id: child_id ).new
-        end
-      end
-    end
-
-    TopicRelationship.import relations
-    logger.info " --> TopicsRelations created"
     logger.close
   end
 
-
 end