diff --git a/Gemfile b/Gemfile index d4f33ea316da3d1fad8ad212de25f686241954c0..9dde2cacc987b91882e5340eacbec51531858b79 100644 --- a/Gemfile +++ b/Gemfile @@ -149,3 +149,5 @@ gem 'pundit' # elasticsearch integration gem 'searchkick' + +gem 'activerecord-import', '~> 0.11.0' diff --git a/app/services/learning_object_relations.rb b/app/services/learning_object_relations.rb index ffa0fc9415ecc53c376ac768232c4c2587990887..4e1976cdeca9821660a51ba7212d8500cdceceeb 100644 --- a/app/services/learning_object_relations.rb +++ b/app/services/learning_object_relations.rb @@ -1,12 +1,5 @@ class LearningObjectRelations - attr_reader :institutions, :topics - - def initialize - @institutions = {} - @topics = {} - end - def parse_topics values topics = [] values.compact! @@ -16,25 +9,5 @@ class LearningObjectRelations topics end - def create_topic name - topic = @topics[name] - if topic.nil? - topic = Topic.where(name: name).first_or_initialize - logger.create "Topic: #{topic.id.to_s} created" - end - @topics[name] = topic - topic - end - - def create_institution name - institution = @institutions[name] - if institution.nil? - institution = Institution.where(name: name).first_or_initialize - logger.create "Institution: #{institution.id.to_s} created" - end - @institutions[name] = institution - institution - end - end diff --git a/lib/tasks/create_relation.rake b/lib/tasks/create_relation.rake deleted file mode 100644 index 273de91c5929a63376f6632855f4bb5104325d79..0000000000000000000000000000000000000000 --- a/lib/tasks/create_relation.rake +++ /dev/null @@ -1,62 +0,0 @@ -namespace :topics_and_intitutions do - desc "Create relations" - - task :importer, [:log] => [:environment] do |t, args| - desc "Create LearningObject relations based on its metadata" - include Log::Logging - - args.with_defaults(:log => STDOUT) - - Log::Logging.logger = Log::DatabaseLogger.new(args.log) - logger.level = Log::DatabaseLogger::CREATE - - limit = 500 - offset = 5140 - created_topics = [] - institutions = [] - helper = LearningObjectRelations.new - - loop do - logger.info " --> Creating LearningObjects relations from #{offset} to #{offset+limit}" - - begin - #Get LearningObjects from PostgreSQL (from offset to offset+limit) - learning_objects = LearningObject.limit(limit).offset(offset) - - rescue => e - logger.warn "Database error, going to sleep" - logger.error e - sleep(10.seconds) - next - else - break if learning_objects.empty? - - learning_objects.each do |learning_object| - subjects_categories = learning_object.get_metadata_values_of "dc.subject.category" - institution = learning_object.get_metadata_values_of("dc.creator").first - topics = helper.parse_topics subjects_categories - - topics.each do |topic| - if !helper.topics[topic].nil? - created_topics.push helper.create_topic topic - end - end - - if !institution.nil? && !helper.institutions[institution].nil? - institutions.push helper.create_institution institution - end - end - - end - offset += limit - end - - Institution.import institutions - logger.info " --> Institutions imported" - Topic.import created_topics - logger.info " --> topics imported" - logger.close - end - - -end diff --git a/lib/tasks/create_relations.rake b/lib/tasks/create_relations.rake new file mode 100644 index 0000000000000000000000000000000000000000..0848093c44c382f778c5dc84ed199c8893779cb8 --- /dev/null +++ b/lib/tasks/create_relations.rake @@ -0,0 +1,74 @@ +namespace :create_relations do + desc "Create relations" + + task :learning_object, [:log] => [:environment] do |t, args| + desc "Create LearningObject relations based on its metadata" + include Log::Logging + + args.with_defaults(:log => STDOUT) + + Log::Logging.logger = Log::DatabaseLogger.new(args.log) + logger.level = Log::DatabaseLogger::CREATE + + new_topics = {} + new_institutions = {} + relations = [] + relations_order = [] + + limit = 500 + offset = 0 + helper = LearningObjectRelations.new + + loop do + logger.info " --> Creating LearningObjects relations from #{offset} to #{offset+limit}" + + begin + learning_objects = LearningObject.limit(limit).offset(offset) + + rescue => e + logger.warn "Database error, going to sleep" + logger.error e + sleep(10.seconds) + next + else + break if learning_objects.empty? + + learning_objects.each do |learning_object| + lo_topics = helper.parse_topics ( learning_object.get_metadata_values_of "dc.subject.category" ) + institution_name = learning_object.get_metadata_values_of("dc.creator").first + + lo_topics.each do |topic_name| + unless Topic.exists?(name: topic_name) + new_topics[topic_name] ||= Topic.where(name: topic_name).first_or_initialize + new_topics[topic_name].learning_objects << learning_object + relations_order.push(topic_name) + end + end + + unless institution_name.nil? || Institution.exists?(name:institution_name) + new_institutions[institution_name] ||= Institution.where(name: institution_name).first_or_initialize + end + + end + end + offset += limit + end + + Institution.import new_institutions.values + logger.info " --> Institutions imported" + Topic.import new_topics.values + logger.info " --> Topics imported" + + relations_order.each_with_index do |relation, i| + parent_id = Topic.find_by_name(relations_order[i - 1]).id + child_id = Topic.find_by_name(relation).id + relations.push TopicRelationship.where(parent_id: parent_id, child_id: child_id ).first_or_initialize unless i == 0 + end + + TopicRelationship.import relations + logger.info " --> TopicsRelations created" + logger.close + end + + +end