Skip to content
Snippets Groups Projects
Commit eada17ea authored by Mateus Rambo Strey's avatar Mateus Rambo Strey
Browse files

fix dspace item import

parent 2ce57584
No related branches found
No related tags found
No related merge requests found
...@@ -9,8 +9,8 @@ module Metadatable ...@@ -9,8 +9,8 @@ module Metadatable
def get_metadata_values_of key def get_metadata_values_of key
values = [] values = []
if @metadata.is_a? Array if metadata.is_a? String
@metadata.each do |m| JSON.parse(metadata).each do |m|
m = m.with_indifferent_access m = m.with_indifferent_access
values << m["value"] if m["key"] == key values << m["value"] if m["key"] == key
end end
...@@ -18,4 +18,4 @@ module Metadatable ...@@ -18,4 +18,4 @@ module Metadatable
values values
end end
end end
\ No newline at end of file
class LearningObject < ActiveRecord::Base class LearningObject < ActiveRecord::Base
include Metadatable
has_and_belongs_to_many :subjects has_and_belongs_to_many :subjects
has_many :collection_items, as: :collectionable has_many :collection_items, as: :collectionable
......
class CreateLearningObjects < ActiveRecord::Migration class CreateLearningObjects < ActiveRecord::Migration
def change def change
create_table :learning_objects do |t| create_table :learning_objects do |t|
t.integer :id_dspace t.integer :id_dspace, index: true
t.string :name t.string :name
t.string :author t.string :author
t.text :description t.text :description
t.datetime :published_at t.datetime :published_at
t.string :type t.string :object_type
t.integer :score t.integer :score
t.integer :school_level t.integer :school_level
t.string :language t.string :language
t.text :metadata t.text :metadata
t.text :bitstreams t.text :bitstreams
t.string :thumbnail
t.text :keywords t.text :keywords
t.references :published, polymorphic: true, index: true t.references :published, polymorphic: true, index: true
......
...@@ -13,7 +13,7 @@ namespace :dspace do ...@@ -13,7 +13,7 @@ namespace :dspace do
logger.level = Log::DatabaseLogger::CREATE logger.level = Log::DatabaseLogger::CREATE
# Quantity of items fetched on each iteration # Quantity of items fetched on each iteration
limit = 500 limit = 1000
# Start point from where items will be fetched # Start point from where items will be fetched
offset = 0 offset = 0
...@@ -24,7 +24,7 @@ namespace :dspace do ...@@ -24,7 +24,7 @@ namespace :dspace do
begin begin
# Get items from dspace (from offset to offset+limit) # Get items from dspace (from offset to offset+limit)
dspace_items = get_items_from_dspace(limit,offset, ['metadata','bitstreams']) dspace_items = get_items_from_dspace(limit, offset, ['metadata','bitstreams'])
rescue => e rescue => e
logger.warn "Database error, going to sleep" logger.warn "Database error, going to sleep"
logger.error e logger.error e
...@@ -36,6 +36,12 @@ namespace :dspace do ...@@ -36,6 +36,12 @@ namespace :dspace do
# Terminate loop if there are no more items to import # Terminate loop if there are no more items to import
break if dspace_items.empty? break if dspace_items.empty?
# continue if receive an error string
if dspace_items.is_a? String
logger.warn "Received a string instead of items: #{dspace_items}"
next
end
# Increment offset, to get new items on next iteration # Increment offset, to get new items on next iteration
offset = offset + limit offset = offset + limit
...@@ -44,12 +50,13 @@ namespace :dspace do ...@@ -44,12 +50,13 @@ namespace :dspace do
# Initializes new LearningObjects # Initializes new LearningObjects
# and saves them on LearningObjects's repository # and saves them on LearningObjects's repository
dspace_items.each do |item| dspace_items.each do |item|
result = learning_object_repository.get_by_dspace_id item.id if LearningObject.where(id_dspace: item.id).blank?
if result.nil?
learning_object = initialize_learning_object item learning_object = initialize_learning_object item
learning_object_repository.create learning_object if learning_object.save
# Comment the line below if you don't want print the itens logger.create "Imported: #{item.id} -> #{learning_object.id} "
logger.create "Imported: #{item.id} -> #{learning_object.id} " else
logger.warn "Error importing: #{item.id}"
end
end end
end end
end end
...@@ -126,7 +133,7 @@ namespace :dspace do ...@@ -126,7 +133,7 @@ namespace :dspace do
def get_items_from_dspace(limit, offset, expand=[]) def get_items_from_dspace(limit, offset, expand=[])
dspace_repository.items.all( dspace_repository.items.all(
expand: expand, expand: expand.join(','),
limit: limit, offset: offset limit: limit, offset: offset
) )
end end
...@@ -148,22 +155,23 @@ namespace :dspace do ...@@ -148,22 +155,23 @@ namespace :dspace do
def initialize_learning_object item def initialize_learning_object item
metadata = build_array_of(item.metadata) metadata = build_array_of(item.metadata)
bitstreams = build_array_of(item.bit_streams) # bitstreams = build_array_of(item.bit_streams)
current_date = Time.now current_date = Time.now
lo = LearningObject.new( lo = LearningObject.new(
:name => item.name, name: item.name,
:last_modified => current_date, updated_at: current_date,
:created_at => current_date, created_at: current_date,
:id_dspace => item.id, id_dspace: item.id,
:thumbnail => nil, thumbnail: nil,
:bitstreams => bitstreams, bitstreams: item.bit_streams.to_json,
:metadata => metadata metadata: item.metadata.to_json
) )
lo.author = lo.get_metadata_values_of("dc.contributor.author").join(', ') lo.author = lo.get_metadata_values_of('dc.contributor.author').join(', ')
lo.description = lo.get_metadata_value_of("dc.description") lo.description = lo.get_metadata_value_of('dc.description')
lo.type = lo.get_metadata_value_of("dc.type") lo.object_type = lo.get_metadata_value_of('dc.type')
date = lo.get_metadata_value_of("dc.date.issued") lo.language = lo.get_metadata_value_of('dc.language')
date = lo.get_metadata_value_of('dc.date.issued')
lo.published_at = Time.iso8601(date) unless date.nil? lo.published_at = Time.iso8601(date) unless date.nil?
lo lo
end end
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment