diff --git a/lib/tasks/import/learning_objects.rake b/lib/tasks/import/learning_objects.rake index 39f2ae19093012d17b65697ea572de4cb0f3cfe1..9927bbeee96c46b7b576ae9228a8658fae9a93b4 100644 --- a/lib/tasks/import/learning_objects.rake +++ b/lib/tasks/import/learning_objects.rake @@ -46,33 +46,46 @@ namespace :import do n_items = 0 loop do - items = [] - uri.query = URI.encode_www_form(user_params) - response = JSON.parse(Net::HTTP.get(uri)) - if n_items == 0 - n_items = response["response"]["numFound"] - p "#{n_items} items found" - end + begin + items = [] + uri.query = URI.encode_www_form(user_params) + response = JSON.parse(Net::HTTP.get(uri)) + if n_items == 0 + n_items = response["response"]["numFound"] + p "#{n_items} items found" + end - break if params[:start] > n_items - p "--> importing items from #{params[:start]} to #{params[:start] + params[:rows]}" - - response["response"]["docs"].each do |doc| - next if item_exists?(doc["search.resourceid"]) - item = dspace_client.items.find( - id: doc["search.resourceid"], - expand: "metadata,bitstreams,parentCollection" - ) - if items.is_a? String - p "Received a string instead of item: #{item}" - next + break if params[:start] > n_items + p "--> importing items from #{params[:start]} to #{params[:start] + params[:rows]}" + + response["response"]["docs"].each do |doc| + next if item_exists?(doc["search.resourceid"]) + item = dspace_client.items.find( + id: doc["search.resourceid"], + expand: "metadata,bitstreams,parentCollection" + ) + if items.is_a? String + p "Received a string instead of item: #{item}" + next + end + items << item end - items << item + p "#{items.count} items retrieved" + importer = Dspace::LearningObjectImporter.new(items, user, dspace_client) + importer.import + rescue PG::ConnectionBad, Faraday::TimeoutError, Net::HTTP::Persistent::Error, Net::ReadTimeout => e + p "Database error, going to sleep" + p e + p e.class + p e.message + ActiveRecord::Base.clear_active_connections! + # Sleeps for a while to wait database's recovery + sleep(60.seconds) + # Goes to next iteration to retry + next + else + params[:start] += params[:rows] end - params[:start] += params[:rows] - p "#{items.count} items retrieved" - importer = Dspace::LearningObjectImporter.new(items, user, dspace_client) - importer.import end end end