Merge branch 'generate-thumbnails' into sidekiq-workers

Gemfile

@@ -112,3 +112,13 @@ group :test do
   gem 'shoulda'
   gem 'shoulda-callback-matchers', '~> 1.1.1'
 end
+
+gem 'streamio-ffmpeg', '~> 1.0.0'
+
+# sidekiq
+gem 'sidekiq'
+gem 'sinatra', require: false
+gem 'slim'
+
+# CUrl
+gem 'curb', '~> 0.8.8'

app/models/learning_object.rb

@@ -38,6 +38,20 @@ class LearningObject
     values
   end
 
+  def get_bitstream_retrievelink_of name
+    values = @bitstreams.select { |v| v["bundleName"] == name }
+    unless values.empty?
+      return Dspace::Config.rest_url + values.first["retrieveLink"]
+    end
+  end
+
+  def get_bitstream_filename_of name
+    values = @bitstreams.select { |v| v["bundleName"] == name }
+    unless values.empty?
+      return values.first["name"]
+    end
+  end
+
   private
 
   def defaults

app/repositories/orient_db/learning_object_repository.rb

@@ -20,9 +20,9 @@ module OrientDb
     #   list.each do |learning_object|
     #     learning_object.inspect <LearningObject model>
     #   end
-    def all
-      learning_objects_hash = connection.query "SELECT FROM LearningObject"
-      build_objects(learning_objects_hash) || []
+    def all(limit = 100, offset = 0)
+      learning_objects_hash = connection.query "SELECT FROM LearningObject LIMIT #{limit} OFFSET #{offset}", {limit: limit}
+      learning_objects = build_learning_objects(learning_objects_hash) || []
     end
 
     # Usage:
@@ -102,6 +102,10 @@ module OrientDb
 
     private
 
+    def accepted_properties
+      ['thumbnail']
+    end
+
     def create_edges_from_array(edge_class, id, array, unique=false)
       edges = []
       array.each do |o|

app/workers/thumbnail_generator_worker.rb

+class ThumbnailGeneratorWorker
+
+  include Sidekiq::Worker
+
+  def perform(learning_object_id)
+
+    root_dir = Rails.root.join('public')
+    thumbnails_dir = "/thumbnails"
+    create_dir("#{root_dir}#{thumbnails_dir}")
+
+    default_size = "600x600"
+    default_thumbnail = "#{thumbnails_dir}/default_thumbnail.jpg"
+
+    item = item_database_repository.find(learning_object_id)
+    retrieve_link = item.get_bitstream_retrievelink_of "ORIGINAL"
+    filename = item.get_bitstream_filename_of "ORIGINAL"
+
+    unless accepted_formats.include? File.extname(filename)
+      item.thumbnail = default_thumbnail
+    else
+      begin
+        file = download_bitstream(retrieve_link,filename)
+      rescue
+        puts "ERROR!!!"
+      else
+        item_hash = encode_hash_from item.to_json
+
+        item.thumbnail = "#{thumbnails_dir}/#{item_hash}_#{default_size}.jpg"
+
+        thumbnail_output_path = "#{root_dir}#{item.thumbnail}"
+
+        if accepted_video_formats.include? File.extname(filename)
+          generate_video_thumbnail(file,default_size,thumbnail_output_path)
+        else
+          generate_image_thumbnail(file,default_size,thumbnail_output_path)
+        end
+
+        delete_downloaded_bitstream(file)
+      end
+    end
+
+    item_database_repository.update_property(item,"thumbnail",item.thumbnail)
+
+  end
+
+  private
+
+  def item_database_repository
+    @item_database_repository ||= Portalmec::Application.repository.for(:learning_object)
+  end
+
+  def encode_hash_from(object)
+    Digest::SHA1.hexdigest object
+  end
+
+  def create_dir(target)
+    unless File.directory?(target)
+      FileUtils.mkdir(target, :mode => 0700)
+    end
+  end
+
+  def generate_video_thumbnail(input,geom,output)
+    movie = FFMPEG::Movie.new(input)
+    frame = (movie.duration * 25/100).floor
+    movie.screenshot(output,
+                      { seek_time: frame, resolution: geom },
+                      preserve_aspect_ratio: :width)
+  end
+
+  def generate_image_thumbnail(input,geom,output)
+    # Read the image and resize it. The `change_geometry' method
+    # computes the new image geometry and yields to a block. The
+    # return value of the block is the return value of the method.
+    img = Magick::Image.read(input)[0]
+    img.change_geometry!(geom) { |cols, rows| img.thumbnail! cols, rows }
+    img.write(output)
+  end
+
+  def delete_downloaded_bitstream(file)
+    File.unlink(file) if File.exist?(file)
+  end
+
+  def download_bitstream(url, filename)
+    output_dir = "/tmp"
+
+    output_file = "#{output_dir}/#{filename}"
+
+    c = Curl::Easy.new(url)
+    c.ssl_verify_peer = false
+    c.ssl_verify_host = false
+    File.open(output_file, 'wb') do |f|
+      # c.on_progress { |dl_total, dl_now, ul_total, ul_now|
+      #   puts "#{dl_total}, #{dl_now}, #{ul_total}, #{ul_now}"; true
+      # }
+      c.on_body {|data| f << data; data.size }
+      c.perform
+    end
+
+    return output_file
+  end
+
+  def accepted_video_formats
+    [".mp4", ".wmv"]
+  end
+
+  def accepted_image_formats
+    [".jpg", ".gif", ".png"]
+  end
+
+  def accepted_other_formats
+    [".pdf"]
+  end
+
+  def accepted_formats
+    return accepted_video_formats + accepted_image_formats + accepted_other_formats
+  end
+
+end

config/routes.rb

 Rails.application.routes.draw do
+  require 'sidekiq/web'
+
   devise_for :users
 
   namespace :auth do
@@ -18,7 +20,10 @@ Rails.application.routes.draw do
     end
 
     resources :users
+    resources :highlights
     resources :carousels
+
+    mount Sidekiq::Web, at: '/sidekiq'
   end
 
   root 'welcome#index'
@@ -39,4 +44,6 @@ Rails.application.routes.draw do
   get '/contact' => 'welcome#contact', as: 'contact'
   get '/complaint' => 'welcome#complaint', as: 'complaint'
   get '/search' => 'search#index', as: 'search'
+
+  mount Sidekiq::Web => '/sidekiq'
 end

lib/tasks/thumbnail.rake

+namespace :thumbnail do
+
+  desc "Generate Thumbnails"
+  task :generate => :environment do
+
+    # Quantity of items fetched on each iteration
+    limit = 1000
+
+    # Start point from where items will be fetched
+    offset = 0
+
+    loop do
+
+      begin
+        # Get items from dspace (from offset to offset+limit)
+        items = item_database_repository.all(limit,offset)
+      rescue
+        # Sleeps for a while to wait database's recovery
+        sleep(30.seconds)
+        # Goes to next iteration to retry
+        next
+      else
+        # Terminate loop if there are no more items to import
+        break if items.empty?
+
+        # Increment offset, to get new items on next iteration
+        offset = offset + limit
+
+        items.each do |item|
+          ThumbnailGeneratorWorker.perform_async(item.id)
+        end
+
+      end
+    end
+  end
+
+  private
+
+  def item_database_repository
+    @item_database_repository ||= Portalmec::Application.repository.for(:learning_object)
+  end
+
+end