diff --git a/Gemfile b/Gemfile
index 35f19b90ca20f0547204f1613091dda58278fef5..43a380a159c7075835da3bd72b1430ea344b9545 100644
--- a/Gemfile
+++ b/Gemfile
@@ -112,3 +112,13 @@ group :test do
gem 'shoulda'
gem 'shoulda-callback-matchers', '~> 1.1.1'
end
+
+gem 'streamio-ffmpeg', '~> 1.0.0'
+
+# sidekiq
+gem 'sidekiq'
+gem 'sinatra', require: false
+gem 'slim'
+
+# CUrl
+gem 'curb', '~> 0.8.8'
diff --git a/app/models/learning_object.rb b/app/models/learning_object.rb
index f6a4f61b637f15ec32f1fd0c0fcba2205db1957e..b28f497a28be4e68517e93f9ae4d0448bbcbbdd8 100644
--- a/app/models/learning_object.rb
+++ b/app/models/learning_object.rb
@@ -38,6 +38,20 @@ class LearningObject
values
end
+ def get_bitstream_retrievelink_of name
+ values = @bitstreams.select { |v| v["bundleName"] == name }
+ unless values.empty?
+ return Dspace::Config.rest_url + values.first["retrieveLink"]
+ end
+ end
+
+ def get_bitstream_filename_of name
+ values = @bitstreams.select { |v| v["bundleName"] == name }
+ unless values.empty?
+ return values.first["name"]
+ end
+ end
+
private
def defaults
diff --git a/app/repositories/orient_db/learning_object_repository.rb b/app/repositories/orient_db/learning_object_repository.rb
index 989492fd42993fa30351368fad7562cbe88b84ef..2a3d779a4cc5ef2b34cc4118e50f58e9df5fdae0 100644
--- a/app/repositories/orient_db/learning_object_repository.rb
+++ b/app/repositories/orient_db/learning_object_repository.rb
@@ -15,16 +15,6 @@ module OrientDb
create_edge "Likes", user.rid, learning_object.id
end
- # Example:
- # list = repository.for(:learning_objects).all
- # list.each do |learning_object|
- # learning_object.inspect <LearningObject model>
- # end
- def all
- learning_objects_hash = connection.query "SELECT FROM LearningObject"
- build_objects(learning_objects_hash) || []
- end
-
# Usage:
# learning_object = repository.for(:learning_objects).get_by_dspace_id 123
#
@@ -49,6 +39,16 @@ module OrientDb
edges.flatten
end
+ def save(learning_object)
+ result = connection.command "INSERT INTO LearningObject CONTENT #{learning_object.to_json}"
+ end
+
+ def update_property(learning_object,property,value)
+ if accepted_properties.include? property
+ connection.command "UPDATE LearningObject SET #{property}='#{value}' WHERE @rid = #{learning_object.id}"
+ end
+ end
+
##
# To create "index:learningobject_search" on OrientDB, use the following command:
# CREATE INDEX learningobject_search
@@ -95,6 +95,20 @@ module OrientDb
private
+ def accepted_properties
+ ['thumbnail']
+ end
+
+ def create_edges_from_array(edge_class, id, array, unique=false)
+ edges = []
+ array.each do |o|
+ unless unique && edge_exists?(edge_class, id, o.id)
+ edges << create_edge(edge_class, id, o.id)
+ end
+ end
+ edges
+ end
+
def odb_class
"LearningObject"
end
diff --git a/app/workers/massive_likes_creator_worker.rb b/app/workers/massive_likes_creator_worker.rb
new file mode 100644
index 0000000000000000000000000000000000000000..79506a6831d6571a912b2a24990054f0f60ed616
--- /dev/null
+++ b/app/workers/massive_likes_creator_worker.rb
@@ -0,0 +1,21 @@
+class MassiveLikesCreatorWorker
+
+ include Sidekiq::Worker
+ include RepositoriesProxy
+
+ def perform(item_id, users_ids)
+
+ item = learning_object_repository.find item_id
+
+ users_ids.each do |user_id|
+ user = User.find user_id
+ begin
+ learning_object_repository.like user, item
+ rescue
+ next
+ end
+ end
+
+ end
+
+end
diff --git a/app/workers/massive_views_creator_worker.rb b/app/workers/massive_views_creator_worker.rb
new file mode 100644
index 0000000000000000000000000000000000000000..a5903a0a87314ecb87074abfc47cf1e90e4ab418
--- /dev/null
+++ b/app/workers/massive_views_creator_worker.rb
@@ -0,0 +1,21 @@
+class MassiveViewsCreatorWorker
+
+ include Sidekiq::Worker
+ include RepositoriesProxy
+
+ def perform(item_id, users_ids)
+
+ item = learning_object_repository.find item_id
+
+ users_ids.each do |user_id|
+ user = User.find user_id
+ begin
+ learning_object_repository.increment_views user, item
+ rescue
+ next
+ end
+ end
+
+ end
+
+end
diff --git a/app/workers/thumbnail_generator_worker.rb b/app/workers/thumbnail_generator_worker.rb
new file mode 100644
index 0000000000000000000000000000000000000000..75b809f3f6141524b8a98ee833ccf87476ebb84a
--- /dev/null
+++ b/app/workers/thumbnail_generator_worker.rb
@@ -0,0 +1,58 @@
+class ThumbnailGeneratorWorker
+
+ include Sidekiq::Worker
+ include RepositoriesProxy
+ include Thumbnail::Creation
+ include Thumbnail::Formats
+
+ def perform(learning_object_id)
+
+ item = learning_object_repository.find(learning_object_id)
+ filename = item.get_bitstream_filename_of "ORIGINAL"
+
+ size = "530x298"
+
+ unless accepted_formats.include? file_format filename
+ item.thumbnail = default_thumbnail
+ else
+ begin
+ retrieve_link = item.get_bitstream_retrievelink_of "ORIGINAL"
+ file = download_bitstream(retrieve_link,filename)
+ rescue
+ puts "ERROR!!! Some error occurred during file download."
+ else
+ item.thumbnail = generate_thumbnail(file,filename,size)
+ delete_downloaded_bitstream(file)
+ end
+ end
+
+ learning_object_repository.update_property(item,"thumbnail",item.thumbnail)
+
+ end
+
+ private
+
+ def delete_downloaded_bitstream(file)
+ File.unlink(file) if File.exist?(file)
+ end
+
+ def download_bitstream(url, filename)
+ output_dir = "/tmp"
+
+ output_file = "#{output_dir}/#{filename}"
+
+ c = Curl::Easy.new(url)
+ c.ssl_verify_peer = false
+ c.ssl_verify_host = false
+ File.open(output_file, 'wb') do |f|
+ # c.on_progress { |dl_total, dl_now, ul_total, ul_now|
+ # puts "#{dl_total}, #{dl_now}, #{ul_total}, #{ul_now}"; true
+ # }
+ c.on_body {|data| f << data; data.size }
+ c.perform
+ end
+
+ return output_file
+ end
+
+end
diff --git a/config/routes.rb b/config/routes.rb
index 597cf4ff4424ca60fa87624a09dd04e2afbf84ce..dda50d5997d878742652821529a377cb0dc83b2c 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -1,4 +1,6 @@
Rails.application.routes.draw do
+ require 'sidekiq/web'
+
devise_for :users
namespace :auth do
@@ -18,7 +20,10 @@ Rails.application.routes.draw do
end
resources :users
+ resources :highlights
resources :carousels
+
+ mount Sidekiq::Web, at: '/sidekiq'
end
root 'welcome#index'
diff --git a/lib/dspace/client.rb b/lib/dspace/client.rb
index a88bb8477206f7742e6ef4b7b675ffc8c191d031..e3dd70c904575bf94a3a9665e5051d3dc3ec4e06 100644
--- a/lib/dspace/client.rb
+++ b/lib/dspace/client.rb
@@ -1,2 +1,16 @@
-class OrientDb::Client
+class Dspace::Client
+ @@client = nil
+
+ def self.instance
+ if !@@client.nil?
+ return @@client
+ end
+
+ begin
+ @@client = DspaceClient.new(Dspace::Config.rest_url)
+ rescue
+ raise 'Wrong orient db credentials'
+ end
+ end
+
end
diff --git a/lib/tasks/dspace.rake b/lib/tasks/dspace.rake
new file mode 100644
index 0000000000000000000000000000000000000000..f2c2770d952d93686f000382b9e5be6ec6100523
--- /dev/null
+++ b/lib/tasks/dspace.rake
@@ -0,0 +1,101 @@
+require 'dspace_rest_client'
+
+namespace :dspace do
+ desc "Dspace tasks"
+
+ task import: :environment do
+ desc "Import DSpace items to LearningObject Repositoy"
+
+ include RepositoriesProxy
+
+ # Quantity of items fetched on each iteration
+ limit = 500
+
+ # Start point from where items will be fetched
+ offset = 0
+
+ loop do
+
+ puts " --> importing items from #{offset} to #{offset+limit}"
+
+ begin
+ # Get items from dspace (from offset to offset+limit)
+ dspace_items = get_items_from_dspace(limit,offset)
+ rescue
+ # Sleeps for a while to wait database's recovery
+ sleep(10.seconds)
+ # Goes to next iteration to retry
+ next
+ else
+ # Terminate loop if there are no more items to import
+ break if dspace_items.empty?
+
+ # Increment offset, to get new items on next iteration
+ offset = offset + limit
+
+ # Iterates through items
+ # Verifies if item is already on repository
+ # Initializes new LearningObjects
+ # and saves them on LearningObjects's repository
+ dspace_items.each do |item|
+ result = learning_object_repository.get_by_dspace_id item.id
+ if result.nil?
+ learning_object = initialize_learning_object item
+ learning_object_repository.save learning_object
+ end
+ end
+
+ end
+
+ end
+
+ end
+
+ private
+
+ def dspace_repository
+ @dspace_repository ||= Dspace::Client.instance.repository.item_repository
+ end
+
+ def get_items_from_dspace(limit, offset)
+ dspace_repository.get_all_items(
+ expand: ['metadata','bitstreams'],
+ limit: limit, offset: offset
+ )
+ end
+
+ def initialize_learning_object item
+ metadata = build_array_of(item.metadata)
+ bitstreams = build_array_of(item.bit_streams)
+ current_date = Time.new
+
+ LearningObject.new(
+ :name => item.name,
+ :description => select_value_of(metadata, "dc.description"),
+ :thumbnail => '/thumbnails/default_thumbnail.jpg',
+ :created_at => current_date,
+ :last_modified => current_date,
+ :id_dspace => item.id,
+ :type => item.type,
+ :bitstreams => bitstreams,
+ :metadata => metadata
+ )
+ end
+
+ def build_array_of(item_content=[])
+ return item_content if item_content.nil?
+ content_array = []
+ item_content.each do |i|
+ content_array << i.to_h
+ end
+ return content_array
+ end
+
+ def select_value_of(array, key)
+ descriptions = array.select { |a| a[:key] == key }
+ unless descriptions.empty?
+ descriptions.first[:value]
+ end
+ end
+
+end
diff --git a/lib/tasks/fakeData.rake b/lib/tasks/fakeData.rake
new file mode 100644
index 0000000000000000000000000000000000000000..07f90f587c7cc4d7c781eb0b039ca14ca40a26be
--- /dev/null
+++ b/lib/tasks/fakeData.rake
@@ -0,0 +1,89 @@
+namespace :fakedata do
+ desc "Fake data generation tasks"
+
+ task generate_users: :environment do
+ desc "Generate fake Users"
+
+ include RepositoriesProxy
+
+ total_new_users = 50
+
+ (1..total_new_users).each do |i|
+ user = User.new(
+ :name => "User#{i}",
+ :email => "user#{i}@inf.ufpr.br",
+ :password => "1234567890",
+ :password_confirmation => "1234567890"
+ )
+ user.save
+ end
+ end
+
+ task generate_data: :environment do
+ desc "Generate fake user Likes/Views data"
+
+ include RepositoriesProxy
+
+ # Quantity of items fetched on each iteration
+ limit = 100
+
+ # Start point from where items will be fetched
+ offset = 0
+
+ users = User.all
+
+ loop do
+
+ begin
+ # Get items from dspace (from offset to offset+limit)
+ learning_objects = learning_object_repository.all_from_offset_to_limit(limit,offset)
+ rescue
+ # Sleeps for a while to wait database's recovery
+ sleep(30.seconds)
+ # Goes to next iteration to retry
+ next
+ else
+ # Terminate loop if there are no more items to import
+ break if learning_objects.empty?
+
+ # Increment offset, to get new items on next iteration
+ offset = offset + limit
+
+ learning_objects.each do |lo|
+
+ users_views = []
+ users_likes = []
+
+ users.each do |user|
+ # Randomize "Views"
+ if random_check(30)
+ users_views << user. id
+ # Randomize "Likes"
+ if random_check(70)
+ users_likes << user.id
+ end
+ end
+ end
+
+ MassiveViewsCreatorWorker.perform_async(lo.id, users_views)
+ MassiveLikesCreatorWorker.perform_async(lo.id, users_likes)
+
+ end
+
+ end
+
+ end
+
+ end
+
+ private
+
+ def random_check(threshold)
+ if rand(1..100) <= threshold
+ return true
+ else
+ return false
+ end
+ end
+
+end
diff --git a/lib/tasks/thumbnail.rake b/lib/tasks/thumbnail.rake
new file mode 100644
index 0000000000000000000000000000000000000000..7f7a393b1117441b0ddcd5f0a419a0cf1c241295
--- /dev/null
+++ b/lib/tasks/thumbnail.rake
@@ -0,0 +1,39 @@
+namespace :thumbnail do
+
+ desc "Generate Thumbnails"
+ task :generate => :environment do
+
+ include RepositoriesProxy
+
+ # Quantity of items fetched on each iteration
+ limit = 500
+
+ # Start point from where items will be fetched
+ offset = 0
+
+ loop do
+
+ begin
+ # Get items from dspace (from offset to offset+limit)
+ items = learning_object_repository.all_from_offset_to_limit(limit,offset)
+ rescue
+ # Sleeps for a while to wait database's recovery
+ sleep(30.seconds)
+ # Goes to next iteration to retry
+ next
+ else
+ # Terminate loop if there are no more items to import
+ break if items.empty?
+
+ # Increment offset, to get new items on next iteration
+ offset = offset + limit
+
+ items.each do |item|
+ ThumbnailGeneratorWorker.perform_async(item.id)
+ end
+
+ end
+ end
+ end
+
+end
diff --git a/lib/thumbnail/creation.rb b/lib/thumbnail/creation.rb
new file mode 100644
index 0000000000000000000000000000000000000000..0531f65155074be2661e8ce889e30e7fd48f33f3
--- /dev/null
+++ b/lib/thumbnail/creation.rb
@@ -0,0 +1,68 @@
+module Thumbnail
+ module Creation
+
+ include Formats
+
+ def generate_thumbnail(input, filename, size)
+ unless accepted_formats.include? file_format(filename)
+ return default_thumbnail
+ else
+ thumbnail = thumbnail_path(filename, size)
+ output = "#{root_dir}#{thumbnail}"
+ begin
+ if accepted_video_formats.include? File.extname(input)
+ generate_video_thumbnail(input, output, size)
+ else
+ generate_image_thumbnail(input, output, size)
+ end
+ rescue
+ return default_thumbnail
+ else
+ return thumbnail
+ end
+ end
+
+ end
+
+ def default_thumbnail
+ @default_thumbnail ||= nil
+ end
+
+ private
+
+ def generate_video_thumbnail(input,output,size)
+ movie = FFMPEG::Movie.new(input)
+ frame = (movie.duration * 25/100).floor
+ movie.screenshot(output,
+ { seek_time: frame, resolution: size },
+ preserve_aspect_ratio: :width)
+ end
+
+ def generate_image_thumbnail(input,output,size)
+ # Read the image and resize it. The `change_geometry' method
+ # computes the new image geometry and yields to a block. The
+ # return value of the block is the return value of the method.
+ img = Magick::Image.read(input)[0]
+ img.change_geometry!(size) { |cols, rows| img.thumbnail! cols, rows }
+ img.write(output)
+ end
+
+ def encode_hash_from(object)
+ Digest::SHA1.hexdigest object
+ end
+
+ def thumbnail_path(filename, size)
+ thumbnail_name = encode_hash_from filename
+ thumbnail_path = "#{thumbnails_dir}/#{thumbnail_name}_#{size}.jpg"
+ end
+
+ def thumbnails_dir
+ @thumbnails_dir ||= "/thumbnails"
+ end
+
+ def root_dir
+ @root_dir ||= Rails.root.join('public')
+ end
+
+ end
+end
diff --git a/lib/thumbnail/formats.rb b/lib/thumbnail/formats.rb
new file mode 100644
index 0000000000000000000000000000000000000000..4df246e9bc30576c0886551952073f63750650cc
--- /dev/null
+++ b/lib/thumbnail/formats.rb
@@ -0,0 +1,36 @@
+module Thumbnail
+ module Formats
+
+ def file_format file
+ unless file.nil?
+ file_format = File.extname(file)
+ else
+ file_format = ""
+ end
+ file_format
+ end
+
+ def accepted_video_formats
+ lower = [".mp4", ".wmv", ".3gp", ".asf", ".avi", ".flv", ".mov", ".mpg", ".mpeg", ".rmvb", ".vob", ".webm"]
+ upper = [".MP4", ".WMV", ".3GP", ".ASF", ".AVI", ".FLV", ".MOV", ".MPG", ".MPEG", ".RMVB", ".VOB", ".WEBM"]
+ return lower + upper
+ end
+
+ def accepted_image_formats
+ lower = [".jpg", ".jpeg", ".gif", ".png", ".bmp", ".tif"]
+ upper = [".JPG", ".JPEG", ".GIF", ".PNG", ".BMP", ".TIF"]
+ return lower + upper
+ end
+
+ def accepted_other_formats
+ lower = [".pdf", ".pps"]
+ upper = [".PDF", ".PPS"]
+ return lower + upper
+ end
+
+ def accepted_formats
+ return accepted_video_formats + accepted_image_formats + accepted_other_formats
+ end
+
+ end
+end