Merge branch 'elastic-6' into 'master'

Updated search code to work with Elasticsearch 6.2.4

See merge request portalmec/portalmec!506

Gemfile.lock

 GIT
-  remote: git://github.com/mgiacomini/dspace-rest-client.git
-  revision: 7bc3a0a4fe8222663b9642d580de17acfdb76688
+  remote: git://github.com/c3sl/dspace-rest-client.git
+  revision: 5489fc45c8bd1b8d1e364fc8fb83750c29ba5d68
   branch: master
   specs:
     dspace_rest_client (2.2.8)
@@ -107,7 +107,7 @@ GEM
     coderay (1.1.1)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
-    concurrent-ruby (1.0.5)
+    concurrent-ruby (1.1.3)
     connection_pool (2.2.1)
     curb (0.8.8)
     cvss (0.99.0)
@@ -194,12 +194,12 @@ GEM
       dm-core (~> 1.2.0)
     do_sqlite3 (0.10.17)
       data_objects (= 0.10.17)
-    elasticsearch (5.0.4)
-      elasticsearch-api (= 5.0.4)
-      elasticsearch-transport (= 5.0.4)
-    elasticsearch-api (5.0.4)
+    elasticsearch (6.1.0)
+      elasticsearch-api (= 6.1.0)
+      elasticsearch-transport (= 6.1.0)
+    elasticsearch-api (6.1.0)
       multi_json
-    elasticsearch-transport (5.0.4)
+    elasticsearch-transport (6.1.0)
       faraday
       multi_json
     equalizer (0.0.11)
@@ -236,11 +236,12 @@ GEM
     haml (5.0.1)
       temple (>= 0.8.0)
       tilt
-    hashie (3.5.5)
+    hashie (3.5.7)
     heapy (0.1.2)
     httparty (0.14.0)
       multi_xml (>= 0.5.2)
-    i18n (0.8.1)
+    i18n (0.9.5)
+      concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
     immigrant (0.3.6)
       activerecord (>= 3.0)
@@ -267,13 +268,13 @@ GEM
       open4 (~> 1.3.4)
       rake
     mini_portile2 (2.1.0)
-    minitest (5.10.2)
+    minitest (5.11.3)
     minitest-reporters (1.1.14)
       ansi
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    multi_json (1.12.1)
+    multi_json (1.13.1)
     multi_xml (0.6.0)
     multipart-post (2.0.0)
     mustache (1.0.5)
@@ -383,7 +384,7 @@ GEM
       parser (>= 2.4.0.0, < 2.5)
       rainbow (~> 2.0)
     request_store (1.3.2)
-    resource_kit (0.1.6)
+    resource_kit (0.1.7)
       addressable (>= 2.3.6, < 3.0.0)
     responders (2.4.0)
       actionpack (>= 4.2.0, < 5.3)
@@ -439,9 +440,9 @@ GEM
     sdoc (0.4.2)
       json (~> 1.7, >= 1.7.7)
       rdoc (~> 4.0)
-    searchkick (2.3.0)
-      activemodel (>= 4.1)
-      elasticsearch (>= 1)
+    searchkick (3.1.2)
+      activemodel (>= 4.2)
+      elasticsearch (>= 5)
       hashie
     sexp_processor (4.9.0)
     shoulda (3.5.0)
@@ -484,7 +485,7 @@ GEM
     thor (0.19.4)
     thread_safe (0.3.6)
     tilt (2.0.7)
-    tzinfo (1.2.3)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
     unicode-display_width (1.1.3)
     uniform_notifier (1.10.0)
@@ -577,4 +578,4 @@ DEPENDENCIES
   whenever
 
 BUNDLED WITH
-   1.12.5
+   1.14.6

app/models/collection.rb

@@ -69,7 +69,15 @@ class Collection < ApplicationRecord
 
   scope :from_user, ->(user) { where(owner: user) }
 
-  searchkick language: 'brazilian', match: :word_start, searchable: [:name, :description, :author], callbacks: :async
+  searchkick language: 'brazilian', match: :word_start, searchable: [:name, :description, :author], callbacks: :async, merge_mappings: true, mappings: {
+    collection: {
+      properties: {
+        created_at: {
+          type: "date"
+        }
+      }
+    }
+  }
 
   has_paper_trail
 

app/models/learning_object.rb

@@ -88,11 +88,20 @@ class LearningObject < ApplicationRecord
   default_scope { includes(:object_type, :attachment, :attachments) }
   scope :missing_thumbnail, ->() { where(thumbnail_file_name: nil) }
 
-  searchkick language: 'brazilian', match: :word_start, searchable: [:name, :description, :author, :object_type, :tags, :subjects, :educational_stages], callbacks: :async
+  searchkick language: 'brazilian', match: :word_start, searchable: [:name, :description, :author, :object_type, :tags, :subjects, :educational_stages], callbacks: :async, merge_mappings: true, mappings: {
+    learning_object: {
+      properties: {
+        published_at: {
+          type: "date"
+        }
+      }
+    }
+  }
 
   acts_as_paranoid
 
   scope :search_import, -> { includes(:object_type, :tags, :subjects, :educational_stages, :publisher) }
+
   def search_data
     source = !publisher.nil? ? publisher.name : ""
     type = object_type.try(:id)

app/serializers/learning_object_serializer.rb

@@ -72,6 +72,7 @@ class LearningObjectSerializer < ActiveModel::Serializer
     :complained,
     :shares_count,
     :created_at,
+    :published_at,
     :terms_of_service,
     :updated_at,
     :magnetlink

app/services/search_service/collection.rb

@@ -20,7 +20,7 @@
 module SearchService
   class Collection < Model
     def search
-      ::Collection.search(body: mount_query, order: order_hash, page: @search.page, per_page: @search.results_per_page)
+      ::Collection.search(body: mount_query)
     end
 
     def autocomplete
@@ -42,9 +42,9 @@ module SearchService
     def mount_filter
       filter = []
       filter << { term: { privacy: 'public' } }
-      filter << { in: { tags: @search.tags } } unless @search.tags.blank?
-      filter << { in: { subjects: @search.subjects } } unless @search.subjects.blank?
-      filter << { in: { educational_stages: @search.educational_stages } } unless @search.educational_stages.blank?
+      filter << { terms: { tags: @search.tags } } unless @search.tags.blank?
+      filter << { terms: { subjects: @search.subjects } } unless @search.subjects.blank?
+      filter << { terms: { educational_stages: @search.educational_stages } } unless @search.educational_stages.blank?
       filter << { terms: { empty: show_empty } }
       filter
     end
@@ -52,8 +52,8 @@ module SearchService
     def order_hash
       case @search.order
       when 'author' then { owner: { order: :asc, unmapped_type: :string } }
-      when 'publicationasc' then { created_at: { order: :asc, unmapped_type: :timestamp } }
-      when 'publicationdesc' then { created_at: { order: :desc, unmapped_type: :timestamp } }
+      when 'publicationasc' then { created_at: { order: :asc, unmapped_type: :date } }
+      when 'publicationdesc' then { created_at: { order: :desc, unmapped_type: :date } }
       when 'title' then { name: { order: :asc, unmapped_type: :string } }
       when 'likes' then { likes: { order: :desc } }
       when 'downloads' then { downloads: { order: :desc } }

app/services/search_service/learning_object.rb

@@ -21,7 +21,7 @@ module SearchService
   class LearningObject < Model
     def search
       fields = [:license, :tags, :subjects, :educational_stages, :taggings, :publisher, :language, :attachments, :reviews]
-      ::LearningObject.search(body: mount_query, includes: fields, order: order_hash, page: @search.page, per_page: @search.results_per_page)
+      ::LearningObject.search(body: mount_query, includes: fields)
     end
 
     def autocomplete
@@ -42,10 +42,10 @@ module SearchService
 
     def mount_filter
       filter = []
-      filter << { in: { tags: @search.tags } } unless @search.tags.blank?
-      filter << { in: { subjects: @search.subjects } } unless @search.subjects.blank?
-      filter << { in: { educational_stages: @search.educational_stages } } unless @search.educational_stages.blank?
-      filter << { in: { object_type: @search.object_types.map(&:to_i) } } unless @search.object_types.blank?
+      filter << { terms: { tags: @search.tags } } unless @search.tags.blank?
+      filter << { terms: { subjects: @search.subjects } } unless @search.subjects.blank?
+      filter << { terms: { educational_stages: @search.educational_stages } } unless @search.educational_stages.blank?
+      filter << { terms: { object_type: @search.object_types.map(&:to_i) } } unless @search.object_types.blank?
       filter << { terms: { state: validate_object } }
       filter
     end
@@ -53,8 +53,8 @@ module SearchService
     def order_hash
       case @search.order
       when 'author' then { author: { order: :asc, unmapped_type: :string } }
-      when 'publicationasc' then { published_at: { order: :asc, unmapped_type: :timestamp } }
-      when 'publicationdesc' then { published_at: { order: :desc, unmapped_type: :timestamp } }
+      when 'publicationasc' then { published_at: { order: :asc, unmapped_type: :date } }
+      when 'publicationdesc' then { published_at: { order: :desc, unmapped_type: :date } }
       when 'title' then { name: { order: :asc, unmapped_type: :string } }
       when 'likes' then { likes: { order: :desc } }
       when 'downloads' then { downloads: { order: :desc } }

app/services/search_service/model.rb

@@ -78,7 +78,7 @@ module SearchService
               must: { dis_max: { queries: mount_queries } },
               filter: mount_filter
             } },
-            functions: [{ script_score: { script: { lang: 'painless', file: 'calculate_score' } } }]
+            functions: [{ script_score: { script: { source: "_score * doc['score'].value" } } }]
           }
         },
         # https://www.elastic.co/guide/en/elasticsearch/guide/current/pagination.html

app/services/search_service/user.rb

@@ -20,7 +20,7 @@
 module SearchService
   class User < Model
     def search
-      ::User.search(body: mount_query, order: order_hash, page: @search.page, per_page: @search.results_per_page)
+      ::User.search(body: mount_query)
     end
 
     def autocomplete