Adding Search to Rails with MeiliSearch

Native MeiliSearch Rails Integration

class Student < ApplicationRecord 
include MeiliSearch

meilisearch do
attribute :name, :subject
end
end
hits = Student.search('physics')

Using MeiliSearch with an Admin App

  1. A rake task in MissionControl finds all changed records and updates the index
  2. A second rake task in MissionControl finds all deleted records and removes them from the index.
  3. Both tasks are scheduled with CRONJobs to run regularly and frequently.
  4. We can remove the entire search index if needed and the next run will rebuild it from scratch.
require 'meilisearch' $ms_client = MeiliSearch::Client.new(ENV.fetch('MEILISEARCH_URL') { 'http://127.0.0.1:7700' })

Indexing new and changed objects

desc "Indexes all docs" task :run, [:name] => [:environment] do |t, args|   # only run one of these tasks at the time   is_running = $redis.get("mc_index_running")   if is_running == "1"       puts "Another index process is running"       next   end   begin     $redis.set("mc_index_running", 1)     BATCH_SIZE = 100 
INDEX_NAME = :searchable
# this filters records that we don't want to index class_filters = { ::User => -> (x) { x.account.nil? },
::Account => -> (x) { x.main_user.nil? },
# ... more classes here
} # any class we want to index and the attributes on them we want indexed classes_to_index = { ::User => [
:email,
:name,
:phone_number,
:updated_at,
{ company_name: ->(x) { x.account.name } } ], ::Account => [
:name,
:company_name,
:updated_at,
{ owner_email: -> (x) { x.main_user.email }}, ], # ... more classes here ] } index = $ms_client.index(INDEX_NAME) # take the cutoff point
cutoff = Time.now.utc
puts "Cutting off at #{cutoff}" filter = nil if args.count > 0 filter = args[:name] end
classes_to_index.each_pair do |clazz, attributes|
name = clazz.name.to_s.downcase next if !filter.nil? && name != filter puts "Indexing #{name}" # get the last index cut off index_metadata = ::IndexMetadata.find_by(name: name) last_cut_off = index_metadata&.last_index
puts "Last cutoff was #{last_cut_off ? last_cut_off : 'never'}"

db_attributes = attributes.filter { |x| x.is_a? Symbol }
calculated_attributes = attributes.filter { |x| x.is_a? Hash } if last_cut_off query = clazz.select(:id, db_attributes).where('updated_at >= ?', last_cut_off)

puts "#{clazz.where('updated_at >= ?', last_cut_off).count} new records found"
else query = clazz.select(:id, db_attributes) end query.find_in_batches(batch_size: BATCH_SIZE) do |group| to_index = group.map do |x| next nil if class_filters[clazz].call(x) atts = x.attributes.dup atts['id'] = "#{name}-#{x.id}"
atts['kind'] = name
calculated_attributes.each do |c_attrs| c_attrs.each_pair do |key, value| if value.is_a? Proc atts[key] = value.call(x) end end
end

next atts.symbolize_keys
end to_index.compact! index.add_documents(to_index, 'id') unless to_index.empty? rescue => exc puts exc.message pp to_index raise end
if index_metadata
index_metadata.update({ name: name, last_index: cutoff }) else ::IndexMetadata.create({ name: name, last_index: cutoff }) end end ensure $redis.set("mc_index_running", 0) end end
::User => [           :email, 
:name,
:phone_number,
:updated_at,
{ company_name: ->(x) { x.account.name } }
],
index_metadata = ::IndexMetadata.find_by(name: name) last_cut_off = index_metadata&.last_index if last_cut_off            query = clazz.select(:id, db_attributes).where('updated_at >= ?', last_cut_off)            puts "#{clazz.where('updated_at >= ?', last_cut_off).count} new records found"          else            query = clazz.select(:id, db_attributes)           end
query.find_in_batches(batch_size: BATCH_SIZE) do |group| 
# ...
end
query.find_in_batches(batch_size: BATCH_SIZE) do |group|            to_index = group.map do |x|              next nil if class_filters[clazz].call(x) 
atts = x.attributes.filter{ |x| x != 'params'}.dup
atts['id'] = "#{name}-#{x.id}"
atts['kind'] = name
calculated_attributes.each do |c_attrs| c_attrs.each_pair do |key, value| if value.is_a? Proc atts[key] = value.call(x) end end end next atts.symbolize_keys end
to_index.compact!
index.add_documents(to_index, 'id') unless to_index.empty? rescue => exc puts exc.message pp to_index raise end
if index_metadata    index_metadata.update({ name: name, last_index: cutoff }) else     ::IndexMetadata.create({ name: name, last_index: cutoff }) end

Removing deleted object

desc "Delete removed docs"     task :purge, [:name] => [:environment] do |t, args|         INDEX_NAME = :searchable         FETCH_LIMIT = 500   
index = $ms_client.index(INDEX_NAME)
# fetch all docs from the search index in batches id_list = { 'user' => [],
'account' => [],
# more classes here } puts "Fetching all documents in search index" offset = 0 loop do ids = index.documents(limit: FETCH_LIMIT, offset: offset, attributesToRetrieve: 'id') # parse the IDs ids.each { |id| parts = id['id'].split('-'); id_list[parts[0]] << parts[1].to_i } offset = offset + FETCH_LIMIT break if ids.count < FETCH_LIMIT # finished end
# for each index fetch all IDs.
def compare(id_list, idx) puts "Comparing #{id_list[idx].count} #{idx.pluralize} with database" offset = 0 ids = [] loop do db_ids = ::ActiveRecord::Base.establish_connection(:main).connection.execute("SELECT id FROM #{idx.pluralize} LIMIT #{offset},#{FETCH_LIMIT}").to_a.flatten ids << db_ids offset = offset + FETCH_LIMIT break if db_ids.count < FETCH_LIMIT # finished end ids.flatten!
return id_list[idx] - ids
end
to_delete = []
if args.count == 0 to_delete << compare(id_list, 'user').map { |x| "user-#{x}"}.
to_delete << compare(id_list, 'account').map { |x| "account-#{x}"}
# more classes here else to_delete << compare(id_list, args[:name]).map { |x| "#{args[:name]}-#{x}"} end to_delete.flatten! puts "Deleting #{to_delete.count} documents from search index" index.delete_documents(to_delete) end
  1. Fetch the IDs of all documents in the index (this is very quick in MeiliSearch)
  2. Fetch the IDs of all documents in the database (this can be improved using VALUES if you use MySQL 8.0.19+) but still very fast and memory efficient.
  3. Compare the ID sets and find the missing ones
  4. Delete the missing ones from the index

Scheduling

Searching

--

--

DevOps-as-a-Service to help developers build, deploy and maintain apps on any Cloud. Sign-up for a free trial by visting: www.cloud66.com

Love podcasts or audiobooks? Learn on the go with our new app.

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
Cloud 66

Cloud 66

DevOps-as-a-Service to help developers build, deploy and maintain apps on any Cloud. Sign-up for a free trial by visting: www.cloud66.com