Skip to content

Commit a08c1fb

Browse files
danielsdeleodanielsdeleo
committed
rewrite chef-solr indexer hot paths to be fast
* use fast_xs if available for XML escaping * rewrite Chef::Solr::Indexer flatten and expand to eliminate branching, redundant value checking, be simpler and generate less garbage * rewrite Chef::Solr#add to build the XML doc by concatenating strings becuase this is twice as fast and the document is very simple so code clarity is not harmed.
1 parent bf970b0 commit a08c1fb

5 files changed

Lines changed: 206 additions & 269 deletions

File tree

‎chef-solr/lib/chef/solr.rb‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -75,27 +75,31 @@ def post_to_solr(doc)
7575
res
7676
end
7777

78-
def solr_add(data)
79-
data = [data] unless data.kind_of?(Array)
78+
START_XML = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<add><doc>"
79+
END_XML = "</doc></add>\n"
80+
FIELD_ATTR = '<field name="'
81+
FIELD_ATTR_END = '">'
82+
CLOSE_FIELD = "</field>"
8083

84+
def solr_add(data)
8185
Chef::Log.debug("adding to SOLR: #{data.inspect}")
82-
xml_document = LibXML::XML::Document.new
83-
xml_add = LibXML::XML::Node.new("add")
84-
data.each do |doc|
85-
xml_doc = LibXML::XML::Node.new("doc")
86-
doc.each do |field, values|
87-
values = [values] unless values.kind_of?(Array)
88-
values.each do |v|
89-
xml_field = LibXML::XML::Node.new("field")
90-
xml_field["name"] = field
91-
xml_field.content = xml_escape(v.to_s)
92-
xml_doc << xml_field
93-
end
86+
87+
xml = ""
88+
xml << START_XML
89+
90+
data.each do |field, values|
91+
values.each do |v|
92+
xml << FIELD_ATTR
93+
xml << field
94+
xml << FIELD_ATTR_END
95+
xml << xml_escape(v)
96+
xml << CLOSE_FIELD
9497
end
95-
xml_add << xml_doc
9698
end
97-
xml_document.root = xml_add
98-
post_to_solr(xml_document.to_s(:indent => false))
99+
xml << END_XML
100+
xml
101+
102+
post_to_solr(xml)
99103
end
100104

101105
def solr_commit(opts={})

‎chef-solr/lib/chef/solr/index.rb‎

Lines changed: 46 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@
66
# Licensed under the Apache License, Version 2.0 (the "License");
77
# you may not use this file except in compliance with the License.
88
# You may obtain a copy of the License at
9-
#
9+
#
1010
# http://www.apache.org/licenses/LICENSE-2.0
11-
#
11+
#
1212
# Unless required by applicable law or agreed to in writing, software
1313
# distributed under the License is distributed on an "AS IS" BASIS,
1414
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -26,12 +26,24 @@ class Chef
2626
class Solr
2727
class Index < Solr
2828

29+
UNDERSCORE = '_'
30+
X = 'X'
31+
32+
X_CHEF_id_CHEF_X = 'X_CHEF_id_CHEF_X'
33+
X_CHEF_database_CHEF_X = 'X_CHEF_database_CHEF_X'
34+
X_CHEF_type_CHEF_X = 'X_CHEF_type_CHEF_X'
35+
2936
def add(id, database, type, item)
30-
raise ArgumentError, "Object must respond to keys!" unless item.respond_to?(:keys)
37+
unless item.respond_to?(:keys)
38+
raise ArgumentError, "#{self.class.name} can only index Hash-like objects. You gave #{item.inspect}"
39+
end
40+
3141
to_index = flatten_and_expand(item)
32-
to_index["X_CHEF_id_CHEF_X"] = id
33-
to_index["X_CHEF_database_CHEF_X"] = database
34-
to_index["X_CHEF_type_CHEF_X"] = type
42+
43+
to_index[X_CHEF_id_CHEF_X] = [id]
44+
to_index[X_CHEF_database_CHEF_X] = [database]
45+
to_index[X_CHEF_type_CHEF_X] = [type]
46+
3547
solr_add(to_index)
3648
to_index
3749
end
@@ -44,117 +56,48 @@ def delete_by_query(query)
4456
solr_delete_by_query(query)
4557
end
4658

47-
def flatten_and_expand(item, fields=Hash.new, parent=nil)
48-
item.keys.each do |key|
49-
# If we have a parent, we want to add the current key as a value
50-
if parent
51-
# foo_bar = bar
52-
set_field_value(fields, parent, key)
53-
# foo_X = bar, etc.
54-
make_expando_fields(parent).each do |ex_key|
55-
set_field_value(fields, ex_key, key)
56-
end
57-
end
58-
case item[key]
59-
when Hash
60-
parent_key = parent ? "#{parent}_#{key}" : key
61-
flatten_and_expand(item[key], fields, parent_key)
62-
else
63-
parent_key = parent ? "#{parent}_#{key}" : key
64-
set_field_value(fields, key, item[key])
65-
set_field_value(fields, parent_key, item[key]) if parent
66-
make_expando_fields(parent_key).each do |ex_key|
67-
set_field_value(fields, ex_key, item[key])
68-
end
69-
end
70-
end
71-
fields
72-
end
73-
74-
def make_expando_fields(key)
75-
key = key.to_s
76-
fields = Array.new
77-
parts = key.split("_")
78-
length = parts.length
79-
parts.each_index do |i|
80-
beginning = nil
81-
remainder = nil
82-
if i == 0
83-
beginning = "X"
84-
else
85-
beginning = parts[0..i-1].join("_")
86-
end
87-
88-
if i == length-1
89-
remainder = "X"
90-
else
91-
remainder = parts[i+1..-1].join("_")
92-
end
59+
def flatten_and_expand(item)
60+
@flattened_item = Hash.new {|hash, key| hash[key] = []}
9361

94-
if beginning == "X" || remainder == "X"
95-
unless beginning == "X" && remainder == "X"
96-
fields << "#{beginning}_#{remainder}"
97-
end
98-
else
99-
fields << "#{beginning}_X_#{remainder}"
100-
end
62+
item.each do |key, value|
63+
flatten_each([key.to_s], value)
10164
end
102-
fields
103-
end
10465

105-
def set_field_value(fields, key, value)
106-
key = key.to_s
107-
if fields.has_key?(key)
108-
convert_field_to_array(fields, key, value) unless fields[key].kind_of?(Array)
109-
add_value_to_field_array(fields, key, value)
110-
else
111-
check_value(value)
112-
if value.kind_of?(Array)
113-
fields[key] = Array.new
114-
value.each do |v|
115-
if v.kind_of?(Hash)
116-
flatten_and_expand(v, fields, key)
117-
else
118-
fields[key] << v.to_s
119-
end
120-
end
121-
else
122-
fields[key] = value.to_s
123-
end
124-
end
125-
fields
66+
@flattened_item.each_value { |values| values.uniq! }
67+
@flattened_item
12668
end
12769

128-
def add_value_to_field_array(fields, key, value)
129-
check_value(value)
130-
if value.kind_of?(Array)
131-
value.each do |v|
132-
if v.kind_of?(Hash)
133-
flatten_and_expand(v, fields, key)
134-
else
135-
fields[key] << v.to_s unless fields[key].include?(v.to_s)
136-
end
70+
def flatten_each(keys, values)
71+
case values
72+
when Hash
73+
values.each do |child_key, child_value|
74+
add_field_value(keys, child_key)
75+
flatten_each(keys + [child_key.to_s], child_value)
13776
end
77+
when Array
78+
values.each { |child_value| flatten_each(keys, child_value) }
13879
else
139-
fields[key] << value.to_s unless fields[key].include?(value.to_s)
80+
add_field_value(keys, values)
14081
end
141-
fields
14282
end
14383

144-
def convert_field_to_array(fields, key, value)
145-
if fields[key] != value
146-
safe = fields[key]
147-
fields[key] = [ safe ]
148-
end
149-
fields
84+
def add_field_value(keys, value)
85+
value = value.to_s
86+
each_expando_field(keys) { |expando_field| @flattened_item[expando_field] << value }
87+
@flattened_item[keys.join(UNDERSCORE)] << value
88+
@flattened_item[keys.last] << value
15089
end
15190

152-
def check_value(value)
153-
raise ArgumentError, "Value must not be a type of hash!" if value.kind_of?(Hash)
154-
value
91+
def each_expando_field(keys)
92+
return if keys.size == 1
93+
0.upto(keys.size - 1) do |index|
94+
original = keys[index]
95+
keys[index] = X
96+
yield keys.join(UNDERSCORE)
97+
keys[index] = original
98+
end
15599
end
156100

157101
end
158102
end
159103
end
160-

0 commit comments

Comments
 (0)