Skip to content

Commit 3245151

Browse files
danielsdeleoadamhjk
danielsdeleo
authored and
adamhjk
committed
bring back the syntax check cache for 5x improvement on re-uploads
1 parent 050eddb commit 3245151

5 files changed

Lines changed: 356 additions & 40 deletions

File tree

‎chef/lib/chef/cookbook/syntax_check.rb‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
#
2+
# Author:: Daniel DeLeo (<dan@opscode.com>)
3+
# Copyright:: Copyright (c) 2010 Opscode, Inc.
4+
# License:: Apache License, Version 2.0
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
19+
require 'chef/cache/checksum'
20+
require 'chef/mixin/shell_out'
21+
22+
class Chef
23+
class Cookbook
24+
class SyntaxCheck
25+
include Chef::Mixin::ShellOut
26+
27+
attr_reader :cookbook_path
28+
29+
def self.for_cookbook(cookbook_name, cookbook_path=nil)
30+
cookbook_path ||= Chef::Config.cookbook_path
31+
unless cookbook_path
32+
raise ArgumentError, "Cannot find cookbook #{cookbook_name} unless Chef::Config.cookbook_path is set or an explicit cookbook path is given"
33+
end
34+
new(File.join(cookbook_path, cookbook_name.to_s))
35+
end
36+
37+
def initialize(cookbook_path)
38+
@cookbook_path = cookbook_path
39+
end
40+
41+
def cache
42+
Chef::Cache::Checksum.instance
43+
end
44+
45+
def ruby_files
46+
Dir[File.join(cookbook_path, '**', '*.rb')]
47+
end
48+
49+
def untested_ruby_files
50+
ruby_files.reject do |file|
51+
if validated?(file)
52+
Chef::Log.debug("ruby file #{file} is unchanged skipping syntax check")
53+
true
54+
else
55+
false
56+
end
57+
end
58+
end
59+
60+
def template_files
61+
Dir[File.join(cookbook_path, '**', '*.erb')]
62+
end
63+
64+
def untested_template_files
65+
template_files.reject do |file|
66+
if validated?(file)
67+
Chef::Log.debug("template #{file} is unchanged, skipping syntax check")
68+
true
69+
else
70+
false
71+
end
72+
end
73+
end
74+
75+
def validated?(file)
76+
!!cache.lookup_checksum(cache_key(file), File.stat(file))
77+
end
78+
79+
def validated(file)
80+
cache.generate_checksum(cache_key(file), file, File.stat(file))
81+
end
82+
83+
def cache_key(file)
84+
@cache_keys ||= {}
85+
@cache_keys[file] ||= cache.generate_key(file, "chef-test")
86+
end
87+
88+
def validate_ruby_files
89+
untested_ruby_files.each do |ruby_file|
90+
return false unless validate_ruby_file(ruby_file)
91+
validated(ruby_file)
92+
end
93+
end
94+
95+
def validate_templates
96+
untested_template_files.each do |template|
97+
return false unless validate_template(template)
98+
validated(template)
99+
end
100+
end
101+
102+
def validate_template(erb_file)
103+
Chef::Log.debug("Testing template #{erb_file} for syntax errors...")
104+
result = shell_out("sh -c 'erubis -x #{erb_file} | ruby -c'")
105+
result.error!
106+
true
107+
rescue Chef::Exceptions::ShellCommandFailed
108+
file_relative_path = erb_file[/^#{Regexp.escape(cookbook_path+File::Separator)}(.*)/, 1]
109+
Chef::Log.fatal("Erb template #{file_relative_path} has a syntax error:")
110+
result.stderr.each_line { |l| Chef::Log.fatal(l.chomp) }
111+
false
112+
end
113+
114+
def validate_ruby_file(ruby_file)
115+
Chef::Log.debug("Testing #{ruby_file} for syntax errors...")
116+
result = shell_out("ruby -c #{ruby_file}")
117+
result.error!
118+
true
119+
rescue Chef::Exceptions::ShellCommandFailed
120+
file_relative_path = ruby_file[/^#{Regexp.escape(cookbook_path+File::Separator)}(.*)/, 1]
121+
Chef::Log.fatal("Cookbook file #{file_relative_path} has a ruby syntax error:")
122+
result.stderr.each_line { |l| Chef::Log.fatal(l.chomp) }
123+
false
124+
end
125+
126+
end
127+
end
128+
end

‎chef/lib/chef/knife/cookbook_upload.rb‎

Lines changed: 16 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
require 'chef/cache/checksum'
2525
require 'chef/sandbox'
2626
require 'chef/cookbook_version'
27+
require 'chef/cookbook/syntax_check'
2728
require 'chef/cookbook/file_system_file_vendor'
2829

2930
class Chef
@@ -50,6 +51,8 @@ def run
5051
else
5152
config[:cookbook_path] = Chef::Config[:cookbook_path]
5253
end
54+
# Ugh, manipulating globals causes bugs.
55+
@user_cookbook_path = config[:cookbook_path]
5356

5457
Chef::Cookbook::FileVendor.on_create { |manifest| Chef::Cookbook::FileSystemFileVendor.new(manifest) }
5558

@@ -70,23 +73,12 @@ def run
7073
end
7174
end
7275

73-
def test_ruby(cookbook_dir)
74-
Chef::Log.info("Validating ruby files")
75-
Dir[File.join(cookbook_dir, '**', '*.rb')].each do |ruby_file|
76-
test_ruby_file(cookbook_dir, ruby_file)
77-
end
78-
end
79-
80-
def test_templates(cookbook_dir)
81-
Chef::Log.info("Validating templates")
82-
Dir[File.join(cookbook_dir, '**', '*.erb')].each do |erb_file|
83-
test_template_file(cookbook_dir, erb_file)
84-
end
85-
end
86-
8776
def upload_cookbook(cookbook)
8877
Chef::Log.info("Saving #{cookbook.name}")
8978

79+
# Validate the cookbook before staging it or else the syntax checker's
80+
# cache will not be helpful.
81+
validate_cookbook(cookbook)
9082
# create build directory
9183
tmp_cookbook_dir = create_build_dir(cookbook)
9284

@@ -173,10 +165,6 @@ def create_build_dir(cookbook)
173165
end
174166
end
175167

176-
# Validate ruby files and templates
177-
test_ruby(tmp_cookbook_dir)
178-
test_templates(tmp_cookbook_dir)
179-
180168
# First, generate metadata
181169
Chef::Log.debug("Generating metadata")
182170
kcm = Chef::Knife::CookbookMetadata.new
@@ -201,30 +189,18 @@ def catch_auth_exceptions
201189
end
202190
end
203191

204-
private
205-
206-
def test_template_file(cookbook_dir, erb_file)
207-
Chef::Log.debug("Testing template #{erb_file} for syntax errors...")
208-
result = shell_out("sh -c 'erubis -x #{erb_file} | ruby -c'")
209-
result.error!
210-
rescue Chef::Exceptions::ShellCommandFailed
211-
file_relative_path = erb_file[/^#{Regexp.escape(cookbook_dir+File::Separator)}(.*)/, 1]
212-
Chef::Log.fatal("Erb template #{file_relative_path} has a syntax error:")
213-
result.stderr.each_line { |l| Chef::Log.fatal(l.chomp) }
214-
exit(1)
192+
def validate_cookbook(cookbook)
193+
pp :validate_cookbook => {:cookbook_name => cookbook.name}
194+
syntax_checker = Chef::Cookbook::SyntaxCheck.for_cookbook(cookbook.name, @user_cookbook_path)
195+
pp syntax_checker
196+
Chef::Log.info("Validating ruby files")
197+
exit(1) unless syntax_checker.validate_ruby_files
198+
Chef::Log.info("Validating templates")
199+
exit(1) unless syntax_checker.validate_templates
200+
Chef::Log.info("Syntax OK")
201+
true
215202
end
216203

217-
def test_ruby_file(cookbook_dir, ruby_file)
218-
Chef::Log.debug("Testing #{ruby_file} for syntax errors...")
219-
result = shell_out("ruby -c #{ruby_file}")
220-
result.error!
221-
rescue Chef::Exceptions::ShellCommandFailed
222-
file_relative_path = ruby_file[/^#{Regexp.escape(cookbook_dir+File::Separator)}(.*)/, 1]
223-
Chef::Log.fatal("Cookbook file #{file_relative_path} has a syntax error:")
224-
result.stderr.each_line { |l| Chef::Log.fatal(l.chomp) }
225-
exit(1)
226-
end
227-
228204
end
229205
end
230206
end

‎chef/spec/data/cookbooks/borken/recipes/default.rb‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
a cat walked on the keyboard one day...
2+
(*&(*&(*&(*&(*^%$%^%#^^&(*)(*{}}}}}}}}+++++===))))))

‎chef/spec/data/cookbooks/borken/templates/default/borken.erb‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
a cat walked on the keyboard one day...
2+
<%= (*&)(*^^^^*******++_+_--- }}}}]]]end)%>

0 commit comments

Comments
 (0)