|
| 1 | +# Checks that every benchmark under code/ has the same shape, so they all run, |
| 2 | +# print a comparison, and are measured the same way: |
| 3 | +# |
| 4 | +# - every Benchmark.ips block calls x.compare! |
| 5 | +# - no custom timing (Benchmark.ips(20), x.time = 20, x.warmup = 5, x.config(time: 20) in any hash syntax), |
| 6 | +# so every file uses the default |
| 7 | +# - no Benchmark.ips sits inside a method that never runs from the top of the file, |
| 8 | +# which would benchmark nothing at all |
| 9 | +# |
| 10 | +# Usage: ruby .github/scripts/lint-benchmarks.rb [files...] (needs Ruby 3.3+) |
| 11 | +require "prism" |
| 12 | + |
| 13 | +# Calls named `name` that have a block, not looking inside the ones found. |
| 14 | +def find_calls(node, name, found = []) |
| 15 | + return found unless node |
| 16 | + |
| 17 | + if node.is_a?(Prism::CallNode) && node.name == name && node.block |
| 18 | + found << node |
| 19 | + else |
| 20 | + node.compact_child_nodes.each { |child| find_calls(child, name, found) } |
| 21 | + end |
| 22 | + found |
| 23 | +end |
| 24 | + |
| 25 | +def any_call?(node, &test) |
| 26 | + return false unless node |
| 27 | + return true if node.is_a?(Prism::CallNode) && test.call(node) |
| 28 | + |
| 29 | + node.compact_child_nodes.any? { |child| any_call?(child, &test) } |
| 30 | +end |
| 31 | + |
| 32 | +TIMING_KEYS = %w[time warmup].freeze |
| 33 | + |
| 34 | +# x.time = 20, x.warmup = 5, or x.config with a time or warmup key, |
| 35 | +# in any hash syntax (time: 20, :time => 20). |
| 36 | +def sets_timing?(node) |
| 37 | + any_call?(node) do |call| |
| 38 | + next true if %i[time= warmup=].include?(call.name) |
| 39 | + next false unless call.name == :config && call.arguments |
| 40 | + |
| 41 | + call.arguments.arguments.any? do |arg| |
| 42 | + next false unless arg.is_a?(Prism::KeywordHashNode) || arg.is_a?(Prism::HashNode) |
| 43 | + |
| 44 | + arg.elements.any? do |element| |
| 45 | + element.is_a?(Prism::AssocNode) && element.key.is_a?(Prism::SymbolNode) && |
| 46 | + TIMING_KEYS.include?(element.key.unescaped) |
| 47 | + end |
| 48 | + end |
| 49 | + end |
| 50 | +end |
| 51 | + |
| 52 | +# Every method is known by its name, and by its scope |
| 53 | +# ("Foo#run", or "#run" at the top of the file). |
| 54 | +# A class method is also known by its full class name ("A::B.run"), |
| 55 | +# whether defined as `def self.run` or inside `class << self`, |
| 56 | +# so a call like `A.run` does not count as calling `C.run`. |
| 57 | +Definition = Struct.new(:keys, :node, :owner) |
| 58 | + |
| 59 | +def nested_name(owner, node) |
| 60 | + [owner, node.constant_path.slice].compact.join("::") |
| 61 | +end |
| 62 | + |
| 63 | +def definitions(node, owner = nil, singleton = false, found = []) |
| 64 | + return found unless node |
| 65 | + |
| 66 | + case node |
| 67 | + when Prism::ClassNode, Prism::ModuleNode |
| 68 | + owner = nested_name(owner, node) |
| 69 | + singleton = false |
| 70 | + when Prism::SingletonClassNode |
| 71 | + singleton = node.expression.is_a?(Prism::SelfNode) |
| 72 | + when Prism::DefNode |
| 73 | + class_method = owner && (singleton || node.receiver.is_a?(Prism::SelfNode)) |
| 74 | + keys = [node.name.to_s, "#{owner}##{node.name}"] |
| 75 | + keys << "#{owner}.#{node.name}" if class_method |
| 76 | + found << Definition.new(keys, node, owner) |
| 77 | + end |
| 78 | + node.compact_child_nodes.each { |child| definitions(child, owner, singleton, found) } |
| 79 | + found |
| 80 | +end |
| 81 | + |
| 82 | +# Keys a call can reach: |
| 83 | +# - a call without a receiver reaches a method in the same class or at the |
| 84 | +# top of the file when the file defines one, otherwise any method of that |
| 85 | +# name (one from a parent class or an included module); |
| 86 | +# - `Foo.run` reaches "Foo.run" when the file defines it, otherwise any `run`; |
| 87 | +# - a call on another object (`Foo.new.run`) reaches any `run`; |
| 88 | +# - calls on a variable (x.report, x.compare!) are the benchmark's own API. |
| 89 | +def keys_of(call, owner, known) |
| 90 | + receiver = call.receiver |
| 91 | + case receiver |
| 92 | + when Prism::LocalVariableReadNode |
| 93 | + [] |
| 94 | + when nil, Prism::SelfNode |
| 95 | + scoped = ["##{call.name}", *("#{owner}##{call.name}" if owner)] |
| 96 | + scoped.intersect?(known) ? scoped : [call.name.to_s] |
| 97 | + when Prism::ConstantReadNode, Prism::ConstantPathNode |
| 98 | + key = "#{receiver.slice}.#{call.name}" |
| 99 | + known.include?(key) ? [key] : [call.name.to_s] |
| 100 | + else |
| 101 | + [call.name.to_s] |
| 102 | + end |
| 103 | +end |
| 104 | + |
| 105 | +# Keys of the methods called anywhere under `node`, from code in class |
| 106 | +# `owner`. With `top_level: true`, only calls outside any method: |
| 107 | +# the file's entry points. |
| 108 | +def called_keys(node, known, owner: nil, top_level: false, keys: []) |
| 109 | + return keys unless node |
| 110 | + return keys if top_level && node.is_a?(Prism::DefNode) |
| 111 | + |
| 112 | + owner = nested_name(owner, node) if node.is_a?(Prism::ClassNode) || node.is_a?(Prism::ModuleNode) |
| 113 | + keys.concat(keys_of(node, owner, known)) if node.is_a?(Prism::CallNode) |
| 114 | + node.compact_child_nodes.each do |child| |
| 115 | + called_keys(child, known, owner: owner, top_level: top_level, keys: keys) |
| 116 | + end |
| 117 | + keys |
| 118 | +end |
| 119 | + |
| 120 | +# Methods that run when the file runs: called from top-level code, or from a method that does. |
| 121 | +def reachable_methods(root, methods) |
| 122 | + known = methods.flat_map { |m| m.keys.drop(1) } |
| 123 | + keys = called_keys(root, known, top_level: true) |
| 124 | + reached = [] |
| 125 | + loop do |
| 126 | + newly = (methods - reached).select { |m| m.keys.intersect?(keys) } |
| 127 | + break if newly.empty? |
| 128 | + |
| 129 | + reached.concat(newly) |
| 130 | + newly.each { |m| keys.concat(called_keys(m.node.body, known, owner: m.owner)) } |
| 131 | + end |
| 132 | + reached |
| 133 | +end |
| 134 | + |
| 135 | +def lint(file) |
| 136 | + result = Prism.parse_file(file) |
| 137 | + return ["does not parse: #{result.errors.first.message}"] if result.failure? |
| 138 | + |
| 139 | + blocks = find_calls(result.value, :ips) |
| 140 | + return ["no Benchmark.ips block"] if blocks.empty? |
| 141 | + |
| 142 | + methods = definitions(result.value) |
| 143 | + reached = reachable_methods(result.value, methods) |
| 144 | + problems = [] |
| 145 | + |
| 146 | + blocks.each_with_index do |ips, index| |
| 147 | + where = blocks.size > 1 ? "block #{index + 1} (line #{ips.location.start_line})" : "Benchmark.ips" |
| 148 | + |
| 149 | + problems << "#{where}: no x.compare!" unless any_call?(ips.block) { |call| call.name == :compare! } |
| 150 | + problems << "#{where}: remove the timing arguments, use the default" if ips.arguments |
| 151 | + problems << "#{where}: remove the timing settings, use the default" if sets_timing?(ips.block) |
| 152 | + |
| 153 | + # The innermost method around this Benchmark.ips, if any. |
| 154 | + owner = methods.select do |m| |
| 155 | + m.node.location.start_offset <= ips.location.start_offset && ips.location.end_offset <= m.node.location.end_offset |
| 156 | + end.min_by { |m| m.node.location.length } |
| 157 | + |
| 158 | + if owner && !reached.include?(owner) |
| 159 | + name = owner.keys.last.delete_prefix("#") |
| 160 | + problems << "#{where}: inside `def #{name}`, which never runs from the top of the file" |
| 161 | + end |
| 162 | + end |
| 163 | + problems |
| 164 | +end |
| 165 | + |
| 166 | +files = ARGV.empty? ? Dir["code/**/*.rb"].sort : ARGV |
| 167 | +problems = files.flat_map { |file| lint(file).map { |problem| "#{file}: #{problem}" } } |
| 168 | + |
| 169 | +if problems.empty? |
| 170 | + puts "All #{files.size} benchmark files have the expected shape." |
| 171 | +else |
| 172 | + noun = problems.size == 1 ? "problem" : "problems" |
| 173 | + puts problems, "", "#{problems.size} #{noun}, see \"Note on entry\" in CONTRIBUTING.md." |
| 174 | + exit 1 |
| 175 | +end |
0 commit comments