Skip to content

Commit cf4de7a

Browse files
andrewvcandrewvc
committed
Simple vertex hashing
Fixes elastic#8836
1 parent 590e47f commit cf4de7a

17 files changed

Lines changed: 286 additions & 304 deletions

File tree

‎logstash-core/lib/logstash/compiler/lscl.rb‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,10 @@ def javaify_sexpr(sexpr)
262262
java_f_branch = f_branch && javaify_sexpr(f_branch)
263263

264264
if java_t_branch || java_f_branch
265-
jdsl.iIf(condition, java_t_branch || jdsl.noop, java_f_branch || jdsl.noop)
265+
# We use the condition as the source with metadata because it hashes correctly
266+
# It's hard to use the 'real' source due to the re-branching from if / elsif into if/else only
267+
# branches. We should come back and improve this at some point if that makes a difference
268+
jdsl.iIf(condition.source_with_metadata, condition, java_t_branch || jdsl.noop, java_f_branch || jdsl.noop)
266269
else
267270
jdsl.noop()
268271
end
@@ -297,7 +300,8 @@ def expr
297300

298301
all_elements = [first_element, *rest_elements]
299302

300-
if all_elements.size == 1
303+
304+
res = if all_elements.size == 1
301305
elem = all_elements.first
302306
if elem.is_a?(Selector)
303307
eventValue = elem.recursive_select(SelectorElement).first.expr
@@ -310,6 +314,7 @@ def expr
310314
else
311315
join_conditions(all_elements)
312316
end
317+
res
313318
end
314319

315320
def precedence(op)
@@ -339,13 +344,13 @@ def jconvert(sexpr)
339344

340345
case op
341346
when :and
342-
return jdsl.eAnd(left, right);
347+
return jdsl.eAnd(source_meta, left, right);
343348
when :nand
344-
return jdsl.eNand(left, right);
349+
return jdsl.eNand(source_meta, left, right);
345350
when :or
346-
return jdsl.eOr(left, right);
351+
return jdsl.eOr(source_meta, left, right);
347352
when :xor
348-
return jdsl.eXor(left, right);
353+
return jdsl.eXor(source_meta, left, right);
349354
else
350355
raise "Unknown op #{jop}"
351356
end
@@ -394,7 +399,7 @@ def stack_to_expr(stack)
394399
while elem = stack.pop
395400
if elem.is_a?(::Method)
396401
right, left = working_stack.pop, working_stack.pop
397-
working_stack << elem.call(left, right)
402+
working_stack << elem.call(source_meta, left, right)
398403
else
399404
working_stack << elem
400405
end
@@ -472,13 +477,13 @@ def expr
472477
# Handle string rvalues, they just get turned into regexps
473478
# Maybe we really shouldn't handle these anymore...
474479
if regexp.class == org.logstash.config.ir.expression.ValueExpression
475-
regexp = jdsl.eRegex(regexp.get)
480+
regexp = jdsl.eRegex(source_meta, regexp.get)
476481
end
477482

478483
raise "Expected a selector in #{text_value}!" unless selector
479484
raise "Expected a regexp in #{text_value}!" unless regexp
480485

481-
operator_method.call(source_meta, selector, regexp);
486+
operator_method.call(source_meta, selector, regexp)
482487
end
483488
end
484489

@@ -512,9 +517,9 @@ module RegExpOperator
512517

513518
def expr
514519
if self.text_value == '!~'
515-
jdsl.method(:eRegexNeq)
520+
jdsl.java_method(:eRegexNeq, [org.logstash.common.SourceWithMetadata, org.logstash.config.ir.expression.Expression, org.logstash.config.ir.expression.ValueExpression])
516521
elsif self.text_value == '=~'
517-
jdsl.method(:eRegexEq)
522+
jdsl.java_method(:eRegexEq, [org.logstash.common.SourceWithMetadata, org.logstash.config.ir.expression.Expression, org.logstash.config.ir.expression.ValueExpression])
518523
else
519524
raise "Unknown regex operator #{self.text_value}"
520525
end

‎logstash-core/lib/logstash/compiler/lscl/helpers.rb‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,10 @@ def self.jdsl
4949
org.logstash.config.ir.DSL
5050
end
5151

52-
AND_METHOD = jdsl.method(:eAnd)
53-
NAND_METHOD = jdsl.method(:eNand)
54-
OR_METHOD = jdsl.method(:eOr)
55-
XOR_METHOD = jdsl.method(:eXor)
52+
BOOLEAN_DSL_METHOD_SIGNATURE = [org.logstash.common.SourceWithMetadata, org.logstash.config.ir.expression.Expression, org.logstash.config.ir.expression.Expression]
53+
AND_METHOD = jdsl.java_method(:eAnd, BOOLEAN_DSL_METHOD_SIGNATURE)
54+
NAND_METHOD = jdsl.java_method(:eNand, BOOLEAN_DSL_METHOD_SIGNATURE)
55+
OR_METHOD = jdsl.java_method(:eOr, BOOLEAN_DSL_METHOD_SIGNATURE)
56+
XOR_METHOD = jdsl.java_method(:eXor, BOOLEAN_DSL_METHOD_SIGNATURE)
5657
end
5758
end; end; end; end; end

‎logstash-core/spec/logstash/compiler/compiler_spec.rb‎

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ def j
88
Java::OrgLogstashConfigIr::DSL
99
end
1010

11+
def rand_meta
12+
org.logstash.common.SourceWithMetadata.new("test", SecureRandom.uuid, 1, 1, SecureRandom.uuid)
13+
end
14+
1115
let(:source_protocol) { "test_proto" }
1216

1317
let(:settings) { mock_settings({}) }
@@ -173,7 +177,7 @@ def j
173177
let(:plugin_source) { "generator {}" }
174178

175179
it "should contain the plugin" do
176-
expect(c_plugin).to ir_eql(j.iPlugin(INPUT, "generator"))
180+
expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, INPUT, "generator"))
177181
end
178182
end
179183

@@ -190,7 +194,7 @@ def j
190194
end
191195

192196
it "should contain the plugin" do
193-
expect(c_plugin).to ir_eql(j.iPlugin(INPUT, "generator", expected_plugin_args))
197+
expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, INPUT, "generator", expected_plugin_args))
194198
end
195199
end
196200

@@ -203,7 +207,7 @@ def j
203207
end
204208

205209
it "should contain the plugin" do
206-
expect(c_plugin).to ir_eql(j.iPlugin(INPUT, "generator", expected_plugin_args))
210+
expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, INPUT, "generator", expected_plugin_args))
207211
end
208212
end
209213

@@ -216,7 +220,7 @@ def j
216220
end
217221

218222
it "should contain the plugin" do
219-
expect(c_plugin).to ir_eql(j.iPlugin(INPUT, "generator", expected_plugin_args))
223+
expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, INPUT, "generator", expected_plugin_args))
220224
end
221225
end
222226

@@ -245,7 +249,7 @@ def j
245249
end
246250

247251
it "should merge the contents of the individual directives" do
248-
expect(c_plugin).to ir_eql(j.iPlugin(FILTER, "grok", expected_plugin_args))
252+
expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, FILTER, "grok", expected_plugin_args))
249253
end
250254

251255
describe "a filter plugin that has nested Hash directives" do
@@ -300,7 +304,7 @@ def j
300304
end
301305

302306
it "should produce a nested ::Hash object" do
303-
expect(c_plugin).to ir_eql(j.iPlugin(FILTER, "matryoshka", expected_plugin_args))
307+
expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, FILTER, "matryoshka", expected_plugin_args))
304308
end
305309
end
306310
end
@@ -313,7 +317,7 @@ def j
313317
let(:source) { "input { generator {} }" }
314318

315319
it "should contain the single input" do
316-
expect(input).to ir_eql(j.iPlugin(INPUT, "generator"))
320+
expect(input).to ir_eql(j.iPlugin(rand_meta, INPUT, "generator"))
317321
end
318322

319323
it_should_behave_like("component source_with_metadata") do
@@ -326,8 +330,8 @@ def j
326330

327331
it "should contain both inputs" do
328332
expect(input).to ir_eql(j.iComposeParallel(
329-
j.iPlugin(INPUT, "generator", {"count" => 1}),
330-
j.iPlugin(INPUT, "generator", {"count" => 2})
333+
j.iPlugin(rand_meta, INPUT, "generator", {"count" => 1}),
334+
j.iPlugin(rand_meta, INPUT, "generator", {"count" => 2})
331335
))
332336
end
333337
end
@@ -351,7 +355,7 @@ def j
351355
let (:compiled_section) { compiled[section] }
352356

353357
def splugin(*args)
354-
j.iPlugin(section_name_enum, *args)
358+
j.iPlugin(rand_meta, section_name_enum, *args)
355359
end
356360

357361
def compose(*statements)
@@ -635,6 +639,7 @@ def compose(*statements)
635639

636640
it "should compile correctly" do
637641
expect(compiled_section).to ir_eql(j.iIf(
642+
rand_meta,
638643
j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
639644
splugin("grok")
640645
)
@@ -647,6 +652,7 @@ def compose(*statements)
647652

648653
it "should compile correctly" do
649654
expect(compiled_section).to ir_eql(j.iIf(
655+
rand_meta,
650656
j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
651657
j.noop,
652658
splugin("fplugin"),
@@ -660,6 +666,7 @@ def compose(*statements)
660666

661667
it "should compile correctly" do
662668
expect(compiled_section).to ir_eql(j.iIf(
669+
rand_meta,
663670
j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
664671
j.noop,
665672
j.noop
@@ -673,6 +680,7 @@ def compose(*statements)
673680

674681
it "should compile correctly" do
675682
expect(compiled_section).to ir_eql(j.iIf(
683+
rand_meta,
676684
j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
677685
splugin("tplugin"),
678686
splugin("fplugin")
@@ -686,9 +694,11 @@ def compose(*statements)
686694

687695
it "should compile correctly" do
688696
expect(compiled_section).to ir_eql(j.iIf(
697+
rand_meta,
689698
j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
690699
splugin("tplugin"),
691700
j.iIf(
701+
rand_meta,
692702
j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")),
693703
splugin("eifplugin"),
694704
splugin("fplugin")
@@ -712,12 +722,15 @@ def compose(*statements)
712722

713723
it "should compile correctly" do
714724
expect(compiled_section).to ir_eql(j.iIf(
725+
rand_meta,
715726
j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
716727
splugin("tplugin"),
717728
j.iIf(
729+
rand_meta,
718730
j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")),
719731
splugin("eifplugin"),
720732
j.iIf(
733+
rand_meta,
721734
j.eEq(j.eEventValue("[baz]"), j.eEventValue("[bot]")),
722735
splugin("eeifplugin"),
723736
splugin("fplugin")
@@ -744,15 +757,18 @@ def compose(*statements)
744757

745758
it "should compile correctly" do
746759
expect(compiled_section).to ir_eql(j.iIf(
760+
rand_meta,
747761
j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
748-
j.iIf(j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")),
762+
j.iIf(rand_meta, j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")),
749763
splugin("aplugin"),
750764
j.noop
751765
),
752766
j.iIf(
767+
rand_meta,
753768
j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")),
754769
splugin("bplugin"),
755770
j.iIf(
771+
rand_meta,
756772
j.eEq(j.eEventValue("[baz]"), j.eEventValue("[bot]")),
757773
splugin("cplugin"),
758774
splugin("dplugin")
@@ -773,7 +789,7 @@ def compose(*statements)
773789
let(:source) { "input { } filter { grok {} } output { }" }
774790

775791
it "should contain the single filter" do
776-
expect(filter).to ir_eql(j.iPlugin(FILTER, "grok"))
792+
expect(filter).to ir_eql(j.iPlugin(rand_meta, FILTER, "grok"))
777793
end
778794

779795
it_should_behave_like("component source_with_metadata") do
@@ -791,7 +807,7 @@ def compose(*statements)
791807
let(:source) { "input { } output { stdout {} }" }
792808

793809
it "should contain the single input" do
794-
expect(output).to ir_eql(j.iPlugin(OUTPUT, "stdout"))
810+
expect(output).to ir_eql(j.iPlugin(rand_meta, OUTPUT, "stdout"))
795811
end
796812

797813
it_should_behave_like("component source_with_metadata") do

0 commit comments

Comments
 (0)