engine: more performance improvements

ruleset/expr.go

@@ -59,6 +59,8 @@ type compiledExprRule struct {
 	Log               bool
 	ModInstance       modifier.Instance
 	Program           *vm.Program
+	Native            nativeExpr
+	AnalyzerRefs      map[string]analyzerRuleRef
 	GeoSiteConditions []string
 	StartTimeSecs     int // seconds since midnight, -1 if unset
 	StopTimeSecs      int // seconds since midnight, -1 if unset
@@ -67,6 +69,7 @@ type compiledExprRule struct {
 }
 
 var _ Ruleset = (*exprRuleset)(nil)
+var _ LogFinalizer = (*exprRuleset)(nil)
 
 var (
 	envPool = sync.Pool{
@@ -102,10 +105,12 @@ func (r *exprRuleset) Match(info StreamInfo) MatchResult {
 		}()
 	}
 
-	env := envPool.Get().(map[string]any)
-	clear(env)
-	macMap, ipMap, portMap := populateExprEnv(env, info)
+	var env map[string]any
+	var macMap, ipMap, portMap map[string]any
 	releaseEnv := func() {
+		if env == nil {
+			return
+		}
 		clear(env)
 		envPool.Put(env)
 		putSubMap(macMap)
@@ -113,31 +118,45 @@ func (r *exprRuleset) Match(info StreamInfo) MatchResult {
 		putSubMap(portMap)
 	}
 	now := time.Now()
+	logged := false
 	for _, rule := range r.Rules {
 		if !matchTime(now, rule.StartTimeSecs, rule.StopTimeSecs, rule.Weekdays, rule.WeekdaysNegated) {
 			continue
 		}
-		v, err := vm.Run(rule.Program, env)
-		if err != nil {
-			if r.stats != nil {
-				r.stats.MatchErrors.Add(1)
+		matched := false
+		if rule.Native != nil {
+			matched = rule.Native.Match(info)
+		} else {
+			if env == nil {
+				env = envPool.Get().(map[string]any)
+				clear(env)
+				macMap, ipMap, portMap = populateExprEnv(env, info)
 			}
-			r.Logger.MatchError(info, rule.Name, err)
-			continue
+			v, err := vm.Run(rule.Program, env)
+			if err != nil {
+				if r.stats != nil {
+					r.stats.MatchErrors.Add(1)
+				}
+				r.Logger.MatchError(info, rule.Name, err)
+				continue
+			}
+			matched, _ = v.(bool)
 		}
-		if vBool, ok := v.(bool); ok && vBool {
+		if matched {
 			if rule.Log {
 				logInfo := info
 				if len(rule.GeoSiteConditions) > 0 && r.GeoMatcher != nil {
 					logInfo = addGeoSiteLogMetadata(logInfo, r.GeoMatcher, rule.GeoSiteConditions)
 				}
 				r.Logger.Log(logInfo, rule.Name)
+				logged = true
 			}
 			if rule.Action != nil {
 				releaseEnv()
 				return MatchResult{
 					Action:      *rule.Action,
 					ModInstance: rule.ModInstance,
+					Logged:      logged,
 				}
 			}
 		}
@@ -145,9 +164,40 @@ func (r *exprRuleset) Match(info StreamInfo) MatchResult {
 	releaseEnv()
 	return MatchResult{
 		Action: ActionMaybe,
+		Logged: logged,
 	}
 }
 
+func (r *exprRuleset) CanFinalizeAfterLog(info StreamInfo, activeAnalyzers []string) bool {
+	active := make(map[string]bool, len(activeAnalyzers))
+	for _, name := range activeAnalyzers {
+		active[name] = true
+	}
+	for _, rule := range r.Rules {
+		if rule.Action == nil {
+			continue
+		}
+		if *rule.Action == ActionModify {
+			return false
+		}
+		if rule.StartTimeSecs != -1 || rule.StopTimeSecs != -1 || len(rule.Weekdays) != 0 {
+			return false
+		}
+		for name, ref := range rule.AnalyzerRefs {
+			if !active[name] {
+				continue
+			}
+			if ref.ResponseSide {
+				return false
+			}
+			if _, ok := info.Props[name]; !ok {
+				return false
+			}
+		}
+	}
+	return true
+}
+
 func (r *exprRuleset) Stats() Stats {
 	if r == nil || r.stats == nil {
 		return Stats{}
@@ -242,17 +292,23 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier
 		if err != nil {
 			return nil, fmt.Errorf("rule %q has invalid weekdays: %w", rule.Name, err)
 		}
+		var analyzerRefs map[string]analyzerRuleRef
+		if refTree, err := parser.Parse(rule.Expr); err == nil && refTree != nil {
+			analyzerRefs = collectAnalyzerRefs(refTree.Node, fullAnMap)
+		}
 		cr := compiledExprRule{
 			Name:              rule.Name,
 			Action:            action,
 			Log:               rule.Log,
 			Program:           program,
+			AnalyzerRefs:      analyzerRefs,
 			GeoSiteConditions: extractGeoSiteConditions(rule.Expr),
 			StartTimeSecs:     startSecs,
 			StopTimeSecs:      stopSecs,
 			Weekdays:          weekdays,
 			WeekdaysNegated:   weekdaysNegated,
 		}
+		cr.Native = compileNativeExpr(rule.Expr, funcMap, geoMatcher)
 		if action != nil && *action == ActionModify {
 			mod, ok := fullModMap[rule.Modifier.Name]
 			if !ok {
@@ -266,9 +322,16 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier
 		}
 		compiledRules = append(compiledRules, cr)
 	}
+	depAns := make([]analyzer.Analyzer, 0, len(depAnMap))
+	for _, a := range ans {
+		if depAnMap[a.Name()] != nil {
+			depAns = append(depAns, a)
+		}
+	}
+
 	return &exprRuleset{
 		Rules:      compiledRules,
-		Ans:        ans,
+		Ans:        depAns,
 		Logger:     config.Logger,
 		GeoMatcher: geoMatcher,
 		stats:      stats,
@@ -373,6 +436,58 @@ func (v *idVisitor) Visit(node *ast.Node) {
 	}
 }
 
+type analyzerRuleRef struct {
+	ResponseSide bool
+}
+
+type analyzerRefVisitor struct {
+	Analyzers map[string]analyzer.Analyzer
+	Refs      map[string]analyzerRuleRef
+}
+
+func collectAnalyzerRefs(root ast.Node, analyzers map[string]analyzer.Analyzer) map[string]analyzerRuleRef {
+	visitor := &analyzerRefVisitor{
+		Analyzers: analyzers,
+		Refs:      make(map[string]analyzerRuleRef),
+	}
+	ast.Walk(&root, visitor)
+	return visitor.Refs
+}
+
+func (v *analyzerRefVisitor) Visit(node *ast.Node) {
+	switch n := (*node).(type) {
+	case *ast.IdentifierNode:
+		if _, ok := v.Analyzers[n.Value]; ok {
+			v.add(n.Value, false)
+		}
+	case *ast.MemberNode:
+		path := memberPath(n)
+		if len(path) == 0 {
+			return
+		}
+		name := path[0]
+		if _, ok := v.Analyzers[name]; !ok {
+			return
+		}
+		v.add(name, len(path) > 1 && isResponseSideAnalyzerPath(path[1]))
+	}
+}
+
+func (v *analyzerRefVisitor) add(name string, responseSide bool) {
+	ref := v.Refs[name]
+	ref.ResponseSide = ref.ResponseSide || responseSide
+	v.Refs[name] = ref
+}
+
+func isResponseSideAnalyzerPath(name string) bool {
+	switch name {
+	case "resp", "server", "answers", "response":
+		return true
+	default:
+		return false
+	}
+}
+
 // idPatcher patches the AST during expr compilation, replacing certain values with
 // their internal representations for better runtime performance.
 type idPatcher struct {