Difuse/Mellaris
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improves flow handling and adds runtime stats APIs

Browse Source 
Refactors TCP and UDP flow managers to enhance analyzer selection and flow binding accuracy, including O(1) UDP stream rebinding by 5-tuple.
Introduces runtime stats tracking for engine and ruleset operations, exposing new APIs for granular performance and error metrics.
Optimizes GeoMatcher with result caching and supports efficient geosite set matching, reducing redundant computation in ruleset expressions.
This commit is contained in:
hayzam
2026-05-13 06:10:38 +05:30
parent 3f895adb43
commit 7a3f6e945d
23 changed files with 1440 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ruleset/expr.go
+203 -24
View File
@@ -60,8 +60,8 @@ type compiledExprRule struct {
ModInstance modifier.Instance
Program *vm.Program
GeoSiteConditions []string
StartTimeSecs int // seconds since midnight, -1 if unset
StopTimeSecs int // seconds since midnight, -1 if unset
StartTimeSecs int // seconds since midnight, -1 if unset
StopTimeSecs int // seconds since midnight, -1 if unset
Weekdays []time.Weekday
WeekdaysNegated bool
}
@@ -86,6 +86,7 @@ type exprRuleset struct {
Ans []analyzer.Analyzer
Logger Logger
GeoMatcher *geo.GeoMatcher
stats *statsCounters
}
func (r *exprRuleset) Analyzers(info StreamInfo) []analyzer.Analyzer {
@@ -93,9 +94,24 @@ func (r *exprRuleset) Analyzers(info StreamInfo) []analyzer.Analyzer {
}
func (r *exprRuleset) Match(info StreamInfo) MatchResult {
start := time.Now()
if r.stats != nil {
r.stats.MatchCalls.Add(1)
defer func() {
r.stats.MatchLatencyNanos.Add(uint64(time.Since(start).Nanoseconds()))
}()
}
env := envPool.Get().(map[string]any)
clear(env)
populateExprEnv(env, info)
macMap, ipMap, portMap := populateExprEnv(env, info)
releaseEnv := func() {
clear(env)
envPool.Put(env)
putSubMap(macMap)
putSubMap(ipMap)
putSubMap(portMap)
}
now := time.Now()
for _, rule := range r.Rules {
if !matchTime(now, rule.StartTimeSecs, rule.StopTimeSecs, rule.Weekdays, rule.WeekdaysNegated) {
@@ -103,6 +119,9 @@ func (r *exprRuleset) Match(info StreamInfo) MatchResult {
}
v, err := vm.Run(rule.Program, env)
if err != nil {
if r.stats != nil {
r.stats.MatchErrors.Add(1)
}
r.Logger.MatchError(info, rule.Name, err)
continue
}
@@ -115,7 +134,7 @@ func (r *exprRuleset) Match(info StreamInfo) MatchResult {
r.Logger.Log(logInfo, rule.Name)
}
if rule.Action != nil {
envPool.Put(env)
releaseEnv()
return MatchResult{
Action: *rule.Action,
ModInstance: rule.ModInstance,
@@ -123,12 +142,26 @@ func (r *exprRuleset) Match(info StreamInfo) MatchResult {
}
}
}
envPool.Put(env)
releaseEnv()
return MatchResult{
Action: ActionMaybe,
}
}
func (r *exprRuleset) Stats() Stats {
if r == nil || r.stats == nil {
return Stats{}
}
return Stats{
MatchCalls: r.stats.MatchCalls.Load(),
MatchErrors: r.stats.MatchErrors.Load(),
MatchLatencyNanos: r.stats.MatchLatencyNanos.Load(),
LookupCalls: r.stats.LookupCalls.Load(),
LookupErrors: r.stats.LookupErrors.Load(),
LookupLatencyNanos: r.stats.LookupLatencyNanos.Load(),
}
}
// CompileExprRules compiles a list of expression rules into a ruleset.
// It returns an error if any of the rules are invalid, or if any of the analyzers
// used by the rules are unknown (not provided in the analyzer list).
@@ -137,7 +170,8 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier
fullAnMap := analyzersToMap(ans)
fullModMap := modifiersToMap(mods)
depAnMap := make(map[string]analyzer.Analyzer)
funcMap, geoMatcher := buildFunctionMap(config)
stats := &statsCounters{}
funcMap, geoMatcher := buildFunctionMap(config, stats)
// Compile all rules and build a map of analyzers that are used by the rules.
for _, rule := range rules {
if rule.Action == "" && !rule.Log {
@@ -152,7 +186,7 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier
action = &a
}
visitor := &idVisitor{Variables: make(map[string]bool), Identifiers: make(map[string]bool)}
patcher := &idPatcher{FuncMap: funcMap}
patcher := &idPatcher{FuncMap: funcMap, GeoMatcher: geoMatcher}
program, err := expr.Compile(rule.Expr,
func(c *conf.Config) {
c.Strict = false
@@ -242,29 +276,47 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier
Ans: depAns,
Logger: config.Logger,
GeoMatcher: geoMatcher,
stats: stats,
}, nil
}
func populateExprEnv(m map[string]any, info StreamInfo) {
func populateExprEnv(m map[string]any, info StreamInfo) (macMap, ipMap, portMap map[string]any) {
macMap = getSubMap()
ipMap = getSubMap()
portMap = getSubMap()
macMap["src"] = info.SrcMAC.String()
macMap["dst"] = info.DstMAC.String()
ipMap["src"] = info.SrcIP.String()
ipMap["dst"] = info.DstIP.String()
portMap["src"] = info.SrcPort
portMap["dst"] = info.DstPort
m["id"] = info.ID
m["proto"] = info.Protocol.String()
m["mac"] = map[string]string{
"src": info.SrcMAC.String(),
"dst": info.DstMAC.String(),
}
m["ip"] = map[string]string{
"src": info.SrcIP.String(),
"dst": info.DstIP.String(),
}
m["port"] = map[string]uint16{
"src": info.SrcPort,
"dst": info.DstPort,
}
m["mac"] = macMap
m["ip"] = ipMap
m["port"] = portMap
for anName, anProps := range info.Props {
if len(anProps) != 0 {
m[anName] = anProps
}
}
return macMap, ipMap, portMap
}
func getSubMap() map[string]any {
m := subMapPool.Get().(map[string]any)
clear(m)
return m
}
func putSubMap(m map[string]any) {
if m == nil {
return
}
clear(m)
subMapPool.Put(m)
}
func isBuiltInAnalyzer(name string) bool {
@@ -329,11 +381,15 @@ func (v *idVisitor) Visit(node *ast.Node) {
// idPatcher patches the AST during expr compilation, replacing certain values with
// their internal representations for better runtime performance.
type idPatcher struct {
FuncMap map[string]*Function
Err error
FuncMap map[string]*Function
GeoMatcher *geo.GeoMatcher
Err error
}
func (p *idPatcher) Visit(node *ast.Node) {
if p.tryPatchGeoSiteORChain(node) {
return
}
switch (*node).(type) {
case *ast.CallNode:
callNode := (*node).(*ast.CallNode)
@@ -352,6 +408,108 @@ func (p *idPatcher) Visit(node *ast.Node) {
}
}
func (p *idPatcher) tryPatchGeoSiteORChain(node *ast.Node) bool {
if p == nil || p.GeoMatcher == nil {
return false
}
terms, ok := collectGeoSiteORChain(*node)
if !ok || len(terms) < 2 {
return false
}
hostExpr := strings.TrimSpace(terms[0].hostExpr)
if hostExpr == "" {
return false
}
conditions := make([]string, 0, len(terms))
for _, term := range terms {
if strings.TrimSpace(term.hostExpr) != hostExpr {
return false
}
conditions = append(conditions, term.condition)
}
normalized := normalizeUniqueLowerStrings(conditions)
if len(normalized) < 2 {
return false
}
hostNode, err := parser.Parse(hostExpr)
if err != nil || hostNode == nil || hostNode.Node == nil {
return false
}
call := &ast.CallNode{
Callee: &ast.IdentifierNode{Value: "geosite_set"},
Arguments: []ast.Node{
hostNode.Node,
&ast.ConstantNode{Value: &geo.SiteConditionSet{Conditions: normalized}},
},
}
ast.Patch(node, call)
return true
}
type geositeTerm struct {
hostExpr string
condition string
}
func collectGeoSiteORChain(node ast.Node) ([]geositeTerm, bool) {
switch n := node.(type) {
case *ast.BinaryNode:
if n.Operator != "or" && n.Operator != "||" {
return nil, false
}
left, ok := collectGeoSiteORChain(n.Left)
if !ok {
return nil, false
}
right, ok := collectGeoSiteORChain(n.Right)
if !ok {
return nil, false
}
out := make([]geositeTerm, 0, len(left)+len(right))
out = append(out, left...)
out = append(out, right...)
return out, true
case *ast.CallNode:
idNode, ok := n.Callee.(*ast.IdentifierNode)
if !ok || len(n.Arguments) < 2 {
return nil, false
}
name := strings.ToLower(idNode.Value)
if name == "geosite" {
condNode, ok := n.Arguments[1].(*ast.StringNode)
if !ok {
return nil, false
}
return []geositeTerm{{
hostExpr: n.Arguments[0].String(),
condition: condNode.Value,
}}, true
}
if name != "geosite_set" {
return nil, false
}
setNode, ok := n.Arguments[1].(*ast.ConstantNode)
if !ok || setNode.Value == nil {
return nil, false
}
set, ok := setNode.Value.(*geo.SiteConditionSet)
if !ok || set == nil {
return nil, false
}
if len(set.Conditions) == 0 {
return nil, false
}
out := make([]geositeTerm, 0, len(set.Conditions))
hostExpr := n.Arguments[0].String()
for _, condition := range set.Conditions {
out = append(out, geositeTerm{hostExpr: hostExpr, condition: condition})
}
return out, true
default:
return nil, false
}
}
type Function struct {
InitFunc func() error
PatchFunc func(args *[]ast.Node) error
@@ -359,7 +517,7 @@ type Function struct {
Types []reflect.Type
}
func buildFunctionMap(config *BuiltinConfig) (map[string]*Function, *geo.GeoMatcher) {
func buildFunctionMap(config *BuiltinConfig, stats *statsCounters) (map[string]*Function, *geo.GeoMatcher) {
geoMatcher := geo.NewGeoMatcher(config.GeoSiteFilename, config.GeoIpFilename)
return map[string]*Function{
"geoip": {
@@ -378,6 +536,16 @@ func buildFunctionMap(config *BuiltinConfig) (map[string]*Function, *geo.GeoMatc
},
Types: []reflect.Type{reflect.TypeOf(geoMatcher.MatchGeoSite)},
},
"geosite_set": {
InitFunc: geoMatcher.LoadGeoSite,
PatchFunc: nil,
Func: func(params ...any) (any, error) {
return geoMatcher.MatchGeoSiteSet(params[0].(string), params[1].(*geo.SiteConditionSet)), nil
},
Types: []reflect.Type{
reflect.TypeOf((func(string, *geo.SiteConditionSet) bool)(nil)),
},
},
"cidr": {
InitFunc: nil,
PatchFunc: func(args *[]ast.Node) error {
@@ -425,9 +593,20 @@ func buildFunctionMap(config *BuiltinConfig) (map[string]*Function, *geo.GeoMatc
return nil
},
Func: func(params ...any) (any, error) {
start := time.Now()
if stats != nil {
stats.LookupCalls.Add(1)
defer func() {
stats.LookupLatencyNanos.Add(uint64(time.Since(start).Nanoseconds()))
}()
}
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
defer cancel()
return params[1].(*net.Resolver).LookupHost(ctx, params[0].(string))
out, err := params[1].(*net.Resolver).LookupHost(ctx, params[0].(string))
if err != nil && stats != nil {
stats.LookupErrors.Add(1)
}
return out, err
},
Types: []reflect.Type{
reflect.TypeOf((func(string, *net.Resolver) []string)(nil)),