chore: rule-provider now read yaml line-by-line

This commit is contained in:
wwqgtxx 2023-04-01 14:11:09 +08:00
parent 54cad53f5f
commit 54c2fa98b4
4 changed files with 148 additions and 66 deletions

View file

@ -37,11 +37,15 @@ func (c *classicalStrategy) ShouldFindProcess() bool {
return c.shouldFindProcess return c.shouldFindProcess
} }
func (c *classicalStrategy) OnUpdate(rules []string) { func (c *classicalStrategy) Reset() {
var classicalRules []C.Rule c.rules = nil
shouldResolveIP := false c.count = 0
for _, rawRule := range rules { c.shouldFindProcess = false
ruleType, rule, params := ruleParse(rawRule) c.shouldResolveIP = false
}
func (c *classicalStrategy) Insert(rule string) {
ruleType, rule, params := ruleParse(rule)
if ruleType == "PROCESS-NAME" { if ruleType == "PROCESS-NAME" {
c.shouldFindProcess = true c.shouldFindProcess = true
@ -51,22 +55,20 @@ func (c *classicalStrategy) OnUpdate(rules []string) {
if err != nil { if err != nil {
log.Warnln("parse rule error:[%s]", err.Error()) log.Warnln("parse rule error:[%s]", err.Error())
} else { } else {
if !shouldResolveIP { if r.ShouldResolveIP() {
shouldResolveIP = r.ShouldResolveIP() c.shouldResolveIP = true
}
if r.ShouldFindProcess() {
c.shouldFindProcess = true
} }
if !c.shouldFindProcess { c.rules = append(c.rules, r)
c.shouldFindProcess = r.ShouldFindProcess() c.count++
} }
classicalRules = append(classicalRules, r)
}
}
c.rules = classicalRules
c.count = len(classicalRules)
} }
func (c *classicalStrategy) FinishInsert() {}
func ruleParse(ruleRaw string) (string, string, []string) { func ruleParse(ruleRaw string) (string, string, []string) {
item := strings.Split(ruleRaw, ",") item := strings.Split(ruleRaw, ",")
if len(item) == 1 { if len(item) == 1 {

View file

@ -8,7 +8,8 @@ import (
type domainStrategy struct { type domainStrategy struct {
count int count int
domainRules *trie.DomainSet domainTrie *trie.DomainTrie[struct{}]
domainSet *trie.DomainSet
} }
func (d *domainStrategy) ShouldFindProcess() bool { func (d *domainStrategy) ShouldFindProcess() bool {
@ -16,7 +17,7 @@ func (d *domainStrategy) ShouldFindProcess() bool {
} }
func (d *domainStrategy) Match(metadata *C.Metadata) bool { func (d *domainStrategy) Match(metadata *C.Metadata) bool {
return d.domainRules != nil && d.domainRules.Has(metadata.RuleHost()) return d.domainSet != nil && d.domainSet.Has(metadata.RuleHost())
} }
func (d *domainStrategy) Count() int { func (d *domainStrategy) Count() int {
@ -27,16 +28,24 @@ func (d *domainStrategy) ShouldResolveIP() bool {
return false return false
} }
func (d *domainStrategy) OnUpdate(rules []string) { func (d *domainStrategy) Reset() {
domainTrie := trie.New[struct{}]() d.domainTrie = trie.New[struct{}]()
for _, rule := range rules { d.domainSet = nil
err := domainTrie.Insert(rule, struct{}{}) d.count = 0
}
func (d *domainStrategy) Insert(rule string) {
err := d.domainTrie.Insert(rule, struct{}{})
if err != nil { if err != nil {
log.Warnln("invalid domain:[%s]", rule) log.Warnln("invalid domain:[%s]", rule)
} else {
d.count++
} }
} }
d.domainRules = domainTrie.NewDomainSet()
d.count = len(rules) func (d *domainStrategy) FinishInsert() {
d.domainSet = d.domainTrie.NewDomainSet()
d.domainTrie = nil
} }
func NewDomainStrategy() *domainStrategy { func NewDomainStrategy() *domainStrategy {

View file

@ -28,23 +28,24 @@ func (i *ipcidrStrategy) ShouldResolveIP() bool {
return i.shouldResolveIP return i.shouldResolveIP
} }
func (i *ipcidrStrategy) OnUpdate(rules []string) { func (i *ipcidrStrategy) Reset() {
ipCidrTrie := trie.NewIpCidrTrie() i.trie = trie.NewIpCidrTrie()
count := 0 i.count = 0
for _, rule := range rules { i.shouldResolveIP = false
err := ipCidrTrie.AddIpCidrForString(rule) }
func (i *ipcidrStrategy) Insert(rule string) {
err := i.trie.AddIpCidrForString(rule)
if err != nil { if err != nil {
log.Warnln("invalid Ipcidr:[%s]", rule) log.Warnln("invalid Ipcidr:[%s]", rule)
} else { } else {
count++ i.shouldResolveIP = true
i.count++
} }
}
i.trie = ipCidrTrie
i.count = count
i.shouldResolveIP = i.count > 0
} }
func (i *ipcidrStrategy) FinishInsert() {}
func NewIPCidrStrategy() *ipcidrStrategy { func NewIPCidrStrategy() *ipcidrStrategy {
return &ipcidrStrategy{} return &ipcidrStrategy{}
} }

View file

@ -1,13 +1,19 @@
package provider package provider
import ( import (
"bufio"
"bytes"
"encoding/json" "encoding/json"
"errors"
"gopkg.in/yaml.v3"
"io"
"runtime"
"time"
"github.com/Dreamacro/clash/common/pool"
"github.com/Dreamacro/clash/component/resource" "github.com/Dreamacro/clash/component/resource"
C "github.com/Dreamacro/clash/constant" C "github.com/Dreamacro/clash/constant"
P "github.com/Dreamacro/clash/constant/provider" P "github.com/Dreamacro/clash/constant/provider"
"gopkg.in/yaml.v3"
"runtime"
"time"
) )
var ( var (
@ -29,8 +35,8 @@ type RulePayload struct {
key: Domain or IP Cidr key: Domain or IP Cidr
value: Rule type or is empty value: Rule type or is empty
*/ */
Rules []string `yaml:"payload"` Payload []string `yaml:"payload"`
Rules2 []string `yaml:"rules"` Rules []string `yaml:"rules"`
} }
type ruleStrategy interface { type ruleStrategy interface {
@ -38,7 +44,9 @@ type ruleStrategy interface {
Count() int Count() int
ShouldResolveIP() bool ShouldResolveIP() bool
ShouldFindProcess() bool ShouldFindProcess() bool
OnUpdate(rules []string) Reset()
Insert(rule string)
FinishInsert()
} }
func RuleProviders() map[string]P.RuleProvider { func RuleProviders() map[string]P.RuleProvider {
@ -114,13 +122,12 @@ func NewRuleSetProvider(name string, behavior P.RuleType, interval time.Duration
} }
onUpdate := func(elm interface{}) { onUpdate := func(elm interface{}) {
rulesRaw := elm.([]string) strategy := elm.(ruleStrategy)
rp.strategy.OnUpdate(rulesRaw) rp.strategy = strategy
} }
fetcher := resource.NewFetcher(name, interval, vehicle, rulesParse, onUpdate)
rp.Fetcher = fetcher
rp.strategy = newStrategy(behavior, parse) rp.strategy = newStrategy(behavior, parse)
rp.Fetcher = resource.NewFetcher(name, interval, vehicle, func(bytes []byte) (any, error) { return rulesParse(bytes, newStrategy(behavior, parse)) }, onUpdate)
wrapper := &RuleSetProvider{ wrapper := &RuleSetProvider{
rp, rp,
@ -147,12 +154,75 @@ func newStrategy(behavior P.RuleType, parse func(tp, payload, target string, par
} }
} }
func rulesParse(buf []byte) (any, error) { var ErrNoPayload = errors.New("file must have a `payload` field")
rulePayload := RulePayload{}
err := yaml.Unmarshal(buf, &rulePayload) func rulesParse(buf []byte, strategy ruleStrategy) (any, error) {
strategy.Reset()
schema := &RulePayload{}
reader := bufio.NewReader(bytes.NewReader(buf))
firstLineBuffer := pool.GetBuffer()
defer pool.PutBuffer(firstLineBuffer)
firstLineLength := 0
for {
line, isPrefix, err := reader.ReadLine()
if err != nil { if err != nil {
if err == io.EOF {
if firstLineLength == 0 { // find payload head
return nil, ErrNoPayload
}
break
}
return nil, err return nil, err
} }
firstLineBuffer.Write(line) // need a copy because the returned buffer is only valid until the next call to ReadLine
if isPrefix {
// If the line was too long for the buffer then isPrefix is set and the
// beginning of the line is returned. The rest of the line will be returned
// from future calls.
continue
}
if firstLineLength == 0 { // find payload head
firstLineBuffer.WriteByte('\n')
firstLineLength = firstLineBuffer.Len()
firstLineBuffer.WriteString(" - ''") // a test line
return append(rulePayload.Rules, rulePayload.Rules2...), nil err = yaml.Unmarshal(firstLineBuffer.Bytes(), schema)
firstLineBuffer.Truncate(firstLineLength)
if err == nil && (len(schema.Rules) > 0 || len(schema.Payload) > 0) { // found
continue
}
// not found or err!=nil
firstLineBuffer.Truncate(0)
firstLineLength = 0
continue
}
// parse payload body
err = yaml.Unmarshal(firstLineBuffer.Bytes(), schema)
firstLineBuffer.Truncate(firstLineLength)
if err != nil {
continue
}
var str string
if len(schema.Rules) > 0 {
str = schema.Rules[0]
}
if len(schema.Payload) > 0 {
str = schema.Payload[0]
}
if str == "" {
continue
}
strategy.Insert(str)
}
strategy.FinishInsert()
return strategy, nil
} }