From 09ffc53d95c10e0f06248ac9d111aa33892f7bb0 Mon Sep 17 00:00:00 2001 From: gVisor bot Date: Wed, 26 Jan 2022 12:01:14 +0800 Subject: [PATCH] [chore] Replace Country.mmdb with GeoIP.dat --- Makefile | 1 + component/geodata/geodata.go | 4 +- component/geodata/router/condition.go | 42 ++++ component/geodata/router/condition_geoip.go | 243 ++++++++++++++++++++ component/mmdb/mmdb.go | 45 ---- config/initial.go | 50 ++-- constant/path.go | 4 +- dns/filters.go | 41 +++- rule/common/geoip.go | 53 ++++- 9 files changed, 383 insertions(+), 100 deletions(-) create mode 100644 component/geodata/router/condition_geoip.go delete mode 100644 component/mmdb/mmdb.go diff --git a/Makefile b/Makefile index 14e234f2..a9644fa8 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ PLATFORM_LIST = \ darwin-amd64 \ darwin-arm64 \ linux-amd64 \ + linux-amd64-AutoIptables\ linux-armv5 \ linux-armv6 \ linux-armv7 \ diff --git a/component/geodata/geodata.go b/component/geodata/geodata.go index cdd141fb..2a5b02f4 100644 --- a/component/geodata/geodata.go +++ b/component/geodata/geodata.go @@ -14,7 +14,7 @@ type loader struct { } func (l *loader) LoadGeoSite(list string) ([]*router.Domain, error) { - return l.LoadGeoSiteWithAttr("geosite.dat", list) + return l.LoadGeoSiteWithAttr("GeoSite.dat", list) } func (l *loader) LoadGeoSiteWithAttr(file string, siteWithAttr string) ([]*router.Domain, error) { @@ -58,7 +58,7 @@ func (l *loader) LoadGeoSiteWithAttr(file string, siteWithAttr string) ([]*route } func (l *loader) LoadGeoIP(country string) ([]*router.CIDR, error) { - return l.LoadIP("geoip.dat", country) + return l.LoadIP("GeoIP.dat", country) } var loaders map[string]func() LoaderImplementation diff --git a/component/geodata/router/condition.go b/component/geodata/router/condition.go index 4b63ddfc..6b3f8b86 100644 --- a/component/geodata/router/condition.go +++ b/component/geodata/router/condition.go @@ -2,6 +2,7 @@ package router import ( "fmt" + "net" "strings" "github.com/Dreamacro/clash/component/geodata/strmatcher" @@ -69,3 +70,44 @@ func NewDomainMatcher(domains []*Domain) (*DomainMatcher, error) { func (m *DomainMatcher) ApplyDomain(domain string) bool { return len(m.matchers.Match(strings.ToLower(domain))) > 0 } + +type MultiGeoIPMatcher struct { + matchers []*GeoIPMatcher +} + +func NewGeoIPMatcher(geoip *GeoIP) (*GeoIPMatcher, error) { + matcher, err := globalGeoIPContainer.Add(geoip) + if err != nil { + return nil, err + } + + return matcher, nil +} + +func (m *MultiGeoIPMatcher) ApplyIp(ip net.IP) bool { + + for _, matcher := range m.matchers { + if matcher.Match(ip) { + return true + } + } + + return false +} + +func NewMultiGeoIPMatcher(geoips []*GeoIP) (*MultiGeoIPMatcher, error) { + var matchers []*GeoIPMatcher + for _, geoip := range geoips { + matcher, err := globalGeoIPContainer.Add(geoip) + if err != nil { + return nil, err + } + matchers = append(matchers, matcher) + } + + matcher := &MultiGeoIPMatcher{ + matchers: matchers, + } + + return matcher, nil +} diff --git a/component/geodata/router/condition_geoip.go b/component/geodata/router/condition_geoip.go new file mode 100644 index 00000000..5a4bb5ca --- /dev/null +++ b/component/geodata/router/condition_geoip.go @@ -0,0 +1,243 @@ +package router + +import ( + "encoding/binary" + "fmt" + "net" + "sort" +) + +// CIDRList is an alias of []*CIDR to provide sort.Interface. +type CIDRList []*CIDR + +// Len implements sort.Interface. +func (l *CIDRList) Len() int { + return len(*l) +} + +// Less implements sort.Interface. +func (l *CIDRList) Less(i int, j int) bool { + ci := (*l)[i] + cj := (*l)[j] + + if len(ci.Ip) < len(cj.Ip) { + return true + } + + if len(ci.Ip) > len(cj.Ip) { + return false + } + + for k := 0; k < len(ci.Ip); k++ { + if ci.Ip[k] < cj.Ip[k] { + return true + } + + if ci.Ip[k] > cj.Ip[k] { + return false + } + } + + return ci.Prefix < cj.Prefix +} + +// Swap implements sort.Interface. +func (l *CIDRList) Swap(i int, j int) { + (*l)[i], (*l)[j] = (*l)[j], (*l)[i] +} + +type ipv6 struct { + a uint64 + b uint64 +} + +type GeoIPMatcher struct { + countryCode string + reverseMatch bool + ip4 []uint32 + prefix4 []uint8 + ip6 []ipv6 + prefix6 []uint8 +} + +func normalize4(ip uint32, prefix uint8) uint32 { + return (ip >> (32 - prefix)) << (32 - prefix) +} + +func normalize6(ip ipv6, prefix uint8) ipv6 { + if prefix <= 64 { + ip.a = (ip.a >> (64 - prefix)) << (64 - prefix) + ip.b = 0 + } else { + ip.b = (ip.b >> (128 - prefix)) << (128 - prefix) + } + return ip +} + +func (m *GeoIPMatcher) Init(cidrs []*CIDR) error { + ip4Count := 0 + ip6Count := 0 + + for _, cidr := range cidrs { + ip := cidr.Ip + switch len(ip) { + case 4: + ip4Count++ + case 16: + ip6Count++ + default: + return fmt.Errorf("unexpect ip length: %d", len(ip)) + } + } + + cidrList := CIDRList(cidrs) + sort.Sort(&cidrList) + + m.ip4 = make([]uint32, 0, ip4Count) + m.prefix4 = make([]uint8, 0, ip4Count) + m.ip6 = make([]ipv6, 0, ip6Count) + m.prefix6 = make([]uint8, 0, ip6Count) + + for _, cidr := range cidrs { + ip := cidr.Ip + prefix := uint8(cidr.Prefix) + switch len(ip) { + case 4: + m.ip4 = append(m.ip4, normalize4(binary.BigEndian.Uint32(ip), prefix)) + m.prefix4 = append(m.prefix4, prefix) + case 16: + ip6 := ipv6{ + a: binary.BigEndian.Uint64(ip[0:8]), + b: binary.BigEndian.Uint64(ip[8:16]), + } + ip6 = normalize6(ip6, prefix) + + m.ip6 = append(m.ip6, ip6) + m.prefix6 = append(m.prefix6, prefix) + } + } + + return nil +} + +func (m *GeoIPMatcher) SetReverseMatch(isReverseMatch bool) { + m.reverseMatch = isReverseMatch +} + +func (m *GeoIPMatcher) match4(ip uint32) bool { + if len(m.ip4) == 0 { + return false + } + + if ip < m.ip4[0] { + return false + } + + size := uint32(len(m.ip4)) + l := uint32(0) + r := size + for l < r { + x := ((l + r) >> 1) + if ip < m.ip4[x] { + r = x + continue + } + + nip := normalize4(ip, m.prefix4[x]) + if nip == m.ip4[x] { + return true + } + + l = x + 1 + } + + return l > 0 && normalize4(ip, m.prefix4[l-1]) == m.ip4[l-1] +} + +func less6(a ipv6, b ipv6) bool { + return a.a < b.a || (a.a == b.a && a.b < b.b) +} + +func (m *GeoIPMatcher) match6(ip ipv6) bool { + if len(m.ip6) == 0 { + return false + } + + if less6(ip, m.ip6[0]) { + return false + } + + size := uint32(len(m.ip6)) + l := uint32(0) + r := size + for l < r { + x := (l + r) / 2 + if less6(ip, m.ip6[x]) { + r = x + continue + } + + if normalize6(ip, m.prefix6[x]) == m.ip6[x] { + return true + } + + l = x + 1 + } + + return l > 0 && normalize6(ip, m.prefix6[l-1]) == m.ip6[l-1] +} + +// Match returns true if the given ip is included by the GeoIP. +func (m *GeoIPMatcher) Match(ip net.IP) bool { + switch len(ip) { + case 4: + if m.reverseMatch { + return !m.match4(binary.BigEndian.Uint32(ip)) + } + return m.match4(binary.BigEndian.Uint32(ip)) + case 16: + if m.reverseMatch { + return !m.match6(ipv6{ + a: binary.BigEndian.Uint64(ip[0:8]), + b: binary.BigEndian.Uint64(ip[8:16]), + }) + } + return m.match6(ipv6{ + a: binary.BigEndian.Uint64(ip[0:8]), + b: binary.BigEndian.Uint64(ip[8:16]), + }) + default: + return false + } +} + +// GeoIPMatcherContainer is a container for GeoIPMatchers. It keeps unique copies of GeoIPMatcher by country code. +type GeoIPMatcherContainer struct { + matchers []*GeoIPMatcher +} + +// Add adds a new GeoIP set into the container. +// If the country code of GeoIP is not empty, GeoIPMatcherContainer will try to find an existing one, instead of adding a new one. +func (c *GeoIPMatcherContainer) Add(geoip *GeoIP) (*GeoIPMatcher, error) { + if len(geoip.CountryCode) > 0 { + for _, m := range c.matchers { + if m.countryCode == geoip.CountryCode && m.reverseMatch == geoip.ReverseMatch { + return m, nil + } + } + } + + m := &GeoIPMatcher{ + countryCode: geoip.CountryCode, + reverseMatch: geoip.ReverseMatch, + } + if err := m.Init(geoip.Cidr); err != nil { + return nil, err + } + if len(geoip.CountryCode) > 0 { + c.matchers = append(c.matchers, m) + } + return m, nil +} + +var globalGeoIPContainer GeoIPMatcherContainer diff --git a/component/mmdb/mmdb.go b/component/mmdb/mmdb.go deleted file mode 100644 index e120055d..00000000 --- a/component/mmdb/mmdb.go +++ /dev/null @@ -1,45 +0,0 @@ -package mmdb - -import ( - "sync" - - C "github.com/Dreamacro/clash/constant" - "github.com/Dreamacro/clash/log" - - "github.com/oschwald/geoip2-golang" -) - -var ( - mmdb *geoip2.Reader - once sync.Once -) - -func LoadFromBytes(buffer []byte) { - once.Do(func() { - var err error - mmdb, err = geoip2.FromBytes(buffer) - if err != nil { - log.Fatalln("Can't load mmdb: %s", err.Error()) - } - }) -} - -func Verify() bool { - instance, err := geoip2.Open(C.Path.MMDB()) - if err == nil { - instance.Close() - } - return err == nil -} - -func Instance() *geoip2.Reader { - once.Do(func() { - var err error - mmdb, err = geoip2.Open(C.Path.MMDB()) - if err != nil { - log.Fatalln("Can't load mmdb: %s", err.Error()) - } - }) - - return mmdb -} diff --git a/config/initial.go b/config/initial.go index e9869e7e..5363d3ba 100644 --- a/config/initial.go +++ b/config/initial.go @@ -6,13 +6,12 @@ import ( "net/http" "os" - "github.com/Dreamacro/clash/component/mmdb" C "github.com/Dreamacro/clash/constant" "github.com/Dreamacro/clash/log" ) -func downloadMMDB(path string) (err error) { - resp, err := http.Get("https://cdn.jsdelivr.net/gh/Loyalsoldier/geoip@release/Country.mmdb") +func downloadGeoIP(path string) (err error) { + resp, err := http.Get("https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@release/geoip.dat") if err != nil { return } @@ -28,28 +27,6 @@ func downloadMMDB(path string) (err error) { return err } -func initMMDB() error { - if _, err := os.Stat(C.Path.MMDB()); os.IsNotExist(err) { - log.Infoln("Can't find MMDB, start download") - if err := downloadMMDB(C.Path.MMDB()); err != nil { - return fmt.Errorf("can't download MMDB: %s", err.Error()) - } - } - - if !mmdb.Verify() { - log.Warnln("MMDB invalid, remove and download") - if err := os.Remove(C.Path.MMDB()); err != nil { - return fmt.Errorf("can't remove invalid MMDB: %s", err.Error()) - } - - if err := downloadMMDB(C.Path.MMDB()); err != nil { - return fmt.Errorf("can't download MMDB: %s", err.Error()) - } - } - - return nil -} - func downloadGeoSite(path string) (err error) { resp, err := http.Get("https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@release/geosite.dat") if err != nil { @@ -79,6 +56,18 @@ func initGeoSite() error { return nil } +func initGeoIP() error { + if _, err := os.Stat(C.Path.GeoIP()); os.IsNotExist(err) { + log.Infoln("Need GeoIP but can't find GeoIP.dat, start download") + if err := downloadGeoIP(C.Path.GeoIP()); err != nil { + return fmt.Errorf("can't download GeoIP.dat: %s", err.Error()) + } + log.Infoln("Download GeoIP.dat finish") + } + + return nil +} + // Init prepare necessary files func Init(dir string) error { // initial homedir @@ -99,14 +88,9 @@ func Init(dir string) error { f.Close() } - //// initial GeoIP - //if err := initGeoIP(); err != nil { - // return fmt.Errorf("can't initial GeoIP: %w", err) - //} - - // initial mmdb - if err := initMMDB(); err != nil { - return fmt.Errorf("can't initial MMDB: %w", err) + // initial GeoIP + if err := initGeoIP(); err != nil { + return fmt.Errorf("can't initial GeoIP: %w", err) } return nil diff --git a/constant/path.go b/constant/path.go index f0f4e8c3..1461e85b 100644 --- a/constant/path.go +++ b/constant/path.go @@ -65,11 +65,11 @@ func (p *path) Cache() string { } func (p *path) GeoIP() string { - return P.Join(p.homeDir, "geoip.dat") + return P.Join(p.homeDir, "GeoIP.dat") } func (p *path) GeoSite() string { - return P.Join(p.homeDir, "geosite.dat") + return P.Join(p.homeDir, "GeoSite.dat") } func (p *path) ScriptDir() string { diff --git a/dns/filters.go b/dns/filters.go index 9c1be0d0..cc814ebd 100644 --- a/dns/filters.go +++ b/dns/filters.go @@ -1,13 +1,11 @@ package dns import ( - "net" - "strings" - + "github.com/Dreamacro/clash/component/geodata" "github.com/Dreamacro/clash/component/geodata/router" - "github.com/Dreamacro/clash/component/mmdb" "github.com/Dreamacro/clash/component/trie" - C "github.com/Dreamacro/clash/constant" + "github.com/Dreamacro/clash/log" + "net" ) type fallbackIPFilter interface { @@ -18,9 +16,38 @@ type geoipFilter struct { code string } +var geoIPMatcher *router.GeoIPMatcher + func (gf *geoipFilter) Match(ip net.IP) bool { - record, _ := mmdb.Instance().Country(ip) - return !strings.EqualFold(record.Country.IsoCode, gf.code) && !ip.IsPrivate() && !ip.Equal(C.TunBroadcastAddr) + if geoIPMatcher == nil { + countryCode := "cn" + geoLoader, err := geodata.GetGeoDataLoader("standard") + if err != nil { + log.Errorln("[GeoIPFilter] GetGeoDataLoader error: %s", err.Error()) + return false + } + + records, err := geoLoader.LoadGeoIP(countryCode) + if err != nil { + log.Errorln("[GeoIPFilter] LoadGeoIP error: %s", err.Error()) + return false + } + + geoIP := &router.GeoIP{ + CountryCode: countryCode, + Cidr: records, + ReverseMatch: false, + } + + geoIPMatcher, err = router.NewGeoIPMatcher(geoIP) + + if err != nil { + log.Errorln("[GeoIPFilter] NewGeoIPMatcher error: %s", err.Error()) + return false + } + } + + return !geoIPMatcher.Match(ip) } type ipnetFilter struct { diff --git a/rule/common/geoip.go b/rule/common/geoip.go index 1485a335..26205cc4 100644 --- a/rule/common/geoip.go +++ b/rule/common/geoip.go @@ -1,17 +1,21 @@ package common import ( + "fmt" + "github.com/Dreamacro/clash/component/geodata" + "github.com/Dreamacro/clash/component/geodata/router" "strings" - "github.com/Dreamacro/clash/component/mmdb" C "github.com/Dreamacro/clash/constant" + "github.com/Dreamacro/clash/log" ) type GEOIP struct { - country string - adapter string - noResolveIP bool - ruleExtra *C.RuleExtra + country string + adapter string + noResolveIP bool + ruleExtra *C.RuleExtra + geoIPMatcher *router.GeoIPMatcher } func (g *GEOIP) RuleType() C.RuleType { @@ -27,8 +31,8 @@ func (g *GEOIP) Match(metadata *C.Metadata) bool { if strings.EqualFold(g.country, "LAN") || C.TunBroadcastAddr.Equal(ip) { return ip.IsPrivate() } - record, _ := mmdb.Instance().Country(ip) - return strings.EqualFold(record.Country.IsoCode, g.country) + + return g.geoIPMatcher.Match(ip) } func (g *GEOIP) Adapter() string { @@ -52,11 +56,38 @@ func (g *GEOIP) GetCountry() string { } func NewGEOIP(country string, adapter string, noResolveIP bool, ruleExtra *C.RuleExtra) (*GEOIP, error) { + + geoLoaderName := "standard" + //geoLoaderName := "memconservative" + geoLoader, err := geodata.GetGeoDataLoader(geoLoaderName) + if err != nil { + return nil, fmt.Errorf("[GeoIP] %s", err.Error()) + } + + records, err := geoLoader.LoadGeoIP(strings.ReplaceAll(country, "!", "")) + if err != nil { + return nil, fmt.Errorf("[GeoIP] %s", err.Error()) + } + + geoIP := &router.GeoIP{ + CountryCode: country, + Cidr: records, + ReverseMatch: strings.Contains(country, "!"), + } + + geoIPMatcher, err := router.NewGeoIPMatcher(geoIP) + + if err != nil { + return nil, fmt.Errorf("[GeoIP] %s", err.Error()) + } + + log.Infoln("Start initial GeoIP rule %s => %s, records: %d", country, adapter, len(records)) geoip := &GEOIP{ - country: country, - adapter: adapter, - noResolveIP: noResolveIP, - ruleExtra: ruleExtra, + country: country, + adapter: adapter, + noResolveIP: noResolveIP, + ruleExtra: ruleExtra, + geoIPMatcher: geoIPMatcher, } return geoip, nil