[chore] Replace Country.mmdb with GeoIP.dat

Makefile

@@ -20,6 +20,7 @@ PLATFORM_LIST = \
 	darwin-amd64 \
 	darwin-arm64 \
 	linux-amd64 \
+	linux-amd64-AutoIptables\
 	linux-armv5 \
 	linux-armv6 \
 	linux-armv7 \

component/geodata/geodata.go

@@ -14,7 +14,7 @@ type loader struct {
 }
 
 func (l *loader) LoadGeoSite(list string) ([]*router.Domain, error) {
-	return l.LoadGeoSiteWithAttr("geosite.dat", list)
+	return l.LoadGeoSiteWithAttr("GeoSite.dat", list)
 }
 
 func (l *loader) LoadGeoSiteWithAttr(file string, siteWithAttr string) ([]*router.Domain, error) {
@@ -58,7 +58,7 @@ func (l *loader) LoadGeoSiteWithAttr(file string, siteWithAttr string) ([]*route
 }
 
 func (l *loader) LoadGeoIP(country string) ([]*router.CIDR, error) {
-	return l.LoadIP("geoip.dat", country)
+	return l.LoadIP("GeoIP.dat", country)
 }
 
 var loaders map[string]func() LoaderImplementation

component/geodata/router/condition.go

@@ -2,6 +2,7 @@ package router
 
 import (
 	"fmt"
+	"net"
 	"strings"
 
 	"github.com/Dreamacro/clash/component/geodata/strmatcher"
@@ -69,3 +70,44 @@ func NewDomainMatcher(domains []*Domain) (*DomainMatcher, error) {
 func (m *DomainMatcher) ApplyDomain(domain string) bool {
 	return len(m.matchers.Match(strings.ToLower(domain))) > 0
 }
+
+type MultiGeoIPMatcher struct {
+	matchers []*GeoIPMatcher
+}
+
+func NewGeoIPMatcher(geoip *GeoIP) (*GeoIPMatcher, error) {
+	matcher, err := globalGeoIPContainer.Add(geoip)
+	if err != nil {
+		return nil, err
+	}
+
+	return matcher, nil
+}
+
+func (m *MultiGeoIPMatcher) ApplyIp(ip net.IP) bool {
+
+	for _, matcher := range m.matchers {
+		if matcher.Match(ip) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func NewMultiGeoIPMatcher(geoips []*GeoIP) (*MultiGeoIPMatcher, error) {
+	var matchers []*GeoIPMatcher
+	for _, geoip := range geoips {
+		matcher, err := globalGeoIPContainer.Add(geoip)
+		if err != nil {
+			return nil, err
+		}
+		matchers = append(matchers, matcher)
+	}
+
+	matcher := &MultiGeoIPMatcher{
+		matchers: matchers,
+	}
+
+	return matcher, nil
+}

component/geodata/router/condition_geoip.go

@@ -0,0 +1,243 @@
+package router
+
+import (
+	"encoding/binary"
+	"fmt"
+	"net"
+	"sort"
+)
+
+// CIDRList is an alias of []*CIDR to provide sort.Interface.
+type CIDRList []*CIDR
+
+// Len implements sort.Interface.
+func (l *CIDRList) Len() int {
+	return len(*l)
+}
+
+// Less implements sort.Interface.
+func (l *CIDRList) Less(i int, j int) bool {
+	ci := (*l)[i]
+	cj := (*l)[j]
+
+	if len(ci.Ip) < len(cj.Ip) {
+		return true
+	}
+
+	if len(ci.Ip) > len(cj.Ip) {
+		return false
+	}
+
+	for k := 0; k < len(ci.Ip); k++ {
+		if ci.Ip[k] < cj.Ip[k] {
+			return true
+		}
+
+		if ci.Ip[k] > cj.Ip[k] {
+			return false
+		}
+	}
+
+	return ci.Prefix < cj.Prefix
+}
+
+// Swap implements sort.Interface.
+func (l *CIDRList) Swap(i int, j int) {
+	(*l)[i], (*l)[j] = (*l)[j], (*l)[i]
+}
+
+type ipv6 struct {
+	a uint64
+	b uint64
+}
+
+type GeoIPMatcher struct {
+	countryCode  string
+	reverseMatch bool
+	ip4          []uint32
+	prefix4      []uint8
+	ip6          []ipv6
+	prefix6      []uint8
+}
+
+func normalize4(ip uint32, prefix uint8) uint32 {
+	return (ip >> (32 - prefix)) << (32 - prefix)
+}
+
+func normalize6(ip ipv6, prefix uint8) ipv6 {
+	if prefix <= 64 {
+		ip.a = (ip.a >> (64 - prefix)) << (64 - prefix)
+		ip.b = 0
+	} else {
+		ip.b = (ip.b >> (128 - prefix)) << (128 - prefix)
+	}
+	return ip
+}
+
+func (m *GeoIPMatcher) Init(cidrs []*CIDR) error {
+	ip4Count := 0
+	ip6Count := 0
+
+	for _, cidr := range cidrs {
+		ip := cidr.Ip
+		switch len(ip) {
+		case 4:
+			ip4Count++
+		case 16:
+			ip6Count++
+		default:
+			return fmt.Errorf("unexpect ip length: %d", len(ip))
+		}
+	}
+
+	cidrList := CIDRList(cidrs)
+	sort.Sort(&cidrList)
+
+	m.ip4 = make([]uint32, 0, ip4Count)
+	m.prefix4 = make([]uint8, 0, ip4Count)
+	m.ip6 = make([]ipv6, 0, ip6Count)
+	m.prefix6 = make([]uint8, 0, ip6Count)
+
+	for _, cidr := range cidrs {
+		ip := cidr.Ip
+		prefix := uint8(cidr.Prefix)
+		switch len(ip) {
+		case 4:
+			m.ip4 = append(m.ip4, normalize4(binary.BigEndian.Uint32(ip), prefix))
+			m.prefix4 = append(m.prefix4, prefix)
+		case 16:
+			ip6 := ipv6{
+				a: binary.BigEndian.Uint64(ip[0:8]),
+				b: binary.BigEndian.Uint64(ip[8:16]),
+			}
+			ip6 = normalize6(ip6, prefix)
+
+			m.ip6 = append(m.ip6, ip6)
+			m.prefix6 = append(m.prefix6, prefix)
+		}
+	}
+
+	return nil
+}
+
+func (m *GeoIPMatcher) SetReverseMatch(isReverseMatch bool) {
+	m.reverseMatch = isReverseMatch
+}
+
+func (m *GeoIPMatcher) match4(ip uint32) bool {
+	if len(m.ip4) == 0 {
+		return false
+	}
+
+	if ip < m.ip4[0] {
+		return false
+	}
+
+	size := uint32(len(m.ip4))
+	l := uint32(0)
+	r := size
+	for l < r {
+		x := ((l + r) >> 1)
+		if ip < m.ip4[x] {
+			r = x
+			continue
+		}
+
+		nip := normalize4(ip, m.prefix4[x])
+		if nip == m.ip4[x] {
+			return true
+		}
+
+		l = x + 1
+	}
+
+	return l > 0 && normalize4(ip, m.prefix4[l-1]) == m.ip4[l-1]
+}
+
+func less6(a ipv6, b ipv6) bool {
+	return a.a < b.a || (a.a == b.a && a.b < b.b)
+}
+
+func (m *GeoIPMatcher) match6(ip ipv6) bool {
+	if len(m.ip6) == 0 {
+		return false
+	}
+
+	if less6(ip, m.ip6[0]) {
+		return false
+	}
+
+	size := uint32(len(m.ip6))
+	l := uint32(0)
+	r := size
+	for l < r {
+		x := (l + r) / 2
+		if less6(ip, m.ip6[x]) {
+			r = x
+			continue
+		}
+
+		if normalize6(ip, m.prefix6[x]) == m.ip6[x] {
+			return true
+		}
+
+		l = x + 1
+	}
+
+	return l > 0 && normalize6(ip, m.prefix6[l-1]) == m.ip6[l-1]
+}
+
+// Match returns true if the given ip is included by the GeoIP.
+func (m *GeoIPMatcher) Match(ip net.IP) bool {
+	switch len(ip) {
+	case 4:
+		if m.reverseMatch {
+			return !m.match4(binary.BigEndian.Uint32(ip))
+		}
+		return m.match4(binary.BigEndian.Uint32(ip))
+	case 16:
+		if m.reverseMatch {
+			return !m.match6(ipv6{
+				a: binary.BigEndian.Uint64(ip[0:8]),
+				b: binary.BigEndian.Uint64(ip[8:16]),
+			})
+		}
+		return m.match6(ipv6{
+			a: binary.BigEndian.Uint64(ip[0:8]),
+			b: binary.BigEndian.Uint64(ip[8:16]),
+		})
+	default:
+		return false
+	}
+}
+
+// GeoIPMatcherContainer is a container for GeoIPMatchers. It keeps unique copies of GeoIPMatcher by country code.
+type GeoIPMatcherContainer struct {
+	matchers []*GeoIPMatcher
+}
+
+// Add adds a new GeoIP set into the container.
+// If the country code of GeoIP is not empty, GeoIPMatcherContainer will try to find an existing one, instead of adding a new one.
+func (c *GeoIPMatcherContainer) Add(geoip *GeoIP) (*GeoIPMatcher, error) {
+	if len(geoip.CountryCode) > 0 {
+		for _, m := range c.matchers {
+			if m.countryCode == geoip.CountryCode && m.reverseMatch == geoip.ReverseMatch {
+				return m, nil
+			}
+		}
+	}
+
+	m := &GeoIPMatcher{
+		countryCode:  geoip.CountryCode,
+		reverseMatch: geoip.ReverseMatch,
+	}
+	if err := m.Init(geoip.Cidr); err != nil {
+		return nil, err
+	}
+	if len(geoip.CountryCode) > 0 {
+		c.matchers = append(c.matchers, m)
+	}
+	return m, nil
+}
+
+var globalGeoIPContainer GeoIPMatcherContainer

component/mmdb/mmdb.go

@@ -1,45 +0,0 @@
-package mmdb
-
-import (
-	"sync"
-
-	C "github.com/Dreamacro/clash/constant"
-	"github.com/Dreamacro/clash/log"
-
-	"github.com/oschwald/geoip2-golang"
-)
-
-var (
-	mmdb *geoip2.Reader
-	once sync.Once
-)
-
-func LoadFromBytes(buffer []byte) {
-	once.Do(func() {
-		var err error
-		mmdb, err = geoip2.FromBytes(buffer)
-		if err != nil {
-			log.Fatalln("Can't load mmdb: %s", err.Error())
-		}
-	})
-}
-
-func Verify() bool {
-	instance, err := geoip2.Open(C.Path.MMDB())
-	if err == nil {
-		instance.Close()
-	}
-	return err == nil
-}
-
-func Instance() *geoip2.Reader {
-	once.Do(func() {
-		var err error
-		mmdb, err = geoip2.Open(C.Path.MMDB())
-		if err != nil {
-			log.Fatalln("Can't load mmdb: %s", err.Error())
-		}
-	})
-
-	return mmdb
-}

config/initial.go

@@ -6,13 +6,12 @@ import (
 	"net/http"
 	"os"
 
-	"github.com/Dreamacro/clash/component/mmdb"
 	C "github.com/Dreamacro/clash/constant"
 	"github.com/Dreamacro/clash/log"
 )
 
-func downloadMMDB(path string) (err error) {
+func downloadGeoIP(path string) (err error) {
-	resp, err := http.Get("https://cdn.jsdelivr.net/gh/Loyalsoldier/geoip@release/Country.mmdb")
+	resp, err := http.Get("https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@release/geoip.dat")
 	if err != nil {
 		return
 	}
@@ -28,28 +27,6 @@ func downloadMMDB(path string) (err error) {
 	return err
 }
 
-func initMMDB() error {
-	if _, err := os.Stat(C.Path.MMDB()); os.IsNotExist(err) {
-		log.Infoln("Can't find MMDB, start download")
-		if err := downloadMMDB(C.Path.MMDB()); err != nil {
-			return fmt.Errorf("can't download MMDB: %s", err.Error())
-		}
-	}
-
-	if !mmdb.Verify() {
-		log.Warnln("MMDB invalid, remove and download")
-		if err := os.Remove(C.Path.MMDB()); err != nil {
-			return fmt.Errorf("can't remove invalid MMDB: %s", err.Error())
-		}
-
-		if err := downloadMMDB(C.Path.MMDB()); err != nil {
-			return fmt.Errorf("can't download MMDB: %s", err.Error())
-		}
-	}
-
-	return nil
-}
-
 func downloadGeoSite(path string) (err error) {
 	resp, err := http.Get("https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@release/geosite.dat")
 	if err != nil {
@@ -79,6 +56,18 @@ func initGeoSite() error {
 	return nil
 }
 
+func initGeoIP() error {
+	if _, err := os.Stat(C.Path.GeoIP()); os.IsNotExist(err) {
+		log.Infoln("Need GeoIP but can't find GeoIP.dat, start download")
+		if err := downloadGeoIP(C.Path.GeoIP()); err != nil {
+			return fmt.Errorf("can't download GeoIP.dat: %s", err.Error())
+		}
+		log.Infoln("Download GeoIP.dat finish")
+	}
+
+	return nil
+}
+
 // Init prepare necessary files
 func Init(dir string) error {
 	// initial homedir
@@ -99,14 +88,9 @@ func Init(dir string) error {
 		f.Close()
 	}
 
-	//// initial GeoIP
+	// initial GeoIP
-	//if err := initGeoIP(); err != nil {
+	if err := initGeoIP(); err != nil {
-	//	return fmt.Errorf("can't initial GeoIP: %w", err)
+		return fmt.Errorf("can't initial GeoIP: %w", err)
-	//}
-
-	// initial mmdb
-	if err := initMMDB(); err != nil {
-		return fmt.Errorf("can't initial MMDB: %w", err)
 	}
 
 	return nil

constant/path.go

@@ -65,11 +65,11 @@ func (p *path) Cache() string {
 }
 
 func (p *path) GeoIP() string {
-	return P.Join(p.homeDir, "geoip.dat")
+	return P.Join(p.homeDir, "GeoIP.dat")
 }
 
 func (p *path) GeoSite() string {
-	return P.Join(p.homeDir, "geosite.dat")
+	return P.Join(p.homeDir, "GeoSite.dat")
 }
 
 func (p *path) ScriptDir() string {

dns/filters.go

@@ -1,13 +1,11 @@
 package dns
 
 import (
-	"net"
+	"github.com/Dreamacro/clash/component/geodata"
-	"strings"
-
 	"github.com/Dreamacro/clash/component/geodata/router"
-	"github.com/Dreamacro/clash/component/mmdb"
 	"github.com/Dreamacro/clash/component/trie"
-	C "github.com/Dreamacro/clash/constant"
+	"github.com/Dreamacro/clash/log"
+	"net"
 )
 
 type fallbackIPFilter interface {
@@ -18,9 +16,38 @@ type geoipFilter struct {
 	code string
 }
 
+var geoIPMatcher *router.GeoIPMatcher
+
 func (gf *geoipFilter) Match(ip net.IP) bool {
-	record, _ := mmdb.Instance().Country(ip)
+	if geoIPMatcher == nil {
-	return !strings.EqualFold(record.Country.IsoCode, gf.code) && !ip.IsPrivate() && !ip.Equal(C.TunBroadcastAddr)
+		countryCode := "cn"
+		geoLoader, err := geodata.GetGeoDataLoader("standard")
+		if err != nil {
+			log.Errorln("[GeoIPFilter] GetGeoDataLoader error: %s", err.Error())
+			return false
+		}
+
+		records, err := geoLoader.LoadGeoIP(countryCode)
+		if err != nil {
+			log.Errorln("[GeoIPFilter] LoadGeoIP error: %s", err.Error())
+			return false
+		}
+
+		geoIP := &router.GeoIP{
+			CountryCode:  countryCode,
+			Cidr:         records,
+			ReverseMatch: false,
+		}
+
+		geoIPMatcher, err = router.NewGeoIPMatcher(geoIP)
+
+		if err != nil {
+			log.Errorln("[GeoIPFilter] NewGeoIPMatcher error: %s", err.Error())
+			return false
+		}
+	}
+
+	return !geoIPMatcher.Match(ip)
 }
 
 type ipnetFilter struct {

rule/common/geoip.go

@@ -1,10 +1,13 @@
 package common
 
 import (
+	"fmt"
+	"github.com/Dreamacro/clash/component/geodata"
+	"github.com/Dreamacro/clash/component/geodata/router"
 	"strings"
 
-	"github.com/Dreamacro/clash/component/mmdb"
 	C "github.com/Dreamacro/clash/constant"
+	"github.com/Dreamacro/clash/log"
 )
 
 type GEOIP struct {
@@ -12,6 +15,7 @@ type GEOIP struct {
 	adapter      string
 	noResolveIP  bool
 	ruleExtra    *C.RuleExtra
+	geoIPMatcher *router.GeoIPMatcher
 }
 
 func (g *GEOIP) RuleType() C.RuleType {
@@ -27,8 +31,8 @@ func (g *GEOIP) Match(metadata *C.Metadata) bool {
 	if strings.EqualFold(g.country, "LAN") || C.TunBroadcastAddr.Equal(ip) {
 		return ip.IsPrivate()
 	}
-	record, _ := mmdb.Instance().Country(ip)
+
-	return strings.EqualFold(record.Country.IsoCode, g.country)
+	return g.geoIPMatcher.Match(ip)
 }
 
 func (g *GEOIP) Adapter() string {
@@ -52,11 +56,38 @@ func (g *GEOIP) GetCountry() string {
 }
 
 func NewGEOIP(country string, adapter string, noResolveIP bool, ruleExtra *C.RuleExtra) (*GEOIP, error) {
+
+	geoLoaderName := "standard"
+	//geoLoaderName := "memconservative"
+	geoLoader, err := geodata.GetGeoDataLoader(geoLoaderName)
+	if err != nil {
+		return nil, fmt.Errorf("[GeoIP] %s", err.Error())
+	}
+
+	records, err := geoLoader.LoadGeoIP(strings.ReplaceAll(country, "!", ""))
+	if err != nil {
+		return nil, fmt.Errorf("[GeoIP] %s", err.Error())
+	}
+
+	geoIP := &router.GeoIP{
+		CountryCode:  country,
+		Cidr:         records,
+		ReverseMatch: strings.Contains(country, "!"),
+	}
+
+	geoIPMatcher, err := router.NewGeoIPMatcher(geoIP)
+
+	if err != nil {
+		return nil, fmt.Errorf("[GeoIP] %s", err.Error())
+	}
+
+	log.Infoln("Start initial GeoIP rule %s => %s, records: %d", country, adapter, len(records))
 	geoip := &GEOIP{
 		country:      country,
 		adapter:      adapter,
 		noResolveIP:  noResolveIP,
 		ruleExtra:    ruleExtra,
+		geoIPMatcher: geoIPMatcher,
 	}
 
 	return geoip, nil