xmengnet/mihomo
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[chore] Replace Country.mmdb with GeoIP.dat

Browse source
This commit is contained in:
gVisor bot 2022-01-26 12:01:14 +08:00
parent eb15f2c5f1
commit 09ffc53d95
9 changed files with 383 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Makefile
View file

@ -20,6 +20,7 @@ PLATFORM_LIST = \
darwin-amd64 \
darwin-arm64 \
linux-amd64 \
linux-amd64-AutoIptables\
linux-armv5 \
linux-armv6 \
linux-armv7 \

4
component/geodata/geodata.go
View file

@ -14,7 +14,7 @@ type loader struct {
}
func (l *loader) LoadGeoSite(list string) ([]*router.Domain, error) {
return l.LoadGeoSiteWithAttr("geosite.dat", list)
return l.LoadGeoSiteWithAttr("GeoSite.dat", list)
}
func (l *loader) LoadGeoSiteWithAttr(file string, siteWithAttr string) ([]*router.Domain, error) {
@ -58,7 +58,7 @@ func (l *loader) LoadGeoSiteWithAttr(file string, siteWithAttr string) ([]*route
}
func (l *loader) LoadGeoIP(country string) ([]*router.CIDR, error) {
return l.LoadIP("geoip.dat", country)
return l.LoadIP("GeoIP.dat", country)
}
var loaders map[string]func() LoaderImplementation

42
component/geodata/router/condition.go
View file

@ -2,6 +2,7 @@ package router
import (
"fmt"
"net"
"strings"
"github.com/Dreamacro/clash/component/geodata/strmatcher"
@ -69,3 +70,44 @@ func NewDomainMatcher(domains []*Domain) (*DomainMatcher, error) {
func (m *DomainMatcher) ApplyDomain(domain string) bool {
return len(m.matchers.Match(strings.ToLower(domain))) > 0
}
type MultiGeoIPMatcher struct {
matchers []*GeoIPMatcher
}
func NewGeoIPMatcher(geoip *GeoIP) (*GeoIPMatcher, error) {
matcher, err := globalGeoIPContainer.Add(geoip)
if err != nil {
return nil, err
}
return matcher, nil
}
func (m *MultiGeoIPMatcher) ApplyIp(ip net.IP) bool {
for _, matcher := range m.matchers {
if matcher.Match(ip) {
return true
}
}
return false
}
func NewMultiGeoIPMatcher(geoips []*GeoIP) (*MultiGeoIPMatcher, error) {
var matchers []*GeoIPMatcher
for _, geoip := range geoips {
matcher, err := globalGeoIPContainer.Add(geoip)
if err != nil {
return nil, err
}
matchers = append(matchers, matcher)
}
matcher := &MultiGeoIPMatcher{
matchers: matchers,
}
return matcher, nil
}

243
component/geodata/router/condition_geoip.go Normal file
View file

@ -0,0 +1,243 @@
package router
import (
"encoding/binary"
"fmt"
"net"
"sort"
)
// CIDRList is an alias of []*CIDR to provide sort.Interface.
type CIDRList []*CIDR
// Len implements sort.Interface.
func (l *CIDRList) Len() int {
return len(*l)
}
// Less implements sort.Interface.
func (l *CIDRList) Less(i int, j int) bool {
ci := (*l)[i]
cj := (*l)[j]
if len(ci.Ip) < len(cj.Ip) {
return true
}
if len(ci.Ip) > len(cj.Ip) {
return false
}
for k := 0; k < len(ci.Ip); k++ {
if ci.Ip[k] < cj.Ip[k] {
return true
}
if ci.Ip[k] > cj.Ip[k] {
return false
}
}
return ci.Prefix < cj.Prefix
}
// Swap implements sort.Interface.
func (l *CIDRList) Swap(i int, j int) {
(*l)[i], (*l)[j] = (*l)[j], (*l)[i]
}
type ipv6 struct {
a uint64
b uint64
}
type GeoIPMatcher struct {
countryCode string
reverseMatch bool
ip4 []uint32
prefix4 []uint8
ip6 []ipv6
prefix6 []uint8
}
func normalize4(ip uint32, prefix uint8) uint32 {
return (ip >> (32 - prefix)) << (32 - prefix)
}
func normalize6(ip ipv6, prefix uint8) ipv6 {
if prefix <= 64 {
ip.a = (ip.a >> (64 - prefix)) << (64 - prefix)
ip.b = 0
} else {
ip.b = (ip.b >> (128 - prefix)) << (128 - prefix)
}
return ip
}
func (m *GeoIPMatcher) Init(cidrs []*CIDR) error {
ip4Count := 0
ip6Count := 0
for _, cidr := range cidrs {
ip := cidr.Ip
switch len(ip) {
case 4:
ip4Count++
case 16:
ip6Count++
default:
return fmt.Errorf("unexpect ip length: %d", len(ip))
}
}
cidrList := CIDRList(cidrs)
sort.Sort(&cidrList)
m.ip4 = make([]uint32, 0, ip4Count)
m.prefix4 = make([]uint8, 0, ip4Count)
m.ip6 = make([]ipv6, 0, ip6Count)
m.prefix6 = make([]uint8, 0, ip6Count)
for _, cidr := range cidrs {
ip := cidr.Ip
prefix := uint8(cidr.Prefix)
switch len(ip) {
case 4:
m.ip4 = append(m.ip4, normalize4(binary.BigEndian.Uint32(ip), prefix))
m.prefix4 = append(m.prefix4, prefix)
case 16:
ip6 := ipv6{
a: binary.BigEndian.Uint64(ip[0:8]),
b: binary.BigEndian.Uint64(ip[8:16]),
}
ip6 = normalize6(ip6, prefix)
m.ip6 = append(m.ip6, ip6)
m.prefix6 = append(m.prefix6, prefix)
}
}
return nil
}
func (m *GeoIPMatcher) SetReverseMatch(isReverseMatch bool) {
m.reverseMatch = isReverseMatch
}
func (m *GeoIPMatcher) match4(ip uint32) bool {
if len(m.ip4) == 0 {
return false
}
if ip < m.ip4[0] {
return false
}
size := uint32(len(m.ip4))
l := uint32(0)
r := size
for l < r {
x := ((l + r) >> 1)
if ip < m.ip4[x] {
r = x
continue
}
nip := normalize4(ip, m.prefix4[x])
if nip == m.ip4[x] {
return true
}
l = x + 1
}
return l > 0 && normalize4(ip, m.prefix4[l-1]) == m.ip4[l-1]
}
func less6(a ipv6, b ipv6) bool {
return a.a < b.a || (a.a == b.a && a.b < b.b)
}
func (m *GeoIPMatcher) match6(ip ipv6) bool {
if len(m.ip6) == 0 {
return false
}
if less6(ip, m.ip6[0]) {
return false
}
size := uint32(len(m.ip6))
l := uint32(0)
r := size
for l < r {
x := (l + r) / 2
if less6(ip, m.ip6[x]) {
r = x
continue
}
if normalize6(ip, m.prefix6[x]) == m.ip6[x] {
return true
}
l = x + 1
}
return l > 0 && normalize6(ip, m.prefix6[l-1]) == m.ip6[l-1]
}
// Match returns true if the given ip is included by the GeoIP.
func (m *GeoIPMatcher) Match(ip net.IP) bool {
switch len(ip) {
case 4:
if m.reverseMatch {
return !m.match4(binary.BigEndian.Uint32(ip))
}
return m.match4(binary.BigEndian.Uint32(ip))
case 16:
if m.reverseMatch {
return !m.match6(ipv6{
a: binary.BigEndian.Uint64(ip[0:8]),
b: binary.BigEndian.Uint64(ip[8:16]),
})
}
return m.match6(ipv6{
a: binary.BigEndian.Uint64(ip[0:8]),
b: binary.BigEndian.Uint64(ip[8:16]),
})
default:
return false
}
}
// GeoIPMatcherContainer is a container for GeoIPMatchers. It keeps unique copies of GeoIPMatcher by country code.
type GeoIPMatcherContainer struct {
matchers []*GeoIPMatcher
}
// Add adds a new GeoIP set into the container.
// If the country code of GeoIP is not empty, GeoIPMatcherContainer will try to find an existing one, instead of adding a new one.
func (c *GeoIPMatcherContainer) Add(geoip *GeoIP) (*GeoIPMatcher, error) {
if len(geoip.CountryCode) > 0 {
for _, m := range c.matchers {
if m.countryCode == geoip.CountryCode && m.reverseMatch == geoip.ReverseMatch {
return m, nil
}
}
}
m := &GeoIPMatcher{
countryCode: geoip.CountryCode,
reverseMatch: geoip.ReverseMatch,
}
if err := m.Init(geoip.Cidr); err != nil {
return nil, err
}
if len(geoip.CountryCode) > 0 {
c.matchers = append(c.matchers, m)
}
return m, nil
}
var globalGeoIPContainer GeoIPMatcherContainer

45
component/mmdb/mmdb.go
View file

@ -1,45 +0,0 @@
package mmdb
import (
"sync"
C "github.com/Dreamacro/clash/constant"
"github.com/Dreamacro/clash/log"
"github.com/oschwald/geoip2-golang"
)
var (
mmdb *geoip2.Reader
once sync.Once
)
func LoadFromBytes(buffer []byte) {
once.Do(func() {
var err error
mmdb, err = geoip2.FromBytes(buffer)
if err != nil {
log.Fatalln("Can't load mmdb: %s", err.Error())
}
})
}
func Verify() bool {
instance, err := geoip2.Open(C.Path.MMDB())
if err == nil {
instance.Close()
}
return err == nil
}
func Instance() *geoip2.Reader {
once.Do(func() {
var err error
mmdb, err = geoip2.Open(C.Path.MMDB())
if err != nil {
log.Fatalln("Can't load mmdb: %s", err.Error())
}
})
return mmdb
}

50
config/initial.go
View file

@ -6,13 +6,12 @@ import (
"net/http"
"os"
"github.com/Dreamacro/clash/component/mmdb"
C "github.com/Dreamacro/clash/constant"
"github.com/Dreamacro/clash/log"
)
func downloadMMDB(path string) (err error) {
resp, err := http.Get("https://cdn.jsdelivr.net/gh/Loyalsoldier/geoip@release/Country.mmdb")
func downloadGeoIP(path string) (err error) {
resp, err := http.Get("https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@release/geoip.dat")
if err != nil {
return
}
@ -28,28 +27,6 @@ func downloadMMDB(path string) (err error) {
return err
}
func initMMDB() error {
if _, err := os.Stat(C.Path.MMDB()); os.IsNotExist(err) {
log.Infoln("Can't find MMDB, start download")
if err := downloadMMDB(C.Path.MMDB()); err != nil {
return fmt.Errorf("can't download MMDB: %s", err.Error())
}
}
if !mmdb.Verify() {
log.Warnln("MMDB invalid, remove and download")
if err := os.Remove(C.Path.MMDB()); err != nil {
return fmt.Errorf("can't remove invalid MMDB: %s", err.Error())
}
if err := downloadMMDB(C.Path.MMDB()); err != nil {
return fmt.Errorf("can't download MMDB: %s", err.Error())
}
}
return nil
}
func downloadGeoSite(path string) (err error) {
resp, err := http.Get("https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@release/geosite.dat")
if err != nil {
@ -79,6 +56,18 @@ func initGeoSite() error {
return nil
}
func initGeoIP() error {
if _, err := os.Stat(C.Path.GeoIP()); os.IsNotExist(err) {
log.Infoln("Need GeoIP but can't find GeoIP.dat, start download")
if err := downloadGeoIP(C.Path.GeoIP()); err != nil {
return fmt.Errorf("can't download GeoIP.dat: %s", err.Error())
}
log.Infoln("Download GeoIP.dat finish")
}
return nil
}
// Init prepare necessary files
func Init(dir string) error {
// initial homedir
@ -99,14 +88,9 @@ func Init(dir string) error {
f.Close()
}
//// initial GeoIP
//if err := initGeoIP(); err != nil {
// return fmt.Errorf("can't initial GeoIP: %w", err)
//}
// initial mmdb
if err := initMMDB(); err != nil {
return fmt.Errorf("can't initial MMDB: %w", err)
// initial GeoIP
if err := initGeoIP(); err != nil {
return fmt.Errorf("can't initial GeoIP: %w", err)
}
return nil

4
constant/path.go
View file

@ -65,11 +65,11 @@ func (p *path) Cache() string {
}
func (p *path) GeoIP() string {
return P.Join(p.homeDir, "geoip.dat")
return P.Join(p.homeDir, "GeoIP.dat")
}
func (p *path) GeoSite() string {
return P.Join(p.homeDir, "geosite.dat")
return P.Join(p.homeDir, "GeoSite.dat")
}
func (p *path) ScriptDir() string {

41
dns/filters.go
View file

@ -1,13 +1,11 @@
package dns
import (
"net"
"strings"
"github.com/Dreamacro/clash/component/geodata"
"github.com/Dreamacro/clash/component/geodata/router"
"github.com/Dreamacro/clash/component/mmdb"
"github.com/Dreamacro/clash/component/trie"
C "github.com/Dreamacro/clash/constant"
"github.com/Dreamacro/clash/log"
"net"
)
type fallbackIPFilter interface {
@ -18,9 +16,38 @@ type geoipFilter struct {
code string
}
var geoIPMatcher *router.GeoIPMatcher
func (gf *geoipFilter) Match(ip net.IP) bool {
record, _ := mmdb.Instance().Country(ip)
return !strings.EqualFold(record.Country.IsoCode, gf.code) && !ip.IsPrivate() && !ip.Equal(C.TunBroadcastAddr)
if geoIPMatcher == nil {
countryCode := "cn"
geoLoader, err := geodata.GetGeoDataLoader("standard")
if err != nil {
log.Errorln("[GeoIPFilter] GetGeoDataLoader error: %s", err.Error())
return false
}
records, err := geoLoader.LoadGeoIP(countryCode)
if err != nil {
log.Errorln("[GeoIPFilter] LoadGeoIP error: %s", err.Error())
return false
}
geoIP := &router.GeoIP{
CountryCode: countryCode,
Cidr: records,
ReverseMatch: false,
}
geoIPMatcher, err = router.NewGeoIPMatcher(geoIP)
if err != nil {
log.Errorln("[GeoIPFilter] NewGeoIPMatcher error: %s", err.Error())
return false
}
}
return !geoIPMatcher.Match(ip)
}
type ipnetFilter struct {

37
rule/common/geoip.go
View file

@ -1,10 +1,13 @@
package common
import (
"fmt"
"github.com/Dreamacro/clash/component/geodata"
"github.com/Dreamacro/clash/component/geodata/router"
"strings"
"github.com/Dreamacro/clash/component/mmdb"
C "github.com/Dreamacro/clash/constant"
"github.com/Dreamacro/clash/log"
)
type GEOIP struct {
@ -12,6 +15,7 @@ type GEOIP struct {
adapter string
noResolveIP bool
ruleExtra *C.RuleExtra
geoIPMatcher *router.GeoIPMatcher
}
func (g *GEOIP) RuleType() C.RuleType {
@ -27,8 +31,8 @@ func (g *GEOIP) Match(metadata *C.Metadata) bool {
if strings.EqualFold(g.country, "LAN") || C.TunBroadcastAddr.Equal(ip) {
return ip.IsPrivate()
}
record, _ := mmdb.Instance().Country(ip)
return strings.EqualFold(record.Country.IsoCode, g.country)
return g.geoIPMatcher.Match(ip)
}
func (g *GEOIP) Adapter() string {
@ -52,11 +56,38 @@ func (g *GEOIP) GetCountry() string {
}
func NewGEOIP(country string, adapter string, noResolveIP bool, ruleExtra *C.RuleExtra) (*GEOIP, error) {
geoLoaderName := "standard"
//geoLoaderName := "memconservative"
geoLoader, err := geodata.GetGeoDataLoader(geoLoaderName)
if err != nil {
return nil, fmt.Errorf("[GeoIP] %s", err.Error())
}
records, err := geoLoader.LoadGeoIP(strings.ReplaceAll(country, "!", ""))
if err != nil {
return nil, fmt.Errorf("[GeoIP] %s", err.Error())
}
geoIP := &router.GeoIP{
CountryCode: country,
Cidr: records,
ReverseMatch: strings.Contains(country, "!"),
}
geoIPMatcher, err := router.NewGeoIPMatcher(geoIP)
if err != nil {
return nil, fmt.Errorf("[GeoIP] %s", err.Error())
}
log.Infoln("Start initial GeoIP rule %s => %s, records: %d", country, adapter, len(records))
geoip := &GEOIP{
country: country,
adapter: adapter,
noResolveIP: noResolveIP,
ruleExtra: ruleExtra,
geoIPMatcher: geoIPMatcher,
}
return geoip, nil