Refactor: Optimize Memory Usage At Startup

https://github.com/XTLS/Xray-core/issues/68#issuecomment-745231528
This commit is contained in:
RPRX 2020-12-15 20:27:18 +08:00 committed by GitHub
parent 2e942e0303
commit 45f44c401a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 101 additions and 30 deletions

View File

@ -2,6 +2,7 @@ package conf
import (
"encoding/json"
"runtime"
"strconv"
"strings"
@ -147,46 +148,109 @@ func ParseIP(s string) (*router.CIDR, error) {
}
}
func loadGeoIP(country string) ([]*router.CIDR, error) {
return loadIP("geoip.dat", country)
func loadGeoIP(code string) ([]*router.CIDR, error) {
return loadIP("geoip.dat", code)
}
func loadIP(filename, country string) ([]*router.CIDR, error) {
geoipBytes, err := filesystem.ReadAsset(filename)
if err != nil {
return nil, newError("failed to open file: ", filename).Base(err)
}
var geoipList router.GeoIPList
if err := proto.Unmarshal(geoipBytes, &geoipList); err != nil {
return nil, err
}
var (
FileCache = make(map[string][]byte)
IPCache = make(map[string]*router.GeoIP)
SiteCache = make(map[string]*router.GeoSite)
)
for _, geoip := range geoipList.Entry {
if geoip.CountryCode == country {
return geoip.Cidr, nil
func loadFile(file string) ([]byte, error) {
if FileCache[file] == nil {
bs, err := filesystem.ReadAsset(file)
if err != nil {
return nil, newError("failed to open file: ", file).Base(err)
}
if len(bs) == 0 {
return nil, newError("empty file: ", file)
}
// Do not cache file, may save RAM when there
// are many files, but consume CPU each time.
return bs, nil
FileCache[file] = bs
}
return nil, newError("country not found in ", filename, ": ", country)
return FileCache[file], nil
}
func loadSite(filename, country string) ([]*router.Domain, error) {
geositeBytes, err := filesystem.ReadAsset(filename)
if err != nil {
return nil, newError("failed to open file: ", filename).Base(err)
}
var geositeList router.GeoSiteList
if err := proto.Unmarshal(geositeBytes, &geositeList); err != nil {
return nil, err
}
for _, site := range geositeList.Entry {
if site.CountryCode == country {
return site.Domain, nil
func loadIP(file, code string) ([]*router.CIDR, error) {
index := file + ":" + code
if IPCache[index] == nil {
bs, err := loadFile(file)
if err != nil {
return nil, newError("failed to load file: ", file).Base(err)
}
bs = find(bs, []byte(code))
if bs == nil {
return nil, newError("code not found in ", file, ": ", code)
}
var geoip router.GeoIP
if err := proto.Unmarshal(bs, &geoip); err != nil {
return nil, newError("error unmarshal IP in ", file, ": ", code).Base(err)
}
defer runtime.GC() // or debug.FreeOSMemory()
return geoip.Cidr, nil // do not cache geoip
IPCache[index] = &geoip
}
return IPCache[index].Cidr, nil
}
return nil, newError("list not found in ", filename, ": ", country)
func loadSite(file, code string) ([]*router.Domain, error) {
index := file + ":" + code
if SiteCache[index] == nil {
bs, err := loadFile(file)
if err != nil {
return nil, newError("failed to load file: ", file).Base(err)
}
bs = find(bs, []byte(code))
if bs == nil {
return nil, newError("list not found in ", file, ": ", code)
}
var geosite router.GeoSite
if err := proto.Unmarshal(bs, &geosite); err != nil {
return nil, newError("error unmarshal Site in ", file, ": ", code).Base(err)
}
defer runtime.GC() // or debug.FreeOSMemory()
return geosite.Domain, nil // do not cache geosite
SiteCache[index] = &geosite
}
return SiteCache[index].Domain, nil
}
func find(data, code []byte) []byte {
codeL := len(code)
if codeL == 0 {
return nil
}
for {
dataL := len(data)
if dataL < 2 {
return nil
}
x, y := proto.DecodeVarint(data[1:])
if x == 0 && y == 0 {
return nil
}
headL, bodyL := 1+y, int(x)
dataL -= headL
if dataL < bodyL {
return nil
}
data = data[headL:]
if int(data[1]) == codeL {
for i := 0; i < codeL && data[2+i] == code[i]; i++ {
if i+1 == codeL {
return data[:bodyL]
}
}
}
if dataL == bodyL {
return nil
}
data = data[bodyL:]
}
}
type AttributeMatcher interface {

View File

@ -9,12 +9,14 @@ import (
"path"
"path/filepath"
"runtime"
"runtime/debug"
"strings"
"syscall"
"github.com/xtls/xray-core/common/cmdarg"
"github.com/xtls/xray-core/common/platform"
"github.com/xtls/xray-core/core"
"github.com/xtls/xray-core/infra/conf"
"github.com/xtls/xray-core/main/commands/base"
)
@ -78,8 +80,13 @@ func executeRun(cmd *base.Command, args []string) {
}
defer server.Close()
conf.FileCache = nil
conf.IPCache = nil
conf.SiteCache = nil
// Explicitly triggering GC to remove garbage from config loading.
runtime.GC()
debug.FreeOSMemory()
{
osSignals := make(chan os.Signal, 1)