Fix args index num

The codes should start from the third arg.

.github/workflows/build.yaml

@@ -0,0 +1,37 @@
+name: Build
+on:
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Get latest go version
+        id: version
+        run: |
+          echo ::set-output name=go_version::$(curl -s https://raw.githubusercontent.com/actions/go-versions/main/versions-manifest.json | grep -oE '"version": "[0-9]{1}.[0-9]{1,}(.[0-9]{1,})?"' | head -1 | cut -d':' -f2 | sed 's/ //g; s/"//g')
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ steps.version.outputs.go_version }}
+      - name: Build geoip
+        id: build
+        env:
+          NO_SKIP: true
+        run: go run -v .
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: geoip.db
+          path: geoip.db
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: geoip-cn.db
+          path: geoip-cn.db

.github/workflows/release.yaml

@@ -0,0 +1,81 @@
+name: Release
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 8 12 * *"
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Get latest go version
+        id: version
+        run: |
+          echo ::set-output name=go_version::$(curl -s https://raw.githubusercontent.com/actions/go-versions/main/versions-manifest.json | grep -oE '"version": "[0-9]{1}.[0-9]{1,}(.[0-9]{1,})?"' | head -1 | cut -d':' -f2 | sed 's/ //g; s/"//g')
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ steps.version.outputs.go_version }}
+      - name: Build geoip
+        id: build
+        run: go run -v .
+      - name: Generate sha256 hash
+        if: steps.build.outputs.skip != 'true'
+        run: |
+          sha256sum geoip.db > geoip.db.sha256sum
+          sha256sum geoip-cn.db > geoip-cn.db.sha256sum
+      - name: Create a release
+        if: steps.build.outputs.skip != 'true'
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ steps.build.outputs.tag }}
+          release_name: ${{ steps.build.outputs.tag }}
+          draft: false
+          prerelease: false
+      - name: Release geoip.db
+        if: steps.build.outputs.skip != 'true'
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./geoip.db
+          asset_name: geoip.db
+          asset_content_type: application/octet-stream
+      - name: Release geoip.db sha256sum
+        if: steps.build.outputs.skip != 'true'
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./geoip.db.sha256sum
+          asset_name: geoip.db.sha256sum
+          asset_content_type: text/plain
+      - name: Release geoip-cn.db
+        if: steps.build.outputs.skip != 'true'
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./geoip-cn.db
+          asset_name: geoip-cn.db
+          asset_content_type: application/octet-stream
+      - name: Release geoip.db sha256sum
+        if: steps.build.outputs.skip != 'true'
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./geoip-cn.db.sha256sum
+          asset_name: geoip-cn.db.sha256sum
+          asset_content_type: text/plain

.gitignore

@@ -0,0 +1,4 @@
+/.idea/
+/vendor/
+/*.db
+*.mmdb

.golangci.yml

@@ -0,0 +1,54 @@
+run:
+  timeout: 5m
+
+linters:
+  enable-all: true
+  disable:
+    - errcheck
+    - wrapcheck
+    - varnamelen
+    - stylecheck
+    - nonamedreturns
+    - nlreturn
+    - ireturn
+    - gomnd
+    - exhaustivestruct
+    - ifshort
+    - goerr113
+    - gochecknoglobals
+    - forcetypeassert
+    - exhaustruct
+    - exhaustive
+    - cyclop
+    - containedctx
+    - wsl
+    - nestif
+    - lll
+    - funlen
+    - goconst
+    - godot
+    - gocognit
+    - golint
+    - goimports
+    - gochecknoinits
+    - maligned
+    - tagliatelle
+    - gocyclo
+    - maintidx
+    - gocritic
+    - nakedret
+
+linters-settings:
+  revive:
+    rules:
+      - name: var-naming
+        disabled: true
+  govet:
+    enable-all: true
+    disable:
+      - composites
+      - fieldalignment
+      - shadow
+  gosec:
+    excludes:
+      - G404

LICENSE

@@ -0,0 +1,14 @@
+Copyright (C) 2022 by nekohasekai <contact-sagernet@sekai.icu>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.

format.go

@@ -0,0 +1,7 @@
+package main
+
+//go:generate go install -v mvdan.cc/gofumpt@latest
+//go:generate go install -v github.com/daixiang0/gci@latest
+//go:generate gofumpt -l -w .
+//go:generate gofmt -s -w .
+//go:generate gci write .

geoip-cn.db.sha256sum

@@ -1 +0,0 @@
-464da4039de5c8622d21e9f2938a36d80bfed33fcfbe1293ba74eef94b9fb49e  geoip-cn.db

geoip.db.sha256sum

@@ -1 +0,0 @@
-71484cf35bb48453e26bcc3373a0988a2536588f8e3ca96cda59ff742af6c392  geoip.db

go.mod

@@ -0,0 +1,19 @@
+module sing-geoip
+
+go 1.18
+
+require (
+	github.com/google/go-github/v45 v45.2.0
+	github.com/maxmind/mmdbwriter v0.0.0-20230530195610-0798c256659b
+	github.com/oschwald/geoip2-golang v1.8.0
+	github.com/oschwald/maxminddb-golang v1.10.0
+	github.com/sagernet/sing v0.2.4
+	github.com/sirupsen/logrus v1.9.3
+)
+
+require (
+	github.com/google/go-querystring v1.1.0 // indirect
+	go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d // indirect
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
+	golang.org/x/sys v0.7.0 // indirect
+)

go.sum

@@ -0,0 +1,35 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI=
+github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/maxmind/mmdbwriter v0.0.0-20230530195610-0798c256659b h1:+RXmWNst9zn/gJRLz6e3KpmjwoyDCWiyOPORL8k/Xtk=
+github.com/maxmind/mmdbwriter v0.0.0-20230530195610-0798c256659b/go.mod h1:/3IBsHZu2HKvnlBELgdw2ewEIkdCKCI+86iUnBYjEEw=
+github.com/oschwald/geoip2-golang v1.8.0 h1:KfjYB8ojCEn/QLqsDU0AzrJ3R5Qa9vFlx3z6SLNcKTs=
+github.com/oschwald/geoip2-golang v1.8.0/go.mod h1:R7bRvYjOeaoenAp9sKRS8GX5bJWcZ0laWO5+DauEktw=
+github.com/oschwald/maxminddb-golang v1.10.0 h1:Xp1u0ZhqkSuopaKmk1WwHtjF0H9Hd9181uj2MQ5Vndg=
+github.com/oschwald/maxminddb-golang v1.10.0/go.mod h1:Y2ELenReaLAZ0b400URyGwvYxHV1dLIxBuyOsyYjHK0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sagernet/sing v0.2.4 h1:gC8BR5sglbJZX23RtMyFa8EETP9YEUADhfbEzU1yVbo=
+github.com/sagernet/sing v0.2.4/go.mod h1:Ta8nHnDLAwqySzKhGoKk4ZIB+vJ3GTKj7UPrWYvM+4w=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d h1:ggxwEf5eu0l8v+87VhX1czFh8zJul3hK16Gmruxn7hw=
+go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d/go.mod h1:tgPU4N2u9RByaTN3NC2p9xOzyFpte4jYwsIIRF7XlSc=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

main.go

@@ -0,0 +1,248 @@
+package main
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/sagernet/sing/common"
+	E "github.com/sagernet/sing/common/exceptions"
+	"github.com/sagernet/sing/common/rw"
+
+	"github.com/google/go-github/v45/github"
+	"github.com/maxmind/mmdbwriter"
+	"github.com/maxmind/mmdbwriter/inserter"
+	"github.com/maxmind/mmdbwriter/mmdbtype"
+	"github.com/oschwald/geoip2-golang"
+	"github.com/oschwald/maxminddb-golang"
+	"github.com/sirupsen/logrus"
+)
+
+var githubClient *github.Client
+
+func init() {
+	accessToken, loaded := os.LookupEnv("ACCESS_TOKEN")
+	if !loaded {
+		githubClient = github.NewClient(nil)
+		return
+	}
+	transport := &github.BasicAuthTransport{
+		Username: accessToken,
+	}
+	githubClient = github.NewClient(transport.Client())
+}
+
+func fetch(from string) (*github.RepositoryRelease, error) {
+	names := strings.SplitN(from, "/", 2)
+	latestRelease, _, err := githubClient.Repositories.GetLatestRelease(context.Background(), names[0], names[1])
+	if err != nil {
+		return nil, err
+	}
+	return latestRelease, err
+}
+
+func get(downloadURL *string) ([]byte, error) {
+	logrus.Info("download ", *downloadURL)
+	response, err := http.Get(*downloadURL)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	return io.ReadAll(response.Body)
+}
+
+func download(release *github.RepositoryRelease) ([]byte, error) {
+	geoipAsset := common.Find(release.Assets, func(it *github.ReleaseAsset) bool {
+		return *it.Name == "Country.mmdb"
+	})
+	if geoipAsset == nil {
+		return nil, E.New("Country.mmdb not found in upstream release ", release.Name)
+	}
+	return get(geoipAsset.BrowserDownloadURL)
+}
+
+func parse(binary []byte) (metadata maxminddb.Metadata, countryMap map[string][]*net.IPNet, err error) {
+	database, err := maxminddb.FromBytes(binary)
+	if err != nil {
+		return
+	}
+	metadata = database.Metadata
+	networks := database.Networks(maxminddb.SkipAliasedNetworks)
+	countryMap = make(map[string][]*net.IPNet)
+	var country geoip2.Enterprise
+	var ipNet *net.IPNet
+	for networks.Next() {
+		ipNet, err = networks.Network(&country)
+		if err != nil {
+			return
+		}
+		var code string
+		if country.Country.IsoCode != "" {
+			code = strings.ToLower(country.Country.IsoCode)
+		} else if country.RegisteredCountry.IsoCode != "" {
+			code = strings.ToLower(country.RegisteredCountry.IsoCode)
+		} else if country.RepresentedCountry.IsoCode != "" {
+			code = strings.ToLower(country.RepresentedCountry.IsoCode)
+		} else if country.Continent.Code != "" {
+			code = strings.ToLower(country.Continent.Code)
+		} else {
+			continue
+		}
+		countryMap[code] = append(countryMap[code], ipNet)
+	}
+	err = networks.Err()
+	return
+}
+
+func newWriter(metadata maxminddb.Metadata, codes []string) (*mmdbwriter.Tree, error) {
+	return mmdbwriter.New(mmdbwriter.Options{
+		DatabaseType:            "sing-geoip",
+		Languages:               codes,
+		IPVersion:               int(metadata.IPVersion),
+		RecordSize:              int(metadata.RecordSize),
+		Inserter:                inserter.ReplaceWith,
+		DisableIPv4Aliasing:     true,
+		IncludeReservedNetworks: true,
+	})
+}
+
+func open(path string, codes []string) (*mmdbwriter.Tree, error) {
+	reader, err := maxminddb.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	if reader.Metadata.DatabaseType != "sing-geoip" {
+		return nil, E.New("invalid sing-geoip database")
+	}
+	reader.Close()
+
+	return mmdbwriter.Load(path, mmdbwriter.Options{
+		Languages: append(reader.Metadata.Languages, common.Filter(codes, func(it string) bool {
+			return !common.Contains(reader.Metadata.Languages, it)
+		})...),
+		Inserter: inserter.ReplaceWith,
+	})
+}
+
+func write(writer *mmdbwriter.Tree, dataMap map[string][]*net.IPNet, output string, codes []string) error {
+	if len(codes) == 0 {
+		codes = make([]string, 0, len(dataMap))
+		for code := range dataMap {
+			codes = append(codes, code)
+		}
+	}
+	sort.Strings(codes)
+	codeMap := make(map[string]bool)
+	for _, code := range codes {
+		codeMap[code] = true
+	}
+	for code, data := range dataMap {
+		if !codeMap[code] {
+			continue
+		}
+		for _, item := range data {
+			err := writer.Insert(item, mmdbtype.String(code))
+			if err != nil {
+				return err
+			}
+		}
+	}
+	outputFile, err := os.Create(output)
+	if err != nil {
+		return err
+	}
+	defer outputFile.Close()
+	_, err = writer.WriteTo(outputFile)
+	return err
+}
+
+func local(input string, output string, codes []string) error {
+	binary, err := os.ReadFile(input)
+	if err != nil {
+		return err
+	}
+	metadata, countryMap, err := parse(binary)
+	if err != nil {
+		return err
+	}
+	var writer *mmdbwriter.Tree
+	if rw.FileExists(output) {
+		writer, err = open(output, codes)
+	} else {
+		writer, err = newWriter(metadata, codes)
+	}
+	if err != nil {
+		return err
+	}
+	return write(writer, countryMap, output, codes)
+}
+
+func release(source string, destination string) error {
+	sourceRelease, err := fetch(source)
+	if err != nil {
+		return err
+	}
+	destinationRelease, err := fetch(destination)
+	if err != nil {
+		logrus.Warn("missing destination latest release")
+	} else {
+		if os.Getenv("NO_SKIP") != "true" && strings.Contains(*destinationRelease.Name, *sourceRelease.Name) {
+			logrus.Info("already latest")
+			setActionOutput("skip", "true")
+			return nil
+		}
+	}
+	binary, err := download(sourceRelease)
+	if err != nil {
+		return err
+	}
+	metadata, countryMap, err := parse(binary)
+	if err != nil {
+		return err
+	}
+	allCodes := make([]string, 0, len(countryMap))
+	for code := range countryMap {
+		allCodes = append(allCodes, code)
+	}
+	writer, err := newWriter(metadata, allCodes)
+	if err != nil {
+		return err
+	}
+	err = write(writer, countryMap, "geoip.db", nil)
+	if err != nil {
+		return err
+	}
+	writer, err = newWriter(metadata, []string{"cn"})
+	if err != nil {
+		return err
+	}
+	err = write(writer, countryMap, "geoip-cn.db", []string{"cn"})
+	if err != nil {
+		return err
+	}
+	if err != nil {
+		return err
+	}
+	setActionOutput("tag", *sourceRelease.Name)
+	return nil
+}
+
+func setActionOutput(name string, content string) {
+	os.Stdout.WriteString("::set-output name=" + name + "::" + content + "\n")
+}
+
+func main() {
+	var err error
+	if len(os.Args) >= 3 {
+		err = local(os.Args[1], os.Args[2], os.Args[3:])
+	} else {
+		err = release("Dreamacro/maxmind-geoip", "sagernet/sing-geoip")
+	}
+	if err != nil {
+		logrus.Fatal(err)
+	}
+}