Update dependencies

The codes should start from the third arg.

.github/release-branch.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+mkdir -p release
+cd release
+git init
+git config --local user.email "github-action@users.noreply.github.com"
+git config --local user.name "GitHub Action"
+git remote add origin https://github-action:$GITHUB_TOKEN@github.com/SagerNet/sing-geoip.git
+git branch -M release
+cp ../*.db ../*.sha256sum .
+git add .
+git commit -m "Update release"
+git push -f origin release

.github/release-rule-set.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+cd rule-set
+git init
+git config --local user.email "github-action@users.noreply.github.com"
+git config --local user.name "GitHub Action"
+git remote add origin https://github-action:$GITHUB_TOKEN@github.com/SagerNet/sing-geoip.git
+git branch -M rule-set
+git add .
+git commit -m "Update rule-set"
+git push -f origin rule-set

.github/update_dependencies.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+PROJECTS=$(dirname "$0")/../..
+
+go get -x github.com/sagernet/sing-box@$(git -C $PROJECTS/sing-box rev-parse HEAD)
+go mod tidy

.github/workflows/build.yaml

@@ -0,0 +1,32 @@
+name: Build
+on:
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Get latest go version
+        id: version
+        run: |
+          echo ::set-output name=go_version::$(curl -s https://raw.githubusercontent.com/actions/go-versions/main/versions-manifest.json | grep -oE '"version": "[0-9]{1}.[0-9]{1,}(.[0-9]{1,})?"' | head -1 | cut -d':' -f2 | sed 's/ //g; s/"//g')
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ steps.version.outputs.go_version }}
+      - name: Build geoip
+        id: build
+        env:
+          NO_SKIP: true
+        run: go run -v .
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: geoip.db
+          path: geoip.db

.github/workflows/release.yaml

@@ -0,0 +1,55 @@
+name: Release
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 8 12 * *"
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Get latest go version
+        id: version
+        run: |
+          echo ::set-output name=go_version::$(curl -s https://raw.githubusercontent.com/actions/go-versions/main/versions-manifest.json | grep -oE '"version": "[0-9]{1}.[0-9]{1,}(.[0-9]{1,})?"' | head -1 | cut -d':' -f2 | sed 's/ //g; s/"//g')
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ steps.version.outputs.go_version }}
+      - name: Build geoip
+        id: build
+        run: go run -v .
+      - name: Release rule sets
+        if: steps.build.outputs.skip != 'true'
+        run: .github/release-rule-set.sh
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Generate sha256 hash
+        if: steps.build.outputs.skip != 'true'
+        run: |
+          sha256sum geoip.db > geoip.db.sha256sum
+          sha256sum geoip-cn.db > geoip-cn.db.sha256sum
+      - name: Release release branch
+        if: steps.build.outputs.skip != 'true'
+        run: .github/release-branch.sh
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: dev-drprasad/delete-older-releases@v0.3.2
+        with:
+          keep_latest: 3
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Release geoip
+        if: steps.build.outputs.skip != 'true'
+        uses: softprops/action-gh-release@v1
+        with:
+          tag_name: ${{ steps.build.outputs.tag }}
+          files: |
+            geoip.db
+            geoip.db.sha256sum
+            geoip-cn.db
+            geoip-cn.db.sha256sum

.gitignore

@@ -0,0 +1,6 @@
+/.idea/
+/vendor/
+/*.db
+*.mmdb
+/rule-set/
+/release/

.golangci.yml

@@ -0,0 +1,17 @@
+linters:
+  disable-all: true
+  enable:
+    - gofumpt
+    - govet
+    - gci
+    - staticcheck
+
+linters-settings:
+  gci:
+    custom-order: true
+    sections:
+      - standard
+      - prefix(github.com/sagernet/)
+      - default
+  staticcheck:
+    go: '1.20'

LICENSE

@@ -0,0 +1,14 @@
+Copyright (C) 2022 by nekohasekai <contact-sagernet@sekai.icu>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.

Makefile

@@ -0,0 +1,17 @@
+fmt:
+	@gofumpt -l -w .
+	@gofmt -s -w .
+	@gci write --custom-order -s standard -s "prefix(github.com/sagernet/)" -s "default" .
+
+fmt_install:
+	go install -v mvdan.cc/gofumpt@latest
+	go install -v github.com/daixiang0/gci@latest
+
+lint:
+	golangci-lint run ./...
+
+lint_install:
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
+
+test:
+	go test -v ./...

geoip-cn.db.sha256sum

@@ -1 +0,0 @@
-464da4039de5c8622d21e9f2938a36d80bfed33fcfbe1293ba74eef94b9fb49e  geoip-cn.db

geoip.db.sha256sum

@@ -1 +0,0 @@
-71484cf35bb48453e26bcc3373a0988a2536588f8e3ca96cda59ff742af6c392  geoip.db

go.mod

@@ -0,0 +1,26 @@
+module github.com/sagernet/sing-geoip
+
+go 1.18
+
+require (
+	github.com/google/go-github/v45 v45.2.0
+	github.com/maxmind/mmdbwriter v1.0.0
+	github.com/oschwald/geoip2-golang v1.9.0
+	github.com/oschwald/maxminddb-golang v1.12.0
+	github.com/sagernet/sing v0.2.20-0.20231212123824-8836b6754226
+	github.com/sagernet/sing-box v1.8.0-beta.3
+)
+
+require (
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
+	github.com/miekg/dns v1.1.57 // indirect
+	github.com/sagernet/sing-dns v0.1.11 // indirect
+	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
+	golang.org/x/crypto v0.16.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/tools v0.16.0 // indirect
+)

go.sum

@@ -0,0 +1,41 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI=
+github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
+github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/maxmind/mmdbwriter v1.0.0 h1:bieL4P6yaYaHvbtLSwnKtEvScUKKD6jcKaLiTM3WSMw=
+github.com/maxmind/mmdbwriter v1.0.0/go.mod h1:noBMCUtyN5PUQ4H8ikkOvGSHhzhLok51fON2hcrpKj8=
+github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
+github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
+github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc=
+github.com/oschwald/geoip2-golang v1.9.0/go.mod h1:BHK6TvDyATVQhKNbQBdrj9eAvuwOMi2zSFXizL3K81Y=
+github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs=
+github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/sagernet/sing v0.2.20-0.20231212123824-8836b6754226 h1:rcII71ho6F/7Nyx7n2kESLcnvNMdcU4i8ZUGF2Fi7yA=
+github.com/sagernet/sing v0.2.20-0.20231212123824-8836b6754226/go.mod h1:Ce5LNojQOgOiWhiD8pPD6E9H7e2KgtOe3Zxx4Ou5u80=
+github.com/sagernet/sing-box v1.8.0-beta.3 h1:2uotZm8GQrx6lztuHzdmHR4Fk/FV6G7eETWmcTmCWi8=
+github.com/sagernet/sing-box v1.8.0-beta.3/go.mod h1:WOnlC8xcuQDYvkSXi0k6Szfqn5CXSV2nmQzCIiZ/nRI=
+github.com/sagernet/sing-dns v0.1.11 h1:PPrMCVVrAeR3f5X23I+cmvacXJ+kzuyAsBiWyUKhGSE=
+github.com/sagernet/sing-dns v0.1.11/go.mod h1:zJ/YjnYB61SYE+ubMcMqVdpaSvsyQ2iShQGO3vuLvvE=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
+golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
+golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM=
+golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

main.go

@@ -0,0 +1,257 @@
+package main
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/sagernet/sing-box/common/srs"
+	C "github.com/sagernet/sing-box/constant"
+	"github.com/sagernet/sing-box/log"
+	"github.com/sagernet/sing-box/option"
+	"github.com/sagernet/sing/common"
+	E "github.com/sagernet/sing/common/exceptions"
+
+	"github.com/google/go-github/v45/github"
+	"github.com/maxmind/mmdbwriter"
+	"github.com/maxmind/mmdbwriter/inserter"
+	"github.com/maxmind/mmdbwriter/mmdbtype"
+	"github.com/oschwald/geoip2-golang"
+	"github.com/oschwald/maxminddb-golang"
+)
+
+var githubClient *github.Client
+
+func init() {
+	accessToken, loaded := os.LookupEnv("ACCESS_TOKEN")
+	if !loaded {
+		githubClient = github.NewClient(nil)
+		return
+	}
+	transport := &github.BasicAuthTransport{
+		Username: accessToken,
+	}
+	githubClient = github.NewClient(transport.Client())
+}
+
+func fetch(from string) (*github.RepositoryRelease, error) {
+	names := strings.SplitN(from, "/", 2)
+	latestRelease, _, err := githubClient.Repositories.GetLatestRelease(context.Background(), names[0], names[1])
+	if err != nil {
+		return nil, err
+	}
+	return latestRelease, err
+}
+
+func get(downloadURL *string) ([]byte, error) {
+	log.Info("download ", *downloadURL)
+	response, err := http.Get(*downloadURL)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	return io.ReadAll(response.Body)
+}
+
+func download(release *github.RepositoryRelease) ([]byte, error) {
+	geoipAsset := common.Find(release.Assets, func(it *github.ReleaseAsset) bool {
+		return *it.Name == "Country.mmdb"
+	})
+	if geoipAsset == nil {
+		return nil, E.New("Country.mmdb not found in upstream release ", release.Name)
+	}
+	return get(geoipAsset.BrowserDownloadURL)
+}
+
+func parse(binary []byte) (metadata maxminddb.Metadata, countryMap map[string][]*net.IPNet, err error) {
+	database, err := maxminddb.FromBytes(binary)
+	if err != nil {
+		return
+	}
+	metadata = database.Metadata
+	networks := database.Networks(maxminddb.SkipAliasedNetworks)
+	countryMap = make(map[string][]*net.IPNet)
+	var country geoip2.Enterprise
+	var ipNet *net.IPNet
+	for networks.Next() {
+		ipNet, err = networks.Network(&country)
+		if err != nil {
+			return
+		}
+		var code string
+		if country.Country.IsoCode != "" {
+			code = strings.ToLower(country.Country.IsoCode)
+		} else if country.RegisteredCountry.IsoCode != "" {
+			code = strings.ToLower(country.RegisteredCountry.IsoCode)
+		} else if country.RepresentedCountry.IsoCode != "" {
+			code = strings.ToLower(country.RepresentedCountry.IsoCode)
+		} else if country.Continent.Code != "" {
+			code = strings.ToLower(country.Continent.Code)
+		} else {
+			continue
+		}
+		countryMap[code] = append(countryMap[code], ipNet)
+	}
+	err = networks.Err()
+	return
+}
+
+func newWriter(metadata maxminddb.Metadata, codes []string) (*mmdbwriter.Tree, error) {
+	return mmdbwriter.New(mmdbwriter.Options{
+		DatabaseType:            "sing-geoip",
+		Languages:               codes,
+		IPVersion:               int(metadata.IPVersion),
+		RecordSize:              int(metadata.RecordSize),
+		Inserter:                inserter.ReplaceWith,
+		DisableIPv4Aliasing:     true,
+		IncludeReservedNetworks: true,
+	})
+}
+
+func open(path string, codes []string) (*mmdbwriter.Tree, error) {
+	reader, err := maxminddb.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	if reader.Metadata.DatabaseType != "sing-geoip" {
+		return nil, E.New("invalid sing-geoip database")
+	}
+	reader.Close()
+
+	return mmdbwriter.Load(path, mmdbwriter.Options{
+		Languages: append(reader.Metadata.Languages, common.Filter(codes, func(it string) bool {
+			return !common.Contains(reader.Metadata.Languages, it)
+		})...),
+		Inserter: inserter.ReplaceWith,
+	})
+}
+
+func write(writer *mmdbwriter.Tree, dataMap map[string][]*net.IPNet, output string, codes []string) error {
+	if len(codes) == 0 {
+		codes = make([]string, 0, len(dataMap))
+		for code := range dataMap {
+			codes = append(codes, code)
+		}
+	}
+	sort.Strings(codes)
+	codeMap := make(map[string]bool)
+	for _, code := range codes {
+		codeMap[code] = true
+	}
+	for code, data := range dataMap {
+		if !codeMap[code] {
+			continue
+		}
+		for _, item := range data {
+			err := writer.Insert(item, mmdbtype.String(code))
+			if err != nil {
+				return err
+			}
+		}
+	}
+	outputFile, err := os.Create(output)
+	if err != nil {
+		return err
+	}
+	defer outputFile.Close()
+	_, err = writer.WriteTo(outputFile)
+	return err
+}
+
+func release(source string, destination string, output string, ruleSetOutput string) error {
+	sourceRelease, err := fetch(source)
+	if err != nil {
+		return err
+	}
+	destinationRelease, err := fetch(destination)
+	if err != nil {
+		log.Warn("missing destination latest release")
+	} else {
+		if os.Getenv("NO_SKIP") != "true" && strings.Contains(*destinationRelease.Name, *sourceRelease.Name) {
+			log.Info("already latest")
+			setActionOutput("skip", "true")
+			return nil
+		}
+	}
+	binary, err := download(sourceRelease)
+	if err != nil {
+		return err
+	}
+	metadata, countryMap, err := parse(binary)
+	if err != nil {
+		return err
+	}
+	allCodes := make([]string, 0, len(countryMap))
+	for code := range countryMap {
+		allCodes = append(allCodes, code)
+	}
+
+	writer, err := newWriter(metadata, allCodes)
+	if err != nil {
+		return err
+	}
+	err = write(writer, countryMap, output, nil)
+	if err != nil {
+		return err
+	}
+
+	writer, err = newWriter(metadata, []string{"cn"})
+	if err != nil {
+		return err
+	}
+	err = write(writer, countryMap, "geoip-cn.db", []string{"cn"})
+	if err != nil {
+		return err
+	}
+
+	os.RemoveAll(ruleSetOutput)
+	err = os.MkdirAll(ruleSetOutput, 0o755)
+	if err != nil {
+		return err
+	}
+	for countryCode, ipNets := range countryMap {
+		var headlessRule option.DefaultHeadlessRule
+		headlessRule.IPCIDR = make([]string, 0, len(ipNets))
+		for _, cidr := range ipNets {
+			headlessRule.IPCIDR = append(headlessRule.IPCIDR, cidr.String())
+		}
+		var plainRuleSet option.PlainRuleSet
+		plainRuleSet.Rules = []option.HeadlessRule{
+			{
+				Type:           C.RuleTypeDefault,
+				DefaultOptions: headlessRule,
+			},
+		}
+		srsPath, _ := filepath.Abs(filepath.Join(ruleSetOutput, "geoip-"+countryCode+".srs"))
+		os.Stderr.WriteString("write " + srsPath + "\n")
+		outputRuleSet, err := os.Create(srsPath)
+		if err != nil {
+			return err
+		}
+		err = srs.Write(outputRuleSet, plainRuleSet)
+		if err != nil {
+			outputRuleSet.Close()
+			return err
+		}
+		outputRuleSet.Close()
+	}
+
+	setActionOutput("tag", *sourceRelease.Name)
+	return nil
+}
+
+func setActionOutput(name string, content string) {
+	os.Stdout.WriteString("::set-output name=" + name + "::" + content + "\n")
+}
+
+func main() {
+	err := release("Dreamacro/maxmind-geoip", "sagernet/sing-geoip", "geoip.db", "rule-set")
+	if err != nil {
+		log.Fatal(err)
+	}
+}