Add delete-older-releases action

.github/update_dependencies.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+PROJECTS=$(dirname "$0")/../..
+
+go get -x github.com/sagernet/sing-box@$(git -C $PROJECTS/sing-box rev-parse HEAD)
+go mod tidy

.github/workflows/build.yaml

@@ -0,0 +1,33 @@
+name: Build
+on:
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Get latest go version
+        id: version
+        run: |
+          echo ::set-output name=go_version::$(curl -s https://raw.githubusercontent.com/actions/go-versions/main/versions-manifest.json | grep -oE '"version": "[0-9]{1}.[0-9]{1,}(.[0-9]{1,})?"' | head -1 | cut -d':' -f2 | sed 's/ //g; s/"//g')
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ steps.version.outputs.go_version }}
+      - name: Build geosite
+        id: build
+        env:
+          NO_SKIP: true
+        run: |
+          go run -v .
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: geosite.db
+          path: geosite.db

.github/workflows/release.yaml

@@ -0,0 +1,66 @@
+name: Release
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * *"
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Get latest go version
+        id: version
+        run: |
+          echo ::set-output name=go_version::$(curl -s https://raw.githubusercontent.com/actions/go-versions/main/versions-manifest.json | grep -oE '"version": "[0-9]{1}.[0-9]{1,}(.[0-9]{1,})?"' | head -1 | cut -d':' -f2 | sed 's/ //g; s/"//g')
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ steps.version.outputs.go_version }}
+      - name: Build geosite
+        id: build
+        run: |
+          go run -v .
+      - name: Generate sha256 hash
+        if: steps.build.outputs.skip != 'true'
+        run: |
+          sha256sum geosite.db > geosite.db.sha256sum
+      - uses: dev-drprasad/delete-older-releases@v0.3.2
+        with:
+          keep_latest: 3
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create a release
+        if: steps.build.outputs.skip != 'true'
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ steps.build.outputs.tag }}
+          release_name: ${{ steps.build.outputs.tag }}
+          draft: false
+          prerelease: false
+      - name: Release geosite.db
+        if: steps.build.outputs.skip != 'true'
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./geosite.db
+          asset_name: geosite.db
+          asset_content_type: application/octet-stream
+      - name: Release geosite.db sha256sum
+        if: steps.build.outputs.skip != 'true'
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./geosite.db.sha256sum
+          asset_name: geosite.db.sha256sum
+          asset_content_type: text/plain

.gitignore

@@ -0,0 +1,3 @@
+/.idea/
+/vendor/
+/geosite.db

.golangci.yml

@@ -0,0 +1,54 @@
+run:
+  timeout: 5m
+
+linters:
+  enable-all: true
+  disable:
+    - errcheck
+    - wrapcheck
+    - varnamelen
+    - stylecheck
+    - nonamedreturns
+    - nlreturn
+    - ireturn
+    - gomnd
+    - exhaustivestruct
+    - ifshort
+    - goerr113
+    - gochecknoglobals
+    - forcetypeassert
+    - exhaustruct
+    - exhaustive
+    - cyclop
+    - containedctx
+    - wsl
+    - nestif
+    - lll
+    - funlen
+    - goconst
+    - godot
+    - gocognit
+    - golint
+    - goimports
+    - gochecknoinits
+    - maligned
+    - tagliatelle
+    - gocyclo
+    - maintidx
+    - gocritic
+    - nakedret
+
+linters-settings:
+  revive:
+    rules:
+      - name: var-naming
+        disabled: true
+  govet:
+    enable-all: true
+    disable:
+      - composites
+      - fieldalignment
+      - shadow
+  gosec:
+    excludes:
+      - G404

LICENSE

@@ -0,0 +1,14 @@
+Copyright (C) 2022 by nekohasekai <contact-sagernet@sekai.icu>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.

format.go

@@ -0,0 +1,7 @@
+package main
+
+//go:generate go install -v mvdan.cc/gofumpt@latest
+//go:generate go install -v github.com/daixiang0/gci@latest
+//go:generate gofumpt -l -w .
+//go:generate gofmt -s -w .
+//go:generate gci write .

geosite-cn.db.sha256sum

@@ -1 +0,0 @@
-4a5028c9bbab52ae9869380ad8aa11fa40f1318e5f5d69ca268e7f6e89ad7548  geosite-cn.db

geosite.db.sha256sum

@@ -1 +0,0 @@
-dd47e49ad9731940b2153c8981a51cc13216a50cec317a906fbfb3a5e16e5557  geosite.db

go.mod

@@ -0,0 +1,25 @@
+module sing-geosite
+
+go 1.18
+
+require (
+	github.com/google/go-github/v45 v45.2.0
+	github.com/sagernet/sing v0.2.17
+	github.com/sagernet/sing-box v1.6.4
+	github.com/sirupsen/logrus v1.9.3
+	github.com/v2fly/v2ray-core/v5 v5.11.0
+	google.golang.org/protobuf v1.31.0
+)
+
+require (
+	github.com/adrg/xdg v0.4.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/miekg/dns v1.1.56 // indirect
+	github.com/sagernet/sing-dns v0.1.10 // indirect
+	golang.org/x/crypto v0.15.0 // indirect
+	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
+)

go.sum

@@ -0,0 +1,53 @@
+github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
+github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI=
+github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE=
+github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sagernet/sing v0.2.17 h1:vMPKb3MV0Aa5ws4dCJkRI8XEjrsUcDn810czd0FwmzI=
+github.com/sagernet/sing v0.2.17/go.mod h1:OL6k2F0vHmEzXz2KW19qQzu172FDgSbUSODylighuVo=
+github.com/sagernet/sing-box v1.6.4 h1:4cz6kQSVehLlRzjfwYrQT52/W1nNYxulSDm7Hd+NHG8=
+github.com/sagernet/sing-box v1.6.4/go.mod h1:JEvY+zKVPCxiiEuri2mw6bu2YZh38WCd6Iv9TlHzuOM=
+github.com/sagernet/sing-dns v0.1.10 h1:iIU7nRBlUYj+fF2TaktGIvRiTFFrHwSMedLQsvlTZCI=
+github.com/sagernet/sing-dns v0.1.10/go.mod h1:vtUimtf7Nq9EdvD5WTpfCr69KL1M7bcgOVKiYBiAY/c=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/v2fly/v2ray-core/v5 v5.11.0 h1:+fTB6LdCf+pAyRn4IRXU0kAKCkFaBQW1rCCW2HgNMps=
+github.com/v2fly/v2ray-core/v5 v5.11.0/go.mod h1:o4G6VGWp7uLp96YE9V6CxJcNOVqepwbWHBEA1vXE5TU=
+golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
+golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
+golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
+golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
+golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
+golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
+golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

main.go

@@ -0,0 +1,219 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/google/go-github/v45/github"
+	"github.com/sagernet/sing-box/common/geosite"
+	"github.com/sagernet/sing/common"
+	E "github.com/sagernet/sing/common/exceptions"
+	"github.com/sirupsen/logrus"
+	"github.com/v2fly/v2ray-core/v5/app/router/routercommon"
+	"google.golang.org/protobuf/proto"
+)
+
+var githubClient *github.Client
+
+func init() {
+	accessToken, loaded := os.LookupEnv("ACCESS_TOKEN")
+	if !loaded {
+		githubClient = github.NewClient(nil)
+		return
+	}
+	transport := &github.BasicAuthTransport{
+		Username: accessToken,
+	}
+	githubClient = github.NewClient(transport.Client())
+}
+
+func fetch(from string) (*github.RepositoryRelease, error) {
+	names := strings.SplitN(from, "/", 2)
+	latestRelease, _, err := githubClient.Repositories.GetLatestRelease(context.Background(), names[0], names[1])
+	if err != nil {
+		return nil, err
+	}
+	return latestRelease, err
+}
+
+func get(downloadURL *string) ([]byte, error) {
+	logrus.Info("download ", *downloadURL)
+	response, err := http.Get(*downloadURL)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	return io.ReadAll(response.Body)
+}
+
+func download(release *github.RepositoryRelease) ([]byte, error) {
+	geositeAsset := common.Find(release.Assets, func(it *github.ReleaseAsset) bool {
+		return *it.Name == "dlc.dat"
+	})
+	geositeChecksumAsset := common.Find(release.Assets, func(it *github.ReleaseAsset) bool {
+		return *it.Name == "dlc.dat.sha256sum"
+	})
+	if geositeAsset == nil {
+		return nil, E.New("geosite asset not found in upstream release ", release.Name)
+	}
+	if geositeChecksumAsset == nil {
+		return nil, E.New("geosite asset not found in upstream release ", release.Name)
+	}
+	data, err := get(geositeAsset.BrowserDownloadURL)
+	if err != nil {
+		return nil, err
+	}
+	remoteChecksum, err := get(geositeChecksumAsset.BrowserDownloadURL)
+	if err != nil {
+		return nil, err
+	}
+	checksum := sha256.Sum256(data)
+	if hex.EncodeToString(checksum[:]) != string(remoteChecksum[:64]) {
+		return nil, E.New("checksum mismatch")
+	}
+	return data, nil
+}
+
+func parse(vGeositeData []byte) (map[string][]geosite.Item, error) {
+	vGeositeList := routercommon.GeoSiteList{}
+	err := proto.Unmarshal(vGeositeData, &vGeositeList)
+	if err != nil {
+		return nil, err
+	}
+	domainMap := make(map[string][]geosite.Item)
+	for _, vGeositeEntry := range vGeositeList.Entry {
+		code := strings.ToLower(vGeositeEntry.CountryCode)
+		domains := make([]geosite.Item, 0, len(vGeositeEntry.Domain)*2)
+		attributes := make(map[string][]*routercommon.Domain)
+		for _, domain := range vGeositeEntry.Domain {
+			if len(domain.Attribute) > 0 {
+				for _, attribute := range domain.Attribute {
+					attributes[attribute.Key] = append(attributes[attribute.Key], domain)
+				}
+			}
+			switch domain.Type {
+			case routercommon.Domain_Plain:
+				domains = append(domains, geosite.Item{
+					Type:  geosite.RuleTypeDomainKeyword,
+					Value: domain.Value,
+				})
+			case routercommon.Domain_Regex:
+				domains = append(domains, geosite.Item{
+					Type:  geosite.RuleTypeDomainRegex,
+					Value: domain.Value,
+				})
+			case routercommon.Domain_RootDomain:
+				if strings.Contains(domain.Value, ".") {
+					domains = append(domains, geosite.Item{
+						Type:  geosite.RuleTypeDomain,
+						Value: domain.Value,
+					})
+				}
+				domains = append(domains, geosite.Item{
+					Type:  geosite.RuleTypeDomainSuffix,
+					Value: "." + domain.Value,
+				})
+			case routercommon.Domain_Full:
+				domains = append(domains, geosite.Item{
+					Type:  geosite.RuleTypeDomain,
+					Value: domain.Value,
+				})
+			}
+		}
+		domainMap[code] = common.Uniq(domains)
+		for attribute, attributeEntries := range attributes {
+			attributeDomains := make([]geosite.Item, 0, len(attributeEntries)*2)
+			for _, domain := range attributeEntries {
+				switch domain.Type {
+				case routercommon.Domain_Plain:
+					attributeDomains = append(attributeDomains, geosite.Item{
+						Type:  geosite.RuleTypeDomainKeyword,
+						Value: domain.Value,
+					})
+				case routercommon.Domain_Regex:
+					attributeDomains = append(attributeDomains, geosite.Item{
+						Type:  geosite.RuleTypeDomainRegex,
+						Value: domain.Value,
+					})
+				case routercommon.Domain_RootDomain:
+					if strings.Contains(domain.Value, ".") {
+						attributeDomains = append(attributeDomains, geosite.Item{
+							Type:  geosite.RuleTypeDomain,
+							Value: domain.Value,
+						})
+					}
+					attributeDomains = append(attributeDomains, geosite.Item{
+						Type:  geosite.RuleTypeDomainSuffix,
+						Value: "." + domain.Value,
+					})
+				case routercommon.Domain_Full:
+					attributeDomains = append(attributeDomains, geosite.Item{
+						Type:  geosite.RuleTypeDomain,
+						Value: domain.Value,
+					})
+				}
+			}
+			domainMap[code+"@"+attribute] = common.Uniq(attributeDomains)
+		}
+	}
+	return domainMap, nil
+}
+
+func generate(release *github.RepositoryRelease, output string) error {
+	outputFile, err := os.Create(output)
+	if err != nil {
+		return err
+	}
+	defer outputFile.Close()
+	vData, err := download(release)
+	if err != nil {
+		return err
+	}
+	domainMap, err := parse(vData)
+	if err != nil {
+		return err
+	}
+	outputPath, _ := filepath.Abs(output)
+	os.Stderr.WriteString("write " + outputPath + "\n")
+	return geosite.Write(outputFile, domainMap)
+}
+
+func setActionOutput(name string, content string) {
+	os.Stdout.WriteString("::set-output name=" + name + "::" + content + "\n")
+}
+
+func release(source string, destination string, output string) error {
+	sourceRelease, err := fetch(source)
+	if err != nil {
+		return err
+	}
+	destinationRelease, err := fetch(destination)
+	if err != nil {
+		logrus.Warn("missing destination latest release")
+	} else {
+		if os.Getenv("NO_SKIP") != "true" && strings.Contains(*destinationRelease.Name, *sourceRelease.Name) {
+			logrus.Info("already latest")
+			setActionOutput("skip", "true")
+			return nil
+		}
+	}
+	err = generate(sourceRelease, output)
+	if err != nil {
+		return err
+	}
+	setActionOutput("tag", *sourceRelease.Name)
+	return nil
+}
+
+func main() {
+	err := release("v2fly/domain-list-community", "sagernet/sing-geosite", "geosite.db")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+}