From b1c06e270655c842675a8eedbaf05d039430a869 Mon Sep 17 00:00:00 2001 From: iglov Date: Sun, 28 Apr 2024 18:35:14 +0200 Subject: [PATCH] feature #22: changed dataset format to official --- README.md | 4 +- dataset.json | 233 ++++++++++++++++++++++++++++++++++++++---- go.mod | 3 +- main.go | 128 ++++++++++++++--------- testdata/dataset.json | 233 ++++++++++++++++++++++++++++++++++++++---- 5 files changed, 506 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index f684290..ba21851 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,13 @@ Make your own GeoIP database! The simple utility for editing MMDB databases. # How to use ```text -Usage of ./mmdb-editor: +Usage of ./bin/mmdb-editor-linux-amd64: -d string Dataset file path. (default "./dataset.json") -i string Input GeoLite2-City.mmdb file path. (default "./GeoLite2-City.mmdb") + -m string + Merge strategy. It may be: toplevel, recurse or replace. (default "replce") -o string Output modified mmdb file path. (default "./GeoLite2-City-mod.mmdb") -v Print current version and exit. diff --git a/dataset.json b/dataset.json index 3d85b43..5837d78 100644 --- a/dataset.json +++ b/dataset.json @@ -1,24 +1,213 @@ -{ - "data": [ - { - "ips": ["94.198.54.0/23"], - "country": { - "geoname_id": 2017370, - "iso_code": "RU", - "names": { - "en": "Russia" - } +[ + { + "networks": ["214.78.120.0/22"], + "data": { + "city" : { + "geoname_id" : 5391811, + "names" : { + "de" : "San Diego", + "en" : "San Diego", + "es" : "San Diego", + "fr" : "San Diego", + "ja" : "サンディエゴ", + "pt-BR" : "San Diego", + "ru" : "Сан-Диего" + } + }, + "continent" : { + "code" : "NA", + "geoname_id" : 6255149, + "names" : { + "de" : "Nordamerika", + "en" : "North America", + "es" : "Norteamérica", + "fr" : "Amérique du Nord", + "ja" : "北アメリカ", + "pt-BR" : "América do Norte", + "ru" : "Северная Америка", + "zh-CN" : "北美洲" + } + }, + "country" : { + "geoname_id" : 6252001, + "iso_code" : "US", + "names" : { + "de" : "Vereinigte Staaten", + "en" : "United States", + "es" : "Estados Unidos", + "fr" : "États Unis", + "ja" : "アメリカ", + "pt-BR" : "EUA", + "ru" : "США", + "zh-CN" : "美国" + } + }, + "location" : { + "accuracy_radius" : 100, + "latitude" : 32.7405, + "longitude" : -117.0935, + "metro_code" : 825, + "time_zone" : "America/Los_Angeles" + }, + "postal" : { + "code" : "92105" + }, + "registered_country" : { + "geoname_id" : 6252001, + "iso_code" : "US", + "names" : { + "de" : "Vereinigte Staaten", + "en" : "United States", + "es" : "Estados Unidos", + "fr" : "États Unis", + "ja" : "アメリカ", + "pt-BR" : "EUA", + "ru" : "США", + "zh-CN" : "美国" + } + }, + "subdivisions" : [ + { + "geoname_id" : 5332921, + "iso_code" : "CA", + "names" : { + "de" : "Kalifornien", + "en" : "California", + "es" : "California", + "fr" : "Californie", + "ja" : "カリフォルニア州", + "pt-BR" : "Califórnia", + "ru" : "Калифорния", + "zh-CN" : "加州" + } + } + ] } - }, - { - "ips": ["10.200.0.0/24","192.168.33.13/30","127.0.0.1/32"], - "country": { - "geoname_id": 6255148, - "iso_code": "NA", - "names": { - "en": "Iglov's property" - } + }, + { + "networks": ["81.2.69.160/18"], + "data": { + "city" : { + "geoname_id" : 2643743, + "names" : { + "de" : "London", + "en" : "London", + "es" : "Londres", + "fr" : "Londres", + "ja" : "ロンドン", + "pt-BR" : "Londres", + "ru" : "Лондон" + } + }, + "org": "My test company", + "continent" : { + "code" : "EU", + "geoname_id" : 6255148, + "names" : { + "de" : "Europa", + "en" : "Europe", + "es" : "Europa", + "fr" : "Europe", + "ja" : "ヨーロッパ", + "pt-BR" : "Europa", + "ru" : "Европа", + "zh-CN" : "欧洲" + } + }, + "country" : { + "geoname_id" : 2635167, + "iso_code" : "GB", + "names" : { + "de" : "Vereinigtes Königreich", + "en" : "United Kingdom", + "es" : "Reino Unido", + "fr" : "Royaume-Uni", + "ja" : "イギリス", + "pt-BR" : "Reino Unido", + "ru" : "Великобритания", + "zh-CN" : "英国" + } + }, + "location" : { + "accuracy_radius" : 100, + "latitude" : 51.5142, + "longitude" : -0.0931, + "time_zone" : "Europe/London" + }, + "registered_country" : { + "geoname_id" : 6252001, + "iso_code" : "US", + "names" : { + "de" : "USA", + "en" : "United States", + "es" : "Estados Unidos", + "fr" : "États-Unis", + "ja" : "アメリカ合衆国", + "pt-BR" : "Estados Unidos", + "ru" : "США", + "zh-CN" : "美国" + } + }, + "subdivisions" : [ + { + "geoname_id" : 6269131, + "iso_code" : "ENG", + "names" : { + "en" : "England", + "es" : "Inglaterra", + "fr" : "Angleterre", + "pt-BR" : "Inglaterra" + } + } + ] } - } - ] -} + }, + { + "networks": ["10.0.0.0/8"], + "data": { + "country" : { + "geoname_id" : 6255148, + "iso_code" : "NA", + "names" : { + "en" : "My private property" + } + } + } + }, + { + "networks": ["8.8.8.8/32"], + "data": { + "country" : { + "geoname_id" : 6255148, + "iso_code" : "NA", + "names" : { + "en" : "Now it's owned by ME!" + } + } + } + }, + { + "networks": ["8.8.8.8/32","10.200.0.0/24","192.168.33.13/30","127.0.0.1/32"], + "data": { + "country" : { + "geoname_id" : 6255148, + "iso_code" : "NA", + "names" : { + "en" : "Iglov's property" + } + } + } + }, + { + "networks": ["94.198.54.0/23"], + "data": { + "country" : { + "geoname_id" : 2017370, + "iso_code" : "RU", + "names" : { + "en" : "Russia" + } + } + } + } +] diff --git a/go.mod b/go.mod index 012d87c..12a7939 100644 --- a/go.mod +++ b/go.mod @@ -7,13 +7,12 @@ require ( github.com/maxmind/mmdbwriter v1.0.0 github.com/oschwald/maxminddb-golang v1.12.0 github.com/stretchr/testify v1.9.0 + go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d // indirect golang.org/x/sys v0.10.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/main.go b/main.go index 813c3c0..e802518 100644 --- a/main.go +++ b/main.go @@ -4,8 +4,9 @@ import ( "encoding/json" "flag" "fmt" + "go4.org/netipx" "log" - "net" + "net/netip" "os" "github.com/maxmind/mmdbwriter" @@ -18,39 +19,75 @@ var ( inputGeo = flag.String("i", "./GeoLite2-City.mmdb", "Input GeoLite2-City.mmdb file path.") outputGeo = flag.String("o", "./GeoLite2-City-mod.mmdb", "Output modified mmdb file path.") version = flag.Bool("v", false, "Print current version and exit.") - merge = flag.String("m", "replce", "Merge method. It may be: toplevel, recurse or replace. Default: replace") + merge = flag.String("m", "replce", "Merge strategy. It may be: toplevel, recurse or replace.") ) +// Version contains main version of build. Get from compiler variables var Version string +// Dataset is for JSON dataset mapping type Dataset struct { - Dataset []Geo `json:"data"` -} - -type Geo struct { - Ips []string `json:"ips"` - Country Country `json:"country"` -} - -type Country struct { - Iso_code string `json:"iso_code"` - Geoname_id int `json:"geoname_id"` - Names Names `json:"names"` -} - -type Names struct { - En string `json:"en"` + Networks []string `json:"networks"` + Data map[string]any `json:"data"` } +// Check func for defer functions func Check(f func() error) { if err := f(); err != nil { fmt.Println("Received error:", err) } } +// toMMDBType key converts field values read from json into their corresponding mmdbtype.DataType. +// It makes some assumptions for numeric types based on previous knowledge about field types. +func toMMDBType(key string, value any) (mmdbtype.DataType, error) { + switch v := value.(type) { + case bool: + return mmdbtype.Bool(v), nil + case string: + return mmdbtype.String(v), nil + case map[string]any: + m := mmdbtype.Map{} + for innerKey, val := range v { + innerVal, err := toMMDBType(innerKey, val) + if err != nil { + return nil, fmt.Errorf("parsing mmdbtype.Map for key %q: %w", key, err) + } + m[mmdbtype.String(innerKey)] = innerVal + } + return m, nil + case []any: + s := mmdbtype.Slice{} + for _, val := range v { + innerVal, err := toMMDBType(key, val) + if err != nil { + return nil, fmt.Errorf("parsing mmdbtype.Slice for key %q: %w", key, err) + } + s = append(s, innerVal) + } + return s, nil + case float64: + switch key { + case "accuracy_radius", "confidence", "metro_code": + return mmdbtype.Uint16(v), nil + case "autonomous_system_number", "average_income", + "geoname_id", "ipv4_24", "ipv4_32", "ipv6_32", + "ipv6_48", "ipv6_64", "population_density": + return mmdbtype.Uint32(v), nil + case "ip_risk", "latitude", "longitude", "score", + "static_ip_score": + return mmdbtype.Float64(v), nil + default: + return nil, fmt.Errorf("unsupported numeric type for key %q: %T", key, value) + } + default: + return nil, fmt.Errorf("unsupported type for key %q: %T", key, value) + } +} + func main() { - // Main dataset from json - var dataset Dataset + // main data map for json dataset + var dataset []Dataset // validate merge strategy. var mergeStrategy inserter.FuncGenerator @@ -61,19 +98,19 @@ func main() { } if *merge == "toplevel" { mergeStrategy = inserter.TopLevelMergeWith - log.Printf("Using merge method: toplevel") + log.Printf("Using merge strategy: toplevel") } else if *merge == "recurse" { mergeStrategy = inserter.DeepMergeWith - log.Printf("Using merge method: recurse") + log.Printf("Using merge strategy: recurse") } else { mergeStrategy = inserter.ReplaceWith - log.Printf("Using merge method: replace") + log.Printf("Using merge strategy: replace") } log.Printf("Loading mmdb: %v", *inputGeo) // Load the database we wish to enrich. - writer, err := mmdbwriter.Load(*inputGeo, mmdbwriter.Options{ + dbWriter, err := mmdbwriter.Load(*inputGeo, mmdbwriter.Options{ Inserter: mergeStrategy, IncludeReservedNetworks: true, Description: map[string]string{"en": fmt.Sprintf("Compiled with mmdb-editor (%v) https://github.com/iglov/mmdb-editor", Version)}, @@ -82,45 +119,40 @@ func main() { log.Fatal(err) } - content, err := os.ReadFile(*datasetGeo) + file, err := os.Open(*datasetGeo) if err != nil { - log.Fatal("Error when opening file: ", err) + log.Fatal("Opening json file error:", err) } + defer Check(file.Close) - err = json.Unmarshal(content, &dataset) - if err != nil { + if err := json.NewDecoder(file).Decode(&dataset); err != nil { log.Printf("error decoding response: %v", err) if e, ok := err.(*json.SyntaxError); ok { log.Printf("syntax error at byte offset %d", e.Offset) } - log.Printf("response: %q", content) + log.Printf("response: %v", file) log.Fatal("Error during Unmarshal(): ", err) } - log.Printf("Loaded data: %v", dataset.Dataset) - - for i := 0; i < len(dataset.Dataset); i++ { - // Define and insert the new data. - data := mmdbtype.Map{ - "country": mmdbtype.Map{ - "geoname_id": mmdbtype.Uint32(dataset.Dataset[i].Country.Geoname_id), - "iso_code": mmdbtype.String(dataset.Dataset[i].Country.Iso_code), - "names": mmdbtype.Map{ - "en": mmdbtype.String(dataset.Dataset[i].Country.Names.En), - }, - }, - } + for _, record := range dataset { + for _, network := range record.Networks { + prefix, err := netip.ParsePrefix(network) + if err != nil { + log.Fatal("Parsing networks error:", err) + } - for _, ip := range dataset.Dataset[i].Ips { - _, network, err := net.ParseCIDR(ip) + mmdbValue, err := toMMDBType(prefix.String(), record.Data) if err != nil { - log.Fatal(err) + log.Fatal("Converting value to mmdbtype error:", err) } - log.Printf("Modifying net: %q", network) - if err := writer.Insert(network, data); err != nil { + + log.Printf("Modifying net: %s", prefix.String()) + if err := dbWriter.Insert(netipx.PrefixIPNet(prefix), mmdbValue); err != nil { log.Fatal(err) } + } + } log.Printf("Compiling and writing modified data into: %v", *outputGeo) @@ -133,7 +165,7 @@ func main() { defer Check(fh.Close) - _, err = writer.WriteTo(fh) + _, err = dbWriter.WriteTo(fh) if err != nil { log.Fatal(err) } diff --git a/testdata/dataset.json b/testdata/dataset.json index 3d85b43..5837d78 100644 --- a/testdata/dataset.json +++ b/testdata/dataset.json @@ -1,24 +1,213 @@ -{ - "data": [ - { - "ips": ["94.198.54.0/23"], - "country": { - "geoname_id": 2017370, - "iso_code": "RU", - "names": { - "en": "Russia" - } +[ + { + "networks": ["214.78.120.0/22"], + "data": { + "city" : { + "geoname_id" : 5391811, + "names" : { + "de" : "San Diego", + "en" : "San Diego", + "es" : "San Diego", + "fr" : "San Diego", + "ja" : "サンディエゴ", + "pt-BR" : "San Diego", + "ru" : "Сан-Диего" + } + }, + "continent" : { + "code" : "NA", + "geoname_id" : 6255149, + "names" : { + "de" : "Nordamerika", + "en" : "North America", + "es" : "Norteamérica", + "fr" : "Amérique du Nord", + "ja" : "北アメリカ", + "pt-BR" : "América do Norte", + "ru" : "Северная Америка", + "zh-CN" : "北美洲" + } + }, + "country" : { + "geoname_id" : 6252001, + "iso_code" : "US", + "names" : { + "de" : "Vereinigte Staaten", + "en" : "United States", + "es" : "Estados Unidos", + "fr" : "États Unis", + "ja" : "アメリカ", + "pt-BR" : "EUA", + "ru" : "США", + "zh-CN" : "美国" + } + }, + "location" : { + "accuracy_radius" : 100, + "latitude" : 32.7405, + "longitude" : -117.0935, + "metro_code" : 825, + "time_zone" : "America/Los_Angeles" + }, + "postal" : { + "code" : "92105" + }, + "registered_country" : { + "geoname_id" : 6252001, + "iso_code" : "US", + "names" : { + "de" : "Vereinigte Staaten", + "en" : "United States", + "es" : "Estados Unidos", + "fr" : "États Unis", + "ja" : "アメリカ", + "pt-BR" : "EUA", + "ru" : "США", + "zh-CN" : "美国" + } + }, + "subdivisions" : [ + { + "geoname_id" : 5332921, + "iso_code" : "CA", + "names" : { + "de" : "Kalifornien", + "en" : "California", + "es" : "California", + "fr" : "Californie", + "ja" : "カリフォルニア州", + "pt-BR" : "Califórnia", + "ru" : "Калифорния", + "zh-CN" : "加州" + } + } + ] } - }, - { - "ips": ["10.200.0.0/24","192.168.33.13/30","127.0.0.1/32"], - "country": { - "geoname_id": 6255148, - "iso_code": "NA", - "names": { - "en": "Iglov's property" - } + }, + { + "networks": ["81.2.69.160/18"], + "data": { + "city" : { + "geoname_id" : 2643743, + "names" : { + "de" : "London", + "en" : "London", + "es" : "Londres", + "fr" : "Londres", + "ja" : "ロンドン", + "pt-BR" : "Londres", + "ru" : "Лондон" + } + }, + "org": "My test company", + "continent" : { + "code" : "EU", + "geoname_id" : 6255148, + "names" : { + "de" : "Europa", + "en" : "Europe", + "es" : "Europa", + "fr" : "Europe", + "ja" : "ヨーロッパ", + "pt-BR" : "Europa", + "ru" : "Европа", + "zh-CN" : "欧洲" + } + }, + "country" : { + "geoname_id" : 2635167, + "iso_code" : "GB", + "names" : { + "de" : "Vereinigtes Königreich", + "en" : "United Kingdom", + "es" : "Reino Unido", + "fr" : "Royaume-Uni", + "ja" : "イギリス", + "pt-BR" : "Reino Unido", + "ru" : "Великобритания", + "zh-CN" : "英国" + } + }, + "location" : { + "accuracy_radius" : 100, + "latitude" : 51.5142, + "longitude" : -0.0931, + "time_zone" : "Europe/London" + }, + "registered_country" : { + "geoname_id" : 6252001, + "iso_code" : "US", + "names" : { + "de" : "USA", + "en" : "United States", + "es" : "Estados Unidos", + "fr" : "États-Unis", + "ja" : "アメリカ合衆国", + "pt-BR" : "Estados Unidos", + "ru" : "США", + "zh-CN" : "美国" + } + }, + "subdivisions" : [ + { + "geoname_id" : 6269131, + "iso_code" : "ENG", + "names" : { + "en" : "England", + "es" : "Inglaterra", + "fr" : "Angleterre", + "pt-BR" : "Inglaterra" + } + } + ] } - } - ] -} + }, + { + "networks": ["10.0.0.0/8"], + "data": { + "country" : { + "geoname_id" : 6255148, + "iso_code" : "NA", + "names" : { + "en" : "My private property" + } + } + } + }, + { + "networks": ["8.8.8.8/32"], + "data": { + "country" : { + "geoname_id" : 6255148, + "iso_code" : "NA", + "names" : { + "en" : "Now it's owned by ME!" + } + } + } + }, + { + "networks": ["8.8.8.8/32","10.200.0.0/24","192.168.33.13/30","127.0.0.1/32"], + "data": { + "country" : { + "geoname_id" : 6255148, + "iso_code" : "NA", + "names" : { + "en" : "Iglov's property" + } + } + } + }, + { + "networks": ["94.198.54.0/23"], + "data": { + "country" : { + "geoname_id" : 2017370, + "iso_code" : "RU", + "names" : { + "en" : "Russia" + } + } + } + } +]