diff --git a/go.mod b/go.mod index fc20197c..fd0d24ff 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/perimeterx/marshmallow v1.1.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sqids/sqids-go v0.4.1 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/crypto v0.9.0 // indirect golang.org/x/sys v0.8.0 // indirect diff --git a/go.sum b/go.sum index 6dfcb1fc..ff611674 100644 --- a/go.sum +++ b/go.sum @@ -74,6 +74,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sqids/sqids-go v0.4.1 h1:eQKYzmAZbLlRwHeHYPF35QhgxwZHLnlmVj9AkIj/rrw= +github.com/sqids/sqids-go v0.4.1/go.mod h1:EMwHuPQgSNFS0A49jESTfIQS+066XQTVhukrzEPScl8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/ogc/features/datasources/datasource.go b/ogc/features/datasources/datasource.go index 560f9d38..f65e5a3d 100644 --- a/ogc/features/datasources/datasource.go +++ b/ogc/features/datasources/datasource.go @@ -8,7 +8,7 @@ import ( type Datasource interface { // GetFeatures returns a FeatureCollection from the underlying datasource and a Cursor for pagination - GetFeatures(collection string, cursor string, limit int) (*domain.FeatureCollection, domain.Cursor) + GetFeatures(collection string, cursor int64, limit int) (*domain.FeatureCollection, domain.Cursor) // GetFeature returns a specific Feature from the FeatureCollection of the underlying datasource GetFeature(collection string, featureID string) *domain.Feature diff --git a/ogc/features/datasources/fakedb.go b/ogc/features/datasources/fakedb.go index 7027487d..a706e6e6 100644 --- a/ogc/features/datasources/fakedb.go +++ b/ogc/features/datasources/fakedb.go @@ -3,7 +3,6 @@ package datasources import ( "fmt" "sort" - "strconv" "github.com/PDOK/gokoala/ogc/features/domain" "github.com/brianvoe/gofakeit/v6" @@ -28,21 +27,13 @@ func (FakeDB) Close() { // noop } -func (fdb FakeDB) GetFeatures(_ string, cursor string, limit int) (*domain.FeatureCollection, domain.Cursor) { - var low int - if cursor == "" { - low = 0 - } else { - low, _ = strconv.Atoi(cursor) - if low < 0 { - low = 0 - } - } +func (fdb FakeDB) GetFeatures(_ string, cursor int64, limit int) (*domain.FeatureCollection, domain.Cursor) { + low := cursor + high := low + int64(limit) - high := low + limit - last := high > len(fdb.featureCollection.Features) + last := high > int64(len(fdb.featureCollection.Features)) if last { - high = len(fdb.featureCollection.Features) + high = int64(len(fdb.featureCollection.Features)) } if high < 0 { high = 0 @@ -77,7 +68,7 @@ func generateFakeFeatureCollection() *domain.FeatureCollection { "purpose": gofakeit.Blurb(), // we use an explicit cursor column in our fake data to keep things simple - cursorColumnName: i, + cursorColumnName: int64(i), } feature := domain.Feature{} @@ -90,7 +81,7 @@ func generateFakeFeatureCollection() *domain.FeatureCollection { // the collection must be ordered by the cursor column sort.Slice(feats, func(i, j int) bool { - return feats[i].Properties[cursorColumnName].(int) < feats[j].Properties[cursorColumnName].(int) + return feats[i].Properties[cursorColumnName].(int64) < feats[j].Properties[cursorColumnName].(int64) }) fc := domain.FeatureCollection{} diff --git a/ogc/features/datasources/geopackage.go b/ogc/features/datasources/geopackage.go index ead2516a..068c8775 100644 --- a/ogc/features/datasources/geopackage.go +++ b/ogc/features/datasources/geopackage.go @@ -17,9 +17,9 @@ func (GeoPackage) Close() { // TODO: clean up DB connection to gpkg } -func (GeoPackage) GetFeatures(collection string, cursor string, limit int) (*domain.FeatureCollection, domain.Cursor) { +func (GeoPackage) GetFeatures(collection string, cursor int64, limit int) (*domain.FeatureCollection, domain.Cursor) { // TODO: not implemented yet - log.Printf("TODO: return data from gpkg for collection %s using cursor %s with limt %d", + log.Printf("TODO: return data from gpkg for collection %s using cursor %d with limt %d", collection, cursor, limit) return nil, domain.Cursor{} } diff --git a/ogc/features/domain/domain.go b/ogc/features/domain/domain.go index 1a985c17..aae1011d 100644 --- a/ogc/features/domain/domain.go +++ b/ogc/features/domain/domain.go @@ -2,8 +2,22 @@ package domain import ( "log" + "strconv" "github.com/go-spatial/geom/encoding/geojson" + "github.com/sqids/sqids-go" +) + +const ( + cursorAlphabet = "1Vti5BYcjOdTXunDozKPm4syvG6galxLM8eIrUS2bWqZCNkwpR309JFAHfh7EQ" // generated on https://sqids.org/playground +) + +var ( + cursorCodec, _ = sqids.New(sqids.Options{ + Alphabet: cursorAlphabet, + Blocklist: nil, // disable blocklist + MinLength: 8, + }) ) // featureCollectionType allows the GeoJSON type to be automatically set during json marshalling @@ -46,8 +60,8 @@ type Link struct { // Cursor since we use cursor-based pagination as opposed to offset-based pagination type Cursor struct { - Prev int - Next int + Prev EncodedCursor + Next EncodedCursor IsFirst bool IsLast bool @@ -57,7 +71,7 @@ func NewCursor(features []*Feature, column string, limit int, last bool) Cursor if len(features) == 0 { return Cursor{} } - max := len(features) - 1 + max := int64(len(features) - 1) start := features[0].Properties[column] end := features[max].Properties[column] @@ -71,20 +85,49 @@ func NewCursor(features []*Feature, column string, limit int, last bool) Cursor end = 0 } - prev := start.(int) + prev := start.(int64) if prev != 0 { prev -= max if prev < 0 { prev = 0 } } - next := end.(int) + next := end.(int64) return Cursor{ - Prev: prev, - Next: next, + Prev: encodeCursor(prev), + Next: encodeCursor(next), - IsFirst: next < limit, + IsFirst: next < int64(limit), IsLast: last, } } + +// EncodedCursor is a scrambled string representation of a consecutive ordered integer cursor +type EncodedCursor string + +func encodeCursor(value int64) EncodedCursor { + encodedValue, err := cursorCodec.Encode([]uint64{uint64(value)}) + if err != nil { + log.Printf("failed to encode cursor value %d, defaulting to unencoded value.", value) + return EncodedCursor(strconv.FormatInt(value, 10)) + } + return EncodedCursor(encodedValue) +} + +func (c EncodedCursor) Decode() int64 { + value := string(c) + if value == "" { + return 0 + } + decodedValue := cursorCodec.Decode(value) + if len(decodedValue) > 1 { + log.Printf("encountered more than one cursor value after decoding: '%v', "+ + "this is not allowed! Defaulting to first value.", decodedValue) + } + if len(decodedValue) == 0 { + log.Printf("decoding cursor value '%v' failed, defaulting to first page", decodedValue) + return 0 + } + return int64(decodedValue[0]) +} diff --git a/ogc/features/json.go b/ogc/features/json.go index e185f392..dd6cd576 100644 --- a/ogc/features/json.go +++ b/ogc/features/json.go @@ -70,7 +70,7 @@ func (jf *jsonFeatures) createFeatureCollectionLinks(collectionID string, cursor Rel: "next", Title: "Next page", Type: engine.MediaTypeGeoJSON, - Href: fmt.Sprintf("%s?f=json&cursor=%d&limit=%d", featuresBaseURL, cursor.Next, limit), + Href: fmt.Sprintf("%s?f=json&cursor=%s&limit=%d", featuresBaseURL, cursor.Next, limit), }) } if !cursor.IsFirst { @@ -78,7 +78,7 @@ func (jf *jsonFeatures) createFeatureCollectionLinks(collectionID string, cursor Rel: "prev", Title: "Previous page", Type: engine.MediaTypeGeoJSON, - Href: fmt.Sprintf("%s?f=json&cursor=%d&limit=%d", featuresBaseURL, cursor.Prev, limit), + Href: fmt.Sprintf("%s?f=json&cursor=%s&limit=%d", featuresBaseURL, cursor.Prev, limit), }) } return links diff --git a/ogc/features/main.go b/ogc/features/main.go index 1bcd7164..17c20f3a 100644 --- a/ogc/features/main.go +++ b/ogc/features/main.go @@ -9,6 +9,7 @@ import ( "github.com/PDOK/gokoala/engine" "github.com/PDOK/gokoala/ogc/common/geospatial" "github.com/PDOK/gokoala/ogc/features/datasources" + "github.com/PDOK/gokoala/ogc/features/domain" "github.com/go-chi/chi/v5" ) @@ -55,7 +56,7 @@ func NewFeatures(e *engine.Engine, router *chi.Mux) *Features { func (f *Features) CollectionContent() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { collectionID := chi.URLParam(r, "collectionId") - cursorParam := r.URL.Query().Get("cursor") + encodedCursor := domain.EncodedCursor(r.URL.Query().Get("cursor")) limit, err := getLimit(r) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -71,7 +72,7 @@ func (f *Features) CollectionContent() http.HandlerFunc { return } - fc, cursor := f.datasource.GetFeatures(collectionID, cursorParam, limit) + fc, cursor := f.datasource.GetFeatures(collectionID, encodedCursor.Decode(), limit) if fc == nil { http.NotFound(w, r) return diff --git a/ogc/features/main_test.go b/ogc/features/main_test.go index 1893a5e3..84a5dab7 100644 --- a/ogc/features/main_test.go +++ b/ogc/features/main_test.go @@ -74,7 +74,7 @@ func TestFeatures_CollectionContent(t *testing.T) { name: "Request GeoJSON for 'foo' collection using limit of 2 and cursor to next page", fields: fields{ configFile: "ogc/features/testdata/config_features.yaml", - url: "http://localhost:8080/collections/:collectionId/items?cursor=9&limit=2", + url: "http://localhost:8080/collections/tunneldelen/items?f=json&cursor=iUMnUmcz&limit=2", collectionID: "foo", format: "json", }, diff --git a/ogc/features/testdata/expected_foo_collection.json b/ogc/features/testdata/expected_foo_collection.json index 75e54b64..d4b3a6fd 100644 --- a/ogc/features/testdata/expected_foo_collection.json +++ b/ogc/features/testdata/expected_foo_collection.json @@ -16,7 +16,7 @@ "rel": "next", "title": "Next page", "type": "application/geo+json", - "href": "http://localhost:8080/collections/foo/items?f=json\u0026cursor=9\u0026limit=10" + "href": "http://localhost:8080/collections/foo/items?f=json&cursor=iUMnUmcz&limit=10" } ], "numberReturned": 10, diff --git a/ogc/features/testdata/expected_foo_collection_with_cursor.json b/ogc/features/testdata/expected_foo_collection_with_cursor.json index 1d19ef5f..3529645f 100644 --- a/ogc/features/testdata/expected_foo_collection_with_cursor.json +++ b/ogc/features/testdata/expected_foo_collection_with_cursor.json @@ -16,13 +16,13 @@ "rel": "next", "title": "Next page", "type": "application/geo+json", - "href": "http://localhost:8080/collections/foo/items?f=json\u0026cursor=10\u0026limit=2" + "href": "http://localhost:8080/collections/foo/items?f=json&cursor=9GxZGfBr&limit=2" }, { "rel": "prev", "title": "Previous page", "type": "application/geo+json", - "href": "http://localhost:8080/collections/foo/items?f=json\u0026cursor=8\u0026limit=2" + "href": "http://localhost:8080/collections/foo/items?f=json&cursor=VCHYvtZJ&limit=2" } ], "numberReturned": 2, diff --git a/ogc/features/testdata/expected_foo_collection_with_limit.json b/ogc/features/testdata/expected_foo_collection_with_limit.json index 99630258..921adc0a 100644 --- a/ogc/features/testdata/expected_foo_collection_with_limit.json +++ b/ogc/features/testdata/expected_foo_collection_with_limit.json @@ -16,7 +16,7 @@ "rel": "next", "title": "Next page", "type": "application/geo+json", - "href": "http://localhost:8080/collections/foo/items?f=json\u0026cursor=1\u0026limit=2" + "href": "http://localhost:8080/collections/foo/items?f=json&cursor=KXzJBocg&limit=2" } ], "numberReturned": 2,