Merge pull request #61 from PDOK/cursor

Cursor encoding

go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/perimeterx/marshmallow v1.1.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sqids/sqids-go v0.4.1 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	golang.org/x/crypto v0.9.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect

go.sum

@@ -74,6 +74,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sqids/sqids-go v0.4.1 h1:eQKYzmAZbLlRwHeHYPF35QhgxwZHLnlmVj9AkIj/rrw=
+github.com/sqids/sqids-go v0.4.1/go.mod h1:EMwHuPQgSNFS0A49jESTfIQS+066XQTVhukrzEPScl8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=

ogc/features/datasources/datasource.go

@@ -8,7 +8,7 @@ import (
 type Datasource interface {
 
 	// GetFeatures returns a FeatureCollection from the underlying datasource and a Cursor for pagination
-	GetFeatures(collection string, cursor string, limit int) (*domain.FeatureCollection, domain.Cursor)
+	GetFeatures(collection string, cursor int64, limit int) (*domain.FeatureCollection, domain.Cursor)
 
 	// GetFeature returns a specific Feature from the FeatureCollection of the underlying datasource
 	GetFeature(collection string, featureID string) *domain.Feature

ogc/features/datasources/fakedb.go

@@ -3,7 +3,6 @@ package datasources
 import (
 	"fmt"
 	"sort"
-	"strconv"
 
 	"github.com/PDOK/gokoala/ogc/features/domain"
 	"github.com/brianvoe/gofakeit/v6"
@@ -28,21 +27,13 @@ func (FakeDB) Close() {
 	// noop
 }
 
-func (fdb FakeDB) GetFeatures(_ string, cursor string, limit int) (*domain.FeatureCollection, domain.Cursor) {
-	var low int
-	if cursor == "" {
-		low = 0
-	} else {
-		low, _ = strconv.Atoi(cursor)
-		if low < 0 {
-			low = 0
-		}
-	}
+func (fdb FakeDB) GetFeatures(_ string, cursor int64, limit int) (*domain.FeatureCollection, domain.Cursor) {
+	low := cursor
+	high := low + int64(limit)
 
-	high := low + limit
-	last := high > len(fdb.featureCollection.Features)
+	last := high > int64(len(fdb.featureCollection.Features))
 	if last {
-		high = len(fdb.featureCollection.Features)
+		high = int64(len(fdb.featureCollection.Features))
 	}
 	if high < 0 {
 		high = 0
@@ -77,7 +68,7 @@ func generateFakeFeatureCollection() *domain.FeatureCollection {
 			"purpose":    gofakeit.Blurb(),
 
 			// we use an explicit cursor column in our fake data to keep things simple
-			cursorColumnName: i,
+			cursorColumnName: int64(i),
 		}
 
 		feature := domain.Feature{}
@@ -90,7 +81,7 @@ func generateFakeFeatureCollection() *domain.FeatureCollection {
 
 	// the collection must be ordered by the cursor column
 	sort.Slice(feats, func(i, j int) bool {
-		return feats[i].Properties[cursorColumnName].(int) < feats[j].Properties[cursorColumnName].(int)
+		return feats[i].Properties[cursorColumnName].(int64) < feats[j].Properties[cursorColumnName].(int64)
 	})
 
 	fc := domain.FeatureCollection{}

ogc/features/datasources/geopackage.go

@@ -17,9 +17,9 @@ func (GeoPackage) Close() {
 	// TODO: clean up DB connection to gpkg
 }
 
-func (GeoPackage) GetFeatures(collection string, cursor string, limit int) (*domain.FeatureCollection, domain.Cursor) {
+func (GeoPackage) GetFeatures(collection string, cursor int64, limit int) (*domain.FeatureCollection, domain.Cursor) {
 	// TODO: not implemented yet
-	log.Printf("TODO: return data from gpkg for collection %s using cursor %s with limt %d",
+	log.Printf("TODO: return data from gpkg for collection %s using cursor %d with limt %d",
 		collection, cursor, limit)
 	return nil, domain.Cursor{}
 }

ogc/features/domain/domain.go

@@ -2,8 +2,22 @@ package domain
 
 import (
 	"log"
+	"strconv"
 
 	"github.com/go-spatial/geom/encoding/geojson"
+	"github.com/sqids/sqids-go"
+)
+
+const (
+	cursorAlphabet = "1Vti5BYcjOdTXunDozKPm4syvG6galxLM8eIrUS2bWqZCNkwpR309JFAHfh7EQ" // generated on https://sqids.org/playground
+)
+
+var (
+	cursorCodec, _ = sqids.New(sqids.Options{
+		Alphabet:  cursorAlphabet,
+		Blocklist: nil, // disable blocklist
+		MinLength: 8,
+	})
 )
 
 // featureCollectionType allows the GeoJSON type to be automatically set during json marshalling
@@ -46,8 +60,8 @@ type Link struct {
 
 // Cursor since we use cursor-based pagination as opposed to offset-based pagination
 type Cursor struct {
-	Prev int
-	Next int
+	Prev EncodedCursor
+	Next EncodedCursor
 
 	IsFirst bool
 	IsLast  bool
@@ -57,7 +71,7 @@ func NewCursor(features []*Feature, column string, limit int, last bool) Cursor
 	if len(features) == 0 {
 		return Cursor{}
 	}
-	max := len(features) - 1
+	max := int64(len(features) - 1)
 
 	start := features[0].Properties[column]
 	end := features[max].Properties[column]
@@ -71,20 +85,49 @@ func NewCursor(features []*Feature, column string, limit int, last bool) Cursor
 		end = 0
 	}
 
-	prev := start.(int)
+	prev := start.(int64)
 	if prev != 0 {
 		prev -= max
 		if prev < 0 {
 			prev = 0
 		}
 	}
-	next := end.(int)
+	next := end.(int64)
 
 	return Cursor{
-		Prev: prev,
-		Next: next,
+		Prev: encodeCursor(prev),
+		Next: encodeCursor(next),
 
-		IsFirst: next < limit,
+		IsFirst: next < int64(limit),
 		IsLast:  last,
 	}
 }
+
+// EncodedCursor is a scrambled string representation of a consecutive ordered integer cursor
+type EncodedCursor string
+
+func encodeCursor(value int64) EncodedCursor {
+	encodedValue, err := cursorCodec.Encode([]uint64{uint64(value)})
+	if err != nil {
+		log.Printf("failed to encode cursor value %d, defaulting to unencoded value.", value)
+		return EncodedCursor(strconv.FormatInt(value, 10))
+	}
+	return EncodedCursor(encodedValue)
+}
+
+func (c EncodedCursor) Decode() int64 {
+	value := string(c)
+	if value == "" {
+		return 0
+	}
+	decodedValue := cursorCodec.Decode(value)
+	if len(decodedValue) > 1 {
+		log.Printf("encountered more than one cursor value after decoding: '%v', "+
+			"this is not allowed! Defaulting to first value.", decodedValue)
+	}
+	if len(decodedValue) == 0 {
+		log.Printf("decoding cursor value '%v' failed, defaulting to first page", decodedValue)
+		return 0
+	}
+	return int64(decodedValue[0])
+}

ogc/features/json.go

@@ -70,15 +70,15 @@ func (jf *jsonFeatures) createFeatureCollectionLinks(collectionID string, cursor
 			Rel:   "next",
 			Title: "Next page",
 			Type:  engine.MediaTypeGeoJSON,
-			Href:  fmt.Sprintf("%s?f=json&cursor=%d&limit=%d", featuresBaseURL, cursor.Next, limit),
+			Href:  fmt.Sprintf("%s?f=json&cursor=%s&limit=%d", featuresBaseURL, cursor.Next, limit),
 		})
 	}
 	if !cursor.IsFirst {
 		links = append(links, domain.Link{
 			Rel:   "prev",
 			Title: "Previous page",
 			Type:  engine.MediaTypeGeoJSON,
-			Href:  fmt.Sprintf("%s?f=json&cursor=%d&limit=%d", featuresBaseURL, cursor.Prev, limit),
+			Href:  fmt.Sprintf("%s?f=json&cursor=%s&limit=%d", featuresBaseURL, cursor.Prev, limit),
 		})
 	}
 	return links

ogc/features/main.go

@@ -9,6 +9,7 @@ import (
 	"github.com/PDOK/gokoala/engine"
 	"github.com/PDOK/gokoala/ogc/common/geospatial"
 	"github.com/PDOK/gokoala/ogc/features/datasources"
+	"github.com/PDOK/gokoala/ogc/features/domain"
 	"github.com/go-chi/chi/v5"
 )
 
@@ -55,7 +56,7 @@ func NewFeatures(e *engine.Engine, router *chi.Mux) *Features {
 func (f *Features) CollectionContent() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		collectionID := chi.URLParam(r, "collectionId")
-		cursorParam := r.URL.Query().Get("cursor")
+		encodedCursor := domain.EncodedCursor(r.URL.Query().Get("cursor"))
 		limit, err := getLimit(r)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusBadRequest)
@@ -71,7 +72,7 @@ func (f *Features) CollectionContent() http.HandlerFunc {
 			return
 		}
 
-		fc, cursor := f.datasource.GetFeatures(collectionID, cursorParam, limit)
+		fc, cursor := f.datasource.GetFeatures(collectionID, encodedCursor.Decode(), limit)
 		if fc == nil {
 			http.NotFound(w, r)
 			return

ogc/features/main_test.go

@@ -74,7 +74,7 @@ func TestFeatures_CollectionContent(t *testing.T) {
 			name: "Request GeoJSON for 'foo' collection using limit of 2 and cursor to next page",
 			fields: fields{
 				configFile:   "ogc/features/testdata/config_features.yaml",
-				url:          "http://localhost:8080/collections/:collectionId/items?cursor=9&limit=2",
+				url:          "http://localhost:8080/collections/tunneldelen/items?f=json&cursor=iUMnUmcz&limit=2",
 				collectionID: "foo",
 				format:       "json",
 			},

ogc/features/testdata/expected_foo_collection.json

@@ -16,7 +16,7 @@
       "rel": "next",
       "title": "Next page",
       "type": "application/geo+json",
-      "href": "http://localhost:8080/collections/foo/items?f=json\u0026cursor=9\u0026limit=10"
+      "href": "http://localhost:8080/collections/foo/items?f=json&cursor=iUMnUmcz&limit=10"
     }
   ],
   "numberReturned": 10,

ogc/features/testdata/expected_foo_collection_with_cursor.json

@@ -16,13 +16,13 @@
       "rel": "next",
       "title": "Next page",
       "type": "application/geo+json",
-      "href": "http://localhost:8080/collections/foo/items?f=json\u0026cursor=10\u0026limit=2"
+      "href": "http://localhost:8080/collections/foo/items?f=json&cursor=9GxZGfBr&limit=2"
     },
     {
       "rel": "prev",
       "title": "Previous page",
       "type": "application/geo+json",
-      "href": "http://localhost:8080/collections/foo/items?f=json\u0026cursor=8\u0026limit=2"
+      "href": "http://localhost:8080/collections/foo/items?f=json&cursor=VCHYvtZJ&limit=2"
     }
   ],
   "numberReturned": 2,

ogc/features/testdata/expected_foo_collection_with_limit.json

@@ -16,7 +16,7 @@
       "rel": "next",
       "title": "Next page",
       "type": "application/geo+json",
-      "href": "http://localhost:8080/collections/foo/items?f=json\u0026cursor=1\u0026limit=2"
+      "href": "http://localhost:8080/collections/foo/items?f=json&cursor=KXzJBocg&limit=2"
     }
   ],
   "numberReturned": 2,