use badger for tiles

server/.gitignore

@@ -2,6 +2,7 @@ mapserver
 world.mt
 output
 map.sqlite
+mapserver.tiles
 mapserver.sqlite
 mapserver.sqlite-journal
 mapserver.json

server/app/app.go

@@ -9,6 +9,7 @@ import (
 	"mapserver/mapobjectdb"
 	"mapserver/params"
 	"mapserver/settings"
+	"mapserver/tiledb"
 	"mapserver/tilerenderer"
 )
 
@@ -23,6 +24,7 @@ type App struct {
 
 	Blockdb  db.DBAccessor
 	Objectdb mapobjectdb.DBAccessor
+	TileDB   *tiledb.TileDB
 	Settings *settings.Settings
 
 	BlockAccessor    *mapblockaccessor.MapBlockAccessor

server/app/setup.go

@@ -9,6 +9,7 @@ import (
 	sqliteobjdb "mapserver/mapobjectdb/sqlite"
 	"mapserver/params"
 	"mapserver/settings"
+	"mapserver/tiledb"
 	"mapserver/tilerenderer"
 	"mapserver/worldconfig"
 
@@ -95,20 +96,27 @@ func Setup(p params.ParamsType, cfg *Config) *App {
 		panic(err)
 	}
 
-	//migrate tile database
+	//migrate object database
 	err = a.Objectdb.Migrate()
 
 	if err != nil {
 		panic(err)
 	}
 
+	//create tiledb
+	a.TileDB, err = tiledb.New("./mapserver.tiles")
+
+	if err != nil {
+		panic(err)
+	}
+
 	//settings
 	a.Settings = settings.New(a.Objectdb)
 
 	//setup tile renderer
 	a.Tilerenderer = tilerenderer.NewTileRenderer(
 		a.Mapblockrenderer,
-		a.Objectdb,
+		a.TileDB,
 		a.Blockdb,
 		a.Config.Layers,
 	)

server/go.mod

@@ -1,12 +1,16 @@
 module mapserver
 
 require (
+	github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 // indirect
+	github.com/dgraph-io/badger v1.5.4
+	github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f // indirect
 	github.com/disintegration/imaging v1.5.0
 	github.com/gorilla/websocket v1.4.0
 	github.com/lib/pq v1.0.0
 	github.com/mattn/go-sqlite3 v1.10.0
 	github.com/mjibson/esc v0.1.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/pkg/errors v0.8.1 // indirect
 	github.com/prometheus/client_golang v0.9.2
 	github.com/sirupsen/logrus v1.3.0
 	github.com/stretchr/testify v1.2.2

server/go.sum

@@ -1,3 +1,5 @@
+github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 h1:PqzgE6kAMi81xWQA2QIVxjWkFHptGgC547vchpUbtFo=
+github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -5,6 +7,10 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgraph-io/badger v1.5.4 h1:gVTrpUTbbr/T24uvoCaqY2KSHfNLVGm0w+hbee2HMeg=
+github.com/dgraph-io/badger v1.5.4/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
+github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA=
+github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/disintegration/imaging v1.5.0 h1:uYqUhwNmLU4K1FN44vhqS4TZJRAA4RhBINgbQlKyGi0=
 github.com/disintegration/imaging v1.5.0/go.mod h1:9B/deIUIrliYkyMTuXJd6OUFLcrZ2tf+3Qlwnaf/CjU=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@@ -23,6 +29,8 @@ github.com/mjibson/esc v0.1.0 h1:5ch+murgrcwDFLOE2hwj0f7kE4xJfJhkSCAjSLY182o=
 github.com/mjibson/esc v0.1.0/go.mod h1:9Hw9gxxfHulMF5OJKCyhYD7PzlSdhzXyaGEBRPH1OPs=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
@@ -44,6 +52,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/image v0.0.0-20190118043309-183bebdce1b2 h1:FNSSV4jv1PrPsiM2iKGpqLPPgYACqh9Muav7Pollk1k=
 golang.org/x/image v0.0.0-20190118043309-183bebdce1b2/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8=

server/mapobjectdb/accessor.go

@@ -68,19 +68,11 @@ type DBAccessor interface {
 	//migrates the database
 	Migrate() error
 
-	//true = speed,unsafe / false = safe
-	EnableSpeedSafetyTradeoff(enableSpeed bool) error
-
 	//Generic map objects (poi, etc)
 	GetMapData(q SearchQuery) ([]*MapObject, error)
 	RemoveMapData(pos *coords.MapBlockCoords) error
 	AddMapData(data *MapObject) error
 
-	//tile data
-	GetTile(pos *coords.TileCoords) (*Tile, error)
-	SetTile(tile *Tile) error
-	RemoveTile(pos *coords.TileCoords) error
-
 	//Settings
 	GetSetting(key string, defaultvalue string) (string, error)
 	SetSetting(key string, value string) error

server/mapobjectdb/sqlite/mapobjects_test.go

@@ -26,44 +26,6 @@ func TestMigrate(t *testing.T) {
 		panic(err)
 	}
 
-	pos := coords.NewTileCoords(0, 0, 13, 0)
-	tile, err := db.GetTile(pos)
-	if err != nil {
-		panic(err)
-	}
-
-	if tile != nil {
-		t.Fatal("non-empty tile found")
-	}
-
-	data := []byte{0x01, 0x02}
-	tile2 := mapobjectdb.Tile{Pos: pos, Data: data}
-	err = db.SetTile(&tile2)
-
-	if err != nil {
-		panic(err)
-	}
-
-	tile3, err := db.GetTile(pos)
-
-	if err != nil {
-		panic(err)
-	}
-
-	if tile3 == nil {
-		t.Fatal("no data returned")
-	}
-
-	if len(tile2.Data) != len(tile3.Data) {
-		t.Fatal("inserted data does not match")
-	}
-
-	err = db.SetTile(&tile2)
-
-	if err != nil {
-		panic(err)
-	}
-
 }
 
 func TestMapObjects(t *testing.T) {

server/mapobjectdb/sqlite/sql.go

@@ -30,16 +30,6 @@ create table if not exists object_attributes(
 
 create index if not exists object_attributes_key_value on object_attributes(key, value);
 
-create table if not exists tiles(
-  data blob,
-  mtime bigint,
-  layerid int,
-  x int,
-  y int,
-  zoom int,
-  primary key(x,y,zoom,layerid)
-);
-
 create table if not exists settings(
 	key varchar primary key not null,
 	value varchar not null
@@ -75,24 +65,6 @@ object_attributes(objectid, key, value)
 values(?, ?, ?)
 `
 
-const getTileQuery = `
-select data,mtime from tiles t
-where t.layerid = ?
-and t.x = ?
-and t.y = ?
-and t.zoom = ?
-`
-
-const setTileQuery = `
-insert or replace into tiles(x,y,zoom,layerid,data,mtime)
-values(?, ?, ?, ?, ?, ?)
-`
-
-const removeTileQuery = `
-delete from tiles
-where x = ? and y = ? and zoom = ? and layerid = ?
-`
-
 const getSettingQuery = `
 select value from settings where key = ?
 `

server/mapobjectdb/sqlite/tiles.go

@@ -1,49 +0,0 @@
-package sqlite
-
-import (
-	"mapserver/coords"
-	"mapserver/mapobjectdb"
-)
-
-func (db *Sqlite3Accessor) GetTile(pos *coords.TileCoords) (*mapobjectdb.Tile, error) {
-	rows, err := db.db.Query(getTileQuery, pos.LayerId, pos.X, pos.Y, pos.Zoom)
-	if err != nil {
-		return nil, err
-	}
-
-	defer rows.Close()
-
-	if rows.Next() {
-		var data []byte
-		var mtime int64
-
-		err = rows.Scan(&data, &mtime)
-		if err != nil {
-			return nil, err
-		}
-
-		if data == nil {
-			return nil, nil
-		}
-
-		mb := mapobjectdb.Tile{
-			Pos:   pos,
-			Data:  data,
-			Mtime: mtime,
-		}
-
-		return &mb, nil
-	}
-
-	return nil, nil
-}
-
-func (db *Sqlite3Accessor) SetTile(tile *mapobjectdb.Tile) error {
-	_, err := db.db.Exec(setTileQuery, tile.Pos.X, tile.Pos.Y, tile.Pos.Zoom, tile.Pos.LayerId, tile.Data, tile.Mtime)
-	return err
-}
-
-func (db *Sqlite3Accessor) RemoveTile(pos *coords.TileCoords) error {
-	_, err := db.db.Exec(removeTileQuery, pos.X, pos.Y, pos.Zoom, pos.LayerId)
-	return err
-}

server/tiledb/tiledb.go

@@ -0,0 +1,65 @@
+package tiledb
+
+import (
+	"fmt"
+	"mapserver/coords"
+
+	"github.com/dgraph-io/badger"
+)
+
+func New(path string) (*TileDB, error) {
+	opts := badger.DefaultOptions
+	opts.Dir = path
+	opts.ValueDir = path
+	db, err := badger.Open(opts)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &TileDB{
+		db: db,
+	}, nil
+}
+
+type TileDB struct {
+	db *badger.DB
+}
+
+func getKey(pos *coords.TileCoords) []byte {
+	return []byte(fmt.Sprintf("%d/%d/%d/%d", pos.X, pos.Y, pos.Zoom, pos.LayerId))
+}
+
+func (this *TileDB) GetTile(pos *coords.TileCoords) ([]byte, error) {
+	var tile []byte
+	err := this.db.View(func(txn *badger.Txn) error {
+		item, err := txn.Get(getKey(pos))
+		if item != nil {
+			tile, err = item.ValueCopy(nil)
+		}
+		return err
+	})
+	if err != nil {
+		return nil, nil
+	}
+
+	return tile, err
+}
+
+func (this *TileDB) SetTile(pos *coords.TileCoords, tile []byte) error {
+	err := this.db.Update(func(txn *badger.Txn) error {
+		err := txn.Set(getKey(pos), tile)
+		return err
+	})
+
+	return err
+}
+
+func (this *TileDB) RemoveTile(pos *coords.TileCoords) error {
+	err := this.db.Update(func(txn *badger.Txn) error {
+		err := txn.Delete(getKey(pos))
+		return err
+	})
+
+	return err
+}

server/tiledb/tiledb_test.go

@@ -0,0 +1,59 @@
+package tiledb
+
+import (
+	"io/ioutil"
+	"mapserver/coords"
+	"os"
+	"testing"
+)
+
+func TestTileDB(t *testing.T) {
+	tmpfile, err := ioutil.TempDir("", "TestTileDB.*.badger")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(tmpfile)
+
+	db, err := New(tmpfile)
+	if err != nil {
+		panic(err)
+	}
+
+	c := coords.NewTileCoords(0, 0, 1, 2)
+
+	err = db.SetTile(c, []byte{1, 2, 3})
+	if err != nil {
+		panic(err)
+	}
+
+	tile, err := db.GetTile(c)
+	if err != nil {
+		panic(err)
+	}
+
+	if len(tile) != 3 {
+		t.Error("wrong size")
+	}
+
+	db.RemoveTile(c)
+
+	tile, err = db.GetTile(c)
+	if err != nil {
+		panic(err)
+	}
+
+	if tile != nil {
+		t.Error("tile not removed")
+	}
+
+	c2 := coords.NewTileCoords(1, 0, 1, 2)
+	tile, err = db.GetTile(c2)
+	if err != nil {
+		panic(err)
+	}
+
+	if tile != nil {
+		t.Error("tile exists")
+	}
+
+}

server/tilerenderer/renderer.go

@@ -11,7 +11,7 @@ import (
 	"mapserver/eventbus"
 	"mapserver/layer"
 	"mapserver/mapblockrenderer"
-	"mapserver/mapobjectdb"
+	"mapserver/tiledb"
 	"time"
 
 	"github.com/disintegration/imaging"
@@ -21,13 +21,13 @@ import (
 type TileRenderer struct {
 	mapblockrenderer *mapblockrenderer.MapBlockRenderer
 	layers           []layer.Layer
-	tdb              mapobjectdb.DBAccessor
+	tdb              *tiledb.TileDB
 	dba              db.DBAccessor
 	Eventbus         *eventbus.Eventbus
 }
 
 func NewTileRenderer(mapblockrenderer *mapblockrenderer.MapBlockRenderer,
-	tdb mapobjectdb.DBAccessor,
+	tdb *tiledb.TileDB,
 	dba db.DBAccessor,
 	layers []layer.Layer) *TileRenderer {
 
@@ -69,7 +69,7 @@ func (tr *TileRenderer) RenderImage(tc *coords.TileCoords, recursionDepth int) (
 		}
 
 		if cachedtile != nil {
-			reader := bytes.NewReader(cachedtile.Data)
+			reader := bytes.NewReader(cachedtile)
 			cachedimg, err := png.Decode(reader)
 			if err != nil {
 				return nil, nil, err
@@ -84,7 +84,7 @@ func (tr *TileRenderer) RenderImage(tc *coords.TileCoords, recursionDepth int) (
 			draw.Draw(img, rect, cachedimg, image.ZP, draw.Src)
 
 			log.WithFields(logrus.Fields{"x": tc.X, "y": tc.Y, "zoom": tc.Zoom}).Debug("Cached image")
-			return img, cachedtile.Data, nil
+			return img, cachedtile, nil
 		}
 	}
 
@@ -212,8 +212,8 @@ func (tr *TileRenderer) RenderImage(tc *coords.TileCoords, recursionDepth int) (
 	encode := t.Sub(start)
 	start = t
 
-	tile := mapobjectdb.Tile{Pos: tc, Data: buf.Bytes(), Mtime: time.Now().Unix()}
+	tile := buf.Bytes()
-	tr.tdb.SetTile(&tile)
+	tr.tdb.SetTile(tc, tile)
 
 	t = time.Now()
 	cache := t.Sub(start)
@@ -223,7 +223,7 @@ func (tr *TileRenderer) RenderImage(tc *coords.TileCoords, recursionDepth int) (
 		"Y":          tc.Y,
 		"Zoom":       tc.Zoom,
 		"LayerId":    tc.LayerId,
-		"size":       len(tile.Data),
+		"size":       len(tile),
 		"quadrender": quadrender,
 		"quadresize": quadresize,
 		"encode":     encode,

server/tilerenderer/renderer_test.go

@@ -9,8 +9,8 @@ import (
 	"mapserver/layer"
 	"mapserver/mapblockaccessor"
 	"mapserver/mapblockrenderer"
-	sqliteobjdb "mapserver/mapobjectdb/sqlite"
 	"mapserver/testutils"
+	"mapserver/tiledb"
 	"os"
 	"testing"
 
@@ -47,11 +47,10 @@ func TestTileRender(t *testing.T) {
 
 	r := mapblockrenderer.NewMapBlockRenderer(cache, c)
 
-	tiletmpfile, err := ioutil.TempFile("", "TestTileRenderTiles.*.sqlite")
+	tiletmpdir, err := ioutil.TempDir("", "TestTileRenderTiles.*.sqlite")
-	defer os.Remove(tiletmpfile.Name())
+	defer os.RemoveAll(tiletmpdir)
 
-	tdb, _ := sqliteobjdb.New(tiletmpfile.Name())
+	tdb, _ := tiledb.New(tiletmpdir)
-	tdb.Migrate()
 
 	layers := []layer.Layer{
 		layer.Layer{

server/tilerendererjob/job.go

@@ -16,20 +16,7 @@ func Job(ctx *app.App) {
 	}
 
 	if ctx.Settings.GetBool(settings.SETTING_INITIAL_RUN, true) {
-		//fast, unsafe mode
-		err := ctx.Objectdb.EnableSpeedSafetyTradeoff(true)
-		if err != nil {
-			panic(err)
-		}
-
 		initialRender(ctx, jobs)
-
-		//normal, safe mode
-		err = ctx.Objectdb.EnableSpeedSafetyTradeoff(false)
-		if err != nil {
-			panic(err)
-		}
-
 	}
 
 	incrementalRender(ctx, jobs)

server/web/tiles.go

@@ -1,7 +1,6 @@
 package web
 
 import (
-	"github.com/prometheus/client_golang/prometheus"
 	"image/color"
 	"mapserver/app"
 	"mapserver/coords"
@@ -9,6 +8,8 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var (
@@ -46,7 +47,7 @@ func (t *Tiles) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 	zoom, _ := strconv.Atoi(parts[3])
 
 	c := coords.NewTileCoords(x, y, zoom, layerid)
-	tile, err := t.ctx.Objectdb.GetTile(c)
+	tile, err := t.ctx.TileDB.GetTile(c)
 
 	if err != nil {
 		resp.WriteHeader(500)
@@ -60,8 +61,8 @@ func (t *Tiles) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 			//TODO: cache/layer color
 
 		} else {
-			tilesCumulativeSize.Add(float64(len(tile.Data)))
+			tilesCumulativeSize.Add(float64(len(tile)))
-			resp.Write(tile.Data)
+			resp.Write(tile)
 
 		}
 	}