From 26d5de407a05f90c2e9c48ebacfe676d0c2dc154 Mon Sep 17 00:00:00 2001 From: Thomas Rudin Date: Sat, 9 Feb 2019 18:05:40 +0100 Subject: [PATCH] use badger for tiles --- server/.gitignore | 1 + server/app/app.go | 2 + server/app/setup.go | 12 +++- server/go.mod | 4 ++ server/go.sum | 9 +++ server/mapobjectdb/accessor.go | 8 --- server/mapobjectdb/sqlite/mapobjects_test.go | 38 ------------ server/mapobjectdb/sqlite/sql.go | 28 --------- server/mapobjectdb/sqlite/tiles.go | 49 --------------- server/tiledb/tiledb.go | 65 ++++++++++++++++++++ server/tiledb/tiledb_test.go | 59 ++++++++++++++++++ server/tilerenderer/renderer.go | 16 ++--- server/tilerenderer/renderer_test.go | 9 ++- server/tilerendererjob/job.go | 13 ---- server/web/tiles.go | 9 +-- 15 files changed, 167 insertions(+), 155 deletions(-) delete mode 100644 server/mapobjectdb/sqlite/tiles.go create mode 100644 server/tiledb/tiledb.go create mode 100644 server/tiledb/tiledb_test.go diff --git a/server/.gitignore b/server/.gitignore index 183f581..734d68b 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -2,6 +2,7 @@ mapserver world.mt output map.sqlite +mapserver.tiles mapserver.sqlite mapserver.sqlite-journal mapserver.json diff --git a/server/app/app.go b/server/app/app.go index 4cf42ee..f78ca9e 100644 --- a/server/app/app.go +++ b/server/app/app.go @@ -9,6 +9,7 @@ import ( "mapserver/mapobjectdb" "mapserver/params" "mapserver/settings" + "mapserver/tiledb" "mapserver/tilerenderer" ) @@ -23,6 +24,7 @@ type App struct { Blockdb db.DBAccessor Objectdb mapobjectdb.DBAccessor + TileDB *tiledb.TileDB Settings *settings.Settings BlockAccessor *mapblockaccessor.MapBlockAccessor diff --git a/server/app/setup.go b/server/app/setup.go index c9bd22d..83add34 100644 --- a/server/app/setup.go +++ b/server/app/setup.go @@ -9,6 +9,7 @@ import ( sqliteobjdb "mapserver/mapobjectdb/sqlite" "mapserver/params" "mapserver/settings" + "mapserver/tiledb" "mapserver/tilerenderer" "mapserver/worldconfig" @@ -95,20 +96,27 @@ func Setup(p params.ParamsType, cfg *Config) *App { panic(err) } - //migrate tile database + //migrate object database err = a.Objectdb.Migrate() if err != nil { panic(err) } + //create tiledb + a.TileDB, err = tiledb.New("./mapserver.tiles") + + if err != nil { + panic(err) + } + //settings a.Settings = settings.New(a.Objectdb) //setup tile renderer a.Tilerenderer = tilerenderer.NewTileRenderer( a.Mapblockrenderer, - a.Objectdb, + a.TileDB, a.Blockdb, a.Config.Layers, ) diff --git a/server/go.mod b/server/go.mod index d3205d1..e103bfe 100644 --- a/server/go.mod +++ b/server/go.mod @@ -1,12 +1,16 @@ module mapserver require ( + github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 // indirect + github.com/dgraph-io/badger v1.5.4 + github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f // indirect github.com/disintegration/imaging v1.5.0 github.com/gorilla/websocket v1.4.0 github.com/lib/pq v1.0.0 github.com/mattn/go-sqlite3 v1.10.0 github.com/mjibson/esc v0.1.0 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible + github.com/pkg/errors v0.8.1 // indirect github.com/prometheus/client_golang v0.9.2 github.com/sirupsen/logrus v1.3.0 github.com/stretchr/testify v1.2.2 diff --git a/server/go.sum b/server/go.sum index e9e7080..dedc619 100644 --- a/server/go.sum +++ b/server/go.sum @@ -1,3 +1,5 @@ +github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 h1:PqzgE6kAMi81xWQA2QIVxjWkFHptGgC547vchpUbtFo= +github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -5,6 +7,10 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.5.4 h1:gVTrpUTbbr/T24uvoCaqY2KSHfNLVGm0w+hbee2HMeg= +github.com/dgraph-io/badger v1.5.4/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= +github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA= +github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/disintegration/imaging v1.5.0 h1:uYqUhwNmLU4K1FN44vhqS4TZJRAA4RhBINgbQlKyGi0= github.com/disintegration/imaging v1.5.0/go.mod h1:9B/deIUIrliYkyMTuXJd6OUFLcrZ2tf+3Qlwnaf/CjU= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= @@ -23,6 +29,8 @@ github.com/mjibson/esc v0.1.0 h1:5ch+murgrcwDFLOE2hwj0f7kE4xJfJhkSCAjSLY182o= github.com/mjibson/esc v0.1.0/go.mod h1:9Hw9gxxfHulMF5OJKCyhYD7PzlSdhzXyaGEBRPH1OPs= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= @@ -44,6 +52,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/image v0.0.0-20190118043309-183bebdce1b2 h1:FNSSV4jv1PrPsiM2iKGpqLPPgYACqh9Muav7Pollk1k= golang.org/x/image v0.0.0-20190118043309-183bebdce1b2/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= diff --git a/server/mapobjectdb/accessor.go b/server/mapobjectdb/accessor.go index d106300..5c1c8a3 100644 --- a/server/mapobjectdb/accessor.go +++ b/server/mapobjectdb/accessor.go @@ -68,19 +68,11 @@ type DBAccessor interface { //migrates the database Migrate() error - //true = speed,unsafe / false = safe - EnableSpeedSafetyTradeoff(enableSpeed bool) error - //Generic map objects (poi, etc) GetMapData(q SearchQuery) ([]*MapObject, error) RemoveMapData(pos *coords.MapBlockCoords) error AddMapData(data *MapObject) error - //tile data - GetTile(pos *coords.TileCoords) (*Tile, error) - SetTile(tile *Tile) error - RemoveTile(pos *coords.TileCoords) error - //Settings GetSetting(key string, defaultvalue string) (string, error) SetSetting(key string, value string) error diff --git a/server/mapobjectdb/sqlite/mapobjects_test.go b/server/mapobjectdb/sqlite/mapobjects_test.go index b0bd3e9..e29c30a 100644 --- a/server/mapobjectdb/sqlite/mapobjects_test.go +++ b/server/mapobjectdb/sqlite/mapobjects_test.go @@ -26,44 +26,6 @@ func TestMigrate(t *testing.T) { panic(err) } - pos := coords.NewTileCoords(0, 0, 13, 0) - tile, err := db.GetTile(pos) - if err != nil { - panic(err) - } - - if tile != nil { - t.Fatal("non-empty tile found") - } - - data := []byte{0x01, 0x02} - tile2 := mapobjectdb.Tile{Pos: pos, Data: data} - err = db.SetTile(&tile2) - - if err != nil { - panic(err) - } - - tile3, err := db.GetTile(pos) - - if err != nil { - panic(err) - } - - if tile3 == nil { - t.Fatal("no data returned") - } - - if len(tile2.Data) != len(tile3.Data) { - t.Fatal("inserted data does not match") - } - - err = db.SetTile(&tile2) - - if err != nil { - panic(err) - } - } func TestMapObjects(t *testing.T) { diff --git a/server/mapobjectdb/sqlite/sql.go b/server/mapobjectdb/sqlite/sql.go index 859061f..dd8a991 100644 --- a/server/mapobjectdb/sqlite/sql.go +++ b/server/mapobjectdb/sqlite/sql.go @@ -30,16 +30,6 @@ create table if not exists object_attributes( create index if not exists object_attributes_key_value on object_attributes(key, value); -create table if not exists tiles( - data blob, - mtime bigint, - layerid int, - x int, - y int, - zoom int, - primary key(x,y,zoom,layerid) -); - create table if not exists settings( key varchar primary key not null, value varchar not null @@ -75,24 +65,6 @@ object_attributes(objectid, key, value) values(?, ?, ?) ` -const getTileQuery = ` -select data,mtime from tiles t -where t.layerid = ? -and t.x = ? -and t.y = ? -and t.zoom = ? -` - -const setTileQuery = ` -insert or replace into tiles(x,y,zoom,layerid,data,mtime) -values(?, ?, ?, ?, ?, ?) -` - -const removeTileQuery = ` -delete from tiles -where x = ? and y = ? and zoom = ? and layerid = ? -` - const getSettingQuery = ` select value from settings where key = ? ` diff --git a/server/mapobjectdb/sqlite/tiles.go b/server/mapobjectdb/sqlite/tiles.go deleted file mode 100644 index 5bde834..0000000 --- a/server/mapobjectdb/sqlite/tiles.go +++ /dev/null @@ -1,49 +0,0 @@ -package sqlite - -import ( - "mapserver/coords" - "mapserver/mapobjectdb" -) - -func (db *Sqlite3Accessor) GetTile(pos *coords.TileCoords) (*mapobjectdb.Tile, error) { - rows, err := db.db.Query(getTileQuery, pos.LayerId, pos.X, pos.Y, pos.Zoom) - if err != nil { - return nil, err - } - - defer rows.Close() - - if rows.Next() { - var data []byte - var mtime int64 - - err = rows.Scan(&data, &mtime) - if err != nil { - return nil, err - } - - if data == nil { - return nil, nil - } - - mb := mapobjectdb.Tile{ - Pos: pos, - Data: data, - Mtime: mtime, - } - - return &mb, nil - } - - return nil, nil -} - -func (db *Sqlite3Accessor) SetTile(tile *mapobjectdb.Tile) error { - _, err := db.db.Exec(setTileQuery, tile.Pos.X, tile.Pos.Y, tile.Pos.Zoom, tile.Pos.LayerId, tile.Data, tile.Mtime) - return err -} - -func (db *Sqlite3Accessor) RemoveTile(pos *coords.TileCoords) error { - _, err := db.db.Exec(removeTileQuery, pos.X, pos.Y, pos.Zoom, pos.LayerId) - return err -} diff --git a/server/tiledb/tiledb.go b/server/tiledb/tiledb.go new file mode 100644 index 0000000..ea22308 --- /dev/null +++ b/server/tiledb/tiledb.go @@ -0,0 +1,65 @@ +package tiledb + +import ( + "fmt" + "mapserver/coords" + + "github.com/dgraph-io/badger" +) + +func New(path string) (*TileDB, error) { + opts := badger.DefaultOptions + opts.Dir = path + opts.ValueDir = path + db, err := badger.Open(opts) + + if err != nil { + return nil, err + } + + return &TileDB{ + db: db, + }, nil +} + +type TileDB struct { + db *badger.DB +} + +func getKey(pos *coords.TileCoords) []byte { + return []byte(fmt.Sprintf("%d/%d/%d/%d", pos.X, pos.Y, pos.Zoom, pos.LayerId)) +} + +func (this *TileDB) GetTile(pos *coords.TileCoords) ([]byte, error) { + var tile []byte + err := this.db.View(func(txn *badger.Txn) error { + item, err := txn.Get(getKey(pos)) + if item != nil { + tile, err = item.ValueCopy(nil) + } + return err + }) + if err != nil { + return nil, nil + } + + return tile, err +} + +func (this *TileDB) SetTile(pos *coords.TileCoords, tile []byte) error { + err := this.db.Update(func(txn *badger.Txn) error { + err := txn.Set(getKey(pos), tile) + return err + }) + + return err +} + +func (this *TileDB) RemoveTile(pos *coords.TileCoords) error { + err := this.db.Update(func(txn *badger.Txn) error { + err := txn.Delete(getKey(pos)) + return err + }) + + return err +} diff --git a/server/tiledb/tiledb_test.go b/server/tiledb/tiledb_test.go new file mode 100644 index 0000000..beb9939 --- /dev/null +++ b/server/tiledb/tiledb_test.go @@ -0,0 +1,59 @@ +package tiledb + +import ( + "io/ioutil" + "mapserver/coords" + "os" + "testing" +) + +func TestTileDB(t *testing.T) { + tmpfile, err := ioutil.TempDir("", "TestTileDB.*.badger") + if err != nil { + panic(err) + } + defer os.RemoveAll(tmpfile) + + db, err := New(tmpfile) + if err != nil { + panic(err) + } + + c := coords.NewTileCoords(0, 0, 1, 2) + + err = db.SetTile(c, []byte{1, 2, 3}) + if err != nil { + panic(err) + } + + tile, err := db.GetTile(c) + if err != nil { + panic(err) + } + + if len(tile) != 3 { + t.Error("wrong size") + } + + db.RemoveTile(c) + + tile, err = db.GetTile(c) + if err != nil { + panic(err) + } + + if tile != nil { + t.Error("tile not removed") + } + + c2 := coords.NewTileCoords(1, 0, 1, 2) + tile, err = db.GetTile(c2) + if err != nil { + panic(err) + } + + if tile != nil { + t.Error("tile exists") + } + +} diff --git a/server/tilerenderer/renderer.go b/server/tilerenderer/renderer.go index fb83695..16c390e 100644 --- a/server/tilerenderer/renderer.go +++ b/server/tilerenderer/renderer.go @@ -11,7 +11,7 @@ import ( "mapserver/eventbus" "mapserver/layer" "mapserver/mapblockrenderer" - "mapserver/mapobjectdb" + "mapserver/tiledb" "time" "github.com/disintegration/imaging" @@ -21,13 +21,13 @@ import ( type TileRenderer struct { mapblockrenderer *mapblockrenderer.MapBlockRenderer layers []layer.Layer - tdb mapobjectdb.DBAccessor + tdb *tiledb.TileDB dba db.DBAccessor Eventbus *eventbus.Eventbus } func NewTileRenderer(mapblockrenderer *mapblockrenderer.MapBlockRenderer, - tdb mapobjectdb.DBAccessor, + tdb *tiledb.TileDB, dba db.DBAccessor, layers []layer.Layer) *TileRenderer { @@ -69,7 +69,7 @@ func (tr *TileRenderer) RenderImage(tc *coords.TileCoords, recursionDepth int) ( } if cachedtile != nil { - reader := bytes.NewReader(cachedtile.Data) + reader := bytes.NewReader(cachedtile) cachedimg, err := png.Decode(reader) if err != nil { return nil, nil, err @@ -84,7 +84,7 @@ func (tr *TileRenderer) RenderImage(tc *coords.TileCoords, recursionDepth int) ( draw.Draw(img, rect, cachedimg, image.ZP, draw.Src) log.WithFields(logrus.Fields{"x": tc.X, "y": tc.Y, "zoom": tc.Zoom}).Debug("Cached image") - return img, cachedtile.Data, nil + return img, cachedtile, nil } } @@ -212,8 +212,8 @@ func (tr *TileRenderer) RenderImage(tc *coords.TileCoords, recursionDepth int) ( encode := t.Sub(start) start = t - tile := mapobjectdb.Tile{Pos: tc, Data: buf.Bytes(), Mtime: time.Now().Unix()} - tr.tdb.SetTile(&tile) + tile := buf.Bytes() + tr.tdb.SetTile(tc, tile) t = time.Now() cache := t.Sub(start) @@ -223,7 +223,7 @@ func (tr *TileRenderer) RenderImage(tc *coords.TileCoords, recursionDepth int) ( "Y": tc.Y, "Zoom": tc.Zoom, "LayerId": tc.LayerId, - "size": len(tile.Data), + "size": len(tile), "quadrender": quadrender, "quadresize": quadresize, "encode": encode, diff --git a/server/tilerenderer/renderer_test.go b/server/tilerenderer/renderer_test.go index 4efe2f2..5b56566 100644 --- a/server/tilerenderer/renderer_test.go +++ b/server/tilerenderer/renderer_test.go @@ -9,8 +9,8 @@ import ( "mapserver/layer" "mapserver/mapblockaccessor" "mapserver/mapblockrenderer" - sqliteobjdb "mapserver/mapobjectdb/sqlite" "mapserver/testutils" + "mapserver/tiledb" "os" "testing" @@ -47,11 +47,10 @@ func TestTileRender(t *testing.T) { r := mapblockrenderer.NewMapBlockRenderer(cache, c) - tiletmpfile, err := ioutil.TempFile("", "TestTileRenderTiles.*.sqlite") - defer os.Remove(tiletmpfile.Name()) + tiletmpdir, err := ioutil.TempDir("", "TestTileRenderTiles.*.sqlite") + defer os.RemoveAll(tiletmpdir) - tdb, _ := sqliteobjdb.New(tiletmpfile.Name()) - tdb.Migrate() + tdb, _ := tiledb.New(tiletmpdir) layers := []layer.Layer{ layer.Layer{ diff --git a/server/tilerendererjob/job.go b/server/tilerendererjob/job.go index 30f54c8..da00fde 100644 --- a/server/tilerendererjob/job.go +++ b/server/tilerendererjob/job.go @@ -16,20 +16,7 @@ func Job(ctx *app.App) { } if ctx.Settings.GetBool(settings.SETTING_INITIAL_RUN, true) { - //fast, unsafe mode - err := ctx.Objectdb.EnableSpeedSafetyTradeoff(true) - if err != nil { - panic(err) - } - initialRender(ctx, jobs) - - //normal, safe mode - err = ctx.Objectdb.EnableSpeedSafetyTradeoff(false) - if err != nil { - panic(err) - } - } incrementalRender(ctx, jobs) diff --git a/server/web/tiles.go b/server/web/tiles.go index fec9312..1ea7e33 100644 --- a/server/web/tiles.go +++ b/server/web/tiles.go @@ -1,7 +1,6 @@ package web import ( - "github.com/prometheus/client_golang/prometheus" "image/color" "mapserver/app" "mapserver/coords" @@ -9,6 +8,8 @@ import ( "net/http" "strconv" "strings" + + "github.com/prometheus/client_golang/prometheus" ) var ( @@ -46,7 +47,7 @@ func (t *Tiles) ServeHTTP(resp http.ResponseWriter, req *http.Request) { zoom, _ := strconv.Atoi(parts[3]) c := coords.NewTileCoords(x, y, zoom, layerid) - tile, err := t.ctx.Objectdb.GetTile(c) + tile, err := t.ctx.TileDB.GetTile(c) if err != nil { resp.WriteHeader(500) @@ -60,8 +61,8 @@ func (t *Tiles) ServeHTTP(resp http.ResponseWriter, req *http.Request) { //TODO: cache/layer color } else { - tilesCumulativeSize.Add(float64(len(tile.Data))) - resp.Write(tile.Data) + tilesCumulativeSize.Add(float64(len(tile))) + resp.Write(tile) } }