Koldun/mapserver
1
0
Fork 0
forked from MTSR/mapserver
Code Pull Requests Activity

use badger for tiles

Browse Source
This commit is contained in:
Thomas Rudin 2019-02-09 18:05:40 +01:00
parent a6ae6734db
commit 26d5de407a
15 changed files with 167 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
server/.gitignore vendored
View File

@ -2,6 +2,7 @@ mapserver
world.mt
output
map.sqlite
mapserver.tiles
mapserver.sqlite
mapserver.sqlite-journal
mapserver.json

2
server/app/app.go
View File

@ -9,6 +9,7 @@ import (
"mapserver/mapobjectdb"
"mapserver/params"
"mapserver/settings"
"mapserver/tiledb"
"mapserver/tilerenderer"
)
@ -23,6 +24,7 @@ type App struct {
Blockdb db.DBAccessor
Objectdb mapobjectdb.DBAccessor
TileDB *tiledb.TileDB
Settings *settings.Settings
BlockAccessor *mapblockaccessor.MapBlockAccessor

12
server/app/setup.go
View File

@ -9,6 +9,7 @@ import (
sqliteobjdb "mapserver/mapobjectdb/sqlite"
"mapserver/params"
"mapserver/settings"
"mapserver/tiledb"
"mapserver/tilerenderer"
"mapserver/worldconfig"
@ -95,20 +96,27 @@ func Setup(p params.ParamsType, cfg *Config) *App {
panic(err)
}
//migrate tile database
//migrate object database
err = a.Objectdb.Migrate()
if err != nil {
panic(err)
}
//create tiledb
a.TileDB, err = tiledb.New("./mapserver.tiles")
if err != nil {
panic(err)
}
//settings
a.Settings = settings.New(a.Objectdb)
//setup tile renderer
a.Tilerenderer = tilerenderer.NewTileRenderer(
a.Mapblockrenderer,
a.Objectdb,
a.TileDB,
a.Blockdb,
a.Config.Layers,
)

4
server/go.mod
View File

@ -1,12 +1,16 @@
module mapserver
require (
github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 // indirect
github.com/dgraph-io/badger v1.5.4
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f // indirect
github.com/disintegration/imaging v1.5.0
github.com/gorilla/websocket v1.4.0
github.com/lib/pq v1.0.0
github.com/mattn/go-sqlite3 v1.10.0
github.com/mjibson/esc v0.1.0 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.8.1 // indirect
github.com/prometheus/client_golang v0.9.2
github.com/sirupsen/logrus v1.3.0
github.com/stretchr/testify v1.2.2

9
server/go.sum
View File

@ -1,3 +1,5 @@
github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 h1:PqzgE6kAMi81xWQA2QIVxjWkFHptGgC547vchpUbtFo=
github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@ -5,6 +7,10 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger v1.5.4 h1:gVTrpUTbbr/T24uvoCaqY2KSHfNLVGm0w+hbee2HMeg=
github.com/dgraph-io/badger v1.5.4/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA=
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/disintegration/imaging v1.5.0 h1:uYqUhwNmLU4K1FN44vhqS4TZJRAA4RhBINgbQlKyGi0=
github.com/disintegration/imaging v1.5.0/go.mod h1:9B/deIUIrliYkyMTuXJd6OUFLcrZ2tf+3Qlwnaf/CjU=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@ -23,6 +29,8 @@ github.com/mjibson/esc v0.1.0 h1:5ch+murgrcwDFLOE2hwj0f7kE4xJfJhkSCAjSLY182o=
github.com/mjibson/esc v0.1.0/go.mod h1:9Hw9gxxfHulMF5OJKCyhYD7PzlSdhzXyaGEBRPH1OPs=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
@ -44,6 +52,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/image v0.0.0-20190118043309-183bebdce1b2 h1:FNSSV4jv1PrPsiM2iKGpqLPPgYACqh9Muav7Pollk1k=
golang.org/x/image v0.0.0-20190118043309-183bebdce1b2/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8=

8
server/mapobjectdb/accessor.go
View File

@ -68,19 +68,11 @@ type DBAccessor interface {
//migrates the database
Migrate() error
//true = speed,unsafe / false = safe
EnableSpeedSafetyTradeoff(enableSpeed bool) error
//Generic map objects (poi, etc)
GetMapData(q SearchQuery) ([]*MapObject, error)
RemoveMapData(pos *coords.MapBlockCoords) error
AddMapData(data *MapObject) error
//tile data
GetTile(pos *coords.TileCoords) (*Tile, error)
SetTile(tile *Tile) error
RemoveTile(pos *coords.TileCoords) error
//Settings
GetSetting(key string, defaultvalue string) (string, error)
SetSetting(key string, value string) error

38
server/mapobjectdb/sqlite/mapobjects_test.go
View File

@ -26,44 +26,6 @@ func TestMigrate(t *testing.T) {
panic(err)
}
pos := coords.NewTileCoords(0, 0, 13, 0)
tile, err := db.GetTile(pos)
if err != nil {
panic(err)
}
if tile != nil {
t.Fatal("non-empty tile found")
}
data := []byte{0x01, 0x02}
tile2 := mapobjectdb.Tile{Pos: pos, Data: data}
err = db.SetTile(&tile2)
if err != nil {
panic(err)
}
tile3, err := db.GetTile(pos)
if err != nil {
panic(err)
}
if tile3 == nil {
t.Fatal("no data returned")
}
if len(tile2.Data) != len(tile3.Data) {
t.Fatal("inserted data does not match")
}
err = db.SetTile(&tile2)
if err != nil {
panic(err)
}
}
func TestMapObjects(t *testing.T) {

28
server/mapobjectdb/sqlite/sql.go
View File

@ -30,16 +30,6 @@ create table if not exists object_attributes(
create index if not exists object_attributes_key_value on object_attributes(key, value);
create table if not exists tiles(
data blob,
mtime bigint,
layerid int,
x int,
y int,
zoom int,
primary key(x,y,zoom,layerid)
);
create table if not exists settings(
key varchar primary key not null,
value varchar not null
@ -75,24 +65,6 @@ object_attributes(objectid, key, value)
values(?, ?, ?)
`
const getTileQuery = `
select data,mtime from tiles t
where t.layerid = ?
and t.x = ?
and t.y = ?
and t.zoom = ?
`
const setTileQuery = `
insert or replace into tiles(x,y,zoom,layerid,data,mtime)
values(?, ?, ?, ?, ?, ?)
`
const removeTileQuery = `
delete from tiles
where x = ? and y = ? and zoom = ? and layerid = ?
`
const getSettingQuery = `
select value from settings where key = ?
`

49
server/mapobjectdb/sqlite/tiles.go
View File

@ -1,49 +0,0 @@
package sqlite
import (
"mapserver/coords"
"mapserver/mapobjectdb"
)
func (db *Sqlite3Accessor) GetTile(pos *coords.TileCoords) (*mapobjectdb.Tile, error) {
rows, err := db.db.Query(getTileQuery, pos.LayerId, pos.X, pos.Y, pos.Zoom)
if err != nil {
return nil, err
}
defer rows.Close()
if rows.Next() {
var data []byte
var mtime int64
err = rows.Scan(&data, &mtime)
if err != nil {
return nil, err
}
if data == nil {
return nil, nil
}
mb := mapobjectdb.Tile{
Pos: pos,
Data: data,
Mtime: mtime,
}
return &mb, nil
}
return nil, nil
}
func (db *Sqlite3Accessor) SetTile(tile *mapobjectdb.Tile) error {
_, err := db.db.Exec(setTileQuery, tile.Pos.X, tile.Pos.Y, tile.Pos.Zoom, tile.Pos.LayerId, tile.Data, tile.Mtime)
return err
}
func (db *Sqlite3Accessor) RemoveTile(pos *coords.TileCoords) error {
_, err := db.db.Exec(removeTileQuery, pos.X, pos.Y, pos.Zoom, pos.LayerId)
return err
}

65
server/tiledb/tiledb.go Normal file
View File

@ -0,0 +1,65 @@
package tiledb
import (
"fmt"
"mapserver/coords"
"github.com/dgraph-io/badger"
)
func New(path string) (*TileDB, error) {
opts := badger.DefaultOptions
opts.Dir = path
opts.ValueDir = path
db, err := badger.Open(opts)
if err != nil {
return nil, err
}
return &TileDB{
db: db,
}, nil
}
type TileDB struct {
db *badger.DB
}
func getKey(pos *coords.TileCoords) []byte {
return []byte(fmt.Sprintf("%d/%d/%d/%d", pos.X, pos.Y, pos.Zoom, pos.LayerId))
}
func (this *TileDB) GetTile(pos *coords.TileCoords) ([]byte, error) {
var tile []byte
err := this.db.View(func(txn *badger.Txn) error {
item, err := txn.Get(getKey(pos))
if item != nil {
tile, err = item.ValueCopy(nil)
}
return err
})
if err != nil {
return nil, nil
}
return tile, err
}
func (this *TileDB) SetTile(pos *coords.TileCoords, tile []byte) error {
err := this.db.Update(func(txn *badger.Txn) error {
err := txn.Set(getKey(pos), tile)
return err
})
return err
}
func (this *TileDB) RemoveTile(pos *coords.TileCoords) error {
err := this.db.Update(func(txn *badger.Txn) error {
err := txn.Delete(getKey(pos))
return err
})
return err
}

59
server/tiledb/tiledb_test.go Normal file
View File

@ -0,0 +1,59 @@
package tiledb
import (
"io/ioutil"
"mapserver/coords"
"os"
"testing"
)
func TestTileDB(t *testing.T) {
tmpfile, err := ioutil.TempDir("", "TestTileDB.*.badger")
if err != nil {
panic(err)
}
defer os.RemoveAll(tmpfile)
db, err := New(tmpfile)
if err != nil {
panic(err)
}
c := coords.NewTileCoords(0, 0, 1, 2)
err = db.SetTile(c, []byte{1, 2, 3})
if err != nil {
panic(err)
}
tile, err := db.GetTile(c)
if err != nil {
panic(err)
}
if len(tile) != 3 {
t.Error("wrong size")
}
db.RemoveTile(c)
tile, err = db.GetTile(c)
if err != nil {
panic(err)
}
if tile != nil {
t.Error("tile not removed")
}
c2 := coords.NewTileCoords(1, 0, 1, 2)
tile, err = db.GetTile(c2)
if err != nil {
panic(err)
}
if tile != nil {
t.Error("tile exists")
}
}

16
server/tilerenderer/renderer.go
View File

@ -11,7 +11,7 @@ import (
"mapserver/eventbus"
"mapserver/layer"
"mapserver/mapblockrenderer"
"mapserver/mapobjectdb"
"mapserver/tiledb"
"time"
"github.com/disintegration/imaging"
@ -21,13 +21,13 @@ import (
type TileRenderer struct {
mapblockrenderer *mapblockrenderer.MapBlockRenderer
layers []layer.Layer
tdb mapobjectdb.DBAccessor
tdb *tiledb.TileDB
dba db.DBAccessor
Eventbus *eventbus.Eventbus
}
func NewTileRenderer(mapblockrenderer *mapblockrenderer.MapBlockRenderer,
tdb mapobjectdb.DBAccessor,
tdb *tiledb.TileDB,
dba db.DBAccessor,
layers []layer.Layer) *TileRenderer {
@ -69,7 +69,7 @@ func (tr *TileRenderer) RenderImage(tc *coords.TileCoords, recursionDepth int) (
}
if cachedtile != nil {
reader := bytes.NewReader(cachedtile.Data)
reader := bytes.NewReader(cachedtile)
cachedimg, err := png.Decode(reader)
if err != nil {
return nil, nil, err
@ -84,7 +84,7 @@ func (tr *TileRenderer) RenderImage(tc *coords.TileCoords, recursionDepth int) (
draw.Draw(img, rect, cachedimg, image.ZP, draw.Src)
log.WithFields(logrus.Fields{"x": tc.X, "y": tc.Y, "zoom": tc.Zoom}).Debug("Cached image")
return img, cachedtile.Data, nil
return img, cachedtile, nil
}
}
@ -212,8 +212,8 @@ func (tr *TileRenderer) RenderImage(tc *coords.TileCoords, recursionDepth int) (
encode := t.Sub(start)
start = t
tile := mapobjectdb.Tile{Pos: tc, Data: buf.Bytes(), Mtime: time.Now().Unix()}
tr.tdb.SetTile(&tile)
tile := buf.Bytes()
tr.tdb.SetTile(tc, tile)
t = time.Now()
cache := t.Sub(start)
@ -223,7 +223,7 @@ func (tr *TileRenderer) RenderImage(tc *coords.TileCoords, recursionDepth int) (
"Y": tc.Y,
"Zoom": tc.Zoom,
"LayerId": tc.LayerId,
"size": len(tile.Data),
"size": len(tile),
"quadrender": quadrender,
"quadresize": quadresize,
"encode": encode,

9
server/tilerenderer/renderer_test.go
View File

@ -9,8 +9,8 @@ import (
"mapserver/layer"
"mapserver/mapblockaccessor"
"mapserver/mapblockrenderer"
sqliteobjdb "mapserver/mapobjectdb/sqlite"
"mapserver/testutils"
"mapserver/tiledb"
"os"
"testing"
@ -47,11 +47,10 @@ func TestTileRender(t *testing.T) {
r := mapblockrenderer.NewMapBlockRenderer(cache, c)
tiletmpfile, err := ioutil.TempFile("", "TestTileRenderTiles.*.sqlite")
defer os.Remove(tiletmpfile.Name())
tiletmpdir, err := ioutil.TempDir("", "TestTileRenderTiles.*.sqlite")
defer os.RemoveAll(tiletmpdir)
tdb, _ := sqliteobjdb.New(tiletmpfile.Name())
tdb.Migrate()
tdb, _ := tiledb.New(tiletmpdir)
layers := []layer.Layer{
layer.Layer{

13
server/tilerendererjob/job.go
View File

@ -16,20 +16,7 @@ func Job(ctx *app.App) {
}
if ctx.Settings.GetBool(settings.SETTING_INITIAL_RUN, true) {
//fast, unsafe mode
err := ctx.Objectdb.EnableSpeedSafetyTradeoff(true)
if err != nil {
panic(err)
}
initialRender(ctx, jobs)
//normal, safe mode
err = ctx.Objectdb.EnableSpeedSafetyTradeoff(false)
if err != nil {
panic(err)
}
}
incrementalRender(ctx, jobs)

9
server/web/tiles.go
View File

@ -1,7 +1,6 @@
package web
import (
"github.com/prometheus/client_golang/prometheus"
"image/color"
"mapserver/app"
"mapserver/coords"
@ -9,6 +8,8 @@ import (
"net/http"
"strconv"
"strings"
"github.com/prometheus/client_golang/prometheus"
)
var (
@ -46,7 +47,7 @@ func (t *Tiles) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
zoom, _ := strconv.Atoi(parts[3])
c := coords.NewTileCoords(x, y, zoom, layerid)
tile, err := t.ctx.Objectdb.GetTile(c)
tile, err := t.ctx.TileDB.GetTile(c)
if err != nil {
resp.WriteHeader(500)
@ -60,8 +61,8 @@ func (t *Tiles) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
//TODO: cache/layer color
} else {
tilesCumulativeSize.Add(float64(len(tile.Data)))
resp.Write(tile.Data)
tilesCumulativeSize.Add(float64(len(tile)))
resp.Write(tile)
}
}