diff --git a/db/accessor.go b/db/accessor.go index b8ae77a..a9efd7a 100644 --- a/db/accessor.go +++ b/db/accessor.go @@ -12,6 +12,12 @@ type Block struct { type DBAccessor interface { Migrate() error + /** + * find old (pre-mapserver) mapblocks by lastpos + * used only on initial rendering + */ + FindLegacyBlocks(lastpos coords.MapBlockCoords, limit int) ([]Block, error) + FindLatestBlocks(mintime int64, limit int) ([]Block, error) CountBlocks(pos1 coords.MapBlockCoords, pos2 coords.MapBlockCoords) (int, error) GetBlock(pos coords.MapBlockCoords) (*Block, error) diff --git a/db/sqlite.go b/db/sqlite.go index 9b46847..c29adfb 100644 --- a/db/sqlite.go +++ b/db/sqlite.go @@ -69,6 +69,43 @@ func convertRows(pos int64, data []byte, mtime int64) Block { return Block{Pos: c, Data: data, Mtime: mtime} } +const getLegacyBlockQuery = ` +select pos,data,mtime +from blocks b +where b.mtime == 0 +and b.pos >= ? +order by b.pos asc +limit ? +` + +func (db *Sqlite3Accessor) FindLegacyBlocks(lastpos coords.MapBlockCoords, limit int) ([]Block, error) { + blocks := make([]Block, 0) + pc := coords.CoordToPlain(lastpos) + + rows, err := db.db.Query(getLegacyBlockQuery, pc, limit) + if err != nil { + return nil, err + } + + defer rows.Close() + + for rows.Next() { + var pos int64 + var data []byte + var mtime int64 + + err = rows.Scan(&pos, &data, &mtime) + if err != nil { + return nil, err + } + + mb := convertRows(pos, data, mtime) + blocks = append(blocks, mb) + } + + return blocks, nil +} + const getLatestBlockQuery = ` select pos,data,mtime from blocks b diff --git a/initialrenderer/job.go b/initialrenderer/job.go new file mode 100644 index 0000000..859b0bd --- /dev/null +++ b/initialrenderer/job.go @@ -0,0 +1,44 @@ +package initialrenderer + +import ( + "github.com/sirupsen/logrus" + "mapserver/app" + "mapserver/coords" + "time" +) + +func Job(ctx *app.App) { + + fields := logrus.Fields{} + logrus.WithFields(fields).Info("Starting initial rendering") + + lastcoords := coords.NewMapBlockCoords(coords.MinCoord, coords.MinCoord, coords.MinCoord) + + for true { + newlastcoords, mblist, err := ctx.BlockAccessor.FindLegacyMapBlocks(lastcoords, 1000) + + if err != nil { + panic(err) + } + + lastcoords = *newlastcoords + + if len(mblist) == 1 { + logrus.Info("Initial rendering complete") + break + } + + //for _, mb := range mblist { + //} + + fields = logrus.Fields{ + "count": len(mblist), + "X": lastcoords.X, + "Y": lastcoords.Y, + "Z": lastcoords.Z, + } + logrus.WithFields(fields).Info("Initial rendering") + + time.Sleep(100 * time.Millisecond) + } +} diff --git a/initialrenderer/renderer.go b/initialrenderer/renderer.go deleted file mode 100644 index 3c21a1a..0000000 --- a/initialrenderer/renderer.go +++ /dev/null @@ -1,83 +0,0 @@ -package initialrenderer - -import ( - "github.com/sirupsen/logrus" - "mapserver/coords" - "mapserver/layerconfig" - "mapserver/tilerenderer" - "runtime" - "time" -) - -func worker(tr *tilerenderer.TileRenderer, jobs <-chan coords.TileCoords) { - for coord := range jobs { - tr.Render(coord) - } -} - -func Render(tr *tilerenderer.TileRenderer, - layers []layerconfig.Layer) { - - start := time.Now() - complete_count := 256 * 256 - current_count := 0 - perf_count := 0 - - jobs := make(chan coords.TileCoords, 100) - - fields := logrus.Fields{ - "workers": runtime.NumCPU(), - } - logrus.WithFields(fields).Info("Starting initial render progress") - - for i := 0; i < runtime.NumCPU(); i++ { - go worker(tr, jobs) - } - - for _, layer := range layers { - - //zoom 10 iterator - for x := -127; x < 128; x++ { - for y := -127; y < 128; y++ { - tc := coords.NewTileCoords(x, y, 9, layer.Id) - jobs <- tc - current_count++ - perf_count++ - - if time.Now().Sub(start).Seconds() > 2 { - start = time.Now() - progress := float64(current_count) / float64(complete_count) * 100 - - fields := logrus.Fields{ - "x": x, - "y": y, - "progress(%)": progress, - "layer": layer.Name, - "perf": perf_count, - } - - perf_count = 0 - logrus.WithFields(fields).Info("Initial render progress") - } - } - } - - } - - close(jobs) - -} - -// zoom:1 == length=1 -// zoom:2 == length=2 -// zoom:3 == length=4 -// zoom:4 == length=8 -// zoom:5 == length=16 -// zoom:6 == length=32 -// zoom:7 == length=64 -// zoom:8 == length=128 -// zoom:9 == length=256 -// zoom:10 == length=512 -// zoom:11 == length=1024 -// zoom:12 == length=2048 -// zoom:13 == length=4096 diff --git a/main.go b/main.go index 4b38882..e98dd75 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,6 @@ import ( "github.com/sirupsen/logrus" "mapserver/app" "mapserver/initialrenderer" - "mapserver/layerconfig" "mapserver/params" "mapserver/tileupdate" "mapserver/web" @@ -60,7 +59,7 @@ func main() { //run initial rendering if ctx.Config.EnableInitialRendering { - go initialrenderer.Render(ctx.Tilerenderer, layerconfig.DefaultLayers) + go initialrenderer.Job(ctx) } //Incremental update diff --git a/mapblockaccessor/mapblockaccessor.go b/mapblockaccessor/mapblockaccessor.go index c19eb63..4cb7d9e 100644 --- a/mapblockaccessor/mapblockaccessor.go +++ b/mapblockaccessor/mapblockaccessor.go @@ -40,6 +40,46 @@ func (a *MapBlockAccessor) Update(pos coords.MapBlockCoords, mb *mapblockparser. a.c.Set(key, mb, cache.DefaultExpiration) } +func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, limit int) (*coords.MapBlockCoords, []*mapblockparser.MapBlock, error) { + + blocks, err := a.accessor.FindLegacyBlocks(lastpos, limit) + + if err != nil { + return nil, nil, err + } + + mblist := make([]*mapblockparser.MapBlock, 0) + var newlastpos *coords.MapBlockCoords + + for _, block := range blocks { + + fields := logrus.Fields{ + "x": block.Pos.X, + "y": block.Pos.Y, + "z": block.Pos.Z, + } + logrus.WithFields(fields).Debug("legacy mapblock") + + key := getKey(block.Pos) + + mapblock, err := mapblockparser.Parse(block.Data, block.Mtime) + if err != nil { + return nil, nil, err + } + + for _, listener := range a.listeners { + listener.OnParsedMapBlock(mapblock, block.Pos) + } + + a.c.Set(key, mapblock, cache.DefaultExpiration) + mblist = append(mblist, mapblock) + + newlastpos = &block.Pos + } + + return newlastpos, mblist, nil +} + func (a *MapBlockAccessor) FindLatestMapBlocks(mintime int64, limit int) ([]*mapblockparser.MapBlock, error) { blocks, err := a.accessor.FindLatestBlocks(mintime, limit) diff --git a/mapserver.json b/mapserver.json index 09ec7fd..88458ef 100644 --- a/mapserver.json +++ b/mapserver.json @@ -1,7 +1,7 @@ { "port": 8080, - "enableinitialrendering": false, - "enableincrementalupdate": true, + "enableinitialrendering": true, + "enableincrementalupdate": false, "webdev": true, "webapi": { "enablemapblock": true