new initial rendering strategy

db/accessor.go

@@ -12,6 +12,12 @@ type Block struct {
 
 type DBAccessor interface {
 	Migrate() error
+	/**
+	 * find old (pre-mapserver) mapblocks by lastpos
+	 * used only on initial rendering
+	 */
+	FindLegacyBlocks(lastpos coords.MapBlockCoords, limit int) ([]Block, error)
+
 	FindLatestBlocks(mintime int64, limit int) ([]Block, error)
 	CountBlocks(pos1 coords.MapBlockCoords, pos2 coords.MapBlockCoords) (int, error)
 	GetBlock(pos coords.MapBlockCoords) (*Block, error)

db/sqlite.go

@@ -69,6 +69,43 @@ func convertRows(pos int64, data []byte, mtime int64) Block {
 	return Block{Pos: c, Data: data, Mtime: mtime}
 }
 
+const getLegacyBlockQuery = `
+select pos,data,mtime
+from blocks b
+where b.mtime == 0
+and b.pos >= ?
+order by b.pos asc
+limit ?
+`
+
+func (db *Sqlite3Accessor) FindLegacyBlocks(lastpos coords.MapBlockCoords, limit int) ([]Block, error) {
+	blocks := make([]Block, 0)
+	pc := coords.CoordToPlain(lastpos)
+
+	rows, err := db.db.Query(getLegacyBlockQuery, pc, limit)
+	if err != nil {
+		return nil, err
+	}
+
+	defer rows.Close()
+
+	for rows.Next() {
+		var pos int64
+		var data []byte
+		var mtime int64
+
+		err = rows.Scan(&pos, &data, &mtime)
+		if err != nil {
+			return nil, err
+		}
+
+		mb := convertRows(pos, data, mtime)
+		blocks = append(blocks, mb)
+	}
+
+	return blocks, nil
+}
+
 const getLatestBlockQuery = `
 select pos,data,mtime
 from blocks b

initialrenderer/job.go

@@ -0,0 +1,44 @@
+package initialrenderer
+
+import (
+	"github.com/sirupsen/logrus"
+	"mapserver/app"
+	"mapserver/coords"
+	"time"
+)
+
+func Job(ctx *app.App) {
+
+	fields := logrus.Fields{}
+	logrus.WithFields(fields).Info("Starting initial rendering")
+
+	lastcoords := coords.NewMapBlockCoords(coords.MinCoord, coords.MinCoord, coords.MinCoord)
+
+	for true {
+		newlastcoords, mblist, err := ctx.BlockAccessor.FindLegacyMapBlocks(lastcoords, 1000)
+
+		if err != nil {
+			panic(err)
+		}
+
+		lastcoords = *newlastcoords
+
+		if len(mblist) == 1 {
+			logrus.Info("Initial rendering complete")
+			break
+		}
+
+		//for _, mb := range mblist {
+		//}
+
+		fields = logrus.Fields{
+			"count": len(mblist),
+			"X":  lastcoords.X,
+			"Y":  lastcoords.Y,
+			"Z":  lastcoords.Z,
+		}
+		logrus.WithFields(fields).Info("Initial rendering")
+
+		time.Sleep(100 * time.Millisecond)
+	}
+}

initialrenderer/renderer.go

@@ -1,83 +0,0 @@
-package initialrenderer
-
-import (
-	"github.com/sirupsen/logrus"
-	"mapserver/coords"
-	"mapserver/layerconfig"
-	"mapserver/tilerenderer"
-	"runtime"
-	"time"
-)
-
-func worker(tr *tilerenderer.TileRenderer, jobs <-chan coords.TileCoords) {
-	for coord := range jobs {
-		tr.Render(coord)
-	}
-}
-
-func Render(tr *tilerenderer.TileRenderer,
-	layers []layerconfig.Layer) {
-
-	start := time.Now()
-	complete_count := 256 * 256
-	current_count := 0
-	perf_count := 0
-
-	jobs := make(chan coords.TileCoords, 100)
-
-	fields := logrus.Fields{
-		"workers": runtime.NumCPU(),
-	}
-	logrus.WithFields(fields).Info("Starting initial render progress")
-
-	for i := 0; i < runtime.NumCPU(); i++ {
-		go worker(tr, jobs)
-	}
-
-	for _, layer := range layers {
-
-		//zoom 10 iterator
-		for x := -127; x < 128; x++ {
-			for y := -127; y < 128; y++ {
-				tc := coords.NewTileCoords(x, y, 9, layer.Id)
-				jobs <- tc
-				current_count++
-				perf_count++
-
-				if time.Now().Sub(start).Seconds() > 2 {
-					start = time.Now()
-					progress := float64(current_count) / float64(complete_count) * 100
-
-					fields := logrus.Fields{
-						"x":           x,
-						"y":           y,
-						"progress(%)": progress,
-						"layer":       layer.Name,
-						"perf":        perf_count,
-					}
-
-					perf_count = 0
-					logrus.WithFields(fields).Info("Initial render progress")
-				}
-			}
-		}
-
-	}
-
-	close(jobs)
-
-}
-
-// zoom:1 == length=1
-// zoom:2 == length=2
-// zoom:3 == length=4
-// zoom:4 == length=8
-// zoom:5 == length=16
-// zoom:6 == length=32
-// zoom:7 == length=64
-// zoom:8 == length=128
-// zoom:9 == length=256
-// zoom:10 == length=512
-// zoom:11 == length=1024
-// zoom:12 == length=2048
-// zoom:13 == length=4096

main.go

@@ -6,7 +6,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"mapserver/app"
 	"mapserver/initialrenderer"
-	"mapserver/layerconfig"
 	"mapserver/params"
 	"mapserver/tileupdate"
 	"mapserver/web"
@@ -60,7 +59,7 @@ func main() {
 
 	//run initial rendering
 	if ctx.Config.EnableInitialRendering {
-		go initialrenderer.Render(ctx.Tilerenderer, layerconfig.DefaultLayers)
+		go initialrenderer.Job(ctx)
 	}
 
 	//Incremental update

mapblockaccessor/mapblockaccessor.go

@@ -40,6 +40,46 @@ func (a *MapBlockAccessor) Update(pos coords.MapBlockCoords, mb *mapblockparser.
 	a.c.Set(key, mb, cache.DefaultExpiration)
 }
 
+func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, limit int) (*coords.MapBlockCoords, []*mapblockparser.MapBlock, error) {
+
+	blocks, err := a.accessor.FindLegacyBlocks(lastpos, limit)
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	mblist := make([]*mapblockparser.MapBlock, 0)
+	var newlastpos *coords.MapBlockCoords
+
+	for _, block := range blocks {
+
+		fields := logrus.Fields{
+			"x": block.Pos.X,
+			"y": block.Pos.Y,
+			"z": block.Pos.Z,
+		}
+		logrus.WithFields(fields).Debug("legacy mapblock")
+
+		key := getKey(block.Pos)
+
+		mapblock, err := mapblockparser.Parse(block.Data, block.Mtime)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		for _, listener := range a.listeners {
+			listener.OnParsedMapBlock(mapblock, block.Pos)
+		}
+
+		a.c.Set(key, mapblock, cache.DefaultExpiration)
+		mblist = append(mblist, mapblock)
+
+		newlastpos = &block.Pos
+	}
+
+	return newlastpos, mblist, nil
+}
+
 func (a *MapBlockAccessor) FindLatestMapBlocks(mintime int64, limit int) ([]*mapblockparser.MapBlock, error) {
 	blocks, err := a.accessor.FindLatestBlocks(mintime, limit)
 

mapserver.json

@@ -1,7 +1,7 @@
 {
 	"port": 8080,
-	"enableinitialrendering": false,
+	"enableinitialrendering": true,
-	"enableincrementalupdate": true,
+	"enableincrementalupdate": false,
 	"webdev": true,
 	"webapi": {
 		"enablemapblock": true