rendering progress

server/app/config.go

@@ -28,6 +28,7 @@ type WebApiConfig struct {
 type RenderStateType struct {
 	//Initial rendering flag (true=still active)
 	InitialRun bool `json:"initialrun"`
+	LegacyProcessed int `json:"legacyprocessed"`
 
 	//Last initial rendering coords
 	LastX int `json:"lastx"`

server/db/accessor.go

@@ -17,8 +17,8 @@ type DBAccessor interface {
 	 * used only on initial rendering
 	 */
 	FindLegacyBlocks(lastpos coords.MapBlockCoords, limit int) ([]Block, error)
+	CountLegacyBlocks() (int, error)
 
 	FindLatestBlocks(mintime int64, limit int) ([]Block, error)
-	CountBlocks(pos1 coords.MapBlockCoords, pos2 coords.MapBlockCoords) (int, error)
 	GetBlock(pos coords.MapBlockCoords) (*Block, error)
 }

server/db/sqlite.go

@@ -2,11 +2,9 @@ package db
 
 import (
 	"database/sql"
-	"errors"
 	_ "github.com/mattn/go-sqlite3"
 	"github.com/sirupsen/logrus"
 	"mapserver/coords"
-	"strings"
 	"time"
 )
 /*
@@ -110,6 +108,32 @@ func (db *Sqlite3Accessor) FindLegacyBlocks(lastpos coords.MapBlockCoords, limit
 	return blocks, nil
 }
 
+const countLegacyBlocksQuery = `
+select count(*) from blocks b where b.mtime = 0
+`
+
+func (db *Sqlite3Accessor) CountLegacyBlocks() (int, error){
+	rows, err := db.db.Query(countLegacyBlocksQuery)
+	if err != nil {
+		return 0, err
+	}
+
+	defer rows.Close()
+
+	if rows.Next() {
+		var count int64
+
+		err = rows.Scan(&count)
+		if err != nil {
+			return 0, err
+		}
+
+		return int(count), nil
+	}
+
+	return 0, nil
+}
+
 const getLatestBlockQuery = `
 select pos,data,mtime
 from blocks b
@@ -176,59 +200,6 @@ func (db *Sqlite3Accessor) GetBlock(pos coords.MapBlockCoords) (*Block, error) {
 	return nil, nil
 }
 
-func sortAsc(a, b int) (int, int) {
-	if a > b {
-		return b, a
-	} else {
-		return a, b
-	}
-}
-
-func (db *Sqlite3Accessor) CountBlocks(pos1 coords.MapBlockCoords, pos2 coords.MapBlockCoords) (int, error) {
-
-	poslist := make([]interface{}, 0)
-
-	minX, maxX := sortAsc(pos1.X, pos2.X)
-	minY, maxY := sortAsc(pos1.Y, pos2.Y)
-	minZ, maxZ := sortAsc(pos1.Z, pos2.Z)
-
-	for x := minX; x <= maxX; x++ {
-		for y := minY; y <= maxY; y++ {
-			for z := minZ; z <= maxZ; z++ {
-				poslist = append(poslist, coords.CoordToPlain(coords.NewMapBlockCoords(x, y, z)))
-			}
-		}
-	}
-
-	if len(poslist) > 999 {
-		//https://stackoverflow.com/questions/7106016/too-many-sql-variables-error-in-django-witih-sqlite3
-		//TODO: return before nested for loops
-		return -1, nil
-	}
-
-	getBlocksQuery := "select count(*) from blocks b where b.pos in (?" + strings.Repeat(",?", len(poslist)-1) + ")"
-
-	rows, err := db.db.Query(getBlocksQuery, poslist...)
-	if err != nil {
-		return 0, err
-	}
-
-	defer rows.Close()
-
-	if rows.Next() {
-		var count int
-
-		err = rows.Scan(&count)
-		if err != nil {
-			return 0, err
-		}
-
-		return count, nil
-	}
-
-	return 0, errors.New("No rows returned")
-}
-
 func NewSqliteAccessor(filename string) (*Sqlite3Accessor, error) {
 	db, err := sql.Open("sqlite3", filename+"?mode=ro")
 	if err != nil {

server/db/sqlite_test.go

@@ -75,7 +75,7 @@ func TestMigrateAndQuery(t *testing.T) {
 
 }
 
-func TestMigrateAndQueryStride(t *testing.T) {
+func TestMigrateAndQueryCount(t *testing.T) {
 	tmpfile, err := ioutil.TempFile("", "TestMigrateAndQueryStride.*.sqlite")
 	if err != nil {
 		panic(err)
@@ -93,14 +93,12 @@ func TestMigrateAndQueryStride(t *testing.T) {
 		panic(err)
 	}
 
-	count, err := a.CountBlocks(coords.NewMapBlockCoords(0, -1, 0), coords.NewMapBlockCoords(0, 10, 0))
+	count, err := a.CountLegacyBlocks()
-
 	if err != nil {
 		panic(err)
 	}
 
-	if count == 0 {
+	if count <= 0 {
-		t.Fatal("no data")
+		t.Fatal("zero count")
 	}
-
 }

server/initialrenderer/job.go

@@ -17,9 +17,13 @@ func Job(ctx *app.App) {
 
 	fields := logrus.Fields{}
 	logrus.WithFields(fields).Info("Starting initial rendering")
-	blockcount := 0
 	tilecount := 0
 
+	totalLegacyCount, err := ctx.Blockdb.CountLegacyBlocks()
+	if err != nil {
+		panic(err)
+	}
+
 	rstate := ctx.Config.RenderState
 
 	lastcoords := coords.NewMapBlockCoords(rstate.LastX, rstate.LastY, rstate.LastZ)
@@ -27,15 +31,15 @@ func Job(ctx *app.App) {
 	for true {
 		start := time.Now()
 
-		hasMore, newlastcoords, mblist, err := ctx.BlockAccessor.FindLegacyMapBlocks(lastcoords, ctx.Config.InitialRenderingFetchLimit, ctx.Config.Layers)
+		result, err := ctx.BlockAccessor.FindLegacyMapBlocks(lastcoords, ctx.Config.InitialRenderingFetchLimit, ctx.Config.Layers)
 
 		if err != nil {
 			panic(err)
 		}
 
-		if len(mblist) == 0 && !hasMore {
+		if len(result.List) == 0 && !result.HasMore {
 			fields = logrus.Fields{
-				"blocks": blockcount,
+				"blocks": rstate.LegacyProcessed,
 				"tiles":  tilecount,
 			}
 			logrus.WithFields(fields).Info("Initial rendering complete")
@@ -45,13 +49,12 @@ func Job(ctx *app.App) {
 			break
 		}
 
-		blockcount += len(mblist)
+		lastcoords = *result.LastPos
-		lastcoords = *newlastcoords
 
 		tileRenderedMap := make(map[string]bool)
 
 		for i := 12; i >= 1; i-- {
-			for _, mb := range mblist {
+			for _, mb := range result.List {
 				//13
 				tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers)
 
@@ -87,13 +90,18 @@ func Job(ctx *app.App) {
 		rstate.LastX = lastcoords.X
 		rstate.LastY = lastcoords.Y
 		rstate.LastZ = lastcoords.Z
+		rstate.LegacyProcessed += result.UnfilteredCount
 		ctx.Config.Save()
 
 		t := time.Now()
 		elapsed := t.Sub(start)
 
+		progress := int( float64(rstate.LegacyProcessed) / float64(totalLegacyCount) * 100 )
+
 		fields = logrus.Fields{
-			"count":   len(mblist),
+			"count":   len(result.List),
+			"processed": rstate.LegacyProcessed,
+			"progress%": progress,
 			"X":       lastcoords.X,
 			"Y":       lastcoords.Y,
 			"Z":       lastcoords.Z,

server/mapblockaccessor/mapblockaccessor.go

@@ -41,17 +41,27 @@ func (a *MapBlockAccessor) Update(pos coords.MapBlockCoords, mb *mapblockparser.
 	a.c.Set(key, mb, cache.DefaultExpiration)
 }
 
-func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, limit int, layerfilter []layer.Layer) (bool, *coords.MapBlockCoords, []*mapblockparser.MapBlock, error) {
+type LegacyMapBlocksResult struct {
+	HasMore bool
+	LastPos *coords.MapBlockCoords
+	List []*mapblockparser.MapBlock
+	UnfilteredCount int
+}
+
+func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, limit int, layerfilter []layer.Layer) (*LegacyMapBlocksResult, error) {
 
 	blocks, err := a.accessor.FindLegacyBlocks(lastpos, limit)
 
 	if err != nil {
-		return false, nil, nil, err
+		return nil, err
 	}
 
+	result := LegacyMapBlocksResult{}
+
 	mblist := make([]*mapblockparser.MapBlock, 0)
 	var newlastpos *coords.MapBlockCoords
-	hasMore := len(blocks) == limit
+	result.HasMore = len(blocks) == limit
+	result.UnfilteredCount = len(blocks)
 
 	for _, block := range blocks {
 		newlastpos = &block.Pos
@@ -79,7 +89,7 @@ func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, li
 
 		mapblock, err := mapblockparser.Parse(block.Data, block.Mtime, block.Pos)
 		if err != nil {
-			return false, nil, nil, err
+			return nil, err
 		}
 
 		for _, listener := range a.listeners {
@@ -91,7 +101,10 @@ func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, li
 
 	}
 
-	return hasMore, newlastpos, mblist, nil
+	result.LastPos = newlastpos
+	result.List = mblist
+
+	return &result, nil
 }
 
 func (a *MapBlockAccessor) FindLatestMapBlocks(mintime int64, limit int, layerfilter []layer.Layer) ([]*mapblockparser.MapBlock, error) {