db refactoring

server/db/accessor.go

@@ -12,13 +12,7 @@ type Block struct {
 
 type DBAccessor interface {
 	Migrate() error
-	/**
+	FindBlocks(lastpos coords.MapBlockCoords, lastmtime int64, limit int) ([]Block, error)
-	 * find old (pre-mapserver) mapblocks by lastpos
+	CountBlocks(frommtime, tomtime int64) (int, error)
-	 * used only on initial rendering
-	 */
-	FindLegacyBlocks(lastpos coords.MapBlockCoords, limit int) ([]Block, error)
-	CountLegacyBlocks() (int, error)
-
-	FindLatestBlocks(mintime int64, limit int) ([]Block, error)
 	GetBlock(pos coords.MapBlockCoords) (*Block, error)
 }

server/db/sqlite.go

@@ -72,20 +72,20 @@ func convertRows(pos int64, data []byte, mtime int64) Block {
 	return Block{Pos: c, Data: data, Mtime: mtime}
 }
 
-const getLegacyBlockQuery = `
+const getLastBlockQuery = `
 select pos,data,mtime
 from blocks b
-where b.mtime == 0
+where b.mtime > ?
 and b.pos > ?
-order by b.pos asc
+order by b.pos asc, b.mtime asc
 limit ?
 `
 
-func (db *Sqlite3Accessor) FindLegacyBlocks(lastpos coords.MapBlockCoords, limit int) ([]Block, error) {
+func (db *Sqlite3Accessor) FindBlocks(lastpos coords.MapBlockCoords, lastmtime int64, limit int) ([]Block, error) {
 	blocks := make([]Block, 0)
 	pc := coords.CoordToPlain(lastpos)
 
-	rows, err := db.db.Query(getLegacyBlockQuery, pc, limit)
+	rows, err := db.db.Query(getLastBlockQuery, lastmtime, pc, limit)
 	if err != nil {
 		return nil, err
 	}
@@ -109,12 +109,12 @@ func (db *Sqlite3Accessor) FindLegacyBlocks(lastpos coords.MapBlockCoords, limit
 	return blocks, nil
 }
 
-const countLegacyBlocksQuery = `
+const countBlocksQuery = `
-select count(*) from blocks b where b.mtime = 0
+select count(*) from blocks b where b.mtime >= ? and b.mtime <= ?
 `
 
-func (db *Sqlite3Accessor) CountLegacyBlocks() (int, error) {
+func (db *Sqlite3Accessor) CountBlocks(frommtime, tomtime int64) (int, error) {
-	rows, err := db.db.Query(countLegacyBlocksQuery)
+	rows, err := db.db.Query(countBlocksQuery, frommtime, tomtime)
 	if err != nil {
 		return 0, err
 	}
@@ -135,40 +135,6 @@ func (db *Sqlite3Accessor) CountLegacyBlocks() (int, error) {
 	return 0, nil
 }
 
-const getLatestBlockQuery = `
-select pos,data,mtime
-from blocks b
-where b.mtime > ?
-order by b.mtime asc
-limit ?
-`
-
-func (db *Sqlite3Accessor) FindLatestBlocks(mintime int64, limit int) ([]Block, error) {
-	blocks := make([]Block, 0)
-
-	rows, err := db.db.Query(getLatestBlockQuery, mintime, limit)
-	if err != nil {
-		return nil, err
-	}
-
-	defer rows.Close()
-
-	for rows.Next() {
-		var pos int64
-		var data []byte
-		var mtime int64
-
-		err = rows.Scan(&pos, &data, &mtime)
-		if err != nil {
-			return nil, err
-		}
-
-		mb := convertRows(pos, data, mtime)
-		blocks = append(blocks, mb)
-	}
-
-	return blocks, nil
-}
 
 const getBlockQuery = `
 select pos,data,mtime from blocks b where b.pos = ?

server/db/sqlite_test.go

@@ -93,7 +93,7 @@ func TestMigrateAndQueryCount(t *testing.T) {
 		panic(err)
 	}
 
-	count, err := a.CountLegacyBlocks()
+	count, err := a.CountBlocks(0, 0)
 	if err != nil {
 		panic(err)
 	}

server/initialrenderer/job.go

@@ -1,125 +0,0 @@
-package initialrenderer
-
-import (
-	"github.com/sirupsen/logrus"
-	"mapserver/app"
-	"mapserver/coords"
-	"strconv"
-	"time"
-)
-
-func getTileKey(tc *coords.TileCoords) string {
-	return strconv.Itoa(tc.X) + "/" + strconv.Itoa(tc.Y) + "/" + strconv.Itoa(tc.Zoom)
-}
-
-func worker(ctx *app.App, coords <-chan *coords.TileCoords) {
-	for tc := range coords {
-		ctx.Objectdb.RemoveTile(tc)
-		_, err := ctx.Tilerenderer.Render(tc, 2)
-		if err != nil {
-			panic(err)
-		}
-	}
-}
-
-func Job(ctx *app.App) {
-
-	totalLegacyCount, err := ctx.Blockdb.CountLegacyBlocks()
-	if err != nil {
-		panic(err)
-	}
-
-	fields := logrus.Fields{
-		"totalLegacyCount": totalLegacyCount,
-	}
-	logrus.WithFields(fields).Info("Starting initial rendering")
-	tilecount := 0
-
-	rstate := ctx.Config.RenderState
-	lastcoords := coords.NewMapBlockCoords(rstate.LastX, rstate.LastY, rstate.LastZ)
-
-	jobs := make(chan *coords.TileCoords, ctx.Config.InitialRenderingQueue)
-
-	for i := 0; i < ctx.Config.InitialRenderingJobs; i++ {
-		go worker(ctx, jobs)
-	}
-
-	for true {
-		start := time.Now()
-
-		result, err := ctx.BlockAccessor.FindLegacyMapBlocks(lastcoords, ctx.Config.InitialRenderingFetchLimit, ctx.Config.Layers)
-
-		if err != nil {
-			panic(err)
-		}
-
-		if len(result.List) == 0 && !result.HasMore {
-			fields = logrus.Fields{
-				"blocks": rstate.LegacyProcessed,
-				"tiles":  tilecount,
-			}
-			logrus.WithFields(fields).Info("Initial rendering complete")
-			close(jobs)
-			rstate.InitialRun = false
-			ctx.Config.Save()
-
-			break
-		}
-
-		lastcoords = *result.LastPos
-
-		tileRenderedMap := make(map[string]bool)
-
-		for i := 12; i >= 1; i-- {
-			for _, mb := range result.List {
-				//13
-				tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers)
-
-				//12-1
-				tc = tc.ZoomOut(13 - i)
-
-				key := getTileKey(tc)
-
-				if tileRenderedMap[key] {
-					continue
-				}
-
-				tileRenderedMap[key] = true
-
-				fields = logrus.Fields{
-					"X":       tc.X,
-					"Y":       tc.Y,
-					"Zoom":    tc.Zoom,
-					"LayerId": tc.LayerId,
-				}
-				logrus.WithFields(fields).Debug("Dispatching tile rendering (z11-1)")
-
-				tilecount++
-				jobs <- tc
-			}
-		}
-
-		//Save current positions of initial run
-		rstate.LastX = lastcoords.X
-		rstate.LastY = lastcoords.Y
-		rstate.LastZ = lastcoords.Z
-		rstate.LegacyProcessed += result.UnfilteredCount
-		ctx.Config.Save()
-
-		t := time.Now()
-		elapsed := t.Sub(start)
-
-		progress := int(float64(rstate.LegacyProcessed) / float64(totalLegacyCount) * 100)
-
-		fields = logrus.Fields{
-			"count":     len(result.List),
-			"processed": rstate.LegacyProcessed,
-			"progress%": progress,
-			"X":         lastcoords.X,
-			"Y":         lastcoords.Y,
-			"Z":         lastcoords.Z,
-			"elapsed":   elapsed,
-		}
-		logrus.WithFields(fields).Info("Initial rendering")
-	}
-}