From 5f8234e2a99566938e785a5485ae807b2679a52d Mon Sep 17 00:00:00 2001 From: NatureFreshMilk Date: Fri, 15 Feb 2019 08:58:04 +0100 Subject: [PATCH] progress in db impl --- server/db/accessor.go | 1 + server/db/postgres/initialblocks.go | 4 ++- server/db/sqlite/initialblocks.go | 20 +++++++++++++-- server/mapblockaccessor/legacyblocks.go | 2 ++ server/tilerendererjob/initial.go | 34 +++++-------------------- 5 files changed, 30 insertions(+), 31 deletions(-) diff --git a/server/db/accessor.go b/server/db/accessor.go index 28939ab..e5d7028 100644 --- a/server/db/accessor.go +++ b/server/db/accessor.go @@ -16,6 +16,7 @@ type InitialBlocksResult struct { List []*Block UnfilteredCount int HasMore bool + Progress float64 } type DBAccessor interface { diff --git a/server/db/postgres/initialblocks.go b/server/db/postgres/initialblocks.go index 8d85ab7..6ee7df7 100644 --- a/server/db/postgres/initialblocks.go +++ b/server/db/postgres/initialblocks.go @@ -68,7 +68,7 @@ func (this *PostgresAccessor) FindNextInitialBlocks(s settings.Settings, layers "pos2": tcr.Pos2, "tile": tc, } - log.WithFields(fields).Info("Initial-Query") + log.WithFields(fields).Debug("Initial-Query") minX := int(math.Min(float64(tcr.Pos1.X), float64(tcr.Pos2.X))) maxX := int(math.Max(float64(tcr.Pos1.X), float64(tcr.Pos2.X))) @@ -102,6 +102,7 @@ func (this *PostgresAccessor) FindNextInitialBlocks(s settings.Settings, layers s.SetInt(SETTING_LAST_Y_BLOCK, lastyblock+1) result := &db.InitialBlocksResult{} + result.Progress = float64(((lastyblock+128) * 256) + (lastxblock+128)) / float64(256*256) result.HasMore = true return result, nil } @@ -142,6 +143,7 @@ func (this *PostgresAccessor) FindNextInitialBlocks(s settings.Settings, layers s.SetInt(SETTING_LAST_Y_BLOCK, lastyblock) result := &db.InitialBlocksResult{} + result.Progress = float64(((lastyblock+128) * 256) + (lastxblock+128)) / float64(256*256) result.List = blocks result.HasMore = true diff --git a/server/db/sqlite/initialblocks.go b/server/db/sqlite/initialblocks.go index d877fc3..d8ae209 100644 --- a/server/db/sqlite/initialblocks.go +++ b/server/db/sqlite/initialblocks.go @@ -10,6 +10,8 @@ import ( const ( SETTING_LAST_POS = "last_pos" + SETTING_TOTAL_LEGACY_COUNT = "total_legacy_count" + SETTING_PROCESSED_LEGACY_COUNT = "total_processed_legacy_count" ) const getLastBlockQuery = ` @@ -22,13 +24,24 @@ limit ? ` func (this *Sqlite3Accessor) FindNextInitialBlocks(s settings.Settings, layers []*layer.Layer, limit int) (*db.InitialBlocksResult, error) { - result := &db.InitialBlocksResult{} blocks := make([]*db.Block, 0) - lastpos := s.GetInt64(SETTING_LAST_POS, coords.MinPlainCoord-1) + processedcount := s.GetInt64(SETTING_PROCESSED_LEGACY_COUNT, 0) + totallegacycount := s.GetInt64(SETTING_TOTAL_LEGACY_COUNT, -1) + if totallegacycount == -1 { + //Query from db + totallegacycount, err := this.CountBlocks(0, 0) + + if err != nil { + panic(err) + } + + s.SetInt64(SETTING_TOTAL_LEGACY_COUNT, int64(totallegacycount)) + } + rows, err := this.db.Query(getLastBlockQuery, lastpos, limit) if err != nil { return nil, err @@ -62,6 +75,9 @@ func (this *Sqlite3Accessor) FindNextInitialBlocks(s settings.Settings, layers [ } } + s.SetInt64(SETTING_PROCESSED_LEGACY_COUNT, int64(result.UnfilteredCount) + processedcount) + + result.Progress = float64(processedcount) / float64(totallegacycount) result.List = blocks //Save current positions of initial run diff --git a/server/mapblockaccessor/legacyblocks.go b/server/mapblockaccessor/legacyblocks.go index fe6e39a..fa09073 100644 --- a/server/mapblockaccessor/legacyblocks.go +++ b/server/mapblockaccessor/legacyblocks.go @@ -14,6 +14,7 @@ type FindNextLegacyBlocksResult struct { HasMore bool List []*mapblockparser.MapBlock UnfilteredCount int + Progress float64 } func (a *MapBlockAccessor) FindNextLegacyBlocks(s settings.Settings, layers []*layer.Layer, limit int) (*FindNextLegacyBlocksResult, error) { @@ -30,6 +31,7 @@ func (a *MapBlockAccessor) FindNextLegacyBlocks(s settings.Settings, layers []*l mblist := make([]*mapblockparser.MapBlock, 0) result.HasMore = nextResult.HasMore result.UnfilteredCount = nextResult.UnfilteredCount + result.Progress = nextResult.Progress for _, block := range blocks { diff --git a/server/tilerendererjob/initial.go b/server/tilerendererjob/initial.go index b617d91..785cf53 100644 --- a/server/tilerendererjob/initial.go +++ b/server/tilerendererjob/initial.go @@ -10,21 +10,11 @@ import ( ) type InitialRenderEvent struct { - Progress int `json:"progress"` + Progress float64 `json:"progress"` } func initialRender(ctx *app.App, jobs chan *coords.TileCoords) { - - totalLegacyCount, err := ctx.Blockdb.CountBlocks(0, 0) - - if err != nil { - panic(err) - } - - fields := logrus.Fields{ - "totalLegacyCount": totalLegacyCount, - } - logrus.WithFields(fields).Info("Starting initial rendering job") + logrus.Info("Starting initial rendering job") for true { start := time.Now() @@ -35,38 +25,27 @@ func initialRender(ctx *app.App, jobs chan *coords.TileCoords) { panic(err) } - legacyProcessed := ctx.Settings.GetInt(settings.SETTING_LEGACY_PROCESSED, 0) - if len(result.List) == 0 && !result.HasMore { ctx.Settings.SetBool(settings.SETTING_INITIAL_RUN, false) ev := InitialRenderEvent{ - Progress: 100, + Progress: 1, } ctx.WebEventbus.Emit("initial-render-progress", &ev) - fields := logrus.Fields{ - "legacyblocks": legacyProcessed, - } - logrus.WithFields(fields).Info("initial rendering complete") + logrus.Info("initial rendering complete") return } tiles := renderMapblocks(ctx, jobs, result.List) - legacyProcessed += result.UnfilteredCount - - ctx.Settings.SetInt(settings.SETTING_LEGACY_PROCESSED, legacyProcessed) - t := time.Now() elapsed := t.Sub(start) - progress := int(float64(legacyProcessed) / float64(totalLegacyCount) * 100) - ev := InitialRenderEvent{ - Progress: progress, + Progress: result.Progress, } ctx.WebEventbus.Emit("initial-render-progress", &ev) @@ -74,8 +53,7 @@ func initialRender(ctx *app.App, jobs chan *coords.TileCoords) { fields := logrus.Fields{ "mapblocks": len(result.List), "tiles": tiles, - "processed": legacyProcessed, - "progress%": progress, + "progress%": int(result.Progress * 100), "elapsed": elapsed, } logrus.WithFields(fields).Info("Initial rendering")