From 91aa1b6481cc9b8f19a1cd1c9f4c564adedb80d7 Mon Sep 17 00:00:00 2001 From: Thomas Rudin Date: Fri, 15 Feb 2019 17:33:49 +0100 Subject: [PATCH] initial rendering of ALL mapblocks --- server/db/accessor.go | 1 + server/db/postgres/initialblocks.go | 9 +++++++-- server/db/postgres/sql.go | 3 +-- server/db/sqlite/initialblocks.go | 4 ++++ server/go.mod | 1 + server/go.sum | 2 ++ server/mapblockaccessor/legacyblocks.go | 4 +++- server/static/js/LayerManager.js | 2 +- server/tilerendererjob/initial.go | 7 +++++++ 9 files changed, 27 insertions(+), 6 deletions(-) diff --git a/server/db/accessor.go b/server/db/accessor.go index 0909a07..422357b 100644 --- a/server/db/accessor.go +++ b/server/db/accessor.go @@ -17,6 +17,7 @@ type InitialBlocksResult struct { UnfilteredCount int HasMore bool Progress float64 + LastMtime int64 } type DBAccessor interface { diff --git a/server/db/postgres/initialblocks.go b/server/db/postgres/initialblocks.go index 83eb04f..717c612 100644 --- a/server/db/postgres/initialblocks.go +++ b/server/db/postgres/initialblocks.go @@ -92,8 +92,7 @@ func (this *PostgresAccessor) FindNextInitialBlocks(s settings.Settings, layers stridecount := this.intQuery(` select count(*) from blocks where posz >= $1 and posz <= $2 - and posy >= $3 and posy <= $4 - and mtime = 0`, + and posy >= $3 and posy <= $4`, minZ, maxZ, minY, maxY, ) @@ -128,6 +127,7 @@ func (this *PostgresAccessor) FindNextInitialBlocks(s settings.Settings, layers defer rows.Close() blocks := make([]*db.Block, 0) + var lastmtime int64 for { for rows.Next() { @@ -140,6 +140,10 @@ func (this *PostgresAccessor) FindNextInitialBlocks(s settings.Settings, layers return nil, err } + if mtime > lastmtime { + lastmtime = mtime + } + mb := convertRows(posx, posy, posz, data, mtime) blocks = append(blocks, mb) } @@ -153,6 +157,7 @@ func (this *PostgresAccessor) FindNextInitialBlocks(s settings.Settings, layers s.SetInt(SETTING_LAST_Y_BLOCK, lastyblock) result := &db.InitialBlocksResult{} + result.LastMtime = lastmtime result.Progress = float64(((lastyblock+128)*256)+(lastxblock+128)) / float64(256*256) result.List = blocks result.HasMore = true diff --git a/server/db/postgres/sql.go b/server/db/postgres/sql.go index 0807443..37cb594 100644 --- a/server/db/postgres/sql.go +++ b/server/db/postgres/sql.go @@ -3,8 +3,7 @@ package postgres const getBlocksByInitialTileQuery = ` select posx,posy,posz,data,mtime from blocks b -where b.mtime = 0 -and b.posx >= $1 +where b.posx >= $1 and b.posy >= $2 and b.posz >= $3 and b.posx <= $4 diff --git a/server/db/sqlite/initialblocks.go b/server/db/sqlite/initialblocks.go index df158a7..1ae537b 100644 --- a/server/db/sqlite/initialblocks.go +++ b/server/db/sqlite/initialblocks.go @@ -62,6 +62,10 @@ func (this *Sqlite3Accessor) FindNextInitialBlocks(s settings.Settings, layers [ return nil, err } + if mtime > result.LastMtime { + result.LastMtime = mtime + } + mb := convertRows(pos, data, mtime) // new position diff --git a/server/go.mod b/server/go.mod index 813368a..e103bfe 100644 --- a/server/go.mod +++ b/server/go.mod @@ -8,6 +8,7 @@ require ( github.com/gorilla/websocket v1.4.0 github.com/lib/pq v1.0.0 github.com/mattn/go-sqlite3 v1.10.0 + github.com/mjibson/esc v0.1.0 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.8.1 // indirect github.com/prometheus/client_golang v0.9.2 diff --git a/server/go.sum b/server/go.sum index bc9747e..83471a9 100644 --- a/server/go.sum +++ b/server/go.sum @@ -25,6 +25,8 @@ github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK86 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mjibson/esc v0.1.0 h1:5ch+murgrcwDFLOE2hwj0f7kE4xJfJhkSCAjSLY182o= +github.com/mjibson/esc v0.1.0/go.mod h1:9Hw9gxxfHulMF5OJKCyhYD7PzlSdhzXyaGEBRPH1OPs= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= diff --git a/server/mapblockaccessor/legacyblocks.go b/server/mapblockaccessor/legacyblocks.go index 469b7d7..1413797 100644 --- a/server/mapblockaccessor/legacyblocks.go +++ b/server/mapblockaccessor/legacyblocks.go @@ -15,23 +15,25 @@ type FindNextLegacyBlocksResult struct { List []*mapblockparser.MapBlock UnfilteredCount int Progress float64 + LastMtime int64 } func (a *MapBlockAccessor) FindNextLegacyBlocks(s settings.Settings, layers []*layer.Layer, limit int) (*FindNextLegacyBlocksResult, error) { nextResult, err := a.accessor.FindNextInitialBlocks(s, layers, limit) - blocks := nextResult.List if err != nil { return nil, err } + blocks := nextResult.List result := FindNextLegacyBlocksResult{} mblist := make([]*mapblockparser.MapBlock, 0) result.HasMore = nextResult.HasMore result.UnfilteredCount = nextResult.UnfilteredCount result.Progress = nextResult.Progress + result.LastMtime = nextResult.LastMtime for _, block := range blocks { diff --git a/server/static/js/LayerManager.js b/server/static/js/LayerManager.js index 4af098f..b0fb54a 100644 --- a/server/static/js/LayerManager.js +++ b/server/static/js/LayerManager.js @@ -3,7 +3,7 @@ function LayerManager(layers, map){ this.listeners = []; this.currentLayer = layers[0]; - this.layers = layer; + this.layers = layers; map.on('baselayerchange', function (e) { console.log("baselayerchange", e.layer); diff --git a/server/tilerendererjob/initial.go b/server/tilerendererjob/initial.go index 785cf53..6455a8f 100644 --- a/server/tilerendererjob/initial.go +++ b/server/tilerendererjob/initial.go @@ -15,6 +15,7 @@ type InitialRenderEvent struct { func initialRender(ctx *app.App, jobs chan *coords.TileCoords) { logrus.Info("Starting initial rendering job") + lastMtime := ctx.Settings.GetInt64(settings.SETTING_LAST_MTIME, 0) for true { start := time.Now() @@ -25,6 +26,10 @@ func initialRender(ctx *app.App, jobs chan *coords.TileCoords) { panic(err) } + if result.LastMtime > lastMtime { + lastMtime = result.LastMtime + } + if len(result.List) == 0 && !result.HasMore { ctx.Settings.SetBool(settings.SETTING_INITIAL_RUN, false) @@ -58,6 +63,8 @@ func initialRender(ctx *app.App, jobs chan *coords.TileCoords) { } logrus.WithFields(fields).Info("Initial rendering") + ctx.Settings.SetInt64(settings.SETTING_LAST_MTIME, lastMtime) + //tile gc ctx.TileDB.GC()