Koldun/mapserver
1
0
Fork 0
forked from MTSR/mapserver
Code Pull Requests Activity

rendering progress

Browse Source
This commit is contained in:
NatureFreshMilk 2019-01-23 12:46:45 +01:00
parent bb6953ba9c
commit 1cd324f483
6 changed files with 66 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
server/app/config.go
View File

@ -28,6 +28,7 @@ type WebApiConfig struct {
type RenderStateType struct {
//Initial rendering flag (true=still active)
InitialRun bool `json:"initialrun"`
LegacyProcessed int `json:"legacyprocessed"`
//Last initial rendering coords
LastX int `json:"lastx"`

2
server/db/accessor.go
View File

@ -17,8 +17,8 @@ type DBAccessor interface {
* used only on initial rendering
*/
FindLegacyBlocks(lastpos coords.MapBlockCoords, limit int) ([]Block, error)
CountLegacyBlocks() (int, error)
FindLatestBlocks(mintime int64, limit int) ([]Block, error)
CountBlocks(pos1 coords.MapBlockCoords, pos2 coords.MapBlockCoords) (int, error)
GetBlock(pos coords.MapBlockCoords) (*Block, error)
}

81
server/db/sqlite.go
View File

@ -2,11 +2,9 @@ package db
import (
"database/sql"
"errors"
_ "github.com/mattn/go-sqlite3"
"github.com/sirupsen/logrus"
"mapserver/coords"
"strings"
"time"
)
/*
@ -110,6 +108,32 @@ func (db *Sqlite3Accessor) FindLegacyBlocks(lastpos coords.MapBlockCoords, limit
return blocks, nil
}
const countLegacyBlocksQuery = `
select count(*) from blocks b where b.mtime = 0
`
func (db *Sqlite3Accessor) CountLegacyBlocks() (int, error){
rows, err := db.db.Query(countLegacyBlocksQuery)
if err != nil {
return 0, err
}
defer rows.Close()
if rows.Next() {
var count int64
err = rows.Scan(&count)
if err != nil {
return 0, err
}
return int(count), nil
}
return 0, nil
}
const getLatestBlockQuery = `
select pos,data,mtime
from blocks b
@ -176,59 +200,6 @@ func (db *Sqlite3Accessor) GetBlock(pos coords.MapBlockCoords) (*Block, error) {
return nil, nil
}
func sortAsc(a, b int) (int, int) {
if a > b {
return b, a
} else {
return a, b
}
}
func (db *Sqlite3Accessor) CountBlocks(pos1 coords.MapBlockCoords, pos2 coords.MapBlockCoords) (int, error) {
poslist := make([]interface{}, 0)
minX, maxX := sortAsc(pos1.X, pos2.X)
minY, maxY := sortAsc(pos1.Y, pos2.Y)
minZ, maxZ := sortAsc(pos1.Z, pos2.Z)
for x := minX; x <= maxX; x++ {
for y := minY; y <= maxY; y++ {
for z := minZ; z <= maxZ; z++ {
poslist = append(poslist, coords.CoordToPlain(coords.NewMapBlockCoords(x, y, z)))
}
}
}
if len(poslist) > 999 {
//https://stackoverflow.com/questions/7106016/too-many-sql-variables-error-in-django-witih-sqlite3
//TODO: return before nested for loops
return -1, nil
}
getBlocksQuery := "select count(*) from blocks b where b.pos in (?" + strings.Repeat(",?", len(poslist)-1) + ")"
rows, err := db.db.Query(getBlocksQuery, poslist...)
if err != nil {
return 0, err
}
defer rows.Close()
if rows.Next() {
var count int
err = rows.Scan(&count)
if err != nil {
return 0, err
}
return count, nil
}
return 0, errors.New("No rows returned")
}
func NewSqliteAccessor(filename string) (*Sqlite3Accessor, error) {
db, err := sql.Open("sqlite3", filename+"?mode=ro")
if err != nil {

10
server/db/sqlite_test.go
View File

@ -75,7 +75,7 @@ func TestMigrateAndQuery(t *testing.T) {
}
func TestMigrateAndQueryStride(t *testing.T) {
func TestMigrateAndQueryCount(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "TestMigrateAndQueryStride.*.sqlite")
if err != nil {
panic(err)
@ -93,14 +93,12 @@ func TestMigrateAndQueryStride(t *testing.T) {
panic(err)
}
count, err := a.CountBlocks(coords.NewMapBlockCoords(0, -1, 0), coords.NewMapBlockCoords(0, 10, 0))
count, err := a.CountLegacyBlocks()
if err != nil {
panic(err)
}
if count == 0 {
t.Fatal("no data")
if count <= 0 {
t.Fatal("zero count")
}
}

24
server/initialrenderer/job.go
View File

@ -17,9 +17,13 @@ func Job(ctx *app.App) {
fields := logrus.Fields{}
logrus.WithFields(fields).Info("Starting initial rendering")
blockcount := 0
tilecount := 0
totalLegacyCount, err := ctx.Blockdb.CountLegacyBlocks()
if err != nil {
panic(err)
}
rstate := ctx.Config.RenderState
lastcoords := coords.NewMapBlockCoords(rstate.LastX, rstate.LastY, rstate.LastZ)
@ -27,15 +31,15 @@ func Job(ctx *app.App) {
for true {
start := time.Now()
hasMore, newlastcoords, mblist, err := ctx.BlockAccessor.FindLegacyMapBlocks(lastcoords, ctx.Config.InitialRenderingFetchLimit, ctx.Config.Layers)
result, err := ctx.BlockAccessor.FindLegacyMapBlocks(lastcoords, ctx.Config.InitialRenderingFetchLimit, ctx.Config.Layers)
if err != nil {
panic(err)
}
if len(mblist) == 0 && !hasMore {
if len(result.List) == 0 && !result.HasMore {
fields = logrus.Fields{
"blocks": blockcount,
"blocks": rstate.LegacyProcessed,
"tiles": tilecount,
}
logrus.WithFields(fields).Info("Initial rendering complete")
@ -45,13 +49,12 @@ func Job(ctx *app.App) {
break
}
blockcount += len(mblist)
lastcoords = *newlastcoords
lastcoords = *result.LastPos
tileRenderedMap := make(map[string]bool)
for i := 12; i >= 1; i-- {
for _, mb := range mblist {
for _, mb := range result.List {
//13
tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers)
@ -87,13 +90,18 @@ func Job(ctx *app.App) {
rstate.LastX = lastcoords.X
rstate.LastY = lastcoords.Y
rstate.LastZ = lastcoords.Z
rstate.LegacyProcessed += result.UnfilteredCount
ctx.Config.Save()
t := time.Now()
elapsed := t.Sub(start)
progress := int( float64(rstate.LegacyProcessed) / float64(totalLegacyCount) * 100 )
fields = logrus.Fields{
"count": len(mblist),
"count": len(result.List),
"processed": rstate.LegacyProcessed,
"progress%": progress,
"X": lastcoords.X,
"Y": lastcoords.Y,
"Z": lastcoords.Z,

23
server/mapblockaccessor/mapblockaccessor.go
View File

@ -41,17 +41,27 @@ func (a *MapBlockAccessor) Update(pos coords.MapBlockCoords, mb *mapblockparser.
a.c.Set(key, mb, cache.DefaultExpiration)
}
func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, limit int, layerfilter []layer.Layer) (bool, *coords.MapBlockCoords, []*mapblockparser.MapBlock, error) {
type LegacyMapBlocksResult struct {
HasMore bool
LastPos *coords.MapBlockCoords
List []*mapblockparser.MapBlock
UnfilteredCount int
}
func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, limit int, layerfilter []layer.Layer) (*LegacyMapBlocksResult, error) {
blocks, err := a.accessor.FindLegacyBlocks(lastpos, limit)
if err != nil {
return false, nil, nil, err
return nil, err
}
result := LegacyMapBlocksResult{}
mblist := make([]*mapblockparser.MapBlock, 0)
var newlastpos *coords.MapBlockCoords
hasMore := len(blocks) == limit
result.HasMore = len(blocks) == limit
result.UnfilteredCount = len(blocks)
for _, block := range blocks {
newlastpos = &block.Pos
@ -79,7 +89,7 @@ func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, li
mapblock, err := mapblockparser.Parse(block.Data, block.Mtime, block.Pos)
if err != nil {
return false, nil, nil, err
return nil, err
}
for _, listener := range a.listeners {
@ -91,7 +101,10 @@ func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, li
}
return hasMore, newlastpos, mblist, nil
result.LastPos = newlastpos
result.List = mblist
return &result, nil
}
func (a *MapBlockAccessor) FindLatestMapBlocks(mintime int64, limit int, layerfilter []layer.Layer) ([]*mapblockparser.MapBlock, error) {