Koldun/mapserver
1
0
Fork 0
forked from MTSR/mapserver
Code Pull Requests Activity

db refactoring

Browse Source
This commit is contained in:
NatureFreshMilk 2019-01-24 07:22:28 +01:00
parent 86c63d84ba
commit 3748489b22
4 changed files with 12 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
server/db/accessor.go
View File

@ -12,13 +12,7 @@ type Block struct {
type DBAccessor interface {
Migrate() error
/**
* find old (pre-mapserver) mapblocks by lastpos
* used only on initial rendering
*/
FindLegacyBlocks(lastpos coords.MapBlockCoords, limit int) ([]Block, error)
CountLegacyBlocks() (int, error)
FindLatestBlocks(mintime int64, limit int) ([]Block, error)
FindBlocks(lastpos coords.MapBlockCoords, lastmtime int64, limit int) ([]Block, error)
CountBlocks(frommtime, tomtime int64) (int, error)
GetBlock(pos coords.MapBlockCoords) (*Block, error)
}

52
server/db/sqlite.go
View File

@ -72,20 +72,20 @@ func convertRows(pos int64, data []byte, mtime int64) Block {
return Block{Pos: c, Data: data, Mtime: mtime}
}
const getLegacyBlockQuery = `
const getLastBlockQuery = `
select pos,data,mtime
from blocks b
where b.mtime == 0
where b.mtime > ?
and b.pos > ?
order by b.pos asc
order by b.pos asc, b.mtime asc
limit ?
`
func (db *Sqlite3Accessor) FindLegacyBlocks(lastpos coords.MapBlockCoords, limit int) ([]Block, error) {
func (db *Sqlite3Accessor) FindBlocks(lastpos coords.MapBlockCoords, lastmtime int64, limit int) ([]Block, error) {
blocks := make([]Block, 0)
pc := coords.CoordToPlain(lastpos)
rows, err := db.db.Query(getLegacyBlockQuery, pc, limit)
rows, err := db.db.Query(getLastBlockQuery, lastmtime, pc, limit)
if err != nil {
return nil, err
}
@ -109,12 +109,12 @@ func (db *Sqlite3Accessor) FindLegacyBlocks(lastpos coords.MapBlockCoords, limit
return blocks, nil
}
const countLegacyBlocksQuery = `
select count(*) from blocks b where b.mtime = 0
const countBlocksQuery = `
select count(*) from blocks b where b.mtime >= ? and b.mtime <= ?
`
func (db *Sqlite3Accessor) CountLegacyBlocks() (int, error) {
rows, err := db.db.Query(countLegacyBlocksQuery)
func (db *Sqlite3Accessor) CountBlocks(frommtime, tomtime int64) (int, error) {
rows, err := db.db.Query(countBlocksQuery, frommtime, tomtime)
if err != nil {
return 0, err
}
@ -135,40 +135,6 @@ func (db *Sqlite3Accessor) CountLegacyBlocks() (int, error) {
return 0, nil
}
const getLatestBlockQuery = `
select pos,data,mtime
from blocks b
where b.mtime > ?
order by b.mtime asc
limit ?
`
func (db *Sqlite3Accessor) FindLatestBlocks(mintime int64, limit int) ([]Block, error) {
blocks := make([]Block, 0)
rows, err := db.db.Query(getLatestBlockQuery, mintime, limit)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var pos int64
var data []byte
var mtime int64
err = rows.Scan(&pos, &data, &mtime)
if err != nil {
return nil, err
}
mb := convertRows(pos, data, mtime)
blocks = append(blocks, mb)
}
return blocks, nil
}
const getBlockQuery = `
select pos,data,mtime from blocks b where b.pos = ?

2
server/db/sqlite_test.go
View File

@ -93,7 +93,7 @@ func TestMigrateAndQueryCount(t *testing.T) {
panic(err)
}
count, err := a.CountLegacyBlocks()
count, err := a.CountBlocks(0, 0)
if err != nil {
panic(err)
}

125
server/initialrenderer/job.go
View File

@ -1,125 +0,0 @@
package initialrenderer
import (
"github.com/sirupsen/logrus"
"mapserver/app"
"mapserver/coords"
"strconv"
"time"
)
func getTileKey(tc *coords.TileCoords) string {
return strconv.Itoa(tc.X) + "/" + strconv.Itoa(tc.Y) + "/" + strconv.Itoa(tc.Zoom)
}
func worker(ctx *app.App, coords <-chan *coords.TileCoords) {
for tc := range coords {
ctx.Objectdb.RemoveTile(tc)
_, err := ctx.Tilerenderer.Render(tc, 2)
if err != nil {
panic(err)
}
}
}
func Job(ctx *app.App) {
totalLegacyCount, err := ctx.Blockdb.CountLegacyBlocks()
if err != nil {
panic(err)
}
fields := logrus.Fields{
"totalLegacyCount": totalLegacyCount,
}
logrus.WithFields(fields).Info("Starting initial rendering")
tilecount := 0
rstate := ctx.Config.RenderState
lastcoords := coords.NewMapBlockCoords(rstate.LastX, rstate.LastY, rstate.LastZ)
jobs := make(chan *coords.TileCoords, ctx.Config.InitialRenderingQueue)
for i := 0; i < ctx.Config.InitialRenderingJobs; i++ {
go worker(ctx, jobs)
}
for true {
start := time.Now()
result, err := ctx.BlockAccessor.FindLegacyMapBlocks(lastcoords, ctx.Config.InitialRenderingFetchLimit, ctx.Config.Layers)
if err != nil {
panic(err)
}
if len(result.List) == 0 && !result.HasMore {
fields = logrus.Fields{
"blocks": rstate.LegacyProcessed,
"tiles": tilecount,
}
logrus.WithFields(fields).Info("Initial rendering complete")
close(jobs)
rstate.InitialRun = false
ctx.Config.Save()
break
}
lastcoords = *result.LastPos
tileRenderedMap := make(map[string]bool)
for i := 12; i >= 1; i-- {
for _, mb := range result.List {
//13
tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers)
//12-1
tc = tc.ZoomOut(13 - i)
key := getTileKey(tc)
if tileRenderedMap[key] {
continue
}
tileRenderedMap[key] = true
fields = logrus.Fields{
"X": tc.X,
"Y": tc.Y,
"Zoom": tc.Zoom,
"LayerId": tc.LayerId,
}
logrus.WithFields(fields).Debug("Dispatching tile rendering (z11-1)")
tilecount++
jobs <- tc
}
}
//Save current positions of initial run
rstate.LastX = lastcoords.X
rstate.LastY = lastcoords.Y
rstate.LastZ = lastcoords.Z
rstate.LegacyProcessed += result.UnfilteredCount
ctx.Config.Save()
t := time.Now()
elapsed := t.Sub(start)
progress := int(float64(rstate.LegacyProcessed) / float64(totalLegacyCount) * 100)
fields = logrus.Fields{
"count": len(result.List),
"processed": rstate.LegacyProcessed,
"progress%": progress,
"X": lastcoords.X,
"Y": lastcoords.Y,
"Z": lastcoords.Z,
"elapsed": elapsed,
}
logrus.WithFields(fields).Info("Initial rendering")
}
}