diff --git a/app/config.go b/app/config.go index 0fc05b2..c6f2a33 100644 --- a/app/config.go +++ b/app/config.go @@ -9,6 +9,7 @@ import ( type Config struct { Port int `json:"port"` EnableInitialRendering bool `json:"enableinitialrendering"` + EnableIncrementalUpdate bool `json:"enableincrementalupdate"` Webdev bool `json:"webdev"` WebApi WebApiConfig `json:"webapi"` } @@ -25,6 +26,7 @@ func ParseConfig(filename string) (*Config, error) { cfg := Config{ Port: 8080, EnableInitialRendering: true, + EnableIncrementalUpdate: true, Webdev: false, WebApi: webapi, } diff --git a/db/sqlite.go b/db/sqlite.go index 288c17e..9b46847 100644 --- a/db/sqlite.go +++ b/db/sqlite.go @@ -69,8 +69,39 @@ func convertRows(pos int64, data []byte, mtime int64) Block { return Block{Pos: c, Data: data, Mtime: mtime} } +const getLatestBlockQuery = ` +select pos,data,mtime +from blocks b +where b.mtime >= ? +order by b.mtime asc +limit ? +` + func (db *Sqlite3Accessor) FindLatestBlocks(mintime int64, limit int) ([]Block, error) { - return make([]Block, 0), nil + blocks := make([]Block, 0) + + rows, err := db.db.Query(getLatestBlockQuery, mintime, limit) + if err != nil { + return nil, err + } + + defer rows.Close() + + for rows.Next() { + var pos int64 + var data []byte + var mtime int64 + + err = rows.Scan(&pos, &data, &mtime) + if err != nil { + return nil, err + } + + mb := convertRows(pos, data, mtime) + blocks = append(blocks, mb) + } + + return blocks, nil } const getBlockQuery = ` diff --git a/main.go b/main.go index 502438b..6703d13 100644 --- a/main.go +++ b/main.go @@ -8,7 +8,7 @@ import ( "mapserver/layerconfig" "mapserver/params" "mapserver/web" - + "mapserver/tileupdate" "fmt" ) @@ -59,6 +59,11 @@ func main() { go initialrenderer.Render(ctx.Tilerenderer, layerconfig.DefaultLayers) } + //Incremental update + if ctx.Config.EnableIncrementalUpdate { + go tileupdate.Job(ctx) + } + //Start http server web.Serve(ctx) diff --git a/mapblockaccessor/mapblockaccessor.go b/mapblockaccessor/mapblockaccessor.go index cabeb7d..267b369 100644 --- a/mapblockaccessor/mapblockaccessor.go +++ b/mapblockaccessor/mapblockaccessor.go @@ -29,6 +29,30 @@ func (a *MapBlockAccessor) Update(pos coords.MapBlockCoords, mb *mapblockparser. a.c.Set(key, mb, cache.DefaultExpiration) } +func (a *MapBlockAccessor) FindLatestMapBlocks(mintime int64, limit int) ([]*mapblockparser.MapBlock, error){ + blocks, err := a.accessor.FindLatestBlocks(mintime, limit) + + if err != nil { + return nil, err + } + + mblist := make([]*mapblockparser.MapBlock, 0) + + for _, block := range(blocks) { + key := getKey(block.Pos) + + mapblock, err := mapblockparser.Parse(block.Data, block.Mtime) + if err != nil { + return nil, err + } + + a.c.Set(key, mapblock, cache.DefaultExpiration) + mblist = append(mblist, mapblock) + } + + return mblist, nil +} + func (a *MapBlockAccessor) GetMapBlock(pos coords.MapBlockCoords) (*mapblockparser.MapBlock, error) { key := getKey(pos) @@ -46,7 +70,7 @@ func (a *MapBlockAccessor) GetMapBlock(pos coords.MapBlockCoords) (*mapblockpars return nil, nil } - mapblock, err := mapblockparser.Parse(block.Data) + mapblock, err := mapblockparser.Parse(block.Data, block.Mtime) if err != nil { return nil, err } diff --git a/mapblockparser/mapblock.go b/mapblockparser/mapblock.go index 353dca9..ecd5c09 100644 --- a/mapblockparser/mapblock.go +++ b/mapblockparser/mapblock.go @@ -6,6 +6,7 @@ type MapBlock struct { Mapdata []byte `json:"mapdata"` Metadata Metadata `json:"metadata"` BlockMapping map[int]string `json:"blockmapping"` + Mtime int64 `json:"mtime"` } type Metadata struct { diff --git a/mapblockparser/parse.go b/mapblockparser/parse.go index adf251c..4658a8b 100644 --- a/mapblockparser/parse.go +++ b/mapblockparser/parse.go @@ -5,8 +5,10 @@ import ( "strconv" ) -func Parse(data []byte) (*MapBlock, error) { +func Parse(data []byte, mtime int64) (*MapBlock, error) { mapblock := NewMapblock() + mapblock.Mtime = mtime + if len(data) == 0 { return nil, errors.New("no data") } diff --git a/mapblockparser/parse_test.go b/mapblockparser/parse_test.go index 0941999..0a3df2e 100644 --- a/mapblockparser/parse_test.go +++ b/mapblockparser/parse_test.go @@ -38,7 +38,7 @@ func TestParse(t *testing.T) { t.Error(err) } - mapblock, err := Parse(data) + mapblock, err := Parse(data, 0) if err != nil { t.Error(err) @@ -69,7 +69,7 @@ func TestParse2(t *testing.T) { t.Error(err) } - mapblock, err := Parse(data) + mapblock, err := Parse(data, 0) if err != nil { t.Error(err) @@ -87,7 +87,7 @@ func TestParse3(t *testing.T) { t.Error(err) } - _, err = Parse(data) + _, err = Parse(data, 0) if err != nil { t.Error(err) diff --git a/tileupdate/job.go b/tileupdate/job.go new file mode 100644 index 0000000..7f4b82a --- /dev/null +++ b/tileupdate/job.go @@ -0,0 +1,38 @@ +package tileupdate + +import ( + "mapserver/app" + "github.com/sirupsen/logrus" + "time" +) + +func Job(ctx *app.App){ + t := time.Now().Unix() + + fields := logrus.Fields{ + "time": t, + } + logrus.WithFields(fields).Info("Starting incremental update") + + for true { + mblist, err := ctx.BlockAccessor.FindLatestMapBlocks(t, 1000) + + if err != nil { + panic(err) + } + + fields = logrus.Fields{ + "count": len(mblist), + "time": t, + } + logrus.WithFields(fields).Info("incremental update") + + for _, mb := range(mblist) { + if mb.Mtime > t { + t = mb.Mtime+1 + } + } + + time.Sleep(5 * time.Second) + } +}