working incremental detection

app/config.go

@@ -9,6 +9,7 @@ import (
 type Config struct {
 	Port                   int          `json:"port"`
 	EnableInitialRendering bool         `json:"enableinitialrendering"`
+	EnableIncrementalUpdate bool	`json:"enableincrementalupdate"`
 	Webdev                 bool         `json:"webdev"`
 	WebApi                 WebApiConfig `json:"webapi"`
 }
@@ -25,6 +26,7 @@ func ParseConfig(filename string) (*Config, error) {
 	cfg := Config{
 		Port:                   8080,
 		EnableInitialRendering: true,
+		EnableIncrementalUpdate: true,
 		Webdev:                 false,
 		WebApi:                 webapi,
 	}

db/sqlite.go

@@ -69,8 +69,39 @@ func convertRows(pos int64, data []byte, mtime int64) Block {
 	return Block{Pos: c, Data: data, Mtime: mtime}
 }
 
+const getLatestBlockQuery = `
+select pos,data,mtime
+from blocks b
+where b.mtime >= ?
+order by b.mtime asc
+limit ?
+`
+
 func (db *Sqlite3Accessor) FindLatestBlocks(mintime int64, limit int) ([]Block, error) {
-	return make([]Block, 0), nil
+	blocks := make([]Block, 0)
+
+	rows, err := db.db.Query(getLatestBlockQuery, mintime, limit)
+	if err != nil {
+		return nil, err
+	}
+
+	defer rows.Close()
+
+	for rows.Next() {
+		var pos int64
+		var data []byte
+		var mtime int64
+
+		err = rows.Scan(&pos, &data, &mtime)
+		if err != nil {
+			return nil, err
+		}
+
+		mb := convertRows(pos, data, mtime)
+		blocks = append(blocks, mb)
+	}
+
+	return blocks, nil
 }
 
 const getBlockQuery = `

main.go

@@ -8,7 +8,7 @@ import (
 	"mapserver/layerconfig"
 	"mapserver/params"
 	"mapserver/web"
-
+	"mapserver/tileupdate"
 	"fmt"
 )
 
@@ -59,6 +59,11 @@ func main() {
 		go initialrenderer.Render(ctx.Tilerenderer, layerconfig.DefaultLayers)
 	}
 
+	//Incremental update
+	if ctx.Config.EnableIncrementalUpdate {
+		go tileupdate.Job(ctx)
+	}
+
 	//Start http server
 	web.Serve(ctx)
 

mapblockaccessor/mapblockaccessor.go

@@ -29,6 +29,30 @@ func (a *MapBlockAccessor) Update(pos coords.MapBlockCoords, mb *mapblockparser.
 	a.c.Set(key, mb, cache.DefaultExpiration)
 }
 
+func (a *MapBlockAccessor) FindLatestMapBlocks(mintime int64, limit int) ([]*mapblockparser.MapBlock, error){
+	blocks, err := a.accessor.FindLatestBlocks(mintime, limit)
+
+	if err != nil {
+		return nil, err
+	}
+
+	mblist := make([]*mapblockparser.MapBlock, 0)
+
+	for _, block := range(blocks) {
+		key := getKey(block.Pos)
+
+		mapblock, err := mapblockparser.Parse(block.Data, block.Mtime)
+		if err != nil {
+			return nil, err
+		}
+
+		a.c.Set(key, mapblock, cache.DefaultExpiration)
+		mblist = append(mblist, mapblock)
+	}
+
+	return mblist, nil
+}
+
 func (a *MapBlockAccessor) GetMapBlock(pos coords.MapBlockCoords) (*mapblockparser.MapBlock, error) {
 	key := getKey(pos)
 
@@ -46,7 +70,7 @@ func (a *MapBlockAccessor) GetMapBlock(pos coords.MapBlockCoords) (*mapblockpars
 		return nil, nil
 	}
 
-	mapblock, err := mapblockparser.Parse(block.Data)
+	mapblock, err := mapblockparser.Parse(block.Data, block.Mtime)
 	if err != nil {
 		return nil, err
 	}

mapblockparser/mapblock.go

@@ -6,6 +6,7 @@ type MapBlock struct {
 	Mapdata      []byte         `json:"mapdata"`
 	Metadata     Metadata       `json:"metadata"`
 	BlockMapping map[int]string `json:"blockmapping"`
+	Mtime	int64	`json:"mtime"`
 }
 
 type Metadata struct {

mapblockparser/parse.go

@@ -5,8 +5,10 @@ import (
 	"strconv"
 )
 
-func Parse(data []byte) (*MapBlock, error) {
+func Parse(data []byte, mtime int64) (*MapBlock, error) {
 	mapblock := NewMapblock()
+	mapblock.Mtime = mtime
+
 	if len(data) == 0 {
 		return nil, errors.New("no data")
 	}

mapblockparser/parse_test.go

@@ -38,7 +38,7 @@ func TestParse(t *testing.T) {
 		t.Error(err)
 	}
 
-	mapblock, err := Parse(data)
+	mapblock, err := Parse(data, 0)
 
 	if err != nil {
 		t.Error(err)
@@ -69,7 +69,7 @@ func TestParse2(t *testing.T) {
 		t.Error(err)
 	}
 
-	mapblock, err := Parse(data)
+	mapblock, err := Parse(data, 0)
 
 	if err != nil {
 		t.Error(err)
@@ -87,7 +87,7 @@ func TestParse3(t *testing.T) {
 		t.Error(err)
 	}
 
-	_, err = Parse(data)
+	_, err = Parse(data, 0)
 
 	if err != nil {
 		t.Error(err)

tileupdate/job.go

@@ -0,0 +1,38 @@
+package tileupdate
+
+import (
+  "mapserver/app"
+  "github.com/sirupsen/logrus"
+  "time"
+)
+
+func Job(ctx *app.App){
+  t := time.Now().Unix()
+
+  fields := logrus.Fields{
+    "time": t,
+  }
+  logrus.WithFields(fields).Info("Starting incremental update")
+
+  for true {
+    mblist, err := ctx.BlockAccessor.FindLatestMapBlocks(t, 1000)
+
+    if err != nil {
+      panic(err)
+    }
+
+    fields = logrus.Fields{
+      "count": len(mblist),
+      "time": t,
+    }
+  	logrus.WithFields(fields).Info("incremental update")
+
+    for _, mb := range(mblist) {
+      if mb.Mtime > t {
+        t = mb.Mtime+1
+      }
+    }
+
+    time.Sleep(5 * time.Second)
+  }
+}