From 3bb9d97e06b94ff23757e5f41e277e62173ef598 Mon Sep 17 00:00:00 2001 From: NatureFreshMilk Date: Mon, 21 Jan 2019 13:27:36 +0100 Subject: [PATCH] working initial rendering --- coords/tileconvert.go | 22 +++++++++-- coords/tileconvert_test.go | 35 +++++++++++++++-- coords/tilecoords.go | 20 +++++----- initialrenderer/job.go | 43 +++++++++++++++++--- mapblockrenderer/renderer_test.go | 12 +++++- tiledb/accessor.go | 6 +-- tiledb/sqlite.go | 4 +- tilerenderer/renderer.go | 65 +++++++++++-------------------- web/tiles.go | 2 +- 9 files changed, 137 insertions(+), 72 deletions(-) diff --git a/coords/tileconvert.go b/coords/tileconvert.go index cd1c32a..948dd2f 100644 --- a/coords/tileconvert.go +++ b/coords/tileconvert.go @@ -2,17 +2,33 @@ package coords import ( "math" + "mapserver/layer" ) const ( MAX_ZOOM = 13 ) -func GetTileCoordsFromMapBlock(mbc MapBlockCoords) TileCoords { - return TileCoords{X: mbc.X, Y: (mbc.Z + 1) * -1, Zoom: MAX_ZOOM} +func GetTileCoordsFromMapBlock(mbc MapBlockCoords, layers []layer.Layer) *TileCoords { + tc := TileCoords{X: mbc.X, Y: (mbc.Z + 1) * -1, Zoom: MAX_ZOOM} + + var layerid *int + for _, l := range layers { + if mbc.Y > l.From && mbc.Y < l.To { + layerid = &l.Id + } + } + + if layerid == nil { + return nil + } + + tc.LayerId = *layerid + + return &tc } -func GetMapBlockRangeFromTile(tc TileCoords, y int) MapBlockRange { +func GetMapBlockRangeFromTile(tc *TileCoords, y int) MapBlockRange { scaleDiff := float64(MAX_ZOOM - tc.Zoom) scale := int(math.Pow(2, scaleDiff)) diff --git a/coords/tileconvert_test.go b/coords/tileconvert_test.go index 1368830..c75c1c1 100644 --- a/coords/tileconvert_test.go +++ b/coords/tileconvert_test.go @@ -2,13 +2,22 @@ package coords import ( "testing" - + "mapserver/layer" "github.com/stretchr/testify/assert" ) func TestConvertMapblockToTile1(t *testing.T) { mbc := NewMapBlockCoords(0, 0, 0) - tc := GetTileCoordsFromMapBlock(mbc) + layers := []layer.Layer{ + layer.Layer{ + Id: 0, + Name: "Base", + From: -16, + To: 160, + }, + } + + tc := GetTileCoordsFromMapBlock(mbc, layers) if tc.X != 0 { t.Fatal("x does not match") @@ -39,7 +48,16 @@ func TestGetMapBlockRangeFromTile(t *testing.T) { func TestConvertMapblockToTile2(t *testing.T) { mbc := NewMapBlockCoords(1, 0, 1) - tc := GetTileCoordsFromMapBlock(mbc) + layers := []layer.Layer{ + layer.Layer{ + Id: 0, + Name: "Base", + From: -16, + To: 160, + }, + } + + tc := GetTileCoordsFromMapBlock(mbc, layers) if tc.X != 1 { t.Fatal("x does not match") @@ -56,7 +74,16 @@ func TestConvertMapblockToTile2(t *testing.T) { func TestConvertMapblockToTile3(t *testing.T) { mbc := NewMapBlockCoords(-1, 0, -1) - tc := GetTileCoordsFromMapBlock(mbc) + layers := []layer.Layer{ + layer.Layer{ + Id: 0, + Name: "Base", + From: -16, + To: 160, + }, + } + + tc := GetTileCoordsFromMapBlock(mbc, layers) if tc.X != -1 { t.Fatal("x does not match") diff --git a/coords/tilecoords.go b/coords/tilecoords.go index ac61839..ee7a31e 100644 --- a/coords/tilecoords.go +++ b/coords/tilecoords.go @@ -11,31 +11,31 @@ type TileCoords struct { } type TileQuadrants struct { - UpperLeft, UpperRight, LowerLeft, LowerRight TileCoords + UpperLeft, UpperRight, LowerLeft, LowerRight *TileCoords } -func NewTileCoords(x, y, zoom int, layerId int) TileCoords { - return TileCoords{X: x, Y: y, Zoom: zoom, LayerId: layerId} +func NewTileCoords(x, y, zoom int, layerId int) *TileCoords { + return &TileCoords{X: x, Y: y, Zoom: zoom, LayerId: layerId} } -func (tc TileCoords) GetZoomedOutTile() TileCoords { - return TileCoords{ +func (tc *TileCoords) GetZoomedOutTile() *TileCoords { + return &TileCoords{ X: int(math.Floor(float64(tc.X) / 2.0)), Y: int(math.Floor(float64(tc.Y) / 2.0)), Zoom: tc.Zoom - 1, LayerId: tc.LayerId} } -func (tc TileCoords) GetZoomedQuadrantsFromTile() TileQuadrants { +func (tc *TileCoords) GetZoomedQuadrantsFromTile() TileQuadrants { nextZoom := tc.Zoom + 1 nextZoomX := tc.X * 2 nextZoomY := tc.Y * 2 - upperLeft := TileCoords{X: nextZoomX, Y: nextZoomY, Zoom: nextZoom, LayerId: tc.LayerId} - upperRight := TileCoords{X: nextZoomX + 1, Y: nextZoomY, Zoom: nextZoom, LayerId: tc.LayerId} - lowerLeft := TileCoords{X: nextZoomX, Y: nextZoomY + 1, Zoom: nextZoom, LayerId: tc.LayerId} - lowerRight := TileCoords{X: nextZoomX + 1, Y: nextZoomY + 1, Zoom: nextZoom, LayerId: tc.LayerId} + upperLeft := &TileCoords{X: nextZoomX, Y: nextZoomY, Zoom: nextZoom, LayerId: tc.LayerId} + upperRight := &TileCoords{X: nextZoomX + 1, Y: nextZoomY, Zoom: nextZoom, LayerId: tc.LayerId} + lowerLeft := &TileCoords{X: nextZoomX, Y: nextZoomY + 1, Zoom: nextZoom, LayerId: tc.LayerId} + lowerRight := &TileCoords{X: nextZoomX + 1, Y: nextZoomY + 1, Zoom: nextZoom, LayerId: tc.LayerId} return TileQuadrants{ UpperLeft: upperLeft, diff --git a/initialrenderer/job.go b/initialrenderer/job.go index a9f4ec0..d1ab219 100644 --- a/initialrenderer/job.go +++ b/initialrenderer/job.go @@ -4,6 +4,7 @@ import ( "github.com/sirupsen/logrus" "mapserver/app" "mapserver/coords" + "mapserver/mapblockparser" ) func Job(ctx *app.App) { @@ -32,12 +33,43 @@ func Job(ctx *app.App) { lastcoords = *newlastcoords - for _, mb := range mblist { - tc := coords.GetTileCoordsFromMapBlock(mb.Pos) + //only mapblocks with valid layer + validmblist := make([]*mapblockparser.MapBlock, 0) - _, err = ctx.Tilerenderer.Render(tc) - if err != nil { - panic(err) + //Invalidate zoom 12-1 + for _, mb := range mblist { + tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers) + + if tc == nil { + continue + } + + validmblist = append(validmblist, mb) + + for tc.Zoom > 1 { + tc = tc.GetZoomedOutTile() + ctx.Tiledb.RemoveTile(tc) + } + } + + //Render zoom 12-1 + for _, mb := range validmblist { + tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers) + for tc.Zoom > 1 { + tc = tc.GetZoomedOutTile() + + fields = logrus.Fields{ + "X": tc.X, + "Y": tc.Y, + "Zoom": tc.Zoom, + "LayerId": tc.LayerId, + } + logrus.WithFields(fields).Debug("Dispatching tile rendering") + + _, err = ctx.Tilerenderer.Render(tc) + if err != nil { + panic(err) + } } } @@ -52,6 +84,7 @@ func Job(ctx *app.App) { "X": lastcoords.X, "Y": lastcoords.Y, "Z": lastcoords.Z, + "validcount": len(validmblist), } logrus.WithFields(fields).Info("Initial rendering") } diff --git a/mapblockrenderer/renderer_test.go b/mapblockrenderer/renderer_test.go index c9c2b2e..aaf565b 100644 --- a/mapblockrenderer/renderer_test.go +++ b/mapblockrenderer/renderer_test.go @@ -9,6 +9,7 @@ import ( "mapserver/db" "mapserver/mapblockaccessor" "mapserver/testutils" + "mapserver/layer" "os" "testing" ) @@ -16,6 +17,15 @@ import ( func TestSimpleRender(t *testing.T) { logrus.SetLevel(logrus.InfoLevel) + layers := []layer.Layer{ + layer.Layer{ + Id: 0, + Name: "Base", + From: -16, + To: 160, + }, + } + tmpfile, err := ioutil.TempFile("", "TestMigrate.*.sqlite") if err != nil { panic(err) @@ -56,7 +66,7 @@ func TestSimpleRender(t *testing.T) { continue } - tc := coords.GetTileCoordsFromMapBlock(result.Job.Pos1) + tc := coords.GetTileCoordsFromMapBlock(result.Job.Pos1, layers) f, _ := os.Create(fmt.Sprintf("../output/image_%d_%d.png", tc.X, tc.Y)) result.Data.WriteTo(f) f.Close() diff --git a/tiledb/accessor.go b/tiledb/accessor.go index 032a767..02e9ad5 100644 --- a/tiledb/accessor.go +++ b/tiledb/accessor.go @@ -5,14 +5,14 @@ import ( ) type Tile struct { - Pos coords.TileCoords + Pos *coords.TileCoords Data []byte Mtime int64 } type DBAccessor interface { Migrate() error - GetTile(pos coords.TileCoords) (*Tile, error) + GetTile(pos *coords.TileCoords) (*Tile, error) SetTile(tile *Tile) error - RemoveTile(pos coords.TileCoords) error + RemoveTile(pos *coords.TileCoords) error } diff --git a/tiledb/sqlite.go b/tiledb/sqlite.go index a022a7a..af094e2 100644 --- a/tiledb/sqlite.go +++ b/tiledb/sqlite.go @@ -47,7 +47,7 @@ and t.y = ? and t.zoom = ? ` -func (db *Sqlite3Accessor) GetTile(pos coords.TileCoords) (*Tile, error) { +func (db *Sqlite3Accessor) GetTile(pos *coords.TileCoords) (*Tile, error) { rows, err := db.db.Query(getTileQuery, pos.LayerId, pos.X, pos.Y, pos.Zoom) if err != nil { return nil, err @@ -95,7 +95,7 @@ delete from tiles where x = ? and y = ? and zoom = ? and layerid = ? ` -func (db *Sqlite3Accessor) RemoveTile(pos coords.TileCoords) error { +func (db *Sqlite3Accessor) RemoveTile(pos *coords.TileCoords) error { _, err := db.db.Exec(removeTileQuery, pos.X, pos.Y, pos.Zoom, pos.LayerId) return err } diff --git a/tilerenderer/renderer.go b/tilerenderer/renderer.go index 602cf10..786d63e 100644 --- a/tilerenderer/renderer.go +++ b/tilerenderer/renderer.go @@ -41,7 +41,7 @@ const ( IMG_SIZE = 256 ) -func (tr *TileRenderer) Render(tc coords.TileCoords) ([]byte, error) { +func (tr *TileRenderer) Render(tc *coords.TileCoords) ([]byte, error) { //Check cache tile, err := tr.tdb.GetTile(tc) @@ -51,7 +51,7 @@ func (tr *TileRenderer) Render(tc coords.TileCoords) ([]byte, error) { if tile == nil { //No tile in db - img, err := tr.RenderImage(tc) + img, err := tr.RenderImage(tc, false) if err != nil { return nil, err @@ -71,7 +71,7 @@ func (tr *TileRenderer) Render(tc coords.TileCoords) ([]byte, error) { return tile.Data, nil } -func (tr *TileRenderer) RenderImage(tc coords.TileCoords) (*image.NRGBA, error) { +func (tr *TileRenderer) RenderImage(tc *coords.TileCoords, cachedOnly bool) (*image.NRGBA, error) { cachedtile, err := tr.tdb.GetTile(tc) if err != nil { @@ -85,7 +85,19 @@ func (tr *TileRenderer) RenderImage(tc coords.TileCoords) (*image.NRGBA, error) return nil, err } - return cachedimg.(*image.NRGBA), nil + rect := image.Rectangle{ + image.Point{0, 0}, + image.Point{IMG_SIZE, IMG_SIZE}, + } + + img := image.NewNRGBA(rect) + draw.Draw(img, rect, cachedimg, image.ZP, draw.Src) + + return img, nil + } + + if cachedOnly { + return nil, nil } log.WithFields(logrus.Fields{"x": tc.X, "y": tc.Y, "zoom": tc.Zoom}).Debug("RenderImage") @@ -112,67 +124,34 @@ func (tr *TileRenderer) RenderImage(tc coords.TileCoords) (*image.NRGBA, error) mbr.Pos1.Y = layer.From mbr.Pos2.Y = layer.To - //count blocks - count, err := tr.dba.CountBlocks(mbr.Pos1, mbr.Pos2) - - if err != nil { - return nil, err - } - - if count == 0 { - return nil, nil - } - return tr.mapblockrenderer.Render(mbr.Pos1, mbr.Pos2) } - if tc.Zoom == 12 { - //count blocks and ignore empty tiles - mbr := coords.GetMapBlockRangeFromTile(tc, 0) - mbr.Pos1.Y = layer.From - mbr.Pos2.Y = layer.To - - count, err := tr.dba.CountBlocks(mbr.Pos1, mbr.Pos2) - - if err != nil { - return nil, err - } - - if count == 0 { - return nil, nil - } - } - //zoom 1-12 quads := tc.GetZoomedQuadrantsFromTile() - upperLeft, err := tr.RenderImage(quads.UpperLeft) + recursiveCachedOnly := tc.Zoom < 12 + + upperLeft, err := tr.RenderImage(quads.UpperLeft, recursiveCachedOnly) if err != nil { return nil, err } - upperRight, err := tr.RenderImage(quads.UpperRight) + upperRight, err := tr.RenderImage(quads.UpperRight, recursiveCachedOnly) if err != nil { return nil, err } - lowerLeft, err := tr.RenderImage(quads.LowerLeft) + lowerLeft, err := tr.RenderImage(quads.LowerLeft, recursiveCachedOnly) if err != nil { return nil, err } - lowerRight, err := tr.RenderImage(quads.LowerRight) + lowerRight, err := tr.RenderImage(quads.LowerRight, recursiveCachedOnly) if err != nil { return nil, err } - isEmpty := upperLeft == nil && upperRight == nil && lowerLeft == nil && lowerRight == nil - - if isEmpty && (tc.Zoom == 11 || tc.Zoom == 10) { - //don't cache empty zoomed tiles - return nil, nil - } - img := image.NewNRGBA( image.Rectangle{ image.Point{0, 0}, diff --git a/web/tiles.go b/web/tiles.go index b8ae030..c57dd2c 100644 --- a/web/tiles.go +++ b/web/tiles.go @@ -41,7 +41,7 @@ func (t *Tiles) ServeHTTP(resp http.ResponseWriter, req *http.Request) { } else { resp.Write(tile.Data) - + } } }