From 2ec79a16cd40fdcd07834a94025afdd2ac0e085c Mon Sep 17 00:00:00 2001 From: NatureFreshMilk Date: Thu, 21 Feb 2019 11:16:01 +0100 Subject: [PATCH] tiledb on fs --- server/tiledb/prometheus.go | 29 --------------- server/tiledb/tiledb.go | 63 +++++++++++++------------------- server/tiledb/tiledb_test.go | 2 +- server/tilerendererjob/worker.go | 4 +- 4 files changed, 29 insertions(+), 69 deletions(-) delete mode 100644 server/tiledb/prometheus.go diff --git a/server/tiledb/prometheus.go b/server/tiledb/prometheus.go deleted file mode 100644 index d7bfe68..0000000 --- a/server/tiledb/prometheus.go +++ /dev/null @@ -1,29 +0,0 @@ -package tiledb - -import ( - "github.com/prometheus/client_golang/prometheus" -) - -var ( - setDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "tiledb_set_duration", - Help: "Histogram for tiledb set timings", - Buckets: prometheus.LinearBuckets(0.001, 0.01, 10), - }) - getDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "tiledb_get_duration", - Help: "Histogram for tiledb get timings", - Buckets: prometheus.LinearBuckets(0.001, 0.01, 10), - }) - removeDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "tiledb_remove_duration", - Help: "Histogram for tiledb remove timings", - Buckets: prometheus.LinearBuckets(0.001, 0.01, 10), - }) -) - -func init() { - prometheus.MustRegister(setDuration) - prometheus.MustRegister(getDuration) - prometheus.MustRegister(removeDuration) -} diff --git a/server/tiledb/tiledb.go b/server/tiledb/tiledb.go index a5cf569..5cf3f4b 100644 --- a/server/tiledb/tiledb.go +++ b/server/tiledb/tiledb.go @@ -1,67 +1,56 @@ package tiledb import ( - "fmt" + "io/ioutil" "mapserver/coords" - - "github.com/dgraph-io/badger" - "github.com/prometheus/client_golang/prometheus" + "os" + "strconv" ) func New(path string) (*TileDB, error) { - opts := badger.DefaultOptions - opts.Dir = path - opts.ValueDir = path - db, err := badger.Open(opts) - - if err != nil { - return nil, err - } - return &TileDB{ - db: db, + path: path, }, nil } type TileDB struct { - db *badger.DB + path string } -func getKey(pos *coords.TileCoords) []byte { - return []byte(fmt.Sprintf("%d/%d/%d/%d", pos.X, pos.Y, pos.Zoom, pos.LayerId)) +func (this *TileDB) getDirAndFile(pos *coords.TileCoords) (string, string) { + dir := this.path + "/" + + strconv.Itoa(pos.LayerId) + "/" + + strconv.Itoa(pos.Zoom) + "/" + + strconv.Itoa(pos.X) + + file := dir + "/" + strconv.Itoa(pos.Y) + ".png" + return dir, file } func (this *TileDB) GC() { - this.db.RunValueLogGC(0.7) + //No-Op } func (this *TileDB) GetTile(pos *coords.TileCoords) ([]byte, error) { - timer := prometheus.NewTimer(getDuration) - defer timer.ObserveDuration() - - var tile []byte - err := this.db.View(func(txn *badger.Txn) error { - item, err := txn.Get(getKey(pos)) - if item != nil { - tile, err = item.ValueCopy(nil) + _, file := this.getDirAndFile(pos) + info, err := os.Stat(file) + if info != nil && err == nil { + content, err := ioutil.ReadFile(file) + if err != nil { + return nil, err } - return err - }) - if err != nil { - return nil, nil + + return content, err } - return tile, err + return nil, nil } func (this *TileDB) SetTile(pos *coords.TileCoords, tile []byte) error { - timer := prometheus.NewTimer(setDuration) - defer timer.ObserveDuration() + dir, file := this.getDirAndFile(pos) + os.MkdirAll(dir, 0700) - err := this.db.Update(func(txn *badger.Txn) error { - err := txn.Set(getKey(pos), tile) - return err - }) + err := ioutil.WriteFile(file, tile, 0644) return err } diff --git a/server/tiledb/tiledb_test.go b/server/tiledb/tiledb_test.go index 3a29b0e..41b9042 100644 --- a/server/tiledb/tiledb_test.go +++ b/server/tiledb/tiledb_test.go @@ -8,7 +8,7 @@ import ( ) func TestTileDB(t *testing.T) { - tmpfile, err := ioutil.TempDir("", "TestTileDB.*.badger") + tmpfile, err := ioutil.TempDir("", "TestTileDB") if err != nil { panic(err) } diff --git a/server/tilerendererjob/worker.go b/server/tilerendererjob/worker.go index e2aec1a..1388164 100644 --- a/server/tilerendererjob/worker.go +++ b/server/tilerendererjob/worker.go @@ -1,9 +1,9 @@ package tilerendererjob import ( + "github.com/sirupsen/logrus" "mapserver/app" "mapserver/coords" - "github.com/sirupsen/logrus" ) func worker(ctx *app.App, coords <-chan *coords.TileCoords, done chan bool) { @@ -15,7 +15,7 @@ func worker(ctx *app.App, coords <-chan *coords.TileCoords, done chan bool) { "Y": tc.Y, "Zoom": tc.Zoom, "LayerId": tc.LayerId, - "prefix": "tilerenderjob", + "prefix": "tilerenderjob", } logrus.WithFields(fields).Debug("Tile render job tile")