diff --git a/server/app/setup.go b/server/app/setup.go index 419d694..d2cb62d 100644 --- a/server/app/setup.go +++ b/server/app/setup.go @@ -2,7 +2,7 @@ package app import ( "mapserver/colormapping" - "mapserver/db" + "mapserver/db/sqlite" "mapserver/eventbus" "mapserver/mapblockaccessor" "mapserver/mapblockrenderer" @@ -30,7 +30,7 @@ func Setup(p params.ParamsType, cfg *Config) *App { switch a.Worldconfig.Backend { case worldconfig.BACKEND_SQLITE3: - a.Blockdb, err = db.NewSqliteAccessor("map.sqlite") + a.Blockdb, err = sqlite.New("map.sqlite") if err != nil { panic(err) } diff --git a/server/db/postgres/postgres.go b/server/db/postgres/postgres.go new file mode 100644 index 0000000..acb9fe0 --- /dev/null +++ b/server/db/postgres/postgres.go @@ -0,0 +1,133 @@ +package postgres + +import ( + "database/sql" + _ "github.com/mattn/go-sqlite3" + "mapserver/coords" + "mapserver/db" +) + + +type PostgresAccessor struct { + db *sql.DB +} + +func (db *PostgresAccessor) Migrate() error { + return nil +} + +func convertRows(pos int64, data []byte, mtime int64) db.Block { + c := coords.PlainToCoord(pos) + return db.Block{Pos: c, Data: data, Mtime: mtime} +} + + +func (this *PostgresAccessor) FindBlocksByMtime(gtmtime int64, limit int) ([]db.Block, error) { + blocks := make([]db.Block, 0) + + rows, err := this.db.Query(getBlocksByMtimeQuery, gtmtime, limit) + if err != nil { + return nil, err + } + + defer rows.Close() + + for rows.Next() { + var pos int64 + var data []byte + var mtime int64 + + err = rows.Scan(&pos, &data, &mtime) + if err != nil { + return nil, err + } + + mb := convertRows(pos, data, mtime) + blocks = append(blocks, mb) + } + + return blocks, nil +} + +func (this *PostgresAccessor) FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]db.Block, error) { + blocks := make([]db.Block, 0) + pc := coords.CoordToPlain(lastpos) + + rows, err := this.db.Query(getLastBlockQuery, pc, limit) + if err != nil { + return nil, err + } + + defer rows.Close() + + for rows.Next() { + var pos int64 + var data []byte + var mtime int64 + + err = rows.Scan(&pos, &data, &mtime) + if err != nil { + return nil, err + } + + mb := convertRows(pos, data, mtime) + blocks = append(blocks, mb) + } + + return blocks, nil +} + + +func (this *PostgresAccessor) CountBlocks(frommtime, tomtime int64) (int, error) { + rows, err := this.db.Query(countBlocksQuery, frommtime, tomtime) + if err != nil { + return 0, err + } + + defer rows.Close() + + if rows.Next() { + var count int64 + + err = rows.Scan(&count) + if err != nil { + return 0, err + } + + return int(count), nil + } + + return 0, nil +} + + +func (this *PostgresAccessor) GetBlock(pos coords.MapBlockCoords) (*db.Block, error) { + ppos := coords.CoordToPlain(pos) + + rows, err := this.db.Query(getBlockQuery, ppos) + if err != nil { + return nil, err + } + + defer rows.Close() + + if rows.Next() { + var pos int64 + var data []byte + var mtime int64 + + err = rows.Scan(&pos, &data, &mtime) + if err != nil { + return nil, err + } + + mb := convertRows(pos, data, mtime) + return &mb, nil + } + + return nil, nil +} + +func NewPostgresAccessor(filename string) (*PostgresAccessor, error) { + return nil, nil +} diff --git a/server/db/postgres/sql.go b/server/db/postgres/sql.go new file mode 100644 index 0000000..f659ef1 --- /dev/null +++ b/server/db/postgres/sql.go @@ -0,0 +1,42 @@ +package postgres + +const migrateScript = ` +alter table blocks add mtime integer default 0; +create index blocks_mtime on blocks(mtime); + +CREATE TRIGGER update_blocks_mtime_insert after insert on blocks for each row +begin +update blocks set mtime = strftime('%s', 'now') where pos = new.pos; +end; + +CREATE TRIGGER update_blocks_mtime_update after update on blocks for each row +begin +update blocks set mtime = strftime('%s', 'now') where pos = old.pos; +end; +` + +const getBlocksByMtimeQuery = ` +select pos,data,mtime +from blocks b +where b.mtime > ? +order by b.mtime asc +limit ? +` + +const getLastBlockQuery = ` +select pos,data,mtime +from blocks b +where b.mtime = 0 +and b.pos > ? +order by b.pos asc, b.mtime asc +limit ? +` + +const countBlocksQuery = ` +select count(*) from blocks b where b.mtime >= ? and b.mtime <= ? +` + + +const getBlockQuery = ` +select pos,data,mtime from blocks b where b.pos = ? +` diff --git a/server/db/sqlite/logger.go b/server/db/sqlite/logger.go new file mode 100644 index 0000000..486e392 --- /dev/null +++ b/server/db/sqlite/logger.go @@ -0,0 +1,11 @@ +package sqlite + +import ( + "github.com/sirupsen/logrus" +) + +var log *logrus.Entry + +func init() { + log = logrus.WithFields(logrus.Fields{"prefix": "sqlite-db"}) +} diff --git a/server/db/sqlite/sql.go b/server/db/sqlite/sql.go new file mode 100644 index 0000000..2e12ef0 --- /dev/null +++ b/server/db/sqlite/sql.go @@ -0,0 +1,42 @@ +package sqlite + +const migrateScript = ` +alter table blocks add mtime integer default 0; +create index blocks_mtime on blocks(mtime); + +CREATE TRIGGER update_blocks_mtime_insert after insert on blocks for each row +begin +update blocks set mtime = strftime('%s', 'now') where pos = new.pos; +end; + +CREATE TRIGGER update_blocks_mtime_update after update on blocks for each row +begin +update blocks set mtime = strftime('%s', 'now') where pos = old.pos; +end; +` + +const getBlocksByMtimeQuery = ` +select pos,data,mtime +from blocks b +where b.mtime > ? +order by b.mtime asc +limit ? +` + +const getLastBlockQuery = ` +select pos,data,mtime +from blocks b +where b.mtime = 0 +and b.pos > ? +order by b.pos asc, b.mtime asc +limit ? +` + +const countBlocksQuery = ` +select count(*) from blocks b where b.mtime >= ? and b.mtime <= ? +` + + +const getBlockQuery = ` +select pos,data,mtime from blocks b where b.pos = ? +` diff --git a/server/db/sqlite.go b/server/db/sqlite/sqlite.go similarity index 62% rename from server/db/sqlite.go rename to server/db/sqlite/sqlite.go index 0e9f634..6899475 100644 --- a/server/db/sqlite.go +++ b/server/db/sqlite/sqlite.go @@ -1,10 +1,11 @@ -package db +package sqlite import ( "database/sql" _ "github.com/mattn/go-sqlite3" "github.com/sirupsen/logrus" "mapserver/coords" + "mapserver/db" "time" ) @@ -13,24 +14,6 @@ sqlite extract: https://stackoverflow.com/questions/15448373/how-to-dump-a-file- sqlite3 my.db "SELECT writefile('object0.gz', MyBlob) FROM MyTable WHERE id = 1" */ -const migrateScript = ` -alter table blocks add mtime integer default 0; -create index blocks_mtime on blocks(mtime); - -CREATE TRIGGER update_blocks_mtime_insert after insert on blocks for each row -begin -update blocks set mtime = strftime('%s', 'now') where pos = new.pos; -end; - -CREATE TRIGGER update_blocks_mtime_update after update on blocks for each row -begin -update blocks set mtime = strftime('%s', 'now') where pos = old.pos; -end; -` - -//TODO: initial run: https://stackoverflow.com/questions/14468586/efficient-paging-in-sqlite-with-millions-of-records -//TODO: postgres test - type Sqlite3Accessor struct { db *sql.DB filename string @@ -67,23 +50,16 @@ func (db *Sqlite3Accessor) Migrate() error { return nil } -func convertRows(pos int64, data []byte, mtime int64) Block { +func convertRows(pos int64, data []byte, mtime int64) db.Block { c := coords.PlainToCoord(pos) - return Block{Pos: c, Data: data, Mtime: mtime} + return db.Block{Pos: c, Data: data, Mtime: mtime} } -const getBlocksByMtimeQuery = ` -select pos,data,mtime -from blocks b -where b.mtime > ? -order by b.mtime asc -limit ? -` -func (db *Sqlite3Accessor) FindBlocksByMtime(gtmtime int64, limit int) ([]Block, error) { - blocks := make([]Block, 0) +func (this *Sqlite3Accessor) FindBlocksByMtime(gtmtime int64, limit int) ([]db.Block, error) { + blocks := make([]db.Block, 0) - rows, err := db.db.Query(getBlocksByMtimeQuery, gtmtime, limit) + rows, err := this.db.Query(getBlocksByMtimeQuery, gtmtime, limit) if err != nil { return nil, err } @@ -107,20 +83,12 @@ func (db *Sqlite3Accessor) FindBlocksByMtime(gtmtime int64, limit int) ([]Block, return blocks, nil } -const getLastBlockQuery = ` -select pos,data,mtime -from blocks b -where b.mtime = 0 -and b.pos > ? -order by b.pos asc, b.mtime asc -limit ? -` -func (db *Sqlite3Accessor) FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]Block, error) { - blocks := make([]Block, 0) +func (this *Sqlite3Accessor) FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]db.Block, error) { + blocks := make([]db.Block, 0) pc := coords.CoordToPlain(lastpos) - rows, err := db.db.Query(getLastBlockQuery, pc, limit) + rows, err := this.db.Query(getLastBlockQuery, pc, limit) if err != nil { return nil, err } @@ -144,9 +112,7 @@ func (db *Sqlite3Accessor) FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, return blocks, nil } -const countBlocksQuery = ` -select count(*) from blocks b where b.mtime >= ? and b.mtime <= ? -` + func (db *Sqlite3Accessor) CountBlocks(frommtime, tomtime int64) (int, error) { rows, err := db.db.Query(countBlocksQuery, frommtime, tomtime) @@ -170,11 +136,7 @@ func (db *Sqlite3Accessor) CountBlocks(frommtime, tomtime int64) (int, error) { return 0, nil } -const getBlockQuery = ` -select pos,data,mtime from blocks b where b.pos = ? -` - -func (db *Sqlite3Accessor) GetBlock(pos coords.MapBlockCoords) (*Block, error) { +func (db *Sqlite3Accessor) GetBlock(pos coords.MapBlockCoords) (*db.Block, error) { ppos := coords.CoordToPlain(pos) rows, err := db.db.Query(getBlockQuery, ppos) @@ -201,7 +163,7 @@ func (db *Sqlite3Accessor) GetBlock(pos coords.MapBlockCoords) (*Block, error) { return nil, nil } -func NewSqliteAccessor(filename string) (*Sqlite3Accessor, error) { +func New(filename string) (*Sqlite3Accessor, error) { db, err := sql.Open("sqlite3", filename+"?mode=ro&_timeout=2000") if err != nil { return nil, err diff --git a/server/db/sqlite_test.go b/server/db/sqlite/sqlite_test.go similarity index 89% rename from server/db/sqlite_test.go rename to server/db/sqlite/sqlite_test.go index 9bf887c..4d2a777 100644 --- a/server/db/sqlite_test.go +++ b/server/db/sqlite/sqlite_test.go @@ -1,4 +1,4 @@ -package db +package sqlite import ( _ "github.com/mattn/go-sqlite3" @@ -17,7 +17,7 @@ func TestMigrateEmpty(t *testing.T) { defer os.Remove(tmpfile.Name()) testutils.CreateEmptyDatabase(tmpfile.Name()) - a, err := NewSqliteAccessor(tmpfile.Name()) + a, err := New(tmpfile.Name()) if err != nil { panic(err) } @@ -35,7 +35,7 @@ func TestMigrate(t *testing.T) { defer os.Remove(tmpfile.Name()) testutils.CreateEmptyDatabase(tmpfile.Name()) - a, err := NewSqliteAccessor(tmpfile.Name()) + a, err := New(tmpfile.Name()) if err != nil { panic(err) } @@ -53,7 +53,7 @@ func TestMigrateAndQuery(t *testing.T) { defer os.Remove(tmpfile.Name()) testutils.CreateTestDatabase(tmpfile.Name()) - a, err := NewSqliteAccessor(tmpfile.Name()) + a, err := New(tmpfile.Name()) if err != nil { panic(err) } @@ -83,7 +83,7 @@ func TestMigrateAndQueryCount(t *testing.T) { defer os.Remove(tmpfile.Name()) testutils.CreateTestDatabase(tmpfile.Name()) - a, err := NewSqliteAccessor(tmpfile.Name()) + a, err := New(tmpfile.Name()) if err != nil { panic(err) } diff --git a/server/mapblockaccessor/mapblockaccessor_test.go b/server/mapblockaccessor/mapblockaccessor_test.go index 8b24d4b..dc68d32 100644 --- a/server/mapblockaccessor/mapblockaccessor_test.go +++ b/server/mapblockaccessor/mapblockaccessor_test.go @@ -4,7 +4,7 @@ import ( "github.com/sirupsen/logrus" "io/ioutil" "mapserver/coords" - "mapserver/db" + "mapserver/db/sqlite" "mapserver/testutils" "os" "testing" @@ -20,7 +20,7 @@ func TestSimpleAccess(t *testing.T) { defer os.Remove(tmpfile.Name()) testutils.CreateTestDatabase(tmpfile.Name()) - a, err := db.NewSqliteAccessor(tmpfile.Name()) + a, err := sqlite.New(tmpfile.Name()) if err != nil { panic(err) } diff --git a/server/mapblockrenderer/renderer_test.go b/server/mapblockrenderer/renderer_test.go index bb35752..d8da89c 100644 --- a/server/mapblockrenderer/renderer_test.go +++ b/server/mapblockrenderer/renderer_test.go @@ -6,7 +6,7 @@ import ( "io/ioutil" "mapserver/colormapping" "mapserver/coords" - "mapserver/db" + "mapserver/db/sqlite" "mapserver/layer" "mapserver/mapblockaccessor" "mapserver/testutils" @@ -33,7 +33,7 @@ func TestSimpleRender(t *testing.T) { defer os.Remove(tmpfile.Name()) testutils.CreateTestDatabase(tmpfile.Name()) - a, err := db.NewSqliteAccessor(tmpfile.Name()) + a, err := sqlite.New(tmpfile.Name()) if err != nil { panic(err) } diff --git a/server/tilerenderer/renderer_test.go b/server/tilerenderer/renderer_test.go index e59f570..a732460 100644 --- a/server/tilerenderer/renderer_test.go +++ b/server/tilerenderer/renderer_test.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "mapserver/colormapping" "mapserver/coords" - "mapserver/db" + "mapserver/db/sqlite" "mapserver/layer" "mapserver/mapblockaccessor" "mapserver/mapblockrenderer" @@ -28,7 +28,7 @@ func TestTileRender(t *testing.T) { defer os.Remove(tmpfile.Name()) testutils.CreateTestDatabase(tmpfile.Name()) - a, err := db.NewSqliteAccessor(tmpfile.Name()) + a, err := sqlite.New(tmpfile.Name()) if err != nil { panic(err) }