diff --git a/Makefile b/Makefile index d801a1a..9143152 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ $(OUT_DIR): mkdir $@ $(MOD_ZIP): $(OUT_DIR) - cd mod && zip -r ../$(OUT_DIR)/mapserver-mod.zip mapserver + zip -r $(OUT_DIR)/mapserver-mod.zip mapserver_mod clean: rm -rf $(OUT_DIR) diff --git a/doc/dev.md b/doc/dev.md new file mode 100644 index 0000000..61dd3f1 --- /dev/null +++ b/doc/dev.md @@ -0,0 +1,5 @@ + +```sql +update blocks set mtime = 0 where pos = (select max(pos) from blocks); +select max(mtime) from blocks; +``` diff --git a/mapserver_mod/mapserver/depends.txt b/mapserver_mod/mapserver/depends.txt new file mode 100644 index 0000000..d195b71 --- /dev/null +++ b/mapserver_mod/mapserver/depends.txt @@ -0,0 +1,2 @@ +default +digiline? diff --git a/mapserver_mod/mapserver/digimessage.lua b/mapserver_mod/mapserver/digimessage.lua new file mode 100644 index 0000000..30074bc --- /dev/null +++ b/mapserver_mod/mapserver/digimessage.lua @@ -0,0 +1,107 @@ + +local update_formspec = function(meta) + local inv = meta:get_inventory() + + local active = meta:get_int("active") == 1 + local state = "Inactive" + + if active then + state = "Active" + end + + local channel = meta:get_string("channel") + local message = meta:get_string("message") + + meta:set_string("infotext", "Digimessage: Channel=" .. channel .. ", Message=" .. message .. " (" .. state .. ")") + + meta:set_string("formspec", "size[8,2;]" .. + -- col 1 + "field[0,1;8,1;channel;Channel;" .. channel .. "]" .. + + -- col 3 + "button_exit[0,2;4,1;save;Save]" .. + "button_exit[4,2;4,1;toggle;Toggle]" .. + "") + +end + + +minetest.register_node("mapserver:digimessage", { + description = "Mapserver Digiline Message", + tiles = { + "tileserver_digimessage.png", + "tileserver_digimessage.png", + "tileserver_digimessage.png", + "tileserver_digimessage.png", + "tileserver_digimessage.png", + "tileserver_digimessage.png" + }, + groups = {cracky=3,oddly_breakable_by_hand=3}, + sounds = default.node_sound_glass_defaults(), + + digiline = { + receptor = {action = function() end}, + effector = { + action = function(pos, _, channel, msg) + local meta = minetest.env:get_meta(pos) + local set_channel = meta:get_string("channel") + + if channel == set_channel and type(msg) == "string" then + meta:set_string("message", msg) + end + end + }, + }, + + can_dig = function(pos, player) + local meta = minetest.env:get_meta(pos) + local owner = meta:get_string("owner") + + return player and player:get_player_name() == owner + end, + + after_place_node = function(pos, placer) + local meta = minetest.get_meta(pos) + meta:set_string("owner", placer:get_player_name() or "") + end, + + on_construct = function(pos) + local meta = minetest.get_meta(pos) + + last_index = last_index + 5 + + meta:set_string("channel", "digimessage") + meta:set_string("message", "") + meta:set_int("active", 1) + + update_formspec(meta) + end, + + on_receive_fields = function(pos, formname, fields, sender) + local meta = minetest.get_meta(pos) + local playername = sender:get_player_name() + + if playername == meta:get_string("owner") then + -- owner + if fields.save then + last_line = fields.line + meta:set_string("channel", fields.channel) + end + + if fields.toggle then + if meta:get_int("active") == 1 then + meta:set_int("active", 0) + else + meta:set_int("active", 1) + end + end + else + -- non-owner + end + + + update_formspec(meta) + end + + +}) diff --git a/mapserver_mod/mapserver/init.lua b/mapserver_mod/mapserver/init.lua new file mode 100644 index 0000000..86dbb41 --- /dev/null +++ b/mapserver_mod/mapserver/init.lua @@ -0,0 +1,16 @@ + +mapserver = { +} + +local MP = minetest.get_modpath("mapserver") +local has_digiline_mod = minetest.get_modpath("digiline") + +dofile(MP.."/poi.lua") +dofile(MP.."/train.lua") + +if has_digiline_mod then + dofile(MP.."/digimessage.lua") +end + + +print("[OK] Mapserver") diff --git a/mapserver_mod/mapserver/poi.lua b/mapserver_mod/mapserver/poi.lua new file mode 100644 index 0000000..353c984 --- /dev/null +++ b/mapserver_mod/mapserver/poi.lua @@ -0,0 +1,99 @@ + +local update_formspec = function(meta) + local inv = meta:get_inventory() + + local active = meta:get_int("active") == 1 + local state = "Inactive" + + if active then + state = "Active" + end + + local name = meta:get_string("name") + local category = meta:get_string("category") + local url = meta:get_string("url") or "" + + meta:set_string("infotext", "POI: " .. name .. ", " .. category .. " (" .. state .. ")") + + meta:set_string("formspec", "size[8,5;]" .. + -- col 1 + "field[0,1;4,1;name;Name;" .. name .. "]" .. + "button_exit[4,1;4,1;save;Save]" .. + + -- col 2 + "field[0,2.5;4,1;category;Category;" .. category .. "]" .. + "button_exit[4,2;4,1;toggle;Toggle]" .. + + -- col 3 + "field[0,3.5;8,1;url;URL;" .. url .. "]" .. + "") + +end + + +minetest.register_node("mapserver:poi", { + description = "Mapserver POI", + tiles = { + "mapserver_poi.png", + "mapserver_poi.png", + "mapserver_poi.png", + "mapserver_poi.png", + "mapserver_poi.png", + "mapserver_poi.png" + }, + groups = {cracky=3,oddly_breakable_by_hand=3}, + sounds = default.node_sound_glass_defaults(), + + can_dig = function(pos, player) + local meta = minetest.env:get_meta(pos) + local owner = meta:get_string("owner") + + return player and player:get_player_name() == owner + end, + + after_place_node = function(pos, placer) + local meta = minetest.get_meta(pos) + meta:set_string("owner", placer:get_player_name() or "") + end, + + on_construct = function(pos) + local meta = minetest.get_meta(pos) + + meta:set_string("name", "") + meta:set_string("category", "main") + meta:set_string("url", "") + meta:set_int("active", 0) + + update_formspec(meta) + end, + + on_receive_fields = function(pos, formname, fields, sender) + local meta = minetest.get_meta(pos) + local playername = sender:get_player_name() + + if playername == meta:get_string("owner") then + -- owner + if fields.save then + meta:set_string("name", fields.name) + meta:set_string("url", fields.url) + meta:set_string("category", fields.category) + end + + if fields.toggle then + if meta:get_int("active") == 1 then + meta:set_int("active", 0) + else + meta:set_int("active", 1) + end + end + + else + -- non-owner + end + + + update_formspec(meta) + end + + +}) diff --git a/mapserver_mod/mapserver/textures/mapserver_digimessage.png b/mapserver_mod/mapserver/textures/mapserver_digimessage.png new file mode 100644 index 0000000..84fb074 Binary files /dev/null and b/mapserver_mod/mapserver/textures/mapserver_digimessage.png differ diff --git a/mapserver_mod/mapserver/textures/mapserver_poi.png b/mapserver_mod/mapserver/textures/mapserver_poi.png new file mode 100644 index 0000000..f2e92b6 Binary files /dev/null and b/mapserver_mod/mapserver/textures/mapserver_poi.png differ diff --git a/mapserver_mod/mapserver/textures/mapserver_train.png b/mapserver_mod/mapserver/textures/mapserver_train.png new file mode 100644 index 0000000..5f1a100 Binary files /dev/null and b/mapserver_mod/mapserver/textures/mapserver_train.png differ diff --git a/mapserver_mod/mapserver/train.lua b/mapserver_mod/mapserver/train.lua new file mode 100644 index 0000000..0f0a782 --- /dev/null +++ b/mapserver_mod/mapserver/train.lua @@ -0,0 +1,108 @@ + +local last_index = 0 +local last_line = "" + +local update_formspec = function(meta) + local inv = meta:get_inventory() + + local active = meta:get_int("active") == 1 + local state = "Inactive" + + if active then + state = "Active" + end + + local line = meta:get_string("line") + local station = meta:get_string("station") + local index = meta:get_string("index") + + meta:set_string("infotext", "Train: Line=" .. line .. ", Station=" .. station .. " (" .. state .. ")") + + meta:set_string("formspec", "size[8,3;]" .. + -- col 1 + "field[0,1;4,1;line;Line;" .. line .. "]" .. + "button_exit[4,1;4,1;save;Save]" .. + + -- col 2 + "field[0,2.5;4,1;station;Station;" .. station .. "]" .. + "field[4,2.5;4,1;index;Index;" .. index .. "]" .. + + -- col 3 + "button_exit[4,3;4,1;toggle;Toggle]" .. + "") + +end + + +minetest.register_node("mapserver:train", { + description = "Mapserver Train", + tiles = { + "tileserver_train.png", + "tileserver_train.png", + "tileserver_train.png", + "tileserver_train.png", + "tileserver_train.png", + "tileserver_train.png" + }, + groups = {cracky=3,oddly_breakable_by_hand=3}, + sounds = default.node_sound_glass_defaults(), + + can_dig = function(pos, player) + local meta = minetest.env:get_meta(pos) + local owner = meta:get_string("owner") + + return player and player:get_player_name() == owner + end, + + after_place_node = function(pos, placer) + local meta = minetest.get_meta(pos) + meta:set_string("owner", placer:get_player_name() or "") + end, + + on_construct = function(pos) + local meta = minetest.get_meta(pos) + + last_index = last_index + 5 + + meta:set_string("station", "") + meta:set_string("line", last_line) + meta:set_int("active", 1) + meta:set_int("index", last_index) + + update_formspec(meta) + end, + + on_receive_fields = function(pos, formname, fields, sender) + local meta = minetest.get_meta(pos) + local playername = sender:get_player_name() + + if playername == meta:get_string("owner") then + -- owner + if fields.save then + last_line = fields.line + meta:set_string("line", fields.line) + meta:set_string("station", fields.station) + local index = tonumber(fields.index) + if index ~= nil then + last_index = index + meta:set_int("index", index) + end + end + + if fields.toggle then + if meta:get_int("active") == 1 then + meta:set_int("active", 0) + else + meta:set_int("active", 1) + end + end + else + -- non-owner + end + + + update_formspec(meta) + end + + +}) diff --git a/mod/mapserver/init.lua b/mapserver_mod/modpack.txt similarity index 100% rename from mod/mapserver/init.lua rename to mapserver_mod/modpack.txt diff --git a/server/app/app.go b/server/app/app.go index f8a0cbb..7ee9170 100644 --- a/server/app/app.go +++ b/server/app/app.go @@ -13,7 +13,7 @@ import ( ) const ( - Version = "2.0-DEV" + Version = "0.0.1-DEV" ) type App struct { diff --git a/server/app/config.go b/server/app/config.go index efd8943..2b3e89c 100644 --- a/server/app/config.go +++ b/server/app/config.go @@ -6,22 +6,20 @@ import ( "mapserver/coords" "mapserver/layer" "os" - "sync" "runtime" + "sync" ) type Config struct { - Port int `json:"port"` - EnableInitialRendering bool `json:"enableinitialrendering"` - EnableIncrementalUpdate bool `json:"enableincrementalupdate"` - Webdev bool `json:"webdev"` - WebApi *WebApiConfig `json:"webapi"` - RenderState *RenderStateType `json:"renderstate"` - Layers []layer.Layer `json:"layers"` - InitialRenderingFetchLimit int `json:"initialrenderingfetchlimit"` - InitialRenderingJobs int `json:"initialrenderingjobs"` - InitialRenderingQueue int `json:"initialrenderingqueue"` - UpdateRenderingFetchLimit int `json:"updaterenderingfetchlimit"` + Port int `json:"port"` + EnableRendering bool `json:"enablerendering"` + Webdev bool `json:"webdev"` + WebApi *WebApiConfig `json:"webapi"` + RenderState *RenderStateType `json:"renderstate"` + Layers []layer.Layer `json:"layers"` + RenderingFetchLimit int `json:"renderingfetchlimit"` + RenderingJobs int `json:"renderingjobs"` + RenderingQueue int `json:"renderingqueue"` } type WebApiConfig struct { @@ -81,7 +79,7 @@ func ParseConfig(filename string) (*Config, error) { LastX: coords.MinCoord - 1, LastY: coords.MinCoord - 1, LastZ: coords.MinCoord - 1, - LastMtime: 1, + LastMtime: 0, } layers := []layer.Layer{ @@ -94,17 +92,15 @@ func ParseConfig(filename string) (*Config, error) { } cfg := Config{ - Port: 8080, - EnableInitialRendering: true, - EnableIncrementalUpdate: true, - Webdev: false, - WebApi: &webapi, - RenderState: &rstate, - Layers: layers, - InitialRenderingFetchLimit: 1000, - InitialRenderingJobs: runtime.NumCPU(), - InitialRenderingQueue: 100, - UpdateRenderingFetchLimit: 1000, + Port: 8080, + EnableRendering: true, + Webdev: false, + WebApi: &webapi, + RenderState: &rstate, + Layers: layers, + RenderingFetchLimit: 1000, + RenderingJobs: runtime.NumCPU(), + RenderingQueue: 100, } info, err := os.Stat(filename) diff --git a/server/db/accessor.go b/server/db/accessor.go index 122e2a1..8373c2b 100644 --- a/server/db/accessor.go +++ b/server/db/accessor.go @@ -12,13 +12,10 @@ type Block struct { type DBAccessor interface { Migrate() error - /** - * find old (pre-mapserver) mapblocks by lastpos - * used only on initial rendering - */ - FindLegacyBlocks(lastpos coords.MapBlockCoords, limit int) ([]Block, error) - CountLegacyBlocks() (int, error) - FindLatestBlocks(mintime int64, limit int) ([]Block, error) + FindBlocksByMtime(gtmtime int64, limit int) ([]Block, error) + FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]Block, error) + + CountBlocks(frommtime, tomtime int64) (int, error) GetBlock(pos coords.MapBlockCoords) (*Block, error) } diff --git a/server/db/sqlite.go b/server/db/sqlite.go index b1d20cd..59cb169 100644 --- a/server/db/sqlite.go +++ b/server/db/sqlite.go @@ -72,20 +72,18 @@ func convertRows(pos int64, data []byte, mtime int64) Block { return Block{Pos: c, Data: data, Mtime: mtime} } -const getLegacyBlockQuery = ` +const getBlocksByMtimeQuery = ` select pos,data,mtime from blocks b -where b.mtime == 0 -and b.pos > ? -order by b.pos asc +where b.mtime > ? +order by b.mtime asc limit ? ` -func (db *Sqlite3Accessor) FindLegacyBlocks(lastpos coords.MapBlockCoords, limit int) ([]Block, error) { +func (db *Sqlite3Accessor) FindBlocksByMtime(gtmtime int64, limit int) ([]Block, error) { blocks := make([]Block, 0) - pc := coords.CoordToPlain(lastpos) - rows, err := db.db.Query(getLegacyBlockQuery, pc, limit) + rows, err := db.db.Query(getBlocksByMtimeQuery, gtmtime, limit) if err != nil { return nil, err } @@ -109,12 +107,49 @@ func (db *Sqlite3Accessor) FindLegacyBlocks(lastpos coords.MapBlockCoords, limit return blocks, nil } -const countLegacyBlocksQuery = ` -select count(*) from blocks b where b.mtime = 0 +const getLastBlockQuery = ` +select pos,data,mtime +from blocks b +where b.mtime = 0 +and b.pos > ? +order by b.pos asc, b.mtime asc +limit ? ` -func (db *Sqlite3Accessor) CountLegacyBlocks() (int, error) { - rows, err := db.db.Query(countLegacyBlocksQuery) +func (db *Sqlite3Accessor) FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]Block, error) { + blocks := make([]Block, 0) + pc := coords.CoordToPlain(lastpos) + + rows, err := db.db.Query(getLastBlockQuery, pc, limit) + if err != nil { + return nil, err + } + + defer rows.Close() + + for rows.Next() { + var pos int64 + var data []byte + var mtime int64 + + err = rows.Scan(&pos, &data, &mtime) + if err != nil { + return nil, err + } + + mb := convertRows(pos, data, mtime) + blocks = append(blocks, mb) + } + + return blocks, nil +} + +const countBlocksQuery = ` +select count(*) from blocks b where b.mtime >= ? and b.mtime <= ? +` + +func (db *Sqlite3Accessor) CountBlocks(frommtime, tomtime int64) (int, error) { + rows, err := db.db.Query(countBlocksQuery, frommtime, tomtime) if err != nil { return 0, err } @@ -135,41 +170,6 @@ func (db *Sqlite3Accessor) CountLegacyBlocks() (int, error) { return 0, nil } -const getLatestBlockQuery = ` -select pos,data,mtime -from blocks b -where b.mtime > ? -order by b.mtime asc -limit ? -` - -func (db *Sqlite3Accessor) FindLatestBlocks(mintime int64, limit int) ([]Block, error) { - blocks := make([]Block, 0) - - rows, err := db.db.Query(getLatestBlockQuery, mintime, limit) - if err != nil { - return nil, err - } - - defer rows.Close() - - for rows.Next() { - var pos int64 - var data []byte - var mtime int64 - - err = rows.Scan(&pos, &data, &mtime) - if err != nil { - return nil, err - } - - mb := convertRows(pos, data, mtime) - blocks = append(blocks, mb) - } - - return blocks, nil -} - const getBlockQuery = ` select pos,data,mtime from blocks b where b.pos = ? ` diff --git a/server/db/sqlite_test.go b/server/db/sqlite_test.go index 1ce07c5..9bf887c 100644 --- a/server/db/sqlite_test.go +++ b/server/db/sqlite_test.go @@ -93,7 +93,7 @@ func TestMigrateAndQueryCount(t *testing.T) { panic(err) } - count, err := a.CountLegacyBlocks() + count, err := a.CountBlocks(0, 0) if err != nil { panic(err) } diff --git a/server/initialrenderer/job.go b/server/initialrenderer/job.go deleted file mode 100644 index 709eef8..0000000 --- a/server/initialrenderer/job.go +++ /dev/null @@ -1,125 +0,0 @@ -package initialrenderer - -import ( - "github.com/sirupsen/logrus" - "mapserver/app" - "mapserver/coords" - "strconv" - "time" -) - -func getTileKey(tc *coords.TileCoords) string { - return strconv.Itoa(tc.X) + "/" + strconv.Itoa(tc.Y) + "/" + strconv.Itoa(tc.Zoom) -} - -func worker(ctx *app.App, coords <-chan *coords.TileCoords) { - for tc := range coords { - ctx.Objectdb.RemoveTile(tc) - _, err := ctx.Tilerenderer.Render(tc, 2) - if err != nil { - panic(err) - } - } -} - -func Job(ctx *app.App) { - - totalLegacyCount, err := ctx.Blockdb.CountLegacyBlocks() - if err != nil { - panic(err) - } - - fields := logrus.Fields{ - "totalLegacyCount": totalLegacyCount, - } - logrus.WithFields(fields).Info("Starting initial rendering") - tilecount := 0 - - rstate := ctx.Config.RenderState - lastcoords := coords.NewMapBlockCoords(rstate.LastX, rstate.LastY, rstate.LastZ) - - jobs := make(chan *coords.TileCoords, ctx.Config.InitialRenderingQueue) - - for i := 0; i < ctx.Config.InitialRenderingJobs; i++ { - go worker(ctx, jobs) - } - - for true { - start := time.Now() - - result, err := ctx.BlockAccessor.FindLegacyMapBlocks(lastcoords, ctx.Config.InitialRenderingFetchLimit, ctx.Config.Layers) - - if err != nil { - panic(err) - } - - if len(result.List) == 0 && !result.HasMore { - fields = logrus.Fields{ - "blocks": rstate.LegacyProcessed, - "tiles": tilecount, - } - logrus.WithFields(fields).Info("Initial rendering complete") - close(jobs) - rstate.InitialRun = false - ctx.Config.Save() - - break - } - - lastcoords = *result.LastPos - - tileRenderedMap := make(map[string]bool) - - for i := 12; i >= 1; i-- { - for _, mb := range result.List { - //13 - tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers) - - //12-1 - tc = tc.ZoomOut(13 - i) - - key := getTileKey(tc) - - if tileRenderedMap[key] { - continue - } - - tileRenderedMap[key] = true - - fields = logrus.Fields{ - "X": tc.X, - "Y": tc.Y, - "Zoom": tc.Zoom, - "LayerId": tc.LayerId, - } - logrus.WithFields(fields).Debug("Dispatching tile rendering (z11-1)") - - tilecount++ - jobs <- tc - } - } - - //Save current positions of initial run - rstate.LastX = lastcoords.X - rstate.LastY = lastcoords.Y - rstate.LastZ = lastcoords.Z - rstate.LegacyProcessed += result.UnfilteredCount - ctx.Config.Save() - - t := time.Now() - elapsed := t.Sub(start) - - progress := int(float64(rstate.LegacyProcessed) / float64(totalLegacyCount) * 100) - - fields = logrus.Fields{ - "count": len(result.List), - "processed": rstate.LegacyProcessed, - "progress%": progress, - "X": lastcoords.X, - "Y": lastcoords.Y, - "Z": lastcoords.Z, - "elapsed": elapsed, - } - logrus.WithFields(fields).Info("Initial rendering") - } -} diff --git a/server/main.go b/server/main.go index 8af5782..5930460 100644 --- a/server/main.go +++ b/server/main.go @@ -4,10 +4,9 @@ import ( "fmt" "github.com/sirupsen/logrus" "mapserver/app" - "mapserver/initialrenderer" "mapserver/mapobject" "mapserver/params" - "mapserver/tileupdate" + "mapserver/tilerendererjob" "mapserver/web" "runtime" ) @@ -30,10 +29,10 @@ func main() { if p.Version { fmt.Print("Mapserver version: ") - fmt.Print(app.Version) - fmt.Print(" OS: ") - fmt.Print(runtime.GOOS) - fmt.Print(" Architecture: ") + fmt.Println(app.Version) + fmt.Print("OS: ") + fmt.Println(runtime.GOOS) + fmt.Print("Architecture: ") fmt.Println(runtime.GOARCH) return } @@ -62,13 +61,8 @@ func main() { mapobject.Setup(ctx) //run initial rendering - if ctx.Config.EnableInitialRendering && ctx.Config.RenderState.InitialRun { - go initialrenderer.Job(ctx) - } - - //Incremental update - if ctx.Config.EnableIncrementalUpdate { - go tileupdate.Job(ctx) + if ctx.Config.EnableRendering { + go tilerendererjob.Job(ctx) } //Start http server diff --git a/server/mapblockaccessor/mapblockaccessor.go b/server/mapblockaccessor/mapblockaccessor.go index 0711f14..b0e42ab 100644 --- a/server/mapblockaccessor/mapblockaccessor.go +++ b/server/mapblockaccessor/mapblockaccessor.go @@ -41,22 +41,29 @@ func (a *MapBlockAccessor) Update(pos coords.MapBlockCoords, mb *mapblockparser. a.c.Set(key, mb, cache.DefaultExpiration) } -type LegacyMapBlocksResult struct { +type FindMapBlocksResult struct { HasMore bool LastPos *coords.MapBlockCoords + LastMtime int64 List []*mapblockparser.MapBlock UnfilteredCount int } -func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, limit int, layerfilter []layer.Layer) (*LegacyMapBlocksResult, error) { +func (a *MapBlockAccessor) FindMapBlocksByMtime(lastmtime int64, limit int, layerfilter []layer.Layer) (*FindMapBlocksResult, error) { - blocks, err := a.accessor.FindLegacyBlocks(lastpos, limit) + fields := logrus.Fields{ + "lastmtime": lastmtime, + "limit": limit, + } + logrus.WithFields(fields).Debug("FindMapBlocksByMtime") + + blocks, err := a.accessor.FindBlocksByMtime(lastmtime, limit) if err != nil { return nil, err } - result := LegacyMapBlocksResult{} + result := FindMapBlocksResult{} mblist := make([]*mapblockparser.MapBlock, 0) var newlastpos *coords.MapBlockCoords @@ -65,6 +72,9 @@ func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, li for _, block := range blocks { newlastpos = &block.Pos + if result.LastMtime < block.Mtime { + result.LastMtime = block.Mtime + } inLayer := false for _, l := range layerfilter { @@ -83,7 +93,7 @@ func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, li "y": block.Pos.Y, "z": block.Pos.Z, } - logrus.WithFields(fields).Trace("legacy mapblock") + logrus.WithFields(fields).Debug("mapblock") key := getKey(block.Pos) @@ -107,21 +117,40 @@ func (a *MapBlockAccessor) FindLegacyMapBlocks(lastpos coords.MapBlockCoords, li return &result, nil } -func (a *MapBlockAccessor) FindLatestMapBlocks(mintime int64, limit int, layerfilter []layer.Layer) ([]*mapblockparser.MapBlock, error) { - blocks, err := a.accessor.FindLatestBlocks(mintime, limit) +func (a *MapBlockAccessor) FindMapBlocksByPos(lastpos coords.MapBlockCoords, limit int, layerfilter []layer.Layer) (*FindMapBlocksResult, error) { + + fields := logrus.Fields{ + "x": lastpos.X, + "y": lastpos.Y, + "z": lastpos.Z, + "limit": limit, + } + logrus.WithFields(fields).Debug("FindMapBlocksByPos") + + blocks, err := a.accessor.FindLegacyBlocksByPos(lastpos, limit) if err != nil { return nil, err } + result := FindMapBlocksResult{} + mblist := make([]*mapblockparser.MapBlock, 0) + var newlastpos *coords.MapBlockCoords + result.HasMore = len(blocks) == limit + result.UnfilteredCount = len(blocks) for _, block := range blocks { + newlastpos = &block.Pos + if result.LastMtime < block.Mtime { + result.LastMtime = block.Mtime + } inLayer := false for _, l := range layerfilter { if (block.Pos.Y*16) >= l.From && (block.Pos.Y*16) <= l.To { inLayer = true + break } } @@ -134,7 +163,7 @@ func (a *MapBlockAccessor) FindLatestMapBlocks(mintime int64, limit int, layerfi "y": block.Pos.Y, "z": block.Pos.Z, } - logrus.WithFields(fields).Trace("updated mapblock") + logrus.WithFields(fields).Trace("mapblock") key := getKey(block.Pos) @@ -149,9 +178,13 @@ func (a *MapBlockAccessor) FindLatestMapBlocks(mintime int64, limit int, layerfi a.c.Set(key, mapblock, cache.DefaultExpiration) mblist = append(mblist, mapblock) + } - return mblist, nil + result.LastPos = newlastpos + result.List = mblist + + return &result, nil } func (a *MapBlockAccessor) GetMapBlock(pos coords.MapBlockCoords) (*mapblockparser.MapBlock, error) { diff --git a/server/mapblockparser/mapblock.go b/server/mapblockparser/mapblock.go index cb69356..f378d66 100644 --- a/server/mapblockparser/mapblock.go +++ b/server/mapblockparser/mapblock.go @@ -55,6 +55,10 @@ func NewMetadata() *Metadata { return &md } +func (md *Metadata) GetMetadata(x, y, z int) map[string]string { + return md.GetPairsMap(getNodePos(x, y, z)) +} + func (md *Metadata) GetPairsMap(pos int) map[string]string { pairsMap := md.Pairs[pos] if pairsMap == nil { diff --git a/server/mapobject/poi.go b/server/mapobject/poi.go index 031aeaf..74f5fc4 100644 --- a/server/mapobject/poi.go +++ b/server/mapobject/poi.go @@ -6,5 +6,24 @@ import ( ) func onPoiBlock(id int, block *mapblockparser.MapBlock, odb mapobjectdb.DBAccessor) { - panic("OK") //XXX + + for x := 0; x < 16; x++ { + for y := 0; y < 16; y++ { + for z := 0; z < 16; z++ { + name := block.GetNodeName(x, y, z) + if name == "mapserver:poi" { + md := block.Metadata.GetMetadata(x, y, z) + + o := mapobjectdb.NewMapObject(&block.Pos, x, y, z, "poi") + o.Attributes["name"] = md["name"] + o.Attributes["category"] = md["category"] + o.Attributes["url"] = md["url"] + o.Attributes["active"] = md["active"] + o.Attributes["owner"] = md["owner"] + + odb.AddMapData(o) + } + } + } + } } diff --git a/server/mapobject/setup.go b/server/mapobject/setup.go index 3a221ea..694f636 100644 --- a/server/mapobject/setup.go +++ b/server/mapobject/setup.go @@ -10,7 +10,7 @@ type Listener struct { } func (this *Listener) OnParsedMapBlock(block *mapblockparser.MapBlock) { - err := this.ctx.Objectdb.RemoveMapData(block.Pos) + err := this.ctx.Objectdb.RemoveMapData(&block.Pos) if err != nil { panic(err) } diff --git a/server/mapobjectdb/accessor.go b/server/mapobjectdb/accessor.go index b8846cf..1c45622 100644 --- a/server/mapobjectdb/accessor.go +++ b/server/mapobjectdb/accessor.go @@ -2,6 +2,7 @@ package mapobjectdb import ( "mapserver/coords" + "time" ) /* @@ -28,6 +29,20 @@ type MapObject struct { Attributes map[string]string } +func NewMapObject(MBPos *coords.MapBlockCoords, x int, y int, z int, _type string) *MapObject { + o := MapObject{ + MBPos: MBPos, + Type: _type, + X: MBPos.X + x, + Y: MBPos.Y + y, + Z: MBPos.Z + z, + Mtime: time.Now().Unix(), + Attributes: make(map[string]string), + } + + return &o +} + type SearchQuery struct { //block position (not mapblock) Pos1, Pos2 coords.MapBlockCoords @@ -39,8 +54,8 @@ type DBAccessor interface { //Generic map objects (poi, etc) GetMapData(q SearchQuery) ([]MapObject, error) - RemoveMapData(pos coords.MapBlockCoords) error - AddMapData(data MapObject) error + RemoveMapData(pos *coords.MapBlockCoords) error + AddMapData(data *MapObject) error //tile data GetTile(pos *coords.TileCoords) (*Tile, error) diff --git a/server/mapobjectdb/sqlite_mapobjects.go b/server/mapobjectdb/sqlite_mapobjects.go index 99f4407..865a793 100644 --- a/server/mapobjectdb/sqlite_mapobjects.go +++ b/server/mapobjectdb/sqlite_mapobjects.go @@ -12,7 +12,7 @@ const removeMapDataQuery = ` delete from objects where posx = ? and posy = ? and posz = ? ` -func (db *Sqlite3Accessor) RemoveMapData(pos coords.MapBlockCoords) error { +func (db *Sqlite3Accessor) RemoveMapData(pos *coords.MapBlockCoords) error { _, err := db.db.Exec(removeMapDataQuery, pos.X, pos.Y, pos.Z) return err } @@ -29,27 +29,19 @@ object_attributes(objectid, key, value) values(?, ?, ?) ` -func (db *Sqlite3Accessor) AddMapData(data MapObject) error { - tx, err := db.db.Begin() - - if err != nil { - return err - } - +func (db *Sqlite3Accessor) AddMapData(data *MapObject) error { res, err := db.db.Exec(addMapDataQuery, data.X, data.Y, data.Z, data.MBPos.X, data.MBPos.Y, data.MBPos.Z, data.Type, data.Mtime) if err != nil { - tx.Rollback() return err } id, err := res.LastInsertId() if err != nil { - tx.Rollback() return err } @@ -58,11 +50,9 @@ func (db *Sqlite3Accessor) AddMapData(data MapObject) error { _, err := db.db.Exec(addMapDataAttributeQuery, id, k, v) if err != nil { - tx.Rollback() return err } } - tx.Commit() return nil } diff --git a/server/mapobjectdb/sqlite_test.go b/server/mapobjectdb/sqlite_test.go index 7dcbe3d..ce780fe 100644 --- a/server/mapobjectdb/sqlite_test.go +++ b/server/mapobjectdb/sqlite_test.go @@ -69,7 +69,7 @@ func TestMapObjects(t *testing.T) { if err != nil { panic(err) } - defer os.Remove(tmpfile.Name()) + //defer os.Remove(tmpfile.Name()) db, err := NewSqliteAccessor(tmpfile.Name()) if err != nil { @@ -96,7 +96,7 @@ func TestMapObjects(t *testing.T) { Attributes: attrs, } - err = db.AddMapData(o) + err = db.AddMapData(&o) if err != nil { panic(err) } diff --git a/server/mapobjectdb/sqlite_tiles.go b/server/mapobjectdb/sqlite_tiles.go index 7107c43..3066475 100644 --- a/server/mapobjectdb/sqlite_tiles.go +++ b/server/mapobjectdb/sqlite_tiles.go @@ -68,7 +68,10 @@ func (db *Sqlite3Accessor) RemoveTile(pos *coords.TileCoords) error { } func NewSqliteAccessor(filename string) (*Sqlite3Accessor, error) { + //TODO: flag/config for unsafe db access db, err := sql.Open("sqlite3", filename+"?_timeout=500&_journal_mode=MEMORY&_synchronous=OFF") + db.SetMaxOpenConns(1) + if err != nil { return nil, err } diff --git a/server/tilerendererjob/common.go b/server/tilerendererjob/common.go new file mode 100644 index 0000000..df62b39 --- /dev/null +++ b/server/tilerendererjob/common.go @@ -0,0 +1,49 @@ +package tilerendererjob + +import ( + "github.com/sirupsen/logrus" + "mapserver/app" + "mapserver/coords" + "mapserver/mapblockparser" + "strconv" +) + +func getTileKey(tc *coords.TileCoords) string { + return strconv.Itoa(tc.X) + "/" + strconv.Itoa(tc.Y) + "/" + strconv.Itoa(tc.Zoom) +} + +func renderMapblocks(ctx *app.App, jobs chan *coords.TileCoords, mblist []*mapblockparser.MapBlock) int { + tileRenderedMap := make(map[string]bool) + tilecount := 0 + + for i := 12; i >= 1; i-- { + for _, mb := range mblist { + //13 + tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers) + + //12-1 + tc = tc.ZoomOut(13 - i) + + key := getTileKey(tc) + + if tileRenderedMap[key] { + continue + } + + tileRenderedMap[key] = true + + fields := logrus.Fields{ + "X": tc.X, + "Y": tc.Y, + "Zoom": tc.Zoom, + "LayerId": tc.LayerId, + } + logrus.WithFields(fields).Debug("Dispatching tile rendering (z11-1)") + + tilecount++ + jobs <- tc + } + } + + return tilecount +} diff --git a/server/tilerendererjob/incremental.go b/server/tilerendererjob/incremental.go new file mode 100644 index 0000000..c53bad5 --- /dev/null +++ b/server/tilerendererjob/incremental.go @@ -0,0 +1,48 @@ +package tilerendererjob + +import ( + "github.com/sirupsen/logrus" + "mapserver/app" + "mapserver/coords" + "time" +) + +func incrementalRender(ctx *app.App, jobs chan *coords.TileCoords) { + + rstate := ctx.Config.RenderState + + fields := logrus.Fields{ + "LastMtime": rstate.LastMtime, + } + logrus.WithFields(fields).Info("Starting incremental rendering job") + + for true { + start := time.Now() + + result, err := ctx.BlockAccessor.FindMapBlocksByMtime(rstate.LastMtime, ctx.Config.RenderingFetchLimit, ctx.Config.Layers) + + if err != nil { + panic(err) + } + + if len(result.List) == 0 { + time.Sleep(5 * time.Second) + continue + } + + tiles := renderMapblocks(ctx, jobs, result.List) + + rstate.LastMtime = result.LastMtime + ctx.Config.Save() + + t := time.Now() + elapsed := t.Sub(start) + + fields := logrus.Fields{ + "mapblocks": len(result.List), + "tiles": tiles, + "elapsed": elapsed, + } + logrus.WithFields(fields).Info("incremental rendering") + } +} diff --git a/server/tilerendererjob/initial.go b/server/tilerendererjob/initial.go new file mode 100644 index 0000000..e0dc763 --- /dev/null +++ b/server/tilerendererjob/initial.go @@ -0,0 +1,78 @@ +package tilerendererjob + +import ( + "github.com/sirupsen/logrus" + "mapserver/app" + "mapserver/coords" + "time" +) + +func initialRender(ctx *app.App, jobs chan *coords.TileCoords) { + + rstate := ctx.Config.RenderState + totalLegacyCount, err := ctx.Blockdb.CountBlocks(0, 0) + + if err != nil { + panic(err) + } + + fields := logrus.Fields{ + "totalLegacyCount": totalLegacyCount, + "LastMtime": rstate.LastMtime, + } + logrus.WithFields(fields).Info("Starting initial rendering job") + + lastcoords := coords.NewMapBlockCoords(rstate.LastX, rstate.LastY, rstate.LastZ) + + for true { + start := time.Now() + + result, err := ctx.BlockAccessor.FindMapBlocksByPos(lastcoords, ctx.Config.RenderingFetchLimit, ctx.Config.Layers) + + if err != nil { + panic(err) + } + + if len(result.List) == 0 && !result.HasMore { + rstate.InitialRun = false + ctx.Config.Save() + + fields := logrus.Fields{ + "legacyblocks": rstate.LegacyProcessed, + } + logrus.WithFields(fields).Info("initial rendering complete") + + return + } + + tiles := renderMapblocks(ctx, jobs, result.List) + + lastcoords = *result.LastPos + rstate.LastMtime = result.LastMtime + + //Save current positions of initial run + rstate.LastX = lastcoords.X + rstate.LastY = lastcoords.Y + rstate.LastZ = lastcoords.Z + rstate.LegacyProcessed += result.UnfilteredCount + ctx.Config.Save() + + t := time.Now() + elapsed := t.Sub(start) + + progress := int(float64(rstate.LegacyProcessed) / float64(totalLegacyCount) * 100) + + fields := logrus.Fields{ + "mapblocks": len(result.List), + "tiles": tiles, + "processed": rstate.LegacyProcessed, + "progress%": progress, + "X": lastcoords.X, + "Y": lastcoords.Y, + "Z": lastcoords.Z, + "elapsed": elapsed, + } + logrus.WithFields(fields).Info("Initial rendering") + + } +} diff --git a/server/tilerendererjob/job.go b/server/tilerendererjob/job.go new file mode 100644 index 0000000..108b05a --- /dev/null +++ b/server/tilerendererjob/job.go @@ -0,0 +1,23 @@ +package tilerendererjob + +import ( + "mapserver/app" + "mapserver/coords" +) + +func Job(ctx *app.App) { + + rstate := ctx.Config.RenderState + jobs := make(chan *coords.TileCoords, ctx.Config.RenderingQueue) + + for i := 0; i < ctx.Config.RenderingJobs; i++ { + go worker(ctx, jobs) + } + + if rstate.InitialRun { + initialRender(ctx, jobs) + } + + incrementalRender(ctx, jobs) + +} diff --git a/server/tilerendererjob/worker.go b/server/tilerendererjob/worker.go new file mode 100644 index 0000000..5f319d0 --- /dev/null +++ b/server/tilerendererjob/worker.go @@ -0,0 +1,16 @@ +package tilerendererjob + +import ( + "mapserver/app" + "mapserver/coords" +) + +func worker(ctx *app.App, coords <-chan *coords.TileCoords) { + for tc := range coords { + ctx.Objectdb.RemoveTile(tc) + _, err := ctx.Tilerenderer.Render(tc, 2) + if err != nil { + panic(err) + } + } +} diff --git a/server/tileupdate/job.go b/server/tileupdate/job.go deleted file mode 100644 index dae83d5..0000000 --- a/server/tileupdate/job.go +++ /dev/null @@ -1,76 +0,0 @@ -package tileupdate - -import ( - "mapserver/app" - "mapserver/coords" - "time" - - "github.com/sirupsen/logrus" -) - -func Job(ctx *app.App) { - rstate := ctx.Config.RenderState - - fields := logrus.Fields{ - "lastmtime": rstate.LastMtime, - } - logrus.WithFields(fields).Info("Starting incremental update") - - for true { - mblist, err := ctx.BlockAccessor.FindLatestMapBlocks(rstate.LastMtime, ctx.Config.UpdateRenderingFetchLimit, ctx.Config.Layers) - - if err != nil { - panic(err) - } - - for _, mb := range mblist { - if mb.Mtime > rstate.LastMtime { - rstate.LastMtime = mb.Mtime - } - - tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers) - - if tc == nil { - panic("tile not in any layer") - } - - for tc.Zoom > 1 { - tc = tc.GetZoomedOutTile() - ctx.Objectdb.RemoveTile(tc) - } - } - - //Render zoom 12-1 - for _, mb := range mblist { - tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers) - for tc.Zoom > 1 { - tc = tc.GetZoomedOutTile() - - fields = logrus.Fields{ - "X": tc.X, - "Y": tc.Y, - "Zoom": tc.Zoom, - "LayerId": tc.LayerId, - } - logrus.WithFields(fields).Debug("Dispatching tile rendering (update)") - - _, err = ctx.Tilerenderer.Render(tc, 2) - if err != nil { - panic(err) - } - } - } - - ctx.Config.Save() - - if len(mblist) > 0 { - fields = logrus.Fields{ - "count": len(mblist), - "lastmtime": rstate.LastMtime, - } - logrus.WithFields(fields).Info("incremental update") - } - - time.Sleep(5 * time.Second) - } -} diff --git a/server/web/tiles.go b/server/web/tiles.go index efe4e85..d1cae4e 100644 --- a/server/web/tiles.go +++ b/server/web/tiles.go @@ -39,7 +39,7 @@ func (t *Tiles) ServeHTTP(resp http.ResponseWriter, req *http.Request) { resp.Header().Add("content-type", "image/png") if tile == nil { - resp.Write(tilerenderer.CreateBlankTile(color.RGBA{0, 0, 0, 0})) + resp.Write(tilerenderer.CreateBlankTile(color.RGBA{255, 255, 255, 255})) //TODO: cache/layer color } else { diff --git a/server/world_format.txt b/server/world_format.txt new file mode 100644 index 0000000..c5d1d1b --- /dev/null +++ b/server/world_format.txt @@ -0,0 +1,668 @@ +============================= +Minetest World Format 22...27 +============================= + +This applies to a world format carrying the block serialization version +22...27, used at least in +- 0.4.dev-20120322 ... 0.4.dev-20120606 (22...23) +- 0.4.0 (23) +- 24 was never released as stable and existed for ~2 days +- 27 was added in 0.4.15-dev + +The block serialization version does not fully specify every aspect of this +format; if compliance with this format is to be checked, it needs to be +done by detecting if the files and data indeed follows it. + +Legacy stuff +============= +Data can, in theory, be contained in the flat file directory structure +described below in Version 17, but it is not officially supported. Also you +may stumble upon all kinds of oddities in not-so-recent formats. + +Files +====== +Everything is contained in a directory, the name of which is freeform, but +often serves as the name of the world. + +Currently the authentication and ban data is stored on a per-world basis. +It can be copied over from an old world to a newly created world. + +World +|-- auth.txt ----- Authentication data +|-- auth.sqlite -- Authentication data (SQLite alternative) +|-- env_meta.txt - Environment metadata +|-- ipban.txt ---- Banned ips/users +|-- map_meta.txt - Map metadata +|-- map.sqlite --- Map data +|-- players ------ Player directory +| |-- player1 -- Player file +| '-- Foo ------ Player file +`-- world.mt ----- World metadata + +auth.txt +--------- +Contains authentication data, player per line. + :: + +Legacy format (until 0.4.12) of password hash is SHA1'd, +in the base64 encoding. + +Format (since 0.4.13) of password hash is #1##, with the +parts inside <> encoded in the base64 encoding. + is an RFC 2945 compatible SRP verifier, +of the given salt, password, and the player's name lowercased, +using the 2048-bit group specified in RFC 5054 and the SHA-256 hash function. + +Example lines: +- Player "celeron55", no password, privileges "interact" and "shout": + celeron55::interact,shout +- Player "Foo", password "bar", privilege "shout", with a legacy password hash: + foo:iEPX+SQWIR3p67lj/0zigSWTKHg:shout +- Player "Foo", password "bar", privilege "shout", with a 0.4.13 pw hash: + foo:#1#hPpy4O3IAn1hsNK00A6wNw#Kpu6rj7McsrPCt4euTb5RA5ltF7wdcWGoYMcRngwDi11cZhPuuR9i5Bo7o6A877TgcEwoc//HNrj9EjR/CGjdyTFmNhiermZOADvd8eu32FYK1kf7RMC0rXWxCenYuOQCG4WF9mMGiyTPxC63VAjAMuc1nCZzmy6D9zt0SIKxOmteI75pAEAIee2hx4OkSXRIiU4Zrxo1Xf7QFxkMY4x77vgaPcvfmuzom0y/fU1EdSnZeopGPvzMpFx80ODFx1P34R52nmVl0W8h4GNo0k8ZiWtRCdrJxs8xIg7z5P1h3Th/BJ0lwexpdK8sQZWng8xaO5ElthNuhO8UQx1l6FgEA:shout +- Player "bar", no password, no privileges: + bar:: + +auth.sqlite +------------ +Contains authentification data as an SQLite database. This replaces auth.txt +above when auth_backend is set to "sqlite3" in world.mt . + +This database contains two tables "auth" and "user_privileges": + +CREATE TABLE `auth` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `name` VARCHAR(32) UNIQUE, + `password` VARCHAR(512), + `last_login` INTEGER +); +CREATE TABLE `user_privileges` ( + `id` INTEGER, + `privilege` VARCHAR(32), + PRIMARY KEY (id, privilege) + CONSTRAINT fk_id FOREIGN KEY (id) REFERENCES auth (id) ON DELETE CASCADE +); + +The "name" and "password" fields of the auth table are the same as the auth.txt +fields (with modern password hash). The "last_login" field is the last login +time as a unix time stamp. + +The "user_privileges" table contains one entry per privilege and player. +A player with "interact" and "shout" privileges will have two entries, one +with privilege="interact" and the second with privilege="shout". + +env_meta.txt +------------- +Simple global environment variables. +Example content (added indentation): + game_time = 73471 + time_of_day = 19118 + EnvArgsEnd + +ipban.txt +---------- +Banned IP addresses and usernames. +Example content (added indentation): + 123.456.78.9|foo + 123.456.78.10|bar + +map_meta.txt +------------- +Simple global map variables. +Example content (added indentation): + seed = 7980462765762429666 + [end_of_params] + +map.sqlite +----------- +Map data. +See Map File Format below. + +player1, Foo +------------- +Player data. +Filename can be anything. +See Player File Format below. + +world.mt +--------- +World metadata. +Example content (added indentation and - explanations): + gameid = mesetint - name of the game + enable_damage = true - whether damage is enabled or not + creative_mode = false - whether creative mode is enabled or not + backend = sqlite3 - which DB backend to use for blocks (sqlite3, dummy, leveldb, redis, postgresql) + player_backend = sqlite3 - which DB backend to use for player data + readonly_backend = sqlite3 - optionally readonly seed DB (DB file _must_ be located in "readonly" subfolder) + server_announce = false - whether the server is publicly announced or not + load_mod_ = false - whether is to be loaded in this world + auth_backend = files - which DB backend to use for authentication data + +Player File Format +=================== + +- Should be pretty self-explanatory. +- Note: position is in nodes * 10 + +Example content (added indentation): + hp = 11 + name = celeron55 + pitch = 39.77 + position = (-5231.97,15,1961.41) + version = 1 + yaw = 101.37 + PlayerArgsEnd + List main 32 + Item default:torch 13 + Item default:pick_steel 1 50112 + Item experimental:tnt + Item default:cobble 99 + Item default:pick_stone 1 13104 + Item default:shovel_steel 1 51838 + Item default:dirt 61 + Item default:rail 78 + Item default:coal_lump 3 + Item default:cobble 99 + Item default:leaves 22 + Item default:gravel 52 + Item default:axe_steel 1 2045 + Item default:cobble 98 + Item default:sand 61 + Item default:water_source 94 + Item default:glass 2 + Item default:mossycobble + Item default:pick_steel 1 64428 + Item animalmaterials:bone + Item default:sword_steel + Item default:sapling + Item default:sword_stone 1 10647 + Item default:dirt 99 + Empty + Empty + Empty + Empty + Empty + Empty + Empty + Empty + EndInventoryList + List craft 9 + Empty + Empty + Empty + Empty + Empty + Empty + Empty + Empty + Empty + EndInventoryList + List craftpreview 1 + Empty + EndInventoryList + List craftresult 1 + Empty + EndInventoryList + EndInventory + +Map File Format +================ + +Minetest maps consist of MapBlocks, chunks of 16x16x16 nodes. + +In addition to the bulk node data, MapBlocks stored on disk also contain +other things. + +History +-------- +We need a bit of history in here. Initially Minetest stored maps in a +format called the "sectors" format. It was a directory/file structure like +this: + sectors2/XXX/ZZZ/YYYY +For example, the MapBlock at (0,1,-2) was this file: + sectors2/000/ffd/0001 + +Eventually Minetest outgrow this directory structure, as filesystems were +struggling under the amount of files and directories. + +Large servers seriously needed a new format, and thus the base of the +current format was invented, suggested by celeron55 and implemented by +JacobF. + +SQLite3 was slammed in, and blocks files were directly inserted as blobs +in a single table, indexed by integer primary keys, oddly mangled from +coordinates. + +Today we know that SQLite3 allows multiple primary keys (which would allow +storing coordinates separately), but the format has been kept unchanged for +that part. So, this is where it has come. + + +So here goes +------------- +map.sqlite is an sqlite3 database, containing a single table, called +"blocks". It looks like this: + + CREATE TABLE `blocks` (`pos` INT NOT NULL PRIMARY KEY,`data` BLOB); + +The key +-------- +"pos" is created from the three coordinates of a MapBlock using this +algorithm, defined here in Python: + + def getBlockAsInteger(p): + return int64(p[2]*16777216 + p[1]*4096 + p[0]) + + def int64(u): + while u >= 2**63: + u -= 2**64 + while u <= -2**63: + u += 2**64 + return u + +It can be converted the other way by using this code: + + def getIntegerAsBlock(i): + x = unsignedToSigned(i % 4096, 2048) + i = int((i - x) / 4096) + y = unsignedToSigned(i % 4096, 2048) + i = int((i - y) / 4096) + z = unsignedToSigned(i % 4096, 2048) + return x,y,z + + def unsignedToSigned(i, max_positive): + if i < max_positive: + return i + else: + return i - 2*max_positive + +The blob +--------- +The blob is the data that would have otherwise gone into the file. + +See below for description. + +MapBlock serialization format +============================== +NOTE: Byte order is MSB first (big-endian). +NOTE: Zlib data is in such a format that Python's zlib at least can + directly decompress. + +u8 version +- map format version number, see serialisation.h for the latest number + +u8 flags +- Flag bitmasks: + - 0x01: is_underground: Should be set to 0 if there will be no light + obstructions above the block. If/when sunlight of a block is updated + and there is no block above it, this value is checked for determining + whether sunlight comes from the top. + - 0x02: day_night_differs: Whether the lighting of the block is different + on day and night. Only blocks that have this bit set are updated when + day transforms to night. + - 0x04: lighting_expired: Not used in version 27 and above. If true, + lighting is invalid and should be updated. If you can't calculate + lighting in your generator properly, you could try setting this 1 to + everything and setting the uppermost block in every sector as + is_underground=0. I am quite sure it doesn't work properly, though. + - 0x08: generated: True if the block has been generated. If false, block + is mostly filled with CONTENT_IGNORE and is likely to contain eg. parts + of trees of neighboring blocks. + +u16 lighting_complete +- Added in version 27. +- This contains 12 flags, each of them corresponds to a direction. +- Indicates if the light is correct at the sides of a map block. + Lighting may not be correct if the light changed, but a neighbor + block was not loaded at that time. + If these flags are false, Minetest will automatically recompute light + when both this block and its required neighbor are loaded. +- The bit order is: + nothing, nothing, nothing, nothing, + night X-, night Y-, night Z-, night Z+, night Y+, night X+, + day X-, day Y-, day Z-, day Z+, day Y+, day X+. + Where 'day' is for the day light bank, 'night' is for the night + light bank. + The 'nothing' bits should be always set, as they will be used + to indicate if direct sunlight spreading is finished. +- Example: if the block at (0, 0, 0) has + lighting_complete = 0b1111111111111110, + then Minetest will correct lighting in the day light bank when + the block at (1, 0, 0) is also loaded. + +u8 content_width +- Number of bytes in the content (param0) fields of nodes +if map format version <= 23: + - Always 1 +if map format version >= 24: + - Always 2 + +u8 params_width +- Number of bytes used for parameters per node +- Always 2 + +zlib-compressed node data: +if content_width == 1: + - content: + u8[4096]: param0 fields + u8[4096]: param1 fields + u8[4096]: param2 fields +if content_width == 2: + - content: + u16[4096]: param0 fields + u8[4096]: param1 fields + u8[4096]: param2 fields +- The location of a node in each of those arrays is (z*16*16 + y*16 + x). + +zlib-compressed node metadata list +- content: +if map format version <= 22: + u16 version (=1) + u16 count of metadata + foreach count: + u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X) + u16 type_id + u16 content_size + u8[content_size] content of metadata. Format depends on type_id, see below. +if map format version >= 23: + u8 version (=1) -- Note the type is u8, while for map format version <= 22 it's u16 + u16 count of metadata + foreach count: + u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X) + u32 num_vars + foreach num_vars: + u16 key_len + u8[key_len] key + u32 val_len + u8[val_len] value + serialized inventory + +- Node timers +if map format version == 23: + u8 unused version (always 0) +if map format version == 24: (NOTE: Not released as stable) + u8 nodetimer_version + if nodetimer_version == 0: + (nothing else) + if nodetimer_version == 1: + u16 num_of_timers + foreach num_of_timers: + u16 timer position (z*16*16 + y*16 + x) + s32 timeout*1000 + s32 elapsed*1000 +if map format version >= 25: + -- Nothing right here, node timers are serialized later + +u8 static object version: +- Always 0 + +u16 static_object_count + +foreach static_object_count: + u8 type (object type-id) + s32 pos_x_nodes * 10000 + s32 pos_y_nodes * 10000 + s32 pos_z_nodes * 10000 + u16 data_size + u8[data_size] data + +u32 timestamp +- Timestamp when last saved, as seconds from starting the game. +- 0xffffffff = invalid/unknown timestamp, nothing should be done with the time + difference when loaded + +u8 name-id-mapping version +- Always 0 + +u16 num_name_id_mappings + +foreach num_name_id_mappings + u16 id + u16 name_len + u8[name_len] name + +- Node timers +if map format version == 25: + u8 length of the data of a single timer (always 2+4+4=10) + u16 num_of_timers + foreach num_of_timers: + u16 timer position (z*16*16 + y*16 + x) + s32 timeout*1000 + s32 elapsed*1000 + +EOF. + +Format of nodes +---------------- +A node is composed of the u8 fields param0, param1 and param2. + +if map format version <= 23: + The content id of a node is determined as so: + - If param0 < 0x80, + content_id = param0 + - Otherwise + content_id = (param0<<4) + (param2>>4) +if map format version >= 24: + The content id of a node is param0. + +The purpose of param1 and param2 depend on the definition of the node. + +The name-id-mapping +-------------------- +The mapping maps node content ids to node names. + +Node metadata format for map format versions <= 22 +--------------------------------------------------- +The node metadata are serialized depending on the type_id field. + +1: Generic metadata + serialized inventory + u32 len + u8[len] text + u16 len + u8[len] owner + u16 len + u8[len] infotext + u16 len + u8[len] inventory drawspec + u8 allow_text_input (bool) + u8 removal_disabled (bool) + u8 enforce_owner (bool) + u32 num_vars + foreach num_vars + u16 len + u8[len] name + u32 len + u8[len] value + +14: Sign metadata + u16 text_len + u8[text_len] text + +15: Chest metadata + serialized inventory + +16: Furnace metadata + TBD + +17: Locked Chest metadata + u16 len + u8[len] owner + serialized inventory + +Static objects +--------------- +Static objects are persistent freely moving objects in the world. + +Object types: +1: Test object +2: Item +3: Rat (deprecated) +4: Oerkki (deprecated) +5: Firefly (deprecated) +6: MobV2 (deprecated) +7: LuaEntity + +1: Item: + u8 version + version 0: + u16 len + u8[len] itemstring + +7: LuaEntity: + u8 version + version 1: + u16 len + u8[len] entity name + u32 len + u8[len] static data + s16 hp + s32 velocity.x * 10000 + s32 velocity.y * 10000 + s32 velocity.z * 10000 + s32 yaw * 1000 + +Itemstring format +------------------ +eg. 'default:dirt 5' +eg. 'default:pick_wood 21323' +eg. '"default:apple" 2' +eg. 'default:apple' +- The wear value in tools is 0...65535 +- There are also a number of older formats that you might stumble upon: +eg. 'node "default:dirt" 5' +eg. 'NodeItem default:dirt 5' +eg. 'ToolItem WPick 21323' + +Inventory serialization format +------------------------------- +- The inventory serialization format is line-based +- The newline character used is "\n" +- The end condition of a serialized inventory is always "EndInventory\n" +- All the slots in a list must always be serialized. + +Example (format does not include "---"): +--- +List foo 4 +Item default:sapling +Item default:sword_stone 1 10647 +Item default:dirt 99 +Empty +EndInventoryList +List bar 9 +Empty +Empty +Empty +Empty +Empty +Empty +Empty +Empty +Empty +EndInventoryList +EndInventory +--- + +============================================== +Minetest World Format used as of 2011-05 or so +============================================== + +Map data serialization format version 17. + +0.3.1 does not use this format, but a more recent one. This exists here for +historical reasons. + +Directory structure: +sectors/XXXXZZZZ or sectors2/XXX/ZZZ +XXXX, ZZZZ, XXX and ZZZ being the hexadecimal X and Z coordinates. +Under these, the block files are stored, called YYYY. + +There also exists files map_meta.txt and chunk_meta, that are used by the +generator. If they are not found or invalid, the generator will currently +behave quite strangely. + +The MapBlock file format (sectors2/XXX/ZZZ/YYYY): +------------------------------------------------- + +NOTE: Byte order is MSB first. + +u8 version +- map format version number, this one is version 17 + +u8 flags +- Flag bitmasks: + - 0x01: is_underground: Should be set to 0 if there will be no light + obstructions above the block. If/when sunlight of a block is updated and + there is no block above it, this value is checked for determining whether + sunlight comes from the top. + - 0x02: day_night_differs: Whether the lighting of the block is different on + day and night. Only blocks that have this bit set are updated when day + transforms to night. + - 0x04: lighting_expired: If true, lighting is invalid and should be updated. + If you can't calculate lighting in your generator properly, you could try + setting this 1 to everything and setting the uppermost block in every + sector as is_underground=0. I am quite sure it doesn't work properly, + though. + +zlib-compressed map data: +- content: + u8[4096]: content types + u8[4096]: param1 values + u8[4096]: param2 values + +zlib-compressed node metadata +- content: + u16 version (=1) + u16 count of metadata + foreach count: + u16 position (= p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X) + u16 type_id + u16 content_size + u8[content_size] misc. stuff contained in the metadata + +u16 mapblockobject_count +- always write as 0. +- if read != 0, just fail. + +foreach mapblockobject_count: + - deprecated, should not be used. Length of this data can only be known by + properly parsing it. Just hope not to run into any of this. + +u8 static object version: +- currently 0 + +u16 static_object_count + +foreach static_object_count: + u8 type (object type-id) + s32 pos_x * 1000 + s32 pos_y * 1000 + s32 pos_z * 1000 + u16 data_size + u8[data_size] data + +u32 timestamp +- Timestamp when last saved, as seconds from starting the game. +- 0xffffffff = invalid/unknown timestamp, nothing will be done with the time + difference when loaded (recommended) + +Node metadata format: +--------------------- + +Sign metadata: + u16 string_len + u8[string_len] string + +Furnace metadata: + TBD + +Chest metadata: + TBD + +Locking Chest metadata: + u16 string_len + u8[string_len] string + TBD + +// END +