diff --git a/doc/dev.md b/doc/dev.md new file mode 100644 index 0000000..61dd3f1 --- /dev/null +++ b/doc/dev.md @@ -0,0 +1,5 @@ + +```sql +update blocks set mtime = 0 where pos = (select max(pos) from blocks); +select max(mtime) from blocks; +``` diff --git a/server/db/accessor.go b/server/db/accessor.go index a4bcf45..8373c2b 100644 --- a/server/db/accessor.go +++ b/server/db/accessor.go @@ -12,7 +12,10 @@ type Block struct { type DBAccessor interface { Migrate() error - FindBlocks(lastpos coords.MapBlockCoords, lastmtime int64, limit int) ([]Block, error) + + FindBlocksByMtime(gtmtime int64, limit int) ([]Block, error) + FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]Block, error) + CountBlocks(frommtime, tomtime int64) (int, error) GetBlock(pos coords.MapBlockCoords) (*Block, error) } diff --git a/server/db/sqlite.go b/server/db/sqlite.go index 42e4f7d..59cb169 100644 --- a/server/db/sqlite.go +++ b/server/db/sqlite.go @@ -72,20 +72,55 @@ func convertRows(pos int64, data []byte, mtime int64) Block { return Block{Pos: c, Data: data, Mtime: mtime} } -const getLastBlockQuery = ` +const getBlocksByMtimeQuery = ` select pos,data,mtime from blocks b -where b.mtime >= ? -and b.pos > ? -order by b.mtime asc, b.pos asc +where b.mtime > ? +order by b.mtime asc limit ? ` -func (db *Sqlite3Accessor) FindBlocks(lastpos coords.MapBlockCoords, lastmtime int64, limit int) ([]Block, error) { +func (db *Sqlite3Accessor) FindBlocksByMtime(gtmtime int64, limit int) ([]Block, error) { + blocks := make([]Block, 0) + + rows, err := db.db.Query(getBlocksByMtimeQuery, gtmtime, limit) + if err != nil { + return nil, err + } + + defer rows.Close() + + for rows.Next() { + var pos int64 + var data []byte + var mtime int64 + + err = rows.Scan(&pos, &data, &mtime) + if err != nil { + return nil, err + } + + mb := convertRows(pos, data, mtime) + blocks = append(blocks, mb) + } + + return blocks, nil +} + +const getLastBlockQuery = ` +select pos,data,mtime +from blocks b +where b.mtime = 0 +and b.pos > ? +order by b.pos asc, b.mtime asc +limit ? +` + +func (db *Sqlite3Accessor) FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]Block, error) { blocks := make([]Block, 0) pc := coords.CoordToPlain(lastpos) - rows, err := db.db.Query(getLastBlockQuery, lastmtime, pc, limit) + rows, err := db.db.Query(getLastBlockQuery, pc, limit) if err != nil { return nil, err } diff --git a/server/mapblockaccessor/mapblockaccessor.go b/server/mapblockaccessor/mapblockaccessor.go index 1fd77c9..b0e42ab 100644 --- a/server/mapblockaccessor/mapblockaccessor.go +++ b/server/mapblockaccessor/mapblockaccessor.go @@ -49,18 +49,15 @@ type FindMapBlocksResult struct { UnfilteredCount int } -func (a *MapBlockAccessor) FindMapBlocks(lastpos coords.MapBlockCoords, lastmtime int64, limit int, layerfilter []layer.Layer) (*FindMapBlocksResult, error) { +func (a *MapBlockAccessor) FindMapBlocksByMtime(lastmtime int64, limit int, layerfilter []layer.Layer) (*FindMapBlocksResult, error) { fields := logrus.Fields{ - "x": lastpos.X, - "y": lastpos.Y, - "z": lastpos.Z, "lastmtime": lastmtime, "limit": limit, } - logrus.WithFields(fields).Debug("FindMapBlocks") + logrus.WithFields(fields).Debug("FindMapBlocksByMtime") - blocks, err := a.accessor.FindBlocks(lastpos, lastmtime, limit) + blocks, err := a.accessor.FindBlocksByMtime(lastmtime, limit) if err != nil { return nil, err @@ -96,7 +93,77 @@ func (a *MapBlockAccessor) FindMapBlocks(lastpos coords.MapBlockCoords, lastmtim "y": block.Pos.Y, "z": block.Pos.Z, } - logrus.WithFields(fields).Trace("legacy mapblock") + logrus.WithFields(fields).Debug("mapblock") + + key := getKey(block.Pos) + + mapblock, err := mapblockparser.Parse(block.Data, block.Mtime, block.Pos) + if err != nil { + return nil, err + } + + for _, listener := range a.listeners { + listener.OnParsedMapBlock(mapblock) + } + + a.c.Set(key, mapblock, cache.DefaultExpiration) + mblist = append(mblist, mapblock) + + } + + result.LastPos = newlastpos + result.List = mblist + + return &result, nil +} + +func (a *MapBlockAccessor) FindMapBlocksByPos(lastpos coords.MapBlockCoords, limit int, layerfilter []layer.Layer) (*FindMapBlocksResult, error) { + + fields := logrus.Fields{ + "x": lastpos.X, + "y": lastpos.Y, + "z": lastpos.Z, + "limit": limit, + } + logrus.WithFields(fields).Debug("FindMapBlocksByPos") + + blocks, err := a.accessor.FindLegacyBlocksByPos(lastpos, limit) + + if err != nil { + return nil, err + } + + result := FindMapBlocksResult{} + + mblist := make([]*mapblockparser.MapBlock, 0) + var newlastpos *coords.MapBlockCoords + result.HasMore = len(blocks) == limit + result.UnfilteredCount = len(blocks) + + for _, block := range blocks { + newlastpos = &block.Pos + if result.LastMtime < block.Mtime { + result.LastMtime = block.Mtime + } + + inLayer := false + for _, l := range layerfilter { + if (block.Pos.Y*16) >= l.From && (block.Pos.Y*16) <= l.To { + inLayer = true + break + } + } + + if !inLayer { + continue + } + + fields := logrus.Fields{ + "x": block.Pos.X, + "y": block.Pos.Y, + "z": block.Pos.Z, + } + logrus.WithFields(fields).Trace("mapblock") key := getKey(block.Pos) diff --git a/server/tilerendererjob/common.go b/server/tilerendererjob/common.go new file mode 100644 index 0000000..df62b39 --- /dev/null +++ b/server/tilerendererjob/common.go @@ -0,0 +1,49 @@ +package tilerendererjob + +import ( + "github.com/sirupsen/logrus" + "mapserver/app" + "mapserver/coords" + "mapserver/mapblockparser" + "strconv" +) + +func getTileKey(tc *coords.TileCoords) string { + return strconv.Itoa(tc.X) + "/" + strconv.Itoa(tc.Y) + "/" + strconv.Itoa(tc.Zoom) +} + +func renderMapblocks(ctx *app.App, jobs chan *coords.TileCoords, mblist []*mapblockparser.MapBlock) int { + tileRenderedMap := make(map[string]bool) + tilecount := 0 + + for i := 12; i >= 1; i-- { + for _, mb := range mblist { + //13 + tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers) + + //12-1 + tc = tc.ZoomOut(13 - i) + + key := getTileKey(tc) + + if tileRenderedMap[key] { + continue + } + + tileRenderedMap[key] = true + + fields := logrus.Fields{ + "X": tc.X, + "Y": tc.Y, + "Zoom": tc.Zoom, + "LayerId": tc.LayerId, + } + logrus.WithFields(fields).Debug("Dispatching tile rendering (z11-1)") + + tilecount++ + jobs <- tc + } + } + + return tilecount +} diff --git a/server/tilerendererjob/incremental.go b/server/tilerendererjob/incremental.go new file mode 100644 index 0000000..c41ee60 --- /dev/null +++ b/server/tilerendererjob/incremental.go @@ -0,0 +1,47 @@ +package tilerendererjob + +import ( + "github.com/sirupsen/logrus" + "mapserver/app" + "mapserver/coords" + "time" +) + +func incrementalRender(ctx *app.App, jobs chan *coords.TileCoords) { + + rstate := ctx.Config.RenderState + + fields := logrus.Fields{ + "LastMtime": rstate.LastMtime, + } + logrus.WithFields(fields).Info("Starting incremental rendering job") + + for true { + start := time.Now() + + result, err := ctx.BlockAccessor.FindMapBlocksByMtime(rstate.LastMtime, ctx.Config.RenderingFetchLimit, ctx.Config.Layers) + + if err != nil { + panic(err) + } + + if len(result.List) == 0 { + time.Sleep(5 * time.Second) + continue + } + + renderMapblocks(ctx, jobs, result.List) + + rstate.LastMtime = result.LastMtime + ctx.Config.Save() + + t := time.Now() + elapsed := t.Sub(start) + + fields := logrus.Fields{ + "count": len(result.List), + "elapsed": elapsed, + } + logrus.WithFields(fields).Info("incremental rendering") + } +} diff --git a/server/tilerendererjob/initial.go b/server/tilerendererjob/initial.go new file mode 100644 index 0000000..9f23fb4 --- /dev/null +++ b/server/tilerendererjob/initial.go @@ -0,0 +1,77 @@ +package tilerendererjob + +import ( + "github.com/sirupsen/logrus" + "mapserver/app" + "mapserver/coords" + "time" +) + +func initialRender(ctx *app.App, jobs chan *coords.TileCoords) { + + rstate := ctx.Config.RenderState + totalLegacyCount, err := ctx.Blockdb.CountBlocks(0, 0) + + if err != nil { + panic(err) + } + + fields := logrus.Fields{ + "totalLegacyCount": totalLegacyCount, + "LastMtime": rstate.LastMtime, + } + logrus.WithFields(fields).Info("Starting initial rendering job") + + lastcoords := coords.NewMapBlockCoords(rstate.LastX, rstate.LastY, rstate.LastZ) + + for true { + start := time.Now() + + result, err := ctx.BlockAccessor.FindMapBlocksByPos(lastcoords, ctx.Config.RenderingFetchLimit, ctx.Config.Layers) + + if err != nil { + panic(err) + } + + if len(result.List) == 0 && !result.HasMore { + rstate.InitialRun = false + ctx.Config.Save() + + fields := logrus.Fields{ + "legacyblocks": rstate.LegacyProcessed, + } + logrus.WithFields(fields).Info("initial rendering complete") + + return + } + + renderMapblocks(ctx, jobs, result.List) + + lastcoords = *result.LastPos + rstate.LastMtime = result.LastMtime + + //Save current positions of initial run + rstate.LastX = lastcoords.X + rstate.LastY = lastcoords.Y + rstate.LastZ = lastcoords.Z + rstate.LegacyProcessed += result.UnfilteredCount + ctx.Config.Save() + + t := time.Now() + elapsed := t.Sub(start) + + progress := int(float64(rstate.LegacyProcessed) / float64(totalLegacyCount) * 100) + + fields := logrus.Fields{ + "count": len(result.List), + "processed": rstate.LegacyProcessed, + "progress%": progress, + "X": lastcoords.X, + "Y": lastcoords.Y, + "Z": lastcoords.Z, + "elapsed": elapsed, + } + logrus.WithFields(fields).Info("Initial rendering") + + } +} diff --git a/server/tilerendererjob/job.go b/server/tilerendererjob/job.go index 55463df..108b05a 100644 --- a/server/tilerendererjob/job.go +++ b/server/tilerendererjob/job.go @@ -1,156 +1,23 @@ package tilerendererjob import ( - "github.com/sirupsen/logrus" "mapserver/app" "mapserver/coords" - "strconv" - "time" ) -func getTileKey(tc *coords.TileCoords) string { - return strconv.Itoa(tc.X) + "/" + strconv.Itoa(tc.Y) + "/" + strconv.Itoa(tc.Zoom) -} - -func worker(ctx *app.App, coords <-chan *coords.TileCoords) { - for tc := range coords { - ctx.Objectdb.RemoveTile(tc) - _, err := ctx.Tilerenderer.Render(tc, 2) - if err != nil { - panic(err) - } - } -} - func Job(ctx *app.App) { rstate := ctx.Config.RenderState - var totalLegacyCount int - var err error - - if rstate.InitialRun { - totalLegacyCount, err = ctx.Blockdb.CountBlocks(0, 0) - - if err != nil { - panic(err) - } - - fields := logrus.Fields{ - "totalLegacyCount": totalLegacyCount, - "LastMtime": rstate.LastMtime, - } - logrus.WithFields(fields).Info("Starting rendering job") - - } else { - fields := logrus.Fields{ - "LastMtime": rstate.LastMtime, - } - logrus.WithFields(fields).Info("Starting rendering job") - - } - - tilecount := 0 - - lastcoords := coords.NewMapBlockCoords(rstate.LastX, rstate.LastY, rstate.LastZ) - jobs := make(chan *coords.TileCoords, ctx.Config.RenderingQueue) for i := 0; i < ctx.Config.RenderingJobs; i++ { go worker(ctx, jobs) } - for true { - start := time.Now() - - result, err := ctx.BlockAccessor.FindMapBlocks(lastcoords, rstate.LastMtime, ctx.Config.RenderingFetchLimit, ctx.Config.Layers) - - if err != nil { - panic(err) - } - - if len(result.List) == 0 && !result.HasMore { - if rstate.InitialRun { - rstate.InitialRun = false - ctx.Config.Save() - - fields := logrus.Fields{ - "legacyblocks": rstate.LegacyProcessed, - } - logrus.WithFields(fields).Info("initial rendering complete") - } - - time.Sleep(5 * time.Second) - continue - } - - tileRenderedMap := make(map[string]bool) - - for i := 12; i >= 1; i-- { - for _, mb := range result.List { - //13 - tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers) - - //12-1 - tc = tc.ZoomOut(13 - i) - - key := getTileKey(tc) - - if tileRenderedMap[key] { - continue - } - - tileRenderedMap[key] = true - - fields := logrus.Fields{ - "X": tc.X, - "Y": tc.Y, - "Zoom": tc.Zoom, - "LayerId": tc.LayerId, - } - logrus.WithFields(fields).Debug("Dispatching tile rendering (z11-1)") - - tilecount++ - jobs <- tc - } - } - - lastcoords = *result.LastPos - rstate.LastMtime = result.LastMtime - - if rstate.InitialRun { - //Save current positions of initial run - rstate.LastX = lastcoords.X - rstate.LastY = lastcoords.Y - rstate.LastZ = lastcoords.Z - rstate.LegacyProcessed += result.UnfilteredCount - } - - ctx.Config.Save() - - t := time.Now() - elapsed := t.Sub(start) - - if rstate.InitialRun { - progress := int(float64(rstate.LegacyProcessed) / float64(totalLegacyCount) * 100) - - fields := logrus.Fields{ - "count": len(result.List), - "processed": rstate.LegacyProcessed, - "progress%": progress, - "X": lastcoords.X, - "Y": lastcoords.Y, - "Z": lastcoords.Z, - "elapsed": elapsed, - } - logrus.WithFields(fields).Info("Initial rendering") - - } else { - fields := logrus.Fields{ - "count": len(result.List), - "elapsed": elapsed, - } - logrus.WithFields(fields).Info("incremental rendering") - - } + if rstate.InitialRun { + initialRender(ctx, jobs) } + + incrementalRender(ctx, jobs) + } diff --git a/server/tilerendererjob/worker.go b/server/tilerendererjob/worker.go new file mode 100644 index 0000000..5f319d0 --- /dev/null +++ b/server/tilerendererjob/worker.go @@ -0,0 +1,16 @@ +package tilerendererjob + +import ( + "mapserver/app" + "mapserver/coords" +) + +func worker(ctx *app.App, coords <-chan *coords.TileCoords) { + for tc := range coords { + ctx.Objectdb.RemoveTile(tc) + _, err := ctx.Tilerenderer.Render(tc, 2) + if err != nil { + panic(err) + } + } +} diff --git a/server/world_format.txt b/server/world_format.txt new file mode 100644 index 0000000..c5d1d1b --- /dev/null +++ b/server/world_format.txt @@ -0,0 +1,668 @@ +============================= +Minetest World Format 22...27 +============================= + +This applies to a world format carrying the block serialization version +22...27, used at least in +- 0.4.dev-20120322 ... 0.4.dev-20120606 (22...23) +- 0.4.0 (23) +- 24 was never released as stable and existed for ~2 days +- 27 was added in 0.4.15-dev + +The block serialization version does not fully specify every aspect of this +format; if compliance with this format is to be checked, it needs to be +done by detecting if the files and data indeed follows it. + +Legacy stuff +============= +Data can, in theory, be contained in the flat file directory structure +described below in Version 17, but it is not officially supported. Also you +may stumble upon all kinds of oddities in not-so-recent formats. + +Files +====== +Everything is contained in a directory, the name of which is freeform, but +often serves as the name of the world. + +Currently the authentication and ban data is stored on a per-world basis. +It can be copied over from an old world to a newly created world. + +World +|-- auth.txt ----- Authentication data +|-- auth.sqlite -- Authentication data (SQLite alternative) +|-- env_meta.txt - Environment metadata +|-- ipban.txt ---- Banned ips/users +|-- map_meta.txt - Map metadata +|-- map.sqlite --- Map data +|-- players ------ Player directory +| |-- player1 -- Player file +| '-- Foo ------ Player file +`-- world.mt ----- World metadata + +auth.txt +--------- +Contains authentication data, player per line. + :: + +Legacy format (until 0.4.12) of password hash is SHA1'd, +in the base64 encoding. + +Format (since 0.4.13) of password hash is #1##, with the +parts inside <> encoded in the base64 encoding. + is an RFC 2945 compatible SRP verifier, +of the given salt, password, and the player's name lowercased, +using the 2048-bit group specified in RFC 5054 and the SHA-256 hash function. + +Example lines: +- Player "celeron55", no password, privileges "interact" and "shout": + celeron55::interact,shout +- Player "Foo", password "bar", privilege "shout", with a legacy password hash: + foo:iEPX+SQWIR3p67lj/0zigSWTKHg:shout +- Player "Foo", password "bar", privilege "shout", with a 0.4.13 pw hash: + foo:#1#hPpy4O3IAn1hsNK00A6wNw#Kpu6rj7McsrPCt4euTb5RA5ltF7wdcWGoYMcRngwDi11cZhPuuR9i5Bo7o6A877TgcEwoc//HNrj9EjR/CGjdyTFmNhiermZOADvd8eu32FYK1kf7RMC0rXWxCenYuOQCG4WF9mMGiyTPxC63VAjAMuc1nCZzmy6D9zt0SIKxOmteI75pAEAIee2hx4OkSXRIiU4Zrxo1Xf7QFxkMY4x77vgaPcvfmuzom0y/fU1EdSnZeopGPvzMpFx80ODFx1P34R52nmVl0W8h4GNo0k8ZiWtRCdrJxs8xIg7z5P1h3Th/BJ0lwexpdK8sQZWng8xaO5ElthNuhO8UQx1l6FgEA:shout +- Player "bar", no password, no privileges: + bar:: + +auth.sqlite +------------ +Contains authentification data as an SQLite database. This replaces auth.txt +above when auth_backend is set to "sqlite3" in world.mt . + +This database contains two tables "auth" and "user_privileges": + +CREATE TABLE `auth` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `name` VARCHAR(32) UNIQUE, + `password` VARCHAR(512), + `last_login` INTEGER +); +CREATE TABLE `user_privileges` ( + `id` INTEGER, + `privilege` VARCHAR(32), + PRIMARY KEY (id, privilege) + CONSTRAINT fk_id FOREIGN KEY (id) REFERENCES auth (id) ON DELETE CASCADE +); + +The "name" and "password" fields of the auth table are the same as the auth.txt +fields (with modern password hash). The "last_login" field is the last login +time as a unix time stamp. + +The "user_privileges" table contains one entry per privilege and player. +A player with "interact" and "shout" privileges will have two entries, one +with privilege="interact" and the second with privilege="shout". + +env_meta.txt +------------- +Simple global environment variables. +Example content (added indentation): + game_time = 73471 + time_of_day = 19118 + EnvArgsEnd + +ipban.txt +---------- +Banned IP addresses and usernames. +Example content (added indentation): + 123.456.78.9|foo + 123.456.78.10|bar + +map_meta.txt +------------- +Simple global map variables. +Example content (added indentation): + seed = 7980462765762429666 + [end_of_params] + +map.sqlite +----------- +Map data. +See Map File Format below. + +player1, Foo +------------- +Player data. +Filename can be anything. +See Player File Format below. + +world.mt +--------- +World metadata. +Example content (added indentation and - explanations): + gameid = mesetint - name of the game + enable_damage = true - whether damage is enabled or not + creative_mode = false - whether creative mode is enabled or not + backend = sqlite3 - which DB backend to use for blocks (sqlite3, dummy, leveldb, redis, postgresql) + player_backend = sqlite3 - which DB backend to use for player data + readonly_backend = sqlite3 - optionally readonly seed DB (DB file _must_ be located in "readonly" subfolder) + server_announce = false - whether the server is publicly announced or not + load_mod_ = false - whether is to be loaded in this world + auth_backend = files - which DB backend to use for authentication data + +Player File Format +=================== + +- Should be pretty self-explanatory. +- Note: position is in nodes * 10 + +Example content (added indentation): + hp = 11 + name = celeron55 + pitch = 39.77 + position = (-5231.97,15,1961.41) + version = 1 + yaw = 101.37 + PlayerArgsEnd + List main 32 + Item default:torch 13 + Item default:pick_steel 1 50112 + Item experimental:tnt + Item default:cobble 99 + Item default:pick_stone 1 13104 + Item default:shovel_steel 1 51838 + Item default:dirt 61 + Item default:rail 78 + Item default:coal_lump 3 + Item default:cobble 99 + Item default:leaves 22 + Item default:gravel 52 + Item default:axe_steel 1 2045 + Item default:cobble 98 + Item default:sand 61 + Item default:water_source 94 + Item default:glass 2 + Item default:mossycobble + Item default:pick_steel 1 64428 + Item animalmaterials:bone + Item default:sword_steel + Item default:sapling + Item default:sword_stone 1 10647 + Item default:dirt 99 + Empty + Empty + Empty + Empty + Empty + Empty + Empty + Empty + EndInventoryList + List craft 9 + Empty + Empty + Empty + Empty + Empty + Empty + Empty + Empty + Empty + EndInventoryList + List craftpreview 1 + Empty + EndInventoryList + List craftresult 1 + Empty + EndInventoryList + EndInventory + +Map File Format +================ + +Minetest maps consist of MapBlocks, chunks of 16x16x16 nodes. + +In addition to the bulk node data, MapBlocks stored on disk also contain +other things. + +History +-------- +We need a bit of history in here. Initially Minetest stored maps in a +format called the "sectors" format. It was a directory/file structure like +this: + sectors2/XXX/ZZZ/YYYY +For example, the MapBlock at (0,1,-2) was this file: + sectors2/000/ffd/0001 + +Eventually Minetest outgrow this directory structure, as filesystems were +struggling under the amount of files and directories. + +Large servers seriously needed a new format, and thus the base of the +current format was invented, suggested by celeron55 and implemented by +JacobF. + +SQLite3 was slammed in, and blocks files were directly inserted as blobs +in a single table, indexed by integer primary keys, oddly mangled from +coordinates. + +Today we know that SQLite3 allows multiple primary keys (which would allow +storing coordinates separately), but the format has been kept unchanged for +that part. So, this is where it has come. + + +So here goes +------------- +map.sqlite is an sqlite3 database, containing a single table, called +"blocks". It looks like this: + + CREATE TABLE `blocks` (`pos` INT NOT NULL PRIMARY KEY,`data` BLOB); + +The key +-------- +"pos" is created from the three coordinates of a MapBlock using this +algorithm, defined here in Python: + + def getBlockAsInteger(p): + return int64(p[2]*16777216 + p[1]*4096 + p[0]) + + def int64(u): + while u >= 2**63: + u -= 2**64 + while u <= -2**63: + u += 2**64 + return u + +It can be converted the other way by using this code: + + def getIntegerAsBlock(i): + x = unsignedToSigned(i % 4096, 2048) + i = int((i - x) / 4096) + y = unsignedToSigned(i % 4096, 2048) + i = int((i - y) / 4096) + z = unsignedToSigned(i % 4096, 2048) + return x,y,z + + def unsignedToSigned(i, max_positive): + if i < max_positive: + return i + else: + return i - 2*max_positive + +The blob +--------- +The blob is the data that would have otherwise gone into the file. + +See below for description. + +MapBlock serialization format +============================== +NOTE: Byte order is MSB first (big-endian). +NOTE: Zlib data is in such a format that Python's zlib at least can + directly decompress. + +u8 version +- map format version number, see serialisation.h for the latest number + +u8 flags +- Flag bitmasks: + - 0x01: is_underground: Should be set to 0 if there will be no light + obstructions above the block. If/when sunlight of a block is updated + and there is no block above it, this value is checked for determining + whether sunlight comes from the top. + - 0x02: day_night_differs: Whether the lighting of the block is different + on day and night. Only blocks that have this bit set are updated when + day transforms to night. + - 0x04: lighting_expired: Not used in version 27 and above. If true, + lighting is invalid and should be updated. If you can't calculate + lighting in your generator properly, you could try setting this 1 to + everything and setting the uppermost block in every sector as + is_underground=0. I am quite sure it doesn't work properly, though. + - 0x08: generated: True if the block has been generated. If false, block + is mostly filled with CONTENT_IGNORE and is likely to contain eg. parts + of trees of neighboring blocks. + +u16 lighting_complete +- Added in version 27. +- This contains 12 flags, each of them corresponds to a direction. +- Indicates if the light is correct at the sides of a map block. + Lighting may not be correct if the light changed, but a neighbor + block was not loaded at that time. + If these flags are false, Minetest will automatically recompute light + when both this block and its required neighbor are loaded. +- The bit order is: + nothing, nothing, nothing, nothing, + night X-, night Y-, night Z-, night Z+, night Y+, night X+, + day X-, day Y-, day Z-, day Z+, day Y+, day X+. + Where 'day' is for the day light bank, 'night' is for the night + light bank. + The 'nothing' bits should be always set, as they will be used + to indicate if direct sunlight spreading is finished. +- Example: if the block at (0, 0, 0) has + lighting_complete = 0b1111111111111110, + then Minetest will correct lighting in the day light bank when + the block at (1, 0, 0) is also loaded. + +u8 content_width +- Number of bytes in the content (param0) fields of nodes +if map format version <= 23: + - Always 1 +if map format version >= 24: + - Always 2 + +u8 params_width +- Number of bytes used for parameters per node +- Always 2 + +zlib-compressed node data: +if content_width == 1: + - content: + u8[4096]: param0 fields + u8[4096]: param1 fields + u8[4096]: param2 fields +if content_width == 2: + - content: + u16[4096]: param0 fields + u8[4096]: param1 fields + u8[4096]: param2 fields +- The location of a node in each of those arrays is (z*16*16 + y*16 + x). + +zlib-compressed node metadata list +- content: +if map format version <= 22: + u16 version (=1) + u16 count of metadata + foreach count: + u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X) + u16 type_id + u16 content_size + u8[content_size] content of metadata. Format depends on type_id, see below. +if map format version >= 23: + u8 version (=1) -- Note the type is u8, while for map format version <= 22 it's u16 + u16 count of metadata + foreach count: + u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X) + u32 num_vars + foreach num_vars: + u16 key_len + u8[key_len] key + u32 val_len + u8[val_len] value + serialized inventory + +- Node timers +if map format version == 23: + u8 unused version (always 0) +if map format version == 24: (NOTE: Not released as stable) + u8 nodetimer_version + if nodetimer_version == 0: + (nothing else) + if nodetimer_version == 1: + u16 num_of_timers + foreach num_of_timers: + u16 timer position (z*16*16 + y*16 + x) + s32 timeout*1000 + s32 elapsed*1000 +if map format version >= 25: + -- Nothing right here, node timers are serialized later + +u8 static object version: +- Always 0 + +u16 static_object_count + +foreach static_object_count: + u8 type (object type-id) + s32 pos_x_nodes * 10000 + s32 pos_y_nodes * 10000 + s32 pos_z_nodes * 10000 + u16 data_size + u8[data_size] data + +u32 timestamp +- Timestamp when last saved, as seconds from starting the game. +- 0xffffffff = invalid/unknown timestamp, nothing should be done with the time + difference when loaded + +u8 name-id-mapping version +- Always 0 + +u16 num_name_id_mappings + +foreach num_name_id_mappings + u16 id + u16 name_len + u8[name_len] name + +- Node timers +if map format version == 25: + u8 length of the data of a single timer (always 2+4+4=10) + u16 num_of_timers + foreach num_of_timers: + u16 timer position (z*16*16 + y*16 + x) + s32 timeout*1000 + s32 elapsed*1000 + +EOF. + +Format of nodes +---------------- +A node is composed of the u8 fields param0, param1 and param2. + +if map format version <= 23: + The content id of a node is determined as so: + - If param0 < 0x80, + content_id = param0 + - Otherwise + content_id = (param0<<4) + (param2>>4) +if map format version >= 24: + The content id of a node is param0. + +The purpose of param1 and param2 depend on the definition of the node. + +The name-id-mapping +-------------------- +The mapping maps node content ids to node names. + +Node metadata format for map format versions <= 22 +--------------------------------------------------- +The node metadata are serialized depending on the type_id field. + +1: Generic metadata + serialized inventory + u32 len + u8[len] text + u16 len + u8[len] owner + u16 len + u8[len] infotext + u16 len + u8[len] inventory drawspec + u8 allow_text_input (bool) + u8 removal_disabled (bool) + u8 enforce_owner (bool) + u32 num_vars + foreach num_vars + u16 len + u8[len] name + u32 len + u8[len] value + +14: Sign metadata + u16 text_len + u8[text_len] text + +15: Chest metadata + serialized inventory + +16: Furnace metadata + TBD + +17: Locked Chest metadata + u16 len + u8[len] owner + serialized inventory + +Static objects +--------------- +Static objects are persistent freely moving objects in the world. + +Object types: +1: Test object +2: Item +3: Rat (deprecated) +4: Oerkki (deprecated) +5: Firefly (deprecated) +6: MobV2 (deprecated) +7: LuaEntity + +1: Item: + u8 version + version 0: + u16 len + u8[len] itemstring + +7: LuaEntity: + u8 version + version 1: + u16 len + u8[len] entity name + u32 len + u8[len] static data + s16 hp + s32 velocity.x * 10000 + s32 velocity.y * 10000 + s32 velocity.z * 10000 + s32 yaw * 1000 + +Itemstring format +------------------ +eg. 'default:dirt 5' +eg. 'default:pick_wood 21323' +eg. '"default:apple" 2' +eg. 'default:apple' +- The wear value in tools is 0...65535 +- There are also a number of older formats that you might stumble upon: +eg. 'node "default:dirt" 5' +eg. 'NodeItem default:dirt 5' +eg. 'ToolItem WPick 21323' + +Inventory serialization format +------------------------------- +- The inventory serialization format is line-based +- The newline character used is "\n" +- The end condition of a serialized inventory is always "EndInventory\n" +- All the slots in a list must always be serialized. + +Example (format does not include "---"): +--- +List foo 4 +Item default:sapling +Item default:sword_stone 1 10647 +Item default:dirt 99 +Empty +EndInventoryList +List bar 9 +Empty +Empty +Empty +Empty +Empty +Empty +Empty +Empty +Empty +EndInventoryList +EndInventory +--- + +============================================== +Minetest World Format used as of 2011-05 or so +============================================== + +Map data serialization format version 17. + +0.3.1 does not use this format, but a more recent one. This exists here for +historical reasons. + +Directory structure: +sectors/XXXXZZZZ or sectors2/XXX/ZZZ +XXXX, ZZZZ, XXX and ZZZ being the hexadecimal X and Z coordinates. +Under these, the block files are stored, called YYYY. + +There also exists files map_meta.txt and chunk_meta, that are used by the +generator. If they are not found or invalid, the generator will currently +behave quite strangely. + +The MapBlock file format (sectors2/XXX/ZZZ/YYYY): +------------------------------------------------- + +NOTE: Byte order is MSB first. + +u8 version +- map format version number, this one is version 17 + +u8 flags +- Flag bitmasks: + - 0x01: is_underground: Should be set to 0 if there will be no light + obstructions above the block. If/when sunlight of a block is updated and + there is no block above it, this value is checked for determining whether + sunlight comes from the top. + - 0x02: day_night_differs: Whether the lighting of the block is different on + day and night. Only blocks that have this bit set are updated when day + transforms to night. + - 0x04: lighting_expired: If true, lighting is invalid and should be updated. + If you can't calculate lighting in your generator properly, you could try + setting this 1 to everything and setting the uppermost block in every + sector as is_underground=0. I am quite sure it doesn't work properly, + though. + +zlib-compressed map data: +- content: + u8[4096]: content types + u8[4096]: param1 values + u8[4096]: param2 values + +zlib-compressed node metadata +- content: + u16 version (=1) + u16 count of metadata + foreach count: + u16 position (= p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X) + u16 type_id + u16 content_size + u8[content_size] misc. stuff contained in the metadata + +u16 mapblockobject_count +- always write as 0. +- if read != 0, just fail. + +foreach mapblockobject_count: + - deprecated, should not be used. Length of this data can only be known by + properly parsing it. Just hope not to run into any of this. + +u8 static object version: +- currently 0 + +u16 static_object_count + +foreach static_object_count: + u8 type (object type-id) + s32 pos_x * 1000 + s32 pos_y * 1000 + s32 pos_z * 1000 + u16 data_size + u8[data_size] data + +u32 timestamp +- Timestamp when last saved, as seconds from starting the game. +- 0xffffffff = invalid/unknown timestamp, nothing will be done with the time + difference when loaded (recommended) + +Node metadata format: +--------------------- + +Sign metadata: + u16 string_len + u8[string_len] string + +Furnace metadata: + TBD + +Chest metadata: + TBD + +Locking Chest metadata: + u16 string_len + u8[string_len] string + TBD + +// END +