Koldun/mapserver
1
0
Fork 0
forked from MTSR/mapserver
Code Pull Requests Activity

refactored rendering job

Browse Source
This commit is contained in:
NatureFreshMilk 2019-01-24 15:56:37 +01:00
parent 0ebc6fbd2f
commit 7d52ee6507
10 changed files with 986 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
doc/dev.md Normal file
View File

@ -0,0 +1,5 @@
```sql
update blocks set mtime = 0 where pos = (select max(pos) from blocks);
select max(mtime) from blocks;
```

5
server/db/accessor.go
View File

@ -12,7 +12,10 @@ type Block struct {
type DBAccessor interface {
Migrate() error
FindBlocks(lastpos coords.MapBlockCoords, lastmtime int64, limit int) ([]Block, error)
FindBlocksByMtime(gtmtime int64, limit int) ([]Block, error)
FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]Block, error)
CountBlocks(frommtime, tomtime int64) (int, error)
GetBlock(pos coords.MapBlockCoords) (*Block, error)
}

47
server/db/sqlite.go
View File

@ -72,20 +72,55 @@ func convertRows(pos int64, data []byte, mtime int64) Block {
return Block{Pos: c, Data: data, Mtime: mtime}
}
const getLastBlockQuery = `
const getBlocksByMtimeQuery = `
select pos,data,mtime
from blocks b
where b.mtime >= ?
and b.pos > ?
order by b.mtime asc, b.pos asc
where b.mtime > ?
order by b.mtime asc
limit ?
`
func (db *Sqlite3Accessor) FindBlocks(lastpos coords.MapBlockCoords, lastmtime int64, limit int) ([]Block, error) {
func (db *Sqlite3Accessor) FindBlocksByMtime(gtmtime int64, limit int) ([]Block, error) {
blocks := make([]Block, 0)
rows, err := db.db.Query(getBlocksByMtimeQuery, gtmtime, limit)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var pos int64
var data []byte
var mtime int64
err = rows.Scan(&pos, &data, &mtime)
if err != nil {
return nil, err
}
mb := convertRows(pos, data, mtime)
blocks = append(blocks, mb)
}
return blocks, nil
}
const getLastBlockQuery = `
select pos,data,mtime
from blocks b
where b.mtime = 0
and b.pos > ?
order by b.pos asc, b.mtime asc
limit ?
`
func (db *Sqlite3Accessor) FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]Block, error) {
blocks := make([]Block, 0)
pc := coords.CoordToPlain(lastpos)
rows, err := db.db.Query(getLastBlockQuery, lastmtime, pc, limit)
rows, err := db.db.Query(getLastBlockQuery, pc, limit)
if err != nil {
return nil, err
}

81
server/mapblockaccessor/mapblockaccessor.go
View File

@ -49,18 +49,15 @@ type FindMapBlocksResult struct {
UnfilteredCount int
}
func (a *MapBlockAccessor) FindMapBlocks(lastpos coords.MapBlockCoords, lastmtime int64, limit int, layerfilter []layer.Layer) (*FindMapBlocksResult, error) {
func (a *MapBlockAccessor) FindMapBlocksByMtime(lastmtime int64, limit int, layerfilter []layer.Layer) (*FindMapBlocksResult, error) {
fields := logrus.Fields{
"x": lastpos.X,
"y": lastpos.Y,
"z": lastpos.Z,
"lastmtime": lastmtime,
"limit": limit,
}
logrus.WithFields(fields).Debug("FindMapBlocks")
logrus.WithFields(fields).Debug("FindMapBlocksByMtime")
blocks, err := a.accessor.FindBlocks(lastpos, lastmtime, limit)
blocks, err := a.accessor.FindBlocksByMtime(lastmtime, limit)
if err != nil {
return nil, err
@ -96,7 +93,77 @@ func (a *MapBlockAccessor) FindMapBlocks(lastpos coords.MapBlockCoords, lastmtim
"y": block.Pos.Y,
"z": block.Pos.Z,
}
logrus.WithFields(fields).Trace("legacy mapblock")
logrus.WithFields(fields).Debug("mapblock")
key := getKey(block.Pos)
mapblock, err := mapblockparser.Parse(block.Data, block.Mtime, block.Pos)
if err != nil {
return nil, err
}
for _, listener := range a.listeners {
listener.OnParsedMapBlock(mapblock)
}
a.c.Set(key, mapblock, cache.DefaultExpiration)
mblist = append(mblist, mapblock)
}
result.LastPos = newlastpos
result.List = mblist
return &result, nil
}
func (a *MapBlockAccessor) FindMapBlocksByPos(lastpos coords.MapBlockCoords, limit int, layerfilter []layer.Layer) (*FindMapBlocksResult, error) {
fields := logrus.Fields{
"x": lastpos.X,
"y": lastpos.Y,
"z": lastpos.Z,
"limit": limit,
}
logrus.WithFields(fields).Debug("FindMapBlocksByPos")
blocks, err := a.accessor.FindLegacyBlocksByPos(lastpos, limit)
if err != nil {
return nil, err
}
result := FindMapBlocksResult{}
mblist := make([]*mapblockparser.MapBlock, 0)
var newlastpos *coords.MapBlockCoords
result.HasMore = len(blocks) == limit
result.UnfilteredCount = len(blocks)
for _, block := range blocks {
newlastpos = &block.Pos
if result.LastMtime < block.Mtime {
result.LastMtime = block.Mtime
}
inLayer := false
for _, l := range layerfilter {
if (block.Pos.Y*16) >= l.From && (block.Pos.Y*16) <= l.To {
inLayer = true
break
}
}
if !inLayer {
continue
}
fields := logrus.Fields{
"x": block.Pos.X,
"y": block.Pos.Y,
"z": block.Pos.Z,
}
logrus.WithFields(fields).Trace("mapblock")
key := getKey(block.Pos)

49
server/tilerendererjob/common.go Normal file
View File

@ -0,0 +1,49 @@
package tilerendererjob
import (
"github.com/sirupsen/logrus"
"mapserver/app"
"mapserver/coords"
"mapserver/mapblockparser"
"strconv"
)
func getTileKey(tc *coords.TileCoords) string {
return strconv.Itoa(tc.X) + "/" + strconv.Itoa(tc.Y) + "/" + strconv.Itoa(tc.Zoom)
}
func renderMapblocks(ctx *app.App, jobs chan *coords.TileCoords, mblist []*mapblockparser.MapBlock) int {
tileRenderedMap := make(map[string]bool)
tilecount := 0
for i := 12; i >= 1; i-- {
for _, mb := range mblist {
//13
tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers)
//12-1
tc = tc.ZoomOut(13 - i)
key := getTileKey(tc)
if tileRenderedMap[key] {
continue
}
tileRenderedMap[key] = true
fields := logrus.Fields{
"X": tc.X,
"Y": tc.Y,
"Zoom": tc.Zoom,
"LayerId": tc.LayerId,
}
logrus.WithFields(fields).Debug("Dispatching tile rendering (z11-1)")
tilecount++
jobs <- tc
}
}
return tilecount
}

47
server/tilerendererjob/incremental.go Normal file
View File

@ -0,0 +1,47 @@
package tilerendererjob
import (
"github.com/sirupsen/logrus"
"mapserver/app"
"mapserver/coords"
"time"
)
func incrementalRender(ctx *app.App, jobs chan *coords.TileCoords) {
rstate := ctx.Config.RenderState
fields := logrus.Fields{
"LastMtime": rstate.LastMtime,
}
logrus.WithFields(fields).Info("Starting incremental rendering job")
for true {
start := time.Now()
result, err := ctx.BlockAccessor.FindMapBlocksByMtime(rstate.LastMtime, ctx.Config.RenderingFetchLimit, ctx.Config.Layers)
if err != nil {
panic(err)
}
if len(result.List) == 0 {
time.Sleep(5 * time.Second)
continue
}
renderMapblocks(ctx, jobs, result.List)
rstate.LastMtime = result.LastMtime
ctx.Config.Save()
t := time.Now()
elapsed := t.Sub(start)
fields := logrus.Fields{
"count": len(result.List),
"elapsed": elapsed,
}
logrus.WithFields(fields).Info("incremental rendering")
}
}

77
server/tilerendererjob/initial.go Normal file
View File

@ -0,0 +1,77 @@
package tilerendererjob
import (
"github.com/sirupsen/logrus"
"mapserver/app"
"mapserver/coords"
"time"
)
func initialRender(ctx *app.App, jobs chan *coords.TileCoords) {
rstate := ctx.Config.RenderState
totalLegacyCount, err := ctx.Blockdb.CountBlocks(0, 0)
if err != nil {
panic(err)
}
fields := logrus.Fields{
"totalLegacyCount": totalLegacyCount,
"LastMtime": rstate.LastMtime,
}
logrus.WithFields(fields).Info("Starting initial rendering job")
lastcoords := coords.NewMapBlockCoords(rstate.LastX, rstate.LastY, rstate.LastZ)
for true {
start := time.Now()
result, err := ctx.BlockAccessor.FindMapBlocksByPos(lastcoords, ctx.Config.RenderingFetchLimit, ctx.Config.Layers)
if err != nil {
panic(err)
}
if len(result.List) == 0 && !result.HasMore {
rstate.InitialRun = false
ctx.Config.Save()
fields := logrus.Fields{
"legacyblocks": rstate.LegacyProcessed,
}
logrus.WithFields(fields).Info("initial rendering complete")
return
}
renderMapblocks(ctx, jobs, result.List)
lastcoords = *result.LastPos
rstate.LastMtime = result.LastMtime
//Save current positions of initial run
rstate.LastX = lastcoords.X
rstate.LastY = lastcoords.Y
rstate.LastZ = lastcoords.Z
rstate.LegacyProcessed += result.UnfilteredCount
ctx.Config.Save()
t := time.Now()
elapsed := t.Sub(start)
progress := int(float64(rstate.LegacyProcessed) / float64(totalLegacyCount) * 100)
fields := logrus.Fields{
"count": len(result.List),
"processed": rstate.LegacyProcessed,
"progress%": progress,
"X": lastcoords.X,
"Y": lastcoords.Y,
"Z": lastcoords.Z,
"elapsed": elapsed,
}
logrus.WithFields(fields).Info("Initial rendering")
}
}

143
server/tilerendererjob/job.go
View File

@ -1,156 +1,23 @@
package tilerendererjob
import (
"github.com/sirupsen/logrus"
"mapserver/app"
"mapserver/coords"
"strconv"
"time"
)
func getTileKey(tc *coords.TileCoords) string {
return strconv.Itoa(tc.X) + "/" + strconv.Itoa(tc.Y) + "/" + strconv.Itoa(tc.Zoom)
}
func worker(ctx *app.App, coords <-chan *coords.TileCoords) {
for tc := range coords {
ctx.Objectdb.RemoveTile(tc)
_, err := ctx.Tilerenderer.Render(tc, 2)
if err != nil {
panic(err)
}
}
}
func Job(ctx *app.App) {
rstate := ctx.Config.RenderState
var totalLegacyCount int
var err error
if rstate.InitialRun {
totalLegacyCount, err = ctx.Blockdb.CountBlocks(0, 0)
if err != nil {
panic(err)
}
fields := logrus.Fields{
"totalLegacyCount": totalLegacyCount,
"LastMtime": rstate.LastMtime,
}
logrus.WithFields(fields).Info("Starting rendering job")
} else {
fields := logrus.Fields{
"LastMtime": rstate.LastMtime,
}
logrus.WithFields(fields).Info("Starting rendering job")
}
tilecount := 0
lastcoords := coords.NewMapBlockCoords(rstate.LastX, rstate.LastY, rstate.LastZ)
jobs := make(chan *coords.TileCoords, ctx.Config.RenderingQueue)
for i := 0; i < ctx.Config.RenderingJobs; i++ {
go worker(ctx, jobs)
}
for true {
start := time.Now()
result, err := ctx.BlockAccessor.FindMapBlocks(lastcoords, rstate.LastMtime, ctx.Config.RenderingFetchLimit, ctx.Config.Layers)
if err != nil {
panic(err)
}
if len(result.List) == 0 && !result.HasMore {
if rstate.InitialRun {
rstate.InitialRun = false
ctx.Config.Save()
fields := logrus.Fields{
"legacyblocks": rstate.LegacyProcessed,
}
logrus.WithFields(fields).Info("initial rendering complete")
}
time.Sleep(5 * time.Second)
continue
}
tileRenderedMap := make(map[string]bool)
for i := 12; i >= 1; i-- {
for _, mb := range result.List {
//13
tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers)
//12-1
tc = tc.ZoomOut(13 - i)
key := getTileKey(tc)
if tileRenderedMap[key] {
continue
}
tileRenderedMap[key] = true
fields := logrus.Fields{
"X": tc.X,
"Y": tc.Y,
"Zoom": tc.Zoom,
"LayerId": tc.LayerId,
}
logrus.WithFields(fields).Debug("Dispatching tile rendering (z11-1)")
tilecount++
jobs <- tc
}
}
lastcoords = *result.LastPos
rstate.LastMtime = result.LastMtime
if rstate.InitialRun {
//Save current positions of initial run
rstate.LastX = lastcoords.X
rstate.LastY = lastcoords.Y
rstate.LastZ = lastcoords.Z
rstate.LegacyProcessed += result.UnfilteredCount
}
ctx.Config.Save()
t := time.Now()
elapsed := t.Sub(start)
if rstate.InitialRun {
progress := int(float64(rstate.LegacyProcessed) / float64(totalLegacyCount) * 100)
fields := logrus.Fields{
"count": len(result.List),
"processed": rstate.LegacyProcessed,
"progress%": progress,
"X": lastcoords.X,
"Y": lastcoords.Y,
"Z": lastcoords.Z,
"elapsed": elapsed,
}
logrus.WithFields(fields).Info("Initial rendering")
} else {
fields := logrus.Fields{
"count": len(result.List),
"elapsed": elapsed,
}
logrus.WithFields(fields).Info("incremental rendering")
}
if rstate.InitialRun {
initialRender(ctx, jobs)
}
incrementalRender(ctx, jobs)
}

16
server/tilerendererjob/worker.go Normal file
View File

@ -0,0 +1,16 @@
package tilerendererjob
import (
"mapserver/app"
"mapserver/coords"
)
func worker(ctx *app.App, coords <-chan *coords.TileCoords) {
for tc := range coords {
ctx.Objectdb.RemoveTile(tc)
_, err := ctx.Tilerenderer.Render(tc, 2)
if err != nil {
panic(err)
}
}
}

668
server/world_format.txt Normal file
View File

@ -0,0 +1,668 @@
=============================
Minetest World Format 22...27
=============================
This applies to a world format carrying the block serialization version
22...27, used at least in
- 0.4.dev-20120322 ... 0.4.dev-20120606 (22...23)
- 0.4.0 (23)
- 24 was never released as stable and existed for ~2 days
- 27 was added in 0.4.15-dev
The block serialization version does not fully specify every aspect of this
format; if compliance with this format is to be checked, it needs to be
done by detecting if the files and data indeed follows it.
Legacy stuff
=============
Data can, in theory, be contained in the flat file directory structure
described below in Version 17, but it is not officially supported. Also you
may stumble upon all kinds of oddities in not-so-recent formats.
Files
======
Everything is contained in a directory, the name of which is freeform, but
often serves as the name of the world.
Currently the authentication and ban data is stored on a per-world basis.
It can be copied over from an old world to a newly created world.
World
|-- auth.txt ----- Authentication data
|-- auth.sqlite -- Authentication data (SQLite alternative)
|-- env_meta.txt - Environment metadata
|-- ipban.txt ---- Banned ips/users
|-- map_meta.txt - Map metadata
|-- map.sqlite --- Map data
|-- players ------ Player directory
| |-- player1 -- Player file
| '-- Foo ------ Player file
`-- world.mt ----- World metadata
auth.txt
---------
Contains authentication data, player per line.
<name>:<password hash>:<privilege1,...>
Legacy format (until 0.4.12) of password hash is <name><password> SHA1'd,
in the base64 encoding.
Format (since 0.4.13) of password hash is #1#<salt>#<verifier>, with the
parts inside <> encoded in the base64 encoding.
<verifier> is an RFC 2945 compatible SRP verifier,
of the given salt, password, and the player's name lowercased,
using the 2048-bit group specified in RFC 5054 and the SHA-256 hash function.
Example lines:
- Player "celeron55", no password, privileges "interact" and "shout":
celeron55::interact,shout
- Player "Foo", password "bar", privilege "shout", with a legacy password hash:
foo:iEPX+SQWIR3p67lj/0zigSWTKHg:shout
- Player "Foo", password "bar", privilege "shout", with a 0.4.13 pw hash:
foo:#1#hPpy4O3IAn1hsNK00A6wNw#Kpu6rj7McsrPCt4euTb5RA5ltF7wdcWGoYMcRngwDi11cZhPuuR9i5Bo7o6A877TgcEwoc//HNrj9EjR/CGjdyTFmNhiermZOADvd8eu32FYK1kf7RMC0rXWxCenYuOQCG4WF9mMGiyTPxC63VAjAMuc1nCZzmy6D9zt0SIKxOmteI75pAEAIee2hx4OkSXRIiU4Zrxo1Xf7QFxkMY4x77vgaPcvfmuzom0y/fU1EdSnZeopGPvzMpFx80ODFx1P34R52nmVl0W8h4GNo0k8ZiWtRCdrJxs8xIg7z5P1h3Th/BJ0lwexpdK8sQZWng8xaO5ElthNuhO8UQx1l6FgEA:shout
- Player "bar", no password, no privileges:
bar::
auth.sqlite
------------
Contains authentification data as an SQLite database. This replaces auth.txt
above when auth_backend is set to "sqlite3" in world.mt .
This database contains two tables "auth" and "user_privileges":
CREATE TABLE `auth` (
`id` INTEGER PRIMARY KEY AUTOINCREMENT,
`name` VARCHAR(32) UNIQUE,
`password` VARCHAR(512),
`last_login` INTEGER
);
CREATE TABLE `user_privileges` (
`id` INTEGER,
`privilege` VARCHAR(32),
PRIMARY KEY (id, privilege)
CONSTRAINT fk_id FOREIGN KEY (id) REFERENCES auth (id) ON DELETE CASCADE
);
The "name" and "password" fields of the auth table are the same as the auth.txt
fields (with modern password hash). The "last_login" field is the last login
time as a unix time stamp.
The "user_privileges" table contains one entry per privilege and player.
A player with "interact" and "shout" privileges will have two entries, one
with privilege="interact" and the second with privilege="shout".
env_meta.txt
-------------
Simple global environment variables.
Example content (added indentation):
game_time = 73471
time_of_day = 19118
EnvArgsEnd
ipban.txt
----------
Banned IP addresses and usernames.
Example content (added indentation):
123.456.78.9|foo
123.456.78.10|bar
map_meta.txt
-------------
Simple global map variables.
Example content (added indentation):
seed = 7980462765762429666
[end_of_params]
map.sqlite
-----------
Map data.
See Map File Format below.
player1, Foo
-------------
Player data.
Filename can be anything.
See Player File Format below.
world.mt
---------
World metadata.
Example content (added indentation and - explanations):
gameid = mesetint - name of the game
enable_damage = true - whether damage is enabled or not
creative_mode = false - whether creative mode is enabled or not
backend = sqlite3 - which DB backend to use for blocks (sqlite3, dummy, leveldb, redis, postgresql)
player_backend = sqlite3 - which DB backend to use for player data
readonly_backend = sqlite3 - optionally readonly seed DB (DB file _must_ be located in "readonly" subfolder)
server_announce = false - whether the server is publicly announced or not
load_mod_<mod> = false - whether <mod> is to be loaded in this world
auth_backend = files - which DB backend to use for authentication data
Player File Format
===================
- Should be pretty self-explanatory.
- Note: position is in nodes * 10
Example content (added indentation):
hp = 11
name = celeron55
pitch = 39.77
position = (-5231.97,15,1961.41)
version = 1
yaw = 101.37
PlayerArgsEnd
List main 32
Item default:torch 13
Item default:pick_steel 1 50112
Item experimental:tnt
Item default:cobble 99
Item default:pick_stone 1 13104
Item default:shovel_steel 1 51838
Item default:dirt 61
Item default:rail 78
Item default:coal_lump 3
Item default:cobble 99
Item default:leaves 22
Item default:gravel 52
Item default:axe_steel 1 2045
Item default:cobble 98
Item default:sand 61
Item default:water_source 94
Item default:glass 2
Item default:mossycobble
Item default:pick_steel 1 64428
Item animalmaterials:bone
Item default:sword_steel
Item default:sapling
Item default:sword_stone 1 10647
Item default:dirt 99
Empty
Empty
Empty
Empty
Empty
Empty
Empty
Empty
EndInventoryList
List craft 9
Empty
Empty
Empty
Empty
Empty
Empty
Empty
Empty
Empty
EndInventoryList
List craftpreview 1
Empty
EndInventoryList
List craftresult 1
Empty
EndInventoryList
EndInventory
Map File Format
================
Minetest maps consist of MapBlocks, chunks of 16x16x16 nodes.
In addition to the bulk node data, MapBlocks stored on disk also contain
other things.
History
--------
We need a bit of history in here. Initially Minetest stored maps in a
format called the "sectors" format. It was a directory/file structure like
this:
sectors2/XXX/ZZZ/YYYY
For example, the MapBlock at (0,1,-2) was this file:
sectors2/000/ffd/0001
Eventually Minetest outgrow this directory structure, as filesystems were
struggling under the amount of files and directories.
Large servers seriously needed a new format, and thus the base of the
current format was invented, suggested by celeron55 and implemented by
JacobF.
SQLite3 was slammed in, and blocks files were directly inserted as blobs
in a single table, indexed by integer primary keys, oddly mangled from
coordinates.
Today we know that SQLite3 allows multiple primary keys (which would allow
storing coordinates separately), but the format has been kept unchanged for
that part. So, this is where it has come.
</history>
So here goes
-------------
map.sqlite is an sqlite3 database, containing a single table, called
"blocks". It looks like this:
CREATE TABLE `blocks` (`pos` INT NOT NULL PRIMARY KEY,`data` BLOB);
The key
--------
"pos" is created from the three coordinates of a MapBlock using this
algorithm, defined here in Python:
def getBlockAsInteger(p):
return int64(p[2]*16777216 + p[1]*4096 + p[0])
def int64(u):
while u >= 2**63:
u -= 2**64
while u <= -2**63:
u += 2**64
return u
It can be converted the other way by using this code:
def getIntegerAsBlock(i):
x = unsignedToSigned(i % 4096, 2048)
i = int((i - x) / 4096)
y = unsignedToSigned(i % 4096, 2048)
i = int((i - y) / 4096)
z = unsignedToSigned(i % 4096, 2048)
return x,y,z
def unsignedToSigned(i, max_positive):
if i < max_positive:
return i
else:
return i - 2*max_positive
The blob
---------
The blob is the data that would have otherwise gone into the file.
See below for description.
MapBlock serialization format
==============================
NOTE: Byte order is MSB first (big-endian).
NOTE: Zlib data is in such a format that Python's zlib at least can
directly decompress.
u8 version
- map format version number, see serialisation.h for the latest number
u8 flags
- Flag bitmasks:
- 0x01: is_underground: Should be set to 0 if there will be no light
obstructions above the block. If/when sunlight of a block is updated
and there is no block above it, this value is checked for determining
whether sunlight comes from the top.
- 0x02: day_night_differs: Whether the lighting of the block is different
on day and night. Only blocks that have this bit set are updated when
day transforms to night.
- 0x04: lighting_expired: Not used in version 27 and above. If true,
lighting is invalid and should be updated. If you can't calculate
lighting in your generator properly, you could try setting this 1 to
everything and setting the uppermost block in every sector as
is_underground=0. I am quite sure it doesn't work properly, though.
- 0x08: generated: True if the block has been generated. If false, block
is mostly filled with CONTENT_IGNORE and is likely to contain eg. parts
of trees of neighboring blocks.
u16 lighting_complete
- Added in version 27.
- This contains 12 flags, each of them corresponds to a direction.
- Indicates if the light is correct at the sides of a map block.
Lighting may not be correct if the light changed, but a neighbor
block was not loaded at that time.
If these flags are false, Minetest will automatically recompute light
when both this block and its required neighbor are loaded.
- The bit order is:
nothing, nothing, nothing, nothing,
night X-, night Y-, night Z-, night Z+, night Y+, night X+,
day X-, day Y-, day Z-, day Z+, day Y+, day X+.
Where 'day' is for the day light bank, 'night' is for the night
light bank.
The 'nothing' bits should be always set, as they will be used
to indicate if direct sunlight spreading is finished.
- Example: if the block at (0, 0, 0) has
lighting_complete = 0b1111111111111110,
then Minetest will correct lighting in the day light bank when
the block at (1, 0, 0) is also loaded.
u8 content_width
- Number of bytes in the content (param0) fields of nodes
if map format version <= 23:
- Always 1
if map format version >= 24:
- Always 2
u8 params_width
- Number of bytes used for parameters per node
- Always 2
zlib-compressed node data:
if content_width == 1:
- content:
u8[4096]: param0 fields
u8[4096]: param1 fields
u8[4096]: param2 fields
if content_width == 2:
- content:
u16[4096]: param0 fields
u8[4096]: param1 fields
u8[4096]: param2 fields
- The location of a node in each of those arrays is (z*16*16 + y*16 + x).
zlib-compressed node metadata list
- content:
if map format version <= 22:
u16 version (=1)
u16 count of metadata
foreach count:
u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
u16 type_id
u16 content_size
u8[content_size] content of metadata. Format depends on type_id, see below.
if map format version >= 23:
u8 version (=1) -- Note the type is u8, while for map format version <= 22 it's u16
u16 count of metadata
foreach count:
u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
u32 num_vars
foreach num_vars:
u16 key_len
u8[key_len] key
u32 val_len
u8[val_len] value
serialized inventory
- Node timers
if map format version == 23:
u8 unused version (always 0)
if map format version == 24: (NOTE: Not released as stable)
u8 nodetimer_version
if nodetimer_version == 0:
(nothing else)
if nodetimer_version == 1:
u16 num_of_timers
foreach num_of_timers:
u16 timer position (z*16*16 + y*16 + x)
s32 timeout*1000
s32 elapsed*1000
if map format version >= 25:
-- Nothing right here, node timers are serialized later
u8 static object version:
- Always 0
u16 static_object_count
foreach static_object_count:
u8 type (object type-id)
s32 pos_x_nodes * 10000
s32 pos_y_nodes * 10000
s32 pos_z_nodes * 10000
u16 data_size
u8[data_size] data
u32 timestamp
- Timestamp when last saved, as seconds from starting the game.
- 0xffffffff = invalid/unknown timestamp, nothing should be done with the time
difference when loaded
u8 name-id-mapping version
- Always 0
u16 num_name_id_mappings
foreach num_name_id_mappings
u16 id
u16 name_len
u8[name_len] name
- Node timers
if map format version == 25:
u8 length of the data of a single timer (always 2+4+4=10)
u16 num_of_timers
foreach num_of_timers:
u16 timer position (z*16*16 + y*16 + x)
s32 timeout*1000
s32 elapsed*1000
EOF.
Format of nodes
----------------
A node is composed of the u8 fields param0, param1 and param2.
if map format version <= 23:
The content id of a node is determined as so:
- If param0 < 0x80,
content_id = param0
- Otherwise
content_id = (param0<<4) + (param2>>4)
if map format version >= 24:
The content id of a node is param0.
The purpose of param1 and param2 depend on the definition of the node.
The name-id-mapping
--------------------
The mapping maps node content ids to node names.
Node metadata format for map format versions <= 22
---------------------------------------------------
The node metadata are serialized depending on the type_id field.
1: Generic metadata
serialized inventory
u32 len
u8[len] text
u16 len
u8[len] owner
u16 len
u8[len] infotext
u16 len
u8[len] inventory drawspec
u8 allow_text_input (bool)
u8 removal_disabled (bool)
u8 enforce_owner (bool)
u32 num_vars
foreach num_vars
u16 len
u8[len] name
u32 len
u8[len] value
14: Sign metadata
u16 text_len
u8[text_len] text
15: Chest metadata
serialized inventory
16: Furnace metadata
TBD
17: Locked Chest metadata
u16 len
u8[len] owner
serialized inventory
Static objects
---------------
Static objects are persistent freely moving objects in the world.
Object types:
1: Test object
2: Item
3: Rat (deprecated)
4: Oerkki (deprecated)
5: Firefly (deprecated)
6: MobV2 (deprecated)
7: LuaEntity
1: Item:
u8 version
version 0:
u16 len
u8[len] itemstring
7: LuaEntity:
u8 version
version 1:
u16 len
u8[len] entity name
u32 len
u8[len] static data
s16 hp
s32 velocity.x * 10000
s32 velocity.y * 10000
s32 velocity.z * 10000
s32 yaw * 1000
Itemstring format
------------------
eg. 'default:dirt 5'
eg. 'default:pick_wood 21323'
eg. '"default:apple" 2'
eg. 'default:apple'
- The wear value in tools is 0...65535
- There are also a number of older formats that you might stumble upon:
eg. 'node "default:dirt" 5'
eg. 'NodeItem default:dirt 5'
eg. 'ToolItem WPick 21323'
Inventory serialization format
-------------------------------
- The inventory serialization format is line-based
- The newline character used is "\n"
- The end condition of a serialized inventory is always "EndInventory\n"
- All the slots in a list must always be serialized.
Example (format does not include "---"):
---
List foo 4
Item default:sapling
Item default:sword_stone 1 10647
Item default:dirt 99
Empty
EndInventoryList
List bar 9
Empty
Empty
Empty
Empty
Empty
Empty
Empty
Empty
Empty
EndInventoryList
EndInventory
---
==============================================
Minetest World Format used as of 2011-05 or so
==============================================
Map data serialization format version 17.
0.3.1 does not use this format, but a more recent one. This exists here for
historical reasons.
Directory structure:
sectors/XXXXZZZZ or sectors2/XXX/ZZZ
XXXX, ZZZZ, XXX and ZZZ being the hexadecimal X and Z coordinates.
Under these, the block files are stored, called YYYY.
There also exists files map_meta.txt and chunk_meta, that are used by the
generator. If they are not found or invalid, the generator will currently
behave quite strangely.
The MapBlock file format (sectors2/XXX/ZZZ/YYYY):
-------------------------------------------------
NOTE: Byte order is MSB first.
u8 version
- map format version number, this one is version 17
u8 flags
- Flag bitmasks:
- 0x01: is_underground: Should be set to 0 if there will be no light
obstructions above the block. If/when sunlight of a block is updated and
there is no block above it, this value is checked for determining whether
sunlight comes from the top.
- 0x02: day_night_differs: Whether the lighting of the block is different on
day and night. Only blocks that have this bit set are updated when day
transforms to night.
- 0x04: lighting_expired: If true, lighting is invalid and should be updated.
If you can't calculate lighting in your generator properly, you could try
setting this 1 to everything and setting the uppermost block in every
sector as is_underground=0. I am quite sure it doesn't work properly,
though.
zlib-compressed map data:
- content:
u8[4096]: content types
u8[4096]: param1 values
u8[4096]: param2 values
zlib-compressed node metadata
- content:
u16 version (=1)
u16 count of metadata
foreach count:
u16 position (= p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
u16 type_id
u16 content_size
u8[content_size] misc. stuff contained in the metadata
u16 mapblockobject_count
- always write as 0.
- if read != 0, just fail.
foreach mapblockobject_count:
- deprecated, should not be used. Length of this data can only be known by
properly parsing it. Just hope not to run into any of this.
u8 static object version:
- currently 0
u16 static_object_count
foreach static_object_count:
u8 type (object type-id)
s32 pos_x * 1000
s32 pos_y * 1000
s32 pos_z * 1000
u16 data_size
u8[data_size] data
u32 timestamp
- Timestamp when last saved, as seconds from starting the game.
- 0xffffffff = invalid/unknown timestamp, nothing will be done with the time
difference when loaded (recommended)
Node metadata format:
---------------------
Sign metadata:
u16 string_len
u8[string_len] string
Furnace metadata:
TBD
Chest metadata:
TBD
Locking Chest metadata:
u16 string_len
u8[string_len] string
TBD
// END