refactored rendering job

doc/dev.md

@@ -0,0 +1,5 @@
+
+```sql
+update blocks set mtime = 0 where pos = (select max(pos) from blocks);
+select max(mtime) from blocks;
+```

server/db/accessor.go

@@ -12,7 +12,10 @@ type Block struct {
 
 type DBAccessor interface {
 	Migrate() error
-	FindBlocks(lastpos coords.MapBlockCoords, lastmtime int64, limit int) ([]Block, error)
+
+	FindBlocksByMtime(gtmtime int64, limit int) ([]Block, error)
+	FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]Block, error)
+
 	CountBlocks(frommtime, tomtime int64) (int, error)
 	GetBlock(pos coords.MapBlockCoords) (*Block, error)
 }

server/db/sqlite.go

@@ -72,20 +72,55 @@ func convertRows(pos int64, data []byte, mtime int64) Block {
 	return Block{Pos: c, Data: data, Mtime: mtime}
 }
 
-const getLastBlockQuery = `
+const getBlocksByMtimeQuery = `
 select pos,data,mtime
 from blocks b
-where b.mtime >= ?
+where b.mtime > ?
-and b.pos > ?
+order by b.mtime asc
-order by b.mtime asc, b.pos asc
 limit ?
 `
 
-func (db *Sqlite3Accessor) FindBlocks(lastpos coords.MapBlockCoords, lastmtime int64, limit int) ([]Block, error) {
+func (db *Sqlite3Accessor) FindBlocksByMtime(gtmtime int64, limit int) ([]Block, error) {
+	blocks := make([]Block, 0)
+
+	rows, err := db.db.Query(getBlocksByMtimeQuery, gtmtime, limit)
+	if err != nil {
+		return nil, err
+	}
+
+	defer rows.Close()
+
+	for rows.Next() {
+		var pos int64
+		var data []byte
+		var mtime int64
+
+		err = rows.Scan(&pos, &data, &mtime)
+		if err != nil {
+			return nil, err
+		}
+
+		mb := convertRows(pos, data, mtime)
+		blocks = append(blocks, mb)
+	}
+
+	return blocks, nil
+}
+
+const getLastBlockQuery = `
+select pos,data,mtime
+from blocks b
+where b.mtime = 0
+and b.pos > ?
+order by b.pos asc, b.mtime asc
+limit ?
+`
+
+func (db *Sqlite3Accessor) FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]Block, error) {
 	blocks := make([]Block, 0)
 	pc := coords.CoordToPlain(lastpos)
 
-	rows, err := db.db.Query(getLastBlockQuery, lastmtime, pc, limit)
+	rows, err := db.db.Query(getLastBlockQuery, pc, limit)
 	if err != nil {
 		return nil, err
 	}

server/mapblockaccessor/mapblockaccessor.go

@@ -49,18 +49,15 @@ type FindMapBlocksResult struct {
 	UnfilteredCount int
 }
 
-func (a *MapBlockAccessor) FindMapBlocks(lastpos coords.MapBlockCoords, lastmtime int64, limit int, layerfilter []layer.Layer) (*FindMapBlocksResult, error) {
+func (a *MapBlockAccessor) FindMapBlocksByMtime(lastmtime int64, limit int, layerfilter []layer.Layer) (*FindMapBlocksResult, error) {
 
 	fields := logrus.Fields{
-		"x":         lastpos.X,
-		"y":         lastpos.Y,
-		"z":         lastpos.Z,
 		"lastmtime": lastmtime,
 		"limit":     limit,
 	}
-	logrus.WithFields(fields).Debug("FindMapBlocks")
+	logrus.WithFields(fields).Debug("FindMapBlocksByMtime")
 
-	blocks, err := a.accessor.FindBlocks(lastpos, lastmtime, limit)
+	blocks, err := a.accessor.FindBlocksByMtime(lastmtime, limit)
 
 	if err != nil {
 		return nil, err
@@ -96,7 +93,77 @@ func (a *MapBlockAccessor) FindMapBlocks(lastpos coords.MapBlockCoords, lastmtim
 			"y": block.Pos.Y,
 			"z": block.Pos.Z,
 		}
-		logrus.WithFields(fields).Trace("legacy mapblock")
+		logrus.WithFields(fields).Debug("mapblock")
+
+		key := getKey(block.Pos)
+
+		mapblock, err := mapblockparser.Parse(block.Data, block.Mtime, block.Pos)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, listener := range a.listeners {
+			listener.OnParsedMapBlock(mapblock)
+		}
+
+		a.c.Set(key, mapblock, cache.DefaultExpiration)
+		mblist = append(mblist, mapblock)
+
+	}
+
+	result.LastPos = newlastpos
+	result.List = mblist
+
+	return &result, nil
+}
+
+func (a *MapBlockAccessor) FindMapBlocksByPos(lastpos coords.MapBlockCoords, limit int, layerfilter []layer.Layer) (*FindMapBlocksResult, error) {
+
+	fields := logrus.Fields{
+		"x":     lastpos.X,
+		"y":     lastpos.Y,
+		"z":     lastpos.Z,
+		"limit": limit,
+	}
+	logrus.WithFields(fields).Debug("FindMapBlocksByPos")
+
+	blocks, err := a.accessor.FindLegacyBlocksByPos(lastpos, limit)
+
+	if err != nil {
+		return nil, err
+	}
+
+	result := FindMapBlocksResult{}
+
+	mblist := make([]*mapblockparser.MapBlock, 0)
+	var newlastpos *coords.MapBlockCoords
+	result.HasMore = len(blocks) == limit
+	result.UnfilteredCount = len(blocks)
+
+	for _, block := range blocks {
+		newlastpos = &block.Pos
+		if result.LastMtime < block.Mtime {
+			result.LastMtime = block.Mtime
+		}
+
+		inLayer := false
+		for _, l := range layerfilter {
+			if (block.Pos.Y*16) >= l.From && (block.Pos.Y*16) <= l.To {
+				inLayer = true
+				break
+			}
+		}
+
+		if !inLayer {
+			continue
+		}
+
+		fields := logrus.Fields{
+			"x": block.Pos.X,
+			"y": block.Pos.Y,
+			"z": block.Pos.Z,
+		}
+		logrus.WithFields(fields).Trace("mapblock")
 
 		key := getKey(block.Pos)
 

server/tilerendererjob/common.go

@@ -0,0 +1,49 @@
+package tilerendererjob
+
+import (
+	"github.com/sirupsen/logrus"
+	"mapserver/app"
+	"mapserver/coords"
+	"mapserver/mapblockparser"
+	"strconv"
+)
+
+func getTileKey(tc *coords.TileCoords) string {
+	return strconv.Itoa(tc.X) + "/" + strconv.Itoa(tc.Y) + "/" + strconv.Itoa(tc.Zoom)
+}
+
+func renderMapblocks(ctx *app.App, jobs chan *coords.TileCoords, mblist []*mapblockparser.MapBlock) int {
+	tileRenderedMap := make(map[string]bool)
+	tilecount := 0
+
+	for i := 12; i >= 1; i-- {
+		for _, mb := range mblist {
+			//13
+			tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers)
+
+			//12-1
+			tc = tc.ZoomOut(13 - i)
+
+			key := getTileKey(tc)
+
+			if tileRenderedMap[key] {
+				continue
+			}
+
+			tileRenderedMap[key] = true
+
+			fields := logrus.Fields{
+				"X":       tc.X,
+				"Y":       tc.Y,
+				"Zoom":    tc.Zoom,
+				"LayerId": tc.LayerId,
+			}
+			logrus.WithFields(fields).Debug("Dispatching tile rendering (z11-1)")
+
+			tilecount++
+			jobs <- tc
+		}
+	}
+
+	return tilecount
+}

server/tilerendererjob/incremental.go

@@ -0,0 +1,47 @@
+package tilerendererjob
+
+import (
+	"github.com/sirupsen/logrus"
+	"mapserver/app"
+	"mapserver/coords"
+	"time"
+)
+
+func incrementalRender(ctx *app.App, jobs chan *coords.TileCoords) {
+
+	rstate := ctx.Config.RenderState
+
+	fields := logrus.Fields{
+		"LastMtime": rstate.LastMtime,
+	}
+	logrus.WithFields(fields).Info("Starting incremental rendering job")
+
+	for true {
+		start := time.Now()
+
+		result, err := ctx.BlockAccessor.FindMapBlocksByMtime(rstate.LastMtime, ctx.Config.RenderingFetchLimit, ctx.Config.Layers)
+
+		if err != nil {
+			panic(err)
+		}
+
+		if len(result.List) == 0 {
+			time.Sleep(5 * time.Second)
+			continue
+		}
+
+		renderMapblocks(ctx, jobs, result.List)
+
+		rstate.LastMtime = result.LastMtime
+		ctx.Config.Save()
+
+		t := time.Now()
+		elapsed := t.Sub(start)
+
+		fields := logrus.Fields{
+			"count":   len(result.List),
+			"elapsed": elapsed,
+		}
+		logrus.WithFields(fields).Info("incremental rendering")
+	}
+}

server/tilerendererjob/initial.go

@@ -0,0 +1,77 @@
+package tilerendererjob
+
+import (
+	"github.com/sirupsen/logrus"
+	"mapserver/app"
+	"mapserver/coords"
+	"time"
+)
+
+func initialRender(ctx *app.App, jobs chan *coords.TileCoords) {
+
+	rstate := ctx.Config.RenderState
+	totalLegacyCount, err := ctx.Blockdb.CountBlocks(0, 0)
+
+	if err != nil {
+		panic(err)
+	}
+
+	fields := logrus.Fields{
+		"totalLegacyCount": totalLegacyCount,
+		"LastMtime":        rstate.LastMtime,
+	}
+	logrus.WithFields(fields).Info("Starting initial rendering job")
+
+	lastcoords := coords.NewMapBlockCoords(rstate.LastX, rstate.LastY, rstate.LastZ)
+
+	for true {
+		start := time.Now()
+
+		result, err := ctx.BlockAccessor.FindMapBlocksByPos(lastcoords, ctx.Config.RenderingFetchLimit, ctx.Config.Layers)
+
+		if err != nil {
+			panic(err)
+		}
+
+		if len(result.List) == 0 && !result.HasMore {
+			rstate.InitialRun = false
+			ctx.Config.Save()
+
+			fields := logrus.Fields{
+				"legacyblocks": rstate.LegacyProcessed,
+			}
+			logrus.WithFields(fields).Info("initial rendering complete")
+
+			return
+		}
+
+		renderMapblocks(ctx, jobs, result.List)
+
+		lastcoords = *result.LastPos
+		rstate.LastMtime = result.LastMtime
+
+		//Save current positions of initial run
+		rstate.LastX = lastcoords.X
+		rstate.LastY = lastcoords.Y
+		rstate.LastZ = lastcoords.Z
+		rstate.LegacyProcessed += result.UnfilteredCount
+		ctx.Config.Save()
+
+		t := time.Now()
+		elapsed := t.Sub(start)
+
+		progress := int(float64(rstate.LegacyProcessed) / float64(totalLegacyCount) * 100)
+
+		fields := logrus.Fields{
+			"count":     len(result.List),
+			"processed": rstate.LegacyProcessed,
+			"progress%": progress,
+			"X":         lastcoords.X,
+			"Y":         lastcoords.Y,
+			"Z":         lastcoords.Z,
+			"elapsed":   elapsed,
+		}
+		logrus.WithFields(fields).Info("Initial rendering")
+
+	}
+}

server/tilerendererjob/job.go

@@ -1,156 +1,23 @@
 package tilerendererjob
 
 import (
-	"github.com/sirupsen/logrus"
 	"mapserver/app"
 	"mapserver/coords"
-	"strconv"
-	"time"
 )
 
-func getTileKey(tc *coords.TileCoords) string {
-	return strconv.Itoa(tc.X) + "/" + strconv.Itoa(tc.Y) + "/" + strconv.Itoa(tc.Zoom)
-}
-
-func worker(ctx *app.App, coords <-chan *coords.TileCoords) {
-	for tc := range coords {
-		ctx.Objectdb.RemoveTile(tc)
-		_, err := ctx.Tilerenderer.Render(tc, 2)
-		if err != nil {
-			panic(err)
-		}
-	}
-}
-
 func Job(ctx *app.App) {
 
 	rstate := ctx.Config.RenderState
-	var totalLegacyCount int
-	var err error
-
-	if rstate.InitialRun {
-		totalLegacyCount, err = ctx.Blockdb.CountBlocks(0, 0)
-
-		if err != nil {
-			panic(err)
-		}
-
-		fields := logrus.Fields{
-			"totalLegacyCount": totalLegacyCount,
-			"LastMtime":        rstate.LastMtime,
-		}
-		logrus.WithFields(fields).Info("Starting rendering job")
-
-	} else {
-		fields := logrus.Fields{
-			"LastMtime": rstate.LastMtime,
-		}
-		logrus.WithFields(fields).Info("Starting rendering job")
-
-	}
-
-	tilecount := 0
-
-	lastcoords := coords.NewMapBlockCoords(rstate.LastX, rstate.LastY, rstate.LastZ)
-
 	jobs := make(chan *coords.TileCoords, ctx.Config.RenderingQueue)
 
 	for i := 0; i < ctx.Config.RenderingJobs; i++ {
 		go worker(ctx, jobs)
 	}
 
-	for true {
+	if rstate.InitialRun {
-		start := time.Now()
+		initialRender(ctx, jobs)
-
-		result, err := ctx.BlockAccessor.FindMapBlocks(lastcoords, rstate.LastMtime, ctx.Config.RenderingFetchLimit, ctx.Config.Layers)
-
-		if err != nil {
-			panic(err)
-		}
-
-		if len(result.List) == 0 && !result.HasMore {
-			if rstate.InitialRun {
-				rstate.InitialRun = false
-				ctx.Config.Save()
-
-				fields := logrus.Fields{
-					"legacyblocks": rstate.LegacyProcessed,
-				}
-				logrus.WithFields(fields).Info("initial rendering complete")
-			}
-
-			time.Sleep(5 * time.Second)
-			continue
-		}
-
-		tileRenderedMap := make(map[string]bool)
-
-		for i := 12; i >= 1; i-- {
-			for _, mb := range result.List {
-				//13
-				tc := coords.GetTileCoordsFromMapBlock(mb.Pos, ctx.Config.Layers)
-
-				//12-1
-				tc = tc.ZoomOut(13 - i)
-
-				key := getTileKey(tc)
-
-				if tileRenderedMap[key] {
-					continue
-				}
-
-				tileRenderedMap[key] = true
-
-				fields := logrus.Fields{
-					"X":       tc.X,
-					"Y":       tc.Y,
-					"Zoom":    tc.Zoom,
-					"LayerId": tc.LayerId,
-				}
-				logrus.WithFields(fields).Debug("Dispatching tile rendering (z11-1)")
-
-				tilecount++
-				jobs <- tc
-			}
-		}
-
-		lastcoords = *result.LastPos
-		rstate.LastMtime = result.LastMtime
-
-		if rstate.InitialRun {
-			//Save current positions of initial run
-			rstate.LastX = lastcoords.X
-			rstate.LastY = lastcoords.Y
-			rstate.LastZ = lastcoords.Z
-			rstate.LegacyProcessed += result.UnfilteredCount
-		}
-
-		ctx.Config.Save()
-
-		t := time.Now()
-		elapsed := t.Sub(start)
-
-		if rstate.InitialRun {
-			progress := int(float64(rstate.LegacyProcessed) / float64(totalLegacyCount) * 100)
-
-			fields := logrus.Fields{
-				"count":     len(result.List),
-				"processed": rstate.LegacyProcessed,
-				"progress%": progress,
-				"X":         lastcoords.X,
-				"Y":         lastcoords.Y,
-				"Z":         lastcoords.Z,
-				"elapsed":   elapsed,
-			}
-			logrus.WithFields(fields).Info("Initial rendering")
-
-		} else {
-			fields := logrus.Fields{
-				"count":   len(result.List),
-				"elapsed": elapsed,
-			}
-			logrus.WithFields(fields).Info("incremental rendering")
-
-		}
 	}
+
+	incrementalRender(ctx, jobs)
+
 }

server/tilerendererjob/worker.go

@@ -0,0 +1,16 @@
+package tilerendererjob
+
+import (
+	"mapserver/app"
+	"mapserver/coords"
+)
+
+func worker(ctx *app.App, coords <-chan *coords.TileCoords) {
+	for tc := range coords {
+		ctx.Objectdb.RemoveTile(tc)
+		_, err := ctx.Tilerenderer.Render(tc, 2)
+		if err != nil {
+			panic(err)
+		}
+	}
+}

server/world_format.txt

@@ -0,0 +1,668 @@
+=============================
+Minetest World Format 22...27
+=============================
+
+This applies to a world format carrying the block serialization version
+22...27, used at least in
+- 0.4.dev-20120322 ... 0.4.dev-20120606 (22...23)
+- 0.4.0 (23)
+- 24 was never released as stable and existed for ~2 days
+- 27 was added in 0.4.15-dev
+
+The block serialization version does not fully specify every aspect of this
+format; if compliance with this format is to be checked, it needs to be
+done by detecting if the files and data indeed follows it.
+
+Legacy stuff
+=============
+Data can, in theory, be contained in the flat file directory structure
+described below in Version 17, but it is not officially supported. Also you
+may stumble upon all kinds of oddities in not-so-recent formats.
+
+Files
+======
+Everything is contained in a directory, the name of which is freeform, but
+often serves as the name of the world.
+
+Currently the authentication and ban data is stored on a per-world basis.
+It can be copied over from an old world to a newly created world.
+
+World
+|-- auth.txt ----- Authentication data
+|-- auth.sqlite -- Authentication data (SQLite alternative)
+|-- env_meta.txt - Environment metadata
+|-- ipban.txt ---- Banned ips/users
+|-- map_meta.txt - Map metadata
+|-- map.sqlite --- Map data
+|-- players ------ Player directory
+|   |-- player1 -- Player file
+|   '-- Foo ------ Player file
+`-- world.mt ----- World metadata
+
+auth.txt
+---------
+Contains authentication data, player per line.
+  <name>:<password hash>:<privilege1,...>
+
+Legacy format (until 0.4.12) of password hash is <name><password> SHA1'd,
+in the base64 encoding.
+
+Format (since 0.4.13) of password hash is #1#<salt>#<verifier>, with the
+parts inside <> encoded in the base64 encoding.
+<verifier> is an RFC 2945 compatible SRP verifier,
+of the given salt, password, and the player's name lowercased,
+using the 2048-bit group specified in RFC 5054 and the SHA-256 hash function.
+
+Example lines:
+- Player "celeron55", no password, privileges "interact" and "shout":
+    celeron55::interact,shout
+- Player "Foo", password "bar", privilege "shout", with a legacy password hash:
+    foo:iEPX+SQWIR3p67lj/0zigSWTKHg:shout
+- Player "Foo", password "bar", privilege "shout", with a 0.4.13 pw hash:
+    foo:#1#hPpy4O3IAn1hsNK00A6wNw#Kpu6rj7McsrPCt4euTb5RA5ltF7wdcWGoYMcRngwDi11cZhPuuR9i5Bo7o6A877TgcEwoc//HNrj9EjR/CGjdyTFmNhiermZOADvd8eu32FYK1kf7RMC0rXWxCenYuOQCG4WF9mMGiyTPxC63VAjAMuc1nCZzmy6D9zt0SIKxOmteI75pAEAIee2hx4OkSXRIiU4Zrxo1Xf7QFxkMY4x77vgaPcvfmuzom0y/fU1EdSnZeopGPvzMpFx80ODFx1P34R52nmVl0W8h4GNo0k8ZiWtRCdrJxs8xIg7z5P1h3Th/BJ0lwexpdK8sQZWng8xaO5ElthNuhO8UQx1l6FgEA:shout
+- Player "bar", no password, no privileges:
+    bar::
+
+auth.sqlite
+------------
+Contains authentification data as an SQLite database. This replaces auth.txt
+above when auth_backend is set to "sqlite3" in world.mt .
+
+This database contains two tables "auth" and "user_privileges":
+
+CREATE TABLE `auth` (
+  `id` INTEGER PRIMARY KEY AUTOINCREMENT,
+  `name` VARCHAR(32) UNIQUE,
+  `password` VARCHAR(512),
+  `last_login` INTEGER
+);
+CREATE TABLE `user_privileges` (
+  `id` INTEGER,
+  `privilege` VARCHAR(32),
+  PRIMARY KEY (id, privilege)
+  CONSTRAINT fk_id FOREIGN KEY (id) REFERENCES auth (id) ON DELETE CASCADE
+);
+
+The "name" and "password" fields of the auth table are the same as the auth.txt
+fields (with modern password hash). The "last_login" field is the last login
+time as a unix time stamp.
+
+The "user_privileges" table contains one entry per privilege and player.
+A player with "interact" and "shout" privileges will have two entries, one
+with privilege="interact" and the second with privilege="shout".
+
+env_meta.txt
+-------------
+Simple global environment variables.
+Example content (added indentation):
+  game_time = 73471
+  time_of_day = 19118
+  EnvArgsEnd
+
+ipban.txt
+----------
+Banned IP addresses and usernames.
+Example content (added indentation):
+  123.456.78.9|foo
+  123.456.78.10|bar
+
+map_meta.txt
+-------------
+Simple global map variables.
+Example content (added indentation):
+  seed = 7980462765762429666
+  [end_of_params]
+
+map.sqlite
+-----------
+Map data.
+See Map File Format below.
+
+player1, Foo
+-------------
+Player data.
+Filename can be anything.
+See Player File Format below.
+
+world.mt
+---------
+World metadata.
+Example content (added indentation and - explanations):
+  gameid = mesetint             - name of the game
+  enable_damage = true          - whether damage is enabled or not
+  creative_mode = false         - whether creative mode is enabled or not
+  backend = sqlite3             - which DB backend to use for blocks (sqlite3, dummy, leveldb, redis, postgresql)
+  player_backend = sqlite3      - which DB backend to use for player data
+  readonly_backend = sqlite3    - optionally readonly seed DB (DB file _must_ be located in "readonly" subfolder)
+  server_announce = false       - whether the server is publicly announced or not
+  load_mod_<mod> = false        - whether <mod> is to be loaded in this world
+  auth_backend = files          - which DB backend to use for authentication data
+
+Player File Format
+===================
+
+- Should be pretty self-explanatory.
+- Note: position is in nodes * 10
+
+Example content (added indentation):
+  hp = 11
+  name = celeron55
+  pitch = 39.77
+  position = (-5231.97,15,1961.41)
+  version = 1
+  yaw = 101.37
+  PlayerArgsEnd
+  List main 32
+  Item default:torch 13
+  Item default:pick_steel 1 50112
+  Item experimental:tnt
+  Item default:cobble 99
+  Item default:pick_stone 1 13104
+  Item default:shovel_steel 1 51838
+  Item default:dirt 61
+  Item default:rail 78
+  Item default:coal_lump 3
+  Item default:cobble 99
+  Item default:leaves 22
+  Item default:gravel 52
+  Item default:axe_steel 1 2045
+  Item default:cobble 98
+  Item default:sand 61
+  Item default:water_source 94
+  Item default:glass 2
+  Item default:mossycobble
+  Item default:pick_steel 1 64428
+  Item animalmaterials:bone
+  Item default:sword_steel
+  Item default:sapling
+  Item default:sword_stone 1 10647
+  Item default:dirt 99
+  Empty
+  Empty
+  Empty
+  Empty
+  Empty
+  Empty
+  Empty
+  Empty
+  EndInventoryList
+  List craft 9
+  Empty
+  Empty
+  Empty
+  Empty
+  Empty
+  Empty
+  Empty
+  Empty
+  Empty
+  EndInventoryList
+  List craftpreview 1
+  Empty
+  EndInventoryList
+  List craftresult 1
+  Empty
+  EndInventoryList
+  EndInventory
+
+Map File Format
+================
+
+Minetest maps consist of MapBlocks, chunks of 16x16x16 nodes.
+
+In addition to the bulk node data, MapBlocks stored on disk also contain
+other things.
+
+History
+--------
+We need a bit of history in here. Initially Minetest stored maps in a
+format called the "sectors" format. It was a directory/file structure like
+this:
+  sectors2/XXX/ZZZ/YYYY
+For example, the MapBlock at (0,1,-2) was this file:
+  sectors2/000/ffd/0001
+
+Eventually Minetest outgrow this directory structure, as filesystems were
+struggling under the amount of files and directories.
+
+Large servers seriously needed a new format, and thus the base of the
+current format was invented, suggested by celeron55 and implemented by
+JacobF.
+
+SQLite3 was slammed in, and blocks files were directly inserted as blobs
+in a single table, indexed by integer primary keys, oddly mangled from
+coordinates.
+
+Today we know that SQLite3 allows multiple primary keys (which would allow
+storing coordinates separately), but the format has been kept unchanged for
+that part. So, this is where it has come.
+</history>
+
+So here goes
+-------------
+map.sqlite is an sqlite3 database, containing a single table, called
+"blocks". It looks like this:
+
+  CREATE TABLE `blocks` (`pos` INT NOT NULL PRIMARY KEY,`data` BLOB);
+
+The key
+--------
+"pos" is created from the three coordinates of a MapBlock using this
+algorithm, defined here in Python:
+
+  def getBlockAsInteger(p):
+      return int64(p[2]*16777216 + p[1]*4096 + p[0])
+
+  def int64(u):
+      while u >= 2**63:
+          u -= 2**64
+      while u <= -2**63:
+          u += 2**64
+      return u
+
+It can be converted the other way by using this code:
+
+  def getIntegerAsBlock(i):
+      x = unsignedToSigned(i % 4096, 2048)
+      i = int((i - x) / 4096)
+      y = unsignedToSigned(i % 4096, 2048)
+      i = int((i - y) / 4096)
+      z = unsignedToSigned(i % 4096, 2048)
+      return x,y,z
+
+  def unsignedToSigned(i, max_positive):
+      if i < max_positive:
+          return i
+      else:
+          return i - 2*max_positive
+
+The blob
+---------
+The blob is the data that would have otherwise gone into the file.
+
+See below for description.
+
+MapBlock serialization format
+==============================
+NOTE: Byte order is MSB first (big-endian).
+NOTE: Zlib data is in such a format that Python's zlib at least can
+      directly decompress.
+
+u8 version
+- map format version number, see serialisation.h for the latest number
+
+u8 flags
+- Flag bitmasks:
+  - 0x01: is_underground: Should be set to 0 if there will be no light
+    obstructions above the block. If/when sunlight of a block is updated
+    and there is no block above it, this value is checked for determining
+    whether sunlight comes from the top.
+  - 0x02: day_night_differs: Whether the lighting of the block is different
+    on day and night. Only blocks that have this bit set are updated when
+    day transforms to night.
+  - 0x04: lighting_expired: Not used in version 27 and above. If true,
+    lighting is invalid and should be updated.  If you can't calculate
+    lighting in your generator properly, you could try setting this 1 to
+    everything and setting the uppermost block in every sector as
+    is_underground=0. I am quite sure it doesn't work properly, though.
+  - 0x08: generated: True if the block has been generated. If false, block
+    is mostly filled with CONTENT_IGNORE and is likely to contain eg. parts
+    of trees of neighboring blocks.
+
+u16 lighting_complete
+- Added in version 27.
+- This contains 12 flags, each of them corresponds to a direction.
+- Indicates if the light is correct at the sides of a map block.
+  Lighting may not be correct if the light changed, but a neighbor
+  block was not loaded at that time.
+  If these flags are false, Minetest will automatically recompute light
+  when both this block and its required neighbor are loaded.
+- The bit order is:
+  nothing,  nothing,  nothing,  nothing,
+  night X-, night Y-, night Z-, night Z+, night Y+, night X+,
+  day X-,   day Y-,   day Z-,   day Z+,   day Y+,   day X+.
+  Where 'day' is for the day light bank, 'night' is for the night
+  light bank.
+  The 'nothing' bits should be always set, as they will be used
+  to indicate if direct sunlight spreading is finished.
+- Example: if the block at (0, 0, 0) has
+  lighting_complete = 0b1111111111111110,
+  then Minetest will correct lighting in the day light bank when
+  the block at (1, 0, 0) is also loaded.
+
+u8 content_width
+- Number of bytes in the content (param0) fields of nodes
+if map format version <= 23:
+    - Always 1
+if map format version >= 24:
+    - Always 2
+
+u8 params_width
+- Number of bytes used for parameters per node
+- Always 2
+
+zlib-compressed node data:
+if content_width == 1:
+    - content:
+      u8[4096]: param0 fields
+      u8[4096]: param1 fields
+      u8[4096]: param2 fields
+if content_width == 2:
+    - content:
+      u16[4096]: param0 fields
+      u8[4096]: param1 fields
+      u8[4096]: param2 fields
+- The location of a node in each of those arrays is (z*16*16 + y*16 + x).
+
+zlib-compressed node metadata list
+- content:
+if map format version <= 22:
+  u16 version (=1)
+  u16 count of metadata
+  foreach count:
+    u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
+    u16 type_id
+    u16 content_size
+    u8[content_size] content of metadata. Format depends on type_id, see below.
+if map format version >= 23:
+  u8 version (=1) -- Note the type is u8, while for map format version <= 22 it's u16
+  u16 count of metadata
+  foreach count:
+    u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
+    u32 num_vars
+    foreach num_vars:
+      u16 key_len
+      u8[key_len] key
+      u32 val_len
+      u8[val_len] value
+    serialized inventory
+
+- Node timers
+if map format version == 23:
+  u8 unused version (always 0)
+if map format version == 24: (NOTE: Not released as stable)
+  u8 nodetimer_version
+  if nodetimer_version == 0:
+    (nothing else)
+  if nodetimer_version == 1:
+    u16 num_of_timers
+    foreach num_of_timers:
+      u16 timer position (z*16*16 + y*16 + x)
+      s32 timeout*1000
+      s32 elapsed*1000
+if map format version >= 25:
+  -- Nothing right here, node timers are serialized later
+
+u8 static object version:
+- Always 0
+
+u16 static_object_count
+
+foreach static_object_count:
+  u8 type (object type-id)
+  s32 pos_x_nodes * 10000
+  s32 pos_y_nodes * 10000
+  s32 pos_z_nodes * 10000
+  u16 data_size
+  u8[data_size] data
+
+u32 timestamp
+- Timestamp when last saved, as seconds from starting the game.
+- 0xffffffff = invalid/unknown timestamp, nothing should be done with the time
+               difference when loaded
+
+u8 name-id-mapping version
+- Always 0
+
+u16 num_name_id_mappings
+
+foreach num_name_id_mappings
+  u16 id
+  u16 name_len
+  u8[name_len] name
+
+- Node timers
+if map format version == 25:
+  u8 length of the data of a single timer (always 2+4+4=10)
+  u16 num_of_timers
+  foreach num_of_timers:
+    u16 timer position (z*16*16 + y*16 + x)
+    s32 timeout*1000
+    s32 elapsed*1000
+
+EOF.
+
+Format of nodes
+----------------
+A node is composed of the u8 fields param0, param1 and param2.
+
+if map format version <= 23:
+    The content id of a node is determined as so:
+    - If param0 < 0x80,
+        content_id = param0
+    - Otherwise
+        content_id = (param0<<4) + (param2>>4)
+if map format version >= 24:
+    The content id of a node is param0.
+
+The purpose of param1 and param2 depend on the definition of the node.
+
+The name-id-mapping
+--------------------
+The mapping maps node content ids to node names.
+
+Node metadata format for map format versions <= 22
+---------------------------------------------------
+The node metadata are serialized depending on the type_id field.
+
+1: Generic metadata
+  serialized inventory
+  u32 len
+  u8[len] text
+  u16 len
+  u8[len] owner
+  u16 len
+  u8[len] infotext
+  u16 len
+  u8[len] inventory drawspec
+  u8 allow_text_input (bool)
+  u8 removal_disabled (bool)
+  u8 enforce_owner (bool)
+  u32 num_vars
+  foreach num_vars
+    u16 len
+    u8[len] name
+    u32 len
+    u8[len] value
+
+14: Sign metadata
+  u16 text_len
+  u8[text_len] text
+
+15: Chest metadata
+  serialized inventory
+
+16: Furnace metadata
+  TBD
+
+17: Locked Chest metadata
+  u16 len
+  u8[len] owner
+  serialized inventory
+
+Static objects
+---------------
+Static objects are persistent freely moving objects in the world.
+
+Object types:
+1: Test object
+2: Item
+3: Rat (deprecated)
+4: Oerkki (deprecated)
+5: Firefly (deprecated)
+6: MobV2 (deprecated)
+7: LuaEntity
+
+1: Item:
+  u8 version
+  version 0:
+    u16 len
+    u8[len] itemstring
+
+7: LuaEntity:
+  u8 version
+  version 1:
+    u16 len
+    u8[len] entity name
+    u32 len
+    u8[len] static data
+    s16 hp
+    s32 velocity.x * 10000
+    s32 velocity.y * 10000
+    s32 velocity.z * 10000
+    s32 yaw * 1000
+
+Itemstring format
+------------------
+eg. 'default:dirt 5'
+eg. 'default:pick_wood 21323'
+eg. '"default:apple" 2'
+eg. 'default:apple'
+- The wear value in tools is 0...65535
+- There are also a number of older formats that you might stumble upon:
+eg. 'node "default:dirt" 5'
+eg. 'NodeItem default:dirt 5'
+eg. 'ToolItem WPick 21323'
+
+Inventory serialization format
+-------------------------------
+- The inventory serialization format is line-based
+- The newline character used is "\n"
+- The end condition of a serialized inventory is always "EndInventory\n"
+- All the slots in a list must always be serialized.
+
+Example (format does not include "---"):
+---
+List foo 4
+Item default:sapling
+Item default:sword_stone 1 10647
+Item default:dirt 99
+Empty
+EndInventoryList
+List bar 9
+Empty
+Empty
+Empty
+Empty
+Empty
+Empty
+Empty
+Empty
+Empty
+EndInventoryList
+EndInventory
+---
+
+==============================================
+Minetest World Format used as of 2011-05 or so
+==============================================
+
+Map data serialization format version 17.
+
+0.3.1 does not use this format, but a more recent one. This exists here for
+historical reasons.
+
+Directory structure:
+sectors/XXXXZZZZ or sectors2/XXX/ZZZ
+XXXX, ZZZZ, XXX and ZZZ being the hexadecimal X and Z coordinates.
+Under these, the block files are stored, called YYYY.
+
+There also exists files map_meta.txt and chunk_meta, that are used by the
+generator. If they are not found or invalid, the generator will currently
+behave quite strangely.
+
+The MapBlock file format (sectors2/XXX/ZZZ/YYYY):
+-------------------------------------------------
+
+NOTE: Byte order is MSB first.
+
+u8 version
+- map format version number, this one is version 17
+
+u8 flags
+- Flag bitmasks:
+  - 0x01: is_underground: Should be set to 0 if there will be no light
+    obstructions above the block. If/when sunlight of a block is updated and
+	there is no block above it, this value is checked for determining whether
+	sunlight comes from the top.
+  - 0x02: day_night_differs: Whether the lighting of the block is different on
+    day and night. Only blocks that have this bit set are updated when day
+	transforms to night.
+  - 0x04: lighting_expired: If true, lighting is invalid and should be updated.
+    If you can't calculate lighting in your generator properly, you could try
+	setting this 1 to everything and setting the uppermost block in every
+	sector as is_underground=0. I am quite sure it doesn't work properly,
+	though.
+
+zlib-compressed map data:
+- content:
+  u8[4096]: content types
+  u8[4096]: param1 values
+  u8[4096]: param2 values
+
+zlib-compressed node metadata
+- content:
+  u16 version (=1)
+  u16 count of metadata
+  foreach count:
+    u16 position (= p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
+	u16 type_id
+	u16 content_size
+	u8[content_size] misc. stuff contained in the metadata
+
+u16 mapblockobject_count
+- always write as 0.
+- if read != 0, just fail.
+
+foreach mapblockobject_count:
+  - deprecated, should not be used. Length of this data can only be known by
+    properly parsing it. Just hope not to run into any of this.
+
+u8 static object version:
+- currently 0
+
+u16 static_object_count
+
+foreach static_object_count:
+  u8 type (object type-id)
+  s32 pos_x * 1000
+  s32 pos_y * 1000
+  s32 pos_z * 1000
+  u16 data_size
+  u8[data_size] data
+
+u32 timestamp
+- Timestamp when last saved, as seconds from starting the game.
+- 0xffffffff = invalid/unknown timestamp, nothing will be done with the time
+               difference when loaded (recommended)
+
+Node metadata format:
+---------------------
+
+Sign metadata:
+  u16 string_len
+  u8[string_len] string
+
+Furnace metadata:
+  TBD
+
+Chest metadata:
+  TBD
+
+Locking Chest metadata:
+  u16 string_len
+  u8[string_len] string
+  TBD
+
+// END
+