storage backend prepared for sqlite

basic_machines/forceload.lua

@@ -53,6 +53,12 @@ local function chat(player, text)
 	minetest.chat_send_player(player:get_player_name(), "[Techage] "..text)
 end
 
+local function postload_area(pos)
+	if not minetest.forceload_block(pos, true) then
+		minetest.after(60, postload_area, pos)
+	end
+end
+	
 local function add_pos(pos, player)
 	local lPos = minetest.deserialize(player:get_attribute("techage_forceload_blocks")) or {}
 	if not in_list(lPos, pos) and (#lPos < techage.max_num_forceload_blocks or
@@ -259,7 +265,9 @@ minetest.register_on_joinplayer(function(player)
 	for _,pos in ipairs(get_pos_list(player)) do
 		local node = techage.get_node_lvm(pos)
 		if node.name == "techage:forceload" or node.name == "techage:forceloadtile" then
-			minetest.forceload_block(pos, true)
+			if not minetest.forceload_block(pos, true) then
+				minetest.after(60, postload_area, pos)
+			end
 			lPos[#lPos+1] = pos
 		end
 	end

basis/backend_sqlite.lua

@@ -0,0 +1,107 @@
+--[[
+
+	TechAge
+	=======
+
+	Copyright (C) 2020 Joachim Stolberg
+
+	GPL v3
+	See LICENSE.txt for more information
+	
+	Node number management/storage based on SQLite
+
+]]--
+
+local MN = minetest.get_current_modname()
+local WP = minetest.get_worldpath()
+local IE = minetest.request_insecure_environment()
+
+if not IE then
+	error("Please add 'secure.trusted_mods = techage' to minetest.conf!")
+end
+
+local sqlite3 = IE.require("lsqlite3")
+local marshal = IE.require("marshal")
+
+if not sqlite3 then
+	error("Please install sqlite3 via 'luarocks install lsqlite3'")
+end
+if not marshal then
+	error("Please install marshal via 'luarocks install lua-marshal'")
+end
+
+local db = sqlite3.open(WP.."/techage_numbers.sqlite3")
+
+-- Prevent use of this db instance.
+if sqlite3 then sqlite3 = nil end
+
+db:exec[[
+  CREATE TABLE test2(id INTEGER PRIMARY KEY, number INTEGER, x INTEGER, y INTEGER, z INTEGER);
+  CREATE INDEX idx ON test2(number);  
+]]
+
+local insert = db:prepare("INSERT INTO test2 VALUES(NULL, ?, ?, ?, ?);")
+local query = db:prepare("SELECT * FROM test2 WHERE number=?")
+
+local backend = {}
+local storage = minetest.get_mod_storage()
+
+
+local function set(number, pos)
+	insert:reset()
+	insert:bind(1, number)
+	insert:bind(2, pos.x)
+	insert:bind(3, pos.y)
+	insert:bind(4, pos.z)
+	insert:step()	
+end
+
+local function get(number)
+	query:reset()
+	query:bind(1, number)
+	query:step()	
+	local _, _, x, y, z, name = unpack(query:get_values())
+	return {pos = {x = x, y = y, z = z}, name = name}
+end
+
+-------------------------------------------------------------------
+-- API functions
+-------------------------------------------------------------------
+function backend.get_nodepos(number)
+	return minetest.string_to_pos(storage:get_string(number))
+end	
+	
+function backend.set_nodepos(number, pos)
+	storage:get_string(number, minetest.pos_to_string(pos))
+end	
+	
+function backend.add_nodepos(pos)
+	local num = tostring(NextNumber)
+	NextNumber = NextNumber + 1
+	storage:set_int("NextNumber", NextNumber)
+	storage:get_string(num, minetest.pos_to_string(pos))
+	return num
+end	
+	
+function backend.del_nodepos(number)
+	storage:get_string(number, "")
+end	
+
+-- delete invalid entries
+function backend.delete_invalid_entries(node_def)
+	minetest.log("info", "[TechAge] Data maintenance started")
+	for i = 1, NextNumber do
+		local number = tostring(i)
+		if storage:contains(number) then
+			local pos = backend.get_nodepos(number)
+			local name = techage.get_node_lvm(pos).name
+			if not node_def[name] then
+				backend.del_nodepos(number)
+			end
+		end
+	end
+	minetest.log("info", "[TechAge] Data maintenance finished")
+end	
+
+return backend
+

basis/backend_storage.lua

@@ -0,0 +1,96 @@
+--[[
+
+	TechAge
+	=======
+
+	Copyright (C) 2020 Joachim Stolberg
+
+	GPL v3
+	See LICENSE.txt for more information
+	
+	Mod storage based solution
+
+]]--
+
+local backend = {}
+local storage = minetest.get_mod_storage()
+
+-- legacy method
+local function deserialize(s)
+	local tbl = {}
+	for line in s:gmatch("[^;]+") do
+		local num, spos = unpack(string.split(line, "="))
+		tbl[num] = minetest.string_to_pos(spos)
+	end
+	return tbl
+end
+
+
+local Version = minetest.deserialize(storage:get_string("Version")) or 3
+local NextNumber = 0
+
+if Version == 1 then
+	Version = 3
+	local tbl = minetest.deserialize(storage:get_string("Number2Pos")) or {}
+	NextNumber = minetest.deserialize(storage:get_string("NextNumber")) or 1
+	for num, pos in pairs(tbl) do
+		storage:set_string(num, minetest.pos_to_string(pos))
+	end
+	storage:set_string("Number2Pos", "")
+elseif Version == 2 then
+	Version = 3
+	NextNumber = minetest.deserialize(storage:get_string("NextNumber")) or 1
+	local tbl = deserialize(storage:get_string("Number2Pos"))
+	for num, pos in pairs(tbl) do
+		storage:set_string(num, minetest.pos_to_string(pos))
+	end
+	storage:set_string("Number2Pos", "")
+else
+	Version = 3
+	NextNumber = storage:get_int("NextNumber")
+end
+
+storage:set_int("NextNumber", NextNumber)
+storage:set_int("Version", Version)
+
+
+-------------------------------------------------------------------
+-- API functions
+-------------------------------------------------------------------
+function backend.get_nodepos(number)
+	return minetest.string_to_pos(storage:get_string(number))
+end	
+	
+function backend.set_nodepos(number, pos)
+	storage:get_string(number, minetest.pos_to_string(pos))
+end	
+	
+function backend.add_nodepos(pos)
+	local num = tostring(NextNumber)
+	NextNumber = NextNumber + 1
+	storage:set_int("NextNumber", NextNumber)
+	storage:get_string(num, minetest.pos_to_string(pos))
+	return num
+end	
+	
+function backend.del_nodepos(number)
+	storage:get_string(number, "")
+end	
+
+-- delete invalid entries
+function backend.delete_invalid_entries(node_def)
+	minetest.log("info", "[TechAge] Data maintenance started")
+	for i = 1, NextNumber do
+		local number = tostring(i)
+		if storage:contains(number) then
+			local pos = backend.get_nodepos(number)
+			local name = techage.get_node_lvm(pos).name
+			if not node_def[name] then
+				backend.del_nodepos(number)
+			end
+		end
+	end
+	minetest.log("info", "[TechAge] Data maintenance finished")
+end	
+
+return backend

basis/command.lua

@@ -3,7 +3,7 @@
 	TechAge
 	=======
 
-	Copyright (C) 2019 Joachim Stolberg
+	Copyright (C) 2019-2020 Joachim Stolberg
 
 	GPL v3
 	See LICENSE.txt for more information
@@ -14,68 +14,62 @@
 
 --- for lazy programmers
 local S = function(pos) if pos then return minetest.pos_to_string(pos) end end
-local P = minetest.string_to_pos
+--local P = minetest.string_to_pos
-local M = minetest.get_meta
+--local M = minetest.get_meta
 
+local NodeInfoCache = {}
+local MP = minetest.get_modpath("techage")
+local use_database = minetest.settings:get_bool('techage.use_database', false)
+
+-- Localize functions to avoid table lookups (better performance)
+local string_split = string.split
+local NodeDef = techage.NodeDef
+local Tube = techage.Tube
 local check_cart_for_loading = minecart.check_cart_for_loading
 
-local function deserialize(s)
-	local tbl = {}
-	for line in s:gmatch("[^;]+") do
-		local num, spos = unpack(string.split(line, "="))
-		tbl[num] = {pos = minetest.string_to_pos(spos)}
-	end
-	return tbl
-end
-
-local function serialize(data)
-	local tbl = {}
-	for k,v in pairs(data) do
-		tbl[#tbl+1] = k.."="..minetest.pos_to_string(v.pos)
-	end
-	return table.concat(tbl, ";")
-end
-	
-------------------------------------------------------------------
--- Data base storage
 -------------------------------------------------------------------
-local storage = minetest.get_mod_storage()
+-- Database
-local NextNumber = minetest.deserialize(storage:get_string("NextNumber")) or 1
+-------------------------------------------------------------------
-local Version = minetest.deserialize(storage:get_string("Version")) or 2
+local backend
-local Number2Pos
+if use_database then
-if Version == 1 then
+    backend = dofile(MP .. "/basis/backend_sqlite.lua")
-	Version = 2
-	Number2Pos = minetest.deserialize(storage:get_string("Number2Pos")) or {}
 else
-	Number2Pos = deserialize(storage:get_string("Number2Pos"))
+    backend = dofile(MP .. "/basis/backend_storage.lua")
 end
 
-local function update_mod_storage()
+local function update_nodeinfo(number)
-	local t = minetest.get_us_time()
+	local pos = backend.get_nodepos(number)
-	minetest.log("action", "[TechAge] Store data...")
+	if pos then
-	storage:set_string("NextNumber", minetest.serialize(NextNumber))
+		NodeInfoCache[number] = {pos = pos, name = techage.get_node_lvm(pos).name}
-	storage:set_string("Version", minetest.serialize(Version))
+		return NodeInfoCache[number]
-	storage:set_string("Number2Pos", serialize(Number2Pos))
+	end
-	-- store data each hour
-	minetest.after(60*59, update_mod_storage)
-	t = minetest.get_us_time() - t
-	minetest.log("action", "[TechAge] Data stored. t="..t.."us")
 end
 
-minetest.register_on_shutdown(function()
+local function delete_nodeinfo_entry(number)
-	update_mod_storage()
+	if number and NodeInfoCache[number] then
-end)
+		number, _ = next(NodeInfoCache, number)
+		if number then
+			NodeInfoCache[number] = nil
+		end
+	else
+		number, _ = next(NodeInfoCache, nil)
+	end
+	return number
+end
 
--- store data after one hour
+-- Keep the cache size small by deleting entries randomly 
-minetest.after(60*59, update_mod_storage)
+local function keep_small(number)
+	number = delete_nodeinfo_entry(number)
+	minetest.after(2, keep_small, number)
+end
 
--- Key2Number will be generated at runtine
+keep_small()
-local Key2Number = {} 
+
+minetest.after(2, backend.delete_invalid_entries, NodeDef)
 
 -------------------------------------------------------------------
 -- Local helper functions
 -------------------------------------------------------------------
-
 local function in_list(list, x)
 	for _, v in ipairs(list) do
 		if v == x then return true end
@@ -83,26 +77,17 @@ local function in_list(list, x)
 	return false
 end
 
--- Localize functions to avoid table lookups (better performance).
-local string_split = string.split
-local NodeDef = techage.NodeDef
-local Tube = techage.Tube
-
 -- Determine position related node number for addressing purposes
-local function get_number(pos)
+local function get_number(pos, new)
-	local key = minetest.hash_node_position(pos)
+	local meta = minetest.get_meta(pos)
-	if not Key2Number[key] then
+	if meta:contains("node_number") then 
-		Key2Number[key] = NextNumber
+		return meta:get_string("node_number") 
-		NextNumber = NextNumber + 1
 	end
-	return string.format("%u", Key2Number[key])
+	-- generate new number
-end
+	if new then
-
+		local num = backend.add_nodepos(pos)
-local function generate_Key2Number()
+		meta:set_string("node_number", num) 
-	local key
+		return num
-	for num,item in pairs(Number2Pos) do
-		key = minetest.hash_node_position(item.pos)
-		Key2Number[key] = num
 	end
 end
 
@@ -135,16 +120,6 @@ local function register_lbm(name, nodenames)
 	})
 end
 
-
-local DirToSide = {"B", "R", "F", "L", "D", "U"}
-
-local function dir_to_side(dir, param2)
-	if dir < 5 then
-		dir = (((dir - 1) - (param2 % 4)) % 4) + 1
-	end
-	return DirToSide[dir]
-end
-
 local SideToDir = {B=1, R=2, F=3, L=4, D=5, U=6}
 
 local function side_to_dir(side, param2)
@@ -192,26 +167,15 @@ end
 -- API helper functions
 -------------------------------------------------------------------
 	
--- Function returns { pos, name } for the node on the given position number.
+-- Function returns { pos, name } for the node referenced by number
 function techage.get_node_info(dest_num)
-	if Number2Pos[dest_num] then
+	return NodeInfoCache[dest_num] or update_nodeinfo(dest_num)
-		return Number2Pos[dest_num]
-	end
-	return nil
 end	
 
 -- Function returns the node number from the given position or
 -- nil, if no node number for this position is assigned.
 function techage.get_node_number(pos)
-	local key = minetest.hash_node_position(pos)
+	return get_number(pos)
-	local num = Key2Number[key]
-	if num then
-		num = string.format("%u", num)
-		if Number2Pos[num] and Number2Pos[num].name then
-			return num
-		end
-	end
-	return nil
 end	
 
 function techage.get_pos(pos, side)
@@ -226,12 +190,7 @@ end
 -- Function is used for available nodes with lost numbers, only.
 function techage.get_new_number(pos, name)
 	-- store position 
-	local number = get_number(pos)
+	return get_number(pos, true)
-	Number2Pos[number] = {
-		pos = pos, 
-		name = name,
-	}
-	return number
 end
 
 -- extract ident and value from strings like "ident=value"
@@ -252,24 +211,21 @@ function techage.add_node(pos, name)
 		Tube:after_place_node(pos)
 	end
 	-- store position 
-	local number = get_number(pos)
+	return get_number(pos, true)
-	Number2Pos[number] = {
-		pos = pos, 
-		name = name,
-	}
-	return number
 end
 
 -- Function removes the node from the techage lists.
 function techage.remove_node(pos)
 	local number = get_number(pos)
-	local name
+	if number then
-	if Number2Pos[number] then
+		local ninfo = NodeInfoCache[number] or update_nodeinfo(number)
-		name = Number2Pos[number].name
+		if ninfo then
-		Number2Pos[number].name = nil
+			backend.del_nodepos(number)
-	end
+			NodeInfoCache[number] = nil
-	if item_handling_node(name) then
+			if item_handling_node(ninfo.name) then
-		Tube:after_dig_node(pos)
+				Tube:after_dig_node(pos)
+			end
+		end
 	end
 end
 
@@ -314,11 +270,9 @@ end
 -------------------------------------------------------------------
 
 function techage.not_protected(number, placer_name, clicker_name)
-	if Number2Pos[number] and Number2Pos[number].name then
+	local ninfo = NodeInfoCache[number] or update_nodeinfo(number)
-		local data = Number2Pos[number]
+	if ninfo and ninfo.pos then
-		if data.pos	then
+		return not_protected(ninfo.pos, placer_name, clicker_name)
-			return not_protected(data.pos, placer_name, clicker_name)
-		end
 	end
 	return false
 end
@@ -341,10 +295,11 @@ end
 function techage.send_multi(src, numbers, topic, payload)
 	--print("send_multi", src, numbers, topic)
 	for _,num in ipairs(string_split(numbers, " ")) do
-		if Number2Pos[num] and Number2Pos[num].name then
+		local ninfo = NodeInfoCache[num] or update_nodeinfo(num)
-			local data = Number2Pos[num]
+		if ninfo and ninfo.name and ninfo.pos then
-			if data.pos and NodeDef[data.name] and NodeDef[data.name].on_recv_message then
+			local ndef = NodeDef[ninfo.name]
-				NodeDef[data.name].on_recv_message(data.pos, src, topic, payload)
+			if ndef and ndef.on_recv_message then
+				ndef.on_recv_message(ninfo.pos, src, topic, payload)
 			end
 		end
 	end
@@ -352,10 +307,11 @@ end
 
 function techage.send_single(src, number, topic, payload)
 	--print("send_single", src, number, topic)
-	if Number2Pos[number] and Number2Pos[number].name then
+	local ninfo = NodeInfoCache[number] or update_nodeinfo(number)
-		local data = Number2Pos[number]
+	if ninfo and ninfo.name and ninfo.pos then
-		if data.pos and NodeDef[data.name] and NodeDef[data.name].on_recv_message then
+		local ndef = NodeDef[ninfo.name]
-			return NodeDef[data.name].on_recv_message(data.pos, src, topic, payload)
+		if ndef and ndef.on_recv_message then
+			return ndef.on_recv_message(ninfo.pos, src, topic, payload)
 		end
 	end
 	return false
@@ -502,7 +458,6 @@ function techage.put_items(inv, listname, item, idx)
 	return false
 end
 
-
 -- Return "full", "loaded", or "empty" depending
 -- on the inventory load.
 -- Full is returned, when no empty stack is available.
@@ -521,29 +476,3 @@ function techage.get_inv_state(inv, listname)
     end
     return state
 end
-
-
--------------------------------------------------------------------------------
--- Data Maintenance
--------------------------------------------------------------------------------
-local function data_maintenance()
-	minetest.log("info", "[TechAge] Data maintenance started")
-	-- Remove unused positions
-	local tbl = table.copy(Number2Pos)
-	Number2Pos = {}
-	for num,item in pairs(tbl) do
-		local name = techage.get_node_lvm(item.pos).name
-		if NodeDef[name] then
-			Number2Pos[num] = item
-			-- add node names which are not stored as file
-			Number2Pos[num].name = name
-		end
-	end
-	minetest.log("info", "[TechAge] Data maintenance finished")
-end	
-	
-generate_Key2Number()
-
-minetest.after(2, data_maintenance)
-
-

manuals/manual_DE.md

@@ -14,7 +14,9 @@ Suche und fördere Öl, baute Schienenwege zur Ölbeförderung. Ein Kraftwerk li
 TA4: Zukunft (Future Age)
 Regenerative Energiequellen wie Wind, Sonne und Biokraft helfen dir, das Ölzeitalter zu verlassen. Mit modernen Technologien und intelligenten Maschinen machst du dich auf in die Zukunft.
 
-[wlanchip|image]
+Hinweis: Mit Klicken auf die Pluszeichen kommst du in die Unterkapitel dieser Anleitung.
+
+[techage_ta4|image]
 
 
 

manuals/manual_EN.md

@@ -14,7 +14,9 @@ Find and extract oil, built railways for oil transportation. A power plant provi
 TA4: Future Age
 Renewable energy sources such as wind, sun and biofuels help you to leave the oil age. With modern technologies and intelligent machines you set out into the future.
 
-[wlanchip|image]
+Note: With a click on the plus sign you get into the subchapters of this manual.
+
+[techage_ta4|image]