MTSR/techage
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sqlite3 support added, furnace bugfix

Browse Source
This commit is contained in:
Joachim Stolberg 2020-05-23 14:11:35 +02:00
parent 831aaaf291
commit 376e29b854
15 changed files with 556 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
README.md
View File

@ -59,10 +59,11 @@ It is highly recommended that you install the following mods, too:
### Configuration
For servers with many players, it is recommended to use the external Lua library 'lua-marshal' to accelarate
the serialization/deserialization of node metadata.
To be able to use 'lua-marshal', install 'lua-marshal' with:
the serialization/deserialization of node data and the database SQLite (lsqlite3) to store large amounts of data.
To be able to use 'lua-marshal' and 'lsqlite3', install the packages with:
sudo luarocks install lua-marshal
luarocks install lua-marshal
luarocks install lsqlite3
and add 'techage' to the list of trusted mods in minetest.conf:
@ -70,6 +71,11 @@ and add 'techage' to the list of trusted mods in minetest.conf:
For the installation of 'luarocks' (if not already available), see: https://luarocks.org/
If you enable 'lsqlite3' you also have to enable 'lua-marshal'. Available worlds will be converted
to 'lsqlite3' and 'lua-marshal', but there is no way back, so:
** Never disable 'lsqlite3' and 'lua-marshal' for a world, which it was already used!**
### History
- 2019-06-16 V0.01 * First upload
@ -78,5 +84,5 @@ For the installation of 'luarocks' (if not already available), see: https://luar
- 2020-03-14 V0.05 * TA4 Lua controller added
- 2020-04-24 V0.06 * TA4 injector added
- 2020-04-26 V0.07 * English translation added
- 2020-05-20 V0.08 * Support for 'lua-marshal' added
- 2020-05-22 V0.08 * Support for 'lua-marshal' and 'lsqlite3' added

4
basic_machines/legacy_nodes.lua
View File

@ -82,12 +82,12 @@ techage.register_node({"default:furnace", "default:furnace_active"}, {
if minetest.get_craft_result({method="fuel", width=1, items={stack}}).time ~= 0 then
return techage.put_items(inv, "fuel", stack)
else
return techage.put_items(meta, "src", stack)
return techage.put_items(inv, "src", stack)
end
end,
on_unpull_item = function(pos, side, stack)
local meta = minetest.get_meta(pos)
local inv = meta:get_inventory()
return techage.put_items(meta, "dst", stack)
return techage.put_items(inv, "dst", stack)
end,
})

107
basis/backend_sqlite.lua
View File

@ -1,107 +0,0 @@
--[[
TechAge
=======
Copyright (C) 2020 Joachim Stolberg
GPL v3
See LICENSE.txt for more information
Node number management/storage based on SQLite
]]--
local MN = minetest.get_current_modname()
local WP = minetest.get_worldpath()
local IE = minetest.request_insecure_environment()
if not IE then
error("Please add 'secure.trusted_mods = techage' to minetest.conf!")
end
local sqlite3 = IE.require("lsqlite3")
local marshal = IE.require("marshal")
if not sqlite3 then
error("Please install sqlite3 via 'luarocks install lsqlite3'")
end
if not marshal then
error("Please install marshal via 'luarocks install lua-marshal'")
end
local db = sqlite3.open(WP.."/techage_numbers.sqlite3")
-- Prevent use of this db instance.
if sqlite3 then sqlite3 = nil end
db:exec[[
CREATE TABLE test2(id INTEGER PRIMARY KEY, number INTEGER, x INTEGER, y INTEGER, z INTEGER);
CREATE INDEX idx ON test2(number);
]]
local insert = db:prepare("INSERT INTO test2 VALUES(NULL, ?, ?, ?, ?);")
local query = db:prepare("SELECT * FROM test2 WHERE number=?")
local backend = {}
local storage = minetest.get_mod_storage()
local function set(number, pos)
insert:reset()
insert:bind(1, number)
insert:bind(2, pos.x)
insert:bind(3, pos.y)
insert:bind(4, pos.z)
insert:step()
end
local function get(number)
query:reset()
query:bind(1, number)
query:step()
local _, _, x, y, z, name = unpack(query:get_values())
return {pos = {x = x, y = y, z = z}, name = name}
end
-------------------------------------------------------------------
-- API functions
-------------------------------------------------------------------
function backend.get_nodepos(number)
return minetest.string_to_pos(storage:get_string(number))
end
function backend.set_nodepos(number, pos)
storage:get_string(number, minetest.pos_to_string(pos))
end
function backend.add_nodepos(pos)
local num = tostring(NextNumber)
NextNumber = NextNumber + 1
storage:set_int("NextNumber", NextNumber)
storage:get_string(num, minetest.pos_to_string(pos))
return num
end
function backend.del_nodepos(number)
storage:get_string(number, "")
end
-- delete invalid entries
function backend.delete_invalid_entries(node_def)
minetest.log("info", "[TechAge] Data maintenance started")
for i = 1, NextNumber do
local number = tostring(i)
if storage:contains(number) then
local pos = backend.get_nodepos(number)
local name = techage.get_node_lvm(pos).name
if not node_def[name] then
backend.del_nodepos(number)
end
end
end
minetest.log("info", "[TechAge] Data maintenance finished")
end
return backend

12
basis/command.lua
View File

@ -19,7 +19,7 @@ local S = function(pos) if pos then return minetest.pos_to_string(pos) end end
local NodeInfoCache = {}
local MP = minetest.get_modpath("techage")
local use_database = minetest.settings:get_bool('techage_use_database', false)
local techage_use_sqlite = minetest.settings:get_bool('techage_use_sqlite', false)
-- Localize functions to avoid table lookups (better performance)
local string_split = string.split
@ -31,10 +31,10 @@ local check_cart_for_loading = minecart.check_cart_for_loading
-- Database
-------------------------------------------------------------------
local backend
if use_database then
backend = dofile(MP .. "/basis/backend_sqlite.lua")
if techage_use_sqlite then
backend = dofile(MP .. "/basis/numbers_sqlite.lua")
else
backend = dofile(MP .. "/basis/backend_storage.lua")
backend = dofile(MP .. "/basis/numbers_storage.lua")
end
local function update_nodeinfo(number)
@ -47,12 +47,12 @@ end
local function delete_nodeinfo_entry(number)
if number and NodeInfoCache[number] then
number, _ = next(NodeInfoCache, number)
number = next(NodeInfoCache, number)
if number then
NodeInfoCache[number] = nil
end
else
number, _ = next(NodeInfoCache, nil)
number = next(NodeInfoCache, nil)
end
return number
end

169
basis/node_store.lua Normal file
View File

@ -0,0 +1,169 @@
--[[
TechAge
=======
Copyright (C) 2019-2020 Joachim Stolberg
GPL v3
See LICENSE.txt for more information
Data storage system for node related volatile and non-volatile data.
Non-volatile data is stored from time to time and at shutdown.
Volatile data is lost at every shutdown.
]]--
local NvmStore = {} -- non-volatile data cache
local MemStore = {} -- volatile data cache
local N = function(pos) print(minetest.pos_to_string(pos), minetest.get_node(pos).name) end
-------------------------------------------------------------------
-- Backend
-------------------------------------------------------------------
local MP = minetest.get_modpath("techage")
local techage_use_sqlite = minetest.settings:get_bool('techage_use_sqlite', false)
local backend
if techage_use_sqlite then
backend = dofile(MP .. "/basis/nodedata_sqlite.lua")
else
backend = dofile(MP .. "/basis/nodedata_meta.lua")
end
-- return keys for mapblock and inner-mapblock addressing based on the node position
local function get_keys(pos)
local kx1, kx2 = math.floor(pos.x / 16) + 2048, pos.x % 16
local ky1, ky2 = math.floor(pos.y / 16) + 2048, pos.y % 16
local kz1, kz2 = math.floor(pos.z / 16) + 2048, pos.z % 16
return kx1 * 4096 * 4096 + ky1 * 4096 + kz1, kx2 * 16 * 16 + ky2 * 16 + kz2
end
local function pos_from_key(key1, key2)
local x1 = (math.floor(key1 / (4096 * 4096)) - 2048) * 16
local y1 = ((math.floor(key1 / 4096) % 4096) - 2048) * 16
local z1 = ((key1 % 4096) - 2048) * 16
local x2 = math.floor(key2 / (16 * 16))
local y2 = math.floor(key2 / 16) % 16
local z2 = key2 % 16
return {x = x1 + x2, y = y1 + y2, z = z1 + z2}
end
local function debug(key1, item)
--local pos1 = pos_from_key(key1, 0)
--local pos2 = {x = pos1.x + 15, y = pos1.y + 15, z = pos1.z + 15}
--techage.mark_region("mapblock", pos1, pos2, "singleplayer", 5)
local cnt = 0
for key2, tbl in pairs(item) do
if key2 ~= "in_use" then
cnt = cnt + 1
--N(pos_from_key(key1, key2))
end
end
print("mapblock", string.format("%09X", key1), cnt.." nodes")
end
-------------------------------------------------------------------
-- Storage scheduler
-------------------------------------------------------------------
local CYCLE_TIME = 900 -- store data every 15 min
local JobQueue = {}
local first = 0
local last = -1
local SystemTime = 0
local function push(key)
last = last + 1
JobQueue[last] = {key = key, time = SystemTime + CYCLE_TIME}
end
local function pop()
if first > last then return end
local item = JobQueue[first]
if item.time <= SystemTime then
JobQueue[first] = nil -- to allow garbage collection
first = first + 1
return item.key
end
end
-- check every 100 msec if any data has to be stored
minetest.register_globalstep(function(dtime)
SystemTime = SystemTime + dtime
local key = pop()
if key and NvmStore[key] then
--debug(key, NvmStore[key])
local t = minetest.get_us_time()
if NvmStore[key].in_use then
NvmStore[key].in_use = nil
backend.store_mapblock_data(key, NvmStore[key])
push(key)
else
NvmStore[key] = nil -- remove unused data from cache
end
t = minetest.get_us_time() - t
if t > 10000 then
minetest.log("warning", "[TA Storage] duration = "..(t/1000.0).." ms")
end
end
end)
-------------------------------------------------------------------
-- Store/Restore NVM data
-------------------------------------------------------------------
NvmStore = backend.restore_at_startup()
minetest.register_on_shutdown(function()
backend.freeze_at_shutdown(NvmStore)
end)
-------------------------------------------------------------------
-- API functions
-------------------------------------------------------------------
-- Returns volatile node data as table
function techage.get_mem(pos)
local hash = minetest.hash_node_position(pos)
if not MemStore[hash] then
MemStore[hash] = {}
end
return MemStore[hash]
end
-- Returns non-volatile node data as table
function techage.get_nvm(pos)
local key1, key2 = get_keys(pos)
if not NvmStore[key1] then
NvmStore[key1] = backend.get_mapblock_data(key1)
push(key1)
end
local block = NvmStore[key1]
block.in_use = true
if not block[key2] then
block[key2] = backend.get_node_data(pos)
end
return block[key2]
end
function techage.peek_nvm(pos)
local key1, key2 = get_keys(pos)
local block = NvmStore[key1] or {}
return block[key2] or {}
end
-- To be called when a node is removed
function techage.del_mem(pos)
local hash = minetest.hash_node_position(pos)
MemStore[hash] = nil
local key1, key2 = get_keys(pos)
NvmStore[key1] = NvmStore[key1] or backend.get_mapblock_data(key1)
NvmStore[key1][key2] = nil
backend.store_mapblock_data(key1, NvmStore[key1])
end

102
basis/nodedata_meta.lua Normal file
View File

@ -0,0 +1,102 @@
--[[
TechAge
=======
Copyright (C) 2020 Joachim Stolberg
GPL v3
See LICENSE.txt for more information
Storage backend for node related data as node metadata
]]--
-- for lazy programmers
local M = minetest.get_meta
local storage = techage.storage
-------------------------------------------------------------------
-- Marshaling
-------------------------------------------------------------------
local use_marshal = minetest.settings:get_bool('techage_use_marshal', false)
local MAR_MAGIC = 0x8e
-- default functions
local serialize = minetest.serialize
local deserialize = minetest.deserialize
if use_marshal then
if not techage.IE then
error("Please add 'secure.trusted_mods = techage' to minetest.conf!")
end
local marshal = techage.IE.require("marshal")
if not marshal then
error("Please install marshal via 'luarocks install lua-marshal'")
end
serialize = marshal.encode
deserialize = function(s)
if s ~= "" then
if s:byte(1) == MAR_MAGIC then
return marshal.decode(s)
else
return minetest.deserialize(s)
end
end
end
end
-------------------------------------------------------------------
-- API functions
-------------------------------------------------------------------
local api = {}
function api.get_mapblock_data(key)
return {}
end
function api.store_mapblock_data(key, mapblock_data)
for key, item in pairs(mapblock_data) do
if key ~= "in_use" then
local pos = item and item._POS_
if pos then
item._POS_ = nil
local data = serialize(item)
local meta = M(pos)
meta:set_string("ta_data", data)
meta:mark_as_private("ta_data")
end
end
end
end
function api.get_node_data(pos)
local tbl = {}
local s = M(pos):get_string("ta_data")
if s ~= "" then
tbl = deserialize(s) or {}
end
tbl._POS_ = table.copy(pos)
return tbl
end
-- Meta data can't be written reliable at shutdown,
-- so we have to store/restore the data differently
function api.freeze_at_shutdown(data)
storage:set_string("shutdown_nodedata", serialize(data))
end
function api.restore_at_startup()
local s = storage:get_string("shutdown_nodedata")
if s ~= "" then
return deserialize(s) or {}
end
return {}
end
return api

113
basis/nodedata_sqlite.lua Normal file
View File

@ -0,0 +1,113 @@
--[[
TechAge
=======
Copyright (C) 2020 Joachim Stolberg
GPL v3
See LICENSE.txt for more information
Storage backend for node related data via sqlite database
]]--
-- for lazy programmers
local M = minetest.get_meta
-------------------------------------------------------------------
-- Database
-------------------------------------------------------------------
local MN = minetest.get_current_modname()
local WP = minetest.get_worldpath()
local MAR_MAGIC = 0x8e
if not techage.IE then
error("Please add 'secure.trusted_mods = techage' to minetest.conf!")
end
local sqlite3 = techage.IE.require("lsqlite3")
local marshal = techage.IE.require("marshal")
if not sqlite3 then
error("Please install sqlite3 via 'luarocks install lsqlite3'")
end
if not marshal then
error("Please install marshal via 'luarocks install lua-marshal'")
end
local db = sqlite3.open(WP.."/techage_nodedata.sqlite")
local ROW = sqlite3.ROW
-- Prevent use of this db instance.
if sqlite3 then sqlite3 = nil end
db:exec[[
CREATE TABLE mapblocks(id INTEGER PRIMARY KEY, key INTEGER, data BLOB);
CREATE UNIQUE INDEX idx ON mapblocks(key);
]]
local set = db:prepare("INSERT or REPLACE INTO mapblocks VALUES(NULL, ?, ?);")
local get = db:prepare("SELECT * FROM mapblocks WHERE key=?;")
local function set_block(key, data)
set:reset()
set:bind(1, key)
set:bind_blob(2, data)
set:step()
return true
end
local function get_block(key)
get:reset()
get:bind(1, key)
if get:step() == ROW then
return get:get_value(2)
end
end
-------------------------------------------------------------------
-- API functions
-------------------------------------------------------------------
local api = {}
function api.store_mapblock_data(key, mapblock_data)
local s = marshal.encode(mapblock_data)
return set_block(key, s)
end
function api.get_mapblock_data(key)
local s = get_block(key)
if s then
return marshal.decode(s)
end
api.store_mapblock_data(key, {})
return {}
end
function api.get_node_data(pos)
-- legacy data available?
local s = M(pos):get_string("ta_data")
if s ~= "" then
M(pos):set_string("ta_data", "")
if s:byte(1) == MAR_MAGIC then
return marshal.decode(s)
else
return minetest.deserialize(s)
end
end
return {}
end
function api.freeze_at_shutdown(data)
for key, item in pairs(data) do
api.store_mapblock_data(key, item)
end
end
function api.restore_at_startup()
-- nothing to restore
return {}
end
return api

139
basis/numbers_sqlite.lua Normal file
View File

@ -0,0 +1,139 @@
--[[
TechAge
=======
Copyright (C) 2020 Joachim Stolberg
GPL v3
See LICENSE.txt for more information
Storage backend for node number mapping via sqlite database
]]--
-- for lazy programmers
local M = minetest.get_meta
local storage = techage.storage
-------------------------------------------------------------------
-- Database
-------------------------------------------------------------------
local MN = minetest.get_current_modname()
local WP = minetest.get_worldpath()
local MAR_MAGIC = 0x8e
if not techage.IE then
error("Please add 'secure.trusted_mods = techage' to minetest.conf!")
end
local sqlite3 = techage.IE.require("lsqlite3")
local marshal = techage.IE.require("marshal")
if not sqlite3 then
error("Please install sqlite3 via 'luarocks install lsqlite3'")
end
if not marshal then
error("Please install marshal via 'luarocks install lua-marshal'")
end
local db = sqlite3.open(WP.."/techage_numbers.sqlite")
local ROW = sqlite3.ROW
-- Prevent use of this db instance.
if sqlite3 then sqlite3 = nil end
db:exec[[
CREATE TABLE numbers(id INTEGER PRIMARY KEY, number INTEGER, x INTEGER, y INTEGER, z INTEGER);
CREATE UNIQUE INDEX idx ON numbers(number);
]]
local set = db:prepare("INSERT or REPLACE INTO numbers VALUES(NULL, ?, ?, ?, ?);")
local get = db:prepare("SELECT * FROM numbers WHERE number=?;")
local function set_block(number, pos)
set:reset()
set:bind(1, number)
set:bind(2, pos.x)
set:bind(3, pos.y)
set:bind(4, pos.z)
set:step()
return true
end
local function get_block(number)
get:reset()
get:bind(1, number)
if get:step() == ROW then
return {x = get:get_value(2), y = get:get_value(3), z = get:get_value(4)}
end
end
local function del_block(number)
db:exec("DELETE FROM numbers WHERE number="..number..";")
end
-------------------------------------------------------------------
-- Migration from mod storage
-------------------------------------------------------------------
local Version = storage:get_int("Version") or 0
local NextNumber = 0
if Version == 3 then
Version = 4
NextNumber = storage:get_int("NextNumber")
for i = 1, NextNumber do
local number = tostring(i)
if storage:contains(number) then
local pos = minetest.string_to_pos(storage:get_string(number))
set_block(number, pos)
storage:set_string(number, "")
end
end
elseif Version == 4 then
NextNumber = storage:get_int("NextNumber")
else
error("[] Invalid version number for 'number to pos mapping' table!")
end
-------------------------------------------------------------------
-- API functions
-------------------------------------------------------------------
local api = {}
function api.get_nodepos(number)
return get_block(number)
end
function api.set_nodepos(number, pos)
set_block(number, pos)
end
function api.add_nodepos(pos)
local num = tostring(NextNumber)
NextNumber = NextNumber + 1
storage:set_int("NextNumber", NextNumber)
set_block(num, pos)
return num
end
function api.del_nodepos(number)
del_block(number)
end
-- delete invalid entries
function api.delete_invalid_entries(node_def)
minetest.log("info", "[TechAge] Data maintenance started")
for id, num, x, y, z in db:urows('SELECT * FROM numbers') do
local pos = {x = x, y = y, z = z}
local name = techage.get_node_lvm(pos).name
if not node_def[name] then
del_block(num)
end
end
minetest.log("info", "[TechAge] Data maintenance finished")
end
return api

4
basis/backend_storage.lua → basis/numbers_storage.lua
View File

@ -8,12 +8,12 @@
GPL v3
See LICENSE.txt for more information
Mod storage based solution
Storage backend for node number mapping via mod storage
]]--
local backend = {}
local storage = minetest.get_mod_storage()
local storage = techage.storage
-- legacy method
local function deserialize(s)

2
doc/manual_EN.lua
View File

@ -139,7 +139,7 @@ techage.manual_EN.aTitel = {
"3,TA4 Pipe",
"2,Hydrogen",
"3,Electrolyzer",
"3,Fuel VCell",
"3,Fuel Cell",
"2,Chemical Reactor",
"3,TA4 Doser",
"3,TA4 Reactor",

5
init.lua
View File

@ -52,10 +52,13 @@ techage.IE = minetest.request_insecure_environment()
-- Load support for I18n.
techage.S = minetest.get_translator("techage")
-- Load mod storage
techage.storage = minetest.get_mod_storage()
-- Basis features
local MP = minetest.get_modpath("techage")
dofile(MP.."/basis/lib.lua") -- helper functions
dofile(MP.."/basis/storage.lua")
dofile(MP.."/basis/node_store.lua")
dofile(MP.."/basis/gravel_lib.lua") -- ore probability
dofile(MP.."/basis/node_states.lua") -- state model
dofile(MP.."/basis/tubes.lua") -- tubes for item transport

2
manuals/manual_ta4_EN.md
View File

@ -211,7 +211,7 @@ The electrolyzer can draw up to 30 ku of electricity and then generates a hydrog
[ta4_electrolyzer|image]
### Fuel VCell
### Fuel Cell
The fuel cell converts hydrogen into electricity.
It must be supplied with hydrogen from the left by a pump. The power connection is on the right.

2
manuals/toc_EN.md
View File

@ -138,7 +138,7 @@
- [TA4 Pipe](./manual_ta4_EN.md#ta4-pipe)
- [Hydrogen](./manual_ta4_EN.md#hydrogen)
- [Electrolyzer](./manual_ta4_EN.md#electrolyzer)
- [Fuel VCell](./manual_ta4_EN.md#fuel-vcell)
- [Fuel Cell](./manual_ta4_EN.md#fuel-cell)
- [Chemical Reactor](./manual_ta4_EN.md#chemical-reactor)
- [TA4 Doser](./manual_ta4_EN.md#ta4-doser)
- [TA4 Reactor](./manual_ta4_EN.md#ta4-reactor)

10
settingtypes.txt
View File

@ -20,9 +20,9 @@ techage_oil_exploration_seed (techage oil exploration seed) int 1234
# Use the external Lua library 'lua-marshal' for faster serialization/deserialization
# of node meta data. For that, you have to add 'techage' to the list of
# trusted mods in minetest.conf:
#
# secure.trusted_mods = techage
#
# of node meta data. See also 'README.md'.
techage_use_marshal (use lua-marshal as serialize/deserialize functions) bool false
# Use the external library 'lsqlite3' for for faster storing of data.
# See also 'README.md'.
techage_use_sqlite (use sqlite database) bool false

2
ta3_power/tiny_generator.lua
View File

@ -132,7 +132,7 @@ local function node_timer(pos, elapsed)
if techage.is_activeformspec(pos) then
M(pos):set_string("formspec", formspec(State, pos, nvm))
end
return true
return State:is_active(nvm)
end
local function on_receive_fields(pos, formname, fields, player)