ensure that all inserted user-values are valid utf8 strings

fixes #113

go.sum

@@ -175,6 +175,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=

mapobjectdb/postgres/mapobjects.go

@@ -4,6 +4,8 @@ import (
 	"database/sql"
 	"mapserver/coords"
 	"mapserver/mapobjectdb"
+	"github.com/sirupsen/logrus"
+	"unicode/utf8"
 )
 
 func (db *PostgresAccessor) GetMapData(q *mapobjectdb.SearchQuery) ([]*mapobjectdb.MapObject, error) {
@@ -94,6 +96,20 @@ func (db *PostgresAccessor) RemoveMapData(pos *coords.MapBlockCoords) error {
 }
 
 func (db *PostgresAccessor) AddMapData(data *mapobjectdb.MapObject) error {
+
+	for k, v := range data.Attributes {
+		if !utf8.Valid([]byte(v)) {
+			// invalid utf8, skip insert into db
+			fields := logrus.Fields{
+				"type": data.Type,
+				"value": v,
+				"key": k,
+			}
+			log.WithFields(fields).Info("Migration completed")
+			return nil
+		}
+	}
+
 	res := db.db.QueryRow(addMapDataQuery,
 		data.X, data.Y, data.Z,
 		data.MBPos.X, data.MBPos.Y, data.MBPos.Z,

mapobjectdb/sqlite/mapobjects.go

@@ -4,6 +4,8 @@ import (
 	"database/sql"
 	"mapserver/coords"
 	"mapserver/mapobjectdb"
+	"github.com/sirupsen/logrus"
+	"unicode/utf8"
 )
 
 func (db *Sqlite3Accessor) GetMapData(q *mapobjectdb.SearchQuery) ([]*mapobjectdb.MapObject, error) {
@@ -94,6 +96,20 @@ func (db *Sqlite3Accessor) RemoveMapData(pos *coords.MapBlockCoords) error {
 }
 
 func (db *Sqlite3Accessor) AddMapData(data *mapobjectdb.MapObject) error {
+
+	for k, v := range data.Attributes {
+		if !utf8.Valid([]byte(v)) {
+			// invalid utf8, skip insert into db
+			fields := logrus.Fields{
+				"type": data.Type,
+				"value": v,
+				"key": k,
+			}
+			log.WithFields(fields).Info("Migration completed")
+			return nil
+		}
+	}
+
 	res, err := db.db.Exec(addMapDataQuery,
 		data.X, data.Y, data.Z,
 		data.MBPos.X, data.MBPos.Y, data.MBPos.Z,

mapobjectdb/utf8_test.go

@@ -0,0 +1,18 @@
+package mapobjectdb
+
+import (
+	"testing"
+	"unicode/utf8"
+)
+
+func TestInvalidUtf8(t *testing.T) {
+	if utf8.Valid([]byte{0xe1, 0x7f, 0xc7}) {
+		t.Error("should be invalid")
+	}
+}
+
+func TestValidUtf8(t *testing.T) {
+	if !utf8.Valid([]byte("some valid string")) {
+		t.Error("should be valid")
+	}
+}