mapserver/server/db/sqlite.go

package db

import (
	"database/sql"
	"errors"
	_ "github.com/mattn/go-sqlite3"
	"github.com/sirupsen/logrus"
	"mapserver/coords"
	"strings"
	"time"
)
/*
sqlite extract: https://stackoverflow.com/questions/15448373/how-to-dump-a-file-stored-in-a-sqlite-database-as-a-blob
sqlite3 my.db "SELECT writefile('object0.gz', MyBlob) FROM MyTable WHERE id = 1"
*/

const migrateScript = `
alter table blocks add mtime integer default 0;
create index blocks_mtime on blocks(mtime);

CREATE TRIGGER update_blocks_mtime_insert after insert on blocks for each row
begin
update blocks set mtime = strftime('%s', 'now') where pos = new.pos;
end;

CREATE TRIGGER update_blocks_mtime_update after update on blocks for each row
begin
update blocks set mtime = strftime('%s', 'now') where pos = old.pos;
end;
`

//TODO: initial run: https://stackoverflow.com/questions/14468586/efficient-paging-in-sqlite-with-millions-of-records
//TODO: postgres test

type Sqlite3Accessor struct {
	db       *sql.DB
	filename string
}

func (db *Sqlite3Accessor) Migrate() error {

	//RW connection
	rwdb, err := sql.Open("sqlite3", db.filename+"?mode=rw")
	if err != nil {
		return err
	}

	defer rwdb.Close()

	hasMtime := true
	_, err = rwdb.Query("select max(mtime) from blocks")
	if err != nil {
		hasMtime = false
	}

	if !hasMtime {
		log.WithFields(logrus.Fields{"filename": db.filename}).Info("Migrating database")
		start := time.Now()
		_, err = rwdb.Exec(migrateScript)
		if err != nil {
			return err
		}
		t := time.Now()
		elapsed := t.Sub(start)
		log.WithFields(logrus.Fields{"elapsed": elapsed}).Info("Migration completed")
	}

	return nil
}

func convertRows(pos int64, data []byte, mtime int64) Block {
	c := coords.PlainToCoord(pos)
	return Block{Pos: c, Data: data, Mtime: mtime}
}

const getLegacyBlockQuery = `
select pos,data,mtime
from blocks b
where b.mtime == 0
and b.pos > ?
order by b.pos asc
limit ?
`

func (db *Sqlite3Accessor) FindLegacyBlocks(lastpos coords.MapBlockCoords, limit int) ([]Block, error) {
	blocks := make([]Block, 0)
	pc := coords.CoordToPlain(lastpos)

	rows, err := db.db.Query(getLegacyBlockQuery, pc, limit)
	if err != nil {
		return nil, err
	}

	defer rows.Close()

	for rows.Next() {
		var pos int64
		var data []byte
		var mtime int64

		err = rows.Scan(&pos, &data, &mtime)
		if err != nil {
			return nil, err
		}

		mb := convertRows(pos, data, mtime)
		blocks = append(blocks, mb)
	}

	return blocks, nil
}

const getLatestBlockQuery = `
select pos,data,mtime
from blocks b
where b.mtime > ?
order by b.mtime asc
limit ?
`

func (db *Sqlite3Accessor) FindLatestBlocks(mintime int64, limit int) ([]Block, error) {
	blocks := make([]Block, 0)

	rows, err := db.db.Query(getLatestBlockQuery, mintime, limit)
	if err != nil {
		return nil, err
	}

	defer rows.Close()

	for rows.Next() {
		var pos int64
		var data []byte
		var mtime int64

		err = rows.Scan(&pos, &data, &mtime)
		if err != nil {
			return nil, err
		}

		mb := convertRows(pos, data, mtime)
		blocks = append(blocks, mb)
	}

	return blocks, nil
}

const getBlockQuery = `
select pos,data,mtime from blocks b where b.pos = ?
`

func (db *Sqlite3Accessor) GetBlock(pos coords.MapBlockCoords) (*Block, error) {
	ppos := coords.CoordToPlain(pos)

	rows, err := db.db.Query(getBlockQuery, ppos)
	if err != nil {
		return nil, err
	}

	defer rows.Close()

	if rows.Next() {
		var pos int64
		var data []byte
		var mtime int64

		err = rows.Scan(&pos, &data, &mtime)
		if err != nil {
			return nil, err
		}

		mb := convertRows(pos, data, mtime)
		return &mb, nil
	}

	return nil, nil
}

func sortAsc(a, b int) (int, int) {
	if a > b {
		return b, a
	} else {
		return a, b
	}
}

func (db *Sqlite3Accessor) CountBlocks(pos1 coords.MapBlockCoords, pos2 coords.MapBlockCoords) (int, error) {

	poslist := make([]interface{}, 0)

	minX, maxX := sortAsc(pos1.X, pos2.X)
	minY, maxY := sortAsc(pos1.Y, pos2.Y)
	minZ, maxZ := sortAsc(pos1.Z, pos2.Z)

	for x := minX; x <= maxX; x++ {
		for y := minY; y <= maxY; y++ {
			for z := minZ; z <= maxZ; z++ {
				poslist = append(poslist, coords.CoordToPlain(coords.NewMapBlockCoords(x, y, z)))
			}
		}
	}

	if len(poslist) > 999 {
		//https://stackoverflow.com/questions/7106016/too-many-sql-variables-error-in-django-witih-sqlite3
		//TODO: return before nested for loops
		return -1, nil
	}

	getBlocksQuery := "select count(*) from blocks b where b.pos in (?" + strings.Repeat(",?", len(poslist)-1) + ")"

	rows, err := db.db.Query(getBlocksQuery, poslist...)
	if err != nil {
		return 0, err
	}

	defer rows.Close()

	if rows.Next() {
		var count int

		err = rows.Scan(&count)
		if err != nil {
			return 0, err
		}

		return count, nil
	}

	return 0, errors.New("No rows returned")
}

func NewSqliteAccessor(filename string) (*Sqlite3Accessor, error) {
	db, err := sql.Open("sqlite3", filename+"?mode=ro")
	if err != nil {
		return nil, err
	}

	sq := &Sqlite3Accessor{db: db, filename: filename}
	return sq, nil
}