Merge branch 'sqlite_cleanup'

Converts sqlite to async, providing a way forward
for migrations to actually happen
This commit is contained in:
Reto Brunner 2022-11-24 09:45:01 +01:00
commit deeea274da
7 changed files with 258 additions and 270 deletions

View file

@ -147,7 +147,7 @@ class Client {
} }
for (const messageStorage of client.messageStorage) { for (const messageStorage of client.messageStorage) {
messageStorage.enable(); messageStorage.enable().catch((e) => log.error(e));
} }
} }
@ -614,12 +614,12 @@ class Client {
} }
for (const messageStorage of this.messageStorage) { for (const messageStorage of this.messageStorage) {
messageStorage.deleteChannel(target.network, target.chan); messageStorage.deleteChannel(target.network, target.chan).catch((e) => log.error(e));
} }
} }
search(query: SearchQuery) { search(query: SearchQuery) {
if (this.messageProvider === undefined) { if (!this.messageProvider?.isEnabled) {
return Promise.resolve({ return Promise.resolve({
results: [], results: [],
target: "", target: "",
@ -767,7 +767,7 @@ class Client {
}); });
for (const messageStorage of this.messageStorage) { for (const messageStorage of this.messageStorage) {
messageStorage.close(); messageStorage.close().catch((e) => log.error(e));
} }
} }

View file

@ -23,6 +23,7 @@ const Helper = {
parseHostmask, parseHostmask,
compareHostmask, compareHostmask,
compareWithWildcard, compareWithWildcard,
catch_to_error,
password: { password: {
hash: passwordHash, hash: passwordHash,
@ -183,3 +184,17 @@ function compareWithWildcard(a: string, b: string) {
const re = new RegExp(`^${user_regex}$`, "i"); // case insensitive const re = new RegExp(`^${user_regex}$`, "i"); // case insensitive
return re.test(b); return re.test(b);
} }
function catch_to_error(prefix: string, err: any): Error {
let msg: string;
if (err instanceof Error) {
msg = err.message;
} else if (typeof err === "string") {
msg = err;
} else {
msg = err.toString();
}
return new Error(`${prefix}: ${msg}`);
}

View file

@ -260,7 +260,7 @@ class Chan {
} }
for (const messageStorage of client.messageStorage) { for (const messageStorage of client.messageStorage) {
messageStorage.index(target.network, targetChannel, msg); messageStorage.index(target.network, targetChannel, msg).catch((e) => log.error(e));
} }
} }
loadMessages(client: Client, network: Network) { loadMessages(client: Client, network: Network) {

View file

@ -2,11 +2,12 @@ import type {Database} from "sqlite3";
import log from "../../log"; import log from "../../log";
import path from "path"; import path from "path";
import fs from "fs"; import fs from "fs/promises";
import Config from "../../config"; import Config from "../../config";
import Msg, {Message} from "../../models/msg"; import Msg, {Message} from "../../models/msg";
import Client from "../../client"; import Client from "../../client";
import Chan, {Channel} from "../../models/chan"; import Chan, {Channel} from "../../models/chan";
import Helper from "../../helper";
import type { import type {
SearchResponse, SearchResponse,
SearchQuery, SearchQuery,
@ -47,89 +48,85 @@ class SqliteMessageStorage implements ISqliteMessageStorage {
this.isEnabled = false; this.isEnabled = false;
} }
enable() { async enable() {
const logsPath = Config.getUserLogsPath(); const logsPath = Config.getUserLogsPath();
const sqlitePath = path.join(logsPath, `${this.client.name}.sqlite3`); const sqlitePath = path.join(logsPath, `${this.client.name}.sqlite3`);
try { try {
fs.mkdirSync(logsPath, {recursive: true}); await fs.mkdir(logsPath, {recursive: true});
} catch (e: any) { } catch (e) {
log.error("Unable to create logs directory", String(e)); throw Helper.catch_to_error("Unable to create logs directory", e);
return;
} }
this.isEnabled = true; this.isEnabled = true;
this.database = new sqlite3.Database(sqlitePath); this.database = new sqlite3.Database(sqlitePath);
this.database.serialize(() => {
schema.forEach((line) => this.database.run(line));
this.database.get( try {
"SELECT value FROM options WHERE name = 'schema_version'", await this.run_migrations();
(err, row) => { } catch (e) {
if (err) { this.isEnabled = false;
return log.error(`Failed to retrieve schema version: ${err.toString()}`); throw Helper.catch_to_error("Migration failed", e);
} }
// New table
if (row === undefined) {
this.database.serialize(() =>
this.database.run(
"INSERT INTO options (name, value) VALUES ('schema_version', ?)",
currentSchemaVersion
)
);
return;
}
const storedSchemaVersion = parseInt(row.value, 10);
if (storedSchemaVersion === currentSchemaVersion) {
return;
}
if (storedSchemaVersion > currentSchemaVersion) {
return log.error(
`sqlite messages schema version is higher than expected (${storedSchemaVersion} > ${currentSchemaVersion}). Is The Lounge out of date?`
);
}
log.info(
`sqlite messages schema version is out of date (${storedSchemaVersion} < ${currentSchemaVersion}). Running migrations if any.`
);
this.database.serialize(() =>
this.database.run(
"UPDATE options SET value = ? WHERE name = 'schema_version'",
currentSchemaVersion
)
);
}
);
});
} }
close(callback?: (error?: Error | null) => void) { async run_migrations() {
for (const stmt of schema) {
await this.serialize_run(stmt, []);
}
const version = await this.serialize_get(
"SELECT value FROM options WHERE name = 'schema_version'"
);
if (version === undefined) {
// new table
await this.serialize_run(
"INSERT INTO options (name, value) VALUES ('schema_version', ?)",
[currentSchemaVersion]
);
return;
}
const storedSchemaVersion = parseInt(version.value, 10);
if (storedSchemaVersion === currentSchemaVersion) {
return;
}
if (storedSchemaVersion > currentSchemaVersion) {
throw `sqlite messages schema version is higher than expected (${storedSchemaVersion} > ${currentSchemaVersion}). Is The Lounge out of date?`;
}
log.info(
`sqlite messages schema version is out of date (${storedSchemaVersion} < ${currentSchemaVersion}). Running migrations if any.`
);
await this.serialize_run("UPDATE options SET value = ? WHERE name = 'schema_version'", [
currentSchemaVersion,
]);
}
async close() {
if (!this.isEnabled) { if (!this.isEnabled) {
return; return;
} }
this.isEnabled = false; this.isEnabled = false;
this.database.close((err) => { return new Promise<void>((resolve, reject) => {
if (err) { this.database.close((err) => {
log.error(`Failed to close sqlite database: ${err.message}`); if (err) {
} reject(`Failed to close sqlite database: ${err.message}`);
return;
}
if (callback) { resolve();
callback(err); });
}
}); });
} }
index(network: Network, channel: Chan, msg: Msg) { async index(network: Network, channel: Chan, msg: Msg) {
if (!this.isEnabled) { if (!this.isEnabled) {
return; return;
} }
@ -145,78 +142,68 @@ class SqliteMessageStorage implements ISqliteMessageStorage {
return newMsg; return newMsg;
}, {}); }, {});
this.database.serialize(() => await this.serialize_run(
this.database.run( "INSERT INTO messages(network, channel, time, type, msg) VALUES(?, ?, ?, ?, ?)",
"INSERT INTO messages(network, channel, time, type, msg) VALUES(?, ?, ?, ?, ?)", [
network.uuid, network.uuid,
channel.name.toLowerCase(), channel.name.toLowerCase(),
msg.time.getTime(), msg.time.getTime(),
msg.type, msg.type,
JSON.stringify(clonedMsg) JSON.stringify(clonedMsg),
) ]
); );
} }
deleteChannel(network: Network, channel: Channel) { async deleteChannel(network: Network, channel: Channel) {
if (!this.isEnabled) { if (!this.isEnabled) {
return; return;
} }
this.database.serialize(() => await this.serialize_run("DELETE FROM messages WHERE network = ? AND channel = ?", [
this.database.run( network.uuid,
"DELETE FROM messages WHERE network = ? AND channel = ?", channel.name.toLowerCase(),
network.uuid, ]);
channel.name.toLowerCase()
)
);
} }
/** /**
* Load messages for given channel on a given network and resolve a promise with loaded messages. * Load messages for given channel on a given network and resolve a promise with loaded messages.
* *
* @param Network network - Network object where the channel is * @param network Network - Network object where the channel is
* @param Chan channel - Channel object for which to load messages for * @param channel Channel - Channel object for which to load messages for
*/ */
getMessages(network: Network, channel: Channel) { async getMessages(network: Network, channel: Channel): Promise<Message[]> {
if (!this.isEnabled || Config.values.maxHistory === 0) { if (!this.isEnabled || Config.values.maxHistory === 0) {
return Promise.resolve([]); return [];
} }
// If unlimited history is specified, load 100k messages // If unlimited history is specified, load 100k messages
const limit = Config.values.maxHistory < 0 ? 100000 : Config.values.maxHistory; const limit = Config.values.maxHistory < 0 ? 100000 : Config.values.maxHistory;
return new Promise((resolve, reject) => { const rows = await this.serialize_fetchall(
this.database.serialize(() => "SELECT msg, type, time FROM messages WHERE network = ? AND channel = ? ORDER BY time DESC LIMIT ?",
this.database.all( network.uuid,
"SELECT msg, type, time FROM messages WHERE network = ? AND channel = ? ORDER BY time DESC LIMIT ?", channel.name.toLowerCase(),
[network.uuid, channel.name.toLowerCase(), limit], limit
(err, rows) => { );
if (err) {
return reject(err);
}
resolve( return rows.reverse().map((row: any): Message => {
rows.reverse().map((row) => { const msg = JSON.parse(row.msg);
const msg = JSON.parse(row.msg); msg.time = row.time;
msg.time = row.time; msg.type = row.type;
msg.type = row.type;
const newMsg = new Msg(msg); const newMsg = new Msg(msg);
newMsg.id = this.client.idMsg++; newMsg.id = this.client.idMsg++;
return newMsg; return newMsg;
}) });
);
}
)
);
}) as Promise<Message[]>;
} }
search(query: SearchQuery): Promise<SearchResponse | []> { async search(query: SearchQuery): Promise<SearchResponse> {
if (!this.isEnabled) { if (!this.isEnabled) {
// this should never be hit as messageProvider is checked in client.search() // this should never be hit as messageProvider is checked in client.search()
return Promise.resolve([]); throw new Error(
"search called but sqlite provider not enabled. This is a programming error"
);
} }
// Using the '@' character to escape '%' and '_' in patterns. // Using the '@' character to escape '%' and '_' in patterns.
@ -242,30 +229,67 @@ class SqliteMessageStorage implements ISqliteMessageStorage {
params.push(maxResults); params.push(maxResults);
params.push(query.offset); params.push(query.offset);
return new Promise((resolve, reject) => { const rows = await this.serialize_fetchall(select, ...params);
this.database.all(select, params, (err, rows) => { const response: SearchResponse = {
if (err) { searchTerm: query.searchTerm,
reject(err); target: query.channelName,
} else { networkUuid: query.networkUuid,
const response: SearchResponse = { offset: query.offset,
searchTerm: query.searchTerm, results: parseSearchRowsToMessages(query.offset, rows).reverse(),
target: query.channelName, };
networkUuid: query.networkUuid,
offset: query.offset, return response;
results: parseSearchRowsToMessages(query.offset, rows).reverse(),
};
resolve(response);
}
});
});
} }
canProvideMessages() { canProvideMessages() {
return this.isEnabled; return this.isEnabled;
} }
}
export default SqliteMessageStorage; private serialize_run(stmt: string, params: any[]): Promise<void> {
return new Promise((resolve, reject) => {
this.database.serialize(() => {
this.database.run(stmt, params, (err) => {
if (err) {
reject(err);
return;
}
resolve();
});
});
});
}
private serialize_fetchall(stmt: string, ...params: any[]): Promise<any[]> {
return new Promise((resolve, reject) => {
this.database.serialize(() => {
this.database.all(stmt, params, (err, rows) => {
if (err) {
reject(err);
return;
}
resolve(rows);
});
});
});
}
private serialize_get(stmt: string, ...params: any[]): Promise<any> {
return new Promise((resolve, reject) => {
this.database.serialize(() => {
this.database.get(stmt, params, (err, row) => {
if (err) {
reject(err);
return;
}
resolve(row);
});
});
});
}
}
// TODO: type any // TODO: type any
function parseSearchRowsToMessages(id: number, rows: any[]) { function parseSearchRowsToMessages(id: number, rows: any[]) {
@ -284,3 +308,5 @@ function parseSearchRowsToMessages(id: number, rows: any[]) {
return messages; return messages;
} }
export default SqliteMessageStorage;

View file

@ -1,9 +1,8 @@
/* eslint-disable @typescript-eslint/restrict-template-expressions */ /* eslint-disable @typescript-eslint/restrict-template-expressions */
import fs from "fs"; import fs from "fs/promises";
import path from "path"; import path from "path";
import filenamify from "filenamify"; import filenamify from "filenamify";
import log from "../../log";
import Config from "../../config"; import Config from "../../config";
import {MessageStorage} from "./types"; import {MessageStorage} from "./types";
import Client from "../../client"; import Client from "../../client";
@ -20,19 +19,17 @@ class TextFileMessageStorage implements MessageStorage {
this.isEnabled = false; this.isEnabled = false;
} }
enable() { // eslint-disable-next-line @typescript-eslint/require-await
async enable() {
this.isEnabled = true; this.isEnabled = true;
} }
close(callback: () => void) { // eslint-disable-next-line @typescript-eslint/require-await
async close() {
this.isEnabled = false; this.isEnabled = false;
if (callback) {
callback();
}
} }
index(network: Network, channel: Channel, msg: Message) { async index(network: Network, channel: Channel, msg: Message) {
if (!this.isEnabled) { if (!this.isEnabled) {
return; return;
} }
@ -44,10 +41,9 @@ class TextFileMessageStorage implements MessageStorage {
); );
try { try {
fs.mkdirSync(logPath, {recursive: true}); await fs.mkdir(logPath, {recursive: true});
} catch (e: any) { } catch (e) {
log.error("Unable to create logs directory", String(e)); throw new Error(`Unable to create logs directory: ${e}`);
return;
} }
let line = `[${msg.time.toISOString()}] `; let line = `[${msg.time.toISOString()}] `;
@ -106,35 +102,18 @@ class TextFileMessageStorage implements MessageStorage {
line += "\n"; line += "\n";
fs.appendFile( try {
path.join(logPath, TextFileMessageStorage.getChannelFileName(channel)), await fs.appendFile(
line, path.join(logPath, TextFileMessageStorage.getChannelFileName(channel)),
(e) => { line
if (e) { );
log.error("Failed to write user log", e.message); } catch (e) {
} throw new Error(`Failed to write user log: ${e}`);
} }
);
} }
deleteChannel() { async deleteChannel() {
/* TODO: Truncating text logs is disabled, until we figure out some UI for it // Not implemented for text log files
if (!this.isEnabled) {
return;
}
const logPath = path.join(
Config.getUserLogsPath(),
this.client.name,
TextFileMessageStorage.getNetworkFolderName(network),
TextFileMessageStorage.getChannelFileName(channel)
);
fs.truncate(logPath, 0, (e) => {
if (e) {
log.error("Failed to truncate user log", e);
}
});*/
} }
getMessages() { getMessages() {

View file

@ -9,13 +9,13 @@ interface MessageStorage {
client: Client; client: Client;
isEnabled: boolean; isEnabled: boolean;
enable(): void; enable(): Promise<void>;
close(callback?: () => void): void; close(): Promise<void>;
index(network: Network, channel: Channel, msg: Message): void; index(network: Network, channel: Channel, msg: Message): Promise<void>;
deleteChannel(network: Network, channel: Channel); deleteChannel(network: Network, channel: Channel): Promise<void>;
getMessages(network: Network, channel: Channel): Promise<Message[]>; getMessages(network: Network, channel: Channel): Promise<Message[]>;
@ -30,12 +30,11 @@ export type SearchQuery = {
}; };
export type SearchResponse = export type SearchResponse =
| (Omit<SearchQuery, "channelName" | "offset"> & { | Omit<SearchQuery, "channelName" | "offset"> & {
results: Message[]; results: Message[];
target: string; target: string;
offset: number; offset: number;
}) };
| [];
type SearchFunction = (query: SearchQuery) => Promise<SearchResponse>; type SearchFunction = (query: SearchQuery) => Promise<SearchResponse>;

View file

@ -37,18 +37,16 @@ describe("SQLite Message Storage", function () {
fs.rmdir(path.join(Config.getHomePath(), "logs"), done); fs.rmdir(path.join(Config.getHomePath(), "logs"), done);
}); });
it("should resolve an empty array when disabled", function () { it("should resolve an empty array when disabled", async function () {
return store.getMessages(null as any, null as any).then((messages) => { const messages = await store.getMessages(null as any, null as any);
expect(messages).to.be.empty; expect(messages).to.be.empty;
});
}); });
it("should create database file", function () { it("should create database file", async function () {
expect(store.isEnabled).to.be.false; expect(store.isEnabled).to.be.false;
expect(fs.existsSync(expectedPath)).to.be.false; expect(fs.existsSync(expectedPath)).to.be.false;
store.enable(); await store.enable();
expect(store.isEnabled).to.be.true; expect(store.isEnabled).to.be.true;
}); });
@ -90,8 +88,8 @@ describe("SQLite Message Storage", function () {
); );
}); });
it("should store a message", function () { it("should store a message", async function () {
store.index( await store.index(
{ {
uuid: "this-is-a-network-guid", uuid: "this-is-a-network-guid",
} as any, } as any,
@ -105,35 +103,30 @@ describe("SQLite Message Storage", function () {
); );
}); });
it("should retrieve previously stored message", function () { it("should retrieve previously stored message", async function () {
return store const messages = await store.getMessages(
.getMessages( {
{ uuid: "this-is-a-network-guid",
uuid: "this-is-a-network-guid", } as any,
} as any, {
{ name: "#thisisaCHANNEL",
name: "#thisisaCHANNEL", } as any
} as any );
) expect(messages).to.have.lengthOf(1);
.then((messages) => { const msg = messages[0];
expect(messages).to.have.lengthOf(1); expect(msg.text).to.equal("Hello from sqlite world!");
expect(msg.type).to.equal(MessageType.MESSAGE);
const msg = messages[0]; expect(msg.time.getTime()).to.equal(123456789);
expect(msg.text).to.equal("Hello from sqlite world!");
expect(msg.type).to.equal(MessageType.MESSAGE);
expect(msg.time.getTime()).to.equal(123456789);
});
}); });
it("should retrieve latest LIMIT messages in order", function () { it("should retrieve latest LIMIT messages in order", async function () {
const originalMaxHistory = Config.values.maxHistory; const originalMaxHistory = Config.values.maxHistory;
try { try {
Config.values.maxHistory = 2; Config.values.maxHistory = 2;
for (let i = 0; i < 200; ++i) { for (let i = 0; i < 200; ++i) {
store.index( await store.index(
{uuid: "retrieval-order-test-network"} as any, {uuid: "retrieval-order-test-network"} as any,
{name: "#channel"} as any, {name: "#channel"} as any,
new Msg({ new Msg({
@ -143,64 +136,51 @@ describe("SQLite Message Storage", function () {
); );
} }
return store const messages = await store.getMessages(
.getMessages( {uuid: "retrieval-order-test-network"} as any,
{uuid: "retrieval-order-test-network"} as any, {name: "#channel"} as any
{name: "#channel"} as any );
) expect(messages).to.have.lengthOf(2);
.then((messages) => { expect(messages.map((i_1) => i_1.text)).to.deep.equal(["msg 198", "msg 199"]);
expect(messages).to.have.lengthOf(2);
expect(messages.map((i) => i.text)).to.deep.equal(["msg 198", "msg 199"]);
});
} finally { } finally {
Config.values.maxHistory = originalMaxHistory; Config.values.maxHistory = originalMaxHistory;
} }
}); });
it("should search messages", function () { it("should search messages", async function () {
const originalMaxHistory = Config.values.maxHistory; const originalMaxHistory = Config.values.maxHistory;
try { try {
Config.values.maxHistory = 2; Config.values.maxHistory = 2;
return store const search = await store.search({
.search({ searchTerm: "msg",
searchTerm: "msg", networkUuid: "retrieval-order-test-network",
networkUuid: "retrieval-order-test-network", channelName: "",
channelName: "", offset: 0,
offset: 0, });
}) expect(search.results).to.have.lengthOf(100);
.then((messages) => { const expectedMessages: string[] = [];
// @ts-expect-error Property 'results' does not exist on type '[]'.
expect(messages.results).to.have.lengthOf(100);
const expectedMessages: string[] = []; for (let i = 100; i < 200; ++i) {
expectedMessages.push(`msg ${i}`);
}
for (let i = 100; i < 200; ++i) { expect(search.results.map((i_1) => i_1.text)).to.deep.equal(expectedMessages);
expectedMessages.push(`msg ${i}`);
}
// @ts-expect-error Property 'results' does not exist on type '[]'.
expect(messages.results.map((i) => i.text)).to.deep.equal(expectedMessages);
});
} finally { } finally {
Config.values.maxHistory = originalMaxHistory; Config.values.maxHistory = originalMaxHistory;
} }
}); });
it("should search messages with escaped wildcards", function () { it("should search messages with escaped wildcards", async function () {
function assertResults(query, expected) { async function assertResults(query: string, expected: string[]) {
return store const search = await store.search({
.search({ searchTerm: query,
searchTerm: query, networkUuid: "this-is-a-network-guid2",
networkUuid: "this-is-a-network-guid2", channelName: "",
channelName: "", offset: 0,
offset: 0, });
}) expect(search.results.map((i) => i.text)).to.deep.equal(expected);
.then((messages) => {
// @ts-expect-error Property 'results' does not exist on type '[]'.
expect(messages.results.map((i) => i.text)).to.deep.equal(expected);
});
} }
const originalMaxHistory = Config.values.maxHistory; const originalMaxHistory = Config.values.maxHistory;
@ -208,7 +188,7 @@ describe("SQLite Message Storage", function () {
try { try {
Config.values.maxHistory = 3; Config.values.maxHistory = 3;
store.index( await store.index(
{uuid: "this-is-a-network-guid2"} as any, {uuid: "this-is-a-network-guid2"} as any,
{name: "#channel"} as any, {name: "#channel"} as any,
new Msg({ new Msg({
@ -217,7 +197,7 @@ describe("SQLite Message Storage", function () {
} as any) } as any)
); );
store.index( await store.index(
{uuid: "this-is-a-network-guid2"} as any, {uuid: "this-is-a-network-guid2"} as any,
{name: "#channel"} as any, {name: "#channel"} as any,
new Msg({ new Msg({
@ -226,7 +206,7 @@ describe("SQLite Message Storage", function () {
} as any) } as any)
); );
store.index( await store.index(
{uuid: "this-is-a-network-guid2"} as any, {uuid: "this-is-a-network-guid2"} as any,
{name: "#channel"} as any, {name: "#channel"} as any,
new Msg({ new Msg({
@ -235,32 +215,21 @@ describe("SQLite Message Storage", function () {
} as any) } as any)
); );
return ( await assertResults("foo", ["foo % bar _ baz", "foo bar x baz"]);
store await assertResults("%", ["foo % bar _ baz"]);
.getMessages( await assertResults("foo % bar ", ["foo % bar _ baz"]);
{uuid: "this-is-a-network-guid2"} as any, await assertResults("_", ["foo % bar _ baz"]);
{name: "#channel"} as any await assertResults("bar _ baz", ["foo % bar _ baz"]);
) await assertResults("%%", []);
// .getMessages() waits for store.index() transactions to commit await assertResults("@%", []);
.then(() => assertResults("foo", ["foo % bar _ baz", "foo bar x baz"])) await assertResults("@", ["bar @ baz"]);
.then(() => assertResults("%", ["foo % bar _ baz"]))
.then(() => assertResults("foo % bar ", ["foo % bar _ baz"]))
.then(() => assertResults("_", ["foo % bar _ baz"]))
.then(() => assertResults("bar _ baz", ["foo % bar _ baz"]))
.then(() => assertResults("%%", []))
.then(() => assertResults("@%", []))
.then(() => assertResults("@", ["bar @ baz"]))
);
} finally { } finally {
Config.values.maxHistory = originalMaxHistory; Config.values.maxHistory = originalMaxHistory;
} }
}); });
it("should close database", function (done) { it("should close database", async function () {
store.close((err) => { await store.close();
expect(err).to.be.null; expect(fs.existsSync(expectedPath)).to.be.true;
expect(fs.existsSync(expectedPath)).to.be.true;
done();
});
}); });
}); });