From 5cfec76d3a6d19f5904f95bdf2c612482ce9e05c Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 17 Apr 2018 11:06:08 +0300 Subject: [PATCH] Refactor userLog to be the same as sqlite logger Fixes #1392 --- package.json | 1 + src/client.js | 23 +++-- src/helper.js | 6 +- src/models/chan.js | 34 +++----- src/models/network.js | 2 +- src/plugins/{ => messageStorage}/sqlite.js | 16 ++-- src/plugins/messageStorage/text.js | 99 ++++++++++++++++++++++ src/userLog.js | 59 ------------- test/plugins/sqlite.js | 11 ++- yarn.lock | 24 ++++++ 10 files changed, 175 insertions(+), 100 deletions(-) rename src/plugins/{ => messageStorage}/sqlite.js (91%) create mode 100644 src/plugins/messageStorage/text.js delete mode 100644 src/userLog.js diff --git a/package.json b/package.json index ecd06af5..bdddb415 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "cheerio": "0.22.0", "commander": "2.15.1", "express": "4.16.3", + "filenamify": "2.0.0", "fs-extra": "6.0.1", "irc-framework": "2.11.0", "lodash": "4.17.10", diff --git a/src/client.js b/src/client.js index 3752f118..0aa1355b 100644 --- a/src/client.js +++ b/src/client.js @@ -8,9 +8,11 @@ const Msg = require("./models/msg"); const Network = require("./models/network"); const Helper = require("./helper"); const UAParser = require("ua-parser-js"); -const MessageStorage = require("./plugins/sqlite"); const uuidv4 = require("uuid/v4"); +const MessageStorage = require("./plugins/messageStorage/sqlite"); +const TextFileMessageStorage = require("./plugins/messageStorage/text"); + module.exports = Client; const events = [ @@ -76,16 +78,23 @@ function Client(manager, name, config = {}) { networks: [], sockets: manager.sockets, manager: manager, + messageStorage: [], }); const client = this; let delay = 0; - if (!Helper.config.public) { - client.messageStorage = new MessageStorage(client); + if (!Helper.config.public && client.config.log) { + if (Helper.config.messageStorage.includes("sqlite")) { + client.messageStorage.push(new MessageStorage(client)); + } - if (client.config.log && Helper.config.messageStorage.includes("sqlite")) { - client.messageStorage.enable(client.name); + if (Helper.config.messageStorage.includes("text")) { + client.messageStorage.push(new TextFileMessageStorage(client)); + } + + for (const messageStorage of client.messageStorage) { + messageStorage.enable(); } } @@ -489,8 +498,8 @@ Client.prototype.quit = function(signOut) { network.destroy(); }); - if (this.messageStorage) { - this.messageStorage.close(); + for (const messageStorage of this.messageStorage) { + messageStorage.close(); } }; diff --git a/src/helper.js b/src/helper.js index 065a83a5..2fe864aa 100644 --- a/src/helper.js +++ b/src/helper.js @@ -14,6 +14,7 @@ let configPath; let usersPath; let storagePath; let packagesPath; +let userLogsPath; const Helper = { config: null, @@ -89,6 +90,7 @@ function setHome(newPath) { usersPath = path.join(homePath, "users"); storagePath = path.join(homePath, "storage"); packagesPath = path.join(homePath, "packages"); + userLogsPath = path.join(homePath, "logs"); // Reload config from new home location if (fs.existsSync(configPath)) { @@ -145,8 +147,8 @@ function getUserConfigPath(name) { return path.join(usersPath, name + ".json"); } -function getUserLogsPath(name, network) { - return path.join(homePath, "logs", name, network); +function getUserLogsPath() { + return userLogsPath; } function getStoragePath() { diff --git a/src/models/chan.js b/src/models/chan.js index ce8ed98c..45f1661e 100644 --- a/src/models/chan.js +++ b/src/models/chan.js @@ -3,7 +3,6 @@ const _ = require("lodash"); const Helper = require("../helper"); const User = require("./user"); -const userLog = require("../userLog"); const storage = require("../plugins/storage"); module.exports = Chan; @@ -180,13 +179,8 @@ Chan.prototype.getFilteredClone = function(lastActiveChannel, lastMessage) { Chan.prototype.writeUserLog = function(client, msg) { this.messages.push(msg); - // Does this user have logs disabled - if (!client.config.log) { - return; - } - - // Are logs disabled server-wide - if (Helper.config.messageStorage.length === 0) { + // Are there any logs enabled + if (client.messageStorage.length === 0) { return; } @@ -202,27 +196,23 @@ Chan.prototype.writeUserLog = function(client, msg) { return; } - // TODO: Something more pluggable - if (Helper.config.messageStorage.includes("sqlite")) { - client.messageStorage.index(target.network.uuid, this.name, msg); - } - - if (Helper.config.messageStorage.includes("text")) { - userLog.write( - client.name, - target.network.host, // TODO: Fix #1392, multiple connections to same server results in duplicate logs - this.type === Chan.Type.LOBBY ? target.network.host : this.name, - msg - ); + for (const messageStorage of client.messageStorage) { + messageStorage.index(target.network, this, msg); } }; Chan.prototype.loadMessages = function(client, network) { - if (!client.messageStorage || !this.isLoggable()) { + if (!this.isLoggable()) { return; } - client.messageStorage + const messageStorage = client.messageStorage.find((s) => s.canProvideMessages()); + + if (!messageStorage) { + return; + } + + messageStorage .getMessages(network, this) .then((messages) => { if (messages.length === 0) { diff --git a/src/models/network.js b/src/models/network.js index 20eff20d..22574885 100644 --- a/src/models/network.js +++ b/src/models/network.js @@ -129,7 +129,7 @@ Network.prototype.createIrcFramework = function(client) { // Request only new messages from ZNC if we have sqlite logging enabled // See http://wiki.znc.in/Playback - if (client.config.log && Helper.config.messageStorage.includes("sqlite")) { + if (client.config.log && client.messageStorage.find((s) => s.canProvideMessages())) { this.irc.requestCap("znc.in/playback"); } }; diff --git a/src/plugins/sqlite.js b/src/plugins/messageStorage/sqlite.js similarity index 91% rename from src/plugins/sqlite.js rename to src/plugins/messageStorage/sqlite.js index ef1214fe..2b939179 100644 --- a/src/plugins/sqlite.js +++ b/src/plugins/messageStorage/sqlite.js @@ -2,8 +2,8 @@ const path = require("path"); const fsextra = require("fs-extra"); -const Helper = require("../helper"); -const Msg = require("../models/msg"); +const Helper = require("../../helper"); +const Msg = require("../../models/msg"); let sqlite3; @@ -31,9 +31,9 @@ class MessageStorage { this.isEnabled = false; } - enable(name) { - const logsPath = path.join(Helper.getHomePath(), "logs"); - const sqlitePath = path.join(logsPath, `${name}.sqlite3`); + enable() { + const logsPath = Helper.getUserLogsPath(); + const sqlitePath = path.join(logsPath, `${this.client.name}.sqlite3`); try { fsextra.ensureDirSync(logsPath); @@ -114,7 +114,7 @@ class MessageStorage { this.database.serialize(() => this.database.run( "INSERT INTO messages(network, channel, time, type, msg) VALUES(?, ?, ?, ?, ?)", - network, channel.toLowerCase(), msg.time.getTime(), msg.type, JSON.stringify(clonedMsg) + network.uuid, channel.name.toLowerCase(), msg.time.getTime(), msg.type, JSON.stringify(clonedMsg) )); } @@ -152,6 +152,10 @@ class MessageStorage { )); }); } + + canProvideMessages() { + return this.isEnabled; + } } module.exports = MessageStorage; diff --git a/src/plugins/messageStorage/text.js b/src/plugins/messageStorage/text.js new file mode 100644 index 00000000..0b7d436b --- /dev/null +++ b/src/plugins/messageStorage/text.js @@ -0,0 +1,99 @@ +"use strict"; + +const fs = require("fs"); +const fsextra = require("fs-extra"); +const path = require("path"); +const moment = require("moment"); +const filenamify = require("filenamify"); +const Helper = require("../../helper"); + +class TextFileMessageStorage { + constructor(client) { + this.client = client; + this.isEnabled = false; + } + + enable() { + this.isEnabled = true; + } + + close(callback) { + this.isEnabled = false; + + if (callback) { + callback(); + } + } + + index(network, channel, msg) { + if (!this.isEnabled) { + return; + } + + const networkFolderName = cleanFilename(`${network.name}-${network.uuid.substring(network.name.length + 1)}`); + const logPath = path.join(Helper.getUserLogsPath(), this.client.name, networkFolderName); + + try { + fsextra.ensureDirSync(logPath); + } catch (e) { + log.error("Unable to create logs directory", e); + return; + } + + const format = Helper.config.logs.format || "YYYY-MM-DD HH:mm:ss"; + const tz = Helper.config.logs.timezone || "UTC+00:00"; + + const time = moment(msg.time).utcOffset(tz).format(format); + let line = `[${time}] `; + + if (msg.type === "message") { + // Format: + // [2014-01-01 00:00:00] Put that cookie down.. Now!! + line += `<${msg.from.nick}> ${msg.text}`; + } else { + // Format: + // [2014-01-01 00:00:00] * Arnold quit + line += `* ${msg.from.nick} `; + + if (msg.hostmask) { + line += `(${msg.hostmask}) `; + } + + line += msg.type; + + if (msg.new_nick) { // `/nick ` + line += ` ${msg.new_nick}`; + } else if (msg.text) { + line += ` ${msg.text}`; + } + } + + line += "\n"; + + fs.appendFile(path.join(logPath, cleanFilename(channel.name)), line, (e) => { + if (e) { + log.error("Failed to write user log", e); + } + }); + } + + getMessages() { + // Not implemented for text log files + // They do not contain enough data to fully re-create message objects + // Use sqlite storage instead + return Promise.resolve([]); + } + + canProvideMessages() { + return false; + } +} + +module.exports = TextFileMessageStorage; + +function cleanFilename(name) { + name = filenamify(name, {replacement: "_"}); + name = name.toLowerCase(); + + return `${name}.log`; +} diff --git a/src/userLog.js b/src/userLog.js deleted file mode 100644 index 863c6f62..00000000 --- a/src/userLog.js +++ /dev/null @@ -1,59 +0,0 @@ -"use strict"; - -const fs = require("fs"); -const fsextra = require("fs-extra"); -const moment = require("moment"); -const Helper = require("./helper"); - -module.exports.write = function(user, network, chan, msg) { - const path = Helper.getUserLogsPath(user, network); - - try { - fsextra.ensureDirSync(path); - } catch (e) { - log.error("Unable to create logs directory", e); - return; - } - - const format = Helper.config.logs.format || "YYYY-MM-DD HH:mm:ss"; - const tz = Helper.config.logs.timezone || "UTC+00:00"; - - const time = moment(msg.time).utcOffset(tz).format(format); - let line = `[${time}] `; - - const type = msg.type.trim(); - - if (type === "message" || type === "highlight") { - // Format: - // [2014-01-01 00:00:00] Put that cookie down.. Now!! - line += `<${msg.from.nick}> ${msg.text}`; - } else { - // Format: - // [2014-01-01 00:00:00] * Arnold quit - line += `* ${msg.from.nick} `; - - if (msg.hostmask) { - line += `(${msg.hostmask}) `; - } - - line += msg.type; - - if (msg.new_nick) { // `/nick ` - line += ` ${msg.new_nick}`; - } else if (msg.text) { - line += ` ${msg.text}`; - } - } - - fs.appendFile( - // Quick fix to escape pre-escape channel names that contain % using %%, - // and / using %. **This does not escape all reserved words** - path + "/" + chan.replace(/%/g, "%%").replace(/\//g, "%") + ".log", - line + "\n", - function(e) { - if (e) { - log.error("Failed to write user log", e); - } - } - ); -}; diff --git a/test/plugins/sqlite.js b/test/plugins/sqlite.js index f343ac69..a0119691 100644 --- a/test/plugins/sqlite.js +++ b/test/plugins/sqlite.js @@ -5,7 +5,7 @@ const path = require("path"); const expect = require("chai").expect; const Msg = require("../../src/models/msg"); const Helper = require("../../src/helper"); -const MessageStorage = require("../../src/plugins/sqlite.js"); +const MessageStorage = require("../../src/plugins/messageStorage/sqlite.js"); describe("SQLite Message Storage", function() { const expectedPath = path.join(Helper.getHomePath(), "logs", "testUser.sqlite3"); @@ -14,6 +14,7 @@ describe("SQLite Message Storage", function() { // Delete database file from previous test run before(function(done) { store = new MessageStorage({ + name: "testUser", idMsg: 1, }); @@ -35,7 +36,7 @@ describe("SQLite Message Storage", function() { expect(store.isEnabled).to.be.false; expect(fs.existsSync(expectedPath)).to.be.false; - store.enable("testUser"); + store.enable(); expect(store.isEnabled).to.be.true; }); @@ -76,7 +77,11 @@ describe("SQLite Message Storage", function() { it("should store a message", function(done) { store.database.serialize(() => { - store.index("this-is-a-network-guid", "#ThisIsAChannel", new Msg({ + store.index({ + uuid: "this-is-a-network-guid", + }, { + name: "#thisISaCHANNEL", + }, new Msg({ time: 123456789, text: "Hello from sqlite world!", })); diff --git a/yarn.lock b/yarn.lock index d0330abd..4cecbd2a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2821,6 +2821,18 @@ filename-regex@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.1.tgz#c1c4b9bee3e09725ddb106b75c1e301fe2f18b26" +filename-reserved-regex@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/filename-reserved-regex/-/filename-reserved-regex-2.0.0.tgz#abf73dfab735d045440abfea2d91f389ebbfa229" + +filenamify@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/filenamify/-/filenamify-2.0.0.tgz#bd162262c0b6e94bfbcdcf19a3bbb3764f785695" + dependencies: + filename-reserved-regex "^2.0.0" + strip-outer "^1.0.0" + trim-repeated "^1.0.0" + fill-range@^2.1.0: version "2.2.3" resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-2.2.3.tgz#50b77dfd7e469bc7492470963699fe7a8485a723" @@ -7213,6 +7225,12 @@ strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" +strip-outer@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/strip-outer/-/strip-outer-1.0.1.tgz#b2fd2abf6604b9d1e6013057195df836b8a9d631" + dependencies: + escape-string-regexp "^1.0.2" + style-loader@^0.19.1: version "0.19.1" resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-0.19.1.tgz#591ffc80bcefe268b77c5d9ebc0505d772619f85" @@ -7504,6 +7522,12 @@ trim-newlines@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-2.0.0.tgz#b403d0b91be50c331dfc4b82eeceb22c3de16d20" +trim-repeated@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/trim-repeated/-/trim-repeated-1.0.0.tgz#e3646a2ea4e891312bf7eace6cfb05380bc01c21" + dependencies: + escape-string-regexp "^1.0.2" + trim-right@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/trim-right/-/trim-right-1.0.1.tgz#cb2e1203067e0c8de1f614094b9fe45704ea6003"