diff --git a/src/client.js b/src/client.js
index ae363cf6..94a8cc59 100644
--- a/src/client.js
+++ b/src/client.js
@@ -483,7 +483,7 @@ Client.prototype.names = function(data) {
 
 	client.emit("names", {
 		id: target.chan.id,
-		users: target.chan.users,
+		users: target.chan.getSortedUsers(target.network.irc),
 	});
 };
 
diff --git a/src/models/chan.js b/src/models/chan.js
index 029df39b..63b45203 100644
--- a/src/models/chan.js
+++ b/src/models/chan.js
@@ -28,7 +28,7 @@ function Chan(attr) {
 		firstUnread: 0,
 		unread: 0,
 		highlight: 0,
-		users: [],
+		users: new Map(),
 	});
 }
 
@@ -97,7 +97,7 @@ Chan.prototype.dereferencePreviews = function(messages) {
 	});
 };
 
-Chan.prototype.sortUsers = function(irc) {
+Chan.prototype.getSortedUsers = function(irc) {
 	var userModeSortPriority = {};
 	irc.network.options.PREFIX.forEach((prefix, index) => {
 		userModeSortPriority[prefix.symbol] = index;
@@ -105,7 +105,9 @@ Chan.prototype.sortUsers = function(irc) {
 
 	userModeSortPriority[""] = 99; // No mode is lowest
 
-	this.users = this.users.sort(function(a, b) {
+	const users = Array.from(this.users.values());
+
+	return users.sort(function(a, b) {
 		if (a.mode === b.mode) {
 			return a.nick.toLowerCase() < b.nick.toLowerCase() ? -1 : 1;
 		}
@@ -119,13 +121,21 @@ Chan.prototype.findMessage = function(msgId) {
 };
 
 Chan.prototype.findUser = function(nick) {
-	return _.find(this.users, {nick: nick});
+	return this.users.get(nick.toLowerCase());
 };
 
 Chan.prototype.getUser = function(nick) {
 	return this.findUser(nick) || new User({nick: nick});
 };
 
+Chan.prototype.setUser = function(user) {
+	this.users.set(user.nick.toLowerCase(), user);
+};
+
+Chan.prototype.removeUser = function(user) {
+	this.users.delete(user.nick.toLowerCase());
+};
+
 Chan.prototype.toJSON = function() {
 	var clone = _.clone(this);
 	clone.users = []; // Do not send user list, the client will explicitly request it when needed
diff --git a/src/plugins/irc-events/away.js b/src/plugins/irc-events/away.js
index cf64aea5..e0cc9e3b 100644
--- a/src/plugins/irc-events/away.js
+++ b/src/plugins/irc-events/away.js
@@ -1,6 +1,5 @@
 "use strict";
 
-const _ = require("lodash");
 const Msg = require("../../models/msg");
 
 module.exports = function(irc, network) {
@@ -9,7 +8,7 @@ module.exports = function(irc, network) {
 		const away = data.message;
 
 		network.channels.forEach((chan) => {
-			const user = _.find(chan.users, {nick: data.nick});
+			const user = chan.findUser(data.nick);
 
 			if (!user || user.away === away) {
 				return;
diff --git a/src/plugins/irc-events/join.js b/src/plugins/irc-events/join.js
index 40ccebf7..a781d059 100644
--- a/src/plugins/irc-events/join.js
+++ b/src/plugins/irc-events/join.js
@@ -35,8 +35,7 @@ module.exports = function(irc, network) {
 		});
 		chan.pushMessage(client, msg);
 
-		chan.users.push(user);
-		chan.sortUsers(irc);
+		chan.setUser(new User({nick: data.nick}));
 		client.emit("users", {
 			chan: chan.id,
 		});
diff --git a/src/plugins/irc-events/kick.js b/src/plugins/irc-events/kick.js
index 9794e586..90d1ee18 100644
--- a/src/plugins/irc-events/kick.js
+++ b/src/plugins/irc-events/kick.js
@@ -1,6 +1,5 @@
 "use strict";
 
-const _ = require("lodash");
 const Msg = require("../../models/msg");
 
 module.exports = function(irc, network) {
@@ -25,9 +24,9 @@ module.exports = function(irc, network) {
 		chan.pushMessage(client, msg);
 
 		if (data.kicked === irc.user.nick) {
-			chan.users = [];
+			chan.users = new Map();
 		} else {
-			chan.users = _.without(chan.users, msg.target);
+			chan.removeUser(msg.target);
 		}
 
 		client.emit("users", {
diff --git a/src/plugins/irc-events/mode.js b/src/plugins/irc-events/mode.js
index f9f51410..d2d79d65 100644
--- a/src/plugins/irc-events/mode.js
+++ b/src/plugins/irc-events/mode.js
@@ -113,8 +113,6 @@ module.exports = function(irc, network) {
 			// TODO: This is horrible
 			irc.raw("NAMES", data.target);
 		} else {
-			targetChan.sortUsers(irc);
-
 			client.emit("users", {
 				chan: targetChan.id,
 			});
diff --git a/src/plugins/irc-events/names.js b/src/plugins/irc-events/names.js
index 6e9fe2da..77c72869 100644
--- a/src/plugins/irc-events/names.js
+++ b/src/plugins/irc-events/names.js
@@ -1,7 +1,5 @@
 "use strict";
 
-const User = require("../../models/user");
-
 module.exports = function(irc, network) {
 	const client = this;
 
@@ -11,31 +9,16 @@ module.exports = function(irc, network) {
 			return;
 		}
 
-		// Create lookup map of current users,
-		// as we need to keep certain properties
-		// and we can recycle existing User objects
-		const oldUsers = new Map();
+		const newUsers = new Map();
 
-		chan.users.forEach((user) => {
-			oldUsers.set(user.nick, user);
+		data.users.forEach((user) => {
+			const newUser = chan.getUser(user.nick);
+			newUser.setModes(user.modes, network.prefixLookup);
+
+			newUsers.set(user.nick.toLowerCase(), newUser);
 		});
 
-		chan.users = data.users.map((user) => {
-			const oldUser = oldUsers.get(user.nick);
-
-			// For existing users, we only need to update mode
-			if (oldUser) {
-				oldUser.setModes(user.modes, network.prefixLookup);
-				return oldUser;
-			}
-
-			return new User({
-				nick: user.nick,
-				modes: user.modes,
-			}, network.prefixLookup);
-		});
-
-		chan.sortUsers(irc);
+		chan.users = newUsers;
 
 		client.emit("users", {
 			chan: chan.id,
diff --git a/src/plugins/irc-events/nick.js b/src/plugins/irc-events/nick.js
index 9c7869ef..3589453d 100644
--- a/src/plugins/irc-events/nick.js
+++ b/src/plugins/irc-events/nick.js
@@ -32,6 +32,10 @@ module.exports = function(irc, network) {
 				return;
 			}
 
+			chan.removeUser(user);
+			user.nick = data.new_nick;
+			chan.setUser(user);
+
 			msg = new Msg({
 				time: data.time,
 				from: user,
@@ -43,7 +47,6 @@ module.exports = function(irc, network) {
 
 			user.nick = data.new_nick;
 
-			chan.sortUsers(irc);
 			client.emit("users", {
 				chan: chan.id,
 			});
diff --git a/src/plugins/irc-events/part.js b/src/plugins/irc-events/part.js
index 3e23f51d..16a5e7e6 100644
--- a/src/plugins/irc-events/part.js
+++ b/src/plugins/irc-events/part.js
@@ -32,7 +32,7 @@ module.exports = function(irc, network) {
 			});
 			chan.pushMessage(client, msg);
 
-			chan.users = _.without(chan.users, user);
+			chan.removeUser(user);
 			client.emit("users", {
 				chan: chan.id,
 			});
diff --git a/src/plugins/irc-events/quit.js b/src/plugins/irc-events/quit.js
index f5958a33..c8771cb4 100644
--- a/src/plugins/irc-events/quit.js
+++ b/src/plugins/irc-events/quit.js
@@ -1,6 +1,5 @@
 "use strict";
 
-const _ = require("lodash");
 const Msg = require("../../models/msg");
 
 module.exports = function(irc, network) {
@@ -23,7 +22,7 @@ module.exports = function(irc, network) {
 			});
 			chan.pushMessage(client, msg);
 
-			chan.users = _.without(chan.users, user);
+			chan.removeUser(user);
 			client.emit("users", {
 				chan: chan.id,
 			});
diff --git a/test/models/chan.js b/test/models/chan.js
index ce872875..77dc1752 100644
--- a/test/models/chan.js
+++ b/test/models/chan.js
@@ -1,12 +1,31 @@
 "use strict";
 
-var expect = require("chai").expect;
-
-var Chan = require("../../src/models/chan");
-var Msg = require("../../src/models/msg");
-var User = require("../../src/models/user");
+const expect = require("chai").expect;
+const Chan = require("../../src/models/chan");
+const Msg = require("../../src/models/msg");
+const User = require("../../src/models/user");
 
 describe("Chan", function() {
+	const network = {
+		network: {
+			options: {
+				PREFIX: [
+					{symbol: "~", mode: "q"},
+					{symbol: "&", mode: "a"},
+					{symbol: "@", mode: "o"},
+					{symbol: "%", mode: "h"},
+					{symbol: "+", mode: "v"},
+				],
+			},
+		},
+	};
+
+	const prefixLookup = {};
+
+	network.network.options.PREFIX.forEach((mode) => {
+		prefixLookup[mode.mode] = mode.symbol;
+	});
+
 	describe("#findMessage(id)", function() {
 		const chan = new Chan({
 			messages: [
@@ -27,40 +46,51 @@ describe("Chan", function() {
 		});
 	});
 
-	describe("#sortUsers(irc)", function() {
-		var network = {
-			network: {
-				options: {
-					PREFIX: [
-						{symbol: "~", mode: "q"},
-						{symbol: "&", mode: "a"},
-						{symbol: "@", mode: "o"},
-						{symbol: "%", mode: "h"},
-						{symbol: "+", mode: "v"},
-					],
-				},
-			},
-		};
+	describe("#setUser(user)", function() {
+		it("should make key lowercase", function() {
+			const chan = new Chan();
+			chan.setUser(new User({nick: "TestUser"}));
 
-		var prefixLookup = {};
-
-		network.network.options.PREFIX.forEach((mode) => {
-			prefixLookup[mode.mode] = mode.symbol;
+			expect(chan.users.has("testuser")).to.be.true;
 		});
 
-		var makeUser = function(nick) {
-			return new User({nick: nick}, prefixLookup);
-		};
+		it("should update user object", function() {
+			const chan = new Chan();
+			chan.setUser(new User({nick: "TestUser"}, prefixLookup));
+			chan.setUser(new User({nick: "TestUseR", modes: ["o"]}, prefixLookup));
+			const user = chan.getUser("TestUSER");
 
+			expect(user.mode).to.equal("@");
+		});
+	});
+
+	describe("#getUser(nick)", function() {
+		it("should returning existing object", function() {
+			const chan = new Chan();
+			chan.setUser(new User({nick: "TestUseR", modes: ["o"]}, prefixLookup));
+			const user = chan.getUser("TestUSER");
+
+			expect(user.mode).to.equal("@");
+		});
+
+		it("should make new User object if not found", function() {
+			const chan = new Chan();
+			const user = chan.getUser("very-testy-user");
+
+			expect(user.nick).to.equal("very-testy-user");
+		});
+	});
+
+	describe("#getSortedUsers(irc)", function() {
 		var getUserNames = function(chan) {
-			return chan.users.map((u) => u.nick);
+			return chan.getSortedUsers(network).map((u) => u.nick);
 		};
 
 		it("should sort a simple user list", function() {
-			var chan = new Chan({users: [
+			const chan = new Chan();
+			[
 				"JocelynD", "YaManicKill", "astorije", "xPaw", "Max-P",
-			].map(makeUser)});
-			chan.sortUsers(network);
+			].forEach((nick) => chan.setUser(new User({nick: nick}, prefixLookup)));
 
 			expect(getUserNames(chan)).to.deep.equal([
 				"astorije", "JocelynD", "Max-P", "xPaw", "YaManicKill",
@@ -68,14 +98,12 @@ describe("Chan", function() {
 		});
 
 		it("should group users by modes", function() {
-			var chan = new Chan({users: [
-				new User({nick: "JocelynD", modes: ["a", "o"]}, prefixLookup),
-				new User({nick: "YaManicKill", modes: ["v"]}, prefixLookup),
-				new User({nick: "astorije", modes: ["h"]}, prefixLookup),
-				new User({nick: "xPaw", modes: ["q"]}, prefixLookup),
-				new User({nick: "Max-P", modes: ["o"]}, prefixLookup),
-			]});
-			chan.sortUsers(network);
+			const chan = new Chan();
+			chan.setUser(new User({nick: "JocelynD", modes: ["a", "o"]}, prefixLookup));
+			chan.setUser(new User({nick: "YaManicKill", modes: ["v"]}, prefixLookup));
+			chan.setUser(new User({nick: "astorije", modes: ["h"]}, prefixLookup));
+			chan.setUser(new User({nick: "xPaw", modes: ["q"]}, prefixLookup));
+			chan.setUser(new User({nick: "Max-P", modes: ["o"]}, prefixLookup));
 
 			expect(getUserNames(chan)).to.deep.equal([
 				"xPaw", "JocelynD", "Max-P", "astorije", "YaManicKill",
@@ -83,14 +111,12 @@ describe("Chan", function() {
 		});
 
 		it("should sort a mix of users and modes", function() {
-			var chan = new Chan({users: [
-				new User({nick: "JocelynD"}, prefixLookup),
-				new User({nick: "YaManicKill", modes: ["o"]}, prefixLookup),
-				new User({nick: "astorije"}, prefixLookup),
-				new User({nick: "xPaw"}, prefixLookup),
-				new User({nick: "Max-P", modes: ["o"]}, prefixLookup),
-			]});
-			chan.sortUsers(network);
+			const chan = new Chan();
+			chan.setUser(new User({nick: "JocelynD"}, prefixLookup));
+			chan.setUser(new User({nick: "YaManicKill", modes: ["o"]}, prefixLookup));
+			chan.setUser(new User({nick: "astorije"}, prefixLookup));
+			chan.setUser(new User({nick: "xPaw"}, prefixLookup));
+			chan.setUser(new User({nick: "Max-P", modes: ["o"]}, prefixLookup));
 
 			expect(getUserNames(chan)).to.deep.equal(
 				["Max-P", "YaManicKill", "astorije", "JocelynD", "xPaw"]
@@ -98,18 +124,20 @@ describe("Chan", function() {
 		});
 
 		it("should be case-insensitive", function() {
-			var chan = new Chan({users: ["aB", "Ad", "AA", "ac"].map(makeUser)});
-			chan.sortUsers(network);
+			const chan = new Chan();
+			[
+				"aB", "Ad", "AA", "ac",
+			].forEach((nick) => chan.setUser(new User({nick: nick}, prefixLookup)));
 
 			expect(getUserNames(chan)).to.deep.equal(["AA", "aB", "ac", "Ad"]);
 		});
 
 		it("should parse special characters successfully", function() {
-			var chan = new Chan({users: [
+			const chan = new Chan();
+			[
 				"[foo", "]foo", "(foo)", "{foo}", "<foo>", "_foo", "@foo", "^foo",
 				"&foo", "!foo", "+foo", "Foo",
-			].map(makeUser)});
-			chan.sortUsers(network);
+			].forEach((nick) => chan.setUser(new User({nick: nick}, prefixLookup)));
 
 			expect(getUserNames(chan)).to.deep.equal([
 				"!foo", "&foo", "(foo)", "+foo", "<foo>", "@foo", "[foo", "]foo",