Resolve high CPU usage when performing DB reads (#419)

* Disable automatic indexing as we specifically create the required indexes
* Tell SQLite to store temporary tables in memory. This will speed up many read operations that rely on temporary tables, indices, and views.
* Add links & reasoning behind other PRAGMA settings used
* Add new index specifically for driveId & parentId paring
* To force DB schema & index creation, bump DB schema version
* Update handling of skip_dir and skip_file parsing - should only check if the file is excluded if the parent directory is not
* Add another index for selectByPath database queries
* Add new build option to get more DEBUG symbolic information
* Use boolean values rather than on / off values
* Enable auto_vacuum for entry deletes / database cleanup

Makefile

@@ -27,6 +27,14 @@ ifeq ($(notdir $(DC)),ldc2)
 	NOTIF_VERSIONS := $(addprefix -d,$(NOTIF_VERSIONS))
 endif
 
+ifeq ($(DEBUG),1)
+ifeq ($(notdir $(DC)),ldc2)
+DFLAGS += -d-debug -gc
+else
+DFLAGS += -debug -gs
+endif
+endif
+
 DFLAGS += -w -g -ofonedrive -O $(NOTIF_VERSIONS) $(LIBS) -J.
 
 PREFIX ?= /usr/local

README.md

@@ -195,6 +195,9 @@ By passing `NOTIFICATIONS=1` to the `make` call, notifications via
 libnotify are enabled. If `pkg-config` is not used (see above), the necessary
 libraries are `gmodule-2.0`, `glib-2.0`, and `notify`.
 
+By passing `DEBUG=1` to the `make` call, `onedrive` gets built with additional debug
+information, useful (for example) to get `perf`-issued figures.
+
 ### Building using a different compiler (for example [LDC](https://wiki.dlang.org/LDC))
 #### Debian - i386 / i686
 ```text

src/itemdb.d

@@ -31,7 +31,7 @@ struct Item {
 final class ItemDatabase
 {
 	// increment this for every change in the db schema
-	immutable int itemDatabaseVersion = 7;
+	immutable int itemDatabaseVersion = 8;
 
 	Database db;
 	string insertItemStmt;
@@ -59,9 +59,28 @@ final class ItemDatabase
 			db.exec("DROP TABLE item");
 			createTable();
 		}
-		db.exec("PRAGMA foreign_keys = ON");
+		// Set the enforcement of foreign key constraints.
-		db.exec("PRAGMA recursive_triggers = ON");
+		// https://www.sqlite.org/pragma.html#pragma_foreign_keys
+		// PRAGMA foreign_keys = boolean;
+		db.exec("PRAGMA foreign_keys = TRUE");
+		// Set the recursive trigger capability
+		// https://www.sqlite.org/pragma.html#pragma_recursive_triggers
+		// PRAGMA recursive_triggers = boolean;
+		db.exec("PRAGMA recursive_triggers = TRUE");
+		// Set the journal mode for databases associated with the current connection
+		// https://www.sqlite.org/pragma.html#pragma_journal_mode
 		db.exec("PRAGMA journal_mode = WAL");
+		// Automatic indexing is enabled by default as of version 3.7.17
+		// https://www.sqlite.org/pragma.html#pragma_automatic_index 
+		// PRAGMA automatic_index = boolean;
+		db.exec("PRAGMA automatic_index = FALSE");
+		// Tell SQLite to store temporary tables in memory. This will speed up many read operations that rely on temporary tables, indices, and views.
+		// https://www.sqlite.org/pragma.html#pragma_temp_store
+		db.exec("PRAGMA temp_store = MEMORY");
+		// Tell SQlite to cleanup database table size
+		// https://www.sqlite.org/pragma.html#pragma_auto_vacuum
+		// PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
+		db.exec("PRAGMA auto_vacuum = FULL");
 		
 		insertItemStmt = "
 			INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId)
@@ -106,6 +125,8 @@ final class ItemDatabase
 			)");
 		db.exec("CREATE INDEX name_idx ON item (name)");
 		db.exec("CREATE INDEX remote_idx ON item (remoteDriveId, remoteId)");
+		db.exec("CREATE INDEX item_children_idx ON item (driveId, parentId)");
+		db.exec("CREATE INDEX selectByPath_idx ON item (name, driveId, parentId)");
 		db.setVersion(itemDatabaseVersion);
 	}
 	

src/sync.d

@@ -1148,9 +1148,15 @@ final class SyncEngine
 		bool unwanted = false;
 		string path;
 		
-		// Is item.name or the path excluded
+		// Is the path excluded?
-		unwanted = selectiveSync.isFileNameExcluded(item.name);
+		unwanted = selectiveSync.isDirNameExcluded(item.name);
+		
+		// If the path is not excluded, is the filename excluded?
+		if (!unwanted) {
+			unwanted = selectiveSync.isFileNameExcluded(item.name);
+		}
 
+		// If path or filename does not exclude, is this excluded due to use of selective sync?
 		if (!unwanted) {
 			path = itemdb.computePath(item.driveId, item.id);
 			unwanted = selectiveSync.isPathExcluded(path);