From 58c2805237eba002a1266c38b7be42d0c5bca1d7 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Tue, 19 Mar 2019 13:14:07 +1100 Subject: [PATCH 01/10] Update Makefile (#417) * Update makefile so if using 'release' archive to build, the 'version' file is created with the correct version string as .git/HEAD & index is not available. --- Makefile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c111d1c9..466e23bb 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ DC ?= dmd - +RELEASEVER = v2.2.6 pkgconfig := $(shell if [ $(PKGCONFIG) ] && [ "$(PKGCONFIG)" != 0 ] ; then echo 1 ; else echo "" ; fi) notifications := $(shell if [ $(NOTIFICATIONS) ] && [ "$(NOTIFICATIONS)" != 0 ] ; then echo 1 ; else echo "" ; fi) +gitversion := $(shell if [ -f .git/HEAD ] ; then echo 1 ; else echo "" ; fi) ifeq ($(pkgconfig),1) LIBS = $(shell pkg-config --libs sqlite3 libcurl) @@ -126,5 +127,9 @@ endif for i in $(DOCFILES) ; do rm -f $(DESTDIR)$(DOCDIR)/$$i ; done rm -f $(DESTDIR)$(MANDIR)/onedrive.1 -version: .git/HEAD .git/index +version: +ifeq ($(gitversion),1) echo $(shell git describe --tags) > version +else + echo $(RELEASEVER) > version +endif \ No newline at end of file From 6b8b51a7cc913c5e389215a7378f48611a154462 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Thu, 21 Mar 2019 10:29:04 +1100 Subject: [PATCH 02/10] Update sync.d (#422) * Add a check to validate if 'size' is in the JSON response --- src/sync.d | 79 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/src/sync.d b/src/sync.d index 194f86aa..af71d013 100644 --- a/src/sync.d +++ b/src/sync.d @@ -1771,47 +1771,50 @@ final class SyncEngine // Log action to log file log.fileOnly("Uploading new file ", path, " ... done."); - // The file was uploaded - ulong uploadFileSize = response["size"].integer; - - // In some cases the file that was uploaded was not complete, but 'completed' without errors on OneDrive - // This has been seen with PNG / JPG files mainly, which then contributes to generating a 412 error when we attempt to update the metadata - // Validate here that the file uploaded, at least in size, matches in the response to what the size is on disk - if (thisFileSize != uploadFileSize){ - if(disableUploadValidation){ - // Print a warning message - log.log("WARNING: Uploaded file size does not match local file - skipping upload validation"); + // The file was uploaded, or a 4xx / 5xx error was generated + if ("size" in response){ + // The response JSON contains size, high likelihood valid response returned + ulong uploadFileSize = response["size"].integer; + + // In some cases the file that was uploaded was not complete, but 'completed' without errors on OneDrive + // This has been seen with PNG / JPG files mainly, which then contributes to generating a 412 error when we attempt to update the metadata + // Validate here that the file uploaded, at least in size, matches in the response to what the size is on disk + if (thisFileSize != uploadFileSize){ + if(disableUploadValidation){ + // Print a warning message + log.log("WARNING: Uploaded file size does not match local file - skipping upload validation"); + } else { + // OK .. the uploaded file does not match and we did not disable this validation + log.log("Uploaded file size does not match local file - upload failure - retrying"); + // Delete uploaded bad file + onedrive.deleteById(response["parentReference"]["driveId"].str, response["id"].str, response["eTag"].str); + // Re-upload + uploadNewFile(path); + return; + } + } + + // File validation is OK + if ((accountType == "personal") || (thisFileSize == 0)){ + // Update the item's metadata on OneDrive + string id = response["id"].str; + string cTag = response["cTag"].str; + if (exists(path)) { + SysTime mtime = timeLastModified(path).toUTC(); + // use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded + uploadLastModifiedTime(parent.driveId, id, cTag, mtime); + } else { + // will be removed in different event! + log.log("File disappeared after upload: ", path); + } + return; } else { - // OK .. the uploaded file does not match and we did not disable this validation - log.log("Uploaded file size does not match local file - upload failure - retrying"); - // Delete uploaded bad file - onedrive.deleteById(response["parentReference"]["driveId"].str, response["id"].str, response["eTag"].str); - // Re-upload - uploadNewFile(path); + // OneDrive Business Account - always use a session to upload + // The session includes a Request Body element containing lastModifiedDateTime + // which negates the need for a modify event against OneDrive + saveItem(response); return; } - } - - // File validation is OK - if ((accountType == "personal") || (thisFileSize == 0)){ - // Update the item's metadata on OneDrive - string id = response["id"].str; - string cTag = response["cTag"].str; - if (exists(path)) { - SysTime mtime = timeLastModified(path).toUTC(); - // use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded - uploadLastModifiedTime(parent.driveId, id, cTag, mtime); - } else { - // will be removed in different event! - log.log("File disappeared after upload: ", path); - } - return; - } else { - // OneDrive Business Account - always use a session to upload - // The session includes a Request Body element containing lastModifiedDateTime - // which negates the need for a modify event against OneDrive - saveItem(response); - return; } } else { // we are --dry-run - simulate the file upload From f2e007a1d7c0f7ed80634c0347f15c7e9bec7f70 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Fri, 22 Mar 2019 05:30:32 +1100 Subject: [PATCH 03/10] Update skip_dir config handling (#418) * Original skip_dir handling expected an explicit match to path to match. With this patch, wildcard matching for any directory matching the path entry will be skipped --- src/selective.d | 14 +++++++++++++- src/sync.d | 2 ++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/selective.d b/src/selective.d index 08661983..18ca84f6 100644 --- a/src/selective.d +++ b/src/selective.d @@ -39,7 +39,19 @@ final class SelectiveSync // Does the directory name match skip_dir config entry? // Returns true if the name matches a skip_dir config entry // Returns false if no match - return !name.matchFirst(dirmask).empty; + + // Try full path match first + if (!name.matchFirst(dirmask).empty) { + return true; + } else { + // check just the file name + string filename = baseName(name); + if(!filename.matchFirst(dirmask).empty) { + return true; + } + } + // no match + return false; } // config file skip_file parameter diff --git a/src/sync.d b/src/sync.d index af71d013..8514097a 100644 --- a/src/sync.d +++ b/src/sync.d @@ -1485,12 +1485,14 @@ final class SyncEngine // filter out user configured items to skip if (path != ".") { if (isDir(path)) { + log.vdebug("Checking path: ", path); if (selectiveSync.isDirNameExcluded(strip(path,"./"))) { log.vlog("Skipping item - excluded by skip_dir config: ", path); return; } } if (isFile(path)) { + log.vdebug("Checking file: ", path); if (selectiveSync.isFileNameExcluded(strip(path,"./"))) { log.vlog("Skipping item - excluded by skip_file config: ", path); return; From 0dfc165534ac2b10033f47a6176fb320afc8fc7d Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Fri, 22 Mar 2019 08:19:27 +0900 Subject: [PATCH 04/10] Declare version as phony (always outdated) target to guarantee update (#429) * Declare version as phony --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 466e23bb..9889a813 100644 --- a/Makefile +++ b/Makefile @@ -132,4 +132,7 @@ ifeq ($(gitversion),1) echo $(shell git describe --tags) > version else echo $(RELEASEVER) > version -endif \ No newline at end of file +endif + +.PHONY: version + From de98fe849259c4b80562f0731cfb492117120ec2 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Fri, 22 Mar 2019 10:23:21 +1100 Subject: [PATCH 05/10] Fix JSONValue is not an object crash when a 5xx error is returned when uploading files (#426) * Fix JSONValue is not an object crash when a 5xx error is returned when uploading files --- src/sync.d | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/sync.d b/src/sync.d index 8514097a..4ebbb041 100644 --- a/src/sync.d +++ b/src/sync.d @@ -1684,6 +1684,8 @@ final class SyncEngine auto maxUploadFileSize = 16106127360; // 15GB //auto maxUploadFileSize = 21474836480; // 20GB auto thisFileSize = getSize(path); + // To avoid a 409 Conflict error - does the file actually exist on OneDrive already? + JSONValue fileDetailsFromOneDrive; // Can we read the file - as a permissions issue or file corruption will cause a failure // https://github.com/abraunegg/onedrive/issues/113 @@ -1691,10 +1693,6 @@ final class SyncEngine // able to read the file if (thisFileSize <= maxUploadFileSize){ // Resolves: https://github.com/skilion/onedrive/issues/121, https://github.com/skilion/onedrive/issues/294, https://github.com/skilion/onedrive/issues/329 - - // To avoid a 409 Conflict error - does the file actually exist on OneDrive already? - JSONValue fileDetailsFromOneDrive; - // Does this 'file' already exist on OneDrive? try { // test if the local path exists on OneDrive @@ -1841,7 +1839,8 @@ final class SyncEngine // even though some file systems (such as a POSIX-compliant file system) may consider them as different. // Note that NTFS supports POSIX semantics for case sensitivity but this is not the default behavior. - if (fileDetailsFromOneDrive["name"].str == baseName(path)){ + // Check that 'name' is in the JSON response (validates data) and that 'name' == the path we are looking for + if (("name" in fileDetailsFromOneDrive) && (fileDetailsFromOneDrive["name"].str == baseName(path))) { // OneDrive 'name' matches local path name log.vlog("Requested file to upload exists on OneDrive - local database is out of sync for this file: ", path); From 8f28daec8a6b693b18526138c3a494e1d17bfbd0 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Fri, 22 Mar 2019 11:19:53 +0900 Subject: [PATCH 06/10] ignore a path containing multiple dirs in skip_dir --- src/selective.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/selective.d b/src/selective.d index 18ca84f6..35c10f28 100644 --- a/src/selective.d +++ b/src/selective.d @@ -78,7 +78,7 @@ final class SelectiveSync // config sync_list file handling bool isPathExcluded(string path) { - return .isPathExcluded(path, paths) || .isPathMatched(path, mask); + return .isPathExcluded(path, paths) || .isPathMatched(path, mask) || .isPathMatched(path, dirmask); } } From f5c3e7643d9f1b08c13e9c99f4c723cfe1a6e618 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Fri, 22 Mar 2019 15:21:59 +1100 Subject: [PATCH 07/10] Update 'skip_dir' handling to check against OneDrive new downloads (#427) * Add check for directories to match skip_dir entries for OneDrive objects to download * Update logging as to why a OneDrive object was skipped --- src/sync.d | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/sync.d b/src/sync.d index 4ebbb041..c1849ea1 100644 --- a/src/sync.d +++ b/src/sync.d @@ -754,9 +754,17 @@ final class SyncEngine bool unwanted; unwanted |= skippedItems.find(item.parentId).length != 0; if (unwanted) log.vdebug("Flagging as unwanted: find(item.parentId).length != 0"); - unwanted |= selectiveSync.isFileNameExcluded(item.name); - if (unwanted) log.vdebug("Flagging as unwanted: item name is excluded: ", item.name); - + // Check if this is a directory to skip + if (!unwanted) { + unwanted = selectiveSync.isDirNameExcluded(item.name); + if (unwanted) log.vlog("Skipping item - excluded by skip_dir config: ", item.name); + } + // Check if this is a file to skip + if (!unwanted) { + unwanted = selectiveSync.isFileNameExcluded(item.name); + if (unwanted) log.vlog("Skipping item - excluded by skip_file config: ", item.name); + } + // check the item type if (!unwanted) { if (isItemFile(driveItem)) { From 002f9b7aec884edd544cb911b132d746bcc3ce27 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Sat, 23 Mar 2019 10:20:27 +1100 Subject: [PATCH 08/10] Prevent use of --synchronize & --monitor together (#431) * Prevent use of --synchronize & --monitor together --- src/main.d | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main.d b/src/main.d index 80692180..bb9627e7 100644 --- a/src/main.d +++ b/src/main.d @@ -402,7 +402,6 @@ int main(string[] args) // if --synchronize or --monitor not passed in, exit & display help auto performSyncOK = false; - if (synchronize || monitor) { performSyncOK = true; } @@ -420,6 +419,14 @@ int main(string[] args) return EXIT_FAILURE; } + // if --synchronize && --monitor passed in, exit & display help as these conflict with each other + if (synchronize && monitor) { + writeln("\nERROR: --synchronize and --monitor cannot be used together\n"); + writeln("Refer to --help to determine which command option you should use.\n"); + oneDrive.http.shutdown(); + return EXIT_FAILURE; + } + // Initialize the item database log.vlog("Opening the item database ..."); if (!dryRun) { From 79cc5990575c7e3c5dd9101b5683f6783b93f09f Mon Sep 17 00:00:00 2001 From: abraunegg Date: Sun, 24 Mar 2019 11:12:40 +1100 Subject: [PATCH 09/10] Resolve high CPU usage when performing DB reads (#419) * Disable automatic indexing as we specifically create the required indexes * Tell SQLite to store temporary tables in memory. This will speed up many read operations that rely on temporary tables, indices, and views. * Add links & reasoning behind other PRAGMA settings used * Add new index specifically for driveId & parentId paring * To force DB schema & index creation, bump DB schema version * Update handling of skip_dir and skip_file parsing - should only check if the file is excluded if the parent directory is not * Add another index for selectByPath database queries * Add new build option to get more DEBUG symbolic information * Use boolean values rather than on / off values * Enable auto_vacuum for entry deletes / database cleanup --- Makefile | 8 ++++++++ README.md | 3 +++ src/itemdb.d | 27 ++++++++++++++++++++++++--- src/sync.d | 10 ++++++++-- 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 9889a813..86d8e6d1 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,14 @@ ifeq ($(notdir $(DC)),ldc2) NOTIF_VERSIONS := $(addprefix -d,$(NOTIF_VERSIONS)) endif +ifeq ($(DEBUG),1) +ifeq ($(notdir $(DC)),ldc2) +DFLAGS += -d-debug -gc +else +DFLAGS += -debug -gs +endif +endif + DFLAGS += -w -g -ofonedrive -O $(NOTIF_VERSIONS) $(LIBS) -J. PREFIX ?= /usr/local diff --git a/README.md b/README.md index c81bc806..4ed09cb2 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,9 @@ By passing `NOTIFICATIONS=1` to the `make` call, notifications via libnotify are enabled. If `pkg-config` is not used (see above), the necessary libraries are `gmodule-2.0`, `glib-2.0`, and `notify`. +By passing `DEBUG=1` to the `make` call, `onedrive` gets built with additional debug +information, useful (for example) to get `perf`-issued figures. + ### Building using a different compiler (for example [LDC](https://wiki.dlang.org/LDC)) #### Debian - i386 / i686 ```text diff --git a/src/itemdb.d b/src/itemdb.d index 5f0353cc..fd8132eb 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -31,7 +31,7 @@ struct Item { final class ItemDatabase { // increment this for every change in the db schema - immutable int itemDatabaseVersion = 7; + immutable int itemDatabaseVersion = 8; Database db; string insertItemStmt; @@ -59,9 +59,28 @@ final class ItemDatabase db.exec("DROP TABLE item"); createTable(); } - db.exec("PRAGMA foreign_keys = ON"); - db.exec("PRAGMA recursive_triggers = ON"); + // Set the enforcement of foreign key constraints. + // https://www.sqlite.org/pragma.html#pragma_foreign_keys + // PRAGMA foreign_keys = boolean; + db.exec("PRAGMA foreign_keys = TRUE"); + // Set the recursive trigger capability + // https://www.sqlite.org/pragma.html#pragma_recursive_triggers + // PRAGMA recursive_triggers = boolean; + db.exec("PRAGMA recursive_triggers = TRUE"); + // Set the journal mode for databases associated with the current connection + // https://www.sqlite.org/pragma.html#pragma_journal_mode db.exec("PRAGMA journal_mode = WAL"); + // Automatic indexing is enabled by default as of version 3.7.17 + // https://www.sqlite.org/pragma.html#pragma_automatic_index + // PRAGMA automatic_index = boolean; + db.exec("PRAGMA automatic_index = FALSE"); + // Tell SQLite to store temporary tables in memory. This will speed up many read operations that rely on temporary tables, indices, and views. + // https://www.sqlite.org/pragma.html#pragma_temp_store + db.exec("PRAGMA temp_store = MEMORY"); + // Tell SQlite to cleanup database table size + // https://www.sqlite.org/pragma.html#pragma_auto_vacuum + // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL; + db.exec("PRAGMA auto_vacuum = FULL"); insertItemStmt = " INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId) @@ -106,6 +125,8 @@ final class ItemDatabase )"); db.exec("CREATE INDEX name_idx ON item (name)"); db.exec("CREATE INDEX remote_idx ON item (remoteDriveId, remoteId)"); + db.exec("CREATE INDEX item_children_idx ON item (driveId, parentId)"); + db.exec("CREATE INDEX selectByPath_idx ON item (name, driveId, parentId)"); db.setVersion(itemDatabaseVersion); } diff --git a/src/sync.d b/src/sync.d index c1849ea1..9ff2f383 100644 --- a/src/sync.d +++ b/src/sync.d @@ -1148,9 +1148,15 @@ final class SyncEngine bool unwanted = false; string path; - // Is item.name or the path excluded - unwanted = selectiveSync.isFileNameExcluded(item.name); + // Is the path excluded? + unwanted = selectiveSync.isDirNameExcluded(item.name); + + // If the path is not excluded, is the filename excluded? + if (!unwanted) { + unwanted = selectiveSync.isFileNameExcluded(item.name); + } + // If path or filename does not exclude, is this excluded due to use of selective sync? if (!unwanted) { path = itemdb.computePath(item.driveId, item.id); unwanted = selectiveSync.isPathExcluded(path); From 7aad5903e2f84fc4f14b9fade8170006be0cac9f Mon Sep 17 00:00:00 2001 From: abraunegg Date: Mon, 25 Mar 2019 10:41:08 +1100 Subject: [PATCH 10/10] Update release files for 2.3.0 (#434) * Update release files for 2.3.0 --- CHANGELOG.md | 15 +++++++++++++++ Makefile | 2 +- ...nedrive-2.2.6.ebuild => onedrive-2.3.0.ebuild} | 0 onedrive.1.in | 2 +- pacman/PKGBUILD | 2 +- spec/onedrive.spec | 2 +- 6 files changed, 19 insertions(+), 4 deletions(-) rename contrib/gentoo/{onedrive-2.2.6.ebuild => onedrive-2.3.0.ebuild} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index b316a5fa..1b1cbe34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,21 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## 2.3.0 - 2019-03-25 +### Fixed +* Resolve application crash if no 'size' value is returned when uploading a new file +* Resolve application crash if a 5xx error is returned when uploading a new file +* Resolve not 'refreshing' version file when rebuilding +* Resolve unexpected application processing by preventing use of --synchronize & --monitor together +* Resolve high CPU usage when performing DB reads +* Update error logging around directory case-insensitive match +* Update Travis CI and ARM dependencies for LDC 1.14.0 +* Update Makefile due to build failure if building from release archive file +* Update logging as to why a OneDrive object was skipped + +### Added +* Implement config option 'skip_dir' + ## 2.2.6 - 2019-03-12 ### Fixed * Resolve application crash when unable to delete remote folders when business retention policies are enabled diff --git a/Makefile b/Makefile index 86d8e6d1..95ee09d6 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ DC ?= dmd -RELEASEVER = v2.2.6 +RELEASEVER = v2.3.0 pkgconfig := $(shell if [ $(PKGCONFIG) ] && [ "$(PKGCONFIG)" != 0 ] ; then echo 1 ; else echo "" ; fi) notifications := $(shell if [ $(NOTIFICATIONS) ] && [ "$(NOTIFICATIONS)" != 0 ] ; then echo 1 ; else echo "" ; fi) gitversion := $(shell if [ -f .git/HEAD ] ; then echo 1 ; else echo "" ; fi) diff --git a/contrib/gentoo/onedrive-2.2.6.ebuild b/contrib/gentoo/onedrive-2.3.0.ebuild similarity index 100% rename from contrib/gentoo/onedrive-2.2.6.ebuild rename to contrib/gentoo/onedrive-2.3.0.ebuild diff --git a/onedrive.1.in b/onedrive.1.in index d3fa2d74..7ac16253 100644 --- a/onedrive.1.in +++ b/onedrive.1.in @@ -1,4 +1,4 @@ -.TH ONEDRIVE "1" "March 2019" "2.2.6" "User Commands" +.TH ONEDRIVE "1" "March 2019" "2.3.0" "User Commands" .SH NAME onedrive \- folder synchronization with OneDrive .SH SYNOPSIS diff --git a/pacman/PKGBUILD b/pacman/PKGBUILD index a6ca008e..08fc48c9 100644 --- a/pacman/PKGBUILD +++ b/pacman/PKGBUILD @@ -1,5 +1,5 @@ pkgname=onedrive -pkgver=2.2.6 +pkgver=2.3.0 pkgrel=1 #patch-level (Increment this when patch is applied) pkgdesc="A free OneDrive Client for Linux. This is a fork of the https://github.com/skilion/onedrive repository" license=("unknown") diff --git a/spec/onedrive.spec b/spec/onedrive.spec index 1ab4b3fc..2def8266 100644 --- a/spec/onedrive.spec +++ b/spec/onedrive.spec @@ -6,7 +6,7 @@ %endif Name: onedrive -Version: 2.2.6 +Version: 2.3.0 Release: 1%{?dist} Summary: Microsoft OneDrive Client Group: System Environment/Network