diff --git a/README.md b/README.md index 6c81e34c..a714cce3 100644 --- a/README.md +++ b/README.md @@ -249,6 +249,11 @@ skip_file = "= .*|~*" ```text skip_file = "~*" ``` +**Default valid configuration:** +```text +skip_file = "~*|.~*|*.tmp" +``` + Do not use a skip_file entry of `.*` as this will prevent correct searching of local changes to process. ### Important - curl compatibility @@ -313,7 +318,7 @@ Config path = /home/alex/.config/onedrive Config file found in config path = false Config option 'sync_dir' = /home/alex/OneDrive Config option 'skip_dir' = -Config option 'skip_file' = ~* +Config option 'skip_file' = ~*|.~*|*.tmp Config option 'skip_dotfiles' = false Config option 'skip_symlinks' = false Config option 'monitor_interval' = 45 @@ -487,6 +492,11 @@ Files can be skipped in the following fashion: * Explicitly specify the filename and it's full path relative to your sync_dir, eg: 'path/to/file/filename.ext' * Explicitly specify the filename only and skip every instance of this filename, eg: 'filename.ext' +By default, the following files will be skipped: +* Files that start with ~ +* Files that start with .~ (like .~lock.* files generated by LibreOffice) +* Files that end in .tmp + **Note:** after changing `skip_file`, you must perform a full re-synchronization by adding `--resync` to your existing command line - for example: `onedrive --synchronize --resync` **Note:** Do not use a skip_file entry of `.*` as this will prevent correct searching of local changes to process. diff --git a/config b/config index 79cc666b..3095f4f2 100644 --- a/config +++ b/config @@ -1,6 +1,6 @@ # Directory where the files will be synced sync_dir = "~/OneDrive" # Skip files and directories that match this pattern -skip_file = "~*" +skip_file = "~*|.~*|*.tmp" # Wait time (seconds) between sync operations in monitor mode monitor_interval = "45" diff --git a/src/config.d b/src/config.d index 5d07a86a..aff3c68e 100644 --- a/src/config.d +++ b/src/config.d @@ -32,9 +32,11 @@ final class Config setValue("sync_dir", "~/OneDrive"); // Skip Directories - no directories are skipped by default setValue("skip_dir", ""); - // Configure to skip ONLY temp files (~*.doc etc) by default - // Prior configuration was: .*|~* - setValue("skip_file", "~*"); + // 'skilion' configuration was: .*|~* + // Skip files that start with ~ + // Skip files that start with .~ (like .~lock.* files generated by LibreOffice) + // Skip files that end in .tmp + setValue("skip_file", "~*|.~*|*.tmp"); // By default skip dot files & folders are not skipped setValue("skip_dotfiles", "false"); // By default symlinks are not skipped (using string type @@ -51,6 +53,9 @@ final class Config setValue("min_notif_changes", "5"); // Frequency of log messages in monitor, ie after n sync runs ship out a log message setValue("monitor_log_frequency", "5"); + // Number of n sync runs before performing a full local scan of sync_dir + // By default 10 which means every ~7.5 minutes a full disk scan of sync_dir will occur + setValue("monitor_fullscan_frequency", "10"); // Check if we should ignore a directory if a special file (.nosync) is present setValue("check_nosync", "false"); diff --git a/src/itemdb.d b/src/itemdb.d index fd8132eb..8b39f3d5 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -31,7 +31,7 @@ struct Item { final class ItemDatabase { // increment this for every change in the db schema - immutable int itemDatabaseVersion = 8; + immutable int itemDatabaseVersion = 9; Database db; string insertItemStmt; diff --git a/src/main.d b/src/main.d index bb9627e7..32c1d733 100644 --- a/src/main.d +++ b/src/main.d @@ -557,7 +557,7 @@ int main(string[] args) } // Perform the sync - performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, LOG_NORMAL); + performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, LOG_NORMAL, true); } } @@ -621,26 +621,39 @@ int main(string[] args) // monitor loop immutable auto checkInterval = dur!"seconds"(to!long(cfg.getValue("monitor_interval"))); immutable auto logInterval = to!long(cfg.getValue("monitor_log_frequency")); + immutable auto fullScanFrequency = to!long(cfg.getValue("monitor_fullscan_frequency")); auto lastCheckTime = MonoTime.currTime(); auto logMonitorCounter = 0; + auto fullScanCounter = 0; + bool fullScanRequired = true; while (true) { if (!downloadOnly) m.update(online); auto currTime = MonoTime.currTime(); if (currTime - lastCheckTime > checkInterval) { + // log monitor output suppression logMonitorCounter += 1; if (logMonitorCounter > logInterval) logMonitorCounter = 1; + + // full scan of sync_dir + fullScanCounter += 1; + if (fullScanCounter > fullScanFrequency){ + fullScanCounter = 1; + fullScanRequired = true; + } + // log.logAndNotify("DEBUG trying to create checkpoint"); // auto res = itemdb.db_checkpoint(); // log.logAndNotify("Checkpoint return: ", res); // itemdb.dump_open_statements(); + try { if (!initSyncEngine(sync)) { oneDrive.http.shutdown(); return EXIT_FAILURE; } try { - performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT)); + performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired); if (!downloadOnly) { // discard all events that may have been generated by the sync m.update(false); @@ -656,6 +669,7 @@ int main(string[] args) log.log("Cannot initialize connection to OneDrive"); } // performSync complete, set lastCheckTime to current time + fullScanRequired = false; lastCheckTime = MonoTime.currTime(); GC.collect(); } @@ -701,7 +715,7 @@ bool initSyncEngine(SyncEngine sync) } // try to synchronize the folder three times -void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel) +void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired) { int count; string remotePath = "/"; @@ -760,12 +774,17 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo // sync from OneDrive first before uploading files to OneDrive if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ..."); sync.applyDifferences(); - // is this a download only request? - if (!downloadOnly) { - // process local changes - sync.scanForDifferences(localPath); - // ensure that the current remote state is updated locally - sync.applyDifferences(); + // Is a full scan of the entire sync_dir required? + if (fullScanRequired) { + // is this a download only request? + if (!downloadOnly) { + // process local changes walking the entire path checking for changes + // in monitor mode all local changes are captured via inotify + // thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required + sync.scanForDifferences(localPath); + // ensure that the current remote state is updated locally + sync.applyDifferences(); + } } } } diff --git a/src/onedrive.d b/src/onedrive.d index 847be0ec..784dc736 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -265,7 +265,7 @@ final class OneDriveApi // string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; if ((path == ".")||(path == "/")) url = driveUrl ~ "/root/"; else url = itemByPathUrl ~ encodeComponent(path) ~ ":/"; - url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; return get(url); } @@ -277,7 +277,7 @@ final class OneDriveApi const(char)[] url; // string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/"; url = driveByIdUrl ~ driveId ~ "/items/" ~ id; - url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; return get(url); } diff --git a/src/sync.d b/src/sync.d index 9ff2f383..4291d171 100644 --- a/src/sync.d +++ b/src/sync.d @@ -308,6 +308,7 @@ final class SyncEngine void setDisableUploadValidation() { disableUploadValidation = true; + log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments"); } @@ -756,8 +757,11 @@ final class SyncEngine if (unwanted) log.vdebug("Flagging as unwanted: find(item.parentId).length != 0"); // Check if this is a directory to skip if (!unwanted) { - unwanted = selectiveSync.isDirNameExcluded(item.name); - if (unwanted) log.vlog("Skipping item - excluded by skip_dir config: ", item.name); + // Only check path if config is != "" + if (cfg.getValue("skip_dir") != "") { + unwanted = selectiveSync.isDirNameExcluded(item.name); + if (unwanted) log.vlog("Skipping item - excluded by skip_dir config: ", item.name); + } } // Check if this is a file to skip if (!unwanted) { @@ -1360,28 +1364,45 @@ final class SyncEngine writeln("done."); } } else { - // OneDrive Business Account - always use a session to upload - writeln(""); - - try { - response = session.upload(path, item.driveId, item.parentId, baseName(path)); - } catch (OneDriveException e) { - - // Resolve https://github.com/abraunegg/onedrive/issues/36 - if ((e.httpStatusCode == 409) || (e.httpStatusCode == 423)) { - // The file is currently checked out or locked for editing by another user - // We cant upload this file at this time - writeln(" skipped."); - log.fileOnly("Uploading modified file ", path, " ... skipped."); - writeln("", path, " is currently checked out or locked for editing by another user."); - log.fileOnly(path, " is currently checked out or locked for editing by another user."); - return; + // OneDrive Business Account + // We need to always use a session to upload, but handle the changed file correctly + if (accountType == "business"){ + // For logging consistency + writeln(""); + try { + response = session.upload(path, item.driveId, item.parentId, baseName(path), item.eTag); + } catch (OneDriveException e) { + // Resolve https://github.com/abraunegg/onedrive/issues/36 + if ((e.httpStatusCode == 409) || (e.httpStatusCode == 423)) { + // The file is currently checked out or locked for editing by another user + // We cant upload this file at this time + writeln(" skipped."); + log.fileOnly("Uploading modified file ", path, " ... skipped."); + writeln("", path, " is currently checked out or locked for editing by another user."); + log.fileOnly(path, " is currently checked out or locked for editing by another user."); + return; + } + // what is this error????? + else throw e; } + // As the session.upload includes the last modified time, save the response + saveItem(response); } - + // OneDrive documentLibrary + if (accountType == "documentLibrary"){ + // Due to https://github.com/OneDrive/onedrive-api-docs/issues/935 Microsoft modifies all PDF, MS Office & HTML files with added XML content. It is a 'feature' of SharePoint. + // This means, as a session upload, on 'completion' the file is 'moved' and generates a 404 ...... + // Delete record from the local database - file will be uploaded as a new file + writeln(" skipped."); + log.fileOnly("Uploading modified file ", path, " ... skipped."); + log.vlog("Skip Reason: Microsoft Sharepoint 'enrichment' after upload issue"); + log.vlog("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details"); + itemdb.deleteById(item.driveId, item.id); + return; + } + + // log line completion writeln("done."); - // As the session.upload includes the last modified time, save the response - saveItem(response); } log.fileOnly("Uploading modified file ", path, " ... done."); // use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded via simple upload @@ -1500,9 +1521,12 @@ final class SyncEngine if (path != ".") { if (isDir(path)) { log.vdebug("Checking path: ", path); - if (selectiveSync.isDirNameExcluded(strip(path,"./"))) { - log.vlog("Skipping item - excluded by skip_dir config: ", path); - return; + // Only check path if config is != "" + if (cfg.getValue("skip_dir") != "") { + if (selectiveSync.isDirNameExcluded(strip(path,"./"))) { + log.vlog("Skipping item - excluded by skip_dir config: ", path); + return; + } } } if (isFile(path)) { @@ -1886,11 +1910,31 @@ final class SyncEngine // use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded uploadLastModifiedTime(parent.driveId, id, cTag, mtime); } else { - // OneDrive Business account upload handling - writeln(""); - response = session.upload(path, parent.driveId, parent.id, baseName(path)); - writeln(" done."); - saveItem(response); + // OneDrive Business account modified file upload handling + if (accountType == "business"){ + writeln(""); + // session upload + response = session.upload(path, parent.driveId, parent.id, baseName(path), fileDetailsFromOneDrive["eTag"].str); + writeln(" done."); + saveItem(response); + } + + // OneDrive SharePoint account modified file upload handling + if (accountType == "documentLibrary"){ + // If this is a Microsoft SharePoint site, we need to remove the existing file before upload + onedrive.deleteById(fileDetailsFromOneDrive["parentReference"]["driveId"].str, fileDetailsFromOneDrive["id"].str, fileDetailsFromOneDrive["eTag"].str); + // simple upload + response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path)); + writeln(" done."); + saveItem(response); + // Due to https://github.com/OneDrive/onedrive-api-docs/issues/935 Microsoft modifies all PDF, MS Office & HTML files with added XML content. It is a 'feature' of SharePoint. + // So - now the 'local' and 'remote' file is technically DIFFERENT ... thanks Microsoft .. NO way to disable this stupidity + // Download the Microsoft 'modified' file so 'local' is now in sync + log.vlog("Due to Microsoft Sharepoint 'enrichment' of files, downloading 'enriched' file to ensure local file is in-sync"); + log.vlog("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details"); + auto fileSize = response["size"].integer; + onedrive.downloadById(response["parentReference"]["driveId"].str, response["id"].str, path, fileSize); + } } } else { // we are --dry-run - simulate the file upload @@ -2021,10 +2065,11 @@ final class SyncEngine // Takes a JSON input and formats to an item which can be used by the database Item item = makeItem(jsonItem); // Add to the local database + log.vdebug("Adding to database: ", item); itemdb.upsert(item); } else { // log error - log.error("ERROR: OneDrive response missing required 'id' element:"); + log.error("ERROR: OneDrive response missing required 'id' element"); log.error("ERROR: ", jsonItem); } } else {