diff --git a/README.md b/README.md index 6c81e34c..a714cce3 100644 --- a/README.md +++ b/README.md @@ -249,6 +249,11 @@ skip_file = "= .*|~*" ```text skip_file = "~*" ``` +**Default valid configuration:** +```text +skip_file = "~*|.~*|*.tmp" +``` + Do not use a skip_file entry of `.*` as this will prevent correct searching of local changes to process. ### Important - curl compatibility @@ -313,7 +318,7 @@ Config path = /home/alex/.config/onedrive Config file found in config path = false Config option 'sync_dir' = /home/alex/OneDrive Config option 'skip_dir' = -Config option 'skip_file' = ~* +Config option 'skip_file' = ~*|.~*|*.tmp Config option 'skip_dotfiles' = false Config option 'skip_symlinks' = false Config option 'monitor_interval' = 45 @@ -487,6 +492,11 @@ Files can be skipped in the following fashion: * Explicitly specify the filename and it's full path relative to your sync_dir, eg: 'path/to/file/filename.ext' * Explicitly specify the filename only and skip every instance of this filename, eg: 'filename.ext' +By default, the following files will be skipped: +* Files that start with ~ +* Files that start with .~ (like .~lock.* files generated by LibreOffice) +* Files that end in .tmp + **Note:** after changing `skip_file`, you must perform a full re-synchronization by adding `--resync` to your existing command line - for example: `onedrive --synchronize --resync` **Note:** Do not use a skip_file entry of `.*` as this will prevent correct searching of local changes to process. diff --git a/config b/config index 79cc666b..3095f4f2 100644 --- a/config +++ b/config @@ -1,6 +1,6 @@ # Directory where the files will be synced sync_dir = "~/OneDrive" # Skip files and directories that match this pattern -skip_file = "~*" +skip_file = "~*|.~*|*.tmp" # Wait time (seconds) between sync operations in monitor mode monitor_interval = "45" diff --git a/src/config.d b/src/config.d index afc6d4ee..f0921850 100644 --- a/src/config.d +++ b/src/config.d @@ -29,7 +29,7 @@ final class Config // default configuration stringValues["single_directory"] = ""; stringValues["sync_dir"] = "~/OneDrive"; - stringValues["skip_file"] = "~*"; + stringValues["skip_file"] = "~*|.~*|*.tmp"; stringValues["skip_dir"] = ""; stringValues["log_dir"] = "/var/log/onedrive/"; stringValues["drive_id"] = ""; diff --git a/src/itemdb.d b/src/itemdb.d index fd8132eb..8b39f3d5 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -31,7 +31,7 @@ struct Item { final class ItemDatabase { // increment this for every change in the db schema - immutable int itemDatabaseVersion = 8; + immutable int itemDatabaseVersion = 9; Database db; string insertItemStmt; diff --git a/src/onedrive.d b/src/onedrive.d index b39bab86..97b0f39c 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -265,7 +265,7 @@ final class OneDriveApi // string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; if ((path == ".")||(path == "/")) url = driveUrl ~ "/root/"; else url = itemByPathUrl ~ encodeComponent(path) ~ ":/"; - url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; return get(url); } @@ -277,7 +277,7 @@ final class OneDriveApi const(char)[] url; // string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/"; url = driveByIdUrl ~ driveId ~ "/items/" ~ id; - url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; return get(url); } @@ -730,8 +730,8 @@ final class OneDriveApi case 400: // Bad Request .. how should we act? log.vlog("OneDrive returned a 'HTTP 400 - Bad Request' - gracefully handling error"); - break; - + break; + // 412 - Precondition Failed case 412: log.vlog("OneDrive returned a 'HTTP 412 - Precondition Failed' - gracefully handling error"); diff --git a/src/sync.d b/src/sync.d index af8bbad4..2465cf46 100644 --- a/src/sync.d +++ b/src/sync.d @@ -308,6 +308,7 @@ final class SyncEngine void setDisableUploadValidation() { disableUploadValidation = true; + log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments"); } @@ -756,8 +757,11 @@ final class SyncEngine if (unwanted) log.vdebug("Flagging as unwanted: find(item.parentId).length != 0"); // Check if this is a directory to skip if (!unwanted) { - unwanted = selectiveSync.isDirNameExcluded(item.name); - if (unwanted) log.vlog("Skipping item - excluded by skip_dir config: ", item.name); + // Only check path if config is != "" + if (cfg.getValueString("skip_dir") != "") { + unwanted = selectiveSync.isDirNameExcluded(item.name); + if (unwanted) log.vlog("Skipping item - excluded by skip_dir config: ", item.name); + } } // Check if this is a file to skip if (!unwanted) { @@ -1360,28 +1364,45 @@ final class SyncEngine writeln("done."); } } else { - // OneDrive Business Account - always use a session to upload - writeln(""); - - try { - response = session.upload(path, item.driveId, item.parentId, baseName(path)); - } catch (OneDriveException e) { - - // Resolve https://github.com/abraunegg/onedrive/issues/36 - if ((e.httpStatusCode == 409) || (e.httpStatusCode == 423)) { - // The file is currently checked out or locked for editing by another user - // We cant upload this file at this time - writeln(" skipped."); - log.fileOnly("Uploading modified file ", path, " ... skipped."); - writeln("", path, " is currently checked out or locked for editing by another user."); - log.fileOnly(path, " is currently checked out or locked for editing by another user."); - return; + // OneDrive Business Account + // We need to always use a session to upload, but handle the changed file correctly + if (accountType == "business"){ + // For logging consistency + writeln(""); + try { + response = session.upload(path, item.driveId, item.parentId, baseName(path), item.eTag); + } catch (OneDriveException e) { + // Resolve https://github.com/abraunegg/onedrive/issues/36 + if ((e.httpStatusCode == 409) || (e.httpStatusCode == 423)) { + // The file is currently checked out or locked for editing by another user + // We cant upload this file at this time + writeln("skipped."); + log.fileOnly("Uploading modified file ", path, " ... skipped."); + writeln("", path, " is currently checked out or locked for editing by another user."); + log.fileOnly(path, " is currently checked out or locked for editing by another user."); + return; + } + // what is this error????? + else throw e; } + // As the session.upload includes the last modified time, save the response + saveItem(response); } - + // OneDrive documentLibrary + if (accountType == "documentLibrary"){ + // Due to https://github.com/OneDrive/onedrive-api-docs/issues/935 Microsoft modifies all PDF, MS Office & HTML files with added XML content. It is a 'feature' of SharePoint. + // This means, as a session upload, on 'completion' the file is 'moved' and generates a 404 ...... + // Delete record from the local database - file will be uploaded as a new file + writeln("skipped."); + log.fileOnly("Uploading modified file ", path, " ... skipped."); + log.vlog("Skip Reason: Microsoft Sharepoint 'enrichment' after upload issue"); + log.vlog("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details"); + itemdb.deleteById(item.driveId, item.id); + return; + } + + // log line completion writeln("done."); - // As the session.upload includes the last modified time, save the response - saveItem(response); } log.fileOnly("Uploading modified file ", path, " ... done."); // use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded via simple upload @@ -1500,9 +1521,12 @@ final class SyncEngine if (path != ".") { if (isDir(path)) { log.vdebug("Checking path: ", path); - if (selectiveSync.isDirNameExcluded(strip(path,"./"))) { - log.vlog("Skipping item - excluded by skip_dir config: ", path); - return; + // Only check path if config is != "" + if (cfg.getValueString("skip_dir") != "") { + if (selectiveSync.isDirNameExcluded(strip(path,"./"))) { + log.vlog("Skipping item - excluded by skip_dir config: ", path); + return; + } } } if (isFile(path)) { @@ -1886,11 +1910,32 @@ final class SyncEngine // use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded uploadLastModifiedTime(parent.driveId, id, cTag, mtime); } else { - // OneDrive Business account upload handling - writeln(""); - response = session.upload(path, parent.driveId, parent.id, baseName(path)); - writeln(" done."); - saveItem(response); + // OneDrive Business account modified file upload handling + if (accountType == "business"){ + writeln(""); + // session upload + response = session.upload(path, parent.driveId, parent.id, baseName(path), fileDetailsFromOneDrive["eTag"].str); + writeln(" done."); + saveItem(response); + } + + // OneDrive SharePoint account modified file upload handling + if (accountType == "documentLibrary"){ + // If this is a Microsoft SharePoint site, we need to remove the existing file before upload + onedrive.deleteById(fileDetailsFromOneDrive["parentReference"]["driveId"].str, fileDetailsFromOneDrive["id"].str, fileDetailsFromOneDrive["eTag"].str); + // Due to https://github.com/OneDrive/onedrive-api-docs/issues/935 Microsoft modifies all PDF, MS Office & HTML files with added XML content. It is a 'feature' of SharePoint. + // This means, as a session upload, on 'completion' the file is 'moved' and generates a 404 ...... + // Upload modified file via simpleUpload to avoid the session 404 problem + response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path)); + writeln(" done."); + saveItem(response); + // So - now the 'local' and 'remote' file is technically DIFFERENT ... thanks Microsoft .. NO way to disable this stupidity + // Download the Microsoft 'modified' file so 'local' is now in sync + log.vlog("Due to Microsoft Sharepoint 'enrichment' of files, downloading 'enriched' file to ensure local file is in-sync"); + log.vlog("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details"); + auto fileSize = response["size"].integer; + onedrive.downloadById(response["parentReference"]["driveId"].str, response["id"].str, path, fileSize); + } } } else { // we are --dry-run - simulate the file upload @@ -2021,10 +2066,11 @@ final class SyncEngine // Takes a JSON input and formats to an item which can be used by the database Item item = makeItem(jsonItem); // Add to the local database + log.vdebug("Adding to database: ", item); itemdb.upsert(item); } else { // log error - log.error("ERROR: OneDrive response missing required 'id' element:"); + log.error("ERROR: OneDrive response missing required 'id' element"); log.error("ERROR: ", jsonItem); } } else {