deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-08 08:46:40 +02:00
Code Issues Projects Releases Wiki Activity

Resolve file creation loop when working directly in the synced folder with libreoffice (#442)

Browse source 
* Add debug line for flagging to disable upload validation checks
* Handle Microsoft 'enrichment' of certain documents when stored on Sharepoint
* Handle skip_dir checks when nothing to check against (false positive)
* Update default 'skip_file' to include tmp and lock files generated by LibreOffice
* Update code comments & logging output
* Update readme & default config file
* Update database version due to changing defaults of 'skip_file' which will force a rebuild and use of new skip_file default regex
This commit is contained in:
abraunegg 2019-04-02 04:00:22 +11:00 committed by GitHub
parent 4444b9cd8a
commit f38b13dd00
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 94 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
README.md
View file

@ -249,6 +249,11 @@ skip_file = "= .*|~*"
```text
skip_file = "~*"
```
**Default valid configuration:**
```text
skip_file = "~*|.~*|*.tmp"
```
Do not use a skip_file entry of `.*` as this will prevent correct searching of local changes to process.
### Important - curl compatibility
@ -313,7 +318,7 @@ Config path = /home/alex/.config/onedrive
Config file found in config path = false
Config option 'sync_dir' = /home/alex/OneDrive
Config option 'skip_dir' =
Config option 'skip_file' = ~*
Config option 'skip_file' = ~*|.~*|*.tmp
Config option 'skip_dotfiles' = false
Config option 'skip_symlinks' = false
Config option 'monitor_interval' = 45
@ -487,6 +492,11 @@ Files can be skipped in the following fashion:
* Explicitly specify the filename and it's full path relative to your sync_dir, eg: 'path/to/file/filename.ext'
* Explicitly specify the filename only and skip every instance of this filename, eg: 'filename.ext'
By default, the following files will be skipped:
* Files that start with ~
* Files that start with .~ (like .~lock.* files generated by LibreOffice)
* Files that end in .tmp
**Note:** after changing `skip_file`, you must perform a full re-synchronization by adding `--resync` to your existing command line - for example: `onedrive --synchronize --resync`
**Note:** Do not use a skip_file entry of `.*` as this will prevent correct searching of local changes to process.

2
config
View file

@ -1,6 +1,6 @@
# Directory where the files will be synced
sync_dir = "~/OneDrive"
# Skip files and directories that match this pattern
skip_file = "~*"
skip_file = "~*|.~*|*.tmp"
# Wait time (seconds) between sync operations in monitor mode
monitor_interval = "45"

2
src/config.d
View file

@ -29,7 +29,7 @@ final class Config
// default configuration
stringValues["single_directory"] = "";
stringValues["sync_dir"] = "~/OneDrive";
stringValues["skip_file"] = "~*";
stringValues["skip_file"] = "~*|.~*|*.tmp";
stringValues["skip_dir"] = "";
stringValues["log_dir"] = "/var/log/onedrive/";
stringValues["drive_id"] = "";

2
src/itemdb.d
View file

@ -31,7 +31,7 @@ struct Item {
final class ItemDatabase
{
// increment this for every change in the db schema
immutable int itemDatabaseVersion = 8;
immutable int itemDatabaseVersion = 9;
Database db;
string insertItemStmt;

8
src/onedrive.d
View file

@ -265,7 +265,7 @@ final class OneDriveApi
// string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
if ((path == ".")||(path == "/")) url = driveUrl ~ "/root/";
else url = itemByPathUrl ~ encodeComponent(path) ~ ":/";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
return get(url);
}
@ -277,7 +277,7 @@ final class OneDriveApi
const(char)[] url;
// string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
return get(url);
}
@ -730,8 +730,8 @@ final class OneDriveApi
case 400:
// Bad Request .. how should we act?
log.vlog("OneDrive returned a 'HTTP 400 - Bad Request' - gracefully handling error");
break;
break;
// 412 - Precondition Failed
case 412:
log.vlog("OneDrive returned a 'HTTP 412 - Precondition Failed' - gracefully handling error");

106
src/sync.d
View file

@ -308,6 +308,7 @@ final class SyncEngine
void setDisableUploadValidation()
{
disableUploadValidation = true;
log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments");
}
@ -756,8 +757,11 @@ final class SyncEngine
if (unwanted) log.vdebug("Flagging as unwanted: find(item.parentId).length != 0");
// Check if this is a directory to skip
if (!unwanted) {
unwanted = selectiveSync.isDirNameExcluded(item.name);
if (unwanted) log.vlog("Skipping item - excluded by skip_dir config: ", item.name);
// Only check path if config is != ""
if (cfg.getValueString("skip_dir") != "") {
unwanted = selectiveSync.isDirNameExcluded(item.name);
if (unwanted) log.vlog("Skipping item - excluded by skip_dir config: ", item.name);
}
}
// Check if this is a file to skip
if (!unwanted) {
@ -1360,28 +1364,45 @@ final class SyncEngine
writeln("done.");
}
} else {
// OneDrive Business Account - always use a session to upload
writeln("");
try {
response = session.upload(path, item.driveId, item.parentId, baseName(path));
} catch (OneDriveException e) {
// Resolve https://github.com/abraunegg/onedrive/issues/36
if ((e.httpStatusCode == 409) || (e.httpStatusCode == 423)) {
// The file is currently checked out or locked for editing by another user
// We cant upload this file at this time
writeln(" skipped.");
log.fileOnly("Uploading modified file ", path, " ... skipped.");
writeln("", path, " is currently checked out or locked for editing by another user.");
log.fileOnly(path, " is currently checked out or locked for editing by another user.");
return;
// OneDrive Business Account
// We need to always use a session to upload, but handle the changed file correctly
if (accountType == "business"){
// For logging consistency
writeln("");
try {
response = session.upload(path, item.driveId, item.parentId, baseName(path), item.eTag);
} catch (OneDriveException e) {
// Resolve https://github.com/abraunegg/onedrive/issues/36
if ((e.httpStatusCode == 409) || (e.httpStatusCode == 423)) {
// The file is currently checked out or locked for editing by another user
// We cant upload this file at this time
writeln("skipped.");
log.fileOnly("Uploading modified file ", path, " ... skipped.");
writeln("", path, " is currently checked out or locked for editing by another user.");
log.fileOnly(path, " is currently checked out or locked for editing by another user.");
return;
}
// what is this error?????
else throw e;
}
// As the session.upload includes the last modified time, save the response
saveItem(response);
}
// OneDrive documentLibrary
if (accountType == "documentLibrary"){
// Due to https://github.com/OneDrive/onedrive-api-docs/issues/935 Microsoft modifies all PDF, MS Office & HTML files with added XML content. It is a 'feature' of SharePoint.
// This means, as a session upload, on 'completion' the file is 'moved' and generates a 404 ......
// Delete record from the local database - file will be uploaded as a new file
writeln("skipped.");
log.fileOnly("Uploading modified file ", path, " ... skipped.");
log.vlog("Skip Reason: Microsoft Sharepoint 'enrichment' after upload issue");
log.vlog("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details");
itemdb.deleteById(item.driveId, item.id);
return;
}
// log line completion
writeln("done.");
// As the session.upload includes the last modified time, save the response
saveItem(response);
}
log.fileOnly("Uploading modified file ", path, " ... done.");
// use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded via simple upload
@ -1500,9 +1521,12 @@ final class SyncEngine
if (path != ".") {
if (isDir(path)) {
log.vdebug("Checking path: ", path);
if (selectiveSync.isDirNameExcluded(strip(path,"./"))) {
log.vlog("Skipping item - excluded by skip_dir config: ", path);
return;
// Only check path if config is != ""
if (cfg.getValueString("skip_dir") != "") {
if (selectiveSync.isDirNameExcluded(strip(path,"./"))) {
log.vlog("Skipping item - excluded by skip_dir config: ", path);
return;
}
}
}
if (isFile(path)) {
@ -1886,11 +1910,32 @@ final class SyncEngine
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
} else {
// OneDrive Business account upload handling
writeln("");
response = session.upload(path, parent.driveId, parent.id, baseName(path));
writeln(" done.");
saveItem(response);
// OneDrive Business account modified file upload handling
if (accountType == "business"){
writeln("");
// session upload
response = session.upload(path, parent.driveId, parent.id, baseName(path), fileDetailsFromOneDrive["eTag"].str);
writeln(" done.");
saveItem(response);
}
// OneDrive SharePoint account modified file upload handling
if (accountType == "documentLibrary"){
// If this is a Microsoft SharePoint site, we need to remove the existing file before upload
onedrive.deleteById(fileDetailsFromOneDrive["parentReference"]["driveId"].str, fileDetailsFromOneDrive["id"].str, fileDetailsFromOneDrive["eTag"].str);
// Due to https://github.com/OneDrive/onedrive-api-docs/issues/935 Microsoft modifies all PDF, MS Office & HTML files with added XML content. It is a 'feature' of SharePoint.
// This means, as a session upload, on 'completion' the file is 'moved' and generates a 404 ......
// Upload modified file via simpleUpload to avoid the session 404 problem
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
writeln(" done.");
saveItem(response);
// So - now the 'local' and 'remote' file is technically DIFFERENT ... thanks Microsoft .. NO way to disable this stupidity
// Download the Microsoft 'modified' file so 'local' is now in sync
log.vlog("Due to Microsoft Sharepoint 'enrichment' of files, downloading 'enriched' file to ensure local file is in-sync");
log.vlog("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details");
auto fileSize = response["size"].integer;
onedrive.downloadById(response["parentReference"]["driveId"].str, response["id"].str, path, fileSize);
}
}
} else {
// we are --dry-run - simulate the file upload
@ -2021,10 +2066,11 @@ final class SyncEngine
// Takes a JSON input and formats to an item which can be used by the database
Item item = makeItem(jsonItem);
// Add to the local database
log.vdebug("Adding to database: ", item);
itemdb.upsert(item);
} else {
// log error
log.error("ERROR: OneDrive response missing required 'id' element:");
log.error("ERROR: OneDrive response missing required 'id' element");
log.error("ERROR: ", jsonItem);
}
} else {