deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-04-30 13:12:52 +02:00
Code Issues Projects Releases Wiki Activity

Issue #432 and #441 combined (#447)

Browse source 
* Re-add original #432 and #441 fixes to master
This commit is contained in:
abraunegg 2019-04-02 05:21:02 +11:00 committed by GitHub
parent abb82868e4
commit 6a6b8f128f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 126 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
README.md
View file

@ -249,6 +249,11 @@ skip_file = "= .*|~*"
```text
skip_file = "~*"
```
**Default valid configuration:**
```text
skip_file = "~*|.~*|*.tmp"
```
Do not use a skip_file entry of `.*` as this will prevent correct searching of local changes to process.
### Important - curl compatibility
@ -313,7 +318,7 @@ Config path = /home/alex/.config/onedrive
Config file found in config path = false
Config option 'sync_dir' = /home/alex/OneDrive
Config option 'skip_dir' =
Config option 'skip_file' = ~*
Config option 'skip_file' = ~*|.~*|*.tmp
Config option 'skip_dotfiles' = false
Config option 'skip_symlinks' = false
Config option 'monitor_interval' = 45
@ -487,6 +492,11 @@ Files can be skipped in the following fashion:
* Explicitly specify the filename and it's full path relative to your sync_dir, eg: 'path/to/file/filename.ext'
* Explicitly specify the filename only and skip every instance of this filename, eg: 'filename.ext'
By default, the following files will be skipped:
* Files that start with ~
* Files that start with .~ (like .~lock.* files generated by LibreOffice)
* Files that end in .tmp
**Note:** after changing `skip_file`, you must perform a full re-synchronization by adding `--resync` to your existing command line - for example: `onedrive --synchronize --resync`
**Note:** Do not use a skip_file entry of `.*` as this will prevent correct searching of local changes to process.

2
config
View file

@ -1,6 +1,6 @@
# Directory where the files will be synced
sync_dir = "~/OneDrive"
# Skip files and directories that match this pattern
skip_file = "~*"
skip_file = "~*|.~*|*.tmp"
# Wait time (seconds) between sync operations in monitor mode
monitor_interval = "45"

11
src/config.d
View file

@ -32,9 +32,11 @@ final class Config
setValue("sync_dir", "~/OneDrive");
// Skip Directories - no directories are skipped by default
setValue("skip_dir", "");
// Configure to skip ONLY temp files (~*.doc etc) by default
// Prior configuration was: .*|~*
setValue("skip_file", "~*");
// 'skilion' configuration was: .*|~*
// Skip files that start with ~
// Skip files that start with .~ (like .~lock.* files generated by LibreOffice)
// Skip files that end in .tmp
setValue("skip_file", "~*|.~*|*.tmp");
// By default skip dot files & folders are not skipped
setValue("skip_dotfiles", "false");
// By default symlinks are not skipped (using string type
@ -51,6 +53,9 @@ final class Config
setValue("min_notif_changes", "5");
// Frequency of log messages in monitor, ie after n sync runs ship out a log message
setValue("monitor_log_frequency", "5");
// Number of n sync runs before performing a full local scan of sync_dir
// By default 10 which means every ~7.5 minutes a full disk scan of sync_dir will occur
setValue("monitor_fullscan_frequency", "10");
// Check if we should ignore a directory if a special file (.nosync) is present
setValue("check_nosync", "false");

2
src/itemdb.d
View file

@ -31,7 +31,7 @@ struct Item {
final class ItemDatabase
{
// increment this for every change in the db schema
immutable int itemDatabaseVersion = 8;
immutable int itemDatabaseVersion = 9;
Database db;
string insertItemStmt;

37
src/main.d
View file

@ -557,7 +557,7 @@ int main(string[] args)
}
// Perform the sync
performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, LOG_NORMAL);
performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, LOG_NORMAL, true);
}
}
@ -621,26 +621,39 @@ int main(string[] args)
// monitor loop
immutable auto checkInterval = dur!"seconds"(to!long(cfg.getValue("monitor_interval")));
immutable auto logInterval = to!long(cfg.getValue("monitor_log_frequency"));
immutable auto fullScanFrequency = to!long(cfg.getValue("monitor_fullscan_frequency"));
auto lastCheckTime = MonoTime.currTime();
auto logMonitorCounter = 0;
auto fullScanCounter = 0;
bool fullScanRequired = true;
while (true) {
if (!downloadOnly) m.update(online);
auto currTime = MonoTime.currTime();
if (currTime - lastCheckTime > checkInterval) {
// log monitor output suppression
logMonitorCounter += 1;
if (logMonitorCounter > logInterval)
logMonitorCounter = 1;
// full scan of sync_dir
fullScanCounter += 1;
if (fullScanCounter > fullScanFrequency){
fullScanCounter = 1;
fullScanRequired = true;
}
// log.logAndNotify("DEBUG trying to create checkpoint");
// auto res = itemdb.db_checkpoint();
// log.logAndNotify("Checkpoint return: ", res);
// itemdb.dump_open_statements();
try {
if (!initSyncEngine(sync)) {
oneDrive.http.shutdown();
return EXIT_FAILURE;
}
try {
performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT));
performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired);
if (!downloadOnly) {
// discard all events that may have been generated by the sync
m.update(false);
@ -656,6 +669,7 @@ int main(string[] args)
log.log("Cannot initialize connection to OneDrive");
}
// performSync complete, set lastCheckTime to current time
fullScanRequired = false;
lastCheckTime = MonoTime.currTime();
GC.collect();
}
@ -701,7 +715,7 @@ bool initSyncEngine(SyncEngine sync)
}
// try to synchronize the folder three times
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel)
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired)
{
int count;
string remotePath = "/";
@ -760,12 +774,17 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// sync from OneDrive first before uploading files to OneDrive
if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ...");
sync.applyDifferences();
// is this a download only request?
if (!downloadOnly) {
// process local changes
sync.scanForDifferences(localPath);
// ensure that the current remote state is updated locally
sync.applyDifferences();
// Is a full scan of the entire sync_dir required?
if (fullScanRequired) {
// is this a download only request?
if (!downloadOnly) {
// process local changes walking the entire path checking for changes
// in monitor mode all local changes are captured via inotify
// thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required
sync.scanForDifferences(localPath);
// ensure that the current remote state is updated locally
sync.applyDifferences();
}
}
}
}

4
src/onedrive.d
View file

@ -265,7 +265,7 @@ final class OneDriveApi
// string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
if ((path == ".")||(path == "/")) url = driveUrl ~ "/root/";
else url = itemByPathUrl ~ encodeComponent(path) ~ ":/";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
return get(url);
}
@ -277,7 +277,7 @@ final class OneDriveApi
const(char)[] url;
// string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
return get(url);
}

105
src/sync.d
View file

@ -308,6 +308,7 @@ final class SyncEngine
void setDisableUploadValidation()
{
disableUploadValidation = true;
log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments");
}
@ -756,8 +757,11 @@ final class SyncEngine
if (unwanted) log.vdebug("Flagging as unwanted: find(item.parentId).length != 0");
// Check if this is a directory to skip
if (!unwanted) {
unwanted = selectiveSync.isDirNameExcluded(item.name);
if (unwanted) log.vlog("Skipping item - excluded by skip_dir config: ", item.name);
// Only check path if config is != ""
if (cfg.getValue("skip_dir") != "") {
unwanted = selectiveSync.isDirNameExcluded(item.name);
if (unwanted) log.vlog("Skipping item - excluded by skip_dir config: ", item.name);
}
}
// Check if this is a file to skip
if (!unwanted) {
@ -1360,28 +1364,45 @@ final class SyncEngine
writeln("done.");
}
} else {
// OneDrive Business Account - always use a session to upload
writeln("");
try {
response = session.upload(path, item.driveId, item.parentId, baseName(path));
} catch (OneDriveException e) {
// Resolve https://github.com/abraunegg/onedrive/issues/36
if ((e.httpStatusCode == 409) || (e.httpStatusCode == 423)) {
// The file is currently checked out or locked for editing by another user
// We cant upload this file at this time
writeln(" skipped.");
log.fileOnly("Uploading modified file ", path, " ... skipped.");
writeln("", path, " is currently checked out or locked for editing by another user.");
log.fileOnly(path, " is currently checked out or locked for editing by another user.");
return;
// OneDrive Business Account
// We need to always use a session to upload, but handle the changed file correctly
if (accountType == "business"){
// For logging consistency
writeln("");
try {
response = session.upload(path, item.driveId, item.parentId, baseName(path), item.eTag);
} catch (OneDriveException e) {
// Resolve https://github.com/abraunegg/onedrive/issues/36
if ((e.httpStatusCode == 409) || (e.httpStatusCode == 423)) {
// The file is currently checked out or locked for editing by another user
// We cant upload this file at this time
writeln(" skipped.");
log.fileOnly("Uploading modified file ", path, " ... skipped.");
writeln("", path, " is currently checked out or locked for editing by another user.");
log.fileOnly(path, " is currently checked out or locked for editing by another user.");
return;
}
// what is this error?????
else throw e;
}
// As the session.upload includes the last modified time, save the response
saveItem(response);
}
// OneDrive documentLibrary
if (accountType == "documentLibrary"){
// Due to https://github.com/OneDrive/onedrive-api-docs/issues/935 Microsoft modifies all PDF, MS Office & HTML files with added XML content. It is a 'feature' of SharePoint.
// This means, as a session upload, on 'completion' the file is 'moved' and generates a 404 ......
// Delete record from the local database - file will be uploaded as a new file
writeln(" skipped.");
log.fileOnly("Uploading modified file ", path, " ... skipped.");
log.vlog("Skip Reason: Microsoft Sharepoint 'enrichment' after upload issue");
log.vlog("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details");
itemdb.deleteById(item.driveId, item.id);
return;
}
// log line completion
writeln("done.");
// As the session.upload includes the last modified time, save the response
saveItem(response);
}
log.fileOnly("Uploading modified file ", path, " ... done.");
// use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded via simple upload
@ -1500,9 +1521,12 @@ final class SyncEngine
if (path != ".") {
if (isDir(path)) {
log.vdebug("Checking path: ", path);
if (selectiveSync.isDirNameExcluded(strip(path,"./"))) {
log.vlog("Skipping item - excluded by skip_dir config: ", path);
return;
// Only check path if config is != ""
if (cfg.getValue("skip_dir") != "") {
if (selectiveSync.isDirNameExcluded(strip(path,"./"))) {
log.vlog("Skipping item - excluded by skip_dir config: ", path);
return;
}
}
}
if (isFile(path)) {
@ -1886,11 +1910,31 @@ final class SyncEngine
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
} else {
// OneDrive Business account upload handling
writeln("");
response = session.upload(path, parent.driveId, parent.id, baseName(path));
writeln(" done.");
saveItem(response);
// OneDrive Business account modified file upload handling
if (accountType == "business"){
writeln("");
// session upload
response = session.upload(path, parent.driveId, parent.id, baseName(path), fileDetailsFromOneDrive["eTag"].str);
writeln(" done.");
saveItem(response);
}
// OneDrive SharePoint account modified file upload handling
if (accountType == "documentLibrary"){
// If this is a Microsoft SharePoint site, we need to remove the existing file before upload
onedrive.deleteById(fileDetailsFromOneDrive["parentReference"]["driveId"].str, fileDetailsFromOneDrive["id"].str, fileDetailsFromOneDrive["eTag"].str);
// simple upload
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
writeln(" done.");
saveItem(response);
// Due to https://github.com/OneDrive/onedrive-api-docs/issues/935 Microsoft modifies all PDF, MS Office & HTML files with added XML content. It is a 'feature' of SharePoint.
// So - now the 'local' and 'remote' file is technically DIFFERENT ... thanks Microsoft .. NO way to disable this stupidity
// Download the Microsoft 'modified' file so 'local' is now in sync
log.vlog("Due to Microsoft Sharepoint 'enrichment' of files, downloading 'enriched' file to ensure local file is in-sync");
log.vlog("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details");
auto fileSize = response["size"].integer;
onedrive.downloadById(response["parentReference"]["driveId"].str, response["id"].str, path, fileSize);
}
}
} else {
// we are --dry-run - simulate the file upload
@ -2021,10 +2065,11 @@ final class SyncEngine
// Takes a JSON input and formats to an item which can be used by the database
Item item = makeItem(jsonItem);
// Add to the local database
log.vdebug("Adding to database: ", item);
itemdb.upsert(item);
} else {
// log error
log.error("ERROR: OneDrive response missing required 'id' element:");
log.error("ERROR: OneDrive response missing required 'id' element");
log.error("ERROR: ", jsonItem);
}
} else {