mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-15 20:25:18 +02:00
parent
abb82868e4
commit
6a6b8f128f
12
README.md
12
README.md
|
@ -249,6 +249,11 @@ skip_file = "= .*|~*"
|
|||
```text
|
||||
skip_file = "~*"
|
||||
```
|
||||
**Default valid configuration:**
|
||||
```text
|
||||
skip_file = "~*|.~*|*.tmp"
|
||||
```
|
||||
|
||||
Do not use a skip_file entry of `.*` as this will prevent correct searching of local changes to process.
|
||||
|
||||
### Important - curl compatibility
|
||||
|
@ -313,7 +318,7 @@ Config path = /home/alex/.config/onedrive
|
|||
Config file found in config path = false
|
||||
Config option 'sync_dir' = /home/alex/OneDrive
|
||||
Config option 'skip_dir' =
|
||||
Config option 'skip_file' = ~*
|
||||
Config option 'skip_file' = ~*|.~*|*.tmp
|
||||
Config option 'skip_dotfiles' = false
|
||||
Config option 'skip_symlinks' = false
|
||||
Config option 'monitor_interval' = 45
|
||||
|
@ -487,6 +492,11 @@ Files can be skipped in the following fashion:
|
|||
* Explicitly specify the filename and it's full path relative to your sync_dir, eg: 'path/to/file/filename.ext'
|
||||
* Explicitly specify the filename only and skip every instance of this filename, eg: 'filename.ext'
|
||||
|
||||
By default, the following files will be skipped:
|
||||
* Files that start with ~
|
||||
* Files that start with .~ (like .~lock.* files generated by LibreOffice)
|
||||
* Files that end in .tmp
|
||||
|
||||
**Note:** after changing `skip_file`, you must perform a full re-synchronization by adding `--resync` to your existing command line - for example: `onedrive --synchronize --resync`
|
||||
|
||||
**Note:** Do not use a skip_file entry of `.*` as this will prevent correct searching of local changes to process.
|
||||
|
|
2
config
2
config
|
@ -1,6 +1,6 @@
|
|||
# Directory where the files will be synced
|
||||
sync_dir = "~/OneDrive"
|
||||
# Skip files and directories that match this pattern
|
||||
skip_file = "~*"
|
||||
skip_file = "~*|.~*|*.tmp"
|
||||
# Wait time (seconds) between sync operations in monitor mode
|
||||
monitor_interval = "45"
|
||||
|
|
11
src/config.d
11
src/config.d
|
@ -32,9 +32,11 @@ final class Config
|
|||
setValue("sync_dir", "~/OneDrive");
|
||||
// Skip Directories - no directories are skipped by default
|
||||
setValue("skip_dir", "");
|
||||
// Configure to skip ONLY temp files (~*.doc etc) by default
|
||||
// Prior configuration was: .*|~*
|
||||
setValue("skip_file", "~*");
|
||||
// 'skilion' configuration was: .*|~*
|
||||
// Skip files that start with ~
|
||||
// Skip files that start with .~ (like .~lock.* files generated by LibreOffice)
|
||||
// Skip files that end in .tmp
|
||||
setValue("skip_file", "~*|.~*|*.tmp");
|
||||
// By default skip dot files & folders are not skipped
|
||||
setValue("skip_dotfiles", "false");
|
||||
// By default symlinks are not skipped (using string type
|
||||
|
@ -51,6 +53,9 @@ final class Config
|
|||
setValue("min_notif_changes", "5");
|
||||
// Frequency of log messages in monitor, ie after n sync runs ship out a log message
|
||||
setValue("monitor_log_frequency", "5");
|
||||
// Number of n sync runs before performing a full local scan of sync_dir
|
||||
// By default 10 which means every ~7.5 minutes a full disk scan of sync_dir will occur
|
||||
setValue("monitor_fullscan_frequency", "10");
|
||||
// Check if we should ignore a directory if a special file (.nosync) is present
|
||||
setValue("check_nosync", "false");
|
||||
|
||||
|
|
|
@ -31,7 +31,7 @@ struct Item {
|
|||
final class ItemDatabase
|
||||
{
|
||||
// increment this for every change in the db schema
|
||||
immutable int itemDatabaseVersion = 8;
|
||||
immutable int itemDatabaseVersion = 9;
|
||||
|
||||
Database db;
|
||||
string insertItemStmt;
|
||||
|
|
27
src/main.d
27
src/main.d
|
@ -557,7 +557,7 @@ int main(string[] args)
|
|||
}
|
||||
|
||||
// Perform the sync
|
||||
performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, LOG_NORMAL);
|
||||
performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, LOG_NORMAL, true);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -621,26 +621,39 @@ int main(string[] args)
|
|||
// monitor loop
|
||||
immutable auto checkInterval = dur!"seconds"(to!long(cfg.getValue("monitor_interval")));
|
||||
immutable auto logInterval = to!long(cfg.getValue("monitor_log_frequency"));
|
||||
immutable auto fullScanFrequency = to!long(cfg.getValue("monitor_fullscan_frequency"));
|
||||
auto lastCheckTime = MonoTime.currTime();
|
||||
auto logMonitorCounter = 0;
|
||||
auto fullScanCounter = 0;
|
||||
bool fullScanRequired = true;
|
||||
while (true) {
|
||||
if (!downloadOnly) m.update(online);
|
||||
auto currTime = MonoTime.currTime();
|
||||
if (currTime - lastCheckTime > checkInterval) {
|
||||
// log monitor output suppression
|
||||
logMonitorCounter += 1;
|
||||
if (logMonitorCounter > logInterval)
|
||||
logMonitorCounter = 1;
|
||||
|
||||
// full scan of sync_dir
|
||||
fullScanCounter += 1;
|
||||
if (fullScanCounter > fullScanFrequency){
|
||||
fullScanCounter = 1;
|
||||
fullScanRequired = true;
|
||||
}
|
||||
|
||||
// log.logAndNotify("DEBUG trying to create checkpoint");
|
||||
// auto res = itemdb.db_checkpoint();
|
||||
// log.logAndNotify("Checkpoint return: ", res);
|
||||
// itemdb.dump_open_statements();
|
||||
|
||||
try {
|
||||
if (!initSyncEngine(sync)) {
|
||||
oneDrive.http.shutdown();
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
try {
|
||||
performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT));
|
||||
performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired);
|
||||
if (!downloadOnly) {
|
||||
// discard all events that may have been generated by the sync
|
||||
m.update(false);
|
||||
|
@ -656,6 +669,7 @@ int main(string[] args)
|
|||
log.log("Cannot initialize connection to OneDrive");
|
||||
}
|
||||
// performSync complete, set lastCheckTime to current time
|
||||
fullScanRequired = false;
|
||||
lastCheckTime = MonoTime.currTime();
|
||||
GC.collect();
|
||||
}
|
||||
|
@ -701,7 +715,7 @@ bool initSyncEngine(SyncEngine sync)
|
|||
}
|
||||
|
||||
// try to synchronize the folder three times
|
||||
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel)
|
||||
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired)
|
||||
{
|
||||
int count;
|
||||
string remotePath = "/";
|
||||
|
@ -760,9 +774,13 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
// sync from OneDrive first before uploading files to OneDrive
|
||||
if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ...");
|
||||
sync.applyDifferences();
|
||||
// Is a full scan of the entire sync_dir required?
|
||||
if (fullScanRequired) {
|
||||
// is this a download only request?
|
||||
if (!downloadOnly) {
|
||||
// process local changes
|
||||
// process local changes walking the entire path checking for changes
|
||||
// in monitor mode all local changes are captured via inotify
|
||||
// thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required
|
||||
sync.scanForDifferences(localPath);
|
||||
// ensure that the current remote state is updated locally
|
||||
sync.applyDifferences();
|
||||
|
@ -770,6 +788,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
count = -1;
|
||||
} catch (Exception e) {
|
||||
if (++count == 3) {
|
||||
|
|
|
@ -265,7 +265,7 @@ final class OneDriveApi
|
|||
// string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
|
||||
if ((path == ".")||(path == "/")) url = driveUrl ~ "/root/";
|
||||
else url = itemByPathUrl ~ encodeComponent(path) ~ ":/";
|
||||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
|
||||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
|
||||
return get(url);
|
||||
}
|
||||
|
||||
|
@ -277,7 +277,7 @@ final class OneDriveApi
|
|||
const(char)[] url;
|
||||
// string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
|
||||
url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
|
||||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
|
||||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
|
||||
return get(url);
|
||||
}
|
||||
|
||||
|
|
63
src/sync.d
63
src/sync.d
|
@ -308,6 +308,7 @@ final class SyncEngine
|
|||
void setDisableUploadValidation()
|
||||
{
|
||||
disableUploadValidation = true;
|
||||
log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments");
|
||||
}
|
||||
|
||||
|
||||
|
@ -756,9 +757,12 @@ final class SyncEngine
|
|||
if (unwanted) log.vdebug("Flagging as unwanted: find(item.parentId).length != 0");
|
||||
// Check if this is a directory to skip
|
||||
if (!unwanted) {
|
||||
// Only check path if config is != ""
|
||||
if (cfg.getValue("skip_dir") != "") {
|
||||
unwanted = selectiveSync.isDirNameExcluded(item.name);
|
||||
if (unwanted) log.vlog("Skipping item - excluded by skip_dir config: ", item.name);
|
||||
}
|
||||
}
|
||||
// Check if this is a file to skip
|
||||
if (!unwanted) {
|
||||
unwanted = selectiveSync.isFileNameExcluded(item.name);
|
||||
|
@ -1360,13 +1364,14 @@ final class SyncEngine
|
|||
writeln("done.");
|
||||
}
|
||||
} else {
|
||||
// OneDrive Business Account - always use a session to upload
|
||||
// OneDrive Business Account
|
||||
// We need to always use a session to upload, but handle the changed file correctly
|
||||
if (accountType == "business"){
|
||||
// For logging consistency
|
||||
writeln("");
|
||||
|
||||
try {
|
||||
response = session.upload(path, item.driveId, item.parentId, baseName(path));
|
||||
response = session.upload(path, item.driveId, item.parentId, baseName(path), item.eTag);
|
||||
} catch (OneDriveException e) {
|
||||
|
||||
// Resolve https://github.com/abraunegg/onedrive/issues/36
|
||||
if ((e.httpStatusCode == 409) || (e.httpStatusCode == 423)) {
|
||||
// The file is currently checked out or locked for editing by another user
|
||||
|
@ -1377,12 +1382,28 @@ final class SyncEngine
|
|||
log.fileOnly(path, " is currently checked out or locked for editing by another user.");
|
||||
return;
|
||||
}
|
||||
// what is this error?????
|
||||
else throw e;
|
||||
}
|
||||
|
||||
writeln("done.");
|
||||
// As the session.upload includes the last modified time, save the response
|
||||
saveItem(response);
|
||||
}
|
||||
// OneDrive documentLibrary
|
||||
if (accountType == "documentLibrary"){
|
||||
// Due to https://github.com/OneDrive/onedrive-api-docs/issues/935 Microsoft modifies all PDF, MS Office & HTML files with added XML content. It is a 'feature' of SharePoint.
|
||||
// This means, as a session upload, on 'completion' the file is 'moved' and generates a 404 ......
|
||||
// Delete record from the local database - file will be uploaded as a new file
|
||||
writeln(" skipped.");
|
||||
log.fileOnly("Uploading modified file ", path, " ... skipped.");
|
||||
log.vlog("Skip Reason: Microsoft Sharepoint 'enrichment' after upload issue");
|
||||
log.vlog("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details");
|
||||
itemdb.deleteById(item.driveId, item.id);
|
||||
return;
|
||||
}
|
||||
|
||||
// log line completion
|
||||
writeln("done.");
|
||||
}
|
||||
log.fileOnly("Uploading modified file ", path, " ... done.");
|
||||
// use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded via simple upload
|
||||
eTag = response["cTag"].str;
|
||||
|
@ -1500,11 +1521,14 @@ final class SyncEngine
|
|||
if (path != ".") {
|
||||
if (isDir(path)) {
|
||||
log.vdebug("Checking path: ", path);
|
||||
// Only check path if config is != ""
|
||||
if (cfg.getValue("skip_dir") != "") {
|
||||
if (selectiveSync.isDirNameExcluded(strip(path,"./"))) {
|
||||
log.vlog("Skipping item - excluded by skip_dir config: ", path);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (isFile(path)) {
|
||||
log.vdebug("Checking file: ", path);
|
||||
if (selectiveSync.isFileNameExcluded(strip(path,"./"))) {
|
||||
|
@ -1886,12 +1910,32 @@ final class SyncEngine
|
|||
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
|
||||
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
|
||||
} else {
|
||||
// OneDrive Business account upload handling
|
||||
// OneDrive Business account modified file upload handling
|
||||
if (accountType == "business"){
|
||||
writeln("");
|
||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||
// session upload
|
||||
response = session.upload(path, parent.driveId, parent.id, baseName(path), fileDetailsFromOneDrive["eTag"].str);
|
||||
writeln(" done.");
|
||||
saveItem(response);
|
||||
}
|
||||
|
||||
// OneDrive SharePoint account modified file upload handling
|
||||
if (accountType == "documentLibrary"){
|
||||
// If this is a Microsoft SharePoint site, we need to remove the existing file before upload
|
||||
onedrive.deleteById(fileDetailsFromOneDrive["parentReference"]["driveId"].str, fileDetailsFromOneDrive["id"].str, fileDetailsFromOneDrive["eTag"].str);
|
||||
// simple upload
|
||||
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
||||
writeln(" done.");
|
||||
saveItem(response);
|
||||
// Due to https://github.com/OneDrive/onedrive-api-docs/issues/935 Microsoft modifies all PDF, MS Office & HTML files with added XML content. It is a 'feature' of SharePoint.
|
||||
// So - now the 'local' and 'remote' file is technically DIFFERENT ... thanks Microsoft .. NO way to disable this stupidity
|
||||
// Download the Microsoft 'modified' file so 'local' is now in sync
|
||||
log.vlog("Due to Microsoft Sharepoint 'enrichment' of files, downloading 'enriched' file to ensure local file is in-sync");
|
||||
log.vlog("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details");
|
||||
auto fileSize = response["size"].integer;
|
||||
onedrive.downloadById(response["parentReference"]["driveId"].str, response["id"].str, path, fileSize);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// we are --dry-run - simulate the file upload
|
||||
writeln(" done.");
|
||||
|
@ -2021,10 +2065,11 @@ final class SyncEngine
|
|||
// Takes a JSON input and formats to an item which can be used by the database
|
||||
Item item = makeItem(jsonItem);
|
||||
// Add to the local database
|
||||
log.vdebug("Adding to database: ", item);
|
||||
itemdb.upsert(item);
|
||||
} else {
|
||||
// log error
|
||||
log.error("ERROR: OneDrive response missing required 'id' element:");
|
||||
log.error("ERROR: OneDrive response missing required 'id' element");
|
||||
log.error("ERROR: ", jsonItem);
|
||||
}
|
||||
} else {
|
||||
|
|
Loading…
Reference in a new issue