deblan/abraunegg-onedrive
deblan
/
abraunegg-onedrive
mirror of https://github.com/abraunegg/onedrive
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update when DB is updated by OneDrive API data and update when file hash is required to be generated (#2004) 

* Only update the database if the item modified time is different, otherwise we are needlessly updating the database with data that is the same
* Generating a file hash, during the file integrity check is computationally expensive. Only generate a file hash if the modified time of the file is different, otherwise it is pointless to generate the file hash during each integrity check
* Only flush SHM and WAL post integrity check
This commit is contained in:
abraunegg 2022-06-15 09:16:06 +10:00 committed by GitHub
parent c7eabab27b
commit 1935def140
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 40 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/main.d
View File

@ -1485,25 +1485,25 @@ int main(string[] args)
log.log("Cannot initialize connection to OneDrive");
}
// performSync complete, set lastCheckTime to current time
fullScanRequired = false;
if (syncListConfigured) {
syncListConfiguredFullScanOverride = false;
}
lastCheckTime = MonoTime.currTime();
// Display memory details before cleanup
if (displayMemoryUsage) {
log.displayMemoryUsagePreGC();
}
if (displayMemoryUsage) log.displayMemoryUsagePreGC();
// Perform Garbage Cleanup
GC.collect();
// Display memory details after cleanup
if (displayMemoryUsage) {
log.displayMemoryUsagePostGC();
}
if (displayMemoryUsage) log.displayMemoryUsagePostGC();
// If we did a full scan, make sure we merge the conents of the WAL and SHM to disk
if (fullScanRequired) {
// Write WAL and SHM data to file for this loop
log.vdebug("Merge contents of WAL and SHM files into main database file");
itemDb.performVacuum();
}
// reset fullScanRequired and syncListConfiguredFullScanOverride
fullScanRequired = false;
if (syncListConfigured) syncListConfiguredFullScanOverride = false;
// monitor loop complete
logOutputMessage = "################################################ LOOP COMPLETE ###############################################";
@ -1838,7 +1838,7 @@ extern(C) nothrow @nogc @system void exitHandler(int value) {
// was itemDb initialised?
if (itemDb !is null) {
// Make sure the .wal file is incorporated into the main db before we exit
log.log("Shutting down db connection");
log.log("Shutting down db connection and merging temporary data");
itemDb.performVacuum();
destroy(itemDb);
}

23
src/sync.d
View File

@ -2437,6 +2437,7 @@ final class SyncEngine
// update the item
if (cached) {
// the item is in the items.sqlite3 database
log.vdebug("OneDrive change is an update to an existing local item");
applyChangedItem(oldItem, oldPath, item, path);
} else {
@ -2450,6 +2451,7 @@ final class SyncEngine
}
}
}
// apply this new item
applyNewItem(item, path);
}
@ -2458,10 +2460,22 @@ final class SyncEngine
// if the file was detected as malware and NOT downloaded, we dont want to falsify the DB as downloading it as otherwise the next pass will think it was deleted, thus delete the remote item
// Likewise if the download failed, we dont want to falsify the DB as downloading it as otherwise the next pass will think it was deleted, thus delete the remote item
if (cached) {
log.vdebug("Updating local database with item details");
// the item is in the items.sqlite3 database
// Do we need to update the database with the details that were provided by the OneDrive API?
// Is the last modified timestamp in the DB the same as the API data?
SysTime localModifiedTime = oldItem.mtime;
localModifiedTime.fracSecs = Duration.zero;
SysTime remoteModifiedTime = item.mtime;
remoteModifiedTime.fracSecs = Duration.zero;
if (localModifiedTime != remoteModifiedTime) {
// Database update needed for this item because our record is out-of-date
log.vdebug("Updating local database with item details as timestamps of items are different");
itemdb.update(item);
}
} else {
log.vdebug("Inserting item details to local database");
// item is not in the items.sqlite3 database
log.vdebug("Inserting new item details to local database");
itemdb.insert(item);
}
// What was the item that was saved
@ -3028,12 +3042,15 @@ final class SyncEngine
return true;
} else {
log.vlog("The local item has a different modified time ", localModifiedTime, " when compared to ", itemSource, " modified time ", itemModifiedTime);
}
// The file has been modified ... is the hash the same?
// Test the file hash as the date / time stamp is different
// Generating a hash is computationally expensive - only generate the hash if timestamp was modified
if (testFileHash(path, item)) {
return true;
} else {
log.vlog("The local item has a different hash when compared to ", itemSource, " item hash");
}
}
} else {
// Unable to read local file
log.log("Unable to determine the sync state of this file as it cannot be read (file permissions or file corruption): ", path);