deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-02 06:02:53 +02:00
Code Issues Projects Releases Wiki Activity

Add performance output for key functions (#2180)

Browse source 
* Add developer option 'display_processing_time' to control if performance timing is outputted or not
* If option is enabled, print performance data around how long key functions are taking to process data to assist with understanding any performance related questions
This commit is contained in:
abraunegg 2022-10-12 13:34:42 +11:00 committed by GitHub
parent 4f15c35256
commit 7daf8797f8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 114 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/config.d
View file

@ -157,7 +157,11 @@ final class Config
// - Force client to use /children rather than /delta to query changes on OneDrive
// - This option flags nationalCloudDeployment as true, forcing the client to act like it is using a National Cloud Deployment
boolValues["force_children_scan"] = false;
// display_processing_time = true | false
// - Enabling this option will add function processing times to the console output
// - This then enables tracking of where the application is spending most amount of time when processing data when users have questions re performance
boolValues["display_processing_time"] = false;
// EXPAND USERS HOME DIRECTORY
// Determine the users home directory.
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts

51
src/main.d
View file

@ -6,6 +6,7 @@ import std.net.curl: CurlException;
import core.stdc.signal;
import std.traits, std.format;
import std.concurrency: receiveTimeout;
import std.datetime;
static import log;
OneDriveApi oneDrive;
@ -1192,9 +1193,15 @@ int main(string[] args)
// Are we forcing to use /children scan instead of /delta to simulate National Cloud Deployment use of /children?
if (cfg.getValueBool("force_children_scan")) {
log.vdebug("Forcing client to use /children scan rather than /delta to simulate National Cloud Deployment use of /children");
log.log("Forcing client to use /children scan rather than /delta to simulate National Cloud Deployment use of /children");
sync.setNationalCloudDeployment();
}
// Do we need to display the function processing timing
if (cfg.getValueBool("display_processing_time")) {
log.log("Forcing client to display function processing times");
sync.setPerformanceProcessingOutput();
}
// Do we need to validate the syncDir to check for the presence of a '.nosync' file
if (cfg.getValueBool("check_nomount")) {
@ -1547,12 +1554,20 @@ int main(string[] args)
return EXIT_FAILURE;
}
try {
// performance timing
SysTime startSyncProcessingTime = Clock.currTime();
string startMessage = "Starting a sync with OneDrive";
string finishMessage = "Sync with OneDrive is complete";
// perform a --monitor sync
if ((cfg.getValueLong("verbose") > 0) || (logMonitorCounter == logInterval) || (fullScanRequired) ) {
// log to console and log file if enabled
log.log(startMessage);
if (cfg.getValueBool("display_processing_time")) {
log.log(startMessage, " ", startSyncProcessingTime);
} else {
log.log(startMessage);
}
} else {
// log file only if enabled so we know when a sync started when not using --verbose
log.fileOnly(startMessage);
@ -1567,9 +1582,15 @@ int main(string[] args)
log.error("ERROR: The following inotify error was generated: ", e.msg);
}
}
SysTime endSyncProcessingTime = Clock.currTime();
if ((cfg.getValueLong("verbose") > 0) || (logMonitorCounter == logInterval) || (fullScanRequired) ) {
// log to console and log file if enabled
log.log(finishMessage);
if (cfg.getValueBool("display_processing_time")) {
log.log(finishMessage, " ", endSyncProcessingTime);
log.log("Elapsed Sync Time with OneDrive Service: ", (endSyncProcessingTime - startSyncProcessingTime));
} else {
log.log(finishMessage);
}
} else {
// log file only if enabled so we know when a sync completed when not using --verbose
log.fileOnly(finishMessage);
@ -1772,7 +1793,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
}
} else {
// sync from OneDrive first before uploading files to OneDrive
if ((logLevel < MONITOR_LOG_SILENT) || (fullScanRequired)) log.log("Syncing changes from OneDrive ...");
if ((logLevel < MONITOR_LOG_SILENT) || (fullScanRequired)) log.log("Syncing changes and items from OneDrive ...");
// For the initial sync, always use the delta link so that we capture all the right delta changes including adds, moves & deletes
logOutputMessage = "Initial Scan: Call OneDrive Delta API for delta changes as compared to last successful sync.";
@ -1807,13 +1828,21 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
SysTime startIntegrityCheckProcessingTime = Clock.currTime();
if (sync.getPerformanceProcessingOutput()) {
// performance timing for DB and file system integrity check - start
writeln("============================================================");
writeln("Start Integrity Check Processing Time: ", startIntegrityCheckProcessingTime);
}
// What sort of local scan do we want to do?
// In --monitor mode, when performing the DB scan, a race condition occurs where by if a file or folder is moved during this process
// the inotify event is discarded once performSync() is finished (see m.update(false) above), so these events need to be handled
// This can be remediated by breaking the DB and file system scan into separate processes, and handing any applicable inotify events in between
if (!monitorEnabled) {
// --synchronize in use
log.log("Performing a database consistency and integrity check on locally stored data ... ");
// standard process flow
sync.scanForDifferences(localPath);
} else {
@ -1829,7 +1858,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
//
// To change this behaviour adjust 'monitor_interval' and 'monitor_fullscan_frequency' to desired values in the application config file
if (fullScanRequired) {
log.log("Performing a database consistency and integrity check on locally stored data ... ");
log.log("Performing a database consistency and integrity check on locally stored data due to fullscan requirement ... ");
sync.scanForDifferencesDatabaseScan(localPath);
// handle any inotify events that occured 'whilst' we were scanning the database
m.update(true);
@ -1844,7 +1873,15 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// handle any inotify events that occured 'whilst' we were scanning the local filesystem
m.update(true);
}
SysTime endIntegrityCheckProcessingTime = Clock.currTime();
if (sync.getPerformanceProcessingOutput()) {
// performance timing for DB and file system integrity check - finish
writeln("End Integrity Check Processing Time: ", endIntegrityCheckProcessingTime);
writeln("Elapsed Function Processing Time: ", (endIntegrityCheckProcessingTime - startIntegrityCheckProcessingTime));
writeln("============================================================");
}
// At this point, all OneDrive changes / local changes should be uploaded and in sync
// This MAY not be the case when using sync_list, thus a full walk of OneDrive ojects is required

66
src/sync.d
View file

@ -270,6 +270,8 @@ final class SyncEngine
private bool bypassDataPreservation = false;
// is National Cloud Deployments configured
private bool nationalCloudDeployment = false;
// has performance processing timings been requested
private bool displayProcessingTime = false;
// array of all OneDrive driveId's for use with OneDrive Business Folders
private string[] driveIDsArray;
@ -619,6 +621,19 @@ final class SyncEngine
log.vdebug("Setting nationalCloudDeployment = true");
}
// set performance timing flag
void setPerformanceProcessingOutput()
{
displayProcessingTime = true;
log.vdebug("Setting displayProcessingTime = true");
}
// get performance timing flag
bool getPerformanceProcessingOutput()
{
return displayProcessingTime;
}
// set cleanupLocalFiles to true
void setCleanupLocalFiles()
{
@ -1213,6 +1228,20 @@ final class SyncEngine
string deltaLinkAvailable;
bool nationalCloudChildrenScan = false;
// Tracking processing performance
SysTime startFunctionProcessingTime;
SysTime endFunctionProcessingTime;
SysTime startBundleProcessingTime;
SysTime endBundleProcessingTime;
ulong cumulativeOneDriveItemCount = 0;
if (displayProcessingTime) {
writeln("============================================================");
writeln("Querying OneDrive API for relevant 'changes|items' stored online for this account");
startFunctionProcessingTime = Clock.currTime();
writeln("Start Function Processing Time: ", startFunctionProcessingTime);
}
// Update the quota details for this driveId, as this could have changed since we started the application - the user could have added / deleted data online, or purchased additional storage
// Quota details are ONLY available for the main default driveId, as the OneDrive API does not provide quota details for shared folders
try {
@ -1479,6 +1508,13 @@ final class SyncEngine
}
for (;;) {
if (displayProcessingTime) {
writeln("------------------------------------------------------------");
startBundleProcessingTime = Clock.currTime();
writeln("Start 'change|item' API Response Bundle Processing Time: ", startBundleProcessingTime);
}
// Due to differences in OneDrive API's between personal and business we need to get changes only from defaultRootId
// If we used the 'id' passed in & when using --single-directory with a business account we get:
// 'HTTP request returned status code 501 (Not Implemented): view.delta can only be called on the root.'
@ -1846,6 +1882,11 @@ final class SyncEngine
// OneDrive ships 'changes' in ~200 bundles. We display that we are processing X number of objects
// Do not display anything unless we are doing a verbose debug as due to #658 we are essentially doing a --resync each time when using sync_list
// performance logging output
if (displayProcessingTime) {
writeln("Number of 'change|item' in this API Response Bundle from OneDrive to process: ", nrChanges);
}
// is nrChanges >= min_notify_changes (default of min_notify_changes = 5)
if (nrChanges >= cfg.getValueLong("min_notify_changes")) {
// nrChanges is >= than min_notify_changes
@ -1895,7 +1936,10 @@ final class SyncEngine
log.vlog("Number of items from OneDrive to process: ", nrChanges);
}
}
// Add nrChanges to cumulativeOneDriveItemCount so we can detail how may items in total were processed
cumulativeOneDriveItemCount = cumulativeOneDriveItemCount + nrChanges;
foreach (item; changes["value"].array) {
bool isRoot = false;
string thisItemParentPath;
@ -2118,6 +2162,14 @@ final class SyncEngine
log.vdebug("Updating completed deltaLink in DB to: ", deltaLink);
itemdb.setDeltaLink(driveId, id, deltaLink);
}
// Processing Timing for this bundle
if (displayProcessingTime) {
endBundleProcessingTime = Clock.currTime();
writeln("End 'change|item' API Response Bundle Processing Time: ", endBundleProcessingTime);
writeln("Elapsed Processing Time: ", (endBundleProcessingTime - startBundleProcessingTime));
}
if ("@odata.nextLink" in changes) {
// Update deltaLink to next changeSet bundle
deltaLink = changes["@odata.nextLink"].str;
@ -2141,6 +2193,18 @@ final class SyncEngine
// empty the skipped items
skippedItems.length = 0;
assumeSafeAppend(skippedItems);
// Processing timing and metrics for everything that was processed
if (displayProcessingTime) {
endFunctionProcessingTime = Clock.currTime();
// complete the bundle output
writeln("------------------------------------------------------------");
writeln("Start Function Processing Time: ", startFunctionProcessingTime);
writeln("End Function Processing Time: ", endFunctionProcessingTime);
writeln("Elapsed Function Processing Time: ", (endFunctionProcessingTime - startFunctionProcessingTime));
writeln("Total number of OneDrive items processed: ", cumulativeOneDriveItemCount);
writeln("============================================================");
}
}
// process the change of a single DriveItem