mirror of
https://github.com/abraunegg/onedrive
synced 2024-05-17 05:06:55 +02:00
Update PR
* Rework entire #2608 due to errors, memory leaks and handling * Fix 429 'retry-after' header searching to use actual value, not 120 all the time * General cleanup of code from #2608
This commit is contained in:
abraunegg
parent
23c0326822
commit
e3b2ec4ae1
|
|
@ -683,6 +683,7 @@ class ApplicationConfig {
|
|||
|
||||
auto file = File(filename, "r");
|
||||
string lineBuffer;
|
||||
|
||||
scope(exit) {
|
||||
file.close();
|
||||
object.destroy(file);
|
||||
|
|
|
|||
237
src/curlEngine.d
237
src/curlEngine.d
|
|
@ -3,7 +3,7 @@ module curlEngine;
|
|||
|
||||
// What does this module require to function?
|
||||
import std.net.curl;
|
||||
import etc.c.curl: CurlOption;
|
||||
import etc.c.curl;
|
||||
import std.datetime;
|
||||
import std.conv;
|
||||
import std.file;
|
||||
|
|
@ -31,18 +31,17 @@ class CurlResponse {
|
|||
|
||||
void reset() {
|
||||
method = HTTP.Method.undefined;
|
||||
url = null;
|
||||
url = "";
|
||||
requestHeaders = null;
|
||||
postBody = null;
|
||||
|
||||
postBody = [];
|
||||
hasResponse = false;
|
||||
responseHeaders = null;
|
||||
object.destroy(statusLine);
|
||||
content = null;
|
||||
statusLine.reset();
|
||||
content = [];
|
||||
}
|
||||
|
||||
void addRequestHeader(const(char)[] name, const(char)[] value) {
|
||||
requestHeaders[name] = value;
|
||||
requestHeaders[to!string(name)] = to!string(value);
|
||||
}
|
||||
|
||||
void connect(HTTP.Method method, const(char)[] url) {
|
||||
|
|
@ -74,7 +73,7 @@ class CurlResponse {
|
|||
// Return the current value of retryAfterValue
|
||||
int getRetryAfterValue() {
|
||||
int delayBeforeRetry;
|
||||
// is retry-after in the response headers
|
||||
// Is 'retry-after' in the response headers
|
||||
if ("retry-after" in responseHeaders) {
|
||||
// Set the retry-after value
|
||||
addLogEntry("curlEngine.http.perform() => Received a 'Retry-After' Header Response with the following value: " ~ to!string(responseHeaders["retry-after"]), ["debug"]);
|
||||
|
|
@ -85,35 +84,34 @@ class CurlResponse {
|
|||
// This value is based on log files and data when determining correct process for 429 response handling
|
||||
delayBeforeRetry = 120;
|
||||
// Update that we are over-riding the provided value with a default
|
||||
addLogEntry("HTTP Response Header retry-after value was 0 - Using a preconfigured default of: " ~ to!string(delayBeforeRetry), ["debug"]);
|
||||
addLogEntry("HTTP Response Header retry-after value was missing - Using a preconfigured default of: " ~ to!string(delayBeforeRetry), ["debug"]);
|
||||
}
|
||||
|
||||
return delayBeforeRetry;
|
||||
}
|
||||
|
||||
const string parseHeaders(const(immutable(char)[][immutable(char)[]]) headers) {
|
||||
string responseHeadersStr = "";
|
||||
// Ensure headers is not null and iterate over keys safely.
|
||||
if (headers !is null) {
|
||||
foreach (const(char)[] header; headers.byKey()) {
|
||||
// Check if the key actually exists before accessing it to avoid RangeError.
|
||||
if (auto val = header in headers) { // 'in' checks for the key and returns a pointer to the value if found.
|
||||
responseHeadersStr ~= "> " ~ header ~ ": " ~ *val ~ "\n"; // Dereference pointer to get the value.
|
||||
}
|
||||
}
|
||||
}
|
||||
return responseHeadersStr;
|
||||
}
|
||||
|
||||
const string parseHeaders(const(const(char)[][const(char)[]]) headers) {
|
||||
string responseHeadersStr = "";
|
||||
|
||||
const string parseRequestHeaders(const(const(char)[][const(char)[]]) headers) {
|
||||
string requestHeadersStr = "";
|
||||
foreach (string header; headers.byKey()) {
|
||||
if (header == "Authorization") {
|
||||
continue;
|
||||
}
|
||||
// Use the 'in' operator to safely check if the key exists in the associative array.
|
||||
if (auto val = header in headers) {
|
||||
responseHeadersStr ~= "< " ~ header ~ ": " ~ *val ~ "\n";
|
||||
requestHeadersStr ~= "< " ~ header ~ ": " ~ *val ~ "\n";
|
||||
}
|
||||
}
|
||||
return requestHeadersStr;
|
||||
}
|
||||
|
||||
const string parseResponseHeaders(const(immutable(char)[][immutable(char)[]]) headers) {
|
||||
string responseHeadersStr = "";
|
||||
// Ensure response headers is not null and iterate over keys safely.
|
||||
if (headers !is null) {
|
||||
foreach (const(char)[] header; headers.byKey()) {
|
||||
// Check if the key actually exists before accessing it to avoid RangeError.
|
||||
if (auto val = header in headers) { // 'in' checks for the key and returns a pointer to the value if found.
|
||||
responseHeadersStr ~= "> " ~ header ~ ": " ~ *val ~ "\n"; // Dereference pointer to get the value.
|
||||
}
|
||||
}
|
||||
}
|
||||
return responseHeadersStr;
|
||||
|
|
@ -126,14 +124,14 @@ class CurlResponse {
|
|||
string str = "";
|
||||
str ~= format("< %s %s\n", method, url);
|
||||
if (!requestHeaders.empty) {
|
||||
str ~= parseHeaders(requestHeaders);
|
||||
str ~= parseRequestHeaders(requestHeaders);
|
||||
}
|
||||
if (!postBody.empty) {
|
||||
str ~= format("----\n%s\n----\n", postBody);
|
||||
str ~= format("\n----\n%s\n----\n", postBody);
|
||||
}
|
||||
str ~= format("< %s\n", statusLine);
|
||||
if (!responseHeaders.empty) {
|
||||
str ~= parseHeaders(responseHeaders);
|
||||
str ~= parseResponseHeaders(responseHeaders);
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
|
@ -144,7 +142,7 @@ class CurlResponse {
|
|||
|
||||
string str = "";
|
||||
if (!content.empty) {
|
||||
str ~= format("----\n%s\n----\n", content);
|
||||
str ~= format("\n----\n%s\n----\n", content);
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
|
@ -158,75 +156,92 @@ class CurlResponse {
|
|||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
CurlResponse dup() {
|
||||
CurlResponse copy = new CurlResponse();
|
||||
copy.method = method;
|
||||
copy.url = url;
|
||||
copy.requestHeaders = requestHeaders;
|
||||
copy.postBody = postBody;
|
||||
|
||||
copy.responseHeaders = responseHeaders;
|
||||
copy.statusLine = statusLine;
|
||||
copy.content = content;
|
||||
|
||||
return copy;
|
||||
}
|
||||
}
|
||||
|
||||
class CurlEngine {
|
||||
|
||||
__gshared static CurlEngine[] curlEnginePool; // __gshared is used for thread-shared static variables
|
||||
__gshared static CurlEngine[] curlEnginePool; // __gshared is used for thread-shared static variables
|
||||
|
||||
HTTP http;
|
||||
bool keepAlive;
|
||||
ulong dnsTimeout;
|
||||
CurlResponse response;
|
||||
File uploadFile;
|
||||
|
||||
static CurlEngine getCurlInstance() {
|
||||
synchronized (CurlEngine.classinfo) {
|
||||
if (curlEnginePool.empty) {
|
||||
return new CurlEngine; // Constructs a new CurlEngine with a fresh HTTP instance
|
||||
} else {
|
||||
CurlEngine curlEngine = curlEnginePool[$ - 1];
|
||||
curlEnginePool = curlEnginePool[0 .. $ - 1];
|
||||
return curlEngine;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void releaseAll() {
|
||||
synchronized (CurlEngine.classinfo) {
|
||||
foreach (CurlEngine curlEngine; curlEnginePool) {
|
||||
curlEngine.cleanUp(); // Cleanup instance by resetting values
|
||||
curlEngine.shutdown(); // Assume proper cleanup of any resources used by HTTP
|
||||
object.destroy(curlEngine);
|
||||
}
|
||||
curlEnginePool.length = 0;
|
||||
}
|
||||
|
||||
// Cleanup curlEnginePool
|
||||
object.destroy(curlEnginePool);
|
||||
}
|
||||
|
||||
HTTP http;
|
||||
bool keepAlive;
|
||||
ulong dnsTimeout;
|
||||
CurlResponse response;
|
||||
File uploadFile;
|
||||
|
||||
this() {
|
||||
http = HTTP(); // Directly initializes HTTP using its default constructor
|
||||
response = null; // Initialize as null
|
||||
this.http = HTTP(); // Directly initializes HTTP using its default constructor
|
||||
this.response = null; // Initialize as null
|
||||
}
|
||||
|
||||
~this() {
|
||||
// The destructor should only clean up resources owned directly by this instance
|
||||
// Avoid modifying or destroying shared/static resources here
|
||||
if (uploadFile.isOpen())
|
||||
uploadFile.close();
|
||||
// The destructor should only clean up resources owned directly by this instance
|
||||
// Avoid modifying or destroying shared/static resources here
|
||||
if (uploadFile.isOpen()) {
|
||||
uploadFile.close();
|
||||
}
|
||||
|
||||
// Cleanup curlEnginePool
|
||||
object.destroy(curlEnginePool);
|
||||
// Cleanup class memory usage
|
||||
object.destroy(this.uploadFile); // Destroy, however we cant set to null
|
||||
object.destroy(this.response); // Destroy, then set to null
|
||||
this.response = null;
|
||||
|
||||
// Is the http instance is stopped?
|
||||
if (!this.http.isStopped) {
|
||||
|
||||
writeln("TO REMOVE: Calling this.http.shutdown() on this curl instance");
|
||||
|
||||
this.http.shutdown();
|
||||
}
|
||||
|
||||
object.destroy(this.http); // Destroy, however we cant set to null
|
||||
|
||||
}
|
||||
|
||||
|
||||
static CurlEngine getCurlInstance() {
|
||||
synchronized (CurlEngine.classinfo) {
|
||||
if (curlEnginePool.empty) {
|
||||
return new CurlEngine; // Constructs a new CurlEngine with a fresh HTTP instance
|
||||
} else {
|
||||
CurlEngine curlEngine = curlEnginePool[$ - 1];
|
||||
curlEnginePool = curlEnginePool[0 .. $ - 1];
|
||||
|
||||
// Is this engine stopped?
|
||||
if (curlEngine.http.isStopped) {
|
||||
// return a new curl engine as a stopped one cannot be used
|
||||
return new CurlEngine; // Constructs a new CurlEngine with a fresh HTTP instance
|
||||
} else {
|
||||
// return an existing curl engine
|
||||
return curlEngine;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void releaseAllCurlInstances() {
|
||||
synchronized (CurlEngine.classinfo) {
|
||||
// Safely iterate and clean up each CurlEngine instance
|
||||
foreach (CurlEngine curlEngine; curlEnginePool) {
|
||||
try {
|
||||
curlEngine.cleanup(); // Cleanup instance by resetting values
|
||||
curlEngine.shutdown(); // Assume proper cleanup of any resources used by HTTP
|
||||
} catch (Exception e) {
|
||||
// Log the error or handle it appropriately
|
||||
// e.g., writeln("Error during cleanup/shutdown: ", e.toString());
|
||||
}
|
||||
// It's safe to destroy the object here assuming no other references exist
|
||||
object.destroy(curlEngine); // Destroy, then set to null
|
||||
curlEngine = null;
|
||||
}
|
||||
// Clear the array after all instances have been handled
|
||||
curlEnginePool.length = 0; // More explicit than curlEnginePool = [];
|
||||
}
|
||||
// Destroy curlEnginePool, set to null
|
||||
object.destroy(curlEnginePool);
|
||||
curlEnginePool = null;
|
||||
}
|
||||
|
||||
void release() {
|
||||
cleanUp();
|
||||
cleanup();
|
||||
synchronized (CurlEngine.classinfo) {
|
||||
curlEnginePool ~= this;
|
||||
}
|
||||
|
|
@ -381,7 +396,7 @@ class CurlEngine {
|
|||
|
||||
CurlResponse execute() {
|
||||
scope(exit) {
|
||||
cleanUp();
|
||||
cleanup();
|
||||
}
|
||||
setResponseHolder(null);
|
||||
http.onReceive = (ubyte[] data) {
|
||||
|
|
@ -402,7 +417,7 @@ class CurlEngine {
|
|||
|
||||
// function scopes
|
||||
scope(exit) {
|
||||
cleanUp();
|
||||
cleanup();
|
||||
if (file.isOpen()){
|
||||
// close open file
|
||||
file.close();
|
||||
|
|
@ -423,17 +438,27 @@ class CurlEngine {
|
|||
return response;
|
||||
}
|
||||
|
||||
void cleanUp() {
|
||||
void cleanup() {
|
||||
// Reset any values to defaults, freeing any set objects
|
||||
http.clearRequestHeaders();
|
||||
http.onSend = null;
|
||||
http.onReceive = null;
|
||||
http.onReceiveHeader = null;
|
||||
http.onReceiveStatusLine = null;
|
||||
http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) {
|
||||
return 0;
|
||||
};
|
||||
http.contentLength = 0;
|
||||
|
||||
// Is the instance is stopped?
|
||||
if (!http.isStopped) {
|
||||
// A stopped instance is not usable, these cannot be reset
|
||||
http.clearRequestHeaders();
|
||||
http.onSend = null;
|
||||
http.onReceive = null;
|
||||
http.onReceiveHeader = null;
|
||||
http.onReceiveStatusLine = null;
|
||||
http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) {
|
||||
return 0;
|
||||
};
|
||||
http.contentLength = 0;
|
||||
http.flushCookieJar();
|
||||
http.clearSessionCookies();
|
||||
http.clearAllCookies();
|
||||
}
|
||||
|
||||
// set the response to null
|
||||
response = null;
|
||||
|
||||
// close file if open
|
||||
|
|
@ -445,11 +470,9 @@ class CurlEngine {
|
|||
|
||||
void shutdown() {
|
||||
// Shut down the curl instance & close any open sockets
|
||||
http.shutdown();
|
||||
}
|
||||
|
||||
void setDisableSSLVerifyPeer() {
|
||||
addLogEntry("CAUTION: Switching off CurlOption.ssl_verifypeer ... this makes the application insecure.", ["debug"]);
|
||||
http.handle.set(CurlOption.ssl_verifypeer, 0);
|
||||
// Is the instance is stopped?
|
||||
if (!http.isStopped) {
|
||||
http.shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -52,7 +52,13 @@ class LogBuffer {
|
|||
flushThread.isDaemon(true);
|
||||
flushThread.start();
|
||||
}
|
||||
|
||||
|
||||
~this() {
|
||||
object.destroy(bufferLock);
|
||||
object.destroy(condReady);
|
||||
object.destroy(flushThread);
|
||||
}
|
||||
|
||||
void shutdown() {
|
||||
synchronized(bufferLock) {
|
||||
if (!isRunning) return; // Prevent multiple shutdowns
|
||||
|
|
@ -62,6 +68,7 @@ class LogBuffer {
|
|||
flushThread.join(); // Wait for the flush thread to finish
|
||||
flush(); // Perform a final flush to ensure all data is processed
|
||||
}
|
||||
|
||||
shared void logThisMessage(string message, string[] levels = ["info"]) {
|
||||
// Generate the timestamp for this log entry
|
||||
auto timeStamp = leftJustify(Clock.currTime().toString(), 28, '0');
|
||||
|
|
|
|||
304
src/main.d
304
src/main.d
|
|
@ -4,6 +4,7 @@ module main;
|
|||
// What does this module require to function?
|
||||
import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit;
|
||||
import core.stdc.signal;
|
||||
import core.sys.posix.signal;
|
||||
import core.memory;
|
||||
import core.time;
|
||||
import core.thread;
|
||||
|
|
@ -44,7 +45,19 @@ ItemDatabase itemDB;
|
|||
ClientSideFiltering selectiveSync;
|
||||
Monitor filesystemMonitor;
|
||||
|
||||
// Class variables
|
||||
// Flag for performing a synchronised shutdown
|
||||
bool shutdownInProgress = false;
|
||||
// Flag if a --dry-run is being performed, as, on shutdown, once config is destroyed, we have no reference here
|
||||
bool dryRun = false;
|
||||
// Configure the runtime database file path so that it is available to us on shutdown so objects can be destroyed and removed if required
|
||||
// - Typically this will be the default, but in a --dry-run scenario, we use a separate database file
|
||||
string runtimeDatabaseFile = "";
|
||||
|
||||
int main(string[] cliArgs) {
|
||||
// Setup CTRL-C handler
|
||||
setupSignalHandler();
|
||||
|
||||
// Application Start Time - used during monitor loop to detail how long it has been running for
|
||||
auto applicationStartTime = Clock.currTime();
|
||||
// Disable buffering on stdout - this is needed so that when we are using plain write() it will go to the terminal without flushing
|
||||
|
|
@ -63,8 +76,7 @@ int main(string[] cliArgs) {
|
|||
// What is the runtime syncronisation directory that will be used
|
||||
// Typically this will be '~/OneDrive' .. however tilde expansion is unreliable
|
||||
string runtimeSyncDirectory = "";
|
||||
// Configure the runtime database file path. Typically this will be the default, but in a --dry-run scenario, we use a separate database file
|
||||
string runtimeDatabaseFile = "";
|
||||
|
||||
// Verbosity Logging Count - this defines if verbose or debug logging is being used
|
||||
long verbosityCount = 0;
|
||||
// Application Logging Level
|
||||
|
|
@ -86,15 +98,15 @@ int main(string[] cliArgs) {
|
|||
scope(exit) {
|
||||
// Detail what scope was called
|
||||
addLogEntry("Exit scope was called", ["debug"]);
|
||||
// Perform exit tasks
|
||||
performStandardExitProcess("exitScope");
|
||||
// Perform synchronised exit
|
||||
performSynchronisedExitProcess("exitScope");
|
||||
}
|
||||
|
||||
scope(failure) {
|
||||
// Detail what scope was called
|
||||
addLogEntry("Failure scope was called", ["debug"]);
|
||||
// Perform exit tasks
|
||||
performStandardExitProcess("failureScope");
|
||||
// Perform synchronised exit
|
||||
performSynchronisedExitProcess("failureScope");
|
||||
}
|
||||
|
||||
// Read in application options as passed in
|
||||
|
|
@ -140,23 +152,6 @@ int main(string[] cliArgs) {
|
|||
// If we need to enable logging to a file, we can only do this once we know the application configuration which is done slightly later on
|
||||
initialiseLogging(verboseLogging, debugLogging);
|
||||
|
||||
/**
|
||||
// most used
|
||||
addLogEntry("Basic 'info' message", ["info"]); .... or just use addLogEntry("Basic 'info' message");
|
||||
addLogEntry("Basic 'verbose' message", ["verbose"]);
|
||||
addLogEntry("Basic 'debug' message", ["debug"]);
|
||||
// GUI notify only
|
||||
addLogEntry("Basic 'notify' ONLY message and displayed in GUI if notifications are enabled", ["notify"]);
|
||||
// info and notify
|
||||
addLogEntry("Basic 'info and notify' message and displayed in GUI if notifications are enabled", ["info", "notify"]);
|
||||
// log file only
|
||||
addLogEntry("Information sent to the log file only, and only if logging to a file is enabled", ["logFileOnly"]);
|
||||
// Console only (session based upload|download)
|
||||
addLogEntry("Basic 'Console only with new line' message", ["consoleOnly"]);
|
||||
// Console only with no new line
|
||||
addLogEntry("Basic 'Console only with no new line' message", ["consoleOnlyNoNewLine"]);
|
||||
**/
|
||||
|
||||
// Log application start time, log line has start time
|
||||
addLogEntry("Application started", ["debug"]);
|
||||
|
||||
|
|
@ -188,6 +183,9 @@ int main(string[] cliArgs) {
|
|||
// Update the current runtime application configuration (default or 'config' fileread-in options) from any passed in command line arguments
|
||||
appConfig.updateFromArgs(cliArgs);
|
||||
|
||||
// Configure dryRun so that this can be used here & during shutdown
|
||||
dryRun = appConfig.getValueBool("dry_run");
|
||||
|
||||
// As early as possible, now re-configure the logging class, given that we have read in any applicable 'config' file and updated the application running config from CLI input:
|
||||
// - Enable logging to a file if this is required
|
||||
// - Disable GUI notifications if this has been configured
|
||||
|
|
@ -264,9 +262,9 @@ int main(string[] cliArgs) {
|
|||
// Check for --dry-run operation or a 'no-sync' operation where the 'dry-run' DB copy should be used
|
||||
// If this has been requested, we need to ensure that all actions are performed against the dry-run database copy, and,
|
||||
// no actual action takes place - such as deleting files if deleted online, moving files if moved online or local, downloading new & changed files, uploading new & changed files
|
||||
if ((appConfig.getValueBool("dry_run")) || (appConfig.hasNoSyncOperationBeenRequested())) {
|
||||
if (dryRun || (appConfig.hasNoSyncOperationBeenRequested())) {
|
||||
|
||||
if (appConfig.getValueBool("dry_run")) {
|
||||
if (dryRun) {
|
||||
// This is a --dry-run operation
|
||||
addLogEntry("DRY-RUN Configured. Output below shows what 'would' have occurred.");
|
||||
}
|
||||
|
|
@ -279,26 +277,29 @@ int main(string[] cliArgs) {
|
|||
// In a --dry-run --resync scenario, we should not copy the existing database file
|
||||
if (!appConfig.getValueBool("resync")) {
|
||||
// Copy the existing DB file to the dry-run copy
|
||||
if (appConfig.getValueBool("dry_run")) {
|
||||
if (dryRun) {
|
||||
addLogEntry("DRY-RUN: Copying items.sqlite3 to items-dryrun.sqlite3 to use for dry run operations");
|
||||
}
|
||||
copy(appConfig.databaseFilePath,appConfig.databaseFilePathDryRun);
|
||||
} else {
|
||||
// No database copy due to --resync
|
||||
if (appConfig.getValueBool("dry_run")) {
|
||||
if (dryRun) {
|
||||
addLogEntry("DRY-RUN: No database copy created for --dry-run due to --resync also being used");
|
||||
}
|
||||
}
|
||||
}
|
||||
// update runtimeDatabaseFile now that we are using the dry run path
|
||||
runtimeDatabaseFile = appConfig.databaseFilePathDryRun;
|
||||
} else {
|
||||
// Cleanup any existing dry-run elements ... these should never be left hanging around
|
||||
cleanupDryRunDatabaseFiles(appConfig.databaseFilePathDryRun);
|
||||
}
|
||||
|
||||
// Handle --logout as separate item, do not 'resync' on a --logout
|
||||
if (appConfig.getValueBool("logout")) {
|
||||
addLogEntry("--logout requested", ["debug"]);
|
||||
addLogEntry("Deleting the saved authentication status ...");
|
||||
if (!appConfig.getValueBool("dry_run")) {
|
||||
if (!dryRun) {
|
||||
safeRemove(appConfig.refreshTokenFilePath);
|
||||
} else {
|
||||
// --dry-run scenario ... technically we should not be making any local file changes .......
|
||||
|
|
@ -312,7 +313,7 @@ int main(string[] cliArgs) {
|
|||
if (appConfig.getValueBool("reauth")) {
|
||||
addLogEntry("--reauth requested", ["debug"]);
|
||||
addLogEntry("Deleting the saved authentication status ... re-authentication requested");
|
||||
if (!appConfig.getValueBool("dry_run")) {
|
||||
if (!dryRun) {
|
||||
safeRemove(appConfig.refreshTokenFilePath);
|
||||
} else {
|
||||
// --dry-run scenario ... technically we should not be making any local file changes .......
|
||||
|
|
@ -824,10 +825,6 @@ int main(string[] cliArgs) {
|
|||
}
|
||||
};
|
||||
|
||||
// Handle SIGINT and SIGTERM
|
||||
signal(SIGINT, &exitHandler);
|
||||
signal(SIGTERM, &exitHandler);
|
||||
|
||||
// Initialise the local filesystem monitor class using inotify to monitor for local filesystem changes
|
||||
// If we are in a --download-only method of operation, we do not enable local filesystem monitoring
|
||||
if (!appConfig.getValueBool("download_only")) {
|
||||
|
|
@ -1111,78 +1108,6 @@ int main(string[] cliArgs) {
|
|||
}
|
||||
}
|
||||
|
||||
void performStandardExitProcess(string scopeCaller = null) {
|
||||
// Who called this function
|
||||
if (!scopeCaller.empty) {
|
||||
addLogEntry("Running performStandardExitProcess due to: " ~ scopeCaller, ["debug"]);
|
||||
}
|
||||
|
||||
// Shutdown the OneDrive Webhook instance
|
||||
if (oneDriveWebhook !is null) {
|
||||
oneDriveWebhook.stop();
|
||||
object.destroy(oneDriveWebhook);
|
||||
}
|
||||
|
||||
// Shutdown the client side filtering objects
|
||||
if (selectiveSync !is null) {
|
||||
addLogEntry("Shutdown Client Side Filtering instance", ["debug"]);
|
||||
selectiveSync.shutdown();
|
||||
object.destroy(selectiveSync);
|
||||
}
|
||||
|
||||
// Shutdown the application configuration objects
|
||||
if (appConfig !is null) {
|
||||
addLogEntry("Shutdown Application Configuration instance", ["debug"]);
|
||||
// Cleanup any existing dry-run elements ... these should never be left hanging around
|
||||
cleanupDryRunDatabaseFiles(appConfig.databaseFilePathDryRun);
|
||||
object.destroy(appConfig);
|
||||
}
|
||||
|
||||
// Shutdown any local filesystem monitoring
|
||||
if (filesystemMonitor !is null) {
|
||||
addLogEntry("Shutdown Filesystem Monitoring instance", ["debug"]);
|
||||
filesystemMonitor.shutdown();
|
||||
object.destroy(filesystemMonitor);
|
||||
}
|
||||
|
||||
// Shutdown the sync engine
|
||||
if (syncEngineInstance !is null) {
|
||||
addLogEntry("Shutdown Sync Engine instance", ["debug"]);
|
||||
object.destroy(syncEngineInstance);
|
||||
}
|
||||
|
||||
// Shutdown the database
|
||||
if (itemDB !is null) {
|
||||
addLogEntry("Shutdown Database instance", ["debug"]);
|
||||
// Make sure the .wal file is incorporated into the main db before we exit
|
||||
if (itemDB.isDatabaseInitialised()) {
|
||||
itemDB.performVacuum();
|
||||
}
|
||||
object.destroy(itemDB);
|
||||
}
|
||||
|
||||
// Shutdown cached sockets
|
||||
CurlEngine.releaseAll();
|
||||
|
||||
// Set all objects to null
|
||||
if (scopeCaller == "failureScope") {
|
||||
// Set these to be null due to failure scope - prevent 'ERROR: Unable to perform a database vacuum: out of memory' when the exit scope is then called
|
||||
addLogEntry("Setting ALL Class Objects to null due to failure scope", ["debug"]);
|
||||
itemDB = null;
|
||||
appConfig = null;
|
||||
oneDriveWebhook = null;
|
||||
selectiveSync = null;
|
||||
syncEngineInstance = null;
|
||||
} else {
|
||||
addLogEntry("Waiting for all internal threads to complete before exiting application", ["verbose"]);
|
||||
addLogEntry("Application exit", ["debug"]);
|
||||
addLogEntry("#######################################################################################################################################", ["logFileOnly"]);
|
||||
// Destroy the shared logging buffer
|
||||
(cast() logBuffer).shutdown();
|
||||
object.destroy(logBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
void oneDriveWebhookCallback() {
|
||||
// If we are in a --download-only method of operation, there is no filesystem monitoring, so no inotify events to check
|
||||
if (!appConfig.getValueBool("download_only")) {
|
||||
|
|
@ -1352,7 +1277,7 @@ void processResyncDatabaseRemoval(string databaseFilePathToRemove) {
|
|||
destroy(itemDB);
|
||||
// delete application sync state
|
||||
addLogEntry("Deleting the saved application sync status ...");
|
||||
if (!appConfig.getValueBool("dry_run")) {
|
||||
if (!dryRun) {
|
||||
safeRemove(databaseFilePathToRemove);
|
||||
} else {
|
||||
// --dry-run scenario ... technically we should not be making any local file changes .......
|
||||
|
|
@ -1364,7 +1289,7 @@ void cleanupDryRunDatabaseFiles(string dryRunDatabaseFile) {
|
|||
// Temp variables
|
||||
string dryRunShmFile = dryRunDatabaseFile ~ "-shm";
|
||||
string dryRunWalFile = dryRunDatabaseFile ~ "-wal";
|
||||
|
||||
|
||||
// If the dry run database exists, clean this up
|
||||
if (exists(dryRunDatabaseFile)) {
|
||||
// remove the existing file
|
||||
|
|
@ -1406,14 +1331,167 @@ auto assumeNoGC(T) (T t) if (isFunctionPointer!T || isDelegate!T) {
|
|||
return cast(SetFunctionAttributes!(T, functionLinkage!T, attrs)) t;
|
||||
}
|
||||
|
||||
// Catch CTRL-C if user pressed this
|
||||
// Configure the signal handler to catch SIGINT (CTRL-C) and SIGTERM (kill)
|
||||
void setupSignalHandler() {
|
||||
sigaction_t sa;
|
||||
sa.sa_flags = SA_RESETHAND | SA_NODEFER; // Use reset and no defer flags to handle reentrant signals
|
||||
sa.sa_handler = &exitHandler; // Direct function pointer assignment
|
||||
sigemptyset(&sa.sa_mask); // Initialize the signal set to empty
|
||||
|
||||
// Register the signal handler for SIGINT
|
||||
if (sigaction(SIGINT, &sa, null) != 0) {
|
||||
writeln("FATAL: Failed to install SIGINT handler");
|
||||
exit(-1);
|
||||
}
|
||||
|
||||
// Register the signal handler for SIGTERM
|
||||
if (sigaction(SIGTERM, &sa, null) != 0) {
|
||||
writeln("FATAL: Failed to install SIGTERM handler");
|
||||
exit(-1);
|
||||
}
|
||||
}
|
||||
|
||||
// Catch SIGINT (CTRL-C) and SIGTERM (kill), handle rapid repeat presses
|
||||
extern(C) nothrow @nogc @system void exitHandler(int value) {
|
||||
|
||||
if (shutdownInProgress) {
|
||||
return; // Ignore subsequent presses
|
||||
}
|
||||
shutdownInProgress = true;
|
||||
|
||||
try {
|
||||
assumeNoGC ( () {
|
||||
// Force kill any running threads as ^C was used
|
||||
taskPool.finish(false);
|
||||
addLogEntry("\nReceived termination signal, initiating cleanup");
|
||||
// Wait for all parallel jobs that depend on the database to complete
|
||||
|
||||
addLogEntry("Waiting for any existing upload|download process to complete");
|
||||
taskPool.finish(true);
|
||||
|
||||
// Force kill any running threads
|
||||
//addLogEntry("Forcing any active thread to exit");
|
||||
//taskPool.finish(false);
|
||||
|
||||
// Perform the shutdown process
|
||||
performSynchronisedExitProcess("exitHandler");
|
||||
})();
|
||||
} catch(Exception e) {}
|
||||
// Exit with the exitHandler value
|
||||
exit(value);
|
||||
} catch(Exception e) {
|
||||
// Any output here will cause a GC allocation
|
||||
// - Error: `@nogc` function `main.exitHandler` cannot call non-@nogc function `std.stdio.writeln!string.writeln`
|
||||
// - Error: cannot use operator `~` in `@nogc` function `main.exitHandler`
|
||||
// writeln("Exception during shutdown: " ~ e.msg);
|
||||
}
|
||||
// Exit the process with the provided exit code
|
||||
exit(value);
|
||||
|
||||
}
|
||||
|
||||
// Handle application exit
|
||||
void performSynchronisedExitProcess(string scopeCaller = null) {
|
||||
synchronized {
|
||||
// Logging the caller of the shutdown procedure
|
||||
if (!scopeCaller.empty) {
|
||||
addLogEntry("performSynchronisedExitProcess called by: " ~ scopeCaller, ["debug"]);
|
||||
}
|
||||
|
||||
// Perform cleanup and shutdown of various services and resources
|
||||
try {
|
||||
// Shutdown the OneDrive Webhook instance
|
||||
shutdownOneDriveWebhook();
|
||||
// Shutdown the client side filtering objects
|
||||
shutdownSelectiveSync();
|
||||
// Shutdown the sync engine
|
||||
shutdownSyncEngine();
|
||||
// Shutdown any local filesystem monitoring
|
||||
shutdownFilesystemMonitor();
|
||||
// Shutdown the database
|
||||
shutdownDatabase();
|
||||
// Shutdown 'curl' instances
|
||||
shutdownCurlInstances();
|
||||
// Shutdown the application configuration objects
|
||||
shutdownAppConfig();
|
||||
|
||||
} catch (Exception e) {
|
||||
addLogEntry("Error during performStandardExitProcess: " ~ e.toString(), ["error"]);
|
||||
}
|
||||
|
||||
// Finalise all logging and destroy log buffer
|
||||
shutdownApplicationLogging();
|
||||
|
||||
// Memory Garbage Collection
|
||||
GC.collect();
|
||||
}
|
||||
}
|
||||
|
||||
void shutdownOneDriveWebhook() {
|
||||
if (oneDriveWebhook !is null) {
|
||||
addLogEntry("Shutdown OneDrive Webhook instance", ["debug"]);
|
||||
oneDriveWebhook.stop();
|
||||
object.destroy(oneDriveWebhook);
|
||||
oneDriveWebhook = null;
|
||||
}
|
||||
}
|
||||
|
||||
void shutdownFilesystemMonitor() {
|
||||
if (filesystemMonitor !is null) {
|
||||
addLogEntry("Shutdown Filesystem Monitoring instance", ["debug"]);
|
||||
filesystemMonitor.shutdown();
|
||||
object.destroy(filesystemMonitor);
|
||||
filesystemMonitor = null;
|
||||
}
|
||||
}
|
||||
|
||||
void shutdownSelectiveSync() {
|
||||
if (selectiveSync !is null) {
|
||||
addLogEntry("Shutdown Client Side Filtering instance", ["debug"]);
|
||||
selectiveSync.shutdown();
|
||||
object.destroy(selectiveSync);
|
||||
selectiveSync = null;
|
||||
}
|
||||
}
|
||||
|
||||
void shutdownSyncEngine() {
|
||||
if (syncEngineInstance !is null) {
|
||||
addLogEntry("Shutdown Sync Engine instance", ["debug"]);
|
||||
//syncEngineInstance.shutdown(); - potentially need this and also check for a ~this() for class cleanup
|
||||
object.destroy(syncEngineInstance);
|
||||
syncEngineInstance = null;
|
||||
}
|
||||
}
|
||||
|
||||
void shutdownDatabase() {
|
||||
if (itemDB !is null && itemDB.isDatabaseInitialised()) {
|
||||
addLogEntry("Shutdown Database instance", ["debug"]);
|
||||
itemDB.performVacuum();
|
||||
object.destroy(itemDB);
|
||||
itemDB = null;
|
||||
}
|
||||
}
|
||||
|
||||
void shutdownAppConfig() {
|
||||
if (appConfig !is null) {
|
||||
addLogEntry("Shutdown Application Configuration instance", ["debug"]);
|
||||
if (dryRun) {
|
||||
// We were running with --dry-run , clean up the applicable database
|
||||
cleanupDryRunDatabaseFiles(runtimeDatabaseFile);
|
||||
}
|
||||
object.destroy(appConfig);
|
||||
appConfig = null;
|
||||
}
|
||||
}
|
||||
|
||||
void shutdownCurlInstances() {
|
||||
CurlEngine.releaseAllCurlInstances();
|
||||
}
|
||||
|
||||
void shutdownApplicationLogging() {
|
||||
// Join all threads to avoid any thread hanging out there
|
||||
addLogEntry("Waiting for all internal threads to complete before exiting application", ["verbose"]);
|
||||
// Join all threads
|
||||
thread_joinAll();
|
||||
// Log that we are exitintg
|
||||
addLogEntry("Application is exiting.", ["debug"]);
|
||||
addLogEntry("#######################################################################################################################################", ["logFileOnly"]);
|
||||
// Destroy the shared logging buffer
|
||||
(cast() logBuffer).shutdown();
|
||||
object.destroy(logBuffer);
|
||||
}
|
||||
|
|
@ -140,8 +140,7 @@ class MonitorBackgroundWorker {
|
|||
}
|
||||
}
|
||||
|
||||
void startMonitorJob(shared(MonitorBackgroundWorker) worker, Tid callerTid)
|
||||
{
|
||||
void startMonitorJob(shared(MonitorBackgroundWorker) worker, Tid callerTid) {
|
||||
try {
|
||||
worker.watch(callerTid);
|
||||
} catch (OwnerTerminated error) {
|
||||
|
|
@ -282,6 +281,11 @@ final class Monitor {
|
|||
this.selectiveSync = selectiveSync;
|
||||
}
|
||||
|
||||
// The destructor should only clean up resources owned directly by this instance
|
||||
~this() {
|
||||
object.destroy(worker);
|
||||
}
|
||||
|
||||
// Initialise the monitor class
|
||||
void initialise() {
|
||||
// Configure the variables
|
||||
|
|
|
|||
|
|
@ -63,10 +63,12 @@ class OneDriveError: Error {
|
|||
|
||||
// Define the 'OneDriveApi' class
|
||||
class OneDriveApi {
|
||||
// Class variables
|
||||
// Class variables that use other classes
|
||||
ApplicationConfig appConfig;
|
||||
CurlEngine curlEngine;
|
||||
CurlResponse response;
|
||||
|
||||
// Class variables
|
||||
string clientId = "";
|
||||
string companyName = "";
|
||||
string authUrl = "";
|
||||
|
|
@ -115,7 +117,11 @@ class OneDriveApi {
|
|||
}
|
||||
|
||||
~this() {
|
||||
// We cant destroy 'appConfig' here as this leads to a segfault
|
||||
object.destroy(curlEngine);
|
||||
object.destroy(response);
|
||||
curlEngine = null;
|
||||
response = null;
|
||||
}
|
||||
|
||||
// Initialise the OneDrive API class
|
||||
|
|
@ -1114,6 +1120,10 @@ class OneDriveApi {
|
|||
// - This should throw a OneDriveException so that this exception can be handled appropriately elsewhere in the application
|
||||
|
||||
private JSONValue oneDriveErrorHandlerWrapper(CurlResponse delegate(CurlResponse response) executer, bool validateJSONResponse, string callingFunction, int lineno) {
|
||||
// Create a new 'curl' response
|
||||
response = new CurlResponse();
|
||||
|
||||
// Other wrapper variables
|
||||
int retryAttempts = 0;
|
||||
int baseBackoffInterval = 1; // Base backoff interval in seconds
|
||||
int maxRetryCount = 175200; // Approx 365 days based on maxBackoffInterval + appConfig.defaultDataTimeout
|
||||
|
|
@ -1122,12 +1132,10 @@ class OneDriveApi {
|
|||
int thisBackOffInterval = 0;
|
||||
int timestampAlign = 0;
|
||||
JSONValue result;
|
||||
CurlResponse response = new CurlResponse();
|
||||
SysTime currentTime;
|
||||
SysTime retryTime;
|
||||
bool retrySuccess = false;
|
||||
bool transientError = false;
|
||||
|
||||
auto internalThreadId = generateAlphanumericString();
|
||||
|
||||
while (!retrySuccess) {
|
||||
|
|
|
|||
14
src/sync.d
14
src/sync.d
|
|
@ -303,9 +303,17 @@ class SyncEngine {
|
|||
}
|
||||
|
||||
~this() {
|
||||
processPool = null;
|
||||
object.destroy(processPool);
|
||||
object.destroy(oneDriveApiInstance);
|
||||
this.processPool.finish(true);
|
||||
object.destroy(this.processPool); // Destroy, then set to null
|
||||
this.processPool = null;
|
||||
object.destroy(this.oneDriveApiInstance); // Destroy, then set to null
|
||||
this.oneDriveApiInstance = null;
|
||||
object.destroy(this.appConfig); // Destroy, then set to null
|
||||
this.appConfig = null;
|
||||
object.destroy(this.itemDB); // Destroy, then set to null
|
||||
this.itemDB = null;
|
||||
object.destroy(this.selectiveSync); // Destroy, then set to null
|
||||
this.selectiveSync = null;
|
||||
}
|
||||
|
||||
// Initialise the Sync Engine class
|
||||
|
|
|
|||
96
src/util.d
96
src/util.d
|
|
@ -201,8 +201,46 @@ Regex!char wild2regex(const(char)[] pattern) {
|
|||
return regex(str, "i");
|
||||
}
|
||||
|
||||
// Test Internet access to Microsoft OneDrive using a simple HTTP HEAD request
|
||||
// Test Internet access to Microsoft OneDrive
|
||||
bool testInternetReachability(ApplicationConfig appConfig) {
|
||||
CurlEngine curlEngine;
|
||||
bool result = false;
|
||||
try {
|
||||
// Use preconfigured object with all the correct http values assigned
|
||||
curlEngine = CurlEngine.getCurlInstance();
|
||||
curlEngine.initialise(appConfig.getValueLong("dns_timeout"), appConfig.getValueLong("connect_timeout"), appConfig.getValueLong("data_timeout"), appConfig.getValueLong("operation_timeout"), appConfig.defaultMaxRedirects, appConfig.getValueBool("debug_https"), appConfig.getValueString("user_agent"), appConfig.getValueBool("force_http_11"), appConfig.getValueLong("rate_limit"), appConfig.getValueLong("ip_protocol_version"));
|
||||
|
||||
// Configure the remaining items required
|
||||
// URL to use
|
||||
// HTTP connection test method
|
||||
|
||||
curlEngine.connect(HTTP.Method.head, "https://login.microsoftonline.com");
|
||||
addLogEntry("Attempting to contact Microsoft OneDrive Login Service", ["debug"]);
|
||||
curlEngine.http.perform();
|
||||
addLogEntry("Shutting down HTTP engine as successfully reached OneDrive Login Service", ["debug"]);
|
||||
|
||||
// Release
|
||||
curlEngine.release(); // performs curl cleanup()
|
||||
curlEngine = null; // Clean up this memory variable
|
||||
|
||||
// Set that we are online
|
||||
result = true;
|
||||
} catch (SocketException e) {
|
||||
addLogEntry("HTTP Socket Issue", ["debug"]);
|
||||
addLogEntry("Cannot connect to Microsoft OneDrive Login Service - Socket Issue");
|
||||
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
||||
} catch (CurlException e) {
|
||||
addLogEntry("No Network Connection", ["debug"]);
|
||||
addLogEntry("Cannot connect to Microsoft OneDrive Login Service - Network Connection Issue");
|
||||
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
||||
}
|
||||
|
||||
// Return test result
|
||||
return result;
|
||||
}
|
||||
|
||||
// Test Internet access to Microsoft OneDrive using a simple HTTP HEAD request
|
||||
bool testInternetReachabilityAlternate(ApplicationConfig appConfig) {
|
||||
auto http = HTTP();
|
||||
http.url = "https://login.microsoftonline.com";
|
||||
|
||||
|
|
@ -684,27 +722,61 @@ string getFunctionName(alias func)() {
|
|||
return __traits(identifier, __traits(parent, func)) ~ "()\n";
|
||||
}
|
||||
|
||||
JSONValue fetchOnlineURLContent(string url) {
|
||||
// Function variables
|
||||
char[] content;
|
||||
JSONValue onlineContent;
|
||||
|
||||
// Setup HTTP request
|
||||
HTTP http = HTTP();
|
||||
|
||||
// Create an HTTP object within a scope to ensure cleanup
|
||||
scope(exit) {
|
||||
http.shutdown();
|
||||
object.destroy(http);
|
||||
}
|
||||
|
||||
// Configure the URL to access
|
||||
http.url = url;
|
||||
// HTTP the connection method
|
||||
http.method = HTTP.Method.get;
|
||||
|
||||
// Data receive handler
|
||||
http.onReceive = (ubyte[] data) {
|
||||
content ~= data; // Append data as it's received
|
||||
return data.length;
|
||||
};
|
||||
|
||||
// Perform HTTP request
|
||||
http.perform();
|
||||
|
||||
// Parse Content
|
||||
onlineContent = parseJSON(to!string(content));
|
||||
|
||||
// Ensure resources are cleaned up
|
||||
http.shutdown();
|
||||
object.destroy(http);
|
||||
|
||||
// Return onlineResponse
|
||||
return onlineContent;
|
||||
}
|
||||
|
||||
// Get the latest release version from GitHub
|
||||
JSONValue getLatestReleaseDetails() {
|
||||
// Import curl just for this function
|
||||
import std.net.curl;
|
||||
char[] content;
|
||||
JSONValue githubLatest;
|
||||
JSONValue versionDetails;
|
||||
string latestTag;
|
||||
string publishedDate;
|
||||
|
||||
// Query GitHub for the 'latest' release details
|
||||
try {
|
||||
content = get("https://api.github.com/repos/abraunegg/onedrive/releases/latest");
|
||||
githubLatest = content.parseJSON();
|
||||
try {
|
||||
githubLatest = fetchOnlineURLContent("https://api.github.com/repos/abraunegg/onedrive/releases/latest");
|
||||
} catch (CurlException e) {
|
||||
addLogEntry("CurlException: Unable to query GitHub for latest release - " ~ e.msg, ["debug"]);
|
||||
} catch (JSONException e) {
|
||||
addLogEntry("JSONException: Unable to parse GitHub JSON response - " ~ e.msg, ["debug"]);
|
||||
}
|
||||
|
||||
|
||||
// githubLatest has to be a valid JSON object
|
||||
if (githubLatest.type() == JSONType.object){
|
||||
// use the returned tag_name
|
||||
|
|
@ -746,9 +818,6 @@ JSONValue getLatestReleaseDetails() {
|
|||
|
||||
// Get the release details from the 'current' running version
|
||||
JSONValue getCurrentVersionDetails(string thisVersion) {
|
||||
// Import curl just for this function
|
||||
import std.net.curl;
|
||||
char[] content;
|
||||
JSONValue githubDetails;
|
||||
JSONValue versionDetails;
|
||||
string versionTag = "v" ~ thisVersion;
|
||||
|
|
@ -756,9 +825,8 @@ JSONValue getCurrentVersionDetails(string thisVersion) {
|
|||
|
||||
// Query GitHub for the release details to match the running version
|
||||
try {
|
||||
content = get("https://api.github.com/repos/abraunegg/onedrive/releases");
|
||||
githubDetails = content.parseJSON();
|
||||
} catch (CurlException e) {
|
||||
githubDetails = fetchOnlineURLContent("https://api.github.com/repos/abraunegg/onedrive/releases");
|
||||
} catch (CurlException e) {
|
||||
addLogEntry("CurlException: Unable to query GitHub for release details - " ~ e.msg, ["debug"]);
|
||||
return parseJSON(`{"Error": "CurlException", "message": "` ~ e.msg ~ `"}`);
|
||||
} catch (JSONException e) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue