abraunegg-onedrive/src/onedrive.d
2026-01-28 21:21:19 +11:00

2341 lines
No EOL
101 KiB
D
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// What is this module called?
module onedrive;
// What does this module require to function?
import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit;
import core.memory;
import core.thread;
import std.stdio;
import std.string;
import std.utf;
import std.file;
import std.exception;
import std.regex;
import std.json;
import std.algorithm;
import std.net.curl;
import std.datetime;
import std.path;
import std.conv;
import std.math;
import std.uri;
import std.array;
// Required for webhooks
import std.uuid;
// What other modules that we have created do we need to import?
import config;
import log;
import util;
import curlEngine;
import intune;
// Define the 'OneDriveException' class
class OneDriveException : Exception {
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/errors
int httpStatusCode;
const CurlResponse response;
private JSONValue _error;
// Public property to access the JSON error
@property JSONValue error() const {
return _error;
}
this(int httpStatusCode, string reason, const CurlResponse response, string file = __FILE__, size_t line = __LINE__) {
this.httpStatusCode = httpStatusCode;
this.response = response;
this._error = response.json();
string msg = format("HTTP request returned status code %d (%s)\n%s", httpStatusCode, reason, toJSON(_error, true));
super(msg, file, line);
}
this(int httpStatusCode, string reason, string file = __FILE__, size_t line = __LINE__) {
this.httpStatusCode = httpStatusCode;
this.response = null;
super(msg, file, line);
}
}
// Define the 'OneDriveError' class
class OneDriveError: Error {
this(string msg) {
super(msg);
}
}
// Define the 'OneDriveApi' class
class OneDriveApi {
// Class variables that use other classes
ApplicationConfig appConfig;
CurlEngine curlEngine;
CurlResponse response;
// API Endpoint Constants
immutable string defaultDriveUrlAPIEndpoint = "/v1.0/me/drive";
immutable string defaultDriveByIdUrlAPIEndpoint = "/v1.0/drives/";
immutable string defaultSharedWithMeUrlAPIEndpoint = "/v1.0/me/drive/sharedWithMe";
immutable string defaultItemByIdUrlAPIEndpoint = "/v1.0/me/drive/items/";
immutable string defaultItemByPathUrlAPIEndpoint = "/v1.0/me/drive/root:/";
immutable string defaultSiteSearchUrlAPIEndpoint = "/v1.0/sites?search";
immutable string defaultSiteDriveUrlAPIEndpoint = "/v1.0/sites/";
immutable string defaultSubscriptionUrlAPIEndpoint = "/v1.0/subscriptions";
immutable string defaultWebsocketEndpointAPIEndpoint = "/v1.0/me/drive/root/subscriptions/socketIo";
// Class variables
string clientId = "";
string companyName = "";
string authUrl = "";
string deviceAuthUrl = "";
string redirectUrl = "";
string tokenUrl = "";
string driveUrl = "";
string driveByIdUrl = "";
string sharedWithMeUrl = "";
string itemByIdUrl = "";
string itemByPathUrl = "";
string siteSearchUrl = "";
string siteDriveUrl = "";
string subscriptionUrl = "";
string tenantId = "";
string authScope = "";
string websocketEndpoint = "";
string websocketEndpointAPIEndpoint = defaultWebsocketEndpointAPIEndpoint;
const(char)[] refreshToken = "";
bool dryRun = false;
bool keepAlive = false;
this(ApplicationConfig appConfig) {
// Configure the class variable to consume the application configuration
this.appConfig = appConfig;
this.curlEngine = null;
this.response = null;
// Configure the major API Query URL's, based on using application configuration
// These however can be updated by config option 'azure_ad_endpoint', thus handled differently
// Drive Queries
driveUrl = appConfig.globalGraphEndpoint ~ defaultDriveUrlAPIEndpoint;
driveByIdUrl = appConfig.globalGraphEndpoint ~ defaultDriveByIdUrlAPIEndpoint;
// What is 'shared with me' Query
sharedWithMeUrl = appConfig.globalGraphEndpoint ~ defaultSharedWithMeUrlAPIEndpoint;
// Item Queries
itemByIdUrl = appConfig.globalGraphEndpoint ~ defaultItemByIdUrlAPIEndpoint;
itemByPathUrl = appConfig.globalGraphEndpoint ~ defaultItemByPathUrlAPIEndpoint;
// Office 365 / SharePoint Queries
siteSearchUrl = appConfig.globalGraphEndpoint ~ defaultSiteSearchUrlAPIEndpoint;
siteDriveUrl = appConfig.globalGraphEndpoint ~ defaultSiteDriveUrlAPIEndpoint;
// Subscriptions
subscriptionUrl = appConfig.globalGraphEndpoint ~ defaultSubscriptionUrlAPIEndpoint;
// WebSocket Endpoint - sets the default: /v1.0/me/drive/root/subscriptions/socketIo
websocketEndpoint = appConfig.globalGraphEndpoint ~ websocketEndpointAPIEndpoint;
}
// The destructor should only clean up resources owned directly by this instance
~this() {
if (response !is null) {
object.destroy(response); // calls class CurlResponse destructor
response = null;
}
if (curlEngine !is null) {
object.destroy(curlEngine); // calls class CurlEngine destructor
curlEngine = null;
}
if (appConfig !is null) {
appConfig = null;
}
}
// Initialise the OneDrive API class
bool initialise(bool keepAlive=true) {
// Initialise the curl engine
this.keepAlive = keepAlive;
if (curlEngine is null) {
curlEngine = getCurlInstance();
curlEngine.initialise(appConfig.getValueLong("dns_timeout"), appConfig.getValueLong("connect_timeout"), appConfig.getValueLong("data_timeout"), appConfig.getValueLong("operation_timeout"), appConfig.defaultMaxRedirects, appConfig.getValueBool("debug_https"), appConfig.getValueString("user_agent"), appConfig.getValueBool("force_http_11"), appConfig.getValueLong("rate_limit"), appConfig.getValueLong("ip_protocol_version"), appConfig.getValueLong("max_curl_idle"), keepAlive);
// WebSocket capability available in OS cURL version
if (!appConfig.websocketSupportCheckDone) {
// Check the underlying cURL capability to support websockets
if (debugLogging) {addLogEntry("Checking cURL Websocket support ...", ["debug"]);}
bool websocketSupport = curlSupportsWebSockets();
if (debugLogging) {addLogEntry("Checked cURL Websocket support = " ~ to!string(websocketSupport), ["debug"]);}
// Update appConfig flags
appConfig.curlSupportsWebSockets = websocketSupport;
appConfig.websocketSupportCheckDone = true;
// Notify user if cURL version is too old to support websockets, but only if we are in --monitor mode, as this is where this is used
// Are we doing a --monitor operation?
if (appConfig.getValueBool("monitor")) {
if (!websocketSupport) {
// cURL/libcurl version is too old
addLogEntry();
addLogEntry("WARNING: Your libcurl version is too old for WebSocket support. Please upgrade to libcurl 7.86.0 or newer.");
addLogEntry(" The near real-time processing of online changes cannot be enabled on your system.");
addLogEntry();
}
}
}
}
// Authorised value to return
bool authorised = false;
// Did the user specify --dry-run
dryRun = appConfig.getValueBool("dry_run");
// Set clientId to use the configured 'application_id'
clientId = appConfig.getValueString("application_id");
if (clientId != appConfig.defaultApplicationId) {
// a custom 'application_id' was set
companyName = "custom_application";
}
// Do we have a custom Azure Tenant ID?
if (!appConfig.getValueString("azure_tenant_id").empty) {
// Use the value entered by the user
tenantId = appConfig.getValueString("azure_tenant_id");
} else {
// set to common
tenantId = "common";
}
// Did the user specify a 'drive_id' ?
if (!appConfig.getValueString("drive_id").empty) {
// Update base URL's
driveUrl = driveByIdUrl ~ appConfig.getValueString("drive_id");
itemByIdUrl = driveUrl ~ "/items";
itemByPathUrl = driveUrl ~ "/root:/";
// Need to update 'websocketEndpointAPIEndpoint' to /v1.0/drives/{driveId}/root/subscriptions/socketIo
websocketEndpointAPIEndpoint = "/v1.0/drives/" ~ appConfig.getValueString("drive_id") ~ "/root/subscriptions/socketIo";
}
// Configure the authentication scope
if (appConfig.getValueBool("read_only_auth_scope")) {
// read-only authentication scopes has been requested
if (appConfig.getValueBool("use_device_auth")) {
authScope = "&scope=Files.Read%20Files.Read.All%20Sites.Read.All%20offline_access";
} else {
authScope = "&scope=Files.Read%20Files.Read.All%20Sites.Read.All%20offline_access&response_type=code&prompt=login&redirect_uri=";
}
} else {
// read-write authentication scopes will be used (default)
if (appConfig.getValueBool("use_device_auth")) {
authScope = "&scope=Files.ReadWrite%20Files.ReadWrite.All%20Sites.ReadWrite.All%20offline_access";
} else {
authScope = "&scope=Files.ReadWrite%20Files.ReadWrite.All%20Sites.ReadWrite.All%20offline_access&response_type=code&prompt=login&redirect_uri=";
}
}
// Configure Azure AD endpoints if 'azure_ad_endpoint' is configured
string azureConfigValue = appConfig.getValueString("azure_ad_endpoint");
switch(azureConfigValue) {
case "":
if (tenantId == "common") {
if (!appConfig.apiWasInitialised) addLogEntry("Configuring Global Azure AD Endpoints");
} else {
if (!appConfig.apiWasInitialised) addLogEntry("Configuring Global Azure AD Endpoints - Single Tenant Application");
}
// Authentication
authUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
deviceAuthUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/devicecode";
redirectUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
tokenUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
// WebSocket Endpoint
websocketEndpoint = appConfig.globalGraphEndpoint ~ websocketEndpointAPIEndpoint;
break;
case "USL4":
if (!appConfig.apiWasInitialised) addLogEntry("Configuring Azure AD for US Government Endpoints");
// Authentication
authUrl = appConfig.usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
deviceAuthUrl = appConfig.usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/devicecode";
tokenUrl = appConfig.usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
if (clientId == appConfig.defaultApplicationId) {
// application_id == default
if (debugLogging) {addLogEntry("USL4 AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint", ["debug"]);}
redirectUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
} else {
// custom application_id
redirectUrl = appConfig.usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
}
// Drive Queries
driveUrl = appConfig.usl4GraphEndpoint ~ defaultDriveUrlAPIEndpoint;
driveByIdUrl = appConfig.usl4GraphEndpoint ~ defaultDriveByIdUrlAPIEndpoint;
// Item Queries
itemByIdUrl = appConfig.usl4GraphEndpoint ~ defaultItemByIdUrlAPIEndpoint;
itemByPathUrl = appConfig.usl4GraphEndpoint ~ defaultItemByPathUrlAPIEndpoint;
// Office 365 / SharePoint Queries
siteSearchUrl = appConfig.usl4GraphEndpoint ~ defaultSiteSearchUrlAPIEndpoint;
siteDriveUrl = appConfig.usl4GraphEndpoint ~ defaultSiteDriveUrlAPIEndpoint;
// Shared With Me
sharedWithMeUrl = appConfig.usl4GraphEndpoint ~ defaultSharedWithMeUrlAPIEndpoint;
// Subscriptions
subscriptionUrl = appConfig.usl4GraphEndpoint ~ defaultSubscriptionUrlAPIEndpoint;
// WebSocket Endpoint
websocketEndpoint = appConfig.usl4GraphEndpoint ~ websocketEndpointAPIEndpoint;
break;
case "USL5":
if (!appConfig.apiWasInitialised) addLogEntry("Configuring Azure AD for US Government Endpoints (DOD)");
// Authentication
authUrl = appConfig.usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
deviceAuthUrl = appConfig.usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/devicecode";
tokenUrl = appConfig.usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
if (clientId == appConfig.defaultApplicationId) {
// application_id == default
if (debugLogging) {addLogEntry("USL5 AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint", ["debug"]);}
redirectUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
} else {
// custom application_id
redirectUrl = appConfig.usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
}
// Drive Queries
driveUrl = appConfig.usl5GraphEndpoint ~ defaultDriveUrlAPIEndpoint;
driveByIdUrl = appConfig.usl5GraphEndpoint ~ defaultDriveByIdUrlAPIEndpoint;
// Item Queries
itemByIdUrl = appConfig.usl5GraphEndpoint ~ defaultItemByIdUrlAPIEndpoint;
itemByPathUrl = appConfig.usl5GraphEndpoint ~ defaultItemByPathUrlAPIEndpoint;
// Office 365 / SharePoint Queries
siteSearchUrl = appConfig.usl5GraphEndpoint ~ defaultSiteSearchUrlAPIEndpoint;
siteDriveUrl = appConfig.usl5GraphEndpoint ~ defaultSiteDriveUrlAPIEndpoint;
// Shared With Me
sharedWithMeUrl = appConfig.usl5GraphEndpoint ~ defaultSharedWithMeUrlAPIEndpoint;
// Subscriptions
subscriptionUrl = appConfig.usl5GraphEndpoint ~ defaultSubscriptionUrlAPIEndpoint;
// WebSocket Endpoint
websocketEndpoint = appConfig.usl5GraphEndpoint ~ websocketEndpointAPIEndpoint;
break;
case "DE":
if (!appConfig.apiWasInitialised) addLogEntry("Configuring Azure AD Germany");
// Authentication
authUrl = appConfig.deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
deviceAuthUrl = appConfig.deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/devicecode";
tokenUrl = appConfig.deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
if (clientId == appConfig.defaultApplicationId) {
// application_id == default
if (debugLogging) {addLogEntry("DE AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint", ["debug"]);}
redirectUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
} else {
// custom application_id
redirectUrl = appConfig.deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
}
// Drive Queries
driveUrl = appConfig.deGraphEndpoint ~ defaultDriveUrlAPIEndpoint;
driveByIdUrl = appConfig.deGraphEndpoint ~ defaultDriveByIdUrlAPIEndpoint;
// Item Queries
itemByIdUrl = appConfig.deGraphEndpoint ~ defaultItemByIdUrlAPIEndpoint;
itemByPathUrl = appConfig.deGraphEndpoint ~ defaultItemByPathUrlAPIEndpoint;
// Office 365 / SharePoint Queries
siteSearchUrl = appConfig.deGraphEndpoint ~ defaultSiteSearchUrlAPIEndpoint;
siteDriveUrl = appConfig.deGraphEndpoint ~ defaultSiteDriveUrlAPIEndpoint;
// Shared With Me
sharedWithMeUrl = appConfig.deGraphEndpoint ~ defaultSharedWithMeUrlAPIEndpoint;
// Subscriptions
subscriptionUrl = appConfig.deGraphEndpoint ~ defaultSubscriptionUrlAPIEndpoint;
// WebSocket Endpoint
websocketEndpoint = appConfig.deGraphEndpoint ~ websocketEndpointAPIEndpoint;
break;
case "CN":
if (!appConfig.apiWasInitialised) addLogEntry("Configuring AD China operated by VNET");
// Authentication
authUrl = appConfig.cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
deviceAuthUrl = appConfig.cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/devicecode";
tokenUrl = appConfig.cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
if (clientId == appConfig.defaultApplicationId) {
// application_id == default
if (debugLogging) {addLogEntry("CN AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint", ["debug"]);}
redirectUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
} else {
// custom application_id
redirectUrl = appConfig.cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
}
// Drive Queries
driveUrl = appConfig.cnGraphEndpoint ~ defaultDriveUrlAPIEndpoint;
driveByIdUrl = appConfig.cnGraphEndpoint ~ defaultDriveByIdUrlAPIEndpoint;
// Item Queries
itemByIdUrl = appConfig.cnGraphEndpoint ~ defaultItemByIdUrlAPIEndpoint;
itemByPathUrl = appConfig.cnGraphEndpoint ~ defaultItemByPathUrlAPIEndpoint;
// Office 365 / SharePoint Queries
siteSearchUrl = appConfig.cnGraphEndpoint ~ defaultSiteSearchUrlAPIEndpoint;
siteDriveUrl = appConfig.cnGraphEndpoint ~ defaultSiteDriveUrlAPIEndpoint;
// Shared With Me
sharedWithMeUrl = appConfig.cnGraphEndpoint ~ defaultSharedWithMeUrlAPIEndpoint;
// Subscriptions
subscriptionUrl = appConfig.cnGraphEndpoint ~ defaultSubscriptionUrlAPIEndpoint;
// WebSocket Endpoint
websocketEndpoint = appConfig.cnGraphEndpoint ~ websocketEndpointAPIEndpoint;
break;
// Default - all other entries
default:
if (!appConfig.apiWasInitialised) addLogEntry("Unknown Azure AD Endpoint request - using Global Azure AD Endpoints");
}
// Has the application been authenticated?
// How do we authenticate - standard method or via Intune?
if (appConfig.getValueBool("use_intune_sso")) {
// Authenticate via Intune
if (appConfig.accessToken.empty) {
// No authentication via intune yet
authorised = authorise();
} else {
// We are already authenticated
authorised = true;
}
} else {
// Authenticate via standard method
if (!exists(appConfig.refreshTokenFilePath)) {
if (debugLogging) {addLogEntry("Application has no 'refresh_token' thus needs to be authenticated", ["debug"]);}
authorised = authorise();
} else {
// Try and read the value from the appConfig if it is set, rather than trying to read the value from disk
if (!appConfig.refreshToken.empty) {
if (debugLogging) {addLogEntry("Read token from appConfig", ["debug"]);}
refreshToken = strip(appConfig.refreshToken);
authorised = true;
} else {
// Try and read the file from disk
try {
refreshToken = strip(readText(appConfig.refreshTokenFilePath));
// is the refresh_token empty?
if (refreshToken.empty) {
addLogEntry("RefreshToken exists but is empty: " ~ appConfig.refreshTokenFilePath);
authorised = authorise();
} else {
// Existing token not empty
authorised = true;
// update appConfig.refreshToken
appConfig.refreshToken = refreshToken;
}
} catch (FileException exception) {
authorised = authorise();
} catch (std.utf.UTFException exception) {
// path contains characters which generate a UTF exception
addLogEntry("Cannot read refreshToken from: " ~ appConfig.refreshTokenFilePath);
addLogEntry(" Error Reason:" ~ exception.msg);
authorised = false;
}
}
if (refreshToken.empty) {
// PROBLEM ... CODING TO DO ??????????
if (debugLogging) {addLogEntry("DEBUG: refreshToken is empty !!!!!!!!!!", ["debug"]);}
}
}
}
// Return if we are authorised
if (debugLogging) {addLogEntry("Authorised State: " ~ to!string(authorised), ["debug"]);}
return authorised;
}
// If the API has been configured correctly, print the items that been configured
void debugOutputConfiguredAPIItems() {
// Debug output of configured URL's
// Application Identification
if (debugLogging) {
addLogEntry("Configured clientId " ~ clientId, ["debug"]);
addLogEntry("Configured userAgent " ~ appConfig.getValueString("user_agent"), ["debug"]);
// Authentication
addLogEntry("Configured authScope: " ~ authScope, ["debug"]);
addLogEntry("Configured authUrl: " ~ authUrl, ["debug"]);
addLogEntry("Configured redirectUrl: " ~ redirectUrl, ["debug"]);
addLogEntry("Configured tokenUrl: " ~ tokenUrl, ["debug"]);
// Drive Queries
addLogEntry("Configured driveUrl: " ~ driveUrl, ["debug"]);
addLogEntry("Configured driveByIdUrl: " ~ driveByIdUrl, ["debug"]);
// Shared With Me
addLogEntry("Configured sharedWithMeUrl: " ~ sharedWithMeUrl, ["debug"]);
// Item Queries
addLogEntry("Configured itemByIdUrl: " ~ itemByIdUrl, ["debug"]);
addLogEntry("Configured itemByPathUrl: " ~ itemByPathUrl, ["debug"]);
// SharePoint Queries
addLogEntry("Configured siteSearchUrl: " ~ siteSearchUrl, ["debug"]);
addLogEntry("Configured siteDriveUrl: " ~ siteDriveUrl, ["debug"]);
// Websocket
addLogEntry("Configured websocketEndpoint: " ~ websocketEndpoint, ["debug"]);
}
}
// Release CurlEngine bask to the Curl Engine Pool
void releaseCurlEngine() {
// Log that this was called
if ((debugLogging) && (debugHTTPSResponse)) {addLogEntry("OneDrive API releaseCurlEngine() Called", ["debug"]);}
// Release curl instance back to the pool
if (curlEngine !is null) {
curlEngine.releaseEngine();
curlEngine = null;
}
// Release the response
response = null;
// Perform Garbage Collection
GC.collect();
}
// Authenticate this client against Microsoft OneDrive API using one of the 3 authentication methods this client supports
bool authorise() {
// Set this function name
string thisFunctionName = format("%s.%s", strip(__MODULE__) , strip(getFunctionName!({})));
// Has the client been configured to use Intune SSO via Microsoft Identity Broker (microsoft-identity-broker) dbus session
if (appConfig.getValueBool("use_intune_sso")) {
// The client is configured to use Intune SSO via Microsoft Identity Broker dbus session
// Do we have a saved account file?
if (!exists(appConfig.intuneAccountDetailsFilePath)) {
// No file exists locally
auto intuneAuthResult = acquire_token_interactive(appConfig.getValueString("application_id"));
JSONValue intuneBrokerJSONData = intuneAuthResult.brokerTokenResponse;
// Is the response JSON data valid?
if ((intuneBrokerJSONData.type() == JSONType.object)) {
// Does the JSON data have the required authentication elements:
// - accessToken
// - account
// - expiresOn
if ((hasAccessTokenData(intuneBrokerJSONData)) && (hasAccountData(intuneBrokerJSONData)) && (hasExpiresOn(intuneBrokerJSONData))) {
// Details exist
processIntuneResponse(intuneBrokerJSONData);
// Return that we are authenticated
return true;
} else {
// no ... expected values not available
addLogEntry("Required JSON elements are not present in the Intune JSON response");
return false;
}
} else {
// Not a valid JSON response
addLogEntry("Invalid JSON Intune JSON response when attempting access authentication");
return false;
}
} else {
// The account information is available in a saved file. Read this file in and attempt a silent authentication
try {
appConfig.intuneAccountDetails = strip(readText(appConfig.intuneAccountDetailsFilePath));
// Is the 'intune_account' empty?
if (appConfig.intuneAccountDetails.empty) {
addLogEntry("The 'intune_account' file exists but is empty: " ~ appConfig.intuneAccountDetailsFilePath);
// No .. remove the file and perform the interactive authentication
safeRemove(appConfig.intuneAccountDetailsFilePath);
// Attempt interactive authentication
authorise();
return true;
} else {
// We have loaded some Intune Account details, try and use them
auto intuneAuthResult = acquire_token_silently(appConfig.intuneAccountDetails, appConfig.getValueString("application_id"));
JSONValue intuneBrokerJSONData = intuneAuthResult.brokerTokenResponse;
// Is the JSON data valid?
if ((intuneBrokerJSONData.type() == JSONType.object)) {
// Does the JSON data have the required authentication elements:
// - accessToken
// - account
// - expiresOn
if ((hasAccessTokenData(intuneBrokerJSONData)) && (hasAccountData(intuneBrokerJSONData)) && (hasExpiresOn(intuneBrokerJSONData))) {
// Details exist
processIntuneResponse(intuneBrokerJSONData);
// Return that we are authenticated
return true;
} else {
// no ... expected values not available
addLogEntry("Required JSON elements are not present in the Intune JSON response");
return false;
}
} else {
// No .. remove the file and perform the interactive authentication
safeRemove(appConfig.intuneAccountDetailsFilePath);
// Attempt interactive authentication
authorise();
return true;
}
}
} catch (FileException exception) {
return false;
} catch (std.utf.UTFException exception) {
// path contains characters which generate a UTF exception
addLogEntry("Cannot read 'intune_account' file on disk from: " ~ appConfig.intuneAccountDetailsFilePath);
addLogEntry(" Error Reason:" ~ exception.msg);
return false;
}
}
} else {
// There are 2 options here for normal authentication flow
// 1. Use OAuth2 Device Authorisation Flow
// 2. Use OAuth2 Interactive Authorisation Flow (application default)
string authoriseApplicationRequest = "Please authorise this application by visiting the following URL:\n";
if (appConfig.getValueBool("use_device_auth")) {
// Use OAuth2 Device Authorisation Flow
// * deviceAuthUrl: Should already be configured based on client configuration
// * tokenUrl: Should already be configured based on client configuration
// * authScope: Should already be configured with the correct auth scopes
string deviceAuthPostData = "client_id=" ~ clientId ~ authScope;
// Initiating Device Code Request
JSONValue deviceAuthResponse = initiateDeviceAuthorisation(deviceAuthPostData);
// Was a valid JSON response provided?
if (deviceAuthResponse.type() == JSONType.object) {
// A valid JSON was returned
// Extract required values
string deviceCode = deviceAuthResponse["device_code"].str;
string deviceAuthUrl = deviceAuthResponse["verification_uri"].str;
string userCode = deviceAuthResponse["user_code"].str;
long expiresIn = deviceAuthResponse["expires_in"].integer;
long pollInterval = deviceAuthResponse["interval"].integer;
SysTime expiresAt = Clock.currTime + dur!"seconds"(expiresIn);
expiresAt.fracSecs = Duration.zero;
// Display the required items for the user to action
addLogEntry();
addLogEntry(authoriseApplicationRequest, ["consoleOnly"]);
addLogEntry(deviceAuthUrl ~ "\n", ["consoleOnly"]);
addLogEntry("Enter the following code when prompted: " ~ userCode, ["consoleOnly"]);
addLogEntry();
addLogEntry("This code expires at: " ~ to!string(expiresAt), ["consoleOnly"]);
addLogEntry();
// JSON value to store the poll response data
JSONValue deviceAuthPollResponse;
// Construct the polling post submission data
string pollPostData = format(
"client_id=%s&grant_type=urn%%3Aietf%%3Aparams%%3Aoauth%%3Agrant-type%%3Adevice_code&device_code=%s",
clientId,
deviceCode
);
// Poll Microsoft API for authentication to be performed, until the expiry of this device authentication request
while (Clock.currTime < expiresAt) {
// Try the post to poll if the authentication has been done
try {
deviceAuthPollResponse = post(tokenUrl, pollPostData, null, true, "application/x-www-form-urlencoded");
// No error ... break out of the loop so the returned data can be processed
break;
} catch (OneDriveException e) {
// Get the polling error JSON response
JSONValue errorResponse = e.error;
string errorType;
if ("error" in errorResponse) {
errorType = errorResponse["error"].str;
if (errorType == "authorization_pending") {
// Calculate remaining time
Duration timeRemaining = expiresAt - Clock.currTime;
long minutes = timeRemaining.total!"minutes"();
long seconds = timeRemaining.total!"seconds"() % 60;
// Log countdown and status
addLogEntry(format("[%02dm %02ds remaining] Still pending authorisation ...", minutes, seconds));
} else if (errorType == "authorization_declined") {
addLogEntry("Authorisation was declined by the user.");
// return false if we get to this point
// set 'use_device_auth' to false to fall back to interactive authentication flow
appConfig.setValueBool("use_device_auth" , false);
return false;
} else if (errorType == "expired_token") {
addLogEntry("Device code expired before authorisation was completed.");
// return false if we get to this point
// set 'use_device_auth' to false to fall back to interactive authentication flow
appConfig.setValueBool("use_device_auth" , false);
return false;
} else {
addLogEntry("Unhandled error during polling: " ~ errorType);
// return false if we get to this point
// set 'use_device_auth' to false to fall back to interactive authentication flow
appConfig.setValueBool("use_device_auth" , false);
return false;
}
} else {
addLogEntry("Unexpected error response from token polling.");
// return false if we get to this point
// set 'use_device_auth' to false to fall back to interactive authentication flow
appConfig.setValueBool("use_device_auth" , false);
return false;
}
}
// Sleep until next polling interval
Thread.sleep(dur!"seconds"(pollInterval));
}
// Broken out of the polling loop
// Was a valid JSON response provided?
if (deviceAuthPollResponse.type() == JSONType.object) {
// is the required 'access_token' available?
if ("access_token" in deviceAuthPollResponse) {
// We got the applicable access token
addLogEntry("Access token acquired!");
// Process this JSON data
processAuthenticationJSON(deviceAuthPollResponse);
// Return that we are authorised
return true;
}
}
// return false if we get to this point
// set 'use_device_auth' to false to fall back to interactive authentication flow
appConfig.setValueBool("use_device_auth" , false);
return false;
} else {
// No valid JSON response was returned
// set 'use_device_auth' to false to fall back to interactive authentication flow
appConfig.setValueBool("use_device_auth" , false);
return false;
}
} else {
// Use OAuth2 Interactive Authorisation Flow (application default)
char[] response;
// What URL should be presented to the user to access
string url = authUrl ~ "?client_id=" ~ clientId ~ authScope ~ redirectUrl;
// Configure automated authentication if --auth-files authUrlFilePath:responseUrlFilePath is being used
string authFilesString = appConfig.getValueString("auth_files");
string authResponseString = appConfig.getValueString("auth_response");
if (!authResponseString.empty) {
// read the response from authResponseString
response = cast(char[]) authResponseString;
} else if (authFilesString != "") {
string[] authFiles = authFilesString.split(":");
string authUrlFilePath = authFiles[0];
string responseUrlFilePath = authFiles[1];
try {
auto authUrlFile = File(authUrlFilePath, "w");
authUrlFile.write(url);
authUrlFile.close();
} catch (FileException exception) {
// There was a file system error
// display the error message
displayFileSystemErrorMessage(exception.msg, thisFunctionName, authUrlFilePath);
// Must force exit here, allow logging to be done
forceExit();
} catch (ErrnoException exception) {
// There was a file system error
// display the error message
displayFileSystemErrorMessage(exception.msg, thisFunctionName, authUrlFilePath);
// Must force exit here, allow logging to be done
forceExit();
}
// Log we are now waiting
addLogEntry("Client requires authentication before proceeding. Waiting for --auth-files elements to be available.");
while (!exists(responseUrlFilePath)) {
Thread.sleep(dur!("msecs")(100));
}
// read response from provided from OneDrive
try {
response = cast(char[]) read(responseUrlFilePath);
} catch (OneDriveException exception) {
// exception generated
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
return false;
}
// try to remove auth files one at a time
try {
std.file.remove(authUrlFilePath);
} catch (FileException exception) {
addLogEntry("Cannot remove --auth-files elements - details below");
// There was a file system error - display the error message
displayFileSystemErrorMessage(exception.msg, thisFunctionName, authUrlFilePath);
return false;
}
try {
std.file.remove(responseUrlFilePath);
} catch (FileException exception) {
addLogEntry("Cannot remove --auth-files elements - details below");
// There was a file system error - display the error message
displayFileSystemErrorMessage(exception.msg, thisFunctionName, responseUrlFilePath);
return false;
}
} else {
// If we are not running in --dry-run mode, prompt the user to authorise the application
if (!appConfig.getValueBool("dry_run")) {
// Notify the user of the next step: visit the URL to authorise the client
addLogEntry();
addLogEntry(authoriseApplicationRequest, ["consoleOnly"]);
addLogEntry(url ~ "\n", ["consoleOnly"]);
// Prompt the user to paste the full redirect URI (copied from the browser after login)
addLogEntry("After completing the authorisation in your browser, copy the full redirect URI (from the address bar) and paste it below.\n", ["consoleOnly"]);
addLogEntry("Paste redirect URI here: ", ["consoleOnlyNoNewLine"]);
// Read the user's pasted response URI
readln(response);
// Flag that a response URI has been received - at this point could be valid or invalid
appConfig.applicationAuthoriseResponseURIReceived = true;
} else {
// The application cannot be authorised when using --dry-run as we have to write out the authentication data, which negates the whole 'dry-run' process
addLogEntry();
addLogEntry("The application requires authorisation, which involves saving authentication data on your system. Application authorisation cannot be completed when using the '--dry-run' option.");
addLogEntry();
addLogEntry("To authorise the application please use your original command without '--dry-run'.");
addLogEntry();
addLogEntry("To exclusively authorise the application without performing any additional actions, do not add '--sync' or '--monitor' to your command line.");
addLogEntry();
forceExit();
}
}
// match the authorisation code
auto c = matchFirst(strip(response), r"(?:[?&]code=)([^&]+)");
if (c.empty) {
addLogEntry("An empty or invalid response uri was entered");
return false;
}
c.popFront(); // skip the whole match
string authCode = decodeComponent(c.front);
redeemToken(authCode);
return true;
}
}
}
// Process Intune JSON response data
void processIntuneResponse(JSONValue intuneBrokerJSONData) {
// Set this function name
string thisFunctionName = format("%s.%s", strip(__MODULE__) , strip(getFunctionName!({})));
// Use the provided JSON data and configure elements, save JSON data to disk for reuse
long expiresOnMs = intuneBrokerJSONData["expiresOn"].integer();
// Convert to SysTime
SysTime expiryTime = SysTime.fromUnixTime(expiresOnMs / 1000);
// Store in appConfig (to match standard flow)
appConfig.accessTokenExpiration = expiryTime;
addLogEntry("Intune access token expires at: " ~ to!string(appConfig.accessTokenExpiration));
// Configure the 'accessToken' based on Intune response
appConfig.accessToken = "bearer " ~ strip(intuneBrokerJSONData["accessToken"].str);
// Do we print the current access token
debugOutputAccessToken();
// In order to support silent renewal of the access token, when the access token expires, we must store the Intune account data
appConfig.intuneAccountDetails = to!string(intuneBrokerJSONData["account"]);
// try and update the 'intune_account' file on disk for reuse later
try {
if (debugLogging) {addLogEntry("Updating 'intune_account' on disk", ["debug"]);}
std.file.write(appConfig.intuneAccountDetailsFilePath, appConfig.intuneAccountDetails);
if (debugLogging) {addLogEntry("Setting file permissions for: " ~ appConfig.intuneAccountDetailsFilePath, ["debug"]);}
appConfig.intuneAccountDetailsFilePath.setAttributes(appConfig.returnSecureFilePermission());
} catch (FileException exception) {
// display the error message
displayFileSystemErrorMessage(exception.msg, thisFunctionName, appConfig.intuneAccountDetailsFilePath);
}
}
// Initiate OAuth2 Device Authorisation
JSONValue initiateDeviceAuthorisation(string deviceAuthPostData) {
// Device OAuth2 Device Authorisation requires a HTTP POST
return post(deviceAuthUrl, deviceAuthPostData, null, true, "application/x-www-form-urlencoded");
}
// Do we print the current access token
void debugOutputAccessToken() {
if (appConfig.verbosityCount > 1) {
if (appConfig.getValueBool("debug_https")) {
if (appConfig.getValueBool("print_token")) {
// This needs to be highly restricted in output ....
if (debugLogging) {addLogEntry("CAUTION - KEEP THIS SAFE: Current access token: " ~ to!string(appConfig.accessToken), ["debug"]);}
}
}
}
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get
JSONValue getDefaultDriveDetails() {
string url;
url = driveUrl;
return get(url);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getDefaultRootDetails() {
string url;
url = driveUrl ~ "/root";
return get(url);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getDriveIdRoot(string driveId) {
string url;
url = driveByIdUrl ~ driveId ~ "/root";
return get(url);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get
JSONValue getDriveQuota(string driveId) {
string url;
url = driveByIdUrl ~ driveId ~ "/";
url ~= "?select=quota";
return get(url);
}
// Return the details of the specified path, by giving the path we wish to query
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getPathDetails(string path) {
string url;
if ((path == ".")||(path == "/")) {
url = driveUrl ~ "/root/";
} else {
url = itemByPathUrl ~ encodeComponent(path) ~ ":/";
}
// Add select clause
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,createdBy,lastModifiedBy,package";
return get(url);
}
// Return the details of the specified item based on its driveID and itemID
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getPathDetailsById(string driveId, string id) {
string url;
url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,createdBy,lastModifiedBy,webUrl,lastModifiedDateTime,package";
return get(url);
}
// Return all the items that are shared with the user
// https://docs.microsoft.com/en-us/graph/api/drive-sharedwithme
JSONValue getSharedWithMe() {
return get(sharedWithMeUrl);
}
// Create a shareable link for an existing file on OneDrive based on the accessScope JSON permissions
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createlink
JSONValue createShareableLink(string driveId, string id, JSONValue accessScope) {
string url;
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/createLink";
return post(url, accessScope.toString());
}
// Return the requested details of the specified path on the specified drive id and path
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getPathDetailsByDriveId(string driveId, string path) {
string url;
// https://learn.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online
// Required format: /drives/{drive-id}/root:/{item-path}:
url = driveByIdUrl ~ driveId ~ "/root:/" ~ encodeComponent(path) ~ ":";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,createdBy,lastModifiedBy,package";
return get(url);
}
// Track changes for a given driveId
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta
// Your app begins by calling delta without any parameters. The service starts enumerating the drive's hierarchy, returning pages of items and either an @odata.nextLink or an @odata.deltaLink, as described below.
// Your app should continue calling with the @odata.nextLink until you no longer see an @odata.nextLink returned, or you see a response with an empty set of changes.
// After you have finished receiving all the changes, you may apply them to your local state. To check for changes in the future, call delta again with the @odata.deltaLink from the previous successful response.
JSONValue getChangesByItemId(string driveId, string id, string deltaLink) {
string[string] requestHeaders;
// From March 1st 2025, this needs to be added to ensure that Shared Folders are sent in the Delta Query Response
if (appConfig.accountType == "personal") {
// OneDrive Personal Account
addIncludeFeatureRequestHeader(&requestHeaders);
} else {
// Business or SharePoint Library
// Only add if configured to do so
if (appConfig.getValueBool("sync_business_shared_items")) {
// Feature enabled, add headers
addIncludeFeatureRequestHeader(&requestHeaders);
}
}
string url;
// configure deltaLink to query
if (deltaLink.empty) {
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/delta";
// Reduce what we ask for in the response - which reduces the data transferred back to us, and reduces what is held in memory during initial JSON processing
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,createdBy,lastModifiedBy,package";
} else {
url = deltaLink;
}
// get the response
return get(url, false, requestHeaders);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_list_children
JSONValue listChildren(string driveId, string id, string nextLink) {
string[string] requestHeaders;
// From March 1st 2025, this needs to be added to ensure that Shared Folders are sent in the Delta Query Response
if (appConfig.accountType == "personal") {
// OneDrive Personal Account
addIncludeFeatureRequestHeader(&requestHeaders);
} else {
// Business or SharePoint Library
// Only add if configured to do so
if (appConfig.getValueBool("sync_business_shared_items")) {
// Feature enabled, add headers
addIncludeFeatureRequestHeader(&requestHeaders);
}
}
string url;
// configure URL to query
if (nextLink.empty) {
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/children";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,createdBy,lastModifiedBy,package";
} else {
url = nextLink;
}
return get(url, false, requestHeaders);
}
// https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_search
JSONValue searchDriveForPath(string driveId, string path) {
string url;
url = "https://graph.microsoft.com/v1.0/drives/" ~ driveId ~ "/root/search(q='" ~ encodeComponent(path) ~ "')";
return get(url);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_update
JSONValue updateById(const(char)[] driveId, const(char)[] id, JSONValue data, const(char)[] eTag = null) {
string[string] requestHeaders;
const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
if (eTag) requestHeaders["If-Match"] = to!string(eTag);
return patch(url, data.toString(), false, requestHeaders);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delete
void deleteById(const(char)[] driveId, const(char)[] id, const(char)[] eTag = null) {
// string[string] requestHeaders;
const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
//TODO: investigate why this always fail with 412 (Precondition Failed)
// if (eTag) requestHeaders["If-Match"] = eTag;
performDelete(url);
}
// https://learn.microsoft.com/en-us/graph/api/driveitem-permanentdelete?view=graph-rest-1.0
void permanentDeleteById(const(char)[] driveId, const(char)[] id, const(char)[] eTag = null) {
// string[string] requestHeaders;
const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/permanentDelete";
//TODO: investigate why this always fail with 412 (Precondition Failed)
// if (eTag) requestHeaders["If-Match"] = eTag;
// as per documentation, a permanentDelete needs to be a HTTP POST
performPermanentDelete(url);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_post_children
JSONValue createById(string parentDriveId, string parentId, JSONValue item) {
string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ "/children";
return post(url, item.toString());
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
JSONValue simpleUpload(string localPath, string parentDriveId, string parentId, string filename) {
string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/content";
return put(url, localPath);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
JSONValue simpleUploadReplace(string localPath, string driveId, string id) {
string url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content";
return put(url, localPath);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
//JSONValue createUploadSession(string parentDriveId, string parentId, string filename, string eTag = null, JSONValue item = null) {
JSONValue createUploadSession(string parentDriveId, string parentId, string filename, const(char)[] eTag = null, JSONValue item = null) {
string[string] requestHeaders;
string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/createUploadSession";
// eTag If-Match header addition commented out for the moment
// At some point, post the creation of this upload session the eTag is being 'updated' by OneDrive, thus when uploadFragment() is used
// this generates a 412 Precondition Failed and then a 416 Requested Range Not Satisfiable
// This needs to be investigated further as to why this occurs
if (eTag) requestHeaders["If-Match"] = to!string(eTag);
return post(url, item.toString(), requestHeaders);
}
// https://learn.microsoft.com/en-us/graph/api/driveitem-createuploadsession?view=graph-rest-1.0#upload-bytes-to-the-upload-session
JSONValue uploadFragment(string uploadUrl, string filepath, long offset, long offsetSize, long fileSize) {
// If we upload a modified file, with the current known online eTag, this gets changed when the session is started - thus, the tail end of uploading
// a fragment fails with a 412 Precondition Failed and then a 416 Requested Range Not Satisfiable
// For the moment, comment out adding the If-Match header in createUploadSession, which then avoids this issue
string contentRange = "bytes " ~ to!string(offset) ~ "-" ~ to!string(offset + offsetSize - 1) ~ "/" ~ to!string(fileSize);
if (debugLogging) {
addLogEntry("fragment contentRange: " ~ contentRange, ["debug"]);
}
// Before we submit this 'HTTP PUT' request, pre-emptively check token expiry to avoid future 401s during long uploads
checkAccessTokenExpired();
// Perform the HTTP PUT action to upload the file fragment
return put(uploadUrl, filepath, true, contentRange, offset, offsetSize);
}
// https://learn.microsoft.com/en-us/graph/api/driveitem-createuploadsession?view=graph-rest-1.0#resuming-an-in-progress-upload
JSONValue requestUploadStatus(string uploadUrl) {
return get(uploadUrl, true);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/site_search?view=odsp-graph-online
JSONValue o365SiteSearch(string nextLink) {
string url;
// configure URL to query
if (nextLink.empty) {
url = siteSearchUrl ~ "=*";
} else {
url = nextLink;
}
return get(url);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_list?view=odsp-graph-online
JSONValue o365SiteDrives(string site_id, string nextLink){
string url;
// configure URL to query
if (nextLink.empty) {
url = siteDriveUrl ~ site_id ~ "/drives";
} else {
url = nextLink;
}
return get(url);
}
// Create Webhook Subscription
JSONValue createSubscription(string notificationUrl, SysTime expirationDateTime) {
string driveId;
string url = subscriptionUrl;
// What do we set for driveId
if (appConfig.getValueString("drive_id").length) {
// Use the 'config' file option
driveId = appConfig.getValueString("drive_id");
} else {
// use appConfig.defaultDriveId
driveId = appConfig.defaultDriveId;
}
// Create a resource item based on if we have a driveId now configured
string resourceItem;
if (driveId.length) {
resourceItem = "/drives/" ~ driveId ~ "/root";
} else {
resourceItem = "/me/drive/root";
}
// create JSON request to create webhook subscription
const JSONValue request = [
"changeType": "updated",
"notificationUrl": notificationUrl,
"resource": resourceItem,
"expirationDateTime": expirationDateTime.toISOExtString(),
"clientState": randomUUID().toString()
];
return post(url, request.toString());
}
// Renew Webhook Subscription
JSONValue renewSubscription(string subscriptionId, SysTime expirationDateTime) {
string url;
url = subscriptionUrl ~ "/" ~ subscriptionId;
const JSONValue request = [
"expirationDateTime": expirationDateTime.toISOExtString()
];
return patch(url, request.toString(), true);
}
// Delete Webhook subscription
void deleteSubscription(string subscriptionId) {
string url;
url = subscriptionUrl ~ "/" ~ subscriptionId;
performDelete(url);
}
// Obtain the Websocket Notification URL
JSONValue obtainWebSocketNotificationURL() {
if (debugLogging) {addLogEntry("Request a Socket.IO Subscription Endpoint: " ~ websocketEndpoint, ["debug"]);}
return get(websocketEndpoint);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get_content
void downloadById(const(char)[] driveId, const(char)[] itemId, string saveToPath, long fileSize, JSONValue onlineHash, long resumeOffset = 0) {
// Set this function name
string thisFunctionName = format("%s.%s", strip(__MODULE__) , strip(getFunctionName!({})));
// We pass through to 'downloadFile()'
// - resumeOffset
// - onlineHash
// - driveId
// - itemId
scope(failure) {
if (exists(saveToPath)) {
// try and remove the file, catch error
try {
remove(saveToPath);
} catch (FileException exception) {
// display the error message
displayFileSystemErrorMessage(exception.msg, thisFunctionName, saveToPath);
}
}
}
// Create the required local parental path structure if this does not exist
string parentalPath = dirName(saveToPath);
// Does the parental path exist locally?
if (!exists(parentalPath)) {
try {
if (debugLogging) {addLogEntry("Requested local parental path does not exist, creating directory structure: " ~ parentalPath, ["debug"]);}
mkdirRecurse(parentalPath);
// Has the user disabled the setting of filesystem permissions?
if (!appConfig.getValueBool("disable_permission_set")) {
// Configure the applicable permissions for the folder
if (debugLogging) {addLogEntry("Setting directory permissions for: " ~ parentalPath, ["debug"]);}
parentalPath.setAttributes(appConfig.returnRequiredDirectoryPermissions());
} else {
// Use inherited permissions
if (debugLogging) {addLogEntry("Using inherited filesystem permissions for: " ~ parentalPath, ["debug"]);}
}
} catch (FileException exception) {
// display the error message
displayFileSystemErrorMessage(exception.msg, thisFunctionName, parentalPath);
}
}
// Create the URL to download the file
const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ itemId ~ "/content?AVOverride=1";
// Download file using the URL created above
downloadFile(driveId, itemId, url, saveToPath, fileSize, onlineHash, resumeOffset);
// Does downloaded file now exist locally?
if (exists(saveToPath)) {
// Has the user disabled the setting of filesystem permissions?
if (!appConfig.getValueBool("disable_permission_set")) {
// File was downloaded successfully - configure the applicable permissions for the file
if (debugLogging) {addLogEntry("Setting file permissions for: " ~ saveToPath, ["debug"]);}
saveToPath.setAttributes(appConfig.returnRequiredFilePermissions());
} else {
// Use inherited permissions
if (debugLogging) {addLogEntry("Using inherited filesystem permissions for: " ~ saveToPath, ["debug"]);}
}
}
}
// Return the actual siteSearchUrl being used and/or requested when performing 'siteQuery = onedrive.o365SiteSearch(nextLink);' call
string getSiteSearchUrl() {
return siteSearchUrl;
}
// Private OneDrive API Functions
private void addIncludeFeatureRequestHeader(string[string]* headers) {
if (appConfig.accountType == "personal") {
// Add logging message for OneDrive Personal Accounts
if (debugLogging) {addLogEntry("Adding 'Include-Feature=AddToOneDrive' API request header for OneDrive Personal Account Type", ["debug"]);}
} else {
// Add logging message for OneDrive Business Accounts
if (debugLogging) {addLogEntry("Adding 'Include-Feature=AddToOneDrive' API request header as 'sync_business_shared_items' config option is enabled", ["debug"]);}
}
// Add feature to request headers
(*headers)["Prefer"] = "Include-Feature=AddToOneDrive";
}
private void redeemToken(string authCode) {
string postData =
"client_id=" ~ clientId ~
"&redirect_uri=" ~ encodeComponent(redirectUrl) ~
"&code=" ~ encodeComponent(authCode) ~
"&grant_type=authorization_code";
acquireToken(postData.dup);
}
private void acquireToken(char[] postData) {
// Set this function name
string thisFunctionName = format("%s.%s", strip(__MODULE__) , strip(getFunctionName!({})));
// Configure the response JSON
JSONValue response;
// Log what we are doing
if (debugLogging) {
addLogEntry("acquireToken: requesting new access token using refresh token (value redacted)", ["debug"]);
}
// Try and process the 'postData' content
try {
response = post(tokenUrl, postData, null, true, "application/x-www-form-urlencoded");
} catch (OneDriveException exception) {
// an error was generated
if ((exception.httpStatusCode == 400) || (exception.httpStatusCode == 401)) {
// Release curl engine
releaseCurlEngine();
// Handle an unauthorised client
handleClientUnauthorised(exception.httpStatusCode, exception.error);
// Must force exit here, allow logging to be done
forceExit();
} else {
if (exception.httpStatusCode >= 500) {
// There was a HTTP 5xx Server Side Error - retry
acquireToken(postData);
} else {
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
}
}
}
if (response.type() == JSONType.object) {
// Debug the provided response
if (debugLogging) {
string scopes = ("scope" in response) ? response["scope"].str() : "<none>";
string tokenType = ("token_type" in response) ? response["token_type"].str() : "<none>";
long expiresIn = ("expires_in" in response) ? response["expires_in"].integer() : -1;
addLogEntry("acquireToken post response: token_type=" ~ tokenType ~ ", expires_in=" ~ to!string(expiresIn) ~ ", scope=" ~ scopes, ["debug"]);
}
// Has the client been configured to use read_only_auth_scope
if (appConfig.getValueBool("read_only_auth_scope")) {
// read_only_auth_scope has been configured
if ("scope" in response){
string effectiveScopes = response["scope"].str();
// Display the effective authentication scopes
addLogEntry();
if (verboseLogging) {addLogEntry("Effective API Authentication Scopes: " ~ effectiveScopes, ["verbose"]);}
// if we have any write scopes, we need to tell the user to update an remove online prior authentication and exit application
if (canFind(effectiveScopes, "Write")) {
// effective scopes contain write scopes .. so not a read-only configuration
addLogEntry();
addLogEntry("ERROR: You have authentication scopes that allow write operations. You need to remove your existing application access consent");
addLogEntry();
addLogEntry("Please login to https://account.live.com/consent/Manage and remove your existing application access consent");
addLogEntry();
// force exit
releaseCurlEngine();
// Must force exit here, allow logging to be done
forceExit();
}
}
}
if ("access_token" in response) {
// Process the response JSON
processAuthenticationJSON(response);
} else {
// Release curl engine
releaseCurlEngine();
// Log error message
addLogEntry("\nInvalid authentication response from OneDrive. Please check the response uri\n");
// re-authorize
authorise();
}
} else {
// Release curl engine
releaseCurlEngine();
addLogEntry("Invalid response from the Microsoft Graph API. Unable to initialise OneDrive API instance.");
// Must force exit here, allow logging to be done
forceExit();
}
}
// Process the authentication JSON
private void processAuthenticationJSON(JSONValue response) {
// Set this function name
string thisFunctionName = format("%s.%s", strip(__MODULE__) , strip(getFunctionName!({})));
// Use 'access_token' and set in the application configuration
appConfig.accessToken = "bearer " ~ strip(response["access_token"].str);
// Do we print the current access token
debugOutputAccessToken();
// Obtain the 'refresh_token' and its expiry
refreshToken = strip(response["refresh_token"].str);
appConfig.accessTokenExpiration = Clock.currTime() + dur!"seconds"(response["expires_in"].integer());
// Debug this response
if (debugLogging) {addLogEntry("appConfig.accessTokenExpiration = " ~ to!string(appConfig.accessTokenExpiration), ["debug"]);}
if (!dryRun) {
// Update the refreshToken in appConfig so that we can reuse it
if (appConfig.refreshToken.empty) {
// The access token is empty
if (debugLogging) {addLogEntry("Updating appConfig.refreshToken with new refreshToken as appConfig.refreshToken is empty", ["debug"]);}
appConfig.refreshToken = refreshToken;
} else {
// Is the access token different?
if (appConfig.refreshToken != refreshToken) {
// Update the memory version
if (debugLogging) {addLogEntry("Updating appConfig.refreshToken with updated refreshToken", ["debug"]);}
appConfig.refreshToken = refreshToken;
}
}
// try and update the 'refresh_token' file on disk
try {
if (debugLogging) {addLogEntry("Updating 'refresh_token' on disk", ["debug"]);}
std.file.write(appConfig.refreshTokenFilePath, refreshToken);
if (debugLogging) {addLogEntry("Setting file permissions for: " ~ appConfig.refreshTokenFilePath, ["debug"]);}
appConfig.refreshTokenFilePath.setAttributes(appConfig.returnSecureFilePermission());
} catch (FileException exception) {
// display the error message
displayFileSystemErrorMessage(exception.msg, thisFunctionName, appConfig.refreshTokenFilePath);
}
}
}
private void generateNewAccessToken() {
if (debugLogging) {addLogEntry("Need to generate a new access token for Microsoft OneDrive", ["debug"]);}
// Has the client been configured to use Intune SSO via Microsoft Identity Broker (microsoft-identity-broker) dbus session
if (appConfig.getValueBool("use_intune_sso")) {
// The client is configured to use Intune SSO via Microsoft Identity Broker dbus session
auto intuneAuthResult = acquire_token_silently(appConfig.intuneAccountDetails, appConfig.getValueString("application_id"));
JSONValue intuneBrokerJSONData = intuneAuthResult.brokerTokenResponse;
// Is the JSON data valid?
if ((intuneBrokerJSONData.type() == JSONType.object)) {
// Does the JSON data have the required renewal elements:
// - accessToken
// - account
// - expiresOn
if ((hasAccessTokenData(intuneBrokerJSONData)) && (hasAccountData(intuneBrokerJSONData)) && (hasExpiresOn(intuneBrokerJSONData))) {
// Details exist
processIntuneResponse(intuneBrokerJSONData);
} else {
// no ... expected values not available
addLogEntry("Required Intune JSON elements are not present in the Intune JSON response");
}
} else {
// Not a valid JSON response
addLogEntry("Invalid Intune JSON response when attempting access token renewal");
}
} else {
// Normal authentication method
auto postData = appender!(string)();
postData ~= "client_id=" ~ clientId;
postData ~= "&redirect_uri=" ~ redirectUrl;
postData ~= "&refresh_token=" ~ to!string(refreshToken);
postData ~= "&grant_type=refresh_token";
acquireToken(postData.data.dup);
}
}
// Check if the existing access token has expired, if it has, generate a new one
private void checkAccessTokenExpired() {
if (Clock.currTime() >= appConfig.accessTokenExpiration) {
if (debugLogging) {addLogEntry("Microsoft OneDrive OAuth2 Access Token has expired. Must generate a new Microsoft OneDrive OAuth2 Access Token", ["debug"]);}
generateNewAccessToken();
} else {
if (debugLogging) {addLogEntry("Microsoft OneDrive OAuth2 Access Token Valid Until (Local): " ~ to!string(appConfig.accessTokenExpiration), ["debug"]);}
}
}
private string getAccessToken() {
checkAccessTokenExpired();
return to!string(appConfig.accessToken);
}
private void addAccessTokenHeader(string[string]* requestHeaders) {
(*requestHeaders)["Authorization"] = getAccessToken();
}
private void connect(HTTP.Method method, const(char)[] url, bool skipToken, CurlResponse response, string[string] requestHeaders=null) {
// If we are debug logging, output the URL being accessed and the HTTP method being used to access that URL
if (debugLogging) {addLogEntry("HTTP " ~ to!string(method) ~ " request to URL: " ~ to!string(url), ["debug"]);}
// Check access token first in case the request is overridden
if (!skipToken) addAccessTokenHeader(&requestHeaders);
curlEngine.setResponseHolder(response);
foreach(k, v; requestHeaders) {
curlEngine.addRequestHeader(k, v);
}
curlEngine.connect(method, url);
}
private void performDelete(const(char)[] url, string[string] requestHeaders=null, string callingFunction=__FUNCTION__, int lineno=__LINE__) {
bool validateJSONResponse = false;
oneDriveErrorHandlerWrapper((CurlResponse response) {
connect(HTTP.Method.del, url, false, response, requestHeaders);
return curlEngine.execute();
}, validateJSONResponse, callingFunction, lineno);
}
private void performPermanentDelete(const(char)[] url, string[string] requestHeaders=null, string callingFunction=__FUNCTION__, int lineno=__LINE__) {
bool validateJSONResponse = false;
oneDriveErrorHandlerWrapper((CurlResponse response) {
connect(HTTP.Method.post, url, false, response, requestHeaders);
curlEngine.setZeroContentLength();
return curlEngine.execute();
}, validateJSONResponse, callingFunction, lineno);
}
// Download a file based on the URL request
private void downloadFile(const(char)[] driveId, const(char)[] itemId, const(char)[] url, string filename, long fileSize, JSONValue onlineHash, long resumeOffset = 0, string callingFunction=__FUNCTION__, int lineno=__LINE__) {
// Threshold for displaying download bar
long thresholdFileSize = 4 * 2^^20; // 4 MiB
// To support marking of partially-downloaded files
string originalFilename = filename;
string downloadFilename = filename ~ ".partial";
// To support resumable downloads, configure the 'resumable data' file path
string threadResumeDownloadFilePath = appConfig.resumeDownloadFilePath ~ "." ~ generateAlphanumericString();
// Create a JSONValue with download state so this can be used when resuming, to evaluate if the online file has changed, and if we are able to resume in a safe manner
JSONValue resumeDownloadData = JSONValue([
"driveId": JSONValue(to!string(driveId)),
"itemId": JSONValue(to!string(itemId)),
"onlineHash": onlineHash,
"originalFilename": JSONValue(originalFilename),
"downloadFilename": JSONValue(downloadFilename),
"resumeOffset": JSONValue(to!string(resumeOffset))
]);
// ----------------------------------------------------------------------
// Progress state must live for the whole downloadFile() call so that
// retries triggered by oneDriveErrorHandlerWrapper() do NOT reset the
// visible progress bar back to 0%.
// ----------------------------------------------------------------------
size_t expected_total_segments = 20;
SysTime startTime = Clock.currTime();
long start_unix_time = startTime.toUnixTime();
int h, m, s;
string etaString;
bool barInit = false;
real previousProgressPercent = 0.0; // last *displayed* percent
real percentCheck = 5.0;
size_t segmentCount = 0;
// Validate the JSON response
bool validateJSONResponse = false;
oneDriveErrorHandlerWrapper((CurlResponse response) {
connect(HTTP.Method.get, url, false, response);
if (fileSize >= thresholdFileSize) {
// ------------------------------------------------------------------
// Determine an effective resume offset for this attempt.
//
// - Start from the passed-in resumeOffset (from resume_download.*)
// - If a .partial file exists and is larger, prefer its size.
// This ensures we never re-download bytes we already have on disk.
// ------------------------------------------------------------------
long effectiveResumeOffset = resumeOffset;
if (exists(downloadFilename)) {
try {
auto partialSize = cast(long) getSize(downloadFilename);
if (partialSize > effectiveResumeOffset) {
if (debugLogging) {
addLogEntry(
"Resumable download: detected existing partial file '" ~ downloadFilename ~
"' of size " ~ to!string(partialSize) ~ " bytes",
["debug"]
);
addLogEntry(
"Adjusting resumable offset for '" ~ originalFilename ~
"' from " ~ to!string(effectiveResumeOffset) ~
" to " ~ to!string(partialSize),
["debug"]
);
}
effectiveResumeOffset = partialSize;
}
} catch (FileException ex) {
if (debugLogging) {
addLogEntry(
"Failed to obtain size of partial download file '" ~ downloadFilename ~
"': " ~ ex.msg,
["debug"]
);
}
}
}
// If we have a resumable offset to use, set this as the offset to use
if (effectiveResumeOffset > 0) {
curlEngine.setDownloadResumeOffset(effectiveResumeOffset);
// Keep the JSON state in sync with the absolute offset
resumeDownloadData["resumeOffset"] = JSONValue(to!string(effectiveResumeOffset));
}
// Setup progress bar to display
curlEngine.http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) {
string downloadLogEntry = "Downloading: " ~ filename ~ " ... ";
// ------------------------------------------------------------------
// Compute absolute progress as bytes_on_disk + bytes_this_transfer.
// This ensures that after a retry, the percentage continues from
// (for example) 25% instead of restarting at 0%.
// ------------------------------------------------------------------
long absoluteNow = effectiveResumeOffset + cast(long)dlnow;
long absoluteTotal;
if (fileSize > 0) {
absoluteTotal = fileSize;
} else if (dltotal > 0) {
absoluteTotal = effectiveResumeOffset + cast(long)dltotal;
} else {
absoluteTotal = absoluteNow; // best effort; avoids div-by-zero
}
if (absoluteTotal <= 0) {
absoluteTotal = 1; // safety guard
}
// Floor to nearest whole number
real currentDLPercent = floor(
(cast(real) absoluteNow / cast(real) absoluteTotal) * 100.0
);
// Clamp just in case
if (currentDLPercent < 0.0) {
currentDLPercent = 0.0;
} else if (currentDLPercent > 100.0) {
currentDLPercent = 100.0;
}
// Debug logging (optional, but handy while were testing)
if (debugLogging) {
addLogEntry("", ["debug"]);
addLogEntry("absoluteNow = " ~ to!string(absoluteNow), ["debug"]);
addLogEntry("absoluteTotal = " ~ to!string(absoluteTotal), ["debug"]);
addLogEntry("Percent Complete = " ~ to!string(currentDLPercent), ["debug"]);
}
// Have we started downloading (in absolute terms)?
if (currentDLPercent > 0) {
// Has the user set a data rate limit?
// when using rate_limit, we will get odd download rates, for example:
// Percent Complete = 24
// Data Received = 13080163
// Expected Total = 52428800
// Percent Complete = 24
// Data Received = 13685777
// Expected Total = 52428800
// Percent Complete = 26 <---- jumps to 26% missing 25%, thus fmod misses incrementing progress bar
// Data Received = 13685777
// Expected Total = 52428800
// Percent Complete = 26
if (appConfig.getValueLong("rate_limit") > 0) {
// Under rate limiting, libcurl can "jump" the visible percentage,
// e.g. 24% -> 26%, which can skip a clean 5% boundary.
// To keep a stable 5% display (5, 10, 15, ...), we use a
// catch-up loop that prints every missing 5% step up to
// currentDLPercent, based on the *absolute* percentage.
real nextPercent = previousProgressPercent + percentCheck;
// Emit all missing 5% steps below 100%
while (nextPercent < 100.0 && currentDLPercent >= nextPercent) {
if (debugLogging) {
addLogEntry("Incrementing Progress Bar (rate_limit) to " ~ to!string(nextPercent) ~ "%", ["debug"]);
}
segmentCount++;
etaString = formatETA(calc_eta(segmentCount, expected_total_segments, start_unix_time));
string percentage = leftJustify(to!string(cast(int) nextPercent) ~ "%", 5, ' ');
addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]);
previousProgressPercent = nextPercent;
nextPercent += percentCheck;
}
// Handle 100% exactly once
if ((currentDLPercent >= 100.0) && (previousProgressPercent < 100.0)) {
SysTime endTime = Clock.currTime();
long end_unix_time = endTime.toUnixTime();
int download_duration = cast(int)(end_unix_time - start_unix_time);
dur!"seconds"(download_duration).split!("hours", "minutes", "seconds")(h, m, s);
etaString = format!"| DONE in %02d:%02d:%02d"(h, m, s);
string percentage = leftJustify("100%", 5, ' ');
addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]);
previousProgressPercent = 100.0;
}
} else {
// Non-rate-limited case: fmod-based behaviour but applied to the absolute percentage
if ((isIdentical(fmod(currentDLPercent, percentCheck), 0.0)) &&
(previousProgressPercent != currentDLPercent)) {
// currentDLPercent matches a new increment
if (debugLogging) {
addLogEntry("Incrementing Progress Bar using fmod match", ["debug"]);
}
if (currentDLPercent != 100) {
// Not 100% yet
segmentCount++;
etaString = formatETA(calc_eta(segmentCount, expected_total_segments, start_unix_time));
string percentage = leftJustify(to!string(cast(int) currentDLPercent) ~ "%", 5, ' ');
addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]);
} else {
// 100% done
SysTime endTime = Clock.currTime();
long end_unix_time = endTime.toUnixTime();
int download_duration = cast(int)(end_unix_time - start_unix_time);
dur!"seconds"(download_duration).split!("hours", "minutes", "seconds")(h, m, s);
etaString = format!"| DONE in %02d:%02d:%02d"(h, m, s);
string percentage = leftJustify("100%", 5, ' ');
addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]);
}
previousProgressPercent = currentDLPercent;
}
}
// Has our absolute offset advanced?
if (absoluteNow > to!long(resumeDownloadData["resumeOffset"].str)) {
// Update resumeOffset for this progress event with the latest absolute offset
resumeDownloadData["resumeOffset"] = JSONValue(to!string(absoluteNow));
// Save resumable download data - this needs to be saved on every onProgress event that is processed
saveResumeDownloadFile(threadResumeDownloadFilePath, resumeDownloadData);
}
} else {
// We may get frequent progress callbacks at 0%, make sure we initialise the bar once per overall download
if ((currentDLPercent == 0) && (!barInit)) {
etaString = "| ETA --:--:--";
string percentage = leftJustify("0%", 5, ' ');
addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]);
barInit = true;
}
}
return 0;
};
} else {
// No progress bar, no resumable download
}
// Capture the result of the download action
auto result = curlEngine.download(originalFilename, downloadFilename);
// Safe remove 'threadResumeDownloadFilePath' as if we get to this point, the file has been downloaded successfully
safeRemove(threadResumeDownloadFilePath);
// Reset this curlEngine offset value now that the file has been downloaded successfully
curlEngine.resetDownloadResumeOffset();
// Return the applicable result
return result;
}, validateJSONResponse, callingFunction, lineno);
}
// Save the resume download data
private void saveResumeDownloadFile(string threadResumeDownloadFilePath, JSONValue resumeDownloadData) {
// Set this function name
string thisFunctionName = format("%s.%s", strip(__MODULE__) , strip(getFunctionName!({})));
try {
std.file.write(threadResumeDownloadFilePath, resumeDownloadData.toString());
} catch (FileException e) {
// display the error message
displayFileSystemErrorMessage(e.msg, thisFunctionName, threadResumeDownloadFilePath);
}
}
private JSONValue get(string url, bool skipToken = false, string[string] requestHeaders=null, string callingFunction=__FUNCTION__, int lineno=__LINE__) {
bool validateJSONResponse = true;
return oneDriveErrorHandlerWrapper((CurlResponse response) {
connect(HTTP.Method.get, url, skipToken, response, requestHeaders);
return curlEngine.execute();
}, validateJSONResponse, callingFunction, lineno);
}
private JSONValue patch(const(char)[] url, const(char)[] patchData, bool validateJSONResponseInput, string[string] requestHeaders=null, const(char)[] contentType = "application/json", string callingFunction=__FUNCTION__, int lineno=__LINE__) {
bool validateJSONResponse = validateJSONResponseInput;
return oneDriveErrorHandlerWrapper((CurlResponse response) {
connect(HTTP.Method.patch, url, false, response, requestHeaders);
curlEngine.setContent(contentType, patchData);
return curlEngine.execute();
}, validateJSONResponse, callingFunction, lineno);
}
private JSONValue post(const(char)[] url, const(char)[] postData, string[string] requestHeaders=null, bool skipToken = false, const(char)[] contentType = "application/json", string callingFunction=__FUNCTION__, int lineno=__LINE__) {
bool validateJSONResponse = true;
return oneDriveErrorHandlerWrapper((CurlResponse response) {
connect(HTTP.Method.post, url, skipToken, response, requestHeaders);
curlEngine.setContent(contentType, postData);
return curlEngine.execute();
}, validateJSONResponse, callingFunction, lineno);
}
private JSONValue put(const(char)[] url, string filepath, bool skipToken=false, string contentRange=null, ulong offset=0, ulong offsetSize=0, string callingFunction=__FUNCTION__, int lineno=__LINE__) {
bool validateJSONResponse = true;
return oneDriveErrorHandlerWrapper((CurlResponse response) {
connect(HTTP.Method.put, url, skipToken, response);
curlEngine.setFile(filepath, contentRange, offset, offsetSize);
return curlEngine.execute();
}, validateJSONResponse, callingFunction, lineno);
}
// Wrapper function for all requests to OneDrive API
// - This should throw a OneDriveException so that this exception can be handled appropriately elsewhere in the application
private JSONValue oneDriveErrorHandlerWrapper(CurlResponse delegate(CurlResponse response) executer, bool validateJSONResponse, string callingFunction, int lineno) {
// Create a new 'curl' response
response = new CurlResponse();
// Other wrapper variables
int retryAttempts = 0;
int baseBackoffInterval = 1; // Base backoff interval in seconds
int maxRetryCount = 175200; // Approx 365 days based on maxBackoffInterval + appConfig.defaultDataTimeout
//int maxRetryCount = 5; // Temp
int maxBackoffInterval = 120; // Maximum backoff interval in seconds
int thisBackOffInterval = 0;
int timestampAlign = 0;
JSONValue result;
SysTime currentTime;
SysTime retryTime;
bool retrySuccess = false;
bool transientError = false;
bool sslVerifyPeerDisabled = false;
while (!retrySuccess) {
// Reset thisBackOffInterval
thisBackOffInterval = 0;
transientError = false;
if (retryAttempts >= 1) {
// re-try log entry & clock time
retryTime = Clock.currTime();
retryTime.fracSecs = Duration.zero;
addLogEntry("Retrying the respective Microsoft Graph API call for Internal Thread ID: " ~ to!string(curlEngine.internalThreadId) ~ " (Timestamp: " ~ to!string(retryTime) ~ ") ...");
}
try {
response.reset();
response = executer(response);
// Check for a valid response
if (response.hasResponse) {
// Process the response
result = response.json();
// Print response if 'debugHTTPSResponse' is flagged
if (debugHTTPSResponse){
if (debugLogging) {addLogEntry("Microsoft Graph API Response: " ~ response.dumpResponse(), ["debug"]);}
}
// Check http response code, raise a OneDriveException if the operation was not successfully performed
if (checkHttpResponseCode(response.statusLine.code)) {
// 'curl' on platforms like Ubuntu does not reliably provide the 'http.statusLine.reason' when using HTTP/2
// This is a curl bug, but because Ubuntu uses old packages and never updates them, we are stuck with working around this bug
if (response.statusLine.reason.length == 0) {
// No 'reason', fetch what it should have been
response.statusLine.reason = getMicrosoftGraphStatusMessage(response.statusLine.code);
}
// Why are throwing a OneDriveException - do not do this for a 404 error as this is not required as we use a 404 if things are not online, to create them
if (response.statusLine.code != 404) {
if (debugLogging) {
addLogEntry("response.statusLine.code: " ~ to!string(response.statusLine.code), ["debug"]);
addLogEntry("response.statusLine.reason: " ~ to!string(response.statusLine.reason), ["debug"]);
addLogEntry("actual curl response: " ~ to!string(response), ["debug"]);
}
}
// For every HTTP error status code, including those from 3xx (other Redirection codes excluding 302), 4xx (Client Error), and 5xx (Server Error) series, will trigger the following line of code.
throw new OneDriveException(response.statusLine.code, response.statusLine.reason, response);
}
// Do we need to validate the JSON response?
if (validateJSONResponse) {
const code = response.statusLine.code;
// 204 = No Content is a valid success response for some Graph operations (e.g. PATCH/DELETE).
// In that case, there is no JSON payload to validate.
if (code != 204) {
// If caller expects JSON, an empty body is not acceptable
if (response.content.length == 0) {
throw new OneDriveException( 0, "Caller requested a JSON object response, but the response body was empty", response);
}
// Body is present: it must be a JSON object
if (result.type() != JSONType.object) {
throw new OneDriveException(0, "Caller requested a JSON object response, but the response was not a JSON object", response);
}
}
}
// If we get to this point, there is no error from http.perform() on re-try
// If retryAttempts is greater than 1, it means we were re-trying the request
if (retryAttempts > 1) {
// unset the fresh connect option as this then creates performance issues if left enabled
unsetFreshConnectOption();
}
// On successful http.perform() processing, break out of the loop
break;
} else {
// Throw a custom 506 error
// Whilst this error code is a bit more esoteric and typically involves content negotiation issues that lead to a configuration error on the server, but it could be loosely
// interpreted to signal that the response received didn't meet the expected criteria or format.
throw new OneDriveException(506, "Received an unexpected response from Microsoft OneDrive", response);
}
// A 'curl' exception was thrown
} catch (CurlException exception) {
// Handle 'curl' exception errors
// Detail the curl exception, debug output only
if (debugLogging) {
addLogEntry("Handling a curl exception:", ["debug"]);
addLogEntry(to!string(response), ["debug"]);
}
// Parse and display error message received from OneDrive
if (debugLogging) {addLogEntry(callingFunction ~ "() - Generated a OneDrive CurlException", ["debug"]);}
auto errorArray = splitLines(exception.msg);
string errorMessage = errorArray[0];
// Configure libcurl to perform a fresh connection
setFreshConnectOption();
// What is contained in the curl error message?
// Handle the following:
// - Couldn't connect to server on handle
// - Could not connect to server on handle (changed noticed in curl 8.14.1, possibly done earlier ...)
// - Couldn't resolve host name on handle
// - Could not resolve host name on handle (changed noticed in curl 8.14.1, possibly done earlier ...)
// - Timeout was reached on handle
if (canFind(errorMessage, "connect to server on handle") || canFind(errorMessage, "resolve host name on handle") || canFind(errorMessage, "resolve hostname on handle") || canFind(errorMessage, "Timeout was reached on handle")) {
// Connectivity to Microsoft OneDrive was lost
addLogEntry("Internet connectivity to Microsoft OneDrive service has been interrupted .. re-trying in the background");
// What caused the initial curl exception?
// - DNS resolution issue
if (canFind(errorMessage, "resolve host name on handle")) {
if (debugLogging) {addLogEntry("Unable to resolve server - DNS access blocked?", ["debug"]);}
}
// - connection issue
if (canFind(errorMessage, "connect to server on handle")) {
if (debugLogging) {addLogEntry("Unable to connect to server - HTTPS access blocked?", ["debug"]);}
}
// - timeout issue
if (canFind(errorMessage, "Timeout was reached on handle")) {
// Common cause is libcurl trying IPv6 DNS resolution when there are only IPv4 DNS servers available
if (verboseLogging) {
addLogEntry("A libcurl timeout has been triggered - data transfer too slow, no DNS resolution response, no server response or operational timeout", ["verbose"]);
// There are 3 common causes for this issue:
// 1. Usually poor DNS resolution where libcurl flip/flops to use IPv6 and is unable to resolve
// 2. A device between the user and Microsoft OneDrive is unable to correctly handle HTTP/2 communication
// 3. No Internet access from this system at this point in time
addLogEntry(" - IPv6 DNS resolution issues may be causing timeouts. Consider setting 'ip_protocol_version' to IPv4 to potentially avoid this", ["verbose"]);
addLogEntry(" - HTTP/2 compatibility issues might also be interfering with your system. Use 'force_http_11' to switch to HTTP/1.1 to potentially avoid this", ["verbose"]);
addLogEntry(" - Ensure 'operation_timeout' is configured for the conditions of your network, covering DNS lookups, connection setup, TLS negotiation, and how long data transfers normally take", ["verbose"]);
addLogEntry(" - If these options do not resolve this timeout issue, please use --debug-https to diagnose this issue further.", ["verbose"]);
}
}
} else {
// Some other 'libcurl' error was returned
if (canFind(errorMessage, "Problem with the SSL CA cert (path? access rights?) on handle")) {
// error setting certificate verify locations:
// CAfile: /etc/pki/tls/certs/ca-bundle.crt
// CApath: none
//
// Tell the Curl Engine to bypass SSL check - essentially SSL is passing back a bad value due to 'stdio' compile time option
// Further reading:
// https://github.com/curl/curl/issues/6090
// https://github.com/openssl/openssl/issues/7536
// https://stackoverflow.com/questions/45829588/brew-install-fails-curl77-error-setting-certificate-verify
// https://forum.dlang.org/post/vwvkbubufexgeuaxhqfl@forum.dlang.org
string sslCertReadErrorMessage = "System SSL CA certificates are missing or unreadable by libcurl please ensure the correct CA bundle is installed and is accessible.";
addLogEntry("ERROR: " ~ sslCertReadErrorMessage);
throw new OneDriveError(sslCertReadErrorMessage);
} else {
// Was this a curl initialization error?
if (canFind(errorMessage, "Failed initialization on handle")) {
// initialization error ... prevent a run-away process if we have zero disk space
ulong localActualFreeSpace = getAvailableDiskSpace(".");
if (localActualFreeSpace == 0) {
throw new OneDriveError("Zero disk space detected");
}
} else {
// Unknown curl error
displayGeneralErrorMessage(exception, callingFunction, lineno);
// Fallback: Ensure retry interval is enforced in case of unknown CurlException
if (thisBackOffInterval == 0) {
thisBackOffInterval = calculateBackoff(retryAttempts, baseBackoffInterval, maxBackoffInterval);
if (thisBackOffInterval <= 0) {
thisBackOffInterval = 1;
addLogEntry("WARNING: Enforcing minimum backoff interval of 1 second unclassified CurlException");
}
}
}
}
}
// A OneDrive API exception was thrown
} catch (OneDriveException exception) {
// https://developer.overdrive.com/docs/reference-guide
// https://learn.microsoft.com/en-us/onedrive/developer/rest-api/concepts/errors?view=odsp-graph-online
// https://learn.microsoft.com/en-us/graph/errors
/**
HTTP/1.1 Response handling
Errors in the OneDrive API are returned using standard HTTP status codes, as well as a JSON error response object. The following HTTP status codes should be expected.
Status code Status message Description
100 Continue Continue
200 OK Request was handled OK
201 Created This means you've made a successful POST to checkout, lock in a format, or place a hold
204 No Content This means you've made a successful DELETE to remove a hold or return a title
400 Bad Request Cannot process the request because it is malformed or incorrect.
401 Unauthorized Required authentication information is either missing or not valid for the resource.
403 Forbidden Access is denied to the requested resource. The user might not have enough permission.
404 Not Found The requested resource doesnt exist.
405 Method Not Allowed The HTTP method in the request is not allowed on the resource.
406 Not Acceptable This service doesnt support the format requested in the Accept header.
408 Request Time out CUSTOM ERROR - Not expected from OneDrive, but can be used to handle Internet connection failures the same (fallback and try again)
409 Conflict The current state conflicts with what the request expects. For example, the specified parent folder might not exist.
410 Gone The requested resource is no longer available at the server.
411 Length Required A Content-Length header is required on the request.
412 Precondition Failed A precondition provided in the request (such as an if-match header) does not match the resource's current state.
413 Request Entity Too Large The request size exceeds the maximum limit.
415 Unsupported Media Type The content type of the request is a format that is not supported by the service.
416 Requested Range Not Satisfiable The specified byte range is invalid or unavailable.
422 Unprocessable Entity Cannot process the request because it is semantically incorrect.
423 Locked The file is currently checked out or locked for editing by another user
429 Too Many Requests Client application has been throttled and should not attempt to repeat the request until an amount of time has elapsed.
500 Internal Server Error There was an internal server error while processing the request.
501 Not Implemented The requested feature isnt implemented.
502 Bad Gateway The service was unreachable
503 Service Unavailable The service is temporarily unavailable. You may repeat the request after a delay. There may be a Retry-After header.
504 Gateway Timeout The server, which is acting as a gateway or proxy, did not receive a timely response from an upstream server it needed to access in order to complete the request
506 Variant Also Negotiates CUSTOM ERROR - Received an unexpected response from Microsoft OneDrive
507 Insufficient Storage The maximum storage quota has been reached.
509 Bandwidth Limit Exceeded Your app has been throttled for exceeding the maximum bandwidth cap. Your app can retry the request again after more time has elapsed.
HTTP/2 Response handling
0 OK
**/
// Detail the OneDriveAPI exception, debug output only
if (debugLogging) {
addLogEntry("Handling a OneDrive API exception:", ["debug"]);
addLogEntry(to!string(response), ["debug"]);
// Parse and display error message received from OneDrive
addLogEntry(callingFunction ~ "() - Generated a OneDriveException", ["debug"]);
}
// Perform action based on the HTTP Status Code
switch(exception.httpStatusCode) {
// 0 - OK ... HTTP/2 version of 200 OK
case 0:
break;
// 100 - Continue
case 100:
break;
// 408 - Request Time Out
// 429 - Too Many Requests, backoff
case 408,429:
// If OneDrive sends a status code 429 then this function will be used to process the Retry-After response header which contains the value by which we need to wait
if (exception.httpStatusCode == 408) {
addLogEntry("Handling a Microsoft Graph API HTTP 408 Response Code (Request Time Out) - Internal Thread ID: " ~ to!string(curlEngine.internalThreadId));
} else {
addLogEntry("Handling a Microsoft Graph API HTTP 429 Response Code (Too Many Requests) - Internal Thread ID: " ~ to!string(curlEngine.internalThreadId));
}
// Read in the Retry-After HTTP header as set and delay as per this value before retrying the request
thisBackOffInterval = response.getRetryAfterValue();
if (debugLogging) {addLogEntry("Using Retry-After Value = " ~ to!string(thisBackOffInterval), ["debug"]);}
transientError = true;
break;
// Transient errors
// 503 - Service Unavailable
// 504 - Gateway Timeout
case 503,504:
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request
auto errorArray = splitLines(exception.msg);
addLogEntry(to!string(errorArray[0]) ~ " when attempting to query the Microsoft Graph API Service - retrying applicable request in 30 seconds - Internal Thread ID: " ~ to!string(curlEngine.internalThreadId));
if (debugLogging) {addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);}
// Transient error - try again in 30 seconds
thisBackOffInterval = 30;
transientError = true;
break;
// Default
default:
// This exception should be then passed back to the original calling function for handling a OneDriveException
throw new OneDriveException(response.statusLine.code, response.statusLine.reason, response);
}
// A FileSystem exception was thrown from somewhere
} catch (FileException exception) {
// There was a file system error - display the error message
displayFileSystemErrorMessage(exception.msg, callingFunction, ""); // as we have no file path reference here, use a blank input
throw new OneDriveException(0, "There was a file system error during OneDrive request: " ~ exception.msg, response);
// A OneDriveError was thrown
} catch (OneDriveError exception) {
// Disk space error or SSL error caused a OneDriveError to be thrown
/**
DO NOT UNCOMMENT THIS CODE UNLESS TESTING FOR THIS ISSUE: System SSL CA certificates are missing or unreadable by libcurl
// Disk space error or SSL error
if (getAvailableDiskSpace(".") == 0) {
// Must exit
forceExit();
} else {
// Catch the SSL error
addLogEntry("WARNING: Disabling SSL peer verification due to libcurl failing to access the system CA certificate bundle (CAfile missing, unreadable, or misconfigured).");
sslVerifyPeerDisabled = true;
curlEngine.setDisableSSLVerifyPeer();
}
**/
// Must exit
forceExit();
}
// Increment re-try counter
retryAttempts++;
// Configure libcurl to perform a fresh connection on API retry
setFreshConnectOption();
// Has maxRetryCount been reached?
if (retryAttempts > maxRetryCount) {
addLogEntry("ERROR: Unable to reconnect to the Microsoft OneDrive service after " ~ to!string(retryAttempts) ~ " attempts lasting approximately 365 days");
throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?", response);
} else {
// Was 'thisBackOffInterval' set by a 429 event ?
if (thisBackOffInterval == 0) {
// Calculate and apply exponential backoff upto a maximum of 120 seconds before the API call is re-tried
thisBackOffInterval = calculateBackoff(retryAttempts, baseBackoffInterval, maxBackoffInterval);
// If this 'somehow' calculates a negative number, this is not correct .. and this has been seen in testing - unknown cause
//
// Retry attempt: 31 - Internal Thread ID: ICO4ELBlGXFwyTzh
// This attempt timestamp: 2024-Aug-10 10:32:07
// Next retry in approx: -2147483648 seconds
// Next retry approx: 1956-Jul-23 07:17:59
// Illegal instruction (core dumped)
//
// Set to 'maxBackoffInterval' if calculated value is negative
if (thisBackOffInterval < 0) {
thisBackOffInterval = maxBackoffInterval;
}
}
// set the current time for this thread
currentTime = Clock.currTime();
currentTime.fracSecs = Duration.zero;
// If verbose logging, detail when we are re-trying the call
if (verboseLogging) {
auto timeString = currentTime.toString();
addLogEntry("Retry attempt: " ~ to!string(retryAttempts) ~ " - Internal Thread ID: " ~ to!string(curlEngine.internalThreadId), ["verbose"]);
addLogEntry(" This attempt timestamp: " ~ timeString, ["verbose"]);
// Detail when the next attempt will be tried
// Factor in the delay for curl to generate the exception - otherwise the next timestamp appears to be 'out' even though technically correct
auto nextRetry = currentTime + dur!"seconds"(thisBackOffInterval) + dur!"seconds"(timestampAlign);
addLogEntry(" Next retry in approx: " ~ to!string((thisBackOffInterval + timestampAlign)) ~ " seconds");
addLogEntry(" Next retry approx: " ~ to!string(nextRetry), ["verbose"]);
}
// Thread sleep
Thread.sleep(dur!"seconds"(thisBackOffInterval));
}
}
// Reset SSL Peer Validation if it was disabled
if (sslVerifyPeerDisabled) {
curlEngine.setEnableSSLVerifyPeer();
}
// Return the result
return result;
}
// Check the HTTP Response code and determine if a OneDriveException should be thrown
private bool checkHttpResponseCode(int httpResponseCode) {
bool shouldThrow = false;
// Redirect Codes
immutable acceptedRedirectCodes = [301, 302, 304, 307, 308];
//
// This condition checks if the HTTP response code falls within the acceptable range for both HTTP 1.1 and HTTP 2.0.
//
// For HTTP 1.1:
// - Any 1xx response (Informational responses, ranging from 100 to 199)
// - Any 2xx response (Successful responses, ranging from 200 to 299)
// - A 302 response (Temporary Redirect)
//
// For HTTP 2.0:
// - Any 1xx response (Informational responses, ranging from 100 to 199)
// - Any 2xx response (Successful responses, ranging from 200 to 299)
// - A 302 response (Temporary Redirect)
// - A 0 response (Interpreted as 200 OK based on empirical evidence)
//
// If the HTTP response code meets any of these conditions, it is considered acceptable, and no exception will be thrown.
//
if ((httpResponseCode >= 100 && httpResponseCode < 300) || canFind(acceptedRedirectCodes, httpResponseCode) || httpResponseCode == 0) {
shouldThrow = false;
} else {
shouldThrow = true;
}
// return evaluation
return shouldThrow;
}
// Calculates the delay for exponential backoff
private int calculateBackoff(int retryAttempts, int baseInterval, int maxInterval) {
int cappedAttempts = min(retryAttempts, 10); // Prevent exponent overflow
int backoff = baseInterval * (1 << cappedAttempts);
return min(backoff, maxInterval);
}
// Configure libcurl to perform a fresh connection
private void setFreshConnectOption() {
if (debugLogging) {addLogEntry("Configuring libcurl to use a fresh connection for re-try", ["debug"]);}
curlEngine.http.handle.set(CurlOption.fresh_connect,1);
// Set libcurl dns_cache_timeout timeout
// https://curl.se/libcurl/c/CURLOPT_DNS_CACHE_TIMEOUT.html
// https://dlang.org/library/std/net/curl/http.dns_timeout.html
curlEngine.http.dnsTimeout = (dur!"seconds"(0));
}
// Unset the libcurl fresh connection options and reset libcurl DNS Cache Timeout
private void unsetFreshConnectOption() {
if (debugLogging) {addLogEntry("Unsetting libcurl to use a fresh connection as this causes a performance impact if left enabled", ["debug"]);}
curlEngine.http.handle.set(CurlOption.fresh_connect,0);
// Reset libcurl dns_cache_timeout timeout
// https://curl.se/libcurl/c/CURLOPT_DNS_CACHE_TIMEOUT.html
// https://dlang.org/library/std/net/curl/http.dns_timeout.html
curlEngine.http.dnsTimeout = (dur!"seconds"(appConfig.getValueLong("dns_timeout")));
}
// Generate a HTTP 'reason' based on the HTTP 'code'
private string getMicrosoftGraphStatusMessage(ushort code) {
string message;
switch (code) {
case 200:
message = "OK";
break;
case 201:
message = "Created";
break;
case 202:
message = "Accepted";
break;
case 204:
message = "No Content";
break;
case 301:
message = "Moved Permanently";
break;
case 302:
message = "Found";
break;
case 304:
message = "Not Modified";
break;
case 307:
message = "Temporary Redirect";
break;
case 308:
message = "Permanent Redirect";
break;
case 400:
message = "Bad Request";
break;
case 401:
message = "Unauthorized";
break;
case 402:
message = "Payment Required";
break;
case 403:
message = "Forbidden";
break;
case 404:
message = "Not Found";
break;
case 405:
message = "Method Not Allowed";
break;
case 406:
message = "Not Acceptable";
break;
case 409:
message = "Conflict";
break;
case 410:
message = "Gone";
break;
case 411:
message = "Length Required";
break;
case 412:
message = "Precondition Failed";
break;
case 413:
message = "Request Entity Too Large";
break;
case 415:
message = "Unsupported Media Type";
break;
case 416:
message = "Requested Range Not Satisfiable";
break;
case 422:
message = "Unprocessable Entity";
break;
case 423:
message = "Locked";
break;
case 429:
message = "Too Many Requests";
break;
case 500:
message = "Internal Server Error";
break;
case 501:
message = "Not Implemented";
break;
case 503:
message = "Service Unavailable";
break;
case 504:
message = "Gateway Timeout";
break;
case 507:
message = "Insufficient Storage";
break;
case 509:
message = "Bandwidth Limit Exceeded";
break;
default:
message = "Unknown Status Code";
break;
}
return message;
}
}