2024-01-08 23:13:17 +01:00
// What is this module called?
module onedrive ;
// What does this module require to function?
import core.stdc.stdlib : EXIT_SUCCESS , EXIT_FAILURE , exit ;
import core.memory ;
import core.thread ;
import std.stdio ;
import std.string ;
import std.utf ;
import std.file ;
import std.exception ;
import std.regex ;
import std.json ;
2019-10-21 09:04:08 +02:00
import std.algorithm.searching ;
2024-01-08 23:13:17 +01:00
import std.net.curl ;
2022-04-21 00:03:22 +02:00
import std.datetime ;
2024-01-08 23:13:17 +01:00
import std.path ;
import std.conv ;
import std.math ;
import std.uri ;
import std.array ;
2021-11-23 20:54:28 +01:00
2024-01-08 23:13:17 +01:00
// Required for webhooks
import std.uuid ;
2021-11-23 20:54:28 +01:00
2024-01-08 23:13:17 +01:00
// What other modules that we have created do we need to import?
import config ;
import log ;
import util ;
import curlEngine ;
2021-11-23 20:54:28 +01:00
2024-01-08 23:13:17 +01:00
// Shared variables between classes
shared bool debugHTTPResponseOutput = false ;
2018-11-23 20:26:30 +01:00
2024-03-30 22:40:50 +01:00
// Define the 'OneDriveException' class
2024-01-08 23:13:17 +01:00
class OneDriveException : Exception {
2017-12-28 15:03:15 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/errors
2016-12-14 15:17:20 +01:00
int httpStatusCode ;
2024-03-30 21:20:04 +01:00
const CurlResponse response ;
2016-12-13 18:04:16 +01:00
JSONValue error ;
2015-10-04 16:24:45 +02:00
2024-03-30 21:20:04 +01:00
this ( ushort httpStatusCode , string reason , const CurlResponse response , string file = __FILE__ , size_t line = __LINE__ ) {
2016-12-14 15:17:20 +01:00
this . httpStatusCode = httpStatusCode ;
2024-03-30 21:20:04 +01:00
this . response = response ;
this . error = response . json ( ) ;
string msg = format ( "HTTP request returned status code %d (%s)\n%s" , httpStatusCode , reason , toJSON ( error , true ) ) ;
2017-12-28 15:03:15 +01:00
super ( msg , file , line ) ;
2015-10-04 16:24:45 +02:00
}
2024-03-30 22:40:50 +01:00
this ( ushort httpStatusCode , string reason , string file = __FILE__ , size_t line = __LINE__ ) {
this . response = null ;
super ( msg , file , line , null ) ;
}
2024-03-30 21:20:04 +01:00
}
2016-12-13 18:04:16 +01:00
2024-03-30 22:40:50 +01:00
// Define the 'OneDriveError' class
2024-03-30 21:20:04 +01:00
class OneDriveError : Error {
this ( string msg ) {
super ( msg ) ;
2015-10-04 16:24:45 +02:00
}
2015-09-01 20:45:34 +02:00
}
2024-03-30 22:40:50 +01:00
// Define the 'OneDriveApi' class
2024-01-08 23:13:17 +01:00
class OneDriveApi {
// Class variables
ApplicationConfig appConfig ;
CurlEngine curlEngine ;
string clientId = "" ;
string companyName = "" ;
string authUrl = "" ;
string redirectUrl = "" ;
string tokenUrl = "" ;
string driveUrl = "" ;
string driveByIdUrl = "" ;
string sharedWithMeUrl = "" ;
string itemByIdUrl = "" ;
string itemByPathUrl = "" ;
string siteSearchUrl = "" ;
string siteDriveUrl = "" ;
2024-02-20 19:13:26 +01:00
string subscriptionUrl = "" ;
2024-01-08 23:13:17 +01:00
string tenantId = "" ;
string authScope = "" ;
const ( char ) [ ] refreshToken = "" ;
bool dryRun = false ;
bool debugResponse = false ;
2024-03-30 21:20:04 +01:00
bool keepAlive = false ;
2024-02-20 19:13:26 +01:00
2024-01-08 23:13:17 +01:00
this ( ApplicationConfig appConfig ) {
// Configure the class varaible to consume the application configuration
this . appConfig = appConfig ;
2024-03-30 21:20:04 +01:00
this . curlEngine = null ;
2024-01-08 23:13:17 +01:00
// Configure the major API Query URL's, based on using application configuration
// These however can be updated by config option 'azure_ad_endpoint', thus handled differently
// Drive Queries
driveUrl = appConfig . globalGraphEndpoint ~ "/v1.0/me/drive" ;
driveByIdUrl = appConfig . globalGraphEndpoint ~ "/v1.0/drives/" ;
2021-11-23 20:54:28 +01:00
2024-01-08 23:13:17 +01:00
// What is 'shared with me' Query
sharedWithMeUrl = appConfig . globalGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe" ;
// Item Queries
itemByIdUrl = appConfig . globalGraphEndpoint ~ "/v1.0/me/drive/items/" ;
itemByPathUrl = appConfig . globalGraphEndpoint ~ "/v1.0/me/drive/root:/" ;
// Office 365 / SharePoint Queries
siteSearchUrl = appConfig . globalGraphEndpoint ~ "/v1.0/sites?search" ;
siteDriveUrl = appConfig . globalGraphEndpoint ~ "/v1.0/sites/" ;
2024-02-20 19:13:26 +01:00
2024-01-08 23:13:17 +01:00
// Subscriptions
subscriptionUrl = appConfig . globalGraphEndpoint ~ "/v1.0/subscriptions" ;
}
2024-03-30 21:20:04 +01:00
2024-01-08 23:13:17 +01:00
// Initialise the OneDrive API class
2024-01-26 23:55:11 +01:00
bool initialise ( bool keepAlive = true ) {
2024-01-08 23:13:17 +01:00
// Initialise the curl engine
2024-03-30 21:20:04 +01:00
this . keepAlive = keepAlive ;
if ( curlEngine is null ) {
curlEngine = CurlEngine . get ( ) ;
curlEngine . initialise ( appConfig . getValueLong ( "dns_timeout" ) , appConfig . getValueLong ( "connect_timeout" ) , appConfig . getValueLong ( "data_timeout" ) , appConfig . getValueLong ( "operation_timeout" ) , appConfig . defaultMaxRedirects , appConfig . getValueBool ( "debug_https" ) , appConfig . getValueString ( "user_agent" ) , appConfig . getValueBool ( "force_http_11" ) , appConfig . getValueLong ( "rate_limit" ) , appConfig . getValueLong ( "ip_protocol_version" ) , keepAlive ) ;
}
2024-01-08 23:13:17 +01:00
// Authorised value to return
bool authorised = false ;
// Did the user specify --dry-run
dryRun = appConfig . getValueBool ( "dry_run" ) ;
// Did the user specify --debug-https
debugResponse = appConfig . getValueBool ( "debug_https" ) ;
// Flag this so if webhooks are being used, it can also be consumed
debugHTTPResponseOutput = appConfig . getValueBool ( "debug_https" ) ;
// Set clientId to use the configured 'application_id'
clientId = appConfig . getValueString ( "application_id" ) ;
if ( clientId ! = appConfig . defaultApplicationId ) {
// a custom 'application_id' was set
2020-11-25 19:51:29 +01:00
companyName = "custom_application" ;
}
2024-01-08 23:13:17 +01:00
// Do we have a custom Azure Tenant ID?
if ( ! appConfig . getValueString ( "azure_tenant_id" ) . empty ) {
2020-10-05 23:06:04 +02:00
// Use the value entered by the user
2024-01-08 23:13:17 +01:00
tenantId = appConfig . getValueString ( "azure_tenant_id" ) ;
} else {
// set to common
tenantId = "common" ;
2020-10-05 23:06:04 +02:00
}
2024-01-08 23:13:17 +01:00
// Did the user specify a 'drive_id' ?
if ( ! appConfig . getValueString ( "drive_id" ) . empty ) {
// Update base URL's
driveUrl = driveByIdUrl ~ appConfig . getValueString ( "drive_id" ) ;
itemByIdUrl = driveUrl ~ "/items" ;
itemByPathUrl = driveUrl ~ "/root:/" ;
}
// Configure the authentication scope
if ( appConfig . getValueBool ( "read_only_auth_scope" ) ) {
// read-only authentication scopes has been requested
authScope = "&scope=Files.Read%20Files.Read.All%20Sites.Read.All%20offline_access&response_type=code&prompt=login&redirect_uri=" ;
} else {
// read-write authentication scopes will be used (default)
authScope = "&scope=Files.ReadWrite%20Files.ReadWrite.All%20Sites.ReadWrite.All%20offline_access&response_type=code&prompt=login&redirect_uri=" ;
}
2020-06-16 23:57:14 +02:00
// Configure Azure AD endpoints if 'azure_ad_endpoint' is configured
2024-01-08 23:13:17 +01:00
string azureConfigValue = appConfig . getValueString ( "azure_ad_endpoint" ) ;
2020-06-16 23:57:14 +02:00
switch ( azureConfigValue ) {
case "" :
2020-10-10 06:39:04 +02:00
if ( tenantId = = "common" ) {
2024-01-08 23:13:17 +01:00
if ( ! appConfig . apiWasInitialised ) addLogEntry ( "Configuring Global Azure AD Endpoints" ) ;
2020-10-10 06:39:04 +02:00
} else {
2024-01-08 23:13:17 +01:00
if ( ! appConfig . apiWasInitialised ) addLogEntry ( "Configuring Global Azure AD Endpoints - Single Tenant Application" ) ;
2020-10-10 06:39:04 +02:00
}
// Authentication
2024-01-08 23:13:17 +01:00
authUrl = appConfig . globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize" ;
redirectUrl = appConfig . globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
tokenUrl = appConfig . globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token" ;
2020-06-16 23:57:14 +02:00
break ;
case "USL4" :
2024-01-08 23:13:17 +01:00
if ( ! appConfig . apiWasInitialised ) addLogEntry ( "Configuring Azure AD for US Government Endpoints" ) ;
2020-06-16 23:57:14 +02:00
// Authentication
2024-01-08 23:13:17 +01:00
authUrl = appConfig . usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize" ;
tokenUrl = appConfig . usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token" ;
if ( clientId = = appConfig . defaultApplicationId ) {
2020-11-25 19:51:29 +01:00
// application_id == default
2024-01-08 23:13:17 +01:00
addLogEntry ( "USL4 AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint" , [ "debug" ] ) ;
redirectUrl = appConfig . globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
2020-11-25 19:51:29 +01:00
} else {
// custom application_id
2024-01-08 23:13:17 +01:00
redirectUrl = appConfig . usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
2020-11-25 19:51:29 +01:00
}
2021-11-23 20:54:28 +01:00
2020-06-16 23:57:14 +02:00
// Drive Queries
2024-01-08 23:13:17 +01:00
driveUrl = appConfig . usl4GraphEndpoint ~ "/v1.0/me/drive" ;
driveByIdUrl = appConfig . usl4GraphEndpoint ~ "/v1.0/drives/" ;
2020-06-16 23:57:14 +02:00
// Item Queries
2024-01-08 23:13:17 +01:00
itemByIdUrl = appConfig . usl4GraphEndpoint ~ "/v1.0/me/drive/items/" ;
itemByPathUrl = appConfig . usl4GraphEndpoint ~ "/v1.0/me/drive/root:/" ;
2020-06-16 23:57:14 +02:00
// Office 365 / SharePoint Queries
2024-01-08 23:13:17 +01:00
siteSearchUrl = appConfig . usl4GraphEndpoint ~ "/v1.0/sites?search" ;
siteDriveUrl = appConfig . usl4GraphEndpoint ~ "/v1.0/sites/" ;
2020-06-27 11:10:37 +02:00
// Shared With Me
2024-01-08 23:13:17 +01:00
sharedWithMeUrl = appConfig . usl4GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe" ;
2021-11-23 20:54:28 +01:00
// Subscriptions
2024-01-08 23:13:17 +01:00
subscriptionUrl = appConfig . usl4GraphEndpoint ~ "/v1.0/subscriptions" ;
2020-06-16 23:57:14 +02:00
break ;
case "USL5" :
2024-01-08 23:13:17 +01:00
if ( ! appConfig . apiWasInitialised ) addLogEntry ( "Configuring Azure AD for US Government Endpoints (DOD)" ) ;
2020-06-16 23:57:14 +02:00
// Authentication
2024-01-08 23:13:17 +01:00
authUrl = appConfig . usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize" ;
tokenUrl = appConfig . usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token" ;
if ( clientId = = appConfig . defaultApplicationId ) {
2020-11-25 19:51:29 +01:00
// application_id == default
2024-01-08 23:13:17 +01:00
addLogEntry ( "USL5 AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint" , [ "debug" ] ) ;
redirectUrl = appConfig . globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
2020-11-25 19:51:29 +01:00
} else {
// custom application_id
2024-01-08 23:13:17 +01:00
redirectUrl = appConfig . usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
2020-11-25 19:51:29 +01:00
}
2021-11-23 20:54:28 +01:00
2020-06-16 23:57:14 +02:00
// Drive Queries
2024-01-08 23:13:17 +01:00
driveUrl = appConfig . usl5GraphEndpoint ~ "/v1.0/me/drive" ;
driveByIdUrl = appConfig . usl5GraphEndpoint ~ "/v1.0/drives/" ;
2020-06-16 23:57:14 +02:00
// Item Queries
2024-01-08 23:13:17 +01:00
itemByIdUrl = appConfig . usl5GraphEndpoint ~ "/v1.0/me/drive/items/" ;
itemByPathUrl = appConfig . usl5GraphEndpoint ~ "/v1.0/me/drive/root:/" ;
2020-06-16 23:57:14 +02:00
// Office 365 / SharePoint Queries
2024-01-08 23:13:17 +01:00
siteSearchUrl = appConfig . usl5GraphEndpoint ~ "/v1.0/sites?search" ;
siteDriveUrl = appConfig . usl5GraphEndpoint ~ "/v1.0/sites/" ;
2020-06-27 11:10:37 +02:00
// Shared With Me
2024-01-08 23:13:17 +01:00
sharedWithMeUrl = appConfig . usl5GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe" ;
2021-11-23 20:54:28 +01:00
// Subscriptions
2024-01-08 23:13:17 +01:00
subscriptionUrl = appConfig . usl5GraphEndpoint ~ "/v1.0/subscriptions" ;
2020-06-16 23:57:14 +02:00
break ;
case "DE" :
2024-01-08 23:13:17 +01:00
if ( ! appConfig . apiWasInitialised ) addLogEntry ( "Configuring Azure AD Germany" ) ;
2020-06-16 23:57:14 +02:00
// Authentication
2024-01-08 23:13:17 +01:00
authUrl = appConfig . deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize" ;
tokenUrl = appConfig . deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token" ;
if ( clientId = = appConfig . defaultApplicationId ) {
2020-11-25 19:51:29 +01:00
// application_id == default
2024-01-08 23:13:17 +01:00
addLogEntry ( "DE AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint" , [ "debug" ] ) ;
redirectUrl = appConfig . globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
2020-11-25 19:51:29 +01:00
} else {
// custom application_id
2024-01-08 23:13:17 +01:00
redirectUrl = appConfig . deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
2020-11-25 19:51:29 +01:00
}
2021-11-23 20:54:28 +01:00
2020-06-16 23:57:14 +02:00
// Drive Queries
2024-01-08 23:13:17 +01:00
driveUrl = appConfig . deGraphEndpoint ~ "/v1.0/me/drive" ;
driveByIdUrl = appConfig . deGraphEndpoint ~ "/v1.0/drives/" ;
2020-06-16 23:57:14 +02:00
// Item Queries
2024-01-08 23:13:17 +01:00
itemByIdUrl = appConfig . deGraphEndpoint ~ "/v1.0/me/drive/items/" ;
itemByPathUrl = appConfig . deGraphEndpoint ~ "/v1.0/me/drive/root:/" ;
2020-06-16 23:57:14 +02:00
// Office 365 / SharePoint Queries
2024-01-08 23:13:17 +01:00
siteSearchUrl = appConfig . deGraphEndpoint ~ "/v1.0/sites?search" ;
siteDriveUrl = appConfig . deGraphEndpoint ~ "/v1.0/sites/" ;
2020-06-27 11:10:37 +02:00
// Shared With Me
2024-01-08 23:13:17 +01:00
sharedWithMeUrl = appConfig . deGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe" ;
2021-11-23 20:54:28 +01:00
// Subscriptions
2024-01-08 23:13:17 +01:00
subscriptionUrl = appConfig . deGraphEndpoint ~ "/v1.0/subscriptions" ;
2020-06-16 23:57:14 +02:00
break ;
case "CN" :
2024-01-08 23:13:17 +01:00
if ( ! appConfig . apiWasInitialised ) addLogEntry ( "Configuring AD China operated by 21Vianet" ) ;
2020-06-16 23:57:14 +02:00
// Authentication
2024-01-08 23:13:17 +01:00
authUrl = appConfig . cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize" ;
tokenUrl = appConfig . cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token" ;
if ( clientId = = appConfig . defaultApplicationId ) {
2020-11-25 19:51:29 +01:00
// application_id == default
2024-01-08 23:13:17 +01:00
addLogEntry ( "CN AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint" , [ "debug" ] ) ;
redirectUrl = appConfig . globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
2020-11-25 19:51:29 +01:00
} else {
// custom application_id
2024-01-08 23:13:17 +01:00
redirectUrl = appConfig . cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
2020-11-25 19:51:29 +01:00
}
2021-11-23 20:54:28 +01:00
2020-06-16 23:57:14 +02:00
// Drive Queries
2024-01-08 23:13:17 +01:00
driveUrl = appConfig . cnGraphEndpoint ~ "/v1.0/me/drive" ;
driveByIdUrl = appConfig . cnGraphEndpoint ~ "/v1.0/drives/" ;
2020-06-16 23:57:14 +02:00
// Item Queries
2024-01-08 23:13:17 +01:00
itemByIdUrl = appConfig . cnGraphEndpoint ~ "/v1.0/me/drive/items/" ;
itemByPathUrl = appConfig . cnGraphEndpoint ~ "/v1.0/me/drive/root:/" ;
2020-06-16 23:57:14 +02:00
// Office 365 / SharePoint Queries
2024-01-08 23:13:17 +01:00
siteSearchUrl = appConfig . cnGraphEndpoint ~ "/v1.0/sites?search" ;
siteDriveUrl = appConfig . cnGraphEndpoint ~ "/v1.0/sites/" ;
2020-06-27 11:10:37 +02:00
// Shared With Me
2024-01-08 23:13:17 +01:00
sharedWithMeUrl = appConfig . cnGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe" ;
2021-11-23 20:54:28 +01:00
// Subscriptions
2024-01-08 23:13:17 +01:00
subscriptionUrl = appConfig . cnGraphEndpoint ~ "/v1.0/subscriptions" ;
2020-06-16 23:57:14 +02:00
break ;
2021-11-23 20:54:28 +01:00
// Default - all other entries
2020-06-16 23:57:14 +02:00
default :
2024-01-08 23:13:17 +01:00
if ( ! appConfig . apiWasInitialised ) addLogEntry ( "Unknown Azure AD Endpoint request - using Global Azure AD Endpoints" ) ;
2021-01-03 22:28:34 +01:00
}
2022-01-11 23:14:04 +01:00
2024-01-08 23:13:17 +01:00
// Has the application been authenticated?
if ( ! exists ( appConfig . refreshTokenFilePath ) ) {
addLogEntry ( "Application has no 'refresh_token' thus needs to be authenticated" , [ "debug" ] ) ;
authorised = authorise ( ) ;
2019-03-11 07:57:47 +01:00
} else {
2024-01-08 23:13:17 +01:00
// Try and read the value from the appConfig if it is set, rather than trying to read the value from disk
if ( ! appConfig . refreshToken . empty ) {
addLogEntry ( "Read token from appConfig" , [ "debug" ] ) ;
refreshToken = strip ( appConfig . refreshToken ) ;
authorised = true ;
} else {
// Try and read the file from disk
2019-03-11 07:57:47 +01:00
try {
2024-01-08 23:13:17 +01:00
refreshToken = strip ( readText ( appConfig . refreshTokenFilePath ) ) ;
// is the refresh_token empty?
if ( refreshToken . empty ) {
addLogEntry ( "RefreshToken exists but is empty: " ~ appConfig . refreshTokenFilePath ) ;
authorised = authorise ( ) ;
} else {
// existing token not empty
authorised = true ;
// update appConfig.refreshToken
appConfig . refreshToken = refreshToken ;
}
2019-03-11 07:57:47 +01:00
} catch ( FileException e ) {
2024-01-08 23:13:17 +01:00
authorised = authorise ( ) ;
2021-07-06 22:26:52 +02:00
} catch ( std . utf . UTFException e ) {
// path contains characters which generate a UTF exception
2024-01-08 23:13:17 +01:00
addLogEntry ( "Cannot read refreshToken from: " ~ appConfig . refreshTokenFilePath ) ;
addLogEntry ( " Error Reason:" ~ e . msg ) ;
authorised = false ;
2019-03-11 07:57:47 +01:00
}
2024-01-08 23:13:17 +01:00
}
if ( refreshToken . empty ) {
// PROBLEM ... CODING TO DO ??????????
addLogEntry ( "refreshToken is empty !!!!!!!!!! This will cause 4xx errors ... CODING TO DO TO HANDLE ?????" ) ;
2019-03-11 07:57:47 +01:00
}
2016-08-04 23:35:58 +02:00
}
2024-03-30 22:40:50 +01:00
2024-01-08 23:13:17 +01:00
// Return if we are authorised
addLogEntry ( "Authorised State: " ~ to ! string ( authorised ) , [ "debug" ] ) ;
return authorised ;
2015-09-01 20:45:34 +02:00
}
2024-03-30 21:20:04 +01:00
// Reinitialise the OneDrive API class
bool reinitialise ( ) {
shutdown ( ) ;
return initialise ( this . keepAlive ) ;
}
2024-01-08 23:13:17 +01:00
// If the API has been configured correctly, print the items that been configured
void debugOutputConfiguredAPIItems ( ) {
// Debug output of configured URL's
// Application Identification
addLogEntry ( "Configured clientId " ~ clientId , [ "debug" ] ) ;
addLogEntry ( "Configured userAgent " ~ appConfig . getValueString ( "user_agent" ) , [ "debug" ] ) ;
// Authentication
addLogEntry ( "Configured authScope: " ~ authScope , [ "debug" ] ) ;
addLogEntry ( "Configured authUrl: " ~ authUrl , [ "debug" ] ) ;
addLogEntry ( "Configured redirectUrl: " ~ redirectUrl , [ "debug" ] ) ;
addLogEntry ( "Configured tokenUrl: " ~ tokenUrl , [ "debug" ] ) ;
// Drive Queries
addLogEntry ( "Configured driveUrl: " ~ driveUrl , [ "debug" ] ) ;
addLogEntry ( "Configured driveByIdUrl: " ~ driveByIdUrl , [ "debug" ] ) ;
// Shared With Me
addLogEntry ( "Configured sharedWithMeUrl: " ~ sharedWithMeUrl , [ "debug" ] ) ;
// Item Queries
addLogEntry ( "Configured itemByIdUrl: " ~ itemByIdUrl , [ "debug" ] ) ;
addLogEntry ( "Configured itemByPathUrl: " ~ itemByPathUrl , [ "debug" ] ) ;
// SharePoint Queries
addLogEntry ( "Configured siteSearchUrl: " ~ siteSearchUrl , [ "debug" ] ) ;
addLogEntry ( "Configured siteDriveUrl: " ~ siteDriveUrl , [ "debug" ] ) ;
}
// Shutdown OneDrive API Curl Engine
void shutdown ( ) {
2024-01-26 23:55:11 +01:00
// Release curl instance
if ( curlEngine ! is null ) {
curlEngine . release ( ) ;
curlEngine = null ;
}
2024-01-08 23:13:17 +01:00
}
2024-03-30 21:20:04 +01:00
2024-01-08 23:13:17 +01:00
// Authenticate this client against Microsoft OneDrive API
bool authorise ( ) {
char [ ] response ;
// What URL should be presented to the user to access
2022-08-06 00:24:51 +02:00
string url = authUrl ~ "?client_id=" ~ clientId ~ authScope ~ redirectUrl ;
2024-01-08 23:13:17 +01:00
// Configure automated authentication if --auth-files authUrl:responseUrl is being used
string authFilesString = appConfig . getValueString ( "auth_files" ) ;
string authResponseString = appConfig . getValueString ( "auth_response" ) ;
2024-03-30 22:40:50 +01:00
// Is authResponseString not empty
2024-01-08 23:13:17 +01:00
if ( ! authResponseString . empty ) {
// read the response from authResponseString
2021-11-22 21:08:04 +01:00
response = cast ( char [ ] ) authResponseString ;
} else if ( authFilesString ! = "" ) {
2024-03-30 22:40:50 +01:00
// authResponseString empty .. is authFilesString not empty
2019-06-11 09:12:36 +02:00
string [ ] authFiles = authFilesString . split ( ":" ) ;
string authUrl = authFiles [ 0 ] ;
string responseUrl = authFiles [ 1 ] ;
2023-07-23 02:13:03 +02:00
try {
auto authUrlFile = File ( authUrl , "w" ) ;
authUrlFile . write ( url ) ;
authUrlFile . close ( ) ;
2024-01-08 23:13:17 +01:00
} catch ( FileException e ) {
2023-07-23 02:13:03 +02:00
// There was a file system error
// display the error message
displayFileSystemErrorMessage ( e . msg , getFunctionName ! ( { } ) ) ;
2024-01-08 23:13:17 +01:00
// Must force exit here, allow logging to be done
Thread . sleep ( dur ! ( "msecs" ) ( 500 ) ) ;
exit ( - 1 ) ;
} catch ( ErrnoException e ) {
// There was a file system error
// display the error message
displayFileSystemErrorMessage ( e . msg , getFunctionName ! ( { } ) ) ;
// Must force exit here, allow logging to be done
Thread . sleep ( dur ! ( "msecs" ) ( 500 ) ) ;
exit ( - 1 ) ;
2023-07-23 02:13:03 +02:00
}
2024-01-08 23:13:17 +01:00
2024-03-30 22:40:50 +01:00
// Add logging that we are waiting for auth elements to be available
2024-01-08 23:13:17 +01:00
addLogEntry ( "Client requires authentication before proceeding. Waiting for --auth-files elements to be available." ) ;
2023-07-23 02:13:03 +02:00
2019-06-11 09:12:36 +02:00
while ( ! exists ( responseUrl ) ) {
Thread . sleep ( dur ! ( "msecs" ) ( 100 ) ) ;
}
2021-11-23 20:54:28 +01:00
2024-01-08 23:13:17 +01:00
// read response from provided from OneDrive
2020-03-02 02:54:56 +01:00
try {
response = cast ( char [ ] ) read ( responseUrl ) ;
} catch ( OneDriveException e ) {
// exception generated
2020-12-09 04:18:16 +01:00
displayOneDriveErrorMessage ( e . msg , getFunctionName ! ( { } ) ) ;
2020-03-02 02:54:56 +01:00
return false ;
2021-11-23 20:54:28 +01:00
}
2020-03-02 02:54:56 +01:00
// try to remove old files
2019-06-11 09:12:36 +02:00
try {
std . file . remove ( authUrl ) ;
std . file . remove ( responseUrl ) ;
} catch ( FileException e ) {
2024-01-08 23:13:17 +01:00
addLogEntry ( "Cannot remove files " ~ authUrl ~ " " ~ responseUrl ) ;
2020-03-02 02:54:56 +01:00
return false ;
2019-06-11 09:12:36 +02:00
}
2021-11-22 21:08:04 +01:00
} else {
2024-01-15 20:11:16 +01:00
// Are we in a --dry-run scenario?
if ( ! appConfig . getValueBool ( "dry_run" ) ) {
// No --dry-run is being used
addLogEntry ( "Authorise this application by visiting:\n" , [ "consoleOnly" ] ) ;
addLogEntry ( url ~ "\n" , [ "consoleOnly" ] ) ;
addLogEntry ( "Enter the response uri from your browser: " , [ "consoleOnlyNoNewLine" ] ) ;
readln ( response ) ;
appConfig . applicationAuthorizeResponseUri = true ;
} else {
// The application cannot be authorised when using --dry-run as we have to write out the authentication data, which negates the whole 'dry-run' process
addLogEntry ( ) ;
addLogEntry ( "The application requires authorisation, which involves saving authentication data on your system. Note that authorisation cannot be completed with the '--dry-run' option." ) ;
addLogEntry ( ) ;
addLogEntry ( "To exclusively authorise the application without performing any additional actions, use this command: onedrive" ) ;
addLogEntry ( ) ;
forceExit ( ) ;
}
2019-06-11 09:12:36 +02:00
}
2016-06-30 12:51:44 +02:00
// match the authorization code
2020-03-20 21:20:19 +01:00
auto c = matchFirst ( response , r"(?:[\?&]code=)([\w\d-.]+)" ) ;
2016-06-30 12:51:44 +02:00
if ( c . empty ) {
2024-01-08 23:13:17 +01:00
addLogEntry ( "An empty or invalid response uri was entered" ) ;
2016-06-30 12:51:44 +02:00
return false ;
2015-09-01 20:45:34 +02:00
}
2016-06-30 12:51:44 +02:00
c . popFront ( ) ; // skip the whole match
redeemToken ( c . front ) ;
return true ;
2015-09-01 20:45:34 +02:00
}
2022-12-15 20:08:46 +01:00
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get
2024-01-08 23:13:17 +01:00
JSONValue getDefaultDriveDetails ( ) {
string url ;
2020-05-20 03:37:11 +02:00
url = driveUrl ;
2024-01-08 23:13:17 +01:00
return get ( url ) ;
2016-12-25 16:38:55 +01:00
}
2024-01-08 23:13:17 +01:00
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
2024-01-08 23:13:17 +01:00
JSONValue getDefaultRootDetails ( ) {
string url ;
2020-05-20 03:37:11 +02:00
url = driveUrl ~ "/root" ;
return get ( url ) ;
2017-12-27 15:12:38 +01:00
}
2024-01-08 23:13:17 +01:00
2020-06-27 11:10:37 +02:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
2024-01-08 23:13:17 +01:00
JSONValue getDriveIdRoot ( string driveId ) {
string url ;
2020-06-27 11:10:37 +02:00
url = driveByIdUrl ~ driveId ~ "/root" ;
return get ( url ) ;
}
2024-01-08 23:13:17 +01:00
2020-08-08 00:56:00 +02:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get
2024-01-08 23:13:17 +01:00
JSONValue getDriveQuota ( string driveId ) {
string url ;
2020-08-08 00:56:00 +02:00
url = driveByIdUrl ~ driveId ~ "/" ;
url ~ = "?select=quota" ;
return get ( url ) ;
}
2024-01-08 23:13:17 +01:00
// Return the details of the specified path, by giving the path we wish to query
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getPathDetails ( string path ) {
string url ;
if ( ( path = = "." ) | | ( path = = "/" ) ) {
url = driveUrl ~ "/root/" ;
2020-05-20 03:37:11 +02:00
} else {
2024-01-08 23:13:17 +01:00
url = itemByPathUrl ~ encodeComponent ( path ) ~ ":/" ;
2017-12-27 15:12:38 +01:00
}
2015-09-01 20:45:34 +02:00
return get ( url ) ;
}
2024-01-08 23:13:17 +01:00
// Return the details of the specified item based on its driveID and itemID
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getPathDetailsById ( string driveId , string id ) {
string url ;
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ;
//url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
return get ( url ) ;
}
2024-03-08 21:15:11 +01:00
// Return all the items that are shared with the user
// https://docs.microsoft.com/en-us/graph/api/drive-sharedwithme
JSONValue getSharedWithMe ( ) {
checkAccessTokenExpired ( ) ;
return get ( sharedWithMeUrl ) ;
}
2024-01-08 23:13:17 +01:00
// Create a shareable link for an existing file on OneDrive based on the accessScope JSON permissions
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createlink
JSONValue createShareableLink ( string driveId , string id , JSONValue accessScope ) {
string url ;
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/createLink" ;
return post ( url , accessScope . toString ( ) ) ;
}
// Return the requested details of the specified path on the specified drive id and path
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getPathDetailsByDriveId ( string driveId , string path ) {
string url ;
// https://learn.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online
// Required format: /drives/{drive-id}/root:/{item-path}:
url = driveByIdUrl ~ driveId ~ "/root:/" ~ encodeComponent ( path ) ~ ":" ;
return get ( url ) ;
}
2020-06-27 11:10:37 +02:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta
2024-01-08 23:13:17 +01:00
JSONValue viewChangesByItemId ( string driveId , string id , string deltaLink ) {
2024-03-30 21:20:04 +01:00
string [ string ] requestHeaders ;
// If Business Account add Prefer: Include-Feature=AddToOneDrive
2024-01-08 23:13:17 +01:00
if ( ( appConfig . accountType ! = "personal" ) & & ( appConfig . getValueBool ( "sync_business_shared_items" ) ) ) {
2024-03-30 21:20:04 +01:00
addIncludeFeatureRequestHeader ( & requestHeaders ) ;
2024-01-08 23:13:17 +01:00
}
string url ;
// configure deltaLink to query
if ( deltaLink . empty ) {
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/delta" ;
} else {
url = deltaLink ;
2020-06-27 11:10:37 +02:00
}
2024-03-30 21:20:04 +01:00
return get ( url , false , requestHeaders ) ;
2020-06-27 11:10:37 +02:00
}
2024-01-08 23:13:17 +01:00
2020-06-16 23:57:14 +02:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_list_children
2024-01-08 23:13:17 +01:00
JSONValue listChildren ( string driveId , string id , string nextLink ) {
2024-03-30 21:20:04 +01:00
string [ string ] requestHeaders ;
2024-01-08 23:13:17 +01:00
// If Business Account add addIncludeFeatureRequestHeader() which should add Prefer: Include-Feature=AddToOneDrive
if ( ( appConfig . accountType ! = "personal" ) & & ( appConfig . getValueBool ( "sync_business_shared_items" ) ) ) {
2024-03-30 21:20:04 +01:00
addIncludeFeatureRequestHeader ( & requestHeaders ) ;
2024-01-08 23:13:17 +01:00
}
string url ;
2020-06-16 23:57:14 +02:00
// configure URL to query
if ( nextLink . empty ) {
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/children" ;
2024-01-08 23:13:17 +01:00
//url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
2020-06-16 23:57:14 +02:00
} else {
url = nextLink ;
}
2024-03-30 21:20:04 +01:00
return get ( url , false , requestHeaders ) ;
2020-06-16 23:57:14 +02:00
}
2024-01-08 23:13:17 +01:00
// https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_search
JSONValue searchDriveForPath ( string driveId , string path ) {
string url ;
url = "https://graph.microsoft.com/v1.0/drives/" ~ driveId ~ "/root/search(q='" ~ encodeComponent ( path ) ~ "')" ;
return get ( url ) ;
2015-09-01 20:45:34 +02:00
}
2024-01-08 23:13:17 +01:00
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_update
2024-01-08 23:13:17 +01:00
JSONValue updateById ( const ( char ) [ ] driveId , const ( char ) [ ] id , JSONValue data , const ( char ) [ ] eTag = null ) {
2024-03-30 21:20:04 +01:00
string [ string ] requestHeaders ;
2017-12-27 15:12:38 +01:00
const ( char ) [ ] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ;
2024-03-30 21:20:04 +01:00
if ( eTag ) requestHeaders [ "If-Match" ] = to ! string ( eTag ) ;
return patch ( url , data . toString ( ) , requestHeaders ) ;
2015-09-05 21:23:43 +02:00
}
2024-01-08 23:13:17 +01:00
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delete
2024-01-08 23:13:17 +01:00
void deleteById ( const ( char ) [ ] driveId , const ( char ) [ ] id , const ( char ) [ ] eTag = null ) {
2024-03-30 21:20:04 +01:00
// string[string] requestHeaders;
2017-12-27 15:12:38 +01:00
const ( char ) [ ] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ;
2016-12-28 23:47:29 +01:00
//TODO: investigate why this always fail with 412 (Precondition Failed)
2024-03-30 21:20:04 +01:00
// if (eTag) requestHeaders["If-Match"] = eTag;
2024-01-08 23:13:17 +01:00
performDelete ( url ) ;
2015-09-06 10:23:32 +02:00
}
2024-01-08 23:13:17 +01:00
2017-12-31 13:18:11 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_post_children
2024-01-08 23:13:17 +01:00
JSONValue createById ( string parentDriveId , string parentId , JSONValue item ) {
string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ "/children" ;
2015-10-04 16:24:45 +02:00
return post ( url , item . toString ( ) ) ;
2015-09-06 11:06:13 +02:00
}
2024-01-08 23:13:17 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
JSONValue simpleUpload ( string localPath , string parentDriveId , string parentId , string filename ) {
string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent ( filename ) ~ ":/content" ;
2024-03-30 21:20:04 +01:00
return put ( url , localPath ) ;
2020-09-14 09:49:50 +02:00
}
2024-01-08 23:13:17 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
JSONValue simpleUploadReplace ( string localPath , string driveId , string id ) {
string url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content" ;
2024-03-30 21:20:04 +01:00
return put ( url , localPath ) ;
2018-03-14 05:43:40 +01:00
}
2024-01-08 23:13:17 +01:00
2017-12-31 16:11:02 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
2024-01-08 23:13:17 +01:00
//JSONValue createUploadSession(string parentDriveId, string parentId, string filename, string eTag = null, JSONValue item = null) {
JSONValue createUploadSession ( string parentDriveId , string parentId , string filename , const ( char ) [ ] eTag = null , JSONValue item = null ) {
2024-03-30 21:20:04 +01:00
// string[string] requestHeaders;
2024-01-08 23:13:17 +01:00
string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent ( filename ) ~ ":/createUploadSession" ;
// eTag If-Match header addition commented out for the moment
// At some point, post the creation of this upload session the eTag is being 'updated' by OneDrive, thus when uploadFragment() is used
// this generates a 412 Precondition Failed and then a 416 Requested Range Not Satisfiable
// This needs to be investigated further as to why this occurs
2024-03-30 21:20:04 +01:00
// if (eTag) requestHeaders["If-Match"] = eTag;
2018-07-02 23:24:57 +02:00
return post ( url , item . toString ( ) ) ;
2015-09-27 18:47:41 +02:00
}
2024-01-08 23:13:17 +01:00
2015-09-27 18:47:41 +02:00
// https://dev.onedrive.com/items/upload_large_files.htm
2024-01-08 23:13:17 +01:00
JSONValue uploadFragment ( string uploadUrl , string filepath , long offset , long offsetSize , long fileSize ) {
2020-05-20 03:37:11 +02:00
// open file as read-only in binary mode
2024-01-08 23:13:17 +01:00
// If we upload a modified file, with the current known online eTag, this gets changed when the session is started - thus, the tail end of uploading
// a fragment fails with a 412 Precondition Failed and then a 416 Requested Range Not Satisfiable
// For the moment, comment out adding the If-Match header in createUploadSession, which then avoids this issue
2020-05-20 03:37:11 +02:00
string contentRange = "bytes " ~ to ! string ( offset ) ~ "-" ~ to ! string ( offset + offsetSize - 1 ) ~ "/" ~ to ! string ( fileSize ) ;
2024-01-08 23:13:17 +01:00
addLogEntry ( "" , [ "debug" ] ) ; // Add an empty newline before log output
addLogEntry ( "contentRange: " ~ contentRange , [ "debug" ] ) ;
2021-11-23 20:54:28 +01:00
2024-03-30 21:20:04 +01:00
return put ( uploadUrl , filepath , true , contentRange , offset , offsetSize ) ;
2015-09-27 18:47:41 +02:00
}
2024-01-08 23:13:17 +01:00
2015-09-27 18:47:41 +02:00
// https://dev.onedrive.com/items/upload_large_files.htm
2024-01-08 23:13:17 +01:00
JSONValue requestUploadStatus ( string uploadUrl ) {
2017-03-11 11:40:19 +01:00
return get ( uploadUrl , true ) ;
2015-09-27 18:47:41 +02:00
}
2024-01-08 23:13:17 +01:00
2018-12-04 00:59:23 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/site_search?view=odsp-graph-online
2024-01-08 23:13:17 +01:00
JSONValue o365SiteSearch ( string nextLink ) {
string url ;
2021-03-12 19:11:44 +01:00
// configure URL to query
if ( nextLink . empty ) {
url = siteSearchUrl ~ "=*" ;
} else {
url = nextLink ;
}
2018-12-04 00:59:23 +01:00
return get ( url ) ;
}
2024-01-08 23:13:17 +01:00
2018-12-04 00:59:23 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_list?view=odsp-graph-online
JSONValue o365SiteDrives ( string site_id ) {
2024-01-08 23:13:17 +01:00
string url ;
2018-12-04 00:59:23 +01:00
url = siteDriveUrl ~ site_id ~ "/drives" ;
return get ( url ) ;
}
2024-02-20 19:13:26 +01:00
JSONValue createSubscription ( string notificationUrl , SysTime expirationDateTime ) {
checkAccessTokenExpired ( ) ;
string driveId = appConfig . getValueString ( "drive_id" ) ;
string url = subscriptionUrl ;
// Create a resource item based on if we have a driveId
string resourceItem ;
if ( driveId . length ) {
resourceItem = "/drives/" ~ driveId ~ "/root" ;
} else {
resourceItem = "/me/drive/root" ;
}
// create JSON request to create webhook subscription
const JSONValue request = [
"changeType" : "updated" ,
"notificationUrl" : notificationUrl ,
"resource" : resourceItem ,
"expirationDateTime" : expirationDateTime . toISOExtString ( ) ,
"clientState" : randomUUID ( ) . toString ( )
] ;
curlEngine . http . addRequestHeader ( "Content-Type" , "application/json" ) ;
return post ( url , request . toString ( ) ) ;
}
JSONValue renewSubscription ( string subscriptionId , SysTime expirationDateTime ) {
string url ;
url = subscriptionUrl ~ "/" ~ subscriptionId ;
const JSONValue request = [
"expirationDateTime" : expirationDateTime . toISOExtString ( )
] ;
curlEngine . http . addRequestHeader ( "Content-Type" , "application/json" ) ;
return post ( url , request . toString ( ) ) ;
}
void deleteSubscription ( string subscriptionId ) {
string url ;
url = subscriptionUrl ~ "/" ~ subscriptionId ;
performDelete ( url ) ;
}
2024-01-08 23:13:17 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get_content
void downloadById ( const ( char ) [ ] driveId , const ( char ) [ ] id , string saveToPath , long fileSize ) {
scope ( failure ) {
if ( exists ( saveToPath ) ) {
// try and remove the file, catch error
try {
remove ( saveToPath ) ;
} catch ( FileException e ) {
// display the error message
displayFileSystemErrorMessage ( e . msg , getFunctionName ! ( { } ) ) ;
}
}
}
// Create the required local directory
string newPath = dirName ( saveToPath ) ;
// Does the path exist locally?
if ( ! exists ( newPath ) ) {
try {
addLogEntry ( "Requested local path does not exist, creating directory structure: " ~ newPath , [ "debug" ] ) ;
mkdirRecurse ( newPath ) ;
// Configure the applicable permissions for the folder
addLogEntry ( "Setting directory permissions for: " ~ newPath , [ "debug" ] ) ;
newPath . setAttributes ( appConfig . returnRequiredDirectoryPermisions ( ) ) ;
} catch ( FileException e ) {
// display the error message
displayFileSystemErrorMessage ( e . msg , getFunctionName ! ( { } ) ) ;
}
}
const ( char ) [ ] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content?AVOverride=1" ;
// Download file
downloadFile ( url , saveToPath , fileSize ) ;
// Does path exist?
if ( exists ( saveToPath ) ) {
// File was downloaded successfully - configure the applicable permissions for the file
addLogEntry ( "Setting file permissions for: " ~ saveToPath , [ "debug" ] ) ;
saveToPath . setAttributes ( appConfig . returnRequiredFilePermisions ( ) ) ;
}
}
// Return the actual siteSearchUrl being used and/or requested when performing 'siteQuery = onedrive.o365SiteSearch(nextLink);' call
string getSiteSearchUrl ( ) {
return siteSearchUrl ;
}
2024-03-30 22:40:50 +01:00
// Private OneDrive API Functions
2024-03-30 21:20:04 +01:00
private void addIncludeFeatureRequestHeader ( string [ string ] * headers ) {
2024-03-30 22:40:50 +01:00
2024-01-08 23:13:17 +01:00
addLogEntry ( "Adding 'Include-Feature=AddToOneDrive' API request header as 'sync_business_shared_items' config option is enabled" , [ "debug" ] ) ;
2024-03-30 21:20:04 +01:00
( * headers ) [ "Prefer" ] = "Include-Feature=AddToOneDrive" ;
}
private void redeemToken ( char [ ] authCode ) {
char [ ] postData =
"client_id=" ~ clientId ~
"&redirect_uri=" ~ redirectUrl ~
"&code=" ~ authCode ~
"&grant_type=authorization_code" ;
acquireToken ( postData ) ;
2015-09-01 20:45:34 +02:00
}
2024-01-08 23:13:17 +01:00
private void acquireToken ( char [ ] postData ) {
2020-03-02 02:54:56 +01:00
JSONValue response ;
2021-11-23 20:54:28 +01:00
2020-03-02 02:54:56 +01:00
try {
2024-03-30 21:20:04 +01:00
response = post ( tokenUrl , postData , true , "application/x-www-form-urlencoded" ) ;
2020-03-02 02:54:56 +01:00
} catch ( OneDriveException e ) {
2024-03-30 21:20:04 +01:00
if ( e . httpStatusCode > = 500 ) {
// There was a HTTP 5xx Server Side Error - retry
acquireToken ( postData ) ;
2024-01-08 23:13:17 +01:00
} else {
2024-03-30 21:20:04 +01:00
displayOneDriveErrorMessage ( e . msg , getFunctionName ! ( { } ) ) ;
2024-01-08 23:13:17 +01:00
}
2020-03-02 02:54:56 +01:00
}
2021-11-23 20:54:28 +01:00
2019-09-06 05:12:13 +02:00
if ( response . type ( ) = = JSONType . object ) {
2022-08-06 00:24:51 +02:00
// Has the client been configured to use read_only_auth_scope
2024-01-08 23:13:17 +01:00
if ( appConfig . getValueBool ( "read_only_auth_scope" ) ) {
2022-08-06 00:24:51 +02:00
// read_only_auth_scope has been configured
if ( "scope" in response ) {
string effectiveScopes = response [ "scope" ] . str ( ) ;
// Display the effective authentication scopes
2024-01-08 23:13:17 +01:00
addLogEntry ( ) ;
addLogEntry ( "Effective API Authentication Scopes: " ~ effectiveScopes , [ "verbose" ] ) ;
2022-08-06 00:24:51 +02:00
// if we have any write scopes, we need to tell the user to update an remove online prior authentication and exit application
if ( canFind ( effectiveScopes , "Write" ) ) {
// effective scopes contain write scopes .. so not a read-only configuration
2024-01-08 23:13:17 +01:00
addLogEntry ( ) ;
addLogEntry ( "ERROR: You have authentication scopes that allow write operations. You need to remove your existing application access consent" ) ;
addLogEntry ( ) ;
addLogEntry ( "Please login to https://account.live.com/consent/Manage and remove your existing application access consent" ) ;
addLogEntry ( ) ;
2022-08-06 00:24:51 +02:00
// force exit
shutdown ( ) ;
2024-01-08 23:13:17 +01:00
// Must force exit here, allow logging to be done
2024-03-30 21:20:04 +01:00
forceExit ( ) ;
2022-08-06 00:24:51 +02:00
}
}
}
2019-09-06 05:12:13 +02:00
if ( "access_token" in response ) {
2024-01-08 23:13:17 +01:00
appConfig . accessToken = "bearer " ~ strip ( response [ "access_token" ] . str ) ;
// Do we print the current access token
if ( appConfig . verbosityCount > 1 ) {
if ( appConfig . getValueBool ( "debug_https" ) ) {
if ( appConfig . getValueBool ( "print_token" ) ) {
// This needs to be highly restricted in output ....
addLogEntry ( "CAUTION - KEEP THIS SAFE: Current access token: " ~ to ! string ( appConfig . accessToken ) , [ "debug" ] ) ;
}
}
}
refreshToken = strip ( response [ "refresh_token" ] . str ) ;
appConfig . accessTokenExpiration = Clock . currTime ( ) + dur ! "seconds" ( response [ "expires_in" ] . integer ( ) ) ;
if ( ! dryRun ) {
// Update the refreshToken in appConfig so that we can reuse it
if ( appConfig . refreshToken . empty ) {
// The access token is empty
addLogEntry ( "Updating appConfig.refreshToken with new refreshToken as appConfig.refreshToken is empty" , [ "debug" ] ) ;
appConfig . refreshToken = refreshToken ;
} else {
// Is the access token different?
if ( appConfig . refreshToken ! = refreshToken ) {
// Update the memory version
addLogEntry ( "Updating appConfig.refreshToken with updated refreshToken" , [ "debug" ] ) ;
appConfig . refreshToken = refreshToken ;
}
}
// try and update the refresh_token file on disk
2020-11-08 22:06:48 +01:00
try {
2024-01-08 23:13:17 +01:00
addLogEntry ( "Updating refreshToken on disk" , [ "debug" ] ) ;
std . file . write ( appConfig . refreshTokenFilePath , refreshToken ) ;
addLogEntry ( "Setting file permissions for: " ~ appConfig . refreshTokenFilePath , [ "debug" ] ) ;
appConfig . refreshTokenFilePath . setAttributes ( appConfig . returnRequiredFilePermisions ( ) ) ;
2020-11-08 22:06:48 +01:00
} catch ( FileException e ) {
// display the error message
2020-12-09 04:18:16 +01:00
displayFileSystemErrorMessage ( e . msg , getFunctionName ! ( { } ) ) ;
2020-11-08 22:06:48 +01:00
}
2019-09-06 05:12:13 +02:00
}
} else {
2024-01-08 23:13:17 +01:00
addLogEntry ( "\nInvalid authentication response from OneDrive. Please check the response uri\n" ) ;
2019-09-06 05:12:13 +02:00
// re-authorize
2024-01-08 23:13:17 +01:00
authorise ( ) ;
2019-03-12 01:50:21 +01:00
}
} else {
2024-01-08 23:13:17 +01:00
addLogEntry ( "Invalid response from the OneDrive API. Unable to initialise OneDrive API instance." ) ;
// Must force exit here, allow logging to be done
2024-03-30 21:20:04 +01:00
forceExit ( ) ;
2019-03-11 07:57:47 +01:00
}
2015-09-01 20:45:34 +02:00
}
2024-01-08 23:13:17 +01:00
private void newToken ( ) {
addLogEntry ( "Need to generate a new access token for Microsoft OneDrive" , [ "debug" ] ) ;
auto postData = appender ! ( string ) ( ) ;
postData ~ = "client_id=" ~ clientId ;
postData ~ = "&redirect_uri=" ~ redirectUrl ;
postData ~ = "&refresh_token=" ~ to ! string ( refreshToken ) ;
postData ~ = "&grant_type=refresh_token" ;
acquireToken ( postData . data . dup ) ;
}
2024-03-30 21:20:04 +01:00
private void checkAccessTokenExpired ( ) {
if ( Clock . currTime ( ) > = appConfig . accessTokenExpiration ) {
addLogEntry ( "Microsoft OneDrive Access Token has EXPIRED. Must generate a new Microsoft OneDrive Access Token" , [ "debug" ] ) ;
newToken ( ) ;
2015-10-04 16:24:45 +02:00
} else {
2024-03-30 21:20:04 +01:00
addLogEntry ( "Existing Microsoft OneDrive Access Token Expires: " ~ to ! string ( appConfig . accessTokenExpiration ) , [ "debug" ] ) ;
2015-10-04 16:24:45 +02:00
}
}
2024-01-08 23:13:17 +01:00
2024-03-30 21:20:04 +01:00
private string getAccessToken ( ) {
checkAccessTokenExpired ( ) ;
return to ! string ( appConfig . accessToken ) ;
}
2020-05-20 03:37:11 +02:00
2024-03-30 21:20:04 +01:00
private void addAccessTokenHeader ( string [ string ] * requestHeaders ) {
( * requestHeaders ) [ "Authorization" ] = getAccessToken ( ) ;
}
2024-03-30 22:40:50 +01:00
private void connect ( HTTP . Method method , const ( char ) [ ] url , bool skipToken , CurlResponse response , string [ string ] requestHeaders = null ) {
2024-03-30 21:20:04 +01:00
addLogEntry ( "Request URL = " ~ to ! string ( url ) , [ "debug" ] ) ;
// Check access token first in case the request is overridden
if ( ! skipToken ) addAccessTokenHeader ( & requestHeaders ) ;
curlEngine . setResponseHolder ( response ) ;
foreach ( k , v ; requestHeaders ) {
curlEngine . addRequestHeader ( k , v ) ;
}
curlEngine . connect ( method , url ) ;
}
2021-11-23 20:54:28 +01:00
2024-03-30 22:40:50 +01:00
private void performDelete ( const ( char ) [ ] url , string [ string ] requestHeaders = null , string callingFunction = __FUNCTION__ , int lineno = __LINE__ ) {
2024-03-30 21:20:04 +01:00
oneDriveErrorHandlerWrapper ( ( CurlResponse response ) {
connect ( HTTP . Method . del , url , false , response , requestHeaders ) ;
return curlEngine . execute ( ) ;
} , callingFunction , lineno ) ;
}
2024-03-30 22:40:50 +01:00
private void downloadFile ( const ( char ) [ ] url , string filename , long fileSize , string callingFunction = __FUNCTION__ , int lineno = __LINE__ ) {
2024-03-30 21:20:04 +01:00
// Threshold for displaying download bar
long thresholdFileSize = 4 * 2 ^ ^ 20 ; // 4 MiB
// To support marking of partially-downloaded files,
string originalFilename = filename ;
string downloadFilename = filename ~ ".partial" ;
2024-03-30 22:40:50 +01:00
bool validateJSONResponse = false ;
2024-03-30 21:20:04 +01:00
oneDriveErrorHandlerWrapper ( ( CurlResponse response ) {
connect ( HTTP . Method . get , url , false , response ) ;
if ( fileSize > = thresholdFileSize ) {
// Download Progress variables
size_t expected_total_segments = 20 ;
ulong start_unix_time = Clock . currTime . toUnixTime ( ) ;
int h , m , s ;
string etaString ;
bool barInit = false ;
real previousProgressPercent = - 1.0 ;
real percentCheck = 5.0 ;
long segmentCount = - 1 ;
2022-04-21 00:03:22 +02:00
2024-03-30 21:20:04 +01:00
// Setup progress bar to display
curlEngine . http . onProgress = delegate int ( size_t dltotal , size_t dlnow , size_t ultotal , size_t ulnow ) {
// For each onProgress, what is the % of dlnow to dltotal
// floor - rounds down to nearest whole number
real currentDLPercent = floor ( double ( dlnow ) / dltotal * 100 ) ;
string downloadLogEntry = "Downloading: " ~ filename ~ " ... " ;
// Have we started downloading?
if ( currentDLPercent > 0 ) {
// We have started downloading
addLogEntry ( "" , [ "debug" ] ) ; // Debug new line only
addLogEntry ( "Data Received = " ~ to ! string ( dlnow ) , [ "debug" ] ) ;
addLogEntry ( "Expected Total = " ~ to ! string ( dltotal ) , [ "debug" ] ) ;
addLogEntry ( "Percent Complete = " ~ to ! string ( currentDLPercent ) , [ "debug" ] ) ;
// Every 5% download we need to increment the download bar
// Has the user set a data rate limit?
// when using rate_limit, we will get odd download rates, for example:
// Percent Complete = 24
// Data Received = 13080163
// Expected Total = 52428800
// Percent Complete = 24
// Data Received = 13685777
// Expected Total = 52428800
// Percent Complete = 26 <---- jumps to 26% missing 25%, thus fmod misses incrementing progress bar
// Data Received = 13685777
// Expected Total = 52428800
// Percent Complete = 26
if ( appConfig . getValueLong ( "rate_limit" ) > 0 ) {
// User configured rate limit
// How much data should be in each segment to qualify for 5%
ulong dataPerSegment = to ! ulong ( floor ( double ( dltotal ) / expected_total_segments ) ) ;
// How much data received do we need to validate against
ulong thisSegmentData = dataPerSegment * segmentCount ;
ulong nextSegmentData = dataPerSegment * ( segmentCount + 1 ) ;
2022-04-21 00:03:22 +02:00
2024-03-30 21:20:04 +01:00
// Has the data that has been received in a 5% window that we need to increment the progress bar at
if ( ( dlnow > thisSegmentData ) & & ( dlnow < nextSegmentData ) & & ( previousProgressPercent ! = currentDLPercent ) | | ( dlnow = = dltotal ) ) {
// Downloaded data equals approx 5%
addLogEntry ( "Incrementing Progress Bar using calculated 5% of data received" , [ "debug" ] ) ;
// 100% check
if ( currentDLPercent ! = 100 ) {
// Not 100% yet
// Calculate the output
segmentCount + + ;
auto eta = calc_eta ( segmentCount , expected_total_segments , start_unix_time ) ;
dur ! "seconds" ( eta ) . split ! ( "hours" , "minutes" , "seconds" ) ( h , m , s ) ;
etaString = format ! "| ETA %02d:%02d:%02d" ( h , m , s ) ;
string percentage = leftJustify ( to ! string ( currentDLPercent ) ~ "%" , 5 , ' ' ) ;
addLogEntry ( downloadLogEntry ~ percentage ~ etaString , [ "consoleOnly" ] ) ;
} else {
// 100% done
ulong end_unix_time = Clock . currTime . toUnixTime ( ) ;
auto upload_duration = cast ( int ) ( end_unix_time - start_unix_time ) ;
dur ! "seconds" ( upload_duration ) . split ! ( "hours" , "minutes" , "seconds" ) ( h , m , s ) ;
etaString = format ! "| DONE in %02d:%02d:%02d" ( h , m , s ) ;
string percentage = leftJustify ( to ! string ( currentDLPercent ) ~ "%" , 5 , ' ' ) ;
addLogEntry ( downloadLogEntry ~ percentage ~ etaString , [ "consoleOnly" ] ) ;
}
// update values
addLogEntry ( "Setting previousProgressPercent to " ~ to ! string ( currentDLPercent ) , [ "debug" ] ) ;
previousProgressPercent = currentDLPercent ;
addLogEntry ( "Incrementing segmentCount" , [ "debug" ] ) ;
segmentCount + + ;
}
} else {
// Is currentDLPercent divisible by 5 leaving remainder 0 and does previousProgressPercent not equal currentDLPercent
if ( ( isIdentical ( fmod ( currentDLPercent , percentCheck ) , 0.0 ) ) & & ( previousProgressPercent ! = currentDLPercent ) ) {
// currentDLPercent matches a new increment
addLogEntry ( "Incrementing Progress Bar using fmod match" , [ "debug" ] ) ;
// 100% check
if ( currentDLPercent ! = 100 ) {
// Not 100% yet
// Calculate the output
segmentCount + + ;
auto eta = calc_eta ( segmentCount , expected_total_segments , start_unix_time ) ;
dur ! "seconds" ( eta ) . split ! ( "hours" , "minutes" , "seconds" ) ( h , m , s ) ;
etaString = format ! "| ETA %02d:%02d:%02d" ( h , m , s ) ;
string percentage = leftJustify ( to ! string ( currentDLPercent ) ~ "%" , 5 , ' ' ) ;
addLogEntry ( downloadLogEntry ~ percentage ~ etaString , [ "consoleOnly" ] ) ;
} else {
// 100% done
ulong end_unix_time = Clock . currTime . toUnixTime ( ) ;
auto upload_duration = cast ( int ) ( end_unix_time - start_unix_time ) ;
dur ! "seconds" ( upload_duration ) . split ! ( "hours" , "minutes" , "seconds" ) ( h , m , s ) ;
etaString = format ! "| DONE in %02d:%02d:%02d" ( h , m , s ) ;
string percentage = leftJustify ( to ! string ( currentDLPercent ) ~ "%" , 5 , ' ' ) ;
addLogEntry ( downloadLogEntry ~ percentage ~ etaString , [ "consoleOnly" ] ) ;
}
// update values
previousProgressPercent = currentDLPercent ;
2022-04-21 00:03:22 +02:00
}
2019-10-21 09:04:08 +02:00
}
2024-03-30 21:20:04 +01:00
} else {
if ( ( currentDLPercent = = 0 ) & & ( ! barInit ) ) {
// Calculate the output
segmentCount + + ;
etaString = "| ETA --:--:--" ;
string percentage = leftJustify ( to ! string ( currentDLPercent ) ~ "%" , 5 , ' ' ) ;
addLogEntry ( downloadLogEntry ~ percentage ~ etaString , [ "consoleOnly" ] ) ;
barInit = true ;
2019-10-21 09:04:08 +02:00
}
}
2024-03-30 21:20:04 +01:00
return 0 ;
} ;
2019-10-21 09:04:08 +02:00
} else {
2024-03-30 21:20:04 +01:00
// No progress bar
2019-10-21 09:04:08 +02:00
}
2024-03-30 21:20:04 +01:00
return curlEngine . download ( originalFilename , downloadFilename ) ;
2024-03-30 22:40:50 +01:00
} , validateJSONResponse , callingFunction , lineno ) ;
2015-09-01 20:45:34 +02:00
}
2024-03-30 21:20:04 +01:00
2024-03-30 22:40:50 +01:00
private JSONValue get ( string url , bool skipToken = false , string [ string ] requestHeaders = null , string callingFunction = __FUNCTION__ , int lineno = __LINE__ ) {
2024-03-30 21:20:04 +01:00
return oneDriveErrorHandlerWrapper ( ( CurlResponse response ) {
connect ( HTTP . Method . get , url , skipToken , response , requestHeaders ) ;
return curlEngine . execute ( ) ;
} , callingFunction , lineno ) ;
2024-01-08 23:13:17 +01:00
}
2024-03-30 22:40:50 +01:00
private JSONValue patch ( const ( char ) [ ] url , const ( char ) [ ] patchData , string [ string ] requestHeaders = null , const ( char ) [ ] contentType = "application/json" , string callingFunction = __FUNCTION__ , int lineno = __LINE__ ) {
2024-03-30 21:20:04 +01:00
return oneDriveErrorHandlerWrapper ( ( CurlResponse response ) {
connect ( HTTP . Method . patch , url , false , response , requestHeaders ) ;
curlEngine . setContent ( contentType , patchData ) ;
return curlEngine . execute ( ) ;
} , callingFunction , lineno ) ;
}
2021-11-23 20:54:28 +01:00
2024-03-30 22:40:50 +01:00
private JSONValue post ( const ( char ) [ ] url , const ( char ) [ ] postData , bool skipToken = false , const ( char ) [ ] contentType = "application/json" , string callingFunction = __FUNCTION__ , int lineno = __LINE__ ) {
2024-03-30 21:20:04 +01:00
return oneDriveErrorHandlerWrapper ( ( CurlResponse response ) {
connect ( HTTP . Method . post , url , skipToken , response ) ;
curlEngine . setContent ( contentType , postData ) ;
return curlEngine . execute ( ) ;
} , callingFunction , lineno ) ;
2024-01-08 23:13:17 +01:00
}
2024-03-30 22:40:50 +01:00
private JSONValue put ( const ( char ) [ ] url , string filepath , bool skipToken = false , string contentRange = null , ulong offset = 0 , ulong offsetSize = 0 , string callingFunction = __FUNCTION__ , int lineno = __LINE__ ) {
2024-03-30 21:20:04 +01:00
return oneDriveErrorHandlerWrapper ( ( CurlResponse response ) {
string [ string ] requestHeaders ;
// open file as read-only in binary mode
auto file = File ( filepath , "rb" ) ;
if ( ! contentRange . empty )
file . seek ( offset ) ;
// function scopes
scope ( exit ) {
// close file if open
if ( file . isOpen ( ) ) {
// close open file
file . close ( ) ;
}
}
if ( ! contentRange . empty )
requestHeaders [ "Content-Range" ] = contentRange ;
else
offsetSize = file . size ;
connect ( HTTP . Method . put , url , skipToken , response , requestHeaders ) ;
curlEngine . setFile ( & file , offsetSize ) ;
return curlEngine . execute ( ) ;
} , callingFunction , lineno ) ;
2021-03-03 01:12:27 +01:00
}
2015-09-01 20:45:34 +02:00
2024-03-30 21:20:04 +01:00
// Wrapper function for all requests to OneDrive API
2024-03-30 22:40:50 +01:00
// throws a OneDriveException
private JSONValue oneDriveErrorHandlerWrapper ( CurlResponse delegate ( CurlResponse response ) executer , bool validateJSONResponse , string callingFunction , int lineno ) {
2024-03-30 21:20:04 +01:00
int maxRetryCount = 10 ;
int retryAttempts = 0 ;
int backoffInterval = 0 ;
ulong thisBackOffInterval = 64 ;
int maxBackoffInterval = 64 ;
int timestampAlign = 0 ;
bool retrySuccess = false ;
SysTime currentTime ;
CurlResponse response = new CurlResponse ( ) ;
JSONValue result ;
while ( ! retrySuccess & & retryAttempts + + < maxRetryCount ) {
try {
response . reset ( ) ;
response = executer ( response ) ;
if ( response . hasResponse ) {
result = response . json ( ) ;
if ( debugResponse ) {
addLogEntry ( "OneDrive API Response: " ~ response . dumpResponse ( ) , [ "debug" ] ) ;
}
// Check http response code, raise an error if the operation fails
if ( response . statusLine . code / 100 ! = 2 & & response . statusLine . code ! = 302 ) {
throw new OneDriveException ( response . statusLine . code , response . statusLine . reason , response ) ;
}
2024-03-30 22:40:50 +01:00
if ( validateJSONResponse ) {
if ( result . type ( ) ! = JSONType . object ) {
throw new OneDriveException ( 0 , "Caller request a non null JSON response, get null instead" , response ) ;
}
}
2024-03-30 21:20:04 +01:00
} else {
// No valid response is returned
throw new OneDriveException ( 0 , "OneDrive operation returned an invalid response" , response ) ;
2024-01-08 23:13:17 +01:00
}
2024-03-30 21:20:04 +01:00
retrySuccess = true ;
if ( retryAttempts > 1 ) {
// no error from http.perform() on re-try
addLogEntry ( "Internet connectivity to Microsoft OneDrive service has been restored" ) ;
// unset the fresh connect option as this then creates performance issues if left enabled
addLogEntry ( "Unsetting libcurl to use a fresh connection as this causes a performance impact if left enabled" , [ "debug" ] ) ;
curlEngine . http . handle . set ( CurlOption . fresh_connect , 0 ) ;
2024-01-08 23:13:17 +01:00
}
break ;
2024-03-30 21:20:04 +01:00
} catch ( CurlException e ) {
// Parse and display error message received from OneDrive
addLogEntry ( "onedrive.performHTTPOperation() Generated a OneDrive CurlException" , [ "debug" ] ) ;
auto errorArray = splitLines ( e . msg ) ;
string errorMessage = errorArray [ 0 ] ;
addLogEntry ( "Handling Curl expection" ) ;
addLogEntry ( to ! string ( response ) ) ;
2024-01-08 23:13:17 +01:00
2024-03-30 21:20:04 +01:00
// what is contained in the curl error message?
if ( canFind ( errorMessage , "Couldn't connect to server on handle" ) | | canFind ( errorMessage , "Couldn't resolve host name on handle" ) | | canFind ( errorMessage , "Timeout was reached on handle" ) ) {
// Connectivity to Microsoft OneDrive was lost
addLogEntry ( "Internet connectivity to Microsoft OneDrive service has been interrupted .. re-trying in the background" ) ;
// what caused the initial curl exception?
if ( canFind ( errorMessage , "Couldn't connect to server on handle" ) ) addLogEntry ( "Unable to connect to server - HTTPS access blocked?" , [ "debug" ] ) ;
if ( canFind ( errorMessage , "Couldn't resolve host name on handle" ) ) addLogEntry ( "Unable to resolve server - DNS access blocked?" , [ "debug" ] ) ;
if ( canFind ( errorMessage , "Timeout was reached on handle" ) ) {
// Common cause is libcurl trying IPv6 DNS resolution when there are only IPv4 DNS servers available
addLogEntry ( "A libcurl timeout was triggered - data too slow, no DNS resolution response, no server response ... use --debug-https to diagnose this issue further." , [ "verbose" ] ) ;
addLogEntry ( "A common cause is IPv6 DNS resolution. Investigate 'ip_protocol_version' to only use IPv4 network communication to potentially resolve this issue." , [ "verbose" ] ) ;
}
} else {
// what error was returned?
if ( canFind ( errorMessage , "Problem with the SSL CA cert (path? access rights?) on handle" ) ) {
// error setting certificate verify locations:
// CAfile: /etc/pki/tls/certs/ca-bundle.crt
// CApath: none
//
// Tell the Curl Engine to bypass SSL check - essentially SSL is passing back a bad value due to 'stdio' compile time option
// Further reading:
// https://github.com/curl/curl/issues/6090
// https://github.com/openssl/openssl/issues/7536
// https://stackoverflow.com/questions/45829588/brew-install-fails-curl77-error-setting-certificate-verify
// https://forum.dlang.org/post/vwvkbubufexgeuaxhqfl@forum.dlang.org
addLogEntry ( "Problem with reading the SSL CA cert via libcurl - please repair your system SSL CA Certificates" ) ;
throw new OneDriveError ( "OneDrive operation encounter curl lib issue" ) ;
} else {
// Was this a curl initialization error?
if ( canFind ( errorMessage , "Failed initialization on handle" ) ) {
// initialization error ... prevent a run-away process if we have zero disk space
ulong localActualFreeSpace = getAvailableDiskSpace ( "." ) ;
if ( localActualFreeSpace = = 0 ) {
throw new OneDriveError ( "Zero disk space detected" ) ;
}
} else {
// Unknown error
displayGeneralErrorMessage ( e , callingFunction , lineno ) ;
}
}
2024-01-08 23:13:17 +01:00
}
2021-11-23 20:54:28 +01:00
2024-03-30 21:20:04 +01:00
if ( retryAttempts > = maxRetryCount ) {
addLogEntry ( " ERROR: Unable to reconnect to the Microsoft OneDrive service due to curl exception after " ~ to ! string ( maxRetryCount ) ) ;
throw new OneDriveException ( 0 , "Request Timeout after " ~ to ! string ( maxRetryCount ) ~ " attempts - Encounter Curl Exception" , response ) ;
2024-01-08 23:13:17 +01:00
}
2024-03-30 21:20:04 +01:00
// configure libcurl to perform a fresh connection
addLogEntry ( "Configuring libcurl to use a fresh connection for re-try" , [ "debug" ] ) ;
curlEngine . http . handle . set ( CurlOption . fresh_connect , 1 ) ;
// increment backoff interval
backoffInterval + + ;
thisBackOffInterval = retryAttempts * backoffInterval ;
if ( thisBackOffInterval > maxBackoffInterval ) {
thisBackOffInterval = maxBackoffInterval ;
}
} catch ( OneDriveException exception ) {
// https://dev.onedrive.com/misc/errors.htm
// https://developer.overdrive.com/docs/reference-guide
/ *
HTTP / 1.1 Response handling
Errors in the OneDrive API are returned using standard HTTP status codes , as well as a JSON error response object . The following HTTP status codes should be expected .
Status code Status message Description
100 Continue Continue
200 OK Request was handled OK
201 Created This means you ' ve made a successful POST to checkout , lock in a format , or place a hold
204 No Content This means you ' ve made a successful DELETE to remove a hold or return a title
400 Bad Request Cannot process the request because it is malformed or incorrect .
401 Unauthorized Required authentication information is either missing or not valid for the resource .
403 Forbidden Access is denied to the requested resource . The user might not have enough permission .
404 Not Found The requested resource doesn ’ t exist .
405 Method Not Allowed The HTTP method in the request is not allowed on the resource .
406 Not Acceptable This service doesn ’ t support the format requested in the Accept header .
408 Request Time out Not expected from OneDrive , but can be used to handle Internet connection failures the same ( fallback and try again )
409 Conflict The current state conflicts with what the request expects . For example , the specified parent folder might not exist .
410 Gone The requested resource is no longer available at the server .
411 Length Required A Content - Length header is required on the request .
412 Precondition Failed A precondition provided in the request ( such as an if - match header ) does not match the resource ' s current state .
413 Request Entity Too Large The request size exceeds the maximum limit .
415 Unsupported Media Type The content type of the request is a format that is not supported by the service .
416 Requested Range Not Satisfiable The specified byte range is invalid or unavailable .
422 Unprocessable Entity Cannot process the request because it is semantically incorrect .
429 Too Many Requests Client application has been throttled and should not attempt to repeat the request until an amount of time has elapsed .
500 Internal Server Error There was an internal server error while processing the request .
501 Not Implemented The requested feature isn ’ t implemented .
502 Bad Gateway The service was unreachable
503 Service Unavailable The service is temporarily unavailable . You may repeat the request after a delay . There may be a Retry - After header .
507 Insufficient Storage The maximum storage quota has been reached .
509 Bandwidth Limit Exceeded Your app has been throttled for exceeding the maximum bandwidth cap . Your app can retry the request again after more time has elapsed .
HTTP / 2 Response handling
0 OK
* /
// an error was generated
// Dump curl connection
switch ( exception . httpStatusCode ) {
case 400 :
case 401 :
case 408 :
case 503 :
case 504 :
addLogEntry ( "Handling OneDrive expection" ) ;
addLogEntry ( to ! string ( response ) ) ;
break ;
default :
break ;
2021-11-23 20:54:28 +01:00
2024-03-30 21:20:04 +01:00
}
switch ( exception . httpStatusCode ) {
case 400 :
// Try one more time, as OneDrive occasionally returns 400 errors.
if ( retryAttempts < maxRetryCount ) {
retryAttempts = maxRetryCount - 1 ;
thisBackOffInterval = 5 ;
} else
throw exception ;
break ;
case 401 :
handleClientUnauthorised ( exception . httpStatusCode , exception . msg ) ;
break ;
case 429 :
// If OneDrive sends a status code 429 then this function will be used to process the Retry-After response header which contains the value by which we need to wait
addLogEntry ( "Handling a OneDrive HTTP 429 Response Code (Too Many Requests)" ) ;
// Read in the Retry-After HTTP header as set and delay as per this value before retrying the request
thisBackOffInterval = response . getRetryAfterValue ( ) ;
addLogEntry ( "Using Retry-After Value = " ~ to ! string ( thisBackOffInterval ) , [ "debug" ] ) ;
break ;
case 408 :
case 503 :
case 504 :
auto errorArray = splitLines ( exception . msg ) ;
addLogEntry ( to ! string ( errorArray [ 0 ] ) ~ " when attempting to query - retrying applicable request in 30 seconds" ) ;
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
addLogEntry ( "Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request" , [ "debug" ] ) ;
thisBackOffInterval = 30 ;
break ;
default :
throw exception ;
}
2021-11-23 20:54:28 +01:00
2024-03-30 21:20:04 +01:00
if ( retryAttempts > = maxRetryCount ) {
addLogEntry ( " ERROR: Unable to reconnect to the Microsoft OneDrive service after " ~ to ! string ( maxRetryCount ) ) ;
throw exception ;
}
} catch ( ErrnoException e ) {
// There was a file system error
// display the error message
displayFileSystemErrorMessage ( e . msg , callingFunction ) ;
throw new OneDriveException ( 0 , "There was a file system error during OneDrive request: " ~ e . msg , response ) ;
}
2021-11-23 20:54:28 +01:00
2024-03-30 21:20:04 +01:00
// Back off & retry with incremental delay
currentTime = Clock . currTime ( ) ;
// display retry information
currentTime . fracSecs = Duration . zero ;
auto timeString = currentTime . toString ( ) ;
addLogEntry ( " Retry attempt: " ~ to ! string ( retryAttempts ) , [ "verbose" ] ) ;
addLogEntry ( " This attempt timestamp: " ~ timeString , [ "verbose" ] ) ;
// detail when the next attempt will be tried
// factor in the delay for curl to generate the exception - otherwise the next timestamp appears to be 'out' even though technically correct
auto nextRetry = currentTime + dur ! "seconds" ( thisBackOffInterval ) + dur ! "seconds" ( timestampAlign ) ;
addLogEntry ( " Next retry in approx: " ~ to ! string ( ( thisBackOffInterval + timestampAlign ) ) ~ " seconds" ) ;
addLogEntry ( " Next retry approx: " ~ to ! string ( nextRetry ) , [ "verbose" ] ) ;
// thread sleep
Thread . sleep ( dur ! "seconds" ( thisBackOffInterval ) ) ;
addLogEntry ( "Retrying ..." ) ;
}
2018-05-03 08:21:53 +02:00
2024-03-30 21:20:04 +01:00
if ( ! retrySuccess ) {
addLogEntry ( " ERROR: Unable to reconnect to the Microsoft OneDrive service after " ~ to ! string ( maxRetryCount ) ) ;
throw new OneDriveException ( 0 , "Request Timeout after " ~ to ! string ( maxRetryCount ) ~ " attempts" , response ) ;
2016-12-13 18:04:16 +01:00
}
2024-03-30 21:20:04 +01:00
return result ;
2016-12-13 18:04:16 +01:00
}
2024-01-08 23:13:17 +01:00
}