2018-11-10 20:38:41 +01:00
import std.net.curl ;
import etc.c.curl : CurlOption ;
2016-08-04 23:35:58 +02:00
import std.datetime , std . exception , std . file , std . json , std . path ;
2019-06-11 09:12:36 +02:00
import std.stdio , std . string , std . uni , std . uri , std . file ;
import std.array : split ;
2018-08-02 00:25:37 +02:00
import core.stdc.stdlib ;
2018-08-09 23:46:27 +02:00
import core.thread , std . conv , std . math ;
2019-10-21 09:04:08 +02:00
import std.algorithm.searching ;
2018-08-09 23:46:27 +02:00
import progress ;
import config ;
2016-08-04 23:35:58 +02:00
static import log ;
2018-04-15 12:02:39 +02:00
shared bool debugResponse = false ;
2019-04-11 04:26:20 +02:00
private bool dryRun = false ;
private bool simulateNoRefreshTokenFile = false ;
2020-03-19 20:12:47 +01:00
private ulong retryAfterValue = 0 ;
2015-12-29 19:38:15 +01:00
2015-09-01 20:45:34 +02:00
private immutable {
2020-06-16 23:57:14 +02:00
// Azure Active Directory & Graph Explorer Endpoints
// Global & Defaults
string globalAuthEndpoint = "https://login.microsoftonline.com" ;
string globalGraphEndpoint = "https://graph.microsoft.com" ;
2018-12-04 00:59:23 +01:00
2020-06-16 23:57:14 +02:00
// US Government L4
string usl4AuthEndpoint = "https://login.microsoftonline.us" ;
string usl4GraphEndpoint = "https://graph.microsoft.us" ;
// US Government L5
string usl5AuthEndpoint = "https://login.microsoftonline.us" ;
string usl5GraphEndpoint = "https://dod-graph.microsoft.us" ;
// Germany
string deAuthEndpoint = "https://login.microsoftonline.de" ;
string deGraphEndpoint = "https://graph.microsoft.de" ;
// China
string cnAuthEndpoint = "https://login.chinacloudapi.cn" ;
string cnGraphEndpoint = "https://microsoftgraph.chinacloudapi.cn" ;
2015-09-01 20:45:34 +02:00
}
2018-11-23 20:26:30 +01:00
private {
2020-03-22 21:01:51 +01:00
// Client ID / Application ID (abraunegg)
string clientId = "d50ca740-c83f-4d1b-b616-12c519384f0c" ;
// Default User Agent configuration
string isvTag = "ISV" ;
string companyName = "abraunegg" ;
// Application name as per Microsoft Azure application registration
string appTitle = "OneDrive Client for Linux" ;
// Default Drive ID
string driveId = "" ;
2020-06-16 23:57:14 +02:00
// API Query URL's, based on using defaults, but can be updated by config option 'azure_ad_endpoint'
// Authentication
string authUrl = globalAuthEndpoint ~ "/common/oauth2/v2.0/authorize" ;
string redirectUrl = globalAuthEndpoint ~ "/common/oauth2/nativeclient" ;
string tokenUrl = globalAuthEndpoint ~ "/common/oauth2/v2.0/token" ;
// Drive Queries
string driveUrl = globalGraphEndpoint ~ "/v1.0/me/drive" ;
string driveByIdUrl = globalGraphEndpoint ~ "/v1.0/drives/" ;
2020-06-27 11:10:37 +02:00
// What is 'shared with me' Query
string sharedWithMe = globalGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe" ;
2020-06-16 23:57:14 +02:00
// Item Queries
string itemByIdUrl = globalGraphEndpoint ~ "/v1.0/me/drive/items/" ;
string itemByPathUrl = globalGraphEndpoint ~ "/v1.0/me/drive/root:/" ;
// Office 365 / SharePoint Queries
string siteSearchUrl = globalGraphEndpoint ~ "/v1.0/sites?search" ;
string siteDriveUrl = globalGraphEndpoint ~ "/v1.0/sites/" ;
2018-11-23 20:26:30 +01:00
}
2015-09-01 20:45:34 +02:00
class OneDriveException : Exception
{
2017-12-28 15:03:15 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/errors
2016-12-14 15:17:20 +01:00
int httpStatusCode ;
2016-12-13 18:04:16 +01:00
JSONValue error ;
2015-10-04 16:24:45 +02:00
2016-12-14 15:17:20 +01:00
@safe pure this ( int httpStatusCode , string reason , string file = __FILE__ , size_t line = __LINE__ )
2015-10-04 16:24:45 +02:00
{
2016-12-14 15:17:20 +01:00
this . httpStatusCode = httpStatusCode ;
2016-12-13 18:04:16 +01:00
this . error = error ;
2016-12-14 15:17:20 +01:00
string msg = format ( "HTTP request returned status code %d (%s)" , httpStatusCode , reason ) ;
2017-12-28 15:03:15 +01:00
super ( msg , file , line ) ;
2015-10-04 16:24:45 +02:00
}
2016-12-13 18:04:16 +01:00
2016-12-14 15:17:20 +01:00
this ( int httpStatusCode , string reason , ref const JSONValue error , string file = __FILE__ , size_t line = __LINE__ )
2016-12-13 18:04:16 +01:00
{
2016-12-14 15:17:20 +01:00
this . httpStatusCode = httpStatusCode ;
2016-12-13 18:04:16 +01:00
this . error = error ;
2018-03-29 23:14:00 +02:00
string msg = format ( "HTTP request returned status code %d (%s)\n%s" , httpStatusCode , reason , toJSON ( error , true ) ) ;
2017-12-28 15:03:15 +01:00
super ( msg , file , line ) ;
2015-10-04 16:24:45 +02:00
}
2015-09-01 20:45:34 +02:00
}
final class OneDriveApi
{
2016-08-04 23:35:58 +02:00
private Config cfg ;
2015-09-01 20:45:34 +02:00
private string refreshToken , accessToken ;
2015-09-01 22:23:42 +02:00
private SysTime accessTokenExpiration ;
2020-05-20 03:37:11 +02:00
private HTTP http ;
2015-09-01 20:45:34 +02:00
2017-05-28 20:14:50 +02:00
// if true, every new access token is printed
bool printAccessToken ;
2019-04-11 04:26:20 +02:00
this ( Config cfg )
2015-09-01 20:45:34 +02:00
{
2016-08-04 23:35:58 +02:00
this . cfg = cfg ;
2015-09-01 20:45:34 +02:00
http = HTTP ( ) ;
2020-04-23 02:25:41 +02:00
// Curl Timeout Handling
2018-12-02 01:30:50 +01:00
// DNS lookup timeout
2018-04-24 04:10:27 +02:00
http . dnsTimeout = ( dur ! "seconds" ( 5 ) ) ;
2020-04-23 02:25:41 +02:00
// Timeout for connecting
2018-12-02 01:30:50 +01:00
http . connectTimeout = ( dur ! "seconds" ( 10 ) ) ;
// with the following settings we force
// - if there is no data flow for 5min, abort
// - if the download time for one item exceeds 1h, abort
//
// timeout for activity on connection
// this translates into Curl's CURLOPT_LOW_SPEED_TIME
// which says
// It contains the time in number seconds that the
// transfer speed should be below the CURLOPT_LOW_SPEED_LIMIT
// for the library to consider it too slow and abort.
http . dataTimeout = ( dur ! "seconds" ( 300 ) ) ;
// maximum time an operation is allowed to take
// This includes dns resolution, connecting, data transfer, etc.
http . operationTimeout = ( dur ! "seconds" ( 3600 ) ) ;
2018-11-10 20:38:41 +01:00
2018-11-11 20:01:57 +01:00
// Specify how many redirects should be allowed
http . maxRedirects ( 5 ) ;
2018-11-10 20:38:41 +01:00
2019-01-05 19:43:44 +01:00
// Do we enable curl debugging?
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "debug_https" ) ) {
2018-03-14 05:43:40 +01:00
http . verbose = true ;
2018-04-15 12:02:39 +02:00
. debugResponse = true ;
2019-09-24 08:42:40 +02:00
}
2020-06-16 23:57:14 +02:00
2020-10-05 23:06:04 +02:00
// Configure tenant id value, if 'azure_tenant_id' is configured,
// otherwise use the "common" multiplexer
string tenantId = "common" ;
if ( cfg . getValueString ( "azure_tenant_id" ) ! = "" ) {
// Use the value entered by the user
tenantId = cfg . getValueString ( "azure_tenant_id" ) ;
}
2020-06-16 23:57:14 +02:00
// Configure Azure AD endpoints if 'azure_ad_endpoint' is configured
string azureConfigValue = cfg . getValueString ( "azure_ad_endpoint" ) ;
switch ( azureConfigValue ) {
case "" :
2020-10-10 06:39:04 +02:00
if ( tenantId = = "common" ) {
log . log ( "Configuring Global Azure AD Endpoints" ) ;
} else {
log . log ( "Configuring Global Azure AD Endpoints - Single Tenant Application" ) ;
}
// Authentication
authUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize" ;
redirectUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
tokenUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token" ;
2020-06-16 23:57:14 +02:00
break ;
case "USL4" :
log . log ( "Configuring Azure AD for US Government Endpoints" ) ;
// Authentication
2020-10-05 23:06:04 +02:00
authUrl = usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize" ;
redirectUrl = usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
tokenUrl = usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token" ;
2020-06-16 23:57:14 +02:00
// Drive Queries
driveUrl = usl4GraphEndpoint ~ "/v1.0/me/drive" ;
driveByIdUrl = usl4GraphEndpoint ~ "/v1.0/drives/" ;
// Item Queries
itemByIdUrl = usl4GraphEndpoint ~ "/v1.0/me/drive/items/" ;
itemByPathUrl = usl4GraphEndpoint ~ "/v1.0/me/drive/root:/" ;
// Office 365 / SharePoint Queries
siteSearchUrl = usl4GraphEndpoint ~ "/v1.0/sites?search" ;
siteDriveUrl = usl4GraphEndpoint ~ "/v1.0/sites/" ;
2020-06-27 11:10:37 +02:00
// Shared With Me
sharedWithMe = usl4GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe" ;
2020-06-16 23:57:14 +02:00
break ;
case "USL5" :
log . log ( "Configuring Azure AD for US Government Endpoints (DOD)" ) ;
// Authentication
2020-10-05 23:06:04 +02:00
authUrl = usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize" ;
redirectUrl = usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
tokenUrl = usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token" ;
2020-06-16 23:57:14 +02:00
// Drive Queries
driveUrl = usl5GraphEndpoint ~ "/v1.0/me/drive" ;
driveByIdUrl = usl5GraphEndpoint ~ "/v1.0/drives/" ;
// Item Queries
itemByIdUrl = usl5GraphEndpoint ~ "/v1.0/me/drive/items/" ;
itemByPathUrl = usl5GraphEndpoint ~ "/v1.0/me/drive/root:/" ;
// Office 365 / SharePoint Queries
siteSearchUrl = usl5GraphEndpoint ~ "/v1.0/sites?search" ;
siteDriveUrl = usl5GraphEndpoint ~ "/v1.0/sites/" ;
2020-06-27 11:10:37 +02:00
// Shared With Me
sharedWithMe = usl5GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe" ;
2020-06-16 23:57:14 +02:00
break ;
case "DE" :
log . log ( "Configuring Azure AD Germany" ) ;
// Authentication
2020-10-05 23:06:04 +02:00
authUrl = deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize" ;
redirectUrl = deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
tokenUrl = deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token" ;
2020-06-16 23:57:14 +02:00
// Drive Queries
driveUrl = deGraphEndpoint ~ "/v1.0/me/drive" ;
driveByIdUrl = deGraphEndpoint ~ "/v1.0/drives/" ;
// Item Queries
itemByIdUrl = deGraphEndpoint ~ "/v1.0/me/drive/items/" ;
itemByPathUrl = deGraphEndpoint ~ "/v1.0/me/drive/root:/" ;
// Office 365 / SharePoint Queries
siteSearchUrl = deGraphEndpoint ~ "/v1.0/sites?search" ;
siteDriveUrl = deGraphEndpoint ~ "/v1.0/sites/" ;
2020-06-27 11:10:37 +02:00
// Shared With Me
sharedWithMe = deGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe" ;
2020-06-16 23:57:14 +02:00
break ;
case "CN" :
log . log ( "Configuring AD China operated by 21Vianet" ) ;
// Authentication
2020-10-05 23:06:04 +02:00
authUrl = cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize" ;
redirectUrl = cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient" ;
tokenUrl = cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token" ;
2020-06-16 23:57:14 +02:00
// Drive Queries
driveUrl = cnGraphEndpoint ~ "/v1.0/me/drive" ;
driveByIdUrl = cnGraphEndpoint ~ "/v1.0/drives/" ;
// Item Queries
itemByIdUrl = cnGraphEndpoint ~ "/v1.0/me/drive/items/" ;
itemByPathUrl = cnGraphEndpoint ~ "/v1.0/me/drive/root:/" ;
// Office 365 / SharePoint Queries
siteSearchUrl = cnGraphEndpoint ~ "/v1.0/sites?search" ;
siteDriveUrl = cnGraphEndpoint ~ "/v1.0/sites/" ;
2020-06-27 11:10:37 +02:00
// Shared With Me
sharedWithMe = cnGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe" ;
2020-06-16 23:57:14 +02:00
break ;
// Default - all other entries
default :
log . log ( "Unknown Azure AD Endpoint request - using Global Azure AD Endpoints" ) ;
}
2020-03-19 20:12:47 +01:00
// Configure the User Agent string
if ( cfg . getValueString ( "user_agent" ) = = "" ) {
2020-03-22 21:01:51 +01:00
// Application User Agent string defaults
// Comply with OneDrive traffic decoration requirements
2020-03-19 20:12:47 +01:00
// https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
// - Identify as ISV and include Company Name, App Name separated by a pipe character and then adding Version number separated with a slash character
// Note: If you've created an application, the recommendation is to register and use AppID and AppTitle
// The issue here is that currently the application is still using the 'skilion' application ID, thus no idea what the AppTitle used was.
http . setUserAgent = isvTag ~ "|" ~ companyName ~ "|" ~ appTitle ~ "/" ~ strip ( import ( "version" ) ) ;
2019-09-24 08:42:40 +02:00
} else {
2020-03-19 20:12:47 +01:00
// Use the value entered by the user
http . setUserAgent = cfg . getValueString ( "user_agent" ) ;
2019-09-24 08:42:40 +02:00
}
2019-01-05 19:43:44 +01:00
// What version of HTTP protocol do we use?
// Curl >= 7.62.0 defaults to http2 for a significant number of operations
2019-06-20 17:11:29 +02:00
if ( cfg . getValueBool ( "force_http_2" ) ) {
// Use curl defaults
log . vdebug ( "Upgrading all HTTP operations to HTTP/2 where applicable" ) ;
} else {
// Downgrade curl by default due to silent exist issues when using http/2
// See issue #501 for details and discussion
log . vdebug ( "Downgrading all HTTP operations to HTTP/1.1 by default" ) ;
2019-01-05 19:43:44 +01:00
// Downgrade to HTTP 1.1 - yes version = 2 is HTTP 1.1
http . handle . set ( CurlOption . http_version , 2 ) ;
}
2019-03-11 07:57:47 +01:00
// Do we set the dryRun handlers?
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "dry_run" ) ) {
2019-03-11 07:57:47 +01:00
. dryRun = true ;
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "logout" ) ) {
. simulateNoRefreshTokenFile = true ;
}
2019-03-11 07:57:47 +01:00
}
2016-08-04 23:35:58 +02:00
}
2020-05-20 03:37:11 +02:00
// Shutdown OneDrive HTTP construct
void shutdown ( )
{
// reset any values to defaults, freeing any set objects
http . clearRequestHeaders ( ) ;
http . onSend = null ;
http . onReceive = null ;
http . onReceiveHeader = null ;
http . onReceiveStatusLine = null ;
http . contentLength = 0 ;
// shut down the curl instance
http . shutdown ( ) ;
}
2016-08-04 23:35:58 +02:00
bool init ( )
{
2020-03-22 21:01:51 +01:00
// Update clientId if application_id is set in config file
if ( cfg . getValueString ( "application_id" ) ! = "" ) {
// an application_id is set in config file
clientId = cfg . getValueString ( "application_id" ) ;
companyName = "custom_application" ;
}
// detail what we are using for applicaion identification
log . vdebug ( "clientId = " , clientId ) ;
log . vdebug ( "companyName = " , companyName ) ;
log . vdebug ( "appTitle = " , appTitle ) ;
2018-11-23 20:26:30 +01:00
try {
2019-04-11 04:26:20 +02:00
driveId = cfg . getValueString ( "drive_id" ) ;
2018-11-23 20:26:30 +01:00
if ( driveId . length ) {
driveUrl = driveByIdUrl ~ driveId ;
2019-04-11 04:26:20 +02:00
itemByIdUrl = driveUrl ~ "/items" ;
itemByPathUrl = driveUrl ~ "/root:/" ;
2018-11-23 20:26:30 +01:00
}
} catch ( Exception e ) { }
2019-03-11 07:57:47 +01:00
if ( ! . dryRun ) {
// original code
try {
refreshToken = readText ( cfg . refreshTokenFilePath ) ;
} catch ( FileException e ) {
2019-05-13 15:10:19 +02:00
try {
return authorize ( ) ;
} catch ( CurlException e ) {
log . error ( "Cannot authorize with Microsoft OneDrive Service" ) ;
return false ;
}
2019-03-11 07:57:47 +01:00
}
return true ;
} else {
// --dry-run
if ( ! . simulateNoRefreshTokenFile ) {
try {
refreshToken = readText ( cfg . refreshTokenFilePath ) ;
} catch ( FileException e ) {
return authorize ( ) ;
}
return true ;
} else {
// --dry-run & --logout
return authorize ( ) ;
}
2016-08-04 23:35:58 +02:00
}
2015-09-01 20:45:34 +02:00
}
2016-06-30 12:51:44 +02:00
bool authorize ( )
2015-09-01 20:45:34 +02:00
{
import std.stdio , std . regex ;
2016-06-30 12:51:44 +02:00
char [ ] response ;
2020-03-22 21:01:51 +01:00
string url = authUrl ~ "?client_id=" ~ clientId ~ "&scope=Files.ReadWrite%20Files.ReadWrite.all%20Sites.Read.All%20Sites.ReadWrite.All%20offline_access&response_type=code&redirect_uri=" ~ redirectUrl ;
2019-06-11 09:12:36 +02:00
string authFilesString = cfg . getValueString ( "auth_files" ) ;
if ( authFilesString = = "" ) {
log . log ( "Authorize this app visiting:\n" ) ;
write ( url , "\n\n" , "Enter the response uri: " ) ;
readln ( response ) ;
2020-03-14 20:29:44 +01:00
cfg . applicationAuthorizeResponseUri = true ;
2019-06-11 09:12:36 +02:00
} else {
string [ ] authFiles = authFilesString . split ( ":" ) ;
string authUrl = authFiles [ 0 ] ;
string responseUrl = authFiles [ 1 ] ;
auto authUrlFile = File ( authUrl , "w" ) ;
authUrlFile . write ( url ) ;
authUrlFile . close ( ) ;
while ( ! exists ( responseUrl ) ) {
Thread . sleep ( dur ! ( "msecs" ) ( 100 ) ) ;
}
2020-03-02 02:54:56 +01:00
// read response from OneDrive
try {
response = cast ( char [ ] ) read ( responseUrl ) ;
} catch ( OneDriveException e ) {
// exception generated
displayOneDriveErrorMessage ( e . msg ) ;
return false ;
}
// try to remove old files
2019-06-11 09:12:36 +02:00
try {
std . file . remove ( authUrl ) ;
std . file . remove ( responseUrl ) ;
} catch ( FileException e ) {
log . error ( "Cannot remove files " , authUrl , " " , responseUrl ) ;
2020-03-02 02:54:56 +01:00
return false ;
2019-06-11 09:12:36 +02:00
}
}
2016-06-30 12:51:44 +02:00
// match the authorization code
2020-03-20 21:20:19 +01:00
auto c = matchFirst ( response , r"(?:[\?&]code=)([\w\d-.]+)" ) ;
2016-06-30 12:51:44 +02:00
if ( c . empty ) {
2016-08-04 23:35:58 +02:00
log . log ( "Invalid uri" ) ;
2016-06-30 12:51:44 +02:00
return false ;
2015-09-01 20:45:34 +02:00
}
2016-06-30 12:51:44 +02:00
c . popFront ( ) ; // skip the whole match
redeemToken ( c . front ) ;
return true ;
2015-09-01 20:45:34 +02:00
}
2020-03-19 20:12:47 +01:00
ulong getRetryAfterValue ( )
{
// Return the current value of retryAfterValue if it has been set to something other than 0
return . retryAfterValue ;
}
void resetRetryAfterValue ( )
{
// Reset the current value of retryAfterValue to 0 after it has been used
. retryAfterValue = 0 ;
}
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get
2016-12-25 16:38:55 +01:00
JSONValue getDefaultDrive ( )
{
checkAccessTokenExpired ( ) ;
2020-05-20 03:37:11 +02:00
const ( char ) [ ] url ;
url = driveUrl ;
2016-12-25 16:38:55 +01:00
return get ( driveUrl ) ;
}
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getDefaultRoot ( )
{
checkAccessTokenExpired ( ) ;
2020-05-20 03:37:11 +02:00
const ( char ) [ ] url ;
url = driveUrl ~ "/root" ;
return get ( url ) ;
2017-12-27 15:12:38 +01:00
}
2020-06-27 11:10:37 +02:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getDriveIdRoot ( const ( char ) [ ] driveId )
{
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
url = driveByIdUrl ~ driveId ~ "/root" ;
return get ( url ) ;
}
2017-12-27 15:12:38 +01:00
2020-06-27 11:10:37 +02:00
// https://docs.microsoft.com/en-us/graph/api/drive-sharedwithme
JSONValue getSharedWithMe ( )
{
checkAccessTokenExpired ( ) ;
return get ( sharedWithMe ) ;
}
2020-08-08 00:56:00 +02:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get
JSONValue getDriveQuota ( const ( char ) [ ] driveId )
{
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
url = driveByIdUrl ~ driveId ~ "/" ;
url ~ = "?select=quota" ;
return get ( url ) ;
}
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta
2020-06-27 11:10:37 +02:00
JSONValue viewChangesByItemId ( const ( char ) [ ] driveId , const ( char ) [ ] id , const ( char ) [ ] deltaLink )
2015-09-01 20:45:34 +02:00
{
2015-09-01 22:23:42 +02:00
checkAccessTokenExpired ( ) ;
2020-05-20 03:37:11 +02:00
const ( char ) [ ] url ;
// configure deltaLink to query
if ( deltaLink . empty ) {
2017-12-27 15:12:38 +01:00
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/delta" ;
2018-12-28 02:26:03 +01:00
url ~ = "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size" ;
2020-05-20 03:37:11 +02:00
} else {
url = deltaLink ;
2017-12-27 15:12:38 +01:00
}
2015-09-01 20:45:34 +02:00
return get ( url ) ;
}
2020-06-16 23:57:14 +02:00
2020-06-27 11:10:37 +02:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta
JSONValue viewChangesByDriveId ( const ( char ) [ ] driveId , const ( char ) [ ] deltaLink )
{
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url = deltaLink ;
if ( url = = null ) {
url = driveByIdUrl ~ driveId ~ "/root/delta" ;
url ~ = "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size" ;
}
return get ( url ) ;
}
2020-06-16 23:57:14 +02:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_list_children
JSONValue listChildren ( const ( char ) [ ] driveId , const ( char ) [ ] id , const ( char ) [ ] nextLink )
{
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
// configure URL to query
if ( nextLink . empty ) {
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/children" ;
url ~ = "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size" ;
} else {
url = nextLink ;
}
return get ( url ) ;
}
2020-06-27 11:10:37 +02:00
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get_content
2018-08-09 23:46:27 +02:00
void downloadById ( const ( char ) [ ] driveId , const ( char ) [ ] id , string saveToPath , long fileSize )
2015-09-01 20:45:34 +02:00
{
2015-09-01 22:23:42 +02:00
checkAccessTokenExpired ( ) ;
2015-10-04 16:24:45 +02:00
scope ( failure ) {
2015-09-01 20:45:34 +02:00
if ( exists ( saveToPath ) ) remove ( saveToPath ) ;
}
2017-12-27 15:12:38 +01:00
mkdirRecurse ( dirName ( saveToPath ) ) ;
const ( char ) [ ] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content?AVOverride=1" ;
2018-08-09 23:46:27 +02:00
download ( url , saveToPath , fileSize ) ;
2015-09-01 20:45:34 +02:00
}
2017-12-31 02:30:31 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
JSONValue simpleUpload ( string localPath , string parentDriveId , string parentId , string filename , const ( char ) [ ] eTag = null )
{
checkAccessTokenExpired ( ) ;
string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent ( filename ) ~ ":/content" ;
// TODO: investigate why this fails for remote folders
//if (eTag) http.addRequestHeader("If-Match", eTag);
/*else http.addRequestHeader("If-None-Match", "*");*/
return upload ( localPath , url ) ;
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
2017-12-31 13:47:18 +01:00
JSONValue simpleUploadReplace ( string localPath , string driveId , string id , const ( char ) [ ] eTag = null )
2017-12-31 02:30:31 +01:00
{
checkAccessTokenExpired ( ) ;
string url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content" ;
2015-10-04 16:24:45 +02:00
if ( eTag ) http . addRequestHeader ( "If-Match" , eTag ) ;
return upload ( localPath , url ) ;
2015-09-01 20:45:34 +02:00
}
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_update
JSONValue updateById ( const ( char ) [ ] driveId , const ( char ) [ ] id , JSONValue data , const ( char ) [ ] eTag = null )
2015-09-05 21:23:43 +02:00
{
checkAccessTokenExpired ( ) ;
2017-12-27 15:12:38 +01:00
const ( char ) [ ] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ;
2015-09-05 21:23:43 +02:00
if ( eTag ) http . addRequestHeader ( "If-Match" , eTag ) ;
http . addRequestHeader ( "Content-Type" , "application/json" ) ;
2015-10-04 16:24:45 +02:00
return patch ( url , data . toString ( ) ) ;
2015-09-05 21:23:43 +02:00
}
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delete
void deleteById ( const ( char ) [ ] driveId , const ( char ) [ ] id , const ( char ) [ ] eTag = null )
2015-09-06 10:23:32 +02:00
{
checkAccessTokenExpired ( ) ;
2017-12-27 15:12:38 +01:00
const ( char ) [ ] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ;
2016-12-28 23:47:29 +01:00
//TODO: investigate why this always fail with 412 (Precondition Failed)
//if (eTag) http.addRequestHeader("If-Match", eTag);
2015-10-04 16:24:45 +02:00
del ( url ) ;
2015-09-06 10:23:32 +02:00
}
2017-12-31 13:18:11 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_post_children
JSONValue createById ( const ( char ) [ ] parentDriveId , const ( char ) [ ] parentId , JSONValue item )
2015-09-06 11:06:13 +02:00
{
2016-12-28 16:29:55 +01:00
checkAccessTokenExpired ( ) ;
2017-12-31 13:18:11 +01:00
const ( char ) [ ] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ "/children" ;
2018-03-14 05:43:40 +01:00
http . addRequestHeader ( "Content-Type" , "application/json" ) ;
2015-10-04 16:24:45 +02:00
return post ( url , item . toString ( ) ) ;
2015-09-06 11:06:13 +02:00
}
2018-03-14 05:43:40 +01:00
// Return the details of the specified path
JSONValue getPathDetails ( const ( string ) path )
{
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
2018-12-28 02:26:03 +01:00
if ( ( path = = "." ) | | ( path = = "/" ) ) url = driveUrl ~ "/root/" ;
2018-03-14 05:43:40 +01:00
else url = itemByPathUrl ~ encodeComponent ( path ) ~ ":/" ;
2019-04-01 20:21:02 +02:00
url ~ = "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size" ;
2018-03-14 05:43:40 +01:00
return get ( url ) ;
}
2018-04-23 02:58:47 +02:00
// Return the details of the specified id
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
2018-06-17 04:02:58 +02:00
JSONValue getPathDetailsById ( const ( char ) [ ] driveId , const ( char ) [ ] id )
2018-04-23 02:58:47 +02:00
{
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
2018-06-17 04:02:58 +02:00
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ;
2019-04-01 20:21:02 +02:00
url ~ = "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size" ;
2018-04-23 02:58:47 +02:00
return get ( url ) ;
}
2020-06-27 11:10:37 +02:00
// Return the requested details of the specified path on the specified drive id
JSONValue getPathDetailsByDriveId ( const ( char ) [ ] driveId , const ( string ) path )
{
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
// string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
// Required format: /drives/{drive-id}/root:/{item-path}
url = driveByIdUrl ~ driveId ~ "/root:/" ~ encodeComponent ( path ) ;
url ~ = "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size" ;
return get ( url ) ;
}
2018-08-09 23:46:27 +02:00
// Return the requested details of the specified id
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
2018-09-13 00:41:46 +02:00
JSONValue getFileDetails ( const ( char ) [ ] driveId , const ( char ) [ ] id )
2018-08-09 23:46:27 +02:00
{
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ;
2019-08-02 10:43:31 +02:00
url ~ = "?select=size,malware,file,webUrl" ;
2018-08-09 23:46:27 +02:00
return get ( url ) ;
}
2018-04-23 02:58:47 +02:00
2020-09-14 09:49:50 +02:00
// Create an anonymous read-only shareable link for an existing file on OneDrive
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createlink
JSONValue createShareableLink ( const ( char ) [ ] driveId , const ( char ) [ ] id , JSONValue accessScope )
{
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/createLink" ;
http . addRequestHeader ( "Content-Type" , "application/json" ) ;
return post ( url , accessScope . toString ( ) ) ;
}
2018-03-14 05:43:40 +01:00
// https://dev.onedrive.com/items/move.htm
JSONValue moveByPath ( const ( char ) [ ] sourcePath , JSONValue moveData )
{
// Need to use itemByPathUrl
checkAccessTokenExpired ( ) ;
string url = itemByPathUrl ~ encodeComponent ( sourcePath ) ;
http . addRequestHeader ( "Content-Type" , "application/json" ) ;
return move ( url , moveData . toString ( ) ) ;
}
2017-12-31 16:11:02 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
2018-07-02 23:24:57 +02:00
JSONValue createUploadSession ( const ( char ) [ ] parentDriveId , const ( char ) [ ] parentId , const ( char ) [ ] filename , const ( char ) [ ] eTag = null , JSONValue item = null )
2015-09-27 18:47:41 +02:00
{
checkAccessTokenExpired ( ) ;
2017-12-31 16:11:02 +01:00
const ( char ) [ ] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent ( filename ) ~ ":/createUploadSession" ;
2015-09-27 18:47:41 +02:00
if ( eTag ) http . addRequestHeader ( "If-Match" , eTag ) ;
2018-07-02 23:24:57 +02:00
http . addRequestHeader ( "Content-Type" , "application/json" ) ;
return post ( url , item . toString ( ) ) ;
2015-09-27 18:47:41 +02:00
}
// https://dev.onedrive.com/items/upload_large_files.htm
JSONValue uploadFragment ( const ( char ) [ ] uploadUrl , string filepath , long offset , long offsetSize , long fileSize )
{
checkAccessTokenExpired ( ) ;
2020-05-20 03:37:11 +02:00
// open file as read-only in binary mode
auto file = File ( filepath , "rb" ) ;
file . seek ( offset ) ;
string contentRange = "bytes " ~ to ! string ( offset ) ~ "-" ~ to ! string ( offset + offsetSize - 1 ) ~ "/" ~ to ! string ( fileSize ) ;
// function scopes
2015-10-04 16:24:45 +02:00
scope ( exit ) {
http . clearRequestHeaders ( ) ;
http . onSend = null ;
2020-05-20 03:37:11 +02:00
http . onReceive = null ;
http . onReceiveHeader = null ;
http . onReceiveStatusLine = null ;
http . contentLength = 0 ;
// close file if open
if ( file . isOpen ( ) ) {
// close open file
file . close ( ) ;
}
2015-10-04 16:24:45 +02:00
}
2020-05-20 03:37:11 +02:00
2015-09-27 18:47:41 +02:00
http . method = HTTP . Method . put ;
http . url = uploadUrl ;
http . addRequestHeader ( "Content-Range" , contentRange ) ;
2015-10-04 16:24:45 +02:00
http . onSend = data = > file . rawRead ( data ) . length ;
http . contentLength = offsetSize ;
auto response = perform ( ) ;
// TODO: retry on 5xx errors
2019-08-30 21:27:49 +02:00
checkHttpCode ( response ) ;
2015-10-04 16:24:45 +02:00
return response ;
2015-09-27 18:47:41 +02:00
}
// https://dev.onedrive.com/items/upload_large_files.htm
JSONValue requestUploadStatus ( const ( char ) [ ] uploadUrl )
{
2015-10-04 16:24:45 +02:00
checkAccessTokenExpired ( ) ;
2017-03-11 11:40:19 +01:00
// when using microsoft graph the auth code is different
return get ( uploadUrl , true ) ;
2015-09-27 18:47:41 +02:00
}
2018-12-04 00:59:23 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/site_search?view=odsp-graph-online
2020-08-04 03:03:52 +02:00
JSONValue o365SiteSearch ( ) {
2018-12-04 00:59:23 +01:00
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
2020-08-04 03:03:52 +02:00
url = siteSearchUrl ~ "=*" ;
2018-12-04 00:59:23 +01:00
return get ( url ) ;
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_list?view=odsp-graph-online
JSONValue o365SiteDrives ( string site_id ) {
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
url = siteDriveUrl ~ site_id ~ "/drives" ;
return get ( url ) ;
}
2015-09-01 20:45:34 +02:00
private void redeemToken ( const ( char ) [ ] authCode )
{
2016-06-27 19:23:14 +02:00
const ( char ) [ ] postData =
"client_id=" ~ clientId ~
"&redirect_uri=" ~ redirectUrl ~
"&code=" ~ authCode ~
"&grant_type=authorization_code" ;
2015-09-01 20:45:34 +02:00
acquireToken ( postData ) ;
}
private void newToken ( )
{
2016-06-27 19:23:14 +02:00
string postData =
"client_id=" ~ clientId ~
"&redirect_uri=" ~ redirectUrl ~
"&refresh_token=" ~ refreshToken ~
"&grant_type=refresh_token" ;
2015-09-01 20:45:34 +02:00
acquireToken ( postData ) ;
}
private void acquireToken ( const ( char ) [ ] postData )
{
2020-03-02 02:54:56 +01:00
JSONValue response ;
try {
response = post ( tokenUrl , postData ) ;
} catch ( OneDriveException e ) {
// an error was generated
displayOneDriveErrorMessage ( e . msg ) ;
}
2019-09-06 05:12:13 +02:00
if ( response . type ( ) = = JSONType . object ) {
if ( "access_token" in response ) {
accessToken = "bearer " ~ response [ "access_token" ] . str ( ) ;
refreshToken = response [ "refresh_token" ] . str ( ) ;
accessTokenExpiration = Clock . currTime ( ) + dur ! "seconds" ( response [ "expires_in" ] . integer ( ) ) ;
if ( ! . dryRun ) {
std . file . write ( cfg . refreshTokenFilePath , refreshToken ) ;
}
if ( printAccessToken ) writeln ( "New access token: " , accessToken ) ;
} else {
log . error ( "\nInvalid authentication response from OneDrive. Please check the response uri\n" ) ;
// re-authorize
authorize ( ) ;
2019-03-12 01:50:21 +01:00
}
} else {
2019-09-06 05:12:13 +02:00
log . vdebug ( "Invalid JSON response from OneDrive unable to initialize application" ) ;
2019-03-11 07:57:47 +01:00
}
2015-09-01 20:45:34 +02:00
}
2015-09-01 22:23:42 +02:00
private void checkAccessTokenExpired ( )
{
2017-06-15 13:02:04 +02:00
try {
if ( Clock . currTime ( ) > = accessTokenExpiration ) {
newToken ( ) ;
}
} catch ( OneDriveException e ) {
if ( e . httpStatusCode = = 400 | | e . httpStatusCode = = 401 ) {
2020-09-24 22:42:44 +02:00
// flag error and notify
log . errorAndNotify ( "\nERROR: Refresh token invalid, use --logout to authorize the client again.\n" ) ;
// set error message
2017-06-15 13:02:04 +02:00
e . msg ~ = "\nRefresh token invalid, use --logout to authorize the client again" ;
}
2015-09-01 22:23:42 +02:00
}
}
2015-09-05 21:23:43 +02:00
2015-10-04 16:24:45 +02:00
private void addAccessTokenHeader ( )
2015-09-01 20:45:34 +02:00
{
2015-10-04 16:24:45 +02:00
http . addRequestHeader ( "Authorization" , accessToken ) ;
}
2017-03-11 11:40:19 +01:00
private JSONValue get ( const ( char ) [ ] url , bool skipToken = false )
2015-10-04 16:24:45 +02:00
{
scope ( exit ) http . clearRequestHeaders ( ) ;
2020-05-20 03:37:11 +02:00
log . vdebug ( "Request URL = " , url ) ;
2015-10-04 16:24:45 +02:00
http . method = HTTP . Method . get ;
http . url = url ;
2017-03-11 11:40:19 +01:00
if ( ! skipToken ) addAccessTokenHeader ( ) ; // HACK: requestUploadStatus
2020-05-20 03:37:11 +02:00
JSONValue response ;
response = perform ( ) ;
2016-12-13 18:04:16 +01:00
checkHttpCode ( response ) ;
2020-03-19 20:12:47 +01:00
// OneDrive API Response Debugging if --https-debug is being used
2018-04-15 12:02:39 +02:00
if ( . debugResponse ) {
2020-03-19 20:12:47 +01:00
log . vdebug ( "OneDrive API Response: " , response ) ;
2018-04-15 12:02:39 +02:00
}
2015-10-04 16:24:45 +02:00
return response ;
}
private void del ( const ( char ) [ ] url )
{
scope ( exit ) http . clearRequestHeaders ( ) ;
http . method = HTTP . Method . del ;
http . url = url ;
addAccessTokenHeader ( ) ;
2016-12-13 18:04:16 +01:00
auto response = perform ( ) ;
checkHttpCode ( response ) ;
2015-10-04 16:24:45 +02:00
}
2018-08-09 23:46:27 +02:00
private void download ( const ( char ) [ ] url , string filename , long fileSize )
2015-10-04 16:24:45 +02:00
{
2018-08-09 23:46:27 +02:00
// Threshold for displaying download bar
long thresholdFileSize = 4 * 2 ^ ^ 20 ; // 4 MiB
2020-05-20 03:37:11 +02:00
// open file as write in binary mode
auto file = File ( filename , "wb" ) ;
// function scopes
scope ( exit ) {
http . clearRequestHeaders ( ) ;
http . onSend = null ;
http . onReceive = null ;
http . onReceiveHeader = null ;
http . onReceiveStatusLine = null ;
http . contentLength = 0 ;
// Reset onProgress to not display anything for next download
http . onProgress = delegate int ( size_t dltotal , size_t dlnow , size_t ultotal , size_t ulnow )
{
return 0 ;
} ;
// close file if open
if ( file . isOpen ( ) ) {
// close open file
file . close ( ) ;
}
}
2018-08-09 23:46:27 +02:00
2015-10-04 16:24:45 +02:00
http . method = HTTP . Method . get ;
http . url = url ;
addAccessTokenHeader ( ) ;
2020-05-20 03:37:11 +02:00
2015-10-04 16:24:45 +02:00
http . onReceive = ( ubyte [ ] data ) {
2020-05-20 03:37:11 +02:00
file . rawWrite ( data ) ;
2015-10-04 16:24:45 +02:00
return data . length ;
} ;
2018-08-09 23:46:27 +02:00
if ( fileSize > = thresholdFileSize ) {
// Download Progress Bar
size_t iteration = 20 ;
Progress p = new Progress ( iteration ) ;
p . title = "Downloading" ;
writeln ( ) ;
2020-04-25 22:52:29 +02:00
bool barInit = false ;
2018-08-09 23:46:27 +02:00
real previousDLPercent = - 1.0 ;
real percentCheck = 5.0 ;
// Setup progress bar to display
http . onProgress = delegate int ( size_t dltotal , size_t dlnow , size_t ultotal , size_t ulnow )
{
// For each onProgress, what is the % of dlnow to dltotal
2020-04-25 22:52:29 +02:00
// floor - rounds down to nearest whole number
real currentDLPercent = floor ( double ( dlnow ) / dltotal * 100 ) ;
if ( currentDLPercent > 0 ) {
// We have started downloading
// If matching 5% of download, increment progress bar
if ( ( isIdentical ( fmod ( currentDLPercent , percentCheck ) , 0.0 ) ) & & ( previousDLPercent ! = currentDLPercent ) ) {
// What have we downloaded thus far
log . vdebugNewLine ( "Data Received = " , dlnow ) ;
log . vdebug ( "Expected Total = " , dltotal ) ;
log . vdebug ( "Percent Complete = " , currentDLPercent ) ;
// Increment counter & show bar update
p . next ( ) ;
previousDLPercent = currentDLPercent ;
}
} else {
if ( ( currentDLPercent = = 0 ) & & ( ! barInit ) ) {
// Initialise the download bar at 0%
// Downloading 0% | | ETA --:--:--:^C
p . next ( ) ;
barInit = true ;
}
2018-08-09 23:46:27 +02:00
}
return 0 ;
} ;
// Perform download & display progress bar
2020-04-23 02:25:41 +02:00
try {
// try and catch any curl error
http . perform ( ) ;
writeln ( ) ;
2020-05-20 03:37:11 +02:00
// Reset onProgress to not display anything for next download done using exit scope
2020-04-23 02:25:41 +02:00
} catch ( CurlException e ) {
displayOneDriveErrorMessage ( e . msg ) ;
}
2020-05-20 03:37:11 +02:00
// free progress bar memory
p = null ;
2018-08-09 23:46:27 +02:00
} else {
// No progress bar
2020-04-23 02:25:41 +02:00
try {
// try and catch any curl error
http . perform ( ) ;
} catch ( CurlException e ) {
displayOneDriveErrorMessage ( e . msg ) ;
}
2018-08-09 23:46:27 +02:00
}
// Check the HTTP response code
2015-10-04 16:24:45 +02:00
checkHttpCode ( ) ;
2015-09-01 20:45:34 +02:00
}
2015-09-05 21:23:43 +02:00
private auto patch ( T ) ( const ( char ) [ ] url , const ( T ) [ ] patchData )
{
2015-10-04 16:24:45 +02:00
scope ( exit ) http . clearRequestHeaders ( ) ;
http . method = HTTP . Method . patch ;
http . url = url ;
addAccessTokenHeader ( ) ;
auto response = perform ( patchData ) ;
2016-12-13 18:04:16 +01:00
checkHttpCode ( response ) ;
2015-10-04 16:24:45 +02:00
return response ;
2015-09-05 21:23:43 +02:00
}
2015-09-01 20:45:34 +02:00
private auto post ( T ) ( const ( char ) [ ] url , const ( T ) [ ] postData )
{
2015-10-04 16:24:45 +02:00
scope ( exit ) http . clearRequestHeaders ( ) ;
http . method = HTTP . Method . post ;
http . url = url ;
addAccessTokenHeader ( ) ;
auto response = perform ( postData ) ;
2016-12-13 18:04:16 +01:00
checkHttpCode ( response ) ;
2015-10-04 16:24:45 +02:00
return response ;
}
2018-03-14 05:43:40 +01:00
private auto move ( T ) ( const ( char ) [ ] url , const ( T ) [ ] postData )
{
scope ( exit ) http . clearRequestHeaders ( ) ;
http . method = HTTP . Method . patch ;
http . url = url ;
addAccessTokenHeader ( ) ;
auto response = perform ( postData ) ;
checkHttpCode ( ) ;
return response ;
}
2015-10-04 16:24:45 +02:00
private JSONValue upload ( string filepath , string url )
{
2020-05-20 03:37:11 +02:00
checkAccessTokenExpired ( ) ;
// open file as read-only in binary mode
auto file = File ( filepath , "rb" ) ;
// function scopes
2015-10-04 16:24:45 +02:00
scope ( exit ) {
http . clearRequestHeaders ( ) ;
http . onSend = null ;
2020-05-20 03:37:11 +02:00
http . onReceive = null ;
http . onReceiveHeader = null ;
http . onReceiveStatusLine = null ;
2015-10-04 16:24:45 +02:00
http . contentLength = 0 ;
2020-05-20 03:37:11 +02:00
// close file if open
if ( file . isOpen ( ) ) {
// close open file
file . close ( ) ;
}
2015-10-04 16:24:45 +02:00
}
2020-05-20 03:37:11 +02:00
2015-10-04 16:24:45 +02:00
http . method = HTTP . Method . put ;
http . url = url ;
addAccessTokenHeader ( ) ;
http . addRequestHeader ( "Content-Type" , "application/octet-stream" ) ;
http . onSend = data = > file . rawRead ( data ) . length ;
http . contentLength = file . size ;
auto response = perform ( ) ;
2016-12-13 18:04:16 +01:00
checkHttpCode ( response ) ;
2015-10-04 16:24:45 +02:00
return response ;
}
private JSONValue perform ( const ( void ) [ ] sendData )
{
scope ( exit ) {
http . onSend = null ;
http . contentLength = 0 ;
}
if ( sendData ) {
http . contentLength = sendData . length ;
http . onSend = ( void [ ] buf ) {
import std.algorithm : min ;
size_t minLen = min ( buf . length , sendData . length ) ;
if ( minLen = = 0 ) return 0 ;
buf [ 0 . . minLen ] = sendData [ 0 . . minLen ] ;
sendData = sendData [ minLen . . $ ] ;
return minLen ;
} ;
} else {
http . onSend = buf = > 0 ;
}
2020-09-25 04:55:21 +02:00
auto response = perform ( ) ;
return response ;
2015-10-04 16:24:45 +02:00
}
private JSONValue perform ( )
{
scope ( exit ) http . onReceive = null ;
2015-10-04 17:33:48 +02:00
char [ ] content ;
2020-05-20 03:37:11 +02:00
JSONValue json ;
2015-10-04 16:24:45 +02:00
http . onReceive = ( ubyte [ ] data ) {
content ~ = data ;
2020-03-19 20:12:47 +01:00
// HTTP Server Response Code Debugging if --https-debug is being used
2018-11-14 20:08:55 +01:00
if ( . debugResponse ) {
2020-03-19 20:12:47 +01:00
log . vdebug ( "onedrive.perform() => OneDrive HTTP Server Response: " , http . statusLine . code ) ;
2018-11-14 20:08:55 +01:00
}
2015-10-04 16:24:45 +02:00
return data . length ;
} ;
2018-08-02 00:25:37 +02:00
try {
http . perform ( ) ;
2020-03-19 20:12:47 +01:00
// Get the HTTP Response headers - needed for correct 429 handling
auto responseHeaders = http . responseHeaders ( ) ;
// HTTP Server Response Headers Debugging if --https-debug is being used
if ( . debugResponse ) {
log . vdebug ( "onedrive.perform() => HTTP Response Headers: " , responseHeaders ) ;
}
2020-05-20 03:37:11 +02:00
// is retry-after in the response headers
2020-03-19 20:12:47 +01:00
if ( "retry-after" in http . responseHeaders ) {
2020-05-20 03:37:11 +02:00
// Set the retry-after value
2020-03-19 20:12:47 +01:00
log . vdebug ( "onedrive.perform() => Received a 'Retry-After' Header Response with the following value: " , http . responseHeaders [ "retry-after" ] ) ;
log . vdebug ( "onedrive.perform() => Setting retryAfterValue to: " , http . responseHeaders [ "retry-after" ] ) ;
. retryAfterValue = to ! ulong ( http . responseHeaders [ "retry-after" ] ) ;
}
2018-08-02 00:25:37 +02:00
} catch ( CurlException e ) {
2019-10-21 09:04:08 +02:00
// Parse and display error message received from OneDrive
2020-09-01 22:23:41 +02:00
log . vdebug ( "onedrive.perform() Generated a OneDrive CurlException" ) ;
2019-10-21 09:04:08 +02:00
log . error ( "ERROR: OneDrive returned an error with the following message:" ) ;
auto errorArray = splitLines ( e . msg ) ;
string errorMessage = errorArray [ 0 ] ;
2020-03-19 20:12:47 +01:00
2020-05-24 09:39:28 +02:00
if ( canFind ( errorMessage , "Couldn't connect to server on handle" ) | | canFind ( errorMessage , "Couldn't resolve host name on handle" ) ) {
2019-10-21 09:04:08 +02:00
// This is a curl timeout
log . error ( " Error Message: There was a timeout in accessing the Microsoft OneDrive service - Internet connectivity issue?" ) ;
// or 408 request timeout
// https://github.com/abraunegg/onedrive/issues/694
// Back off & retry with incremental delay
int retryCount = 10000 ;
int retryAttempts = 1 ;
int backoffInterval = 1 ;
int maxBackoffInterval = 3600 ;
2019-10-21 09:53:00 +02:00
bool retrySuccess = false ;
while ( ! retrySuccess ) {
2019-10-21 09:04:08 +02:00
backoffInterval + + ;
int thisBackOffInterval = retryAttempts * backoffInterval ;
2020-05-24 09:39:28 +02:00
log . vdebug ( " Retry Attempt: " , retryAttempts ) ;
2019-10-21 09:04:08 +02:00
if ( thisBackOffInterval < = maxBackoffInterval ) {
2020-05-24 09:39:28 +02:00
log . vdebug ( " Retry In (seconds): " , thisBackOffInterval ) ;
2019-10-21 09:04:08 +02:00
Thread . sleep ( dur ! "seconds" ( thisBackOffInterval ) ) ;
} else {
2020-05-24 09:39:28 +02:00
log . vdebug ( " Retry In (seconds): " , maxBackoffInterval ) ;
2019-10-21 09:04:08 +02:00
Thread . sleep ( dur ! "seconds" ( maxBackoffInterval ) ) ;
}
try {
http . perform ( ) ;
// no error from http.perform() on re-try
log . log ( "Internet connectivity to Microsoft OneDrive service has been restored" ) ;
2019-10-21 09:53:00 +02:00
retrySuccess = true ;
2019-10-21 09:04:08 +02:00
} catch ( CurlException e ) {
2020-05-24 09:39:28 +02:00
if ( canFind ( e . msg , "Couldn't connect to server on handle" ) | | canFind ( e . msg , "Couldn't resolve host name on handle" ) ) {
2019-10-21 09:04:08 +02:00
log . error ( " Error Message: There was a timeout in accessing the Microsoft OneDrive service - Internet connectivity issue?" ) ;
// Increment & loop around
retryAttempts + + ;
}
if ( retryAttempts = = retryCount ) {
// we have attempted to re-connect X number of times
// false set this to true to break out of while loop
2019-10-21 09:53:00 +02:00
retrySuccess = true ;
2019-10-21 09:04:08 +02:00
}
}
}
if ( retryAttempts > = retryCount ) {
log . error ( " Error Message: Was unable to reconnect to the Microsoft OneDrive service after 10000 attempts lasting over 1.2 years!" ) ;
throw new OneDriveException ( 408 , "Request Timeout - HTTP 408 or Internet down?" ) ;
}
} else {
// Some other error was returned
log . error ( " Error Message: " , errorMessage ) ;
}
// return an empty JSON for handling
2019-09-06 05:12:13 +02:00
return json ;
2018-08-02 00:25:37 +02:00
}
2017-03-11 11:40:19 +01:00
try {
json = content . parseJSON ( ) ;
} catch ( JSONException e ) {
2019-07-26 21:24:59 +02:00
// Log that a JSON Exception was caught, dont output the HTML response from OneDrive
log . vdebug ( "JSON Exception caught when performing HTTP operations - use --debug-https to diagnose further" ) ;
2017-03-11 11:40:19 +01:00
}
return json ;
2015-09-01 20:45:34 +02:00
}
2015-09-06 10:07:18 +02:00
private void checkHttpCode ( )
2015-09-01 20:45:34 +02:00
{
2018-03-14 05:43:40 +01:00
// https://dev.onedrive.com/misc/errors.htm
// https://developer.overdrive.com/docs/reference-guide
/ *
2018-11-14 20:08:55 +01:00
HTTP / 1.1 Response handling
2018-03-14 05:43:40 +01:00
Errors in the OneDrive API are returned using standard HTTP status codes , as well as a JSON error response object . The following HTTP status codes should be expected .
Status code Status message Description
2020-05-05 23:20:13 +02:00
100 Continue Continue
2018-03-14 05:43:40 +01:00
200 OK Request was handled OK
201 Created This means you ' ve made a successful POST to checkout , lock in a format , or place a hold
204 No Content This means you ' ve made a successful DELETE to remove a hold or return a title
400 Bad Request Cannot process the request because it is malformed or incorrect .
401 Unauthorized Required authentication information is either missing or not valid for the resource .
403 Forbidden Access is denied to the requested resource . The user might not have enough permission .
404 Not Found The requested resource doesn ’ t exist .
405 Method Not Allowed The HTTP method in the request is not allowed on the resource .
406 Not Acceptable This service doesn ’ t support the format requested in the Accept header .
2019-10-21 09:04:08 +02:00
408 Request Time out Not expected from OneDrive , but can be used to handle Internet connection failures the same ( fallback and try again )
2018-03-14 05:43:40 +01:00
409 Conflict The current state conflicts with what the request expects . For example , the specified parent folder might not exist .
410 Gone The requested resource is no longer available at the server .
411 Length Required A Content - Length header is required on the request .
412 Precondition Failed A precondition provided in the request ( such as an if - match header ) does not match the resource ' s current state .
413 Request Entity Too Large The request size exceeds the maximum limit .
415 Unsupported Media Type The content type of the request is a format that is not supported by the service .
416 Requested Range Not Satisfiable The specified byte range is invalid or unavailable .
422 Unprocessable Entity Cannot process the request because it is semantically incorrect .
429 Too Many Requests Client application has been throttled and should not attempt to repeat the request until an amount of time has elapsed .
500 Internal Server Error There was an internal server error while processing the request .
501 Not Implemented The requested feature isn ’ t implemented .
502 Bad Gateway The service was unreachable
503 Service Unavailable The service is temporarily unavailable . You may repeat the request after a delay . There may be a Retry - After header .
507 Insufficient Storage The maximum storage quota has been reached .
509 Bandwidth Limit Exceeded Your app has been throttled for exceeding the maximum bandwidth cap . Your app can retry the request again after more time has elapsed .
2018-11-14 20:08:55 +01:00
HTTP / 2 Response handling
0 OK
2018-03-14 05:43:40 +01:00
* /
switch ( http . statusLine . code )
{
2019-08-30 21:27:49 +02:00
// 0 - OK ... HTTP2 version of 200 OK
2018-11-14 20:08:55 +01:00
case 0 :
break ;
2020-05-05 23:20:13 +02:00
// 100 - Continue
case 100 :
break ;
2018-05-03 08:21:53 +02:00
// 200 - OK
case 200 :
// No Log ..
break ;
// 201 - Created OK
// 202 - Accepted
// 204 - Deleted OK
case 201 , 202 , 204 :
// No actions, but log if verbose logging
2018-05-08 22:47:03 +02:00
//log.vlog("OneDrive Response: '", http.statusLine.code, " - ", http.statusLine.reason, "'");
2018-05-03 08:21:53 +02:00
break ;
2018-11-11 20:01:57 +01:00
// 302 - resource found and available at another location, redirect
case 302 :
break ;
2018-05-03 08:21:53 +02:00
// 400 - Bad Request
case 400 :
// Bad Request .. how should we act?
log . vlog ( "OneDrive returned a 'HTTP 400 - Bad Request' - gracefully handling error" ) ;
2019-08-09 10:14:10 +02:00
break ;
// 403 - Forbidden
case 403 :
// OneDrive responded that the user is forbidden
log . vlog ( "OneDrive returned a 'HTTP 403 - Forbidden' - gracefully handling error" ) ;
break ;
2018-05-03 08:21:53 +02:00
2019-08-09 10:14:10 +02:00
// 404 - Item not found
2018-05-03 08:21:53 +02:00
case 404 :
// Item was not found - do not throw an exception
log . vlog ( "OneDrive returned a 'HTTP 404 - Item not found' - gracefully handling error" ) ;
break ;
2019-10-21 09:04:08 +02:00
// 408 - Request Timeout
case 408 :
// Request to connect to OneDrive service timed out
log . vlog ( "Request Timeout - gracefully handling error" ) ;
throw new OneDriveException ( 408 , "Request Timeout - HTTP 408 or Internet down?" ) ;
2018-05-03 08:21:53 +02:00
// 409 - Conflict
case 409 :
// Conflict handling .. how should we act? This only really gets triggered if we are using --local-first & we remove items.db as the DB thinks the file is not uploaded but it is
log . vlog ( "OneDrive returned a 'HTTP 409 - Conflict' - gracefully handling error" ) ;
break ;
// 412 - Precondition Failed
case 412 :
// A precondition provided in the request (such as an if-match header) does not match the resource's current state.
log . vlog ( "OneDrive returned a 'HTTP 412 - Precondition Failed' - gracefully handling error" ) ;
break ;
// 415 - Unsupported Media Type
case 415 :
// Unsupported Media Type ... sometimes triggered on image files, especially PNG
log . vlog ( "OneDrive returned a 'HTTP 415 - Unsupported Media Type' - gracefully handling error" ) ;
break ;
2018-08-27 02:45:26 +02:00
// 429 - Too Many Requests
case 429 :
// Too many requests in a certain time window
// https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
log . vlog ( "OneDrive returned a 'HTTP 429 - Too Many Requests' - gracefully handling error" ) ;
2020-03-19 20:12:47 +01:00
throw new OneDriveException ( http . statusLine . code , http . statusLine . reason ) ;
2018-08-27 02:45:26 +02:00
2018-05-03 08:21:53 +02:00
// Server side (OneDrive) Errors
// 500 - Internal Server Error
// 502 - Bad Gateway
// 503 - Service Unavailable
// 504 - Gateway Timeout (Issue #320)
2019-04-04 08:13:15 +02:00
case 500 :
2018-05-03 08:21:53 +02:00
// No actions
2019-04-04 08:13:15 +02:00
log . vlog ( "OneDrive returned a 'HTTP 500 Internal Server Error' - gracefully handling error" ) ;
break ;
case 502 :
// No actions
log . vlog ( "OneDrive returned a 'HTTP 502 Bad Gateway Error' - gracefully handling error" ) ;
break ;
case 503 :
// No actions
log . vlog ( "OneDrive returned a 'HTTP 503 Service Unavailable Error' - gracefully handling error" ) ;
break ;
case 504 :
// No actions
log . vlog ( "OneDrive returned a 'HTTP 504 Gateway Timeout Error' - gracefully handling error" ) ;
break ;
2018-05-03 08:21:53 +02:00
// "else"
default :
throw new OneDriveException ( http . statusLine . code , http . statusLine . reason ) ;
2015-09-01 20:45:34 +02:00
}
}
2016-12-13 18:04:16 +01:00
private void checkHttpCode ( ref const JSONValue response )
{
2018-08-27 02:35:15 +02:00
switch ( http . statusLine . code )
{
2019-08-30 21:27:49 +02:00
// 0 - OK ... HTTP2 version of 200 OK
case 0 :
break ;
2020-05-05 23:20:13 +02:00
// 100 - Continue
case 100 :
break ;
2019-08-30 21:27:49 +02:00
// 200 - OK
case 200 :
// No Log ..
break ;
// 201 - Created OK
// 202 - Accepted
// 204 - Deleted OK
case 201 , 202 , 204 :
// No actions, but log if verbose logging
//log.vlog("OneDrive Response: '", http.statusLine.code, " - ", http.statusLine.reason, "'");
break ;
// 302 - resource found and available at another location, redirect
case 302 :
break ;
2019-03-12 01:50:21 +01:00
// 400 - Bad Request
case 400 :
// Bad Request .. how should we act?
2019-11-14 01:53:07 +01:00
// make sure this is thrown so that it is caught
throw new OneDriveException ( http . statusLine . code , http . statusLine . reason , response ) ;
2019-08-09 10:14:10 +02:00
// 403 - Forbidden
case 403 :
// OneDrive responded that the user is forbidden
log . vlog ( "OneDrive returned a 'HTTP 403 - Forbidden' - gracefully handling error" ) ;
// Throw this as a specific exception so this is caught when performing sync.o365SiteSearch
throw new OneDriveException ( http . statusLine . code , http . statusLine . reason , response ) ;
2019-04-01 19:51:25 +02:00
2018-08-27 02:35:15 +02:00
// 412 - Precondition Failed
case 412 :
2019-08-09 10:14:10 +02:00
// Throw this as a specific exception so this is caught when performing sync.uploadLastModifiedTime
2019-07-26 21:24:59 +02:00
throw new OneDriveException ( http . statusLine . code , http . statusLine . reason , response ) ;
2018-08-27 02:35:15 +02:00
// Server side (OneDrive) Errors
// 500 - Internal Server Error
// 502 - Bad Gateway
// 503 - Service Unavailable
// 504 - Gateway Timeout (Issue #320)
2019-04-04 08:13:15 +02:00
case 500 :
2019-08-24 09:28:16 +02:00
// Throw this as a specific exception so this is caught
2019-08-24 07:54:48 +02:00
throw new OneDriveException ( http . statusLine . code , http . statusLine . reason , response ) ;
2019-04-04 08:13:15 +02:00
case 502 :
2019-08-24 09:28:16 +02:00
// Throw this as a specific exception so this is caught
2019-08-24 07:54:48 +02:00
throw new OneDriveException ( http . statusLine . code , http . statusLine . reason , response ) ;
2019-04-04 08:13:15 +02:00
case 503 :
2019-08-24 09:28:16 +02:00
// Throw this as a specific exception so this is caught
2019-08-24 07:54:48 +02:00
throw new OneDriveException ( http . statusLine . code , http . statusLine . reason , response ) ;
2019-04-04 08:13:15 +02:00
case 504 :
2019-08-24 09:28:16 +02:00
// Throw this as a specific exception so this is caught
2019-08-24 07:54:48 +02:00
throw new OneDriveException ( http . statusLine . code , http . statusLine . reason , response ) ;
2018-08-27 02:35:15 +02:00
2018-11-11 20:01:57 +01:00
// Default - all other errors that are not a 2xx or a 302
2018-08-27 02:35:15 +02:00
default :
2018-11-11 20:01:57 +01:00
if ( http . statusLine . code / 100 ! = 2 & & http . statusLine . code ! = 302 ) {
2018-07-15 22:36:31 +02:00
throw new OneDriveException ( http . statusLine . code , http . statusLine . reason , response ) ;
}
2016-12-13 18:04:16 +01:00
}
}
2020-03-02 02:54:56 +01:00
// Parse and display error message received from OneDrive
private void displayOneDriveErrorMessage ( string message ) {
log . error ( "\nERROR: OneDrive returned an error with the following message:" ) ;
auto errorArray = splitLines ( message ) ;
log . error ( " Error Message: " , errorArray [ 0 ] ) ;
// Strip cause from error to leave a JSON
JSONValue errorMessage = parseJSON ( replace ( message , errorArray [ 0 ] , "" ) ) ;
// extra debug
log . vdebug ( "Raw Error Data: " , message ) ;
log . vdebug ( "JSON Message: " , errorMessage ) ;
if ( errorMessage . type ( ) = = JSONType . object ) {
log . error ( " Error Reason: " , errorMessage [ "error_description" ] . str ) ;
}
}
2015-09-01 20:45:34 +02:00
}
2017-03-11 11:40:19 +01:00
unittest
{
string configDirName = expandTilde ( "~/.config/onedrive" ) ;
auto cfg = new config . Config ( configDirName ) ;
cfg . init ( ) ;
OneDriveApi onedrive = new OneDriveApi ( cfg ) ;
onedrive . init ( ) ;
std . file . write ( "/tmp/test" , "test" ) ;
// simpleUpload
auto item = onedrive . simpleUpload ( "/tmp/test" , "/test" ) ;
try {
item = onedrive . simpleUpload ( "/tmp/test" , "/test" ) ;
} catch ( OneDriveException e ) {
assert ( e . httpStatusCode = = 409 ) ;
}
try {
item = onedrive . simpleUpload ( "/tmp/test" , "/test" , "123" ) ;
} catch ( OneDriveException e ) {
assert ( e . httpStatusCode = = 412 ) ;
}
item = onedrive . simpleUpload ( "/tmp/test" , "/test" , item [ "eTag" ] . str ) ;
// deleteById
try {
onedrive . deleteById ( item [ "id" ] . str , "123" ) ;
} catch ( OneDriveException e ) {
assert ( e . httpStatusCode = = 412 ) ;
}
onedrive . deleteById ( item [ "id" ] . str , item [ "eTag" ] . str ) ;
onedrive . http . shutdown ( ) ;
2015-09-01 20:45:34 +02:00
}