2018-11-10 20:38:41 +01:00
import std.net.curl ;
import etc.c.curl : CurlOption ;
2016-08-04 23:35:58 +02:00
import std.datetime , std . exception , std . file , std . json , std . path ;
2015-09-27 18:47:41 +02:00
import std.stdio , std . string , std . uni , std . uri ;
2018-08-02 00:25:37 +02:00
import core.stdc.stdlib ;
2018-08-09 23:46:27 +02:00
import core.thread , std . conv , std . math ;
import progress ;
import config ;
2016-08-04 23:35:58 +02:00
static import log ;
2018-04-15 12:02:39 +02:00
shared bool debugResponse = false ;
2019-04-11 04:26:20 +02:00
private bool dryRun = false ;
private bool simulateNoRefreshTokenFile = false ;
2015-12-29 19:38:15 +01:00
2015-09-01 20:45:34 +02:00
private immutable {
2018-12-04 00:59:23 +01:00
// Client Identifier
2017-03-12 10:30:31 +01:00
string clientId = "22c49a0d-d21c-4792-aed1-8f163c982546" ;
2018-12-04 00:59:23 +01:00
// Personal & Business Queries
2017-03-11 11:40:19 +01:00
string authUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize" ;
2018-01-02 16:32:16 +01:00
string redirectUrl = "https://login.microsoftonline.com/common/oauth2/nativeclient" ;
2017-03-11 11:40:19 +01:00
string tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token" ;
2017-12-27 15:12:38 +01:00
string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/" ;
2018-12-04 00:59:23 +01:00
// Office 365 / SharePoint Queries
string siteSearchUrl = "https://graph.microsoft.com/v1.0/sites?search" ;
string siteDriveUrl = "https://graph.microsoft.com/v1.0/sites/" ;
2015-09-01 20:45:34 +02:00
}
2018-11-23 20:26:30 +01:00
private {
string driveUrl = "https://graph.microsoft.com/v1.0/me/drive" ;
string itemByIdUrl = "https://graph.microsoft.com/v1.0/me/drive/items/" ;
string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/" ;
string driveId = "" ;
}
2015-09-01 20:45:34 +02:00
class OneDriveException : Exception
{
2017-12-28 15:03:15 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/errors
2016-12-14 15:17:20 +01:00
int httpStatusCode ;
2016-12-13 18:04:16 +01:00
JSONValue error ;
2015-10-04 16:24:45 +02:00
2016-12-14 15:17:20 +01:00
@safe pure this ( int httpStatusCode , string reason , string file = __FILE__ , size_t line = __LINE__ )
2015-10-04 16:24:45 +02:00
{
2016-12-14 15:17:20 +01:00
this . httpStatusCode = httpStatusCode ;
2016-12-13 18:04:16 +01:00
this . error = error ;
2016-12-14 15:17:20 +01:00
string msg = format ( "HTTP request returned status code %d (%s)" , httpStatusCode , reason ) ;
2017-12-28 15:03:15 +01:00
super ( msg , file , line ) ;
2015-10-04 16:24:45 +02:00
}
2016-12-13 18:04:16 +01:00
2016-12-14 15:17:20 +01:00
this ( int httpStatusCode , string reason , ref const JSONValue error , string file = __FILE__ , size_t line = __LINE__ )
2016-12-13 18:04:16 +01:00
{
2016-12-14 15:17:20 +01:00
this . httpStatusCode = httpStatusCode ;
2016-12-13 18:04:16 +01:00
this . error = error ;
2018-03-29 23:14:00 +02:00
string msg = format ( "HTTP request returned status code %d (%s)\n%s" , httpStatusCode , reason , toJSON ( error , true ) ) ;
2017-12-28 15:03:15 +01:00
super ( msg , file , line ) ;
2015-10-04 16:24:45 +02:00
}
2015-09-01 20:45:34 +02:00
}
final class OneDriveApi
{
2016-08-04 23:35:58 +02:00
private Config cfg ;
2015-09-01 20:45:34 +02:00
private string refreshToken , accessToken ;
2015-09-01 22:23:42 +02:00
private SysTime accessTokenExpiration ;
2016-02-24 17:07:59 +01:00
/* private */ HTTP http ;
2015-09-01 20:45:34 +02:00
2017-05-28 20:14:50 +02:00
// if true, every new access token is printed
bool printAccessToken ;
2019-04-11 04:26:20 +02:00
this ( Config cfg )
2015-09-01 20:45:34 +02:00
{
2016-08-04 23:35:58 +02:00
this . cfg = cfg ;
2015-09-01 20:45:34 +02:00
http = HTTP ( ) ;
2018-12-02 01:30:50 +01:00
// DNS lookup timeout
2018-04-24 04:10:27 +02:00
http . dnsTimeout = ( dur ! "seconds" ( 5 ) ) ;
2018-12-02 01:30:50 +01:00
// timeout for connecting
http . connectTimeout = ( dur ! "seconds" ( 10 ) ) ;
// Timeouts
// with the following settings we force
// - if there is no data flow for 5min, abort
// - if the download time for one item exceeds 1h, abort
//
// timeout for activity on connection
// this translates into Curl's CURLOPT_LOW_SPEED_TIME
// which says
// It contains the time in number seconds that the
// transfer speed should be below the CURLOPT_LOW_SPEED_LIMIT
// for the library to consider it too slow and abort.
http . dataTimeout = ( dur ! "seconds" ( 300 ) ) ;
// maximum time an operation is allowed to take
// This includes dns resolution, connecting, data transfer, etc.
http . operationTimeout = ( dur ! "seconds" ( 3600 ) ) ;
2018-11-10 20:38:41 +01:00
2018-11-11 20:01:57 +01:00
// Specify how many redirects should be allowed
http . maxRedirects ( 5 ) ;
2018-11-10 20:38:41 +01:00
2019-01-05 19:43:44 +01:00
// Do we enable curl debugging?
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "debug_https" ) ) {
2018-03-14 05:43:40 +01:00
http . verbose = true ;
2018-04-15 12:02:39 +02:00
. debugResponse = true ;
2018-03-14 05:43:40 +01:00
}
2019-01-05 19:43:44 +01:00
// What version of HTTP protocol do we use?
// Curl >= 7.62.0 defaults to http2 for a significant number of operations
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "force_http_11" ) ) {
2019-01-05 19:43:44 +01:00
log . vdebug ( "Downgrading all HTTP operations to HTTP 1.1" ) ;
// Downgrade to HTTP 1.1 - yes version = 2 is HTTP 1.1
http . handle . set ( CurlOption . http_version , 2 ) ;
}
2019-03-11 07:57:47 +01:00
// Do we set the dryRun handlers?
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "dry_run" ) ) {
2019-03-11 07:57:47 +01:00
. dryRun = true ;
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "logout" ) ) {
. simulateNoRefreshTokenFile = true ;
}
2019-03-11 07:57:47 +01:00
}
2016-08-04 23:35:58 +02:00
}
bool init ( )
{
2018-11-23 20:26:30 +01:00
try {
2019-04-11 04:26:20 +02:00
driveId = cfg . getValueString ( "drive_id" ) ;
2018-11-23 20:26:30 +01:00
if ( driveId . length ) {
driveUrl = driveByIdUrl ~ driveId ;
2019-04-11 04:26:20 +02:00
itemByIdUrl = driveUrl ~ "/items" ;
itemByPathUrl = driveUrl ~ "/root:/" ;
2018-11-23 20:26:30 +01:00
}
} catch ( Exception e ) { }
2019-03-11 07:57:47 +01:00
if ( ! . dryRun ) {
// original code
try {
refreshToken = readText ( cfg . refreshTokenFilePath ) ;
} catch ( FileException e ) {
return authorize ( ) ;
}
return true ;
} else {
// --dry-run
if ( ! . simulateNoRefreshTokenFile ) {
try {
refreshToken = readText ( cfg . refreshTokenFilePath ) ;
} catch ( FileException e ) {
return authorize ( ) ;
}
return true ;
} else {
// --dry-run & --logout
return authorize ( ) ;
}
2016-08-04 23:35:58 +02:00
}
2015-09-01 20:45:34 +02:00
}
2016-06-30 12:51:44 +02:00
bool authorize ( )
2015-09-01 20:45:34 +02:00
{
import std.stdio , std . regex ;
2016-06-30 12:51:44 +02:00
char [ ] response ;
2017-03-11 14:52:33 +01:00
string url = authUrl ~ "?client_id=" ~ clientId ~ "&scope=files.readwrite%20files.readwrite.all%20offline_access&response_type=code&redirect_uri=" ~ redirectUrl ;
2016-08-04 23:35:58 +02:00
log . log ( "Authorize this app visiting:\n" ) ;
2016-06-30 12:51:44 +02:00
write ( url , "\n\n" , "Enter the response uri: " ) ;
readln ( response ) ;
// match the authorization code
2018-01-20 21:13:01 +01:00
auto c = matchFirst ( response , r"(?:[\?&]code=)([\w\d-]+)" ) ;
2016-06-30 12:51:44 +02:00
if ( c . empty ) {
2016-08-04 23:35:58 +02:00
log . log ( "Invalid uri" ) ;
2016-06-30 12:51:44 +02:00
return false ;
2015-09-01 20:45:34 +02:00
}
2016-06-30 12:51:44 +02:00
c . popFront ( ) ; // skip the whole match
redeemToken ( c . front ) ;
return true ;
2015-09-01 20:45:34 +02:00
}
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get
2016-12-25 16:38:55 +01:00
JSONValue getDefaultDrive ( )
{
checkAccessTokenExpired ( ) ;
return get ( driveUrl ) ;
}
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getDefaultRoot ( )
{
checkAccessTokenExpired ( ) ;
return get ( driveUrl ~ "/root" ) ;
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta
2017-06-15 13:02:04 +02:00
JSONValue viewChangesById ( const ( char ) [ ] driveId , const ( char ) [ ] id , const ( char ) [ ] deltaLink )
2015-09-01 20:45:34 +02:00
{
2015-09-01 22:23:42 +02:00
checkAccessTokenExpired ( ) ;
2017-12-27 15:12:38 +01:00
const ( char ) [ ] url = deltaLink ;
if ( url = = null ) {
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/delta" ;
2018-12-28 02:26:03 +01:00
url ~ = "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size" ;
2017-12-27 15:12:38 +01:00
}
2015-09-01 20:45:34 +02:00
return get ( url ) ;
}
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get_content
2018-08-09 23:46:27 +02:00
void downloadById ( const ( char ) [ ] driveId , const ( char ) [ ] id , string saveToPath , long fileSize )
2015-09-01 20:45:34 +02:00
{
2015-09-01 22:23:42 +02:00
checkAccessTokenExpired ( ) ;
2015-10-04 16:24:45 +02:00
scope ( failure ) {
2015-09-01 20:45:34 +02:00
if ( exists ( saveToPath ) ) remove ( saveToPath ) ;
}
2017-12-27 15:12:38 +01:00
mkdirRecurse ( dirName ( saveToPath ) ) ;
const ( char ) [ ] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content?AVOverride=1" ;
2018-08-09 23:46:27 +02:00
download ( url , saveToPath , fileSize ) ;
2015-09-01 20:45:34 +02:00
}
2017-12-31 02:30:31 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
JSONValue simpleUpload ( string localPath , string parentDriveId , string parentId , string filename , const ( char ) [ ] eTag = null )
{
checkAccessTokenExpired ( ) ;
string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent ( filename ) ~ ":/content" ;
// TODO: investigate why this fails for remote folders
//if (eTag) http.addRequestHeader("If-Match", eTag);
/*else http.addRequestHeader("If-None-Match", "*");*/
return upload ( localPath , url ) ;
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
2017-12-31 13:47:18 +01:00
JSONValue simpleUploadReplace ( string localPath , string driveId , string id , const ( char ) [ ] eTag = null )
2017-12-31 02:30:31 +01:00
{
checkAccessTokenExpired ( ) ;
string url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content" ;
2015-10-04 16:24:45 +02:00
if ( eTag ) http . addRequestHeader ( "If-Match" , eTag ) ;
return upload ( localPath , url ) ;
2015-09-01 20:45:34 +02:00
}
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_update
JSONValue updateById ( const ( char ) [ ] driveId , const ( char ) [ ] id , JSONValue data , const ( char ) [ ] eTag = null )
2015-09-05 21:23:43 +02:00
{
checkAccessTokenExpired ( ) ;
2017-12-27 15:12:38 +01:00
const ( char ) [ ] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ;
2015-09-05 21:23:43 +02:00
if ( eTag ) http . addRequestHeader ( "If-Match" , eTag ) ;
http . addRequestHeader ( "Content-Type" , "application/json" ) ;
2015-10-04 16:24:45 +02:00
return patch ( url , data . toString ( ) ) ;
2015-09-05 21:23:43 +02:00
}
2017-12-27 15:12:38 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delete
void deleteById ( const ( char ) [ ] driveId , const ( char ) [ ] id , const ( char ) [ ] eTag = null )
2015-09-06 10:23:32 +02:00
{
checkAccessTokenExpired ( ) ;
2017-12-27 15:12:38 +01:00
const ( char ) [ ] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ;
2016-12-28 23:47:29 +01:00
//TODO: investigate why this always fail with 412 (Precondition Failed)
//if (eTag) http.addRequestHeader("If-Match", eTag);
2015-10-04 16:24:45 +02:00
del ( url ) ;
2015-09-06 10:23:32 +02:00
}
2017-12-31 13:18:11 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_post_children
JSONValue createById ( const ( char ) [ ] parentDriveId , const ( char ) [ ] parentId , JSONValue item )
2015-09-06 11:06:13 +02:00
{
2016-12-28 16:29:55 +01:00
checkAccessTokenExpired ( ) ;
2017-12-31 13:18:11 +01:00
const ( char ) [ ] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ "/children" ;
2018-03-14 05:43:40 +01:00
http . addRequestHeader ( "Content-Type" , "application/json" ) ;
2015-10-04 16:24:45 +02:00
return post ( url , item . toString ( ) ) ;
2015-09-06 11:06:13 +02:00
}
2018-03-14 05:43:40 +01:00
// Return the details of the specified path
JSONValue getPathDetails ( const ( string ) path )
{
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
// string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
2018-12-28 02:26:03 +01:00
if ( ( path = = "." ) | | ( path = = "/" ) ) url = driveUrl ~ "/root/" ;
2018-03-14 05:43:40 +01:00
else url = itemByPathUrl ~ encodeComponent ( path ) ~ ":/" ;
2019-04-01 20:21:02 +02:00
url ~ = "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size" ;
2018-03-14 05:43:40 +01:00
return get ( url ) ;
}
2018-04-23 02:58:47 +02:00
// Return the details of the specified id
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
2018-06-17 04:02:58 +02:00
JSONValue getPathDetailsById ( const ( char ) [ ] driveId , const ( char ) [ ] id )
2018-04-23 02:58:47 +02:00
{
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
2018-06-17 04:02:58 +02:00
// string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ;
2019-04-01 20:21:02 +02:00
url ~ = "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size" ;
2018-04-23 02:58:47 +02:00
return get ( url ) ;
}
2018-08-09 23:46:27 +02:00
// Return the requested details of the specified id
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
2018-09-13 00:41:46 +02:00
JSONValue getFileDetails ( const ( char ) [ ] driveId , const ( char ) [ ] id )
2018-08-09 23:46:27 +02:00
{
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
// string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ;
2018-09-13 00:41:46 +02:00
url ~ = "?select=size,malware" ;
2018-08-09 23:46:27 +02:00
return get ( url ) ;
}
2018-04-23 02:58:47 +02:00
2018-03-14 05:43:40 +01:00
// https://dev.onedrive.com/items/move.htm
JSONValue moveByPath ( const ( char ) [ ] sourcePath , JSONValue moveData )
{
// Need to use itemByPathUrl
checkAccessTokenExpired ( ) ;
string url = itemByPathUrl ~ encodeComponent ( sourcePath ) ;
http . addRequestHeader ( "Content-Type" , "application/json" ) ;
return move ( url , moveData . toString ( ) ) ;
}
2017-12-31 16:11:02 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
2018-07-02 23:24:57 +02:00
JSONValue createUploadSession ( const ( char ) [ ] parentDriveId , const ( char ) [ ] parentId , const ( char ) [ ] filename , const ( char ) [ ] eTag = null , JSONValue item = null )
2015-09-27 18:47:41 +02:00
{
checkAccessTokenExpired ( ) ;
2017-12-31 16:11:02 +01:00
const ( char ) [ ] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent ( filename ) ~ ":/createUploadSession" ;
2015-09-27 18:47:41 +02:00
if ( eTag ) http . addRequestHeader ( "If-Match" , eTag ) ;
2018-07-02 23:24:57 +02:00
http . addRequestHeader ( "Content-Type" , "application/json" ) ;
return post ( url , item . toString ( ) ) ;
2015-09-27 18:47:41 +02:00
}
// https://dev.onedrive.com/items/upload_large_files.htm
JSONValue uploadFragment ( const ( char ) [ ] uploadUrl , string filepath , long offset , long offsetSize , long fileSize )
{
checkAccessTokenExpired ( ) ;
2015-10-04 16:24:45 +02:00
scope ( exit ) {
http . clearRequestHeaders ( ) ;
http . onSend = null ;
}
2015-09-27 18:47:41 +02:00
http . method = HTTP . Method . put ;
http . url = uploadUrl ;
2018-11-14 20:08:55 +01:00
2015-09-27 18:47:41 +02:00
import std.conv ;
string contentRange = "bytes " ~ to ! string ( offset ) ~ "-" ~ to ! string ( offset + offsetSize - 1 ) ~ "/" ~ to ! string ( fileSize ) ;
http . addRequestHeader ( "Content-Range" , contentRange ) ;
2015-10-04 16:24:45 +02:00
auto file = File ( filepath , "rb" ) ;
file . seek ( offset ) ;
http . onSend = data = > file . rawRead ( data ) . length ;
http . contentLength = offsetSize ;
auto response = perform ( ) ;
// TODO: retry on 5xx errors
checkHttpCode ( ) ;
return response ;
2015-09-27 18:47:41 +02:00
}
// https://dev.onedrive.com/items/upload_large_files.htm
JSONValue requestUploadStatus ( const ( char ) [ ] uploadUrl )
{
2015-10-04 16:24:45 +02:00
checkAccessTokenExpired ( ) ;
2017-03-11 11:40:19 +01:00
// when using microsoft graph the auth code is different
return get ( uploadUrl , true ) ;
2015-09-27 18:47:41 +02:00
}
2018-12-04 00:59:23 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/site_search?view=odsp-graph-online
JSONValue o365SiteSearch ( string o365SharedLibraryName ) {
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
url = siteSearchUrl ~ "=" ~ o365SharedLibraryName ;
return get ( url ) ;
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_list?view=odsp-graph-online
JSONValue o365SiteDrives ( string site_id ) {
checkAccessTokenExpired ( ) ;
const ( char ) [ ] url ;
url = siteDriveUrl ~ site_id ~ "/drives" ;
return get ( url ) ;
}
2015-09-01 20:45:34 +02:00
private void redeemToken ( const ( char ) [ ] authCode )
{
2016-06-27 19:23:14 +02:00
const ( char ) [ ] postData =
"client_id=" ~ clientId ~
"&redirect_uri=" ~ redirectUrl ~
"&code=" ~ authCode ~
"&grant_type=authorization_code" ;
2015-09-01 20:45:34 +02:00
acquireToken ( postData ) ;
}
private void newToken ( )
{
2016-06-27 19:23:14 +02:00
string postData =
"client_id=" ~ clientId ~
"&redirect_uri=" ~ redirectUrl ~
"&refresh_token=" ~ refreshToken ~
"&grant_type=refresh_token" ;
2015-09-01 20:45:34 +02:00
acquireToken ( postData ) ;
}
private void acquireToken ( const ( char ) [ ] postData )
{
JSONValue response = post ( tokenUrl , postData ) ;
2019-03-12 01:50:21 +01:00
if ( "access_token" in response ) {
accessToken = "bearer " ~ response [ "access_token" ] . str ( ) ;
refreshToken = response [ "refresh_token" ] . str ( ) ;
accessTokenExpiration = Clock . currTime ( ) + dur ! "seconds" ( response [ "expires_in" ] . integer ( ) ) ;
if ( ! . dryRun ) {
std . file . write ( cfg . refreshTokenFilePath , refreshToken ) ;
}
if ( printAccessToken ) writeln ( "New access token: " , accessToken ) ;
} else {
log . error ( "\nInvalid authentication response from OneDrive. Please check the response uri\n" ) ;
// re-authorize
authorize ( ) ;
2019-03-11 07:57:47 +01:00
}
2015-09-01 20:45:34 +02:00
}
2015-09-01 22:23:42 +02:00
private void checkAccessTokenExpired ( )
{
2017-06-15 13:02:04 +02:00
try {
if ( Clock . currTime ( ) > = accessTokenExpiration ) {
newToken ( ) ;
}
} catch ( OneDriveException e ) {
if ( e . httpStatusCode = = 400 | | e . httpStatusCode = = 401 ) {
e . msg ~ = "\nRefresh token invalid, use --logout to authorize the client again" ;
}
2015-09-01 22:23:42 +02:00
}
}
2015-09-05 21:23:43 +02:00
2015-10-04 16:24:45 +02:00
private void addAccessTokenHeader ( )
2015-09-01 20:45:34 +02:00
{
2015-10-04 16:24:45 +02:00
http . addRequestHeader ( "Authorization" , accessToken ) ;
}
2017-03-11 11:40:19 +01:00
private JSONValue get ( const ( char ) [ ] url , bool skipToken = false )
2015-10-04 16:24:45 +02:00
{
scope ( exit ) http . clearRequestHeaders ( ) ;
http . method = HTTP . Method . get ;
http . url = url ;
2017-03-11 11:40:19 +01:00
if ( ! skipToken ) addAccessTokenHeader ( ) ; // HACK: requestUploadStatus
2015-10-04 16:24:45 +02:00
auto response = perform ( ) ;
2016-12-13 18:04:16 +01:00
checkHttpCode ( response ) ;
2018-11-23 21:13:16 +01:00
// OneDrive API Response Debugging
2018-04-15 12:02:39 +02:00
if ( . debugResponse ) {
2018-11-17 21:05:01 +01:00
writeln ( "OneDrive API Response: " , response ) ;
2018-04-15 12:02:39 +02:00
}
2015-10-04 16:24:45 +02:00
return response ;
}
private void del ( const ( char ) [ ] url )
{
scope ( exit ) http . clearRequestHeaders ( ) ;
http . method = HTTP . Method . del ;
http . url = url ;
addAccessTokenHeader ( ) ;
2016-12-13 18:04:16 +01:00
auto response = perform ( ) ;
checkHttpCode ( response ) ;
2015-10-04 16:24:45 +02:00
}
2018-08-09 23:46:27 +02:00
private void download ( const ( char ) [ ] url , string filename , long fileSize )
2015-10-04 16:24:45 +02:00
{
2018-08-09 23:46:27 +02:00
// Threshold for displaying download bar
long thresholdFileSize = 4 * 2 ^ ^ 20 ; // 4 MiB
2015-10-04 16:24:45 +02:00
scope ( exit ) http . clearRequestHeaders ( ) ;
http . method = HTTP . Method . get ;
http . url = url ;
addAccessTokenHeader ( ) ;
auto f = File ( filename , "wb" ) ;
http . onReceive = ( ubyte [ ] data ) {
f . rawWrite ( data ) ;
return data . length ;
} ;
2018-08-09 23:46:27 +02:00
if ( fileSize > = thresholdFileSize ) {
// Download Progress Bar
size_t iteration = 20 ;
Progress p = new Progress ( iteration ) ;
p . title = "Downloading" ;
writeln ( ) ;
real previousDLPercent = - 1.0 ;
real percentCheck = 5.0 ;
// Setup progress bar to display
http . onProgress = delegate int ( size_t dltotal , size_t dlnow , size_t ultotal , size_t ulnow )
{
// For each onProgress, what is the % of dlnow to dltotal
real currentDLPercent = round ( double ( dlnow ) / dltotal * 100 ) ;
// If matching 5% of download, increment progress bar
if ( ( isIdentical ( fmod ( currentDLPercent , percentCheck ) , 0.0 ) ) & & ( previousDLPercent ! = currentDLPercent ) ) {
p . next ( ) ;
previousDLPercent = currentDLPercent ;
}
return 0 ;
} ;
// Perform download & display progress bar
http . perform ( ) ;
writeln ( ) ;
// Reset onProgress to not display anything for next download
http . onProgress = delegate int ( size_t dltotal , size_t dlnow , size_t ultotal , size_t ulnow )
{
return 0 ;
} ;
} else {
// No progress bar
http . perform ( ) ;
}
// Check the HTTP response code
2015-10-04 16:24:45 +02:00
checkHttpCode ( ) ;
2015-09-01 20:45:34 +02:00
}
2015-09-05 21:23:43 +02:00
private auto patch ( T ) ( const ( char ) [ ] url , const ( T ) [ ] patchData )
{
2015-10-04 16:24:45 +02:00
scope ( exit ) http . clearRequestHeaders ( ) ;
http . method = HTTP . Method . patch ;
http . url = url ;
addAccessTokenHeader ( ) ;
auto response = perform ( patchData ) ;
2016-12-13 18:04:16 +01:00
checkHttpCode ( response ) ;
2015-10-04 16:24:45 +02:00
return response ;
2015-09-05 21:23:43 +02:00
}
2015-09-01 20:45:34 +02:00
private auto post ( T ) ( const ( char ) [ ] url , const ( T ) [ ] postData )
{
2015-10-04 16:24:45 +02:00
scope ( exit ) http . clearRequestHeaders ( ) ;
http . method = HTTP . Method . post ;
http . url = url ;
addAccessTokenHeader ( ) ;
auto response = perform ( postData ) ;
2016-12-13 18:04:16 +01:00
checkHttpCode ( response ) ;
2015-10-04 16:24:45 +02:00
return response ;
}
2018-03-14 05:43:40 +01:00
private auto move ( T ) ( const ( char ) [ ] url , const ( T ) [ ] postData )
{
scope ( exit ) http . clearRequestHeaders ( ) ;
http . method = HTTP . Method . patch ;
http . url = url ;
addAccessTokenHeader ( ) ;
auto response = perform ( postData ) ;
checkHttpCode ( ) ;
return response ;
}
2015-10-04 16:24:45 +02:00
private JSONValue upload ( string filepath , string url )
{
scope ( exit ) {
http . clearRequestHeaders ( ) ;
http . onSend = null ;
http . contentLength = 0 ;
}
http . method = HTTP . Method . put ;
http . url = url ;
addAccessTokenHeader ( ) ;
http . addRequestHeader ( "Content-Type" , "application/octet-stream" ) ;
auto file = File ( filepath , "rb" ) ;
http . onSend = data = > file . rawRead ( data ) . length ;
http . contentLength = file . size ;
auto response = perform ( ) ;
2016-12-13 18:04:16 +01:00
checkHttpCode ( response ) ;
2015-10-04 16:24:45 +02:00
return response ;
}
private JSONValue perform ( const ( void ) [ ] sendData )
{
scope ( exit ) {
http . onSend = null ;
http . contentLength = 0 ;
}
if ( sendData ) {
http . contentLength = sendData . length ;
http . onSend = ( void [ ] buf ) {
import std.algorithm : min ;
size_t minLen = min ( buf . length , sendData . length ) ;
if ( minLen = = 0 ) return 0 ;
buf [ 0 . . minLen ] = sendData [ 0 . . minLen ] ;
sendData = sendData [ minLen . . $ ] ;
return minLen ;
} ;
} else {
http . onSend = buf = > 0 ;
}
return perform ( ) ;
}
private JSONValue perform ( )
{
scope ( exit ) http . onReceive = null ;
2015-10-04 17:33:48 +02:00
char [ ] content ;
2015-10-04 16:24:45 +02:00
http . onReceive = ( ubyte [ ] data ) {
content ~ = data ;
2018-11-14 20:08:55 +01:00
// HTTP Server Response Code Debugging
if ( . debugResponse ) {
2018-11-17 21:05:01 +01:00
writeln ( "OneDrive HTTP Server Response: " , http . statusLine . code ) ;
2018-11-14 20:08:55 +01:00
}
2015-10-04 16:24:45 +02:00
return data . length ;
} ;
2018-08-02 00:25:37 +02:00
try {
http . perform ( ) ;
} catch ( CurlException e ) {
// Potentially Timeout was reached on handle error
2018-12-05 20:19:00 +01:00
// we issue warning/error in the catch routines so no need to warn here
// log.error("\nAccess to the Microsoft OneDrive service timed out - Internet connectivity issue?\n");
2018-12-03 03:41:31 +01:00
throw e ;
2018-08-02 00:25:37 +02:00
}
2017-03-11 11:40:19 +01:00
JSONValue json ;
try {
json = content . parseJSON ( ) ;
} catch ( JSONException e ) {
e . msg ~ = "\n" ;
e . msg ~ = content ;
throw e ;
}
return json ;
2015-09-01 20:45:34 +02:00
}
2015-09-06 10:07:18 +02:00
private void checkHttpCode ( )
2015-09-01 20:45:34 +02:00
{
2018-03-14 05:43:40 +01:00
// https://dev.onedrive.com/misc/errors.htm
// https://developer.overdrive.com/docs/reference-guide
/ *
2018-11-14 20:08:55 +01:00
HTTP / 1.1 Response handling
2018-03-14 05:43:40 +01:00
Errors in the OneDrive API are returned using standard HTTP status codes , as well as a JSON error response object . The following HTTP status codes should be expected .
Status code Status message Description
200 OK Request was handled OK
201 Created This means you ' ve made a successful POST to checkout , lock in a format , or place a hold
204 No Content This means you ' ve made a successful DELETE to remove a hold or return a title
400 Bad Request Cannot process the request because it is malformed or incorrect .
401 Unauthorized Required authentication information is either missing or not valid for the resource .
403 Forbidden Access is denied to the requested resource . The user might not have enough permission .
404 Not Found The requested resource doesn ’ t exist .
405 Method Not Allowed The HTTP method in the request is not allowed on the resource .
406 Not Acceptable This service doesn ’ t support the format requested in the Accept header .
409 Conflict The current state conflicts with what the request expects . For example , the specified parent folder might not exist .
410 Gone The requested resource is no longer available at the server .
411 Length Required A Content - Length header is required on the request .
412 Precondition Failed A precondition provided in the request ( such as an if - match header ) does not match the resource ' s current state .
413 Request Entity Too Large The request size exceeds the maximum limit .
415 Unsupported Media Type The content type of the request is a format that is not supported by the service .
416 Requested Range Not Satisfiable The specified byte range is invalid or unavailable .
422 Unprocessable Entity Cannot process the request because it is semantically incorrect .
429 Too Many Requests Client application has been throttled and should not attempt to repeat the request until an amount of time has elapsed .
500 Internal Server Error There was an internal server error while processing the request .
501 Not Implemented The requested feature isn ’ t implemented .
502 Bad Gateway The service was unreachable
503 Service Unavailable The service is temporarily unavailable . You may repeat the request after a delay . There may be a Retry - After header .
507 Insufficient Storage The maximum storage quota has been reached .
509 Bandwidth Limit Exceeded Your app has been throttled for exceeding the maximum bandwidth cap . Your app can retry the request again after more time has elapsed .
2018-11-14 20:08:55 +01:00
HTTP / 2 Response handling
0 OK
2018-03-14 05:43:40 +01:00
* /
switch ( http . statusLine . code )
{
2018-11-14 20:08:55 +01:00
case 0 :
break ;
2018-05-03 08:21:53 +02:00
// 200 - OK
case 200 :
// No Log ..
break ;
// 201 - Created OK
// 202 - Accepted
// 204 - Deleted OK
case 201 , 202 , 204 :
// No actions, but log if verbose logging
2018-05-08 22:47:03 +02:00
//log.vlog("OneDrive Response: '", http.statusLine.code, " - ", http.statusLine.reason, "'");
2018-05-03 08:21:53 +02:00
break ;
2018-11-11 20:01:57 +01:00
// 302 - resource found and available at another location, redirect
case 302 :
break ;
2018-05-03 08:21:53 +02:00
// 400 - Bad Request
case 400 :
// Bad Request .. how should we act?
log . vlog ( "OneDrive returned a 'HTTP 400 - Bad Request' - gracefully handling error" ) ;
break ;
// Item not found
case 404 :
// Item was not found - do not throw an exception
log . vlog ( "OneDrive returned a 'HTTP 404 - Item not found' - gracefully handling error" ) ;
break ;
// 409 - Conflict
case 409 :
// Conflict handling .. how should we act? This only really gets triggered if we are using --local-first & we remove items.db as the DB thinks the file is not uploaded but it is
log . vlog ( "OneDrive returned a 'HTTP 409 - Conflict' - gracefully handling error" ) ;
break ;
// 412 - Precondition Failed
case 412 :
// A precondition provided in the request (such as an if-match header) does not match the resource's current state.
log . vlog ( "OneDrive returned a 'HTTP 412 - Precondition Failed' - gracefully handling error" ) ;
break ;
// 415 - Unsupported Media Type
case 415 :
// Unsupported Media Type ... sometimes triggered on image files, especially PNG
log . vlog ( "OneDrive returned a 'HTTP 415 - Unsupported Media Type' - gracefully handling error" ) ;
break ;
2018-08-27 02:45:26 +02:00
// 429 - Too Many Requests
case 429 :
// Too many requests in a certain time window
// https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
log . vlog ( "OneDrive returned a 'HTTP 429 - Too Many Requests' - gracefully handling error" ) ;
break ;
2018-05-03 08:21:53 +02:00
// Server side (OneDrive) Errors
// 500 - Internal Server Error
// 502 - Bad Gateway
// 503 - Service Unavailable
// 504 - Gateway Timeout (Issue #320)
2019-04-04 08:13:15 +02:00
case 500 :
2018-05-03 08:21:53 +02:00
// No actions
2019-04-04 08:13:15 +02:00
log . vlog ( "OneDrive returned a 'HTTP 500 Internal Server Error' - gracefully handling error" ) ;
break ;
case 502 :
// No actions
log . vlog ( "OneDrive returned a 'HTTP 502 Bad Gateway Error' - gracefully handling error" ) ;
break ;
case 503 :
// No actions
log . vlog ( "OneDrive returned a 'HTTP 503 Service Unavailable Error' - gracefully handling error" ) ;
break ;
case 504 :
// No actions
log . vlog ( "OneDrive returned a 'HTTP 504 Gateway Timeout Error' - gracefully handling error" ) ;
break ;
2018-05-03 08:21:53 +02:00
// "else"
default :
throw new OneDriveException ( http . statusLine . code , http . statusLine . reason ) ;
2015-09-01 20:45:34 +02:00
}
}
2016-12-13 18:04:16 +01:00
private void checkHttpCode ( ref const JSONValue response )
{
2018-08-27 02:35:15 +02:00
switch ( http . statusLine . code )
{
2019-03-12 01:50:21 +01:00
// 400 - Bad Request
case 400 :
// Bad Request .. how should we act?
log . vlog ( "OneDrive returned a 'HTTP 400 - Bad Request' - gracefully handling error" ) ;
2019-04-01 19:51:25 +02:00
break ;
2018-08-27 02:35:15 +02:00
// 412 - Precondition Failed
case 412 :
2018-12-03 20:29:18 +01:00
log . vlog ( "OneDrive returned a 'HTTP 412 - Precondition Failed' - gracefully handling error" ) ;
2018-08-27 02:35:15 +02:00
break ;
// Server side (OneDrive) Errors
// 500 - Internal Server Error
// 502 - Bad Gateway
// 503 - Service Unavailable
// 504 - Gateway Timeout (Issue #320)
2019-04-04 08:13:15 +02:00
case 500 :
// No actions
log . vlog ( "OneDrive returned a 'HTTP 500 Internal Server Error' - gracefully handling error" ) ;
break ;
case 502 :
// No actions
log . vlog ( "OneDrive returned a 'HTTP 502 Bad Gateway Error' - gracefully handling error" ) ;
break ;
case 503 :
// No actions
log . vlog ( "OneDrive returned a 'HTTP 503 Service Unavailable Error' - gracefully handling error" ) ;
break ;
case 504 :
2018-08-27 02:35:15 +02:00
// No actions
2019-04-04 08:13:15 +02:00
log . vlog ( "OneDrive returned a 'HTTP 504 Gateway Timeout Error' - gracefully handling error" ) ;
2018-08-27 02:35:15 +02:00
break ;
2018-11-11 20:01:57 +01:00
// Default - all other errors that are not a 2xx or a 302
2018-08-27 02:35:15 +02:00
default :
2018-11-11 20:01:57 +01:00
if ( http . statusLine . code / 100 ! = 2 & & http . statusLine . code ! = 302 ) {
2018-07-15 22:36:31 +02:00
throw new OneDriveException ( http . statusLine . code , http . statusLine . reason , response ) ;
}
2016-12-13 18:04:16 +01:00
}
}
2015-09-01 20:45:34 +02:00
}
2017-03-11 11:40:19 +01:00
unittest
{
string configDirName = expandTilde ( "~/.config/onedrive" ) ;
auto cfg = new config . Config ( configDirName ) ;
cfg . init ( ) ;
OneDriveApi onedrive = new OneDriveApi ( cfg ) ;
onedrive . init ( ) ;
std . file . write ( "/tmp/test" , "test" ) ;
// simpleUpload
auto item = onedrive . simpleUpload ( "/tmp/test" , "/test" ) ;
try {
item = onedrive . simpleUpload ( "/tmp/test" , "/test" ) ;
} catch ( OneDriveException e ) {
assert ( e . httpStatusCode = = 409 ) ;
}
try {
item = onedrive . simpleUpload ( "/tmp/test" , "/test" , "123" ) ;
} catch ( OneDriveException e ) {
assert ( e . httpStatusCode = = 412 ) ;
}
item = onedrive . simpleUpload ( "/tmp/test" , "/test" , item [ "eTag" ] . str ) ;
// deleteById
try {
onedrive . deleteById ( item [ "id" ] . str , "123" ) ;
} catch ( OneDriveException e ) {
assert ( e . httpStatusCode = = 412 ) ;
}
onedrive . deleteById ( item [ "id" ] . str , item [ "eTag" ] . str ) ;
onedrive . http . shutdown ( ) ;
2015-09-01 20:45:34 +02:00
}