2017-12-31 13:18:11 +01:00
import std.algorithm : find ;
2017-12-29 16:51:08 +01:00
import std.array : array ;
2017-12-31 13:18:11 +01:00
import std.datetime ;
import std.exception : enforce ;
import std.file , std . json , std . path ;
2017-03-24 22:30:03 +01:00
import std.regex ;
2018-03-14 05:43:40 +01:00
import std.stdio , std . string , std . uni , std . uri ;
2017-03-24 22:30:03 +01:00
import config , itemdb , onedrive , selective , upload , util ;
2016-08-04 23:35:58 +02:00
static import log ;
2015-09-27 18:47:41 +02:00
// threshold after which files will be uploaded using an upload session
2017-03-11 11:40:19 +01:00
private long thresholdFileSize = 4 * 2 ^ ^ 20 ; // 4 MiB
2015-09-01 20:45:34 +02:00
private bool isItemFolder ( const ref JSONValue item )
{
2017-03-11 11:40:19 +01:00
return ( "folder" in item ) ! = null ;
2015-09-01 20:45:34 +02:00
}
private bool isItemFile ( const ref JSONValue item )
{
2017-03-11 11:40:19 +01:00
return ( "file" in item ) ! = null ;
2015-09-01 20:45:34 +02:00
}
private bool isItemDeleted ( const ref JSONValue item )
{
2017-12-28 19:58:31 +01:00
return ( "deleted" in item ) ! = null ;
2015-09-01 20:45:34 +02:00
}
2017-03-12 19:40:38 +01:00
private bool isItemRoot ( const ref JSONValue item )
{
2017-12-28 19:58:31 +01:00
return ( "root" in item ) ! = null ;
2015-09-01 20:45:34 +02:00
}
2017-06-14 22:39:36 +02:00
private bool isItemRemote ( const ref JSONValue item )
2015-09-08 18:25:41 +02:00
{
2017-06-14 22:39:36 +02:00
return ( "remoteItem" in item ) ! = null ;
}
2017-12-31 02:30:31 +01:00
// construct an Item struct from a JSON driveItem
private Item makeItem ( const ref JSONValue driveItem )
2015-09-08 18:25:41 +02:00
{
2017-05-28 22:13:19 +02:00
Item item = {
2017-12-31 02:30:31 +01:00
id : driveItem [ "id" ] . str ,
name : "name" in driveItem ? driveItem [ "name" ] . str : null , // name may be missing for deleted files in OneDrive Biz
eTag : "eTag" in driveItem ? driveItem [ "eTag" ] . str : null , // eTag is not returned for the root in OneDrive Biz
cTag : "cTag" in driveItem ? driveItem [ "cTag" ] . str : null , // cTag is missing in old files (and all folders in OneDrive Biz)
2018-04-17 22:28:08 +02:00
// OneDrive API Change: https://github.com/OneDrive/onedrive-api-docs/issues/834
// Fixes issue 'Key not found: lastModifiedDateTime' (#334, #337)
2018-04-20 23:18:46 +02:00
mtime : ( "fileSystemInfo" in driveItem & & "lastModifiedDateTime" in driveItem [ "fileSystemInfo" ] ) ? SysTime . fromISOExtString ( driveItem [ "fileSystemInfo" ] [ "lastModifiedDateTime" ] . str ) : SysTime ( 0 ) ,
2017-05-28 22:13:19 +02:00
} ;
2017-12-31 02:30:31 +01:00
if ( isItemFile ( driveItem ) ) {
item . type = ItemType . file ;
} else if ( isItemFolder ( driveItem ) ) {
item . type = ItemType . dir ;
} else if ( isItemRemote ( driveItem ) ) {
item . type = ItemType . remote ;
} else {
2018-01-01 16:20:28 +01:00
// do not throw exception, item will be removed in applyDifferences()
2017-12-31 02:30:31 +01:00
}
2017-12-28 19:58:31 +01:00
// root and remote items do not have parentReference
2017-12-31 02:30:31 +01:00
if ( ! isItemRoot ( driveItem ) & & ( "parentReference" in driveItem ) ! = null ) {
item . driveId = driveItem [ "parentReference" ] [ "driveId" ] . str ,
item . parentId = driveItem [ "parentReference" ] [ "id" ] . str ;
2017-12-28 19:58:31 +01:00
}
2017-05-28 22:13:19 +02:00
// extract the file hash
2017-12-31 02:30:31 +01:00
if ( isItemFile ( driveItem ) & & ( "hashes" in driveItem [ "file" ] ) ) {
if ( "crc32Hash" in driveItem [ "file" ] [ "hashes" ] ) {
item . crc32Hash = driveItem [ "file" ] [ "hashes" ] [ "crc32Hash" ] . str ;
} else if ( "sha1Hash" in driveItem [ "file" ] [ "hashes" ] ) {
item . sha1Hash = driveItem [ "file" ] [ "hashes" ] [ "sha1Hash" ] . str ;
} else if ( "quickXorHash" in driveItem [ "file" ] [ "hashes" ] ) {
item . quickXorHash = driveItem [ "file" ] [ "hashes" ] [ "quickXorHash" ] . str ;
} else {
log . vlog ( "The file does not have any hash" ) ;
2017-05-28 22:13:19 +02:00
}
}
2017-12-31 02:30:31 +01:00
if ( isItemRemote ( driveItem ) ) {
item . remoteDriveId = driveItem [ "remoteItem" ] [ "parentReference" ] [ "driveId" ] . str ;
item . remoteId = driveItem [ "remoteItem" ] [ "id" ] . str ;
2017-12-27 15:13:28 +01:00
}
2017-05-28 22:13:19 +02:00
return item ;
}
private bool testFileHash ( string path , const ref Item item )
{
if ( item . crc32Hash ) {
if ( item . crc32Hash = = computeCrc32 ( path ) ) return true ;
} else if ( item . sha1Hash ) {
if ( item . sha1Hash = = computeSha1Hash ( path ) ) return true ;
} else if ( item . quickXorHash ) {
if ( item . quickXorHash = = computeQuickXorHash ( path ) ) return true ;
2015-09-08 18:25:41 +02:00
}
return false ;
}
2015-09-01 20:45:34 +02:00
class SyncException : Exception
{
2017-12-28 15:03:15 +01:00
@nogc @safe pure nothrow this ( string msg , string file = __FILE__ , size_t line = __LINE__ )
2015-09-01 20:45:34 +02:00
{
2017-12-28 15:03:15 +01:00
super ( msg , file , line ) ;
2015-09-01 20:45:34 +02:00
}
}
final class SyncEngine
{
2015-09-14 23:56:14 +02:00
private Config cfg ;
private OneDriveApi onedrive ;
private ItemDatabase itemdb ;
2015-09-27 18:47:41 +02:00
private UploadSession session ;
2017-03-24 22:30:03 +01:00
private SelectiveSync selectiveSync ;
2015-12-29 19:38:15 +01:00
// list of items to skip while applying the changes
2015-09-16 10:29:20 +02:00
private string [ ] skippedItems ;
2015-09-18 21:42:27 +02:00
// list of items to delete after the changes has been downloaded
2017-06-14 15:50:02 +02:00
private string [ 2 ] [ ] idsToDelete ;
2018-01-02 15:05:32 +01:00
// default drive id
private string defaultDriveId ;
2018-04-17 22:28:08 +02:00
// type of OneDrive account
private string accountType ;
// default root id
private string defaultRootId ;
2015-09-14 23:56:14 +02:00
2017-03-24 22:30:03 +01:00
this ( Config cfg , OneDriveApi onedrive , ItemDatabase itemdb , SelectiveSync selectiveSync )
2015-09-01 20:45:34 +02:00
{
2017-03-24 22:30:03 +01:00
assert ( onedrive & & itemdb & & selectiveSync ) ;
2015-09-01 20:45:34 +02:00
this . cfg = cfg ;
this . onedrive = onedrive ;
2015-09-14 23:56:14 +02:00
this . itemdb = itemdb ;
2017-03-24 22:30:03 +01:00
this . selectiveSync = selectiveSync ;
2016-08-04 23:35:58 +02:00
session = UploadSession ( onedrive , cfg . uploadStateFilePath ) ;
2015-09-01 20:45:34 +02:00
}
2016-08-04 23:35:58 +02:00
void init ( )
2015-09-01 20:45:34 +02:00
{
2018-04-17 22:28:08 +02:00
// Set accountType, defaultDriveId & defaultRootId once and reuse where possible
auto oneDriveDetails = onedrive . getDefaultDrive ( ) ;
accountType = oneDriveDetails [ "driveType" ] . str ;
defaultDriveId = oneDriveDetails [ "id" ] . str ;
defaultRootId = onedrive . getDefaultRoot [ "id" ] . str ;
// display accountType, defaultDriveId & defaultRootId
log . vlog ( "Account Type: " , accountType ) ;
log . vlog ( "Default Drive ID: " , defaultDriveId ) ;
log . vlog ( "Default Root ID: " , defaultRootId ) ;
2015-09-27 18:47:41 +02:00
// check if there is an interrupted upload session
if ( session . restore ( ) ) {
2016-08-04 23:35:58 +02:00
log . log ( "Continuing the upload session ..." ) ;
2015-09-27 18:47:41 +02:00
auto item = session . upload ( ) ;
saveItem ( item ) ;
}
2015-09-14 23:56:14 +02:00
}
2015-09-01 20:45:34 +02:00
2017-12-31 02:30:31 +01:00
// download all new changes from OneDrive
2015-09-14 23:56:14 +02:00
void applyDifferences ( )
{
2018-03-14 05:43:40 +01:00
// Set defaults for the root folder
2018-04-17 22:28:08 +02:00
string driveId = defaultDriveId ;
string rootId = defaultRootId ;
2017-12-27 15:13:28 +01:00
applyDifferences ( driveId , rootId ) ;
2017-12-31 02:30:31 +01:00
// check all remote folders
// https://github.com/OneDrive/onedrive-api-docs/issues/764
Item [ ] items = itemdb . selectRemoteItems ( ) ;
foreach ( item ; items ) applyDifferences ( item . remoteDriveId , item . remoteId ) ;
2015-09-01 20:45:34 +02:00
}
2018-03-14 05:43:40 +01:00
// download all new changes from a specified folder on OneDrive
void applyDifferencesSingleDirectory ( string path )
{
// test if the path we are going to sync from actually exists on OneDrive
2018-04-17 22:28:08 +02:00
JSONValue onedrivePathDetails ;
2018-03-14 05:43:40 +01:00
try {
2018-04-17 22:28:08 +02:00
onedrivePathDetails = onedrive . getPathDetails ( path ) ;
2018-03-14 05:43:40 +01:00
} catch ( OneDriveException e ) {
if ( e . httpStatusCode = = 404 ) {
// The directory was not found
log . vlog ( "ERROR: The requested single directory to sync was not found on OneDrive" ) ;
2018-04-19 06:31:35 +02:00
// We are expecting this directory however as this is a single directory sync
uploadCreateDir ( path ) ;
2018-03-14 05:43:40 +01:00
}
}
// OK - it should exist, get the driveId and rootId for this folder
log . vlog ( "Checking for differences from OneDrive ..." ) ;
2018-04-19 06:31:35 +02:00
// If the OneDrive Root is not in the local database, creating a remote folder in the selected path will fail
2018-03-14 05:43:40 +01:00
checkDatabaseForOneDriveRoot ( ) ;
2018-04-17 22:28:08 +02:00
// Configure driveID and folderId
2018-03-14 05:43:40 +01:00
string driveId = onedrivePathDetails [ "parentReference" ] [ "driveId" ] . str ; // Should give something like 12345abcde1234a1
string folderId = onedrivePathDetails [ "id" ] . str ; // Should give something like 12345ABCDE1234A1!101
// Apply any differences found on OneDrive for this path (download data)
applyDifferences ( driveId , folderId ) ;
}
// make sure the OneDrive root is in our database
auto checkDatabaseForOneDriveRoot ( )
{
log . vlog ( "Fetching details for OneDrive Root" ) ;
JSONValue rootPathDetails = onedrive . getDefaultRoot ( ) ; // Returns a JSON Value
Item rootPathItem = makeItem ( rootPathDetails ) ;
// Set defaults for the root folder
2018-04-20 23:18:53 +02:00
string driveId = rootPathDetails [ "parentReference" ] [ "driveId" ] . str ; // Should give something like 12345abcde1234a1
string rootId = rootPathDetails [ "id" ] . str ; // Should give something like 12345ABCDE1234A1!101
2018-03-14 05:43:40 +01:00
// Query the database
if ( ! itemdb . selectById ( driveId , rootId , rootPathItem ) ) {
log . vlog ( "OneDrive Root does not exist in the database. We need to add it." ) ;
applyDifference ( rootPathDetails , driveId , true ) ;
} else {
log . vlog ( "OneDrive Root exists in the database" ) ;
}
}
// create a directory on OneDrive without syncing
auto createDirectoryNoSync ( string path )
{
// Attempt to create the requested path within OneDrive without performing a sync
log . vlog ( "Attempting to create the requested path within OneDrive" ) ;
// If the OneDrive Root is not in the local database, creating a remote folder will fail
checkDatabaseForOneDriveRoot ( ) ;
// Handle the remote folder creation and updating of the local database without performing a sync
uploadCreateDir ( path ) ;
}
// delete a directory on OneDrive without syncing
auto deleteDirectoryNoSync ( string path )
{
// Attempt to delete the requested path within OneDrive without performing a sync
log . vlog ( "Attempting to delete the requested path within OneDrive" ) ;
// test if the path we are going to exists on OneDrive
try {
onedrive . getPathDetails ( path ) ;
} catch ( OneDriveException e ) {
if ( e . httpStatusCode = = 404 ) {
// The directory was not found on OneDrive - no need to delete it
log . vlog ( "The requested directory to create was not found on OneDrive - skipping removing the remote directory as it doesnt exist" ) ;
return ;
}
}
Item item ;
if ( ! itemdb . selectByPath ( path , defaultDriveId , item ) ) {
// this is odd .. this directory is not in the local database - just go delete it
log . vlog ( "The requested directory to delete was not found in the local database - pushing delete request direct to OneDrive" ) ;
uploadDeleteItem ( item , path ) ;
} else {
// the folder was in the local database
// Handle the deletion and saving any update to the local database
log . vlog ( "The requested directory to delete was found in the local database. Processing the delection normally" ) ;
deleteByPath ( path ) ;
}
}
// rename a directory on OneDrive without syncing
auto renameDirectoryNoSync ( string source , string destination )
{
try {
// test if the local path exists on OneDrive
onedrive . getPathDetails ( source ) ;
} catch ( OneDriveException e ) {
if ( e . httpStatusCode = = 404 ) {
// The directory was not found
log . vlog ( "The requested directory to rename was not found on OneDrive" ) ;
return ;
}
}
// The OneDrive API returned a 200 OK status, so the folder exists
// Rename the requested directory on OneDrive without performing a sync
moveByPath ( source , destination ) ;
}
2017-12-31 12:36:14 +01:00
// download the new changes of a specific item
2018-01-06 17:50:36 +01:00
// id is the root of the drive or a shared folder
2017-12-31 12:36:14 +01:00
private void applyDifferences ( string driveId , const ( char ) [ ] id )
2017-12-31 02:30:31 +01:00
{
2017-12-27 15:13:28 +01:00
JSONValue changes ;
2017-12-31 02:30:31 +01:00
string deltaLink = itemdb . getDeltaLink ( driveId , id ) ;
2018-03-14 05:43:40 +01:00
log . vlog ( "Applying changes of Path ID: " ~ id ) ;
2018-02-18 18:02:38 +01:00
for ( ; ; ) {
2018-04-19 06:31:35 +02:00
// Will provide a JSON of children of the item id
changes = onedrive . viewChildrenById ( driveId , id ) ;
2018-04-20 23:18:53 +02:00
2018-04-20 23:19:00 +02:00
// For each 'value' item (child resource ..
2018-04-17 22:28:08 +02:00
foreach ( item ; changes [ "value" ] . array ) {
bool isRoot = ( id = = defaultRootId ) ; // fix for https://github.com/skilion/onedrive/issues/269
2018-03-14 05:43:40 +01:00
// Apply the change
2018-01-06 17:50:36 +01:00
applyDifference ( item , driveId , isRoot ) ;
2017-12-27 15:13:28 +01:00
}
2017-12-31 02:30:31 +01:00
// the response may contain either @odata.deltaLink or @odata.nextLink
2017-12-27 15:13:28 +01:00
if ( "@odata.deltaLink" in changes ) deltaLink = changes [ "@odata.deltaLink" ] . str ;
2017-12-31 02:30:31 +01:00
if ( deltaLink ) itemdb . setDeltaLink ( driveId , id , deltaLink ) ;
2017-12-27 15:13:28 +01:00
if ( "@odata.nextLink" in changes ) deltaLink = changes [ "@odata.nextLink" ] . str ;
2018-02-18 18:02:38 +01:00
else break ;
}
2017-12-31 02:30:31 +01:00
// delete items in idsToDelete
if ( idsToDelete . length > 0 ) deleteItems ( ) ;
// empty the skipped items
skippedItems . length = 0 ;
assumeSafeAppend ( skippedItems ) ;
2017-12-27 15:13:28 +01:00
}
2017-12-31 02:30:31 +01:00
// process the change of a single DriveItem
2018-01-06 17:50:36 +01:00
private void applyDifference ( JSONValue driveItem , string driveId , bool isRoot )
2015-09-01 20:45:34 +02:00
{
2018-04-17 22:28:08 +02:00
bool unwanted = false ;
2017-12-31 02:30:31 +01:00
Item item = makeItem ( driveItem ) ;
2018-04-17 22:28:08 +02:00
//if (isItemRoot(driveItem) || !item.parentId || isRoot) {
// Post 'Key not found: lastModifiedDateTime' makeItem change, if the DB has items & OneDrive returns a tombstone, !item.parentID will always return true - thus we get in a loop
// Remove !item.parentId and || qualifier - these two are better at flagging is this the OneDrive root
if ( isItemRoot ( driveItem ) & & isRoot ) {
2018-03-14 05:43:40 +01:00
log . vlog ( "Adding OneDrive Root to the local database" ) ;
2018-01-06 19:27:27 +01:00
item . parentId = null ; // ensures that it has no parent
2018-04-20 23:18:53 +02:00
item . driveId = driveId ; // HACK: makeItem() cannot set the driveId propery of the root
2017-12-28 19:58:31 +01:00
itemdb . upsert ( item ) ;
return ;
}
2018-04-17 22:28:08 +02:00
2017-06-14 22:39:36 +02:00
unwanted | = skippedItems . find ( item . parentId ) . length ! = 0 ;
unwanted | = selectiveSync . isNameExcluded ( item . name ) ;
2017-03-12 16:07:45 +01:00
2018-04-17 22:28:08 +02:00
// what if this item's state is deleted - ie - OneDrive now returns a tombstoned item with no lastModifiedDateTime entry
if ( isItemDeleted ( driveItem ) ) {
log . vlog ( "This remote item is in a deleted state" ) ;
unwanted = true ;
}
2017-12-31 02:30:31 +01:00
// check the item type
if ( ! unwanted ) {
if ( isItemFile ( driveItem ) ) {
2018-03-14 05:43:40 +01:00
//log.vlog("The item we are syncing is a file");
2017-12-31 02:30:31 +01:00
} else if ( isItemFolder ( driveItem ) ) {
2018-03-14 05:43:40 +01:00
//log.vlog("The item we are syncing is a folder");
2017-12-31 02:30:31 +01:00
} else if ( isItemRemote ( driveItem ) ) {
2018-03-14 05:43:40 +01:00
//log.vlog("The item we are syncing is a remote item");
2017-12-31 02:30:31 +01:00
assert ( isItemFolder ( driveItem [ "remoteItem" ] ) , "The remote item is not a folder" ) ;
} else {
2018-03-14 05:43:40 +01:00
log . vlog ( "This item type (" , item . name , ") is not supported" ) ;
2017-12-31 02:30:31 +01:00
unwanted = true ;
}
}
2018-04-17 22:28:08 +02:00
// check for selective path sync
2017-12-31 02:30:31 +01:00
string path ;
if ( ! unwanted ) {
2018-01-02 15:05:32 +01:00
path = itemdb . computePath ( item . driveId , item . parentId ) ~ "/" ~ item . name ;
2017-12-31 02:30:31 +01:00
path = buildNormalizedPath ( path ) ;
unwanted = selectiveSync . isPathExcluded ( path ) ;
2015-09-19 15:38:43 +02:00
}
// skip unwanted items early
2017-06-14 22:39:36 +02:00
if ( unwanted ) {
2018-04-17 22:28:08 +02:00
log . vlog ( "Skipping item: " , item . id ) ;
2017-06-14 22:39:36 +02:00
skippedItems ~ = item . id ;
2017-03-12 16:07:45 +01:00
return ;
}
2017-08-01 19:19:52 +02:00
// check if the item has been seen before
2015-09-19 15:38:43 +02:00
Item oldItem ;
2017-06-14 22:39:36 +02:00
bool cached = itemdb . selectById ( item . driveId , item . id , oldItem ) ;
2017-08-01 19:19:52 +02:00
// check if the item is going to be deleted
2017-12-31 02:30:31 +01:00
if ( isItemDeleted ( driveItem ) ) {
2017-08-01 19:19:52 +02:00
log . vlog ( "The item is marked for deletion" ) ;
if ( cached ) {
// flag to delete
idsToDelete ~ = [ item . driveId , item . id ] ;
} else {
// flag to ignore
skippedItems ~ = item . id ;
}
return ;
}
// rename the local item if it is unsynced and there is a new version of it
string oldPath ;
2017-06-14 22:39:36 +02:00
if ( cached & & item . eTag ! = oldItem . eTag ) {
oldPath = itemdb . computePath ( item . driveId , item . id ) ;
2015-09-19 15:38:43 +02:00
if ( ! isItemSynced ( oldItem , oldPath ) ) {
2016-08-04 23:35:58 +02:00
log . vlog ( "The local item is unsynced, renaming" ) ;
2015-09-19 15:38:43 +02:00
if ( exists ( oldPath ) ) safeRename ( oldPath ) ;
cached = false ;
}
}
2015-09-14 23:56:14 +02:00
2017-12-27 15:13:28 +01:00
// update the item
if ( cached ) {
2017-06-14 22:39:36 +02:00
applyChangedItem ( oldItem , oldPath , item , path ) ;
2017-12-27 15:13:28 +01:00
} else {
applyNewItem ( item , path ) ;
2015-09-17 16:28:24 +02:00
}
2015-09-01 20:45:34 +02:00
2015-09-19 15:38:43 +02:00
// save the item in the db
2017-12-27 15:13:28 +01:00
if ( cached ) {
2017-06-14 22:39:36 +02:00
itemdb . update ( item ) ;
2015-09-19 15:38:43 +02:00
} else {
2017-06-14 22:39:36 +02:00
itemdb . insert ( item ) ;
2015-09-01 20:45:34 +02:00
}
2017-12-27 15:13:28 +01:00
// sync remote folder
2017-12-31 16:11:02 +01:00
// https://github.com/OneDrive/onedrive-api-docs/issues/764
2017-12-31 02:30:31 +01:00
/ * if ( isItemRemote ( driveItem ) ) {
2017-12-27 15:13:28 +01:00
log . log ( "Syncing remote folder: " , path ) ;
applyDifferences ( item . remoteDriveId , item . remoteId ) ;
2017-12-31 02:30:31 +01:00
} * /
2015-09-01 20:45:34 +02:00
}
2017-12-31 02:30:31 +01:00
// download an item that was not synced before
2015-09-19 15:38:43 +02:00
private void applyNewItem ( Item item , string path )
2015-09-01 20:45:34 +02:00
{
2015-09-19 15:38:43 +02:00
if ( exists ( path ) ) {
if ( isItemSynced ( item , path ) ) {
2016-08-04 23:35:58 +02:00
log . vlog ( "The item is already present" ) ;
2015-09-01 20:45:34 +02:00
return ;
} else {
2017-12-31 02:30:31 +01:00
// TODO: force remote sync by deleting local item
2017-06-14 22:39:36 +02:00
log . vlog ( "The local item is out of sync, renaming..." ) ;
2015-09-19 15:38:43 +02:00
safeRename ( path ) ;
2015-09-01 20:45:34 +02:00
}
}
final switch ( item . type ) {
case ItemType . file :
2017-12-31 02:30:31 +01:00
downloadFileItem ( item , path ) ;
2015-09-01 20:45:34 +02:00
break ;
case ItemType . dir :
2017-12-27 15:13:28 +01:00
case ItemType . remote :
2017-12-31 02:30:31 +01:00
log . log ( "Creating directory " , path ) ;
2016-11-23 16:10:37 +01:00
mkdirRecurse ( path ) ;
2015-09-01 20:45:34 +02:00
break ;
}
}
2017-06-14 22:39:36 +02:00
// update a local item
// the local item is assumed to be in sync with the local db
private void applyChangedItem ( Item oldItem , string oldPath , Item newItem , string newPath )
2015-09-01 20:45:34 +02:00
{
2017-06-14 15:50:02 +02:00
assert ( oldItem . driveId = = newItem . driveId ) ;
2015-09-01 20:45:34 +02:00
assert ( oldItem . id = = newItem . id ) ;
2015-09-14 23:56:14 +02:00
assert ( oldItem . type = = newItem . type ) ;
2017-12-31 02:30:31 +01:00
assert ( oldItem . remoteDriveId = = newItem . remoteDriveId ) ;
assert ( oldItem . remoteId = = newItem . remoteId ) ;
2015-09-14 23:56:14 +02:00
if ( oldItem . eTag ! = newItem . eTag ) {
2017-12-31 02:30:31 +01:00
// handle changed name/path
2015-09-19 15:38:43 +02:00
if ( oldPath ! = newPath ) {
2017-12-31 12:36:14 +01:00
log . log ( "Moving " , oldPath , " to " , newPath ) ;
2015-09-19 15:38:43 +02:00
if ( exists ( newPath ) ) {
2017-12-31 02:30:31 +01:00
// TODO: force remote sync by deleting local item
2017-06-14 22:39:36 +02:00
log . vlog ( "The destination is occupied, renaming the conflicting file..." ) ;
2015-09-19 15:38:43 +02:00
safeRename ( newPath ) ;
2015-09-01 20:45:34 +02:00
}
2015-09-19 15:38:43 +02:00
rename ( oldPath , newPath ) ;
2015-09-01 20:45:34 +02:00
}
2017-12-31 02:30:31 +01:00
// handle changed content and mtime
2017-12-31 13:47:18 +01:00
// HACK: use mtime+hash instead of cTag because of https://github.com/OneDrive/onedrive-api-docs/issues/765
if ( newItem . type = = ItemType . file & & oldItem . mtime ! = newItem . mtime & & ! testFileHash ( newPath , newItem ) ) {
2017-12-31 02:30:31 +01:00
downloadFileItem ( newItem , newPath ) ;
2017-12-29 16:51:08 +01:00
} else {
2018-03-14 05:43:40 +01:00
//log.vlog("The item content has not changed");
2017-12-27 15:13:28 +01:00
}
// handle changed time
2017-12-31 13:47:18 +01:00
if ( newItem . type = = ItemType . file & & oldItem . mtime ! = newItem . mtime ) {
2017-12-27 15:13:28 +01:00
setTimes ( newPath , newItem . mtime , newItem . mtime ) ;
2017-12-31 13:47:18 +01:00
}
2015-09-01 20:45:34 +02:00
} else {
2018-03-14 05:43:40 +01:00
//log.vlog("", oldItem.name, " has not changed");
2015-09-01 20:45:34 +02:00
}
}
2017-12-31 02:30:31 +01:00
// downloads a File resource
private void downloadFileItem ( Item item , string path )
{
assert ( item . type = = ItemType . file ) ;
write ( "Downloading " , path , "..." ) ;
onedrive . downloadById ( item . driveId , item . id , path ) ;
setTimes ( path , item . mtime , item . mtime ) ;
writeln ( " done." ) ;
2018-04-10 12:03:28 +02:00
log . log ( "Downloading " , path , "... done." ) ;
2017-12-31 02:30:31 +01:00
}
2015-09-01 20:45:34 +02:00
// returns true if the given item corresponds to the local one
2015-09-19 15:38:43 +02:00
private bool isItemSynced ( Item item , string path )
2015-09-01 20:45:34 +02:00
{
2015-09-19 15:38:43 +02:00
if ( ! exists ( path ) ) return false ;
2015-09-01 20:45:34 +02:00
final switch ( item . type ) {
case ItemType . file :
2015-09-19 15:38:43 +02:00
if ( isFile ( path ) ) {
2017-12-31 02:30:31 +01:00
SysTime localModifiedTime = timeLastModified ( path ) . toUTC ( ) ;
2017-03-12 17:25:51 +01:00
// HACK: reduce time resolution to seconds before comparing
item . mtime . fracSecs = Duration . zero ;
localModifiedTime . fracSecs = Duration . zero ;
2015-09-14 23:56:14 +02:00
if ( localModifiedTime = = item . mtime ) {
return true ;
} else {
2016-08-04 23:35:58 +02:00
log . vlog ( "The local item has a different modified time " , localModifiedTime , " remote is " , item . mtime ) ;
2015-09-01 20:45:34 +02:00
}
2017-05-28 22:13:19 +02:00
if ( testFileHash ( path , item ) ) {
2015-09-19 15:38:43 +02:00
return true ;
} else {
2016-08-04 23:35:58 +02:00
log . vlog ( "The local item has a different hash" ) ;
2015-09-01 20:45:34 +02:00
}
} else {
2016-08-04 23:35:58 +02:00
log . vlog ( "The local item is a directory but should be a file" ) ;
2015-09-01 20:45:34 +02:00
}
break ;
case ItemType . dir :
2017-12-27 15:13:28 +01:00
case ItemType . remote :
2015-09-19 15:38:43 +02:00
if ( isDir ( path ) ) {
2015-09-14 23:56:14 +02:00
return true ;
} else {
2016-08-04 23:35:58 +02:00
log . vlog ( "The local item is a file but should be a directory" ) ;
2015-09-01 20:45:34 +02:00
}
break ;
}
return false ;
}
2015-09-14 23:56:14 +02:00
private void deleteItems ( )
2015-09-01 20:45:34 +02:00
{
2017-06-14 15:50:02 +02:00
foreach_reverse ( i ; idsToDelete ) {
2017-07-02 15:29:33 +02:00
Item item ;
if ( ! itemdb . selectById ( i [ 0 ] , i [ 1 ] , item ) ) continue ; // check if the item is in the db
2017-06-14 15:50:02 +02:00
string path = itemdb . computePath ( i [ 0 ] , i [ 1 ] ) ;
2018-04-10 12:03:28 +02:00
log . log ( "Deleting item " , path ) ;
2017-12-31 12:36:14 +01:00
itemdb . deleteById ( item . driveId , item . id ) ;
2017-12-31 16:56:56 +01:00
if ( item . remoteDriveId ! = null ) {
// delete the linked remote folder
itemdb . deleteById ( item . remoteDriveId , item . remoteId ) ;
}
2015-09-14 23:56:14 +02:00
if ( exists ( path ) ) {
if ( isFile ( path ) ) {
remove ( path ) ;
2015-09-28 13:42:58 +02:00
} else {
try {
2017-12-31 12:36:14 +01:00
if ( item . remoteDriveId = = null ) {
rmdir ( path ) ;
} else {
2017-12-31 16:56:56 +01:00
// children of remote items are not enumerated
2017-12-31 12:36:14 +01:00
rmdirRecurse ( path ) ;
}
2015-09-28 13:42:58 +02:00
} catch ( FileException e ) {
2017-12-31 12:36:14 +01:00
log . log ( e . msg ) ;
2015-09-28 13:42:58 +02:00
}
2015-09-06 22:42:44 +02:00
}
2015-09-01 20:45:34 +02:00
}
}
2016-12-25 23:25:24 +01:00
idsToDelete . length = 0 ;
assumeSafeAppend ( idsToDelete ) ;
2015-09-01 20:45:34 +02:00
}
2017-12-28 19:58:31 +01:00
2017-12-31 12:36:14 +01:00
// scan the given directory for differences and new items
2018-03-14 05:43:40 +01:00
void scanForDifferences ( string path )
2015-09-04 21:00:22 +02:00
{
2018-03-14 05:43:40 +01:00
// Make sure the OneDrive Root is in the database
checkDatabaseForOneDriveRoot ( ) ;
// scan for changes
2017-12-31 12:36:14 +01:00
log . vlog ( "Uploading differences of " , path ) ;
2017-12-28 15:03:15 +01:00
Item item ;
2018-01-02 15:05:32 +01:00
if ( itemdb . selectByPath ( path , defaultDriveId , item ) ) {
2017-12-28 15:03:15 +01:00
uploadDifferences ( item ) ;
2015-09-08 18:25:41 +02:00
}
2017-12-31 12:36:14 +01:00
log . vlog ( "Uploading new items of " , path ) ;
2017-12-28 15:03:15 +01:00
uploadNewItems ( path ) ;
2015-09-04 21:00:22 +02:00
}
2015-09-20 21:21:51 +02:00
private void uploadDifferences ( Item item )
2015-09-11 18:33:22 +02:00
{
2017-12-31 12:36:14 +01:00
log . vlog ( "Processing " , item . name ) ;
2017-03-12 16:07:45 +01:00
2017-12-31 12:36:14 +01:00
string path ;
bool unwanted = selectiveSync . isNameExcluded ( item . name ) ;
if ( ! unwanted ) {
path = itemdb . computePath ( item . driveId , item . id ) ;
unwanted = selectiveSync . isPathExcluded ( path ) ;
2017-03-12 16:07:45 +01:00
}
2017-12-31 12:36:14 +01:00
// skip unwanted items
if ( unwanted ) {
log . vlog ( "Filtered out" ) ;
2017-03-12 16:07:45 +01:00
return ;
}
2018-04-13 03:20:38 +02:00
// Restriction and limitations about windows naming files
if ( ! isValidName ( path ) ) {
log . vlog ( "Skipping item - invalid name (Microsoft Naming Convention): " , path ) ;
return ;
}
2017-03-12 16:07:45 +01:00
2015-09-18 21:42:27 +02:00
final switch ( item . type ) {
case ItemType . dir :
2015-09-19 15:38:43 +02:00
uploadDirDifferences ( item , path ) ;
2015-09-18 21:42:27 +02:00
break ;
case ItemType . file :
2015-09-19 15:38:43 +02:00
uploadFileDifferences ( item , path ) ;
2015-09-18 21:42:27 +02:00
break ;
2017-06-14 15:50:02 +02:00
case ItemType . remote :
2017-12-31 16:56:56 +01:00
uploadRemoteDirDifferences ( item , path ) ;
2017-12-31 12:36:14 +01:00
break ;
2015-09-11 18:33:22 +02:00
}
}
2015-09-19 15:38:43 +02:00
private void uploadDirDifferences ( Item item , string path )
2015-09-04 21:00:22 +02:00
{
2015-09-18 21:42:27 +02:00
assert ( item . type = = ItemType . dir ) ;
2015-09-19 15:38:43 +02:00
if ( exists ( path ) ) {
if ( ! isDir ( path ) ) {
2017-12-31 16:56:56 +01:00
log . vlog ( "The item was a directory but now it is a file" ) ;
2015-09-19 15:38:43 +02:00
uploadDeleteItem ( item , path ) ;
uploadNewFile ( path ) ;
2015-09-18 21:42:27 +02:00
} else {
2016-08-04 23:35:58 +02:00
log . vlog ( "The directory has not changed" ) ;
2015-09-18 21:42:27 +02:00
// loop trough the children
2017-06-14 15:50:02 +02:00
foreach ( Item child ; itemdb . selectChildren ( item . driveId , item . id ) ) {
2015-09-18 21:42:27 +02:00
uploadDifferences ( child ) ;
}
2015-09-17 17:34:58 +02:00
}
2015-09-18 21:42:27 +02:00
} else {
2016-08-04 23:35:58 +02:00
log . vlog ( "The directory has been deleted" ) ;
2015-09-19 15:38:43 +02:00
uploadDeleteItem ( item , path ) ;
2015-09-18 21:42:27 +02:00
}
}
2017-12-31 16:56:56 +01:00
private void uploadRemoteDirDifferences ( Item item , string path )
{
assert ( item . type = = ItemType . remote ) ;
if ( exists ( path ) ) {
if ( ! isDir ( path ) ) {
log . vlog ( "The item was a directory but now it is a file" ) ;
uploadDeleteItem ( item , path ) ;
uploadNewFile ( path ) ;
} else {
log . vlog ( "The directory has not changed" ) ;
// continue trough the linked folder
assert ( item . remoteDriveId & & item . remoteId ) ;
Item remoteItem ;
bool found = itemdb . selectById ( item . remoteDriveId , item . remoteId , remoteItem ) ;
assert ( found ) ;
uploadDifferences ( remoteItem ) ;
}
} else {
log . vlog ( "The directory has been deleted" ) ;
uploadDeleteItem ( item , path ) ;
}
}
2015-09-19 15:38:43 +02:00
private void uploadFileDifferences ( Item item , string path )
2015-09-18 21:42:27 +02:00
{
assert ( item . type = = ItemType . file ) ;
2015-09-19 15:38:43 +02:00
if ( exists ( path ) ) {
if ( isFile ( path ) ) {
2017-12-31 02:30:31 +01:00
SysTime localModifiedTime = timeLastModified ( path ) . toUTC ( ) ;
2017-03-12 17:17:38 +01:00
// HACK: reduce time resolution to seconds before comparing
item . mtime . fracSecs = Duration . zero ;
localModifiedTime . fracSecs = Duration . zero ;
2015-09-18 21:42:27 +02:00
if ( localModifiedTime ! = item . mtime ) {
2016-08-04 23:35:58 +02:00
log . vlog ( "The file last modified time has changed" ) ;
2015-09-18 21:42:27 +02:00
string eTag = item . eTag ;
2017-05-28 22:13:19 +02:00
if ( ! testFileHash ( path , item ) ) {
2016-08-04 23:35:58 +02:00
log . vlog ( "The file content has changed" ) ;
2018-03-14 05:43:40 +01:00
write ( "Uploading file " , path , "..." ) ;
2015-09-27 18:47:41 +02:00
JSONValue response ;
if ( getSize ( path ) < = thresholdFileSize ) {
2017-12-31 16:11:02 +01:00
response = onedrive . simpleUploadReplace ( path , item . driveId , item . id , item . eTag ) ;
writeln ( " done." ) ;
2015-09-27 18:47:41 +02:00
} else {
2017-12-31 16:11:02 +01:00
writeln ( "" ) ;
2018-01-02 15:05:32 +01:00
response = session . upload ( path , item . driveId , item . parentId , baseName ( path ) , eTag ) ;
2015-09-27 18:47:41 +02:00
}
2018-04-10 12:03:28 +02:00
log . log ( "Uploading file " , path , "... done." ) ;
2017-12-31 16:56:56 +01:00
// saveItem(response); redundant
2017-12-31 02:30:31 +01:00
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
2015-09-27 18:47:41 +02:00
eTag = response [ "cTag" ] . str ;
2015-09-17 16:28:24 +02:00
}
2017-12-31 02:30:31 +01:00
uploadLastModifiedTime ( item . driveId , item . id , eTag , localModifiedTime . toUTC ( ) ) ;
2015-09-06 22:42:44 +02:00
} else {
2016-08-04 23:35:58 +02:00
log . vlog ( "The file has not changed" ) ;
2015-09-06 22:42:44 +02:00
}
2015-09-18 21:42:27 +02:00
} else {
2016-08-04 23:35:58 +02:00
log . vlog ( "The item was a file but now is a directory" ) ;
2015-09-19 15:38:43 +02:00
uploadDeleteItem ( item , path ) ;
uploadCreateDir ( path ) ;
2015-09-04 21:00:22 +02:00
}
} else {
2016-08-04 23:35:58 +02:00
log . vlog ( "The file has been deleted" ) ;
2015-09-19 15:38:43 +02:00
uploadDeleteItem ( item , path ) ;
2015-09-11 18:33:22 +02:00
}
}
2015-09-18 21:42:27 +02:00
private void uploadNewItems ( string path )
2015-09-08 18:25:41 +02:00
{
2018-03-14 05:43:40 +01:00
// https://github.com/OneDrive/onedrive-api-docs/issues/443
// If the path is greater than 430 characters, then one drive will return a '400 - Bad Request'
// Need to ensure that the URI is encoded before the check is made
if ( encodeComponent ( path ) . length < 430 ) {
// path is less than 430 characters
2017-03-12 16:07:45 +01:00
2018-03-14 05:43:40 +01:00
// skip unexisting symbolic links
if ( isSymlink ( path ) & & ! exists ( readLink ( path ) ) ) {
2018-04-11 05:02:06 +02:00
log . vlog ( "Skipping item - symbolic link: " , path ) ;
2017-03-12 16:35:47 +01:00
return ;
}
2018-04-13 03:20:38 +02:00
// Restriction and limitations about windows naming files
if ( ! isValidName ( path ) ) {
log . vlog ( "Skipping item - invalid name (Microsoft Naming Convention): " , path ) ;
return ;
}
2017-03-12 16:07:45 +01:00
2018-04-16 13:39:45 +02:00
// filter out user configured items to skip
2018-03-14 05:43:40 +01:00
if ( path ! = "." ) {
if ( selectiveSync . isNameExcluded ( baseName ( path ) ) ) {
2018-04-16 13:39:45 +02:00
log . vlog ( "Skipping item - excluded by skip_file config: " , path ) ;
2018-03-14 05:43:40 +01:00
return ;
}
if ( selectiveSync . isPathExcluded ( path ) ) {
2018-04-11 05:02:06 +02:00
log . vlog ( "Skipping item - path excluded: " , path ) ;
2018-03-14 05:43:40 +01:00
return ;
}
2017-03-12 16:07:45 +01:00
}
2018-03-14 05:43:40 +01:00
2018-04-16 13:39:45 +02:00
// This item passed all the unwanted checks
// We want to upload this new item
2018-03-14 05:43:40 +01:00
if ( isDir ( path ) ) {
Item item ;
if ( ! itemdb . selectByPath ( path , defaultDriveId , item ) ) {
uploadCreateDir ( path ) ;
}
// recursively traverse children
auto entries = dirEntries ( path , SpanMode . shallow , false ) ;
foreach ( DirEntry entry ; entries ) {
uploadNewItems ( entry . name ) ;
}
} else {
2018-04-20 23:19:00 +02:00
Item item ;
if ( ! itemdb . selectByPath ( path , defaultDriveId , item ) ) {
uploadNewFile ( path ) ;
2018-03-14 05:43:40 +01:00
}
2015-09-18 21:42:27 +02:00
}
} else {
2018-03-14 05:43:40 +01:00
// This path was skipped - why?
log . log ( "Skipping item '" , path , "' due to the full path exceeding 430 characters (Microsoft OneDrive limitation)" ) ;
2015-09-06 22:42:44 +02:00
}
2015-09-08 18:25:41 +02:00
}
2018-03-14 05:43:40 +01:00
private void uploadCreateDir ( const ( string ) path )
2015-09-08 18:25:41 +02:00
{
2018-03-14 05:43:40 +01:00
log . vlog ( "OneDrive Client requested to create remote path: " , path ) ;
2017-12-31 13:18:11 +01:00
Item parent ;
2018-03-14 05:43:40 +01:00
// Was the path entered the root path?
if ( path = = "." ) {
// We cant create this directory, as this would essentially equal the users OneDrive root:/
checkDatabaseForOneDriveRoot ( ) ;
} else {
// If this is null or empty - we cant query the database properly
if ( ( parent . driveId = = "" ) & & ( parent . id = = "" ) ) {
// These are both empty .. not good
// What path to use?
string parentPath = dirName ( path ) ; // will be either . or something else
if ( parentPath = = "." ) {
// We cant create this directory, as this would essentially equal the users OneDrive root:/
checkDatabaseForOneDriveRoot ( ) ;
}
2018-04-17 22:28:08 +02:00
JSONValue onedrivePathDetails ;
2018-03-14 05:43:40 +01:00
try {
2018-04-17 22:28:08 +02:00
onedrivePathDetails = onedrive . getPathDetails ( parentPath ) ;
2018-03-14 05:43:40 +01:00
} catch ( OneDriveException e ) {
if ( e . httpStatusCode = = 404 ) {
// Parent does not exist ... need to create parent
uploadCreateDir ( parentPath ) ;
}
}
2018-04-17 22:28:08 +02:00
2018-03-14 05:43:40 +01:00
// configure the data
parent . driveId = onedrivePathDetails [ "parentReference" ] [ "driveId" ] . str ; // Should give something like 12345abcde1234a1
parent . id = onedrivePathDetails [ "id" ] . str ; // This item's ID. Should give something like 12345ABCDE1234A1!101
}
// test if the path we are going to create already exists on OneDrive
try {
onedrive . getPathDetails ( path ) ;
} catch ( OneDriveException e ) {
if ( e . httpStatusCode = = 404 ) {
// The directory was not found
log . vlog ( "The requested directory to create was not found on OneDrive - creating remote directory: " , path ) ;
2015-09-16 10:29:20 +02:00
2018-03-14 05:43:40 +01:00
// Perform the database lookup
enforce ( itemdb . selectById ( parent . driveId , parent . id , parent ) , "The parent item id is not in the database" ) ;
JSONValue driveItem = [
"name" : JSONValue ( baseName ( path ) ) ,
"folder" : parseJSON ( "{}" )
] ;
// Submit the creation request
auto res = onedrive . createById ( parent . driveId , parent . id , driveItem ) ;
// What is returned?
//log.vlog("Create Folder Response JSON: ", res);
saveItem ( res ) ;
log . vlog ( "Sucessfully created the remote directory " , path , " on OneDrive" ) ;
return ;
}
}
log . vlog ( "The requested directory to create was found on OneDrive - skipping creating the directory: " , path ) ;
// Check that this path is in the database
if ( ! itemdb . selectById ( parent . driveId , parent . id , parent ) ) {
// parent for 'path' is NOT in the database
log . vlog ( "The parent for this path is not in the local database - need to add parent to local database" ) ;
string parentPath = dirName ( path ) ;
uploadCreateDir ( parentPath ) ;
} else {
// parent is in database
log . vlog ( "The parent for this path is in the local database - adding requested path (" , path , ") to database" ) ;
auto res = onedrive . getPathDetails ( path ) ;
saveItem ( res ) ;
}
}
}
2015-09-16 10:29:20 +02:00
private void uploadNewFile ( string path )
{
2017-12-31 13:47:18 +01:00
Item parent ;
2018-03-14 05:43:40 +01:00
// Check the database for the parent
enforce ( itemdb . selectByPath ( dirName ( path ) , defaultDriveId , parent ) , "The parent item is not in the local database" ) ;
2018-04-11 23:31:42 +02:00
// Maximum file size upload
// https://support.microsoft.com/en-au/help/3125202/restrictions-and-limitations-when-you-sync-files-and-folders
// 1. OneDrive Business say's 15GB
// 2. Another article updated April 2018 says 20GB:
// https://answers.microsoft.com/en-us/onedrive/forum/odoptions-oddesktop-sdwin10/personal-onedrive-file-upload-size-max/a3621fc9-b766-4a99-99f8-bcc01ccb025f
2018-03-14 05:43:40 +01:00
2018-04-11 23:31:42 +02:00
// Use smaller size for now
auto maxUploadFileSize = 16106127360 ; // 15GB
//auto maxUploadFileSize = 21474836480; // 20GB
auto thisFileSize = getSize ( path ) ;
if ( thisFileSize < = maxUploadFileSize ) {
// Resolves: https://github.com/skilion/onedrive/issues/121, https://github.com/skilion/onedrive/issues/294, https://github.com/skilion/onedrive/issues/329
// To avoid a 409 Conflict error - does the file actually exist on OneDrive already?
JSONValue fileDetailsFromOneDrive ;
// Does this 'file' already exist on OneDrive?
try {
// test if the local path exists on OneDrive
fileDetailsFromOneDrive = onedrive . getPathDetails ( path ) ;
} catch ( OneDriveException e ) {
if ( e . httpStatusCode = = 404 ) {
// The file was not found on OneDrive, need to upload it
write ( "Uploading file " , path , "..." ) ;
JSONValue response ;
if ( getSize ( path ) < = thresholdFileSize ) {
response = onedrive . simpleUpload ( path , parent . driveId , parent . id , baseName ( path ) ) ;
writeln ( " done." ) ;
} else {
writeln ( "" ) ;
response = session . upload ( path , parent . driveId , parent . id , baseName ( path ) ) ;
}
log . log ( "Uploading file " , path , "... done." ) ;
string id = response [ "id" ] . str ;
string cTag = response [ "cTag" ] . str ;
SysTime mtime = timeLastModified ( path ) . toUTC ( ) ;
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
uploadLastModifiedTime ( parent . driveId , id , cTag , mtime ) ;
return ;
}
}
log . vlog ( "Requested file to upload exists on OneDrive - local database is out of sync for this file: " , path ) ;
// Is the local file newer than the uploaded file?
SysTime localFileModifiedTime = timeLastModified ( path ) . toUTC ( ) ;
SysTime remoteFileModifiedTime = SysTime . fromISOExtString ( fileDetailsFromOneDrive [ "fileSystemInfo" ] [ "lastModifiedDateTime" ] . str ) ;
if ( localFileModifiedTime > remoteFileModifiedTime ) {
// local file is newer
log . vlog ( "Requested file to upload is newer than existing file on OneDrive" ) ;
2018-03-14 05:43:40 +01:00
write ( "Uploading file " , path , "..." ) ;
JSONValue response ;
if ( getSize ( path ) < = thresholdFileSize ) {
response = onedrive . simpleUpload ( path , parent . driveId , parent . id , baseName ( path ) ) ;
writeln ( " done." ) ;
} else {
writeln ( "" ) ;
response = session . upload ( path , parent . driveId , parent . id , baseName ( path ) ) ;
}
2018-04-10 12:03:28 +02:00
log . log ( "Uploading file " , path , "... done." ) ;
2018-03-14 05:43:40 +01:00
string id = response [ "id" ] . str ;
string cTag = response [ "cTag" ] . str ;
SysTime mtime = timeLastModified ( path ) . toUTC ( ) ;
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
uploadLastModifiedTime ( parent . driveId , id , cTag , mtime ) ;
} else {
2018-04-11 23:31:42 +02:00
// Save the details of the file that we got from OneDrive
log . vlog ( "Updating the local database with details for this file: " , path ) ;
saveItem ( fileDetailsFromOneDrive ) ;
2018-03-14 05:43:40 +01:00
}
2018-04-12 00:44:52 +02:00
} else {
// Skip file - too large
log . log ( "Skipping uploading this new file as it exceeds the maximum allowed OneDrive size: " , path ) ;
2015-09-27 18:47:41 +02:00
}
2015-09-16 10:29:20 +02:00
}
2018-03-14 05:43:40 +01:00
private void uploadDeleteItem ( Item item , string path )
2015-09-16 10:29:20 +02:00
{
2018-04-10 12:03:28 +02:00
log . log ( "Deleting item from OneDrive: " , path ) ;
2018-03-14 05:43:40 +01:00
if ( ( item . driveId = = "" ) & & ( item . id = = "" ) & & ( item . eTag = = "" ) ) {
// These are empty ... we cannot delete if this is empty ....
JSONValue onedrivePathDetails = onedrive . getPathDetails ( path ) ; // Returns a JSON String for the OneDrive Path
item . driveId = onedrivePathDetails [ "parentReference" ] [ "driveId" ] . str ; // Should give something like 12345abcde1234a1
item . id = onedrivePathDetails [ "id" ] . str ; // This item's ID. Should give something like 12345ABCDE1234A1!101
item . eTag = onedrivePathDetails [ "eTag" ] . str ; // Should be something like aNjM2NjJFRUVGQjY2NjJFMSE5MzUuMA
}
2015-10-04 16:25:31 +02:00
try {
2017-12-29 12:24:26 +01:00
onedrive . deleteById ( item . driveId , item . id , item . eTag ) ;
2015-10-04 16:25:31 +02:00
} catch ( OneDriveException e ) {
2018-04-17 22:28:08 +02:00
if ( e . httpStatusCode = = 404 ) log . log ( "OneDrive reported: The resource could not be found." ) ;
2015-10-04 16:25:31 +02:00
else throw e ;
}
2017-06-14 15:50:02 +02:00
itemdb . deleteById ( item . driveId , item . id ) ;
2017-12-31 16:56:56 +01:00
if ( item . remoteId ! = null ) {
itemdb . deleteById ( item . remoteDriveId , item . remoteId ) ;
}
2015-09-08 18:25:41 +02:00
}
2017-12-27 15:13:28 +01:00
private void uploadLastModifiedTime ( const ( char ) [ ] driveId , const ( char ) [ ] id , const ( char ) [ ] eTag , SysTime mtime )
2015-09-08 18:25:41 +02:00
{
2017-12-31 02:30:31 +01:00
JSONValue data = [
2015-09-08 18:25:41 +02:00
"fileSystemInfo" : JSONValue ( [
"lastModifiedDateTime" : mtime . toISOExtString ( )
] )
] ;
2017-12-31 02:30:31 +01:00
auto response = onedrive . updateById ( driveId , id , data , eTag ) ;
saveItem ( response ) ;
2015-09-08 18:25:41 +02:00
}
2016-12-24 14:12:20 +01:00
private void saveItem ( JSONValue jsonItem )
2015-09-08 18:25:41 +02:00
{
2018-03-14 05:43:40 +01:00
// Takes a JSON input and formats to an item which can be used by the database
2017-05-28 22:13:19 +02:00
Item item = makeItem ( jsonItem ) ;
2018-03-14 05:43:40 +01:00
// Add to the local database
2016-12-24 14:12:20 +01:00
itemdb . upsert ( item ) ;
2015-09-16 10:29:20 +02:00
}
2018-01-01 18:38:08 +01:00
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_move
2015-09-19 15:38:43 +02:00
void uploadMoveItem ( string from , string to )
2015-09-16 10:29:20 +02:00
{
2018-01-01 18:38:08 +01:00
log . log ( "Moving " , from , " to " , to ) ;
2015-09-20 19:07:16 +02:00
Item fromItem , toItem , parentItem ;
2018-01-02 15:05:32 +01:00
if ( ! itemdb . selectByPath ( from , defaultDriveId , fromItem ) ) {
2015-09-20 21:21:51 +02:00
throw new SyncException ( "Can't move an unsynced item" ) ;
2015-09-11 18:33:22 +02:00
}
2018-01-02 13:41:56 +01:00
if ( fromItem . parentId = = null ) {
// the item is a remote folder, need to do the operation on the parent
2018-01-02 15:05:32 +01:00
enforce ( itemdb . selectByPathNoRemote ( from , defaultDriveId , fromItem ) ) ;
2018-01-02 13:41:56 +01:00
}
2018-01-02 15:05:32 +01:00
if ( itemdb . selectByPath ( to , defaultDriveId , toItem ) ) {
2018-01-01 18:38:08 +01:00
// the destination has been overwritten
2015-09-20 19:07:16 +02:00
uploadDeleteItem ( toItem , to ) ;
}
2018-01-02 15:05:32 +01:00
if ( ! itemdb . selectByPath ( dirName ( to ) , defaultDriveId , parentItem ) ) {
2015-09-20 21:21:51 +02:00
throw new SyncException ( "Can't move an item to an unsynced directory" ) ;
2015-09-17 16:28:24 +02:00
}
2018-01-01 18:38:08 +01:00
if ( fromItem . driveId ! = parentItem . driveId ) {
// items cannot be moved between drives
uploadDeleteItem ( fromItem , from ) ;
uploadNewFile ( to ) ;
} else {
SysTime mtime = timeLastModified ( to ) . toUTC ( ) ;
JSONValue diff = [
"name" : JSONValue ( baseName ( to ) ) ,
"parentReference" : JSONValue ( [
"id" : parentItem . id
] ) ,
"fileSystemInfo" : JSONValue ( [
"lastModifiedDateTime" : mtime . toISOExtString ( )
] )
] ;
auto res = onedrive . updateById ( fromItem . driveId , fromItem . id , diff , fromItem . eTag ) ;
// update itemdb
saveItem ( res ) ;
}
2015-09-11 18:33:22 +02:00
}
2018-03-14 05:43:40 +01:00
void deleteByPath ( string path )
2015-09-11 18:33:22 +02:00
{
Item item ;
2018-01-02 15:05:32 +01:00
if ( ! itemdb . selectByPath ( path , defaultDriveId , item ) ) {
2018-03-14 05:43:40 +01:00
throw new SyncException ( "The item to delete is not in the local database" ) ;
2015-09-11 18:33:22 +02:00
}
2018-01-02 13:41:56 +01:00
if ( item . parentId = = null ) {
// the item is a remote folder, need to do the operation on the parent
2018-01-02 15:05:32 +01:00
enforce ( itemdb . selectByPathNoRemote ( path , defaultDriveId , item ) ) ;
2018-01-02 13:41:56 +01:00
}
2015-10-04 16:25:31 +02:00
try {
uploadDeleteItem ( item , path ) ;
} catch ( OneDriveException e ) {
2016-12-14 15:17:20 +01:00
if ( e . httpStatusCode = = 404 ) log . log ( e . msg ) ;
2015-10-04 16:25:31 +02:00
else throw e ;
}
2015-09-11 18:33:22 +02:00
}
2018-03-14 05:43:40 +01:00
// move a OneDrive folder from one name to another
void moveByPath ( const ( string ) source , const ( string ) destination )
{
log . vlog ( "Moving remote folder: " , source , " -> " , destination ) ;
// Source and Destination are relative to ~/OneDrive
string sourcePath = source ;
string destinationBasePath = dirName ( destination ) . idup ;
// if destinationBasePath == '.' then destinationBasePath needs to be ""
if ( destinationBasePath = = "." ) {
destinationBasePath = "" ;
}
string newFolderName = baseName ( destination ) . idup ;
string destinationPathString = "/drive/root:/" ~ destinationBasePath ;
// Build up the JSON changes
JSONValue moveData = [ "name" : newFolderName ] ;
JSONValue destinationPath = [ "path" : destinationPathString ] ;
moveData [ "parentReference" ] = destinationPath ;
// Make the change on OneDrive
auto res = onedrive . moveByPath ( sourcePath , moveData ) ;
}
2015-09-01 20:45:34 +02:00
}