mirror of
https://github.com/abraunegg/onedrive
synced 2024-05-03 06:23:16 +02:00
Update OneDrive API response handling for National Cloud Deployments (#2023)
* Update OneDrive API response handling for National Cloud Deployments * Add developer option to allow easy switch between /children and /delta to query OneDrive for changes
This commit is contained in:
parent
042949f1c1
commit
ca984eba70
|
@ -147,7 +147,12 @@ final class Config
|
||||||
// display_sync_options = true | false
|
// display_sync_options = true | false
|
||||||
// - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging
|
// - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging
|
||||||
boolValues["display_sync_options"] = false;
|
boolValues["display_sync_options"] = false;
|
||||||
|
// force_children_scan = true | false
|
||||||
|
// - Force client to use /children rather than /delta to query changes on OneDrive
|
||||||
|
// - This option flags nationalCloudDeployment as true, forcing the client to act like it is using a National Cloud Deployment
|
||||||
|
boolValues["force_children_scan"] = false;
|
||||||
|
|
||||||
|
// EXPAND USERS HOME DIRECTORY
|
||||||
// Determine the users home directory.
|
// Determine the users home directory.
|
||||||
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
|
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
|
||||||
// Check for HOME environment variable
|
// Check for HOME environment variable
|
||||||
|
|
|
@ -1100,9 +1100,18 @@ int main(string[] args)
|
||||||
// value is configured, is it a valid value?
|
// value is configured, is it a valid value?
|
||||||
if ((cfg.getValueString("azure_ad_endpoint") == "USL4") || (cfg.getValueString("azure_ad_endpoint") == "USL5") || (cfg.getValueString("azure_ad_endpoint") == "DE") || (cfg.getValueString("azure_ad_endpoint") == "CN")) {
|
if ((cfg.getValueString("azure_ad_endpoint") == "USL4") || (cfg.getValueString("azure_ad_endpoint") == "USL5") || (cfg.getValueString("azure_ad_endpoint") == "DE") || (cfg.getValueString("azure_ad_endpoint") == "CN")) {
|
||||||
// valid entries to flag we are using a National Cloud Deployment
|
// valid entries to flag we are using a National Cloud Deployment
|
||||||
|
// National Cloud Deployments do not support /delta as a query
|
||||||
|
// https://docs.microsoft.com/en-us/graph/deployments#supported-features
|
||||||
|
// Flag that we have a valid National Cloud Deployment that cannot use /delta queries
|
||||||
sync.setNationalCloudDeployment();
|
sync.setNationalCloudDeployment();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Are we forcing to use /children scan instead of /delta to simulate National Cloud Deployment use of /children?
|
||||||
|
if (cfg.getValueBool("force_children_scan")) {
|
||||||
|
log.vdebug("Forcing client to use /children scan rather than /delta to simulate National Cloud Deployment use of /children");
|
||||||
|
sync.setNationalCloudDeployment();
|
||||||
|
}
|
||||||
|
|
||||||
// Do we need to validate the syncDir to check for the presence of a '.nosync' file
|
// Do we need to validate the syncDir to check for the presence of a '.nosync' file
|
||||||
if (cfg.getValueBool("check_nomount")) {
|
if (cfg.getValueBool("check_nomount")) {
|
||||||
|
|
94
src/sync.d
94
src/sync.d
|
@ -186,7 +186,7 @@ private Item makeItem(const ref JSONValue driveItem)
|
||||||
item.remoteId = driveItem["remoteItem"]["id"].str;
|
item.remoteId = driveItem["remoteItem"]["id"].str;
|
||||||
}
|
}
|
||||||
|
|
||||||
// National Cloud Deployments (US and DE) do not support /delta as a query
|
// National Cloud Deployments do not support /delta as a query
|
||||||
// Thus we need to track in the database that this item is in sync
|
// Thus we need to track in the database that this item is in sync
|
||||||
// As we are making an item, set the syncStatus to Y
|
// As we are making an item, set the syncStatus to Y
|
||||||
// ONLY when using a National Cloud Deployment, all the existing DB entries will get set to N
|
// ONLY when using a National Cloud Deployment, all the existing DB entries will get set to N
|
||||||
|
@ -1471,36 +1471,39 @@ final class SyncEngine
|
||||||
// To view changes correctly, we need to use the correct path id for the request
|
// To view changes correctly, we need to use the correct path id for the request
|
||||||
if (driveId == defaultDriveId) {
|
if (driveId == defaultDriveId) {
|
||||||
// The drive id matches our users default drive id
|
// The drive id matches our users default drive id
|
||||||
idToQuery = defaultRootId.dup;
|
|
||||||
log.vdebug("Configuring 'idToQuery' as defaultRootId duplicate");
|
log.vdebug("Configuring 'idToQuery' as defaultRootId duplicate");
|
||||||
|
idToQuery = defaultRootId.dup;
|
||||||
} else {
|
} else {
|
||||||
// The drive id does not match our users default drive id
|
// The drive id does not match our users default drive id
|
||||||
// Potentially the 'path id' we are requesting the details of is a Shared Folder (remote item)
|
// Potentially the 'path id' we are requesting the details of is a Shared Folder (remote item)
|
||||||
// Use the 'id' that was passed in (folderId)
|
// Use the 'id' that was passed in (folderId)
|
||||||
idToQuery = id.dup;
|
|
||||||
log.vdebug("Configuring 'idToQuery' as 'id' duplicate");
|
log.vdebug("Configuring 'idToQuery' as 'id' duplicate");
|
||||||
|
idToQuery = id.dup;
|
||||||
}
|
}
|
||||||
// what path id are we going to query?
|
// what path id are we going to query?
|
||||||
log.vdebug("Path object to query configured as 'idToQuery' = ", idToQuery);
|
log.vdebug("Path object to query configured as 'idToQuery' = ", idToQuery);
|
||||||
long deltaChanges = 0;
|
long deltaChanges = 0;
|
||||||
|
|
||||||
// What query do we use?
|
// What query do we use?
|
||||||
// National Cloud Deployments (US and DE) do not support /delta as a query
|
// National Cloud Deployments do not support /delta as a query
|
||||||
// https://docs.microsoft.com/en-us/graph/deployments#supported-features
|
// https://docs.microsoft.com/en-us/graph/deployments#supported-features
|
||||||
// Are we running against a National Cloud Deployments that does not support /delta
|
// Are we running against a National Cloud Deployments that does not support /delta
|
||||||
if (nationalCloudDeployment) {
|
if (nationalCloudDeployment) {
|
||||||
// Have to query /children rather than /delta
|
// National Cloud Deployment that does not support /delta query
|
||||||
|
// Have to query /children and build our own /delta response
|
||||||
nationalCloudChildrenScan = true;
|
nationalCloudChildrenScan = true;
|
||||||
log.vdebug("Using /children call to query drive for items to populate 'changes' and 'changesAvailable'");
|
log.vdebug("Using /children call to query drive for items to populate 'changes' and 'changesAvailable'");
|
||||||
// In a OneDrive Business Shared Folder scenario + nationalCloudDeployment, if ALL items are downgraded, then this leads to local file deletion
|
// In a OneDrive Business Shared Folder scenario + nationalCloudDeployment, if ALL items are downgraded, then this leads to local file deletion
|
||||||
// Downgrade ONLY files associated with this driveId and idToQuery
|
// Downgrade ONLY files associated with this driveId and idToQuery
|
||||||
log.vdebug("Downgrading all children for this driveId (" ~ driveId ~ ") and idToQuery (" ~ idToQuery ~ ") to an out-of-sync state");
|
log.vdebug("Downgrading all children for this driveId (" ~ driveId ~ ") and idToQuery (" ~ idToQuery ~ ") to an out-of-sync state");
|
||||||
|
|
||||||
// Before we get any data, flag any object in the database as out-of-sync for this driveID & ID
|
// Before we get any data, flag any object in the database as out-of-sync for this driveID & ID
|
||||||
auto drivePathChildren = itemdb.selectChildren(driveId, idToQuery);
|
auto drivePathChildren = itemdb.selectChildren(driveId, idToQuery);
|
||||||
if (count(drivePathChildren) > 0) {
|
if (count(drivePathChildren) > 0) {
|
||||||
// Children to process and flag as out-of-sync
|
// Children to process and flag as out-of-sync
|
||||||
foreach (drivePathChild; drivePathChildren) {
|
foreach (drivePathChild; drivePathChildren) {
|
||||||
// Flag any object in the database as out-of-sync for this driveID & ID
|
// Flag any object in the database as out-of-sync for this driveID & ID
|
||||||
|
log.vdebug("Downgrading item as out-of-sync: ", drivePathChild.id);
|
||||||
itemdb.downgradeSyncStatusFlag(drivePathChild.driveId, drivePathChild.id);
|
itemdb.downgradeSyncStatusFlag(drivePathChild.driveId, drivePathChild.id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1779,6 +1782,9 @@ final class SyncEngine
|
||||||
// In some OneDrive Business scenarios, the shared folder /delta response lacks the 'root' drive details
|
// In some OneDrive Business scenarios, the shared folder /delta response lacks the 'root' drive details
|
||||||
// When this occurs, this creates the following error: A database statement execution error occurred: foreign key constraint failed
|
// When this occurs, this creates the following error: A database statement execution error occurred: foreign key constraint failed
|
||||||
// Ensure we query independently the root details for this shared folder and ensure that it is added before we process the /delta response
|
// Ensure we query independently the root details for this shared folder and ensure that it is added before we process the /delta response
|
||||||
|
|
||||||
|
// However, if we are using a National Cloud Deployment, these deployments do not support /delta, so we generate a /delta response via generateDeltaResponse()
|
||||||
|
// This specifically adds the root drive details to the self generated /delta response
|
||||||
if ((!nationalCloudDeployment) && (driveId!= defaultDriveId) && (syncBusinessFolders)) {
|
if ((!nationalCloudDeployment) && (driveId!= defaultDriveId) && (syncBusinessFolders)) {
|
||||||
// fetch this driveId root details to ensure we add this to the database for this remote drive
|
// fetch this driveId root details to ensure we add this to the database for this remote drive
|
||||||
JSONValue rootData;
|
JSONValue rootData;
|
||||||
|
@ -2506,9 +2512,13 @@ final class SyncEngine
|
||||||
SysTime remoteModifiedTime = item.mtime;
|
SysTime remoteModifiedTime = item.mtime;
|
||||||
remoteModifiedTime.fracSecs = Duration.zero;
|
remoteModifiedTime.fracSecs = Duration.zero;
|
||||||
|
|
||||||
if (localModifiedTime != remoteModifiedTime) {
|
// If the timestamp is different, or we are running on a National Cloud Deployment that does not support /delta queries - we have to update the DB with the details from OneDrive
|
||||||
// Database update needed for this item because our record is out-of-date
|
// Unfortunatly because of the consequence of Nataional Cloud Deployments not supporting /delta queries, the application uses the local database to flag what is out-of-date / track changes
|
||||||
log.vdebug("Updating local database with item details as timestamps of items are different");
|
// This means that the constant disk writing to the database fix implemented with https://github.com/abraunegg/onedrive/pull/2004 cannot be utilised when using Nataional Cloud Deployments
|
||||||
|
// as all records are touched / updated when performing the OneDrive sync operations. The only way to change this, is for Microsoft to support /delta queries for Nataional Cloud Deployments
|
||||||
|
if ((localModifiedTime != remoteModifiedTime) || (nationalCloudDeployment)) {
|
||||||
|
// Database update needed for this item because our local record is out-of-date
|
||||||
|
log.vdebug("Updating local database with item details from OneDrive as local record needs to be updated");
|
||||||
itemdb.update(item);
|
itemdb.update(item);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
@ -3198,27 +3208,9 @@ final class SyncEngine
|
||||||
}
|
}
|
||||||
|
|
||||||
// Are we configured to use a National Cloud Deployment
|
// Are we configured to use a National Cloud Deployment
|
||||||
// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
|
if (nationalCloudDeployment) {
|
||||||
// Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query
|
|
||||||
if ((nationalCloudDeployment) || (syncBusinessFolders)) {
|
|
||||||
// Select items that have a out-of-sync flag set
|
// Select items that have a out-of-sync flag set
|
||||||
foreach (driveId; driveIDsArray) {
|
flagNationalCloudDeploymentOutOfSyncItems();
|
||||||
// For each unique OneDrive driveID we know about
|
|
||||||
Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId);
|
|
||||||
foreach (item; outOfSyncItems) {
|
|
||||||
if (!dryRun) {
|
|
||||||
// clean up idsToDelete
|
|
||||||
idsToDelete.length = 0;
|
|
||||||
assumeSafeAppend(idsToDelete);
|
|
||||||
// flag to delete local file as it now is no longer in sync with OneDrive
|
|
||||||
log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive");
|
|
||||||
log.vdebug("item: ", item);
|
|
||||||
idsToDelete ~= [item.driveId, item.id];
|
|
||||||
// delete items in idsToDelete
|
|
||||||
if (idsToDelete.length > 0) deleteItems();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// scan for changes in the path provided
|
// scan for changes in the path provided
|
||||||
|
@ -3299,27 +3291,9 @@ final class SyncEngine
|
||||||
}
|
}
|
||||||
|
|
||||||
// Are we configured to use a National Cloud Deployment
|
// Are we configured to use a National Cloud Deployment
|
||||||
// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
|
if (nationalCloudDeployment) {
|
||||||
// Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query
|
|
||||||
if ((nationalCloudDeployment) || (syncBusinessFolders)) {
|
|
||||||
// Select items that have a out-of-sync flag set
|
// Select items that have a out-of-sync flag set
|
||||||
foreach (driveId; driveIDsArray) {
|
flagNationalCloudDeploymentOutOfSyncItems();
|
||||||
// For each unique OneDrive driveID we know about
|
|
||||||
Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId);
|
|
||||||
foreach (item; outOfSyncItems) {
|
|
||||||
if (!dryRun) {
|
|
||||||
// clean up idsToDelete
|
|
||||||
idsToDelete.length = 0;
|
|
||||||
assumeSafeAppend(idsToDelete);
|
|
||||||
// flag to delete local file as it now is no longer in sync with OneDrive
|
|
||||||
log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive");
|
|
||||||
log.vdebug("item: ", item);
|
|
||||||
idsToDelete ~= [item.driveId, item.id];
|
|
||||||
// delete items in idsToDelete
|
|
||||||
if (idsToDelete.length > 0) deleteItems();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// scan for changes in the path provided
|
// scan for changes in the path provided
|
||||||
|
@ -3360,6 +3334,30 @@ final class SyncEngine
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void flagNationalCloudDeploymentOutOfSyncItems() {
|
||||||
|
// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
|
||||||
|
// Normally, this is done at the end of processing all /delta queries, however National Cloud Deployments do not support /delta as a query
|
||||||
|
// https://docs.microsoft.com/en-us/graph/deployments#supported-features
|
||||||
|
// Select items that have a out-of-sync flag set
|
||||||
|
foreach (driveId; driveIDsArray) {
|
||||||
|
// For each unique OneDrive driveID we know about
|
||||||
|
Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId);
|
||||||
|
foreach (item; outOfSyncItems) {
|
||||||
|
if (!dryRun) {
|
||||||
|
// clean up idsToDelete
|
||||||
|
idsToDelete.length = 0;
|
||||||
|
assumeSafeAppend(idsToDelete);
|
||||||
|
// flag to delete local file as it now is no longer in sync with OneDrive
|
||||||
|
log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive");
|
||||||
|
log.vdebug("item: ", item);
|
||||||
|
idsToDelete ~= [item.driveId, item.id];
|
||||||
|
// delete items in idsToDelete
|
||||||
|
if (idsToDelete.length > 0) deleteItems();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void handleUploadOnlyBusinessSharedFoldersEdgeCase() {
|
void handleUploadOnlyBusinessSharedFoldersEdgeCase() {
|
||||||
// read in the business_shared_folders file contents
|
// read in the business_shared_folders file contents
|
||||||
string[] businessSharedFoldersList;
|
string[] businessSharedFoldersList;
|
||||||
|
|
Loading…
Reference in a new issue