deblan-mirror/abraunegg-onedrive
deblan-mirror
/
abraunegg-onedrive
mirror of https://github.com/abraunegg/onedrive
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update OneDrive API response handling for National Cloud Deployments (#2023) 

* Update OneDrive API response handling for National Cloud Deployments
* Add developer option to allow easy switch between /children and /delta to query OneDrive for changes
This commit is contained in:
abraunegg 2022-06-30 07:08:29 +10:00 committed by GitHub
parent 042949f1c1
commit ca984eba70
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 61 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/config.d
View File

@ -147,7 +147,12 @@ final class Config
// display_sync_options = true | false
// - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging
boolValues["display_sync_options"] = false;
// force_children_scan = true | false
// - Force client to use /children rather than /delta to query changes on OneDrive
// - This option flags nationalCloudDeployment as true, forcing the client to act like it is using a National Cloud Deployment
boolValues["force_children_scan"] = false;
// EXPAND USERS HOME DIRECTORY
// Determine the users home directory.
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
// Check for HOME environment variable

9
src/main.d
View File

@ -1100,9 +1100,18 @@ int main(string[] args)
// value is configured, is it a valid value?
if ((cfg.getValueString("azure_ad_endpoint") == "USL4") || (cfg.getValueString("azure_ad_endpoint") == "USL5") || (cfg.getValueString("azure_ad_endpoint") == "DE") || (cfg.getValueString("azure_ad_endpoint") == "CN")) {
// valid entries to flag we are using a National Cloud Deployment
// National Cloud Deployments do not support /delta as a query
// https://docs.microsoft.com/en-us/graph/deployments#supported-features
// Flag that we have a valid National Cloud Deployment that cannot use /delta queries
sync.setNationalCloudDeployment();
}
}
// Are we forcing to use /children scan instead of /delta to simulate National Cloud Deployment use of /children?
if (cfg.getValueBool("force_children_scan")) {
log.vdebug("Forcing client to use /children scan rather than /delta to simulate National Cloud Deployment use of /children");
sync.setNationalCloudDeployment();
}
// Do we need to validate the syncDir to check for the presence of a '.nosync' file
if (cfg.getValueBool("check_nomount")) {

94
src/sync.d
View File

@ -186,7 +186,7 @@ private Item makeItem(const ref JSONValue driveItem)
item.remoteId = driveItem["remoteItem"]["id"].str;
}
// National Cloud Deployments (US and DE) do not support /delta as a query
// National Cloud Deployments do not support /delta as a query
// Thus we need to track in the database that this item is in sync
// As we are making an item, set the syncStatus to Y
// ONLY when using a National Cloud Deployment, all the existing DB entries will get set to N
@ -1471,36 +1471,39 @@ final class SyncEngine
// To view changes correctly, we need to use the correct path id for the request
if (driveId == defaultDriveId) {
// The drive id matches our users default drive id
idToQuery = defaultRootId.dup;
log.vdebug("Configuring 'idToQuery' as defaultRootId duplicate");
idToQuery = defaultRootId.dup;
} else {
// The drive id does not match our users default drive id
// Potentially the 'path id' we are requesting the details of is a Shared Folder (remote item)
// Use the 'id' that was passed in (folderId)
idToQuery = id.dup;
log.vdebug("Configuring 'idToQuery' as 'id' duplicate");
idToQuery = id.dup;
}
// what path id are we going to query?
log.vdebug("Path object to query configured as 'idToQuery' = ", idToQuery);
long deltaChanges = 0;
// What query do we use?
// National Cloud Deployments (US and DE) do not support /delta as a query
// National Cloud Deployments do not support /delta as a query
// https://docs.microsoft.com/en-us/graph/deployments#supported-features
// Are we running against a National Cloud Deployments that does not support /delta
if (nationalCloudDeployment) {
// Have to query /children rather than /delta
// National Cloud Deployment that does not support /delta query
// Have to query /children and build our own /delta response
nationalCloudChildrenScan = true;
log.vdebug("Using /children call to query drive for items to populate 'changes' and 'changesAvailable'");
// In a OneDrive Business Shared Folder scenario + nationalCloudDeployment, if ALL items are downgraded, then this leads to local file deletion
// Downgrade ONLY files associated with this driveId and idToQuery
log.vdebug("Downgrading all children for this driveId (" ~ driveId ~ ") and idToQuery (" ~ idToQuery ~ ") to an out-of-sync state");
// Before we get any data, flag any object in the database as out-of-sync for this driveID & ID
auto drivePathChildren = itemdb.selectChildren(driveId, idToQuery);
if (count(drivePathChildren) > 0) {
// Children to process and flag as out-of-sync
foreach (drivePathChild; drivePathChildren) {
// Flag any object in the database as out-of-sync for this driveID & ID
log.vdebug("Downgrading item as out-of-sync: ", drivePathChild.id);
itemdb.downgradeSyncStatusFlag(drivePathChild.driveId, drivePathChild.id);
}
}
@ -1779,6 +1782,9 @@ final class SyncEngine
// In some OneDrive Business scenarios, the shared folder /delta response lacks the 'root' drive details
// When this occurs, this creates the following error: A database statement execution error occurred: foreign key constraint failed
// Ensure we query independently the root details for this shared folder and ensure that it is added before we process the /delta response
// However, if we are using a National Cloud Deployment, these deployments do not support /delta, so we generate a /delta response via generateDeltaResponse()
// This specifically adds the root drive details to the self generated /delta response
if ((!nationalCloudDeployment) && (driveId!= defaultDriveId) && (syncBusinessFolders)) {
// fetch this driveId root details to ensure we add this to the database for this remote drive
JSONValue rootData;
@ -2506,9 +2512,13 @@ final class SyncEngine
SysTime remoteModifiedTime = item.mtime;
remoteModifiedTime.fracSecs = Duration.zero;
if (localModifiedTime != remoteModifiedTime) {
// Database update needed for this item because our record is out-of-date
log.vdebug("Updating local database with item details as timestamps of items are different");
// If the timestamp is different, or we are running on a National Cloud Deployment that does not support /delta queries - we have to update the DB with the details from OneDrive
// Unfortunatly because of the consequence of Nataional Cloud Deployments not supporting /delta queries, the application uses the local database to flag what is out-of-date / track changes
// This means that the constant disk writing to the database fix implemented with https://github.com/abraunegg/onedrive/pull/2004 cannot be utilised when using Nataional Cloud Deployments
// as all records are touched / updated when performing the OneDrive sync operations. The only way to change this, is for Microsoft to support /delta queries for Nataional Cloud Deployments
if ((localModifiedTime != remoteModifiedTime) || (nationalCloudDeployment)) {
// Database update needed for this item because our local record is out-of-date
log.vdebug("Updating local database with item details from OneDrive as local record needs to be updated");
itemdb.update(item);
}
} else {
@ -3198,27 +3208,9 @@ final class SyncEngine
}
// Are we configured to use a National Cloud Deployment
// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
// Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query
if ((nationalCloudDeployment) || (syncBusinessFolders)) {
if (nationalCloudDeployment) {
// Select items that have a out-of-sync flag set
foreach (driveId; driveIDsArray) {
// For each unique OneDrive driveID we know about
Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId);
foreach (item; outOfSyncItems) {
if (!dryRun) {
// clean up idsToDelete
idsToDelete.length = 0;
assumeSafeAppend(idsToDelete);
// flag to delete local file as it now is no longer in sync with OneDrive
log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive");
log.vdebug("item: ", item);
idsToDelete ~= [item.driveId, item.id];
// delete items in idsToDelete
if (idsToDelete.length > 0) deleteItems();
}
}
}
flagNationalCloudDeploymentOutOfSyncItems();
}
// scan for changes in the path provided
@ -3299,27 +3291,9 @@ final class SyncEngine
}
// Are we configured to use a National Cloud Deployment
// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
// Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query
if ((nationalCloudDeployment) || (syncBusinessFolders)) {
if (nationalCloudDeployment) {
// Select items that have a out-of-sync flag set
foreach (driveId; driveIDsArray) {
// For each unique OneDrive driveID we know about
Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId);
foreach (item; outOfSyncItems) {
if (!dryRun) {
// clean up idsToDelete
idsToDelete.length = 0;
assumeSafeAppend(idsToDelete);
// flag to delete local file as it now is no longer in sync with OneDrive
log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive");
log.vdebug("item: ", item);
idsToDelete ~= [item.driveId, item.id];
// delete items in idsToDelete
if (idsToDelete.length > 0) deleteItems();
}
}
}
flagNationalCloudDeploymentOutOfSyncItems();
}
// scan for changes in the path provided
@ -3360,6 +3334,30 @@ final class SyncEngine
}
}
void flagNationalCloudDeploymentOutOfSyncItems() {
// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
// Normally, this is done at the end of processing all /delta queries, however National Cloud Deployments do not support /delta as a query
// https://docs.microsoft.com/en-us/graph/deployments#supported-features
// Select items that have a out-of-sync flag set
foreach (driveId; driveIDsArray) {
// For each unique OneDrive driveID we know about
Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId);
foreach (item; outOfSyncItems) {
if (!dryRun) {
// clean up idsToDelete
idsToDelete.length = 0;
assumeSafeAppend(idsToDelete);
// flag to delete local file as it now is no longer in sync with OneDrive
log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive");
log.vdebug("item: ", item);
idsToDelete ~= [item.driveId, item.id];
// delete items in idsToDelete
if (idsToDelete.length > 0) deleteItems();
}
}
}
}
void handleUploadOnlyBusinessSharedFoldersEdgeCase() {
// read in the business_shared_folders file contents
string[] businessSharedFoldersList;