diff --git a/README.md b/README.md index f05cffee..4b095b14 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,11 @@ This client is a 'fork' of the [skilion](https://github.com/skilion/onedrive) cl * Resumable uploads * Support OneDrive for Business (part of Office 365) * Shared folders (OneDrive Personal) -* SharePoint / Office 365 Shared Libraries (refer to [docs/Office365.md](https://github.com/abraunegg/onedrive/blob/master/docs/Office365.md) to configure) +* SharePoint / Office 365 Shared Libraries * Desktop notifications via libnotify * Dry-run capability to test configuration changes * Prevent major OneDrive accidental data deletion after configuration change +* Support for National cloud deployments (Microsoft Cloud for US Government, Microsoft Cloud Germany, Azure and Office 365 operated by 21Vianet in China) ## What's missing * While local changes are uploaded right away, remote changes are delayed until next sync when using --monitor @@ -36,9 +37,12 @@ See [docs/USAGE.md](https://github.com/abraunegg/onedrive/blob/master/docs/USAGE ## Docker support See [docs/Docker.md](https://github.com/abraunegg/onedrive/blob/master/docs/Docker.md) -## Sharepoint group drive in Office 365 business or education +## SharePoint / Office 365 Shared Libraries (Business or Education) See [docs/Office365.md](https://github.com/abraunegg/onedrive/blob/master/docs/Office365.md) +## National Cloud support +See [docs/national-cloud-deployments.md](https://github.com/abraunegg/onedrive/blob/master/docs/national-cloud-deployments.md) + ## Reporting issues If you encounter any bugs you can report them here on Github. Before filing an issue be sure to: diff --git a/docs/Office365.md b/docs/Office365.md index 990de5bd..f61522f1 100644 --- a/docs/Office365.md +++ b/docs/Office365.md @@ -15,7 +15,7 @@ URL: ``` ## Configuring the onedrive client -Once you have obtained the 'drive_id' above, add to your 'onedrive' configuration file (`~/.config/onedrive/config`)the following: +Once you have obtained the 'drive_id' above, add to your 'onedrive' configuration file (`~/.config/onedrive/config`) the following: ```text drive_id = "insert the drive id from above here" ``` diff --git a/docs/images/application_registration.jpg b/docs/images/application_registration.jpg new file mode 100644 index 00000000..0dfbfda9 Binary files /dev/null and b/docs/images/application_registration.jpg differ diff --git a/docs/images/application_registration_done.jpg b/docs/images/application_registration_done.jpg new file mode 100644 index 00000000..e9559d60 Binary files /dev/null and b/docs/images/application_registration_done.jpg differ diff --git a/docs/images/authentication_response_uri.jpg b/docs/images/authentication_response_uri.jpg new file mode 100644 index 00000000..79dc32fa Binary files /dev/null and b/docs/images/authentication_response_uri.jpg differ diff --git a/docs/images/authentication_scopes.jpg b/docs/images/authentication_scopes.jpg new file mode 100644 index 00000000..ae3e5870 Binary files /dev/null and b/docs/images/authentication_scopes.jpg differ diff --git a/docs/national-cloud-deployments.md b/docs/national-cloud-deployments.md new file mode 100644 index 00000000..2edec13c --- /dev/null +++ b/docs/national-cloud-deployments.md @@ -0,0 +1,113 @@ +# How to configure access to specific Microsoft Azure deployments +In some cases it is a requirement to utilise specific Microsoft Azure cloud deployments to conform with data and security reuqirements that requires data to reside within the geographic borders of that country. +Current national clouds that are supported are: +* Microsoft Cloud for US Government +* Microsoft Cloud Germany +* Azure and Office 365 operated by 21Vianet in China + +In order to sucessfully use these specific Microsoft Azure deployments, the following steps are required: +1. Register an application with the Microsoft identity platform using the Azure portal +2. Configure the new application with the appropriate authentication scopes +3. Validate that the authentication / redirect URI is correct for your application registration +4. Configure the onedrive client to use the new application id as provided during application registration +5. Configure the onedrive client to use the right Microsoft Azure deployment region that your application was registered with +6. Authenticate the client + +## Step 1: Register a new application with Microsoft Azure +1. Log into [Microsoft Azure](https://portal.azure.com/) with your applicable identity +2. Select 'Azure Active Directory' as the service you wish to configure +3. Under 'Manage', select 'App registrations' to register a new application +4. Click 'New registration' +5. Type in the appropriate details required as per below: + +![application_registration](./images/application_registration.jpg) + +6. To save the application registration, click 'Register' and something similar to the following will be displayed: + +![application_registration_done](./images/application_registration_done.jpg) + +**Note:** The Application (client) ID UUID as displayed after client registration, is what is required as the 'application_id' for Step 4 below. + +## Step 2: Configure application authentication scopes +Configure the API permissions as per the following: + +| API / Permissions name | Type | Description | Admin consent required | +|---|---|---|---| +| Files.ReadWrite | Delegated | Have full access to user files | No | +| Files.ReadWrite.All | Delegated | Have full access to all files user can access | No | +| offline_access | Delegated | Maintain access to data you have given it access to | No | +| Sites.Read.All | Delegated | Read items in all site collections | No | +| Sites.ReadWrite.All | Delegated | Edit or delete items in all site collections | No | + +![authentication_scopes](./images/authentication_scopes.jpg) + +## Step 3: Validate that the authentication / redirect URI is correct +Add the appropriate redirect URI for your Azure deployment: + +![authentication_response_uri](./images/authentication_response_uri.jpg) + +A valid entry for the response URI should be one of: +* https://login.microsoftonline.us/common/oauth2/nativeclient (Microsoft Cloud for US Government) +* https://login.microsoftonline.de/common/oauth2/nativeclient (Microsoft Cloud Germany) +* https://login.chinacloudapi.cn/common/oauth2/nativeclient (Azure and Office 365 operated by 21Vianet in China) + +## Step 4: Configure the onedrive client to use new application registration +Update to your 'onedrive' configuration file (`~/.config/onedrive/config`) the following: +```text +application_id = "insert valid entry here" +``` + +This will reconfigure the client to use the new application registration you have created. + +**Example:** +```text +application_id = "22c49a0d-d21c-4792-aed1-8f163c982546" +``` + +## Step 5: Confgure the onedrive client to use the specific Microsoft Azure deployment +Update to your 'onedrive' configuration file (`~/.config/onedrive/config`) the following: +```text +azure_ad_endpoint = "insert valid entry here" +``` + +Valid entries are: +* USL4 (Microsoft Cloud for US Government) +* USL5 (Microsoft Cloud for US Government - DOD) +* DE (Microsoft Cloud Germany) +* CN (Azure and Office 365 operated by 21Vianet in China) + +This will configure your client to use the correct Azure AD and Graph endpoints as per [https://docs.microsoft.com/en-us/graph/deployments](https://docs.microsoft.com/en-us/graph/deployments) + +**Example:** +```text +azure_ad_endpoint = "USL4" +``` + +## Step 6: Authenticate the client +Run the application without any additional command switches. + +You will be asked to open a specific URL by using your web browser where you will have to login into your Microsoft Account and give the application the permission to access your files. After giving permission to the application, you will be redirected to a blank page. Copy the URI of the blank page into the application. +```text +[user@hostname ~]$ onedrive + +Authorize this app visiting: + +https://..... + +Enter the response uri: + +``` + +**Example:** +``` +[user@hostname ~]$ onedrive +Authorize this app visiting: + +https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=22c49a0d-d21c-4792-aed1-8f163c982546&scope=Files.ReadWrite%20Files.ReadWrite.all%20Sites.ReadWrite.All%20offline_access&response_type=code&redirect_uri=https://login.microsoftonline.com/common/oauth2/nativeclient + +Enter the response uri: https://login.microsoftonline.com/common/oauth2/nativeclient?code= + +Application has been successfully authorised, however no additional command switches were provided. + +Please use --help for further assistance in regards to running this application. +``` diff --git a/src/config.d b/src/config.d index 050b5822..437b4d97 100644 --- a/src/config.d +++ b/src/config.d @@ -84,6 +84,22 @@ final class Config // Ignore data safety checks and overwrite local data rather than preserve & rename // This is a config file option ONLY boolValues["bypass_data_preservation"] = false; + // Support National Azure AD endpoints as per https://docs.microsoft.com/en-us/graph/deployments + // By default, if empty, use standard Azure AD URL's + // Will support the following options: + // - USL4 + // AD Endpoint: https://login.microsoftonline.us + // Graph Endpoint: https://graph.microsoft.us + // - USL5 + // AD Endpoint: https://login.microsoftonline.us + // Graph Endpoint: https://dod-graph.microsoft.us + // - DE + // AD Endpoint: https://portal.microsoftazure.de + // Graph Endpoint: https://graph.microsoft.de + // - CN + // AD Endpoint: https://login.chinacloudapi.cn + // Graph Endpoint: https://microsoftgraph.chinacloudapi.cn + stringValues["azure_ad_endpoint"] = ""; // DEVELOPER OPTIONS // display_memory = true | false @@ -512,6 +528,30 @@ final class Config if (key == "sync_dir") configFileSyncDir = c.front.dup; if (key == "skip_file") configFileSkipFile = c.front.dup; if (key == "skip_dir") configFileSkipDir = c.front.dup; + // Azure AD Configuration + if (key == "azure_ad_endpoint") { + string azureConfigValue = c.front.dup; + switch(azureConfigValue) { + case "": + log.log("Using config option for Global Azure AD Endpoints"); + break; + case "USL4": + log.log("Using config option for Azure AD for US Government Endpoints"); + break; + case "USL5": + log.log("Using config option for Azure AD for US Government Endpoints (DOD)"); + break; + case "DE": + log.log("Using config option for Azure AD Germany"); + break; + case "CN": + log.log("Using config option for Azure AD China operated by 21Vianet"); + break; + // Default - all other entries + default: + log.log("Unknown Azure AD Endpoint - using Global Azure AD Endpoints"); + } + } } else { auto ppp = key in longValues; if (ppp) { diff --git a/src/itemdb.d b/src/itemdb.d index a8e93807..90568790 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -26,12 +26,13 @@ struct Item { string quickXorHash; string remoteDriveId; string remoteId; + string syncStatus; } final class ItemDatabase { // increment this for every change in the db schema - immutable int itemDatabaseVersion = 9; + immutable int itemDatabaseVersion = 10; Database db; string insertItemStmt; @@ -83,12 +84,12 @@ final class ItemDatabase db.exec("PRAGMA auto_vacuum = FULL"); insertItemStmt = " - INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13) + INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId, syncStatus) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14) "; updateItemStmt = " UPDATE item - SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, crc32Hash = ?9, sha1Hash = ?10, quickXorHash = ?11, remoteDriveId = ?12, remoteId = ?13 + SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, crc32Hash = ?9, sha1Hash = ?10, quickXorHash = ?11, remoteDriveId = ?12, remoteId = ?13, syncStatus = ?14 WHERE driveId = ?1 AND id = ?2 "; selectItemByIdStmt = " @@ -117,6 +118,7 @@ final class ItemDatabase remoteDriveId TEXT, remoteId TEXT, deltaLink TEXT, + syncStatus TEXT, PRIMARY KEY (driveId, id), FOREIGN KEY (driveId, parentId) REFERENCES item (driveId, id) @@ -300,13 +302,14 @@ final class ItemDatabase bind(11, quickXorHash); bind(12, remoteDriveId); bind(13, remoteId); + bind(14, syncStatus); } } private Item buildItem(Statement.Result result) { assert(!result.empty, "The result must not be empty"); - assert(result.front.length == 14, "The result must have 14 columns"); + assert(result.front.length == 15, "The result must have 15 columns"); Item item = { driveId: result.front[0].dup, id: result.front[1].dup, @@ -319,7 +322,8 @@ final class ItemDatabase sha1Hash: result.front[9].dup, quickXorHash: result.front[10].dup, remoteDriveId: result.front[11].dup, - remoteId: result.front[12].dup + remoteId: result.front[12].dup, + syncStatus: result.front[14].dup }; switch (result.front[3]) { case "file": item.type = ItemType.file; break; @@ -417,4 +421,28 @@ final class ItemDatabase stmt.bind(3, deltaLink); stmt.exec(); } + + // National Cloud Deployments (US and DE) do not support /delta as a query + // We need to track in the database that this item is in sync + // As we query /children to get all children from OneDrive, update anything in the database + // to be flagged as not-in-sync, thus, we can use that flag to determing what was previously + // in-sync, but now deleted on OneDrive + void downgradeSyncStatusFlag() + { + db.exec("UPDATE item SET syncStatus = 'N'"); + } + + // National Cloud Deployments (US and DE) do not support /delta as a query + // Select items that have a out-of-sync flag set + Item[] selectOutOfSyncItems() + { + Item[] items; + auto stmt = db.prepare("SELECT * FROM item WHERE syncStatus = 'N'"); + auto res = stmt.exec(); + while (!res.empty) { + items ~= buildItem(res); + res.step(); + } + return items; + } } diff --git a/src/main.d b/src/main.d index ddad65f1..878d4b45 100644 --- a/src/main.d +++ b/src/main.d @@ -681,6 +681,15 @@ int main(string[] args) sync.setBypassDataPreservation(); } + // Are we configured to use a National Cloud Deployment + if (cfg.getValueString("azure_ad_endpoint") != "") { + // value is configured, is it a valid value? + if ((cfg.getValueString("azure_ad_endpoint") == "USL4") || (cfg.getValueString("azure_ad_endpoint") == "USL5") || (cfg.getValueString("azure_ad_endpoint") == "DE") || (cfg.getValueString("azure_ad_endpoint") == "CN")) { + // valid entries to flag we are using a National Cloud Deployment + sync.setNationalCloudDeployment(); + } + } + // Do we need to validate the syncDir to check for the presence of a '.nosync' file if (cfg.getValueBool("check_nomount")) { // we were asked to check the mounts diff --git a/src/onedrive.d b/src/onedrive.d index 0412a97b..1d68e983 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -15,15 +15,26 @@ private bool simulateNoRefreshTokenFile = false; private ulong retryAfterValue = 0; private immutable { - // Personal & Business Queries - string authUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize"; - string redirectUrl = "https://login.microsoftonline.com/common/oauth2/nativeclient"; - string tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token"; - string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/"; + // Azure Active Directory & Graph Explorer Endpoints + // Global & Defaults + string globalAuthEndpoint = "https://login.microsoftonline.com"; + string globalGraphEndpoint = "https://graph.microsoft.com"; - // Office 365 / SharePoint Queries - string siteSearchUrl = "https://graph.microsoft.com/v1.0/sites?search"; - string siteDriveUrl = "https://graph.microsoft.com/v1.0/sites/"; + // US Government L4 + string usl4AuthEndpoint = "https://login.microsoftonline.us"; + string usl4GraphEndpoint = "https://graph.microsoft.us"; + + // US Government L5 + string usl5AuthEndpoint = "https://login.microsoftonline.us"; + string usl5GraphEndpoint = "https://dod-graph.microsoft.us"; + + // Germany + string deAuthEndpoint = "https://login.microsoftonline.de"; + string deGraphEndpoint = "https://graph.microsoft.de"; + + // China + string cnAuthEndpoint = "https://login.chinacloudapi.cn"; + string cnGraphEndpoint = "https://microsoftgraph.chinacloudapi.cn"; } private { @@ -38,11 +49,24 @@ private { // Default Drive ID string driveId = ""; - - // Common URL's - string driveUrl = "https://graph.microsoft.com/v1.0/me/drive"; - string itemByIdUrl = "https://graph.microsoft.com/v1.0/me/drive/items/"; - string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; + + // API Query URL's, based on using defaults, but can be updated by config option 'azure_ad_endpoint' + // Authentication + string authUrl = globalAuthEndpoint ~ "/common/oauth2/v2.0/authorize"; + string redirectUrl = globalAuthEndpoint ~ "/common/oauth2/nativeclient"; + string tokenUrl = globalAuthEndpoint ~ "/common/oauth2/v2.0/token"; + + // Drive Queries + string driveUrl = globalGraphEndpoint ~ "/v1.0/me/drive"; + string driveByIdUrl = globalGraphEndpoint ~ "/v1.0/drives/"; + + // Item Queries + string itemByIdUrl = globalGraphEndpoint ~ "/v1.0/me/drive/items/"; + string itemByPathUrl = globalGraphEndpoint ~ "/v1.0/me/drive/root:/"; + + // Office 365 / SharePoint Queries + string siteSearchUrl = globalGraphEndpoint ~ "/v1.0/sites?search"; + string siteDriveUrl = globalGraphEndpoint ~ "/v1.0/sites/"; } class OneDriveException: Exception @@ -110,7 +134,82 @@ final class OneDriveApi http.verbose = true; .debugResponse = true; } - + + // Configure Azure AD endpoints if 'azure_ad_endpoint' is configured + string azureConfigValue = cfg.getValueString("azure_ad_endpoint"); + switch(azureConfigValue) { + case "": + log.log("Configuring Global Azure AD Endpoints"); + break; + case "USL4": + log.log("Configuring Azure AD for US Government Endpoints"); + // Authentication + authUrl = usl4AuthEndpoint ~ "/common/oauth2/v2.0/authorize"; + redirectUrl = usl4AuthEndpoint ~ "/common/oauth2/nativeclient"; + tokenUrl = usl4AuthEndpoint ~ "/common/oauth2/v2.0/token"; + // Drive Queries + driveUrl = usl4GraphEndpoint ~ "/v1.0/me/drive"; + driveByIdUrl = usl4GraphEndpoint ~ "/v1.0/drives/"; + // Item Queries + itemByIdUrl = usl4GraphEndpoint ~ "/v1.0/me/drive/items/"; + itemByPathUrl = usl4GraphEndpoint ~ "/v1.0/me/drive/root:/"; + // Office 365 / SharePoint Queries + siteSearchUrl = usl4GraphEndpoint ~ "/v1.0/sites?search"; + siteDriveUrl = usl4GraphEndpoint ~ "/v1.0/sites/"; + break; + case "USL5": + log.log("Configuring Azure AD for US Government Endpoints (DOD)"); + // Authentication + authUrl = usl5AuthEndpoint ~ "/common/oauth2/v2.0/authorize"; + redirectUrl = usl5AuthEndpoint ~ "/common/oauth2/nativeclient"; + tokenUrl = usl5AuthEndpoint ~ "/common/oauth2/v2.0/token"; + // Drive Queries + driveUrl = usl5GraphEndpoint ~ "/v1.0/me/drive"; + driveByIdUrl = usl5GraphEndpoint ~ "/v1.0/drives/"; + // Item Queries + itemByIdUrl = usl5GraphEndpoint ~ "/v1.0/me/drive/items/"; + itemByPathUrl = usl5GraphEndpoint ~ "/v1.0/me/drive/root:/"; + // Office 365 / SharePoint Queries + siteSearchUrl = usl5GraphEndpoint ~ "/v1.0/sites?search"; + siteDriveUrl = usl5GraphEndpoint ~ "/v1.0/sites/"; + break; + case "DE": + log.log("Configuring Azure AD Germany"); + // Authentication + authUrl = deAuthEndpoint ~ "/common/oauth2/v2.0/authorize"; + redirectUrl = deAuthEndpoint ~ "/common/oauth2/nativeclient"; + tokenUrl = deAuthEndpoint ~ "/common/oauth2/v2.0/token"; + // Drive Queries + driveUrl = deGraphEndpoint ~ "/v1.0/me/drive"; + driveByIdUrl = deGraphEndpoint ~ "/v1.0/drives/"; + // Item Queries + itemByIdUrl = deGraphEndpoint ~ "/v1.0/me/drive/items/"; + itemByPathUrl = deGraphEndpoint ~ "/v1.0/me/drive/root:/"; + // Office 365 / SharePoint Queries + siteSearchUrl = deGraphEndpoint ~ "/v1.0/sites?search"; + siteDriveUrl = deGraphEndpoint ~ "/v1.0/sites/"; + break; + case "CN": + log.log("Configuring AD China operated by 21Vianet"); + // Authentication + authUrl = cnAuthEndpoint ~ "/common/oauth2/v2.0/authorize"; + redirectUrl = cnAuthEndpoint ~ "/common/oauth2/nativeclient"; + tokenUrl = cnAuthEndpoint ~ "/common/oauth2/v2.0/token"; + // Drive Queries + driveUrl = cnGraphEndpoint ~ "/v1.0/me/drive"; + driveByIdUrl = cnGraphEndpoint ~ "/v1.0/drives/"; + // Item Queries + itemByIdUrl = cnGraphEndpoint ~ "/v1.0/me/drive/items/"; + itemByPathUrl = cnGraphEndpoint ~ "/v1.0/me/drive/root:/"; + // Office 365 / SharePoint Queries + siteSearchUrl = cnGraphEndpoint ~ "/v1.0/sites?search"; + siteDriveUrl = cnGraphEndpoint ~ "/v1.0/sites/"; + break; + // Default - all other entries + default: + log.log("Unknown Azure AD Endpoint request - using Global Azure AD Endpoints"); + } + // Configure the User Agent string if (cfg.getValueString("user_agent") == "") { // Application User Agent string defaults @@ -308,7 +407,22 @@ final class OneDriveApi } return get(url); } - + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_list_children + JSONValue listChildren(const(char)[] driveId, const(char)[] id, const(char)[] nextLink) + { + checkAccessTokenExpired(); + const(char)[] url; + // configure URL to query + if (nextLink.empty) { + url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/children"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; + } else { + url = nextLink; + } + return get(url); + } + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get_content void downloadById(const(char)[] driveId, const(char)[] id, string saveToPath, long fileSize) { @@ -375,7 +489,6 @@ final class OneDriveApi { checkAccessTokenExpired(); const(char)[] url; - // string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; if ((path == ".")||(path == "/")) url = driveUrl ~ "/root/"; else url = itemByPathUrl ~ encodeComponent(path) ~ ":/"; url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; @@ -388,7 +501,6 @@ final class OneDriveApi { checkAccessTokenExpired(); const(char)[] url; - // string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/"; url = driveByIdUrl ~ driveId ~ "/items/" ~ id; url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; return get(url); @@ -400,7 +512,6 @@ final class OneDriveApi { checkAccessTokenExpired(); const(char)[] url; - // string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/"; url = driveByIdUrl ~ driveId ~ "/items/" ~ id; url ~= "?select=size,malware,file,webUrl"; return get(url); diff --git a/src/sync.d b/src/sync.d index 138ad1cd..41b025a1 100644 --- a/src/sync.d +++ b/src/sync.d @@ -171,6 +171,13 @@ private Item makeItem(const ref JSONValue driveItem) item.remoteDriveId = driveItem["remoteItem"]["parentReference"]["driveId"].str; item.remoteId = driveItem["remoteItem"]["id"].str; } + + // National Cloud Deployments (US and DE) do not support /delta as a query + // Thus we need to track in the database that this item is in sync + // As we are making an item, set the syncStatus to Y + // ONLY when using a National Cloud Deployment, all the existing DB entries will get set to N + // so when processing /children, it can be identified what the 'deleted' difference is + item.syncStatus = "Y"; return item; } @@ -241,6 +248,8 @@ final class SyncEngine // is bypass_data_preservation set via config file // Local data loss MAY occur in this scenario private bool bypassDataPreservation = false; + // is National Cloud Deployments configured + private bool nationalCloudDeployment = false; this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync) { @@ -492,6 +501,13 @@ final class SyncEngine log.vdebug("Setting bypassDataPreservation = true"); } + // set nationalCloudDeployment to true + void setNationalCloudDeployment() + { + nationalCloudDeployment = true; + log.vdebug("Setting nationalCloudDeployment = true"); + } + // download all new changes from OneDrive void applyDifferences(bool performFullItemScan) { @@ -715,6 +731,7 @@ final class SyncEngine string syncFolderChildPath; string deltaLink; string deltaLinkAvailable; + bool nationalCloudChildrenScan = false; // Query the name of this folder id try { @@ -925,207 +942,281 @@ final class SyncEngine } // what path id are we going to query? log.vdebug("path idToQuery = ", idToQuery); - - // query for changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink); - try { - // Fetch the changes relative to the path id we want to query - // changes with or without deltaLink - changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink); - if (changes.type() == JSONType.object) { - log.vdebug("Query 'changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink)' performed successfully"); - } - } catch (OneDriveException e) { - // OneDrive threw an error - log.vdebug("------------------------------------------------------------------"); - log.vdebug("Query Error: changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink)"); - log.vdebug("driveId: ", driveId); - log.vdebug("idToQuery: ", idToQuery); - log.vdebug("deltaLink: ", deltaLink); - - // HTTP request returned status code 404 (Not Found) - if (e.httpStatusCode == 404) { - // Stop application - log.log("\n\nOneDrive returned a 'HTTP 404 - Item not found'"); - log.log("The item id to query was not found on OneDrive"); - log.log("\nRemove your '", cfg.databaseFilePath, "' file and try to sync again\n"); - return; - } - - // HTTP request returned status code 410 (The requested resource is no longer available at the server) - if (e.httpStatusCode == 410) { - log.vdebug("Delta link expired for 'onedrive.viewChangesById(driveId, idToQuery, deltaLink)', setting 'deltaLink = null'"); - deltaLink = null; - continue; - } - - // HTTP request returned status code 429 (Too Many Requests) - if (e.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(); - log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query changes from OneDrive using deltaLink"); - } - - // HTTP request returned status code 500 (Internal Server Error) - if (e.httpStatusCode == 500) { - // display what the error is - displayOneDriveErrorMessage(e.msg); - return; - } - - // HTTP request returned status code 504 (Gateway Timeout) or 429 retry - if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) { - // If an error is returned when querying 'changes' and we recall the original function, we go into a never ending loop where the sync never ends - // re-try the specific changes queries - if (e.httpStatusCode == 504) { - log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query for changes - retrying applicable request"); - log.vdebug("changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink) previously threw an error - retrying"); - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); - Thread.sleep(dur!"seconds"(30)); - log.vdebug("Retrying Query - using original deltaLink after delay"); - } - // re-try original request - retried for 429 and 504 - try { - log.vdebug("Retrying Query: changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink)"); - changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink); - log.vdebug("Query 'changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink)' performed successfully on re-try"); - } catch (OneDriveException e) { - // display what the error is - log.vdebug("Query Error: changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink) on re-try after delay"); - if (e.httpStatusCode == 504) { - log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query for changes - retrying applicable request"); - log.vdebug("changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink) previously threw an error - retrying with empty deltaLink"); - try { - // try query with empty deltaLink value - deltaLink = null; - changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink); - log.vdebug("Query 'changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink)' performed successfully on re-try"); - } catch (OneDriveException e) { - // Tried 3 times, give up - displayOneDriveErrorMessage(e.msg); - return; - } - } else { - // error was not a 504 this time - displayOneDriveErrorMessage(e.msg); - return; - } - } - } else { - // Default operation if not 404, 410, 429, 500 or 504 errors - // display what the error is - displayOneDriveErrorMessage(e.msg); - return; - } - } - - // query for changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable); - try { - // Fetch the changes relative to the path id we want to query - // changes based on deltaLink - changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable); - if (changesAvailable.type() == JSONType.object) { - log.vdebug("Query 'changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable)' performed successfully"); - } - } catch (OneDriveException e) { - // OneDrive threw an error - log.vdebug("------------------------------------------------------------------"); - log.vdebug("Query Error: changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable)"); - log.vdebug("driveId: ", driveId); - log.vdebug("idToQuery: ", idToQuery); - log.vdebug("deltaLink: ", deltaLink); - - // HTTP request returned status code 404 (Not Found) - if (e.httpStatusCode == 404) { - // Stop application - log.log("\n\nOneDrive returned a 'HTTP 404 - Item not found'"); - log.log("The item id to query was not found on OneDrive"); - log.log("\nRemove your '", cfg.databaseFilePath, "' file and try to sync again\n"); - return; - } - - // HTTP request returned status code 410 (The requested resource is no longer available at the server) - if (e.httpStatusCode == 410) { - log.vdebug("Delta link expired for 'onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable)', setting 'deltaLinkAvailable = null'"); - deltaLinkAvailable = null; - continue; - } - - // HTTP request returned status code 429 (Too Many Requests) - if (e.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(); - log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query changes from OneDrive using deltaLink"); - } - - // HTTP request returned status code 500 (Internal Server Error) - if (e.httpStatusCode == 500) { - // display what the error is - displayOneDriveErrorMessage(e.msg); - return; - } - - // HTTP request returned status code 504 (Gateway Timeout) or 429 retry - if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) { - // If an error is returned when querying 'changes' and we recall the original function, we go into a never ending loop where the sync never ends - // re-try the specific changes queries - if (e.httpStatusCode == 504) { - log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query for changes - retrying applicable request"); - log.vdebug("changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable) previously threw an error - retrying"); - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); - Thread.sleep(dur!"seconds"(30)); - log.vdebug("Retrying Query - using original deltaLink after delay"); - } - // re-try original request - retried for 429 and 504 - try { - log.vdebug("Retrying Query: changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable)"); - changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable); - log.vdebug("Query 'changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable)' performed successfully on re-try"); - } catch (OneDriveException e) { - // display what the error is - log.vdebug("Query Error: changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable) on re-try after delay"); - if (e.httpStatusCode == 504) { - log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query for changes - retrying applicable request"); - log.vdebug("changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable) previously threw an error - retrying with empty deltaLinkAvailable"); - try { - // try query with empty deltaLinkAvailable value - deltaLinkAvailable = null; - changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable); - log.vdebug("Query 'changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable)' performed successfully on re-try"); - } catch (OneDriveException e) { - // Tried 3 times, give up - displayOneDriveErrorMessage(e.msg); - return; - } - } else { - // error was not a 504 this time - displayOneDriveErrorMessage(e.msg); - return; - } - } - } else { - // Default operation if not 404, 410, 429, 500 or 504 errors - // display what the error is - displayOneDriveErrorMessage(e.msg); - return; - } - } - - // is changesAvailable a valid JSON response long deltaChanges = 0; - if (changesAvailable.type() == JSONType.object) { - // are there any delta changes? - if (("value" in changesAvailable) != null) { - deltaChanges = count(changesAvailable["value"].array); - log.vdebug("changesAvailable query reports that there are " , deltaChanges , " changes that need processing on OneDrive"); + + // What query do we use? + // National Cloud Deployments (US and DE) do not support /delta as a query + // https://docs.microsoft.com/en-us/graph/deployments#supported-features + // Are we running against a National Cloud Deployments that does not support /delta + if (nationalCloudDeployment) { + // have to query /children rather than /delta + nationalCloudChildrenScan = true; + // Before we get any data, flag any object in the database as out of sync + itemdb.downgradeSyncStatusFlag(); + try { + // we have to 'build' our own JSON response that looks like /delta + changes = generateDeltaResponse(driveId, idToQuery); + if (changes.type() == JSONType.object) { + log.vdebug("Query 'changes = generateDeltaResponse(driveId, idToQuery)' performed successfully"); + } + } catch (OneDriveException e) { + // OneDrive threw an error + log.vdebug("------------------------------------------------------------------"); + log.vdebug("Query Error: changes = generateDeltaResponse(driveId, idToQuery)"); + log.vdebug("driveId: ", driveId); + log.vdebug("idToQuery: ", idToQuery); + + // HTTP request returned status code 404 (Not Found) + if (e.httpStatusCode == 404) { + // Stop application + log.log("\n\nOneDrive returned a 'HTTP 404 - Item not found'"); + log.log("The item id to query was not found on OneDrive"); + log.log("\nRemove your '", cfg.databaseFilePath, "' file and try to sync again\n"); + return; + } + + // HTTP request returned status code 429 (Too Many Requests) + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive items"); + } + + // HTTP request returned status code 500 (Internal Server Error) + if (e.httpStatusCode == 500) { + // display what the error is + displayOneDriveErrorMessage(e.msg); + return; + } + + // HTTP request returned status code 504 (Gateway Timeout) or 429 retry + if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) { + // If an error is returned when querying 'changes' and we recall the original function, we go into a never ending loop where the sync never ends + // re-try the specific changes queries + if (e.httpStatusCode == 504) { + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query OneDrive drive items - retrying applicable request"); + log.vdebug("changes = generateDeltaResponse(driveId, idToQuery) previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + log.vdebug("Retrying Query - using original deltaLink after delay"); + } + // re-try original request - retried for 429 and 504 + try { + log.vdebug("Retrying Query: changes = generateDeltaResponse(driveId, idToQuery)"); + changes = generateDeltaResponse(driveId, idToQuery); + log.vdebug("Query 'changes = generateDeltaResponse(driveId, idToQuery)' performed successfully on re-try"); + } catch (OneDriveException e) { + // display what the error is + log.vdebug("Query Error: changes = generateDeltaResponse(driveId, idToQuery) on re-try after delay"); + // error was not a 504 this time + displayOneDriveErrorMessage(e.msg); + return; + } + } else { + // Default operation if not 404, 410, 429, 500 or 504 errors + // display what the error is + displayOneDriveErrorMessage(e.msg); + return; + } + } + } else { + // query for changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink); + try { + // Fetch the changes relative to the path id we want to query + // changes with or without deltaLink + changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink); + if (changes.type() == JSONType.object) { + log.vdebug("Query 'changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink)' performed successfully"); + } + } catch (OneDriveException e) { + // OneDrive threw an error + log.vdebug("------------------------------------------------------------------"); + log.vdebug("Query Error: changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink)"); + log.vdebug("driveId: ", driveId); + log.vdebug("idToQuery: ", idToQuery); + log.vdebug("deltaLink: ", deltaLink); + + // HTTP request returned status code 404 (Not Found) + if (e.httpStatusCode == 404) { + // Stop application + log.log("\n\nOneDrive returned a 'HTTP 404 - Item not found'"); + log.log("The item id to query was not found on OneDrive"); + log.log("\nRemove your '", cfg.databaseFilePath, "' file and try to sync again\n"); + return; + } + + // HTTP request returned status code 410 (The requested resource is no longer available at the server) + if (e.httpStatusCode == 410) { + log.vdebug("Delta link expired for 'onedrive.viewChangesById(driveId, idToQuery, deltaLink)', setting 'deltaLink = null'"); + deltaLink = null; + continue; + } + + // HTTP request returned status code 429 (Too Many Requests) + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query changes from OneDrive using deltaLink"); + } + + // HTTP request returned status code 500 (Internal Server Error) + if (e.httpStatusCode == 500) { + // display what the error is + displayOneDriveErrorMessage(e.msg); + return; + } + + // HTTP request returned status code 504 (Gateway Timeout) or 429 retry + if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) { + // If an error is returned when querying 'changes' and we recall the original function, we go into a never ending loop where the sync never ends + // re-try the specific changes queries + if (e.httpStatusCode == 504) { + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query for changes - retrying applicable request"); + log.vdebug("changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink) previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + log.vdebug("Retrying Query - using original deltaLink after delay"); + } + // re-try original request - retried for 429 and 504 + try { + log.vdebug("Retrying Query: changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink)"); + changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink); + log.vdebug("Query 'changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink)' performed successfully on re-try"); + } catch (OneDriveException e) { + // display what the error is + log.vdebug("Query Error: changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink) on re-try after delay"); + if (e.httpStatusCode == 504) { + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query for changes - retrying applicable request"); + log.vdebug("changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink) previously threw an error - retrying with empty deltaLink"); + try { + // try query with empty deltaLink value + deltaLink = null; + changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink); + log.vdebug("Query 'changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink)' performed successfully on re-try"); + } catch (OneDriveException e) { + // Tried 3 times, give up + displayOneDriveErrorMessage(e.msg); + return; + } + } else { + // error was not a 504 this time + displayOneDriveErrorMessage(e.msg); + return; + } + } + } else { + // Default operation if not 404, 410, 429, 500 or 504 errors + // display what the error is + displayOneDriveErrorMessage(e.msg); + return; + } + } + + // query for changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable); + try { + // Fetch the changes relative to the path id we want to query + // changes based on deltaLink + changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable); + if (changesAvailable.type() == JSONType.object) { + log.vdebug("Query 'changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable)' performed successfully"); + // are there any delta changes? + if (("value" in changesAvailable) != null) { + deltaChanges = count(changesAvailable["value"].array); + log.vdebug("changesAvailable query reports that there are " , deltaChanges , " changes that need processing on OneDrive"); + } + } + } catch (OneDriveException e) { + // OneDrive threw an error + log.vdebug("------------------------------------------------------------------"); + log.vdebug("Query Error: changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable)"); + log.vdebug("driveId: ", driveId); + log.vdebug("idToQuery: ", idToQuery); + log.vdebug("deltaLink: ", deltaLink); + + // HTTP request returned status code 404 (Not Found) + if (e.httpStatusCode == 404) { + // Stop application + log.log("\n\nOneDrive returned a 'HTTP 404 - Item not found'"); + log.log("The item id to query was not found on OneDrive"); + log.log("\nRemove your '", cfg.databaseFilePath, "' file and try to sync again\n"); + return; + } + + // HTTP request returned status code 410 (The requested resource is no longer available at the server) + if (e.httpStatusCode == 410) { + log.vdebug("Delta link expired for 'onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable)', setting 'deltaLinkAvailable = null'"); + deltaLinkAvailable = null; + continue; + } + + // HTTP request returned status code 429 (Too Many Requests) + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query changes from OneDrive using deltaLink"); + } + + // HTTP request returned status code 500 (Internal Server Error) + if (e.httpStatusCode == 500) { + // display what the error is + displayOneDriveErrorMessage(e.msg); + return; + } + + // HTTP request returned status code 504 (Gateway Timeout) or 429 retry + if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) { + // If an error is returned when querying 'changes' and we recall the original function, we go into a never ending loop where the sync never ends + // re-try the specific changes queries + if (e.httpStatusCode == 504) { + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query for changes - retrying applicable request"); + log.vdebug("changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable) previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + log.vdebug("Retrying Query - using original deltaLink after delay"); + } + // re-try original request - retried for 429 and 504 + try { + log.vdebug("Retrying Query: changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable)"); + changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable); + log.vdebug("Query 'changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable)' performed successfully on re-try"); + } catch (OneDriveException e) { + // display what the error is + log.vdebug("Query Error: changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable) on re-try after delay"); + if (e.httpStatusCode == 504) { + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query for changes - retrying applicable request"); + log.vdebug("changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable) previously threw an error - retrying with empty deltaLinkAvailable"); + try { + // try query with empty deltaLinkAvailable value + deltaLinkAvailable = null; + changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable); + log.vdebug("Query 'changesAvailable = onedrive.viewChangesById(driveId, idToQuery, deltaLinkAvailable)' performed successfully on re-try"); + } catch (OneDriveException e) { + // Tried 3 times, give up + displayOneDriveErrorMessage(e.msg); + return; + } + } else { + // error was not a 504 this time + displayOneDriveErrorMessage(e.msg); + return; + } + } + } else { + // Default operation if not 404, 410, 429, 500 or 504 errors + // display what the error is + displayOneDriveErrorMessage(e.msg); + return; + } } } // is changes a valid JSON response if (changes.type() == JSONType.object) { // Are there any changes to process? - if ((("value" in changes) != null) && ((deltaChanges > 0) || (oneDriveFullScanTrigger))) { + if ((("value" in changes) != null) && ((deltaChanges > 0) || (oneDriveFullScanTrigger) ||(nationalCloudChildrenScan))) { auto nrChanges = count(changes["value"].array); auto changeCount = 0; @@ -2239,6 +2330,27 @@ final class SyncEngine // scan the given directory for differences and new items void scanForDifferences(const(string) path) { + // Are we configured to use a National Cloud Deployment + // Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB + // Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query + if (nationalCloudDeployment) { + // Select items that have a out-of-sync flag set + Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(); + foreach (item; outOfSyncItems) { + if (!dryRun) { + // clean up idsToDelete + idsToDelete.length = 0; + assumeSafeAppend(idsToDelete); + // flag to delete local file as it now is no longer in sync with OneDrive + log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive"); + log.vdebug("item: ", item); + idsToDelete ~= [item.driveId, item.id]; + // delete items in idsToDelete + if (idsToDelete.length > 0) deleteItems(); + } + } + } + // scan for changes in the path provided log.vlog("Uploading differences of ", path); Item item; @@ -2260,6 +2372,27 @@ final class SyncEngine // scan the given directory for differences only - for use with --monitor void scanForDifferencesDatabaseScan(const(string) path) { + // Are we configured to use a National Cloud Deployment + // Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB + // Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query + if (nationalCloudDeployment) { + // Select items that have a out-of-sync flag set + Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(); + foreach (item; outOfSyncItems) { + if (!dryRun) { + // clean up idsToDelete + idsToDelete.length = 0; + assumeSafeAppend(idsToDelete); + // flag to delete local file as it now is no longer in sync with OneDrive + log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive"); + log.vdebug("item: ", item); + idsToDelete ~= [item.driveId, item.id]; + // delete items in idsToDelete + if (idsToDelete.length > 0) deleteItems(); + } + } + } + // scan for changes in the path provided log.vlog("Uploading differences of ", path); Item item; @@ -4684,4 +4817,251 @@ final class SyncEngine // Reset retry-after value to zero as we have used this value now and it may be changed in the future to a different value onedrive.resetRetryAfterValue(); } + + // Generage a /delta compatible response when using National Azure AD deployments that do not support /delta queries + // see: https://docs.microsoft.com/en-us/graph/deployments#supported-features + JSONValue generateDeltaResponse(const(char)[] driveId, const(char)[] idToQuery) { + // JSON value which will be responded with + JSONValue deltaResponse; + // initial data + JSONValue rootData; + JSONValue topLevelChildren; + JSONValue[] childrenData; + string nextLink; + + // Get Default Root + try { + rootData = onedrive.getDefaultRoot(); + } catch (OneDriveException e) { + log.vdebug("oneDriveRootDetails = onedrive.getDefaultRoot() generated a OneDriveException"); + // HTTP request returned status code 504 (Gateway Timeout) or 429 retry + if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + if (e.httpStatusCode == 429) { + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - retrying applicable request"); + handleOneDriveThrottleRequest(); + } + if (e.httpStatusCode == 504) { + log.vdebug("Retrying original request that generated the HTTP 504 (Gateway Timeout) - retrying applicable request"); + Thread.sleep(dur!"seconds"(30)); + } + // Retry original request by calling function again to avoid replicating any further error handling + rootData = onedrive.getDefaultRoot(); + } else { + // There was a HTTP 5xx Server Side Error + displayOneDriveErrorMessage(e.msg); + // Must exit here + exit(-1); + } + } + // add root JSON data to array + log.vlog("Adding OneDrive root details for processing"); + childrenData ~= rootData; + + for (;;) { + // query top level children + try { + topLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink); + } catch (OneDriveException e) { + // OneDrive threw an error + log.vdebug("------------------------------------------------------------------"); + log.vdebug("Query Error: topLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)"); + log.vdebug("driveId: ", driveId); + log.vdebug("idToQuery: ", idToQuery); + log.vdebug("nextLink: ", nextLink); + + // HTTP request returned status code 404 (Not Found) + if (e.httpStatusCode == 404) { + // Stop application + log.log("\n\nOneDrive returned a 'HTTP 404 - Item not found'"); + log.log("The item id to query was not found on OneDrive"); + log.log("\nRemove your '", cfg.databaseFilePath, "' file and try to sync again\n"); + } + + // HTTP request returned status code 429 (Too Many Requests) + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive children"); + } + + // HTTP request returned status code 500 (Internal Server Error) + if (e.httpStatusCode == 500) { + // display what the error is + displayOneDriveErrorMessage(e.msg); + } + + // HTTP request returned status code 504 (Gateway Timeout) or 429 retry + if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) { + // re-try the specific changes queries + if (e.httpStatusCode == 504) { + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query OneDrive drive children - retrying applicable request"); + log.vdebug("topLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink) previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429 and 504 + try { + log.vdebug("Retrying Query: topLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)"); + topLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink); + log.vdebug("Query 'topLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)' performed successfully on re-try"); + } catch (OneDriveException e) { + // display what the error is + log.vdebug("Query Error: topLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink) on re-try after delay"); + // error was not a 504 this time + displayOneDriveErrorMessage(e.msg); + } + } else { + // Default operation if not 404, 410, 429, 500 or 504 errors + // display what the error is + displayOneDriveErrorMessage(e.msg); + } + } + + // process top level children + log.vlog("Adding ", count(topLevelChildren["value"].array), " OneDrive items for processing from OneDrive root"); + foreach (child; topLevelChildren["value"].array) { + // add this child to the array of objects + childrenData ~= child; + // is this child a folder? + if (isItemFolder(child)){ + // We have to query this folders children if childCount > 0 + if (child["folder"]["childCount"].integer > 0){ + // This child folder has children + string childIdToQuery = child["id"].str; + string childDriveToQuery = child["parentReference"]["driveId"].str; + auto childParentPath = child["parentReference"]["path"].str.split(":"); + string folderPathToScan = childParentPath[1] ~ "/" ~ child["name"].str; + JSONValue[] grandChildrenData = queryForChildren(childDriveToQuery, childIdToQuery, folderPathToScan); + foreach (grandChild; grandChildrenData.array) { + // add the grandchild to the array + childrenData ~= grandChild; + } + } + } + } + // If a collection exceeds the default page size (200 items), the @odata.nextLink property is returned in the response + // to indicate more items are available and provide the request URL for the next page of items. + if ("@odata.nextLink" in topLevelChildren) { + // Update nextLink to next changeSet bundle + log.vdebug("Setting nextLink to (@odata.nextLink): ", nextLink); + nextLink = topLevelChildren["@odata.nextLink"].str; + } else break; + } + + // craft response from all returned elements + deltaResponse = [ + "@odata.context": JSONValue("https://graph.microsoft.com/v1.0/$metadata#Collection(driveItem)"), + "value": JSONValue(childrenData.array) + ]; + + // return response + return deltaResponse; + } + + // query child for children + JSONValue[] queryForChildren(const(char)[] driveId, const(char)[] idToQuery, const(char)[] childParentPath) { + // function variables + JSONValue thisLevelChildren; + JSONValue[] thisLevelChildrenData; + string nextLink; + + for (;;) { + // query children + try { + thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink); + } catch (OneDriveException e) { + // OneDrive threw an error + log.vdebug("------------------------------------------------------------------"); + log.vdebug("Query Error: thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)"); + log.vdebug("driveId: ", driveId); + log.vdebug("idToQuery: ", idToQuery); + log.vdebug("nextLink: ", nextLink); + + // HTTP request returned status code 404 (Not Found) + if (e.httpStatusCode == 404) { + // Stop application + log.log("\n\nOneDrive returned a 'HTTP 404 - Item not found'"); + log.log("The item id to query was not found on OneDrive"); + log.log("\nRemove your '", cfg.databaseFilePath, "' file and try to sync again\n"); + } + + // HTTP request returned status code 429 (Too Many Requests) + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive children"); + } + + // HTTP request returned status code 500 (Internal Server Error) + if (e.httpStatusCode == 500) { + // display what the error is + displayOneDriveErrorMessage(e.msg); + } + + // HTTP request returned status code 504 (Gateway Timeout) or 429 retry + if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) { + // re-try the specific changes queries + if (e.httpStatusCode == 504) { + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query OneDrive drive children - retrying applicable request"); + log.vdebug("thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink) previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429 and 504 + try { + log.vdebug("Retrying Query: thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)"); + thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink); + log.vdebug("Query 'thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)' performed successfully on re-try"); + } catch (OneDriveException e) { + // display what the error is + log.vdebug("Query Error: thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink) on re-try after delay"); + // error was not a 504 this time + displayOneDriveErrorMessage(e.msg); + } + } else { + // Default operation if not 404, 410, 429, 500 or 504 errors + // display what the error is + displayOneDriveErrorMessage(e.msg); + } + } + + // process this level children + if (!childParentPath.empty) { + log.vlog("Adding ", count(thisLevelChildren["value"].array), " OneDrive items for processing from ", childParentPath); + } + foreach (child; thisLevelChildren["value"].array) { + // add this child to the array of objects + thisLevelChildrenData ~= child; + // is this child a folder? + if (isItemFolder(child)){ + // We have to query this folders children if childCount > 0 + if (child["folder"]["childCount"].integer > 0){ + // This child folder has children + string childIdToQuery = child["id"].str; + string childDriveToQuery = child["parentReference"]["driveId"].str; + auto grandchildParentPath = child["parentReference"]["path"].str.split(":"); + string folderPathToScan = grandchildParentPath[1] ~ "/" ~ child["name"].str; + JSONValue[] grandChildrenData = queryForChildren(childDriveToQuery, childIdToQuery, folderPathToScan); + foreach (grandChild; grandChildrenData.array) { + // add the grandchild to the array + thisLevelChildrenData ~= grandChild; + } + } + } + } + // If a collection exceeds the default page size (200 items), the @odata.nextLink property is returned in the response + // to indicate more items are available and provide the request URL for the next page of items. + if ("@odata.nextLink" in thisLevelChildren) { + // Update nextLink to next changeSet bundle + log.vdebug("Setting nextLink to (@odata.nextLink): ", nextLink); + nextLink = thisLevelChildren["@odata.nextLink"].str; + } else break; + } + + // return response + return thisLevelChildrenData; + } }