mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-01 13:32:16 +02:00
Sync with shared folders/drives for onedrive business (#206)
* Refactor PR104 for latest code base to enable client use with SharePoint libraries * Implement '--disable-upload-validation' as SharePoint modifies files when uploaded * Enable '--disable-upload-validation' by default if account type is documentLibrary - as most likely this is a SharePoint repository
This commit is contained in:
parent
23e79d39e1
commit
c5ee62efd8
79
README.Office365.md
Normal file
79
README.Office365.md
Normal file
|
@ -0,0 +1,79 @@
|
||||||
|
# Show how to access a Sharepoint group drive in Office 365 business or education
|
||||||
|
## Obtaining the Sharepoint Site Details
|
||||||
|
When we click "sync" in a Sharepoint groups Documents page we get an "odopen" URL, for example:
|
||||||
|
```
|
||||||
|
odopen://sync?userId=97858aa6%2Dcd6a%2D4025%2Da6ee%2D29ba80bcbec5&siteId=%7B0936e7d2%2D1285%2D4c54%2D81f9%2D55e9c8b8613b%7D&webId=%7B345db3b7%2D1ebb%2D484f%2D85e3%2D31c785278051%7D&webTitle=FFCCTT%5FIET&listId=%7BA3913FAC%2D26F8%2D4436%2DA9F6%2DAA6B44776343%7D&listTitle=Documentos&userEmail=georg%2Elehner%40ucan%2Eedu%2Eni&listTemplateTypeId=101&webUrl=https%3A%2F%2Fucanedu%2Esharepoint%2Ecom%2Fsites%2FFFCCTT%5FIET&webLogoUrl=%2Fsites%2FFFCCTT%5FIET%2F%5Fapi%2FGroupService%2FGetGroupImage%3Fid%3D%270e6466dc%2D1d90%2D41b0%2D9c9d%2Df6a818721435%27%26hash%3D636549326262064339&isSiteAdmin=1&webTemplate=64&onPrem=0&scope=OPENLIST
|
||||||
|
```
|
||||||
|
|
||||||
|
When we decode this URL we get the following:
|
||||||
|
|
||||||
|
```
|
||||||
|
userId=97858aa6-cd6a-4025-a6ee-29ba80bcbec5
|
||||||
|
siteId={0936e7d2-1285-4c54-81f9-55e9c8b8613b}
|
||||||
|
webId={345db3b7-1ebb-484f-85e3-31c785278051}
|
||||||
|
webTitle=FFCCTT_IET
|
||||||
|
listId={A3913FAC-26F8-4436-A9F6-AA6B44776343}
|
||||||
|
listTitle=Documentos
|
||||||
|
userEmail=georg.lehner@ucan.edu.ni
|
||||||
|
listTemplateTypeId=101
|
||||||
|
webUrl=https://ucanedu.sharepoint.com/sites/FFCCTT_IET
|
||||||
|
webLogoUrl=/sites/FFCCTT_IET/_api/GroupService/GetGroupImage?id='0e6466dc-1d90-41b0-9c9d-f6a818721435'&hash=636549326262064339
|
||||||
|
isSiteAdmin=1
|
||||||
|
webTemplate=64
|
||||||
|
onPrem=0
|
||||||
|
scope=OPENLIST
|
||||||
|
```
|
||||||
|
|
||||||
|
In the following we refer to these parameters with e.g. $userId - wherever there are enclosing braces {} you have to strip them off.
|
||||||
|
|
||||||
|
## Obtaining the Authorization Token
|
||||||
|
|
||||||
|
To get the 'authorization bearer', run the following command:
|
||||||
|
```
|
||||||
|
onedrive --monitor --print-token
|
||||||
|
```
|
||||||
|
|
||||||
|
This will print out something like the following:
|
||||||
|
```
|
||||||
|
Initializing the Synchronization Engine ...
|
||||||
|
New access token: bearer EwBYA8l6......T51Ag==
|
||||||
|
Initializing monitor ...
|
||||||
|
OneDrive monitor interval (seconds): 45
|
||||||
|
```
|
||||||
|
Everything after 'New access token: bearer' is your authorization token.
|
||||||
|
|
||||||
|
## Obtaining the Drive ID
|
||||||
|
To sync an Office365 / Business Shared Folder we need the Drive ID for that shared folder.
|
||||||
|
|
||||||
|
1. Get site drive-id with:
|
||||||
|
```
|
||||||
|
https://graph.microsoft.com/v1.0/sites/$host,$SiteId,$webId/lists/$listId/drive
|
||||||
|
```
|
||||||
|
**Example:**
|
||||||
|
```
|
||||||
|
curl -i 'https://graph.microsoft.com/v1.0/sites/ucanedu.sharepoint.com,0936e7d2-1285-4c54-81f9-55e9c8b8613b,345db3b7-1ebb-484f-85e3-31c785278051/lists/A3913FAC-26F8-4436-A9F6-AA6B44776343/drive' -H "Authorization: bearer <your_token>"
|
||||||
|
```
|
||||||
|
where $host is the hostname in $webUrl. This should return the following response which is the drive id:
|
||||||
|
|
||||||
|
b!0uc2CYUSVEyB-VXpyLhhO7ezXTS7Hk9IheMxx4UngFGsP5Gj-CY2RKn2qmtEd2ND
|
||||||
|
|
||||||
|
2. Get root folder id via drive-id
|
||||||
|
```
|
||||||
|
GET https://graph.microsoft.com/v1.0/drives/$drive-id/root
|
||||||
|
```
|
||||||
|
**Example:**
|
||||||
|
```
|
||||||
|
curl -i 'https://graph.microsoft.com/v1.0/drives/b!0uc2CYUSVEyB-VXpyLhhO7ezXTS7Hk9IheMxx4UngFGsP5Gj-CY2RKn2qmtEd2ND/root' -H "Authorization: bearer <your_token>"
|
||||||
|
```
|
||||||
|
|
||||||
|
This should return the following response:
|
||||||
|
|
||||||
|
016SJBHDN6Y2GOVW7725BZO354PWSELRRZ
|
||||||
|
|
||||||
|
## Configuring the onedrive client
|
||||||
|
Once you have obtained the 'drive id', add to your 'onedrive' configuration file (`~/.config/onedrive/config`)the following:
|
||||||
|
```
|
||||||
|
drive_id = "insert the drive id from above here"
|
||||||
|
```
|
||||||
|
|
||||||
|
The OneDrive client will now sync this shared 'drive id'
|
50
README.md
50
README.md
|
@ -6,7 +6,8 @@
|
||||||
* Real-Time file monitoring with Inotify
|
* Real-Time file monitoring with Inotify
|
||||||
* Resumable uploads
|
* Resumable uploads
|
||||||
* Support OneDrive for Business (part of Office 365)
|
* Support OneDrive for Business (part of Office 365)
|
||||||
* Shared folders (not Business)
|
* Shared folders (OneDrive Personal)
|
||||||
|
* SharePoint / Office 365 Group Drives (refer to README.Office365.md to configure)
|
||||||
|
|
||||||
### What's missing:
|
### What's missing:
|
||||||
* While local changes are uploaded right away, remote changes are delayed
|
* While local changes are uploaded right away, remote changes are delayed
|
||||||
|
@ -393,29 +394,30 @@ If you encounter any bugs you can report them here on Github. Before filing an i
|
||||||
```text
|
```text
|
||||||
Usage: onedrive [OPTION]...
|
Usage: onedrive [OPTION]...
|
||||||
|
|
||||||
no option No Sync and exit
|
no option No Sync and exit
|
||||||
--check-for-nomount Check for the presence of .nosync in the syncdir root. If found, do not perform sync.
|
--check-for-nomount Check for the presence of .nosync in the syncdir root. If found, do not perform sync.
|
||||||
--confdir Set the directory used to store the configuration files
|
--confdir Set the directory used to store the configuration files
|
||||||
--create-directory Create a directory on OneDrive - no sync will be performed.
|
--create-directory Create a directory on OneDrive - no sync will be performed.
|
||||||
--destination-directory Destination directory for renamed or move on OneDrive - no sync will be performed.
|
--destination-directory Destination directory for renamed or move on OneDrive - no sync will be performed.
|
||||||
--debug-http Debug OneDrive HTTP communication.
|
--debug-http Debug OneDrive HTTP communication.
|
||||||
-d --download Only download remote changes
|
--disable-upload-validation Disable upload validation when uploading to OneDrive
|
||||||
--local-first Synchronize from the local directory source first, before downloading changes from OneDrive.
|
-d --download Only download remote changes
|
||||||
--logout Logout the current user
|
--local-first Synchronize from the local directory source first, before downloading changes from OneDrive.
|
||||||
-m --monitor Keep monitoring for local and remote changes
|
--logout Logout the current user
|
||||||
--no-remote-delete Do not delete local file 'deletes' from OneDrive when using --upload-only
|
-m --monitor Keep monitoring for local and remote changes
|
||||||
--print-token Print the access token, useful for debugging
|
--no-remote-delete Do not delete local file 'deletes' from OneDrive when using --upload-only
|
||||||
--resync Forget the last saved state, perform a full sync
|
--print-token Print the access token, useful for debugging
|
||||||
--remove-directory Remove a directory on OneDrive - no sync will be performed.
|
--resync Forget the last saved state, perform a full sync
|
||||||
--single-directory Specify a single local directory within the OneDrive root to sync.
|
--remove-directory Remove a directory on OneDrive - no sync will be performed.
|
||||||
--skip-symlinks Skip syncing of symlinks
|
--single-directory Specify a single local directory within the OneDrive root to sync.
|
||||||
--source-directory Source directory to rename or move on OneDrive - no sync will be performed.
|
--skip-symlinks Skip syncing of symlinks
|
||||||
--syncdir Set the directory used to sync the files that are synced
|
--source-directory Source directory to rename or move on OneDrive - no sync will be performed.
|
||||||
--synchronize Perform a synchronization
|
--syncdir Set the directory used to sync the files that are synced
|
||||||
--upload-only Only upload to OneDrive, do not sync changes from OneDrive locally
|
--synchronize Perform a synchronization
|
||||||
-v --verbose Print more details, useful for debugging
|
--upload-only Only upload to OneDrive, do not sync changes from OneDrive locally
|
||||||
--version Print the version and exit
|
-v --verbose Print more details, useful for debugging
|
||||||
-h --help This help information.
|
--version Print the version and exit
|
||||||
|
-h --help This help information.
|
||||||
```
|
```
|
||||||
|
|
||||||
### File naming
|
### File naming
|
||||||
|
|
|
@ -83,6 +83,9 @@ int main(string[] args)
|
||||||
bool noRemoteDelete;
|
bool noRemoteDelete;
|
||||||
// Are we able to reach the OneDrive Service
|
// Are we able to reach the OneDrive Service
|
||||||
bool online = false;
|
bool online = false;
|
||||||
|
// Does the user want to disable upload validation - https://github.com/abraunegg/onedrive/issues/205
|
||||||
|
// SharePoint will associate some metadata from the library the file is uploaded to directly in the file - thus change file size & checksums
|
||||||
|
bool disableUploadValidation = false;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
auto opt = getopt(
|
auto opt = getopt(
|
||||||
|
@ -94,6 +97,7 @@ int main(string[] args)
|
||||||
"create-directory", "Create a directory on OneDrive - no sync will be performed.", &createDirectory,
|
"create-directory", "Create a directory on OneDrive - no sync will be performed.", &createDirectory,
|
||||||
"destination-directory", "Destination directory for renamed or move on OneDrive - no sync will be performed.", &destinationDirectory,
|
"destination-directory", "Destination directory for renamed or move on OneDrive - no sync will be performed.", &destinationDirectory,
|
||||||
"debug-https", "Debug OneDrive HTTPS communication.", &debugHttp,
|
"debug-https", "Debug OneDrive HTTPS communication.", &debugHttp,
|
||||||
|
"disable-upload-validation", "Disable upload validation when uploading to OneDrive", &disableUploadValidation,
|
||||||
"download|d", "Only download remote changes", &downloadOnly,
|
"download|d", "Only download remote changes", &downloadOnly,
|
||||||
"local-first", "Synchronize from the local directory source first, before downloading changes from OneDrive.", &localFirst,
|
"local-first", "Synchronize from the local directory source first, before downloading changes from OneDrive.", &localFirst,
|
||||||
"logout", "Logout the current user", &logout,
|
"logout", "Logout the current user", &logout,
|
||||||
|
@ -262,6 +266,9 @@ int main(string[] args)
|
||||||
// We should only set noRemoteDelete in an upload-only scenario
|
// We should only set noRemoteDelete in an upload-only scenario
|
||||||
if ((uploadOnly)&&(noRemoteDelete)) sync.setNoRemoteDelete();
|
if ((uploadOnly)&&(noRemoteDelete)) sync.setNoRemoteDelete();
|
||||||
|
|
||||||
|
// Do we configure to disable the upload validation routine
|
||||||
|
if(disableUploadValidation) sync.setDisableUploadValidation();
|
||||||
|
|
||||||
// Do we need to validate the syncDir to check for the presence of a '.nosync' file
|
// Do we need to validate the syncDir to check for the presence of a '.nosync' file
|
||||||
if (checkMount) {
|
if (checkMount) {
|
||||||
// we were asked to check the mounts
|
// we were asked to check the mounts
|
||||||
|
|
|
@ -14,12 +14,16 @@ private immutable {
|
||||||
string authUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize";
|
string authUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize";
|
||||||
string redirectUrl = "https://login.microsoftonline.com/common/oauth2/nativeclient";
|
string redirectUrl = "https://login.microsoftonline.com/common/oauth2/nativeclient";
|
||||||
string tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
|
string tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
|
||||||
string driveUrl = "https://graph.microsoft.com/v1.0/me/drive";
|
|
||||||
string itemByIdUrl = "https://graph.microsoft.com/v1.0/me/drive/items/";
|
|
||||||
string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
|
|
||||||
string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
|
string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private {
|
||||||
|
string driveUrl = "https://graph.microsoft.com/v1.0/me/drive";
|
||||||
|
string itemByIdUrl = "https://graph.microsoft.com/v1.0/me/drive/items/";
|
||||||
|
string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
|
||||||
|
string driveId = "";
|
||||||
|
}
|
||||||
|
|
||||||
class OneDriveException: Exception
|
class OneDriveException: Exception
|
||||||
{
|
{
|
||||||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/errors
|
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/errors
|
||||||
|
@ -71,6 +75,15 @@ final class OneDriveApi
|
||||||
|
|
||||||
bool init()
|
bool init()
|
||||||
{
|
{
|
||||||
|
try {
|
||||||
|
driveId = cfg.getValue("drive_id");
|
||||||
|
if (driveId.length) {
|
||||||
|
driveUrl = driveByIdUrl ~ driveId;
|
||||||
|
itemByIdUrl = driveUrl ~ "/items";
|
||||||
|
itemByPathUrl = driveUrl ~ "/root:/";
|
||||||
|
}
|
||||||
|
} catch (Exception e) {}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
refreshToken = readText(cfg.refreshTokenFilePath);
|
refreshToken = readText(cfg.refreshTokenFilePath);
|
||||||
} catch (FileException e) {
|
} catch (FileException e) {
|
||||||
|
|
69
src/sync.d
69
src/sync.d
|
@ -16,6 +16,9 @@ private long thresholdFileSize = 4 * 2^^20; // 4 MiB
|
||||||
// flag to set whether local files should be deleted
|
// flag to set whether local files should be deleted
|
||||||
private bool noRemoteDelete = false;
|
private bool noRemoteDelete = false;
|
||||||
|
|
||||||
|
// Do we configure to disable the upload validation routine
|
||||||
|
private bool disableUploadValidation = false;
|
||||||
|
|
||||||
private bool isItemFolder(const ref JSONValue item)
|
private bool isItemFolder(const ref JSONValue item)
|
||||||
{
|
{
|
||||||
return ("folder" in item) != null;
|
return ("folder" in item) != null;
|
||||||
|
@ -220,7 +223,13 @@ final class SyncEngine
|
||||||
log.vlog("Default Drive ID: ", defaultDriveId);
|
log.vlog("Default Drive ID: ", defaultDriveId);
|
||||||
log.vlog("Default Root ID: ", defaultRootId);
|
log.vlog("Default Root ID: ", defaultRootId);
|
||||||
log.vlog("Remaining Free Space: ", remainingFreeSpace);
|
log.vlog("Remaining Free Space: ", remainingFreeSpace);
|
||||||
|
|
||||||
|
// If account type is documentLibrary - then most likely this is a SharePoint repository
|
||||||
|
// and files 'may' be modified after upload. See: https://github.com/abraunegg/onedrive/issues/205
|
||||||
|
if(accountType == "documentLibrary") {
|
||||||
|
setDisableUploadValidation();
|
||||||
|
}
|
||||||
|
|
||||||
// Check the local database to ensure the OneDrive Root details are in the database
|
// Check the local database to ensure the OneDrive Root details are in the database
|
||||||
checkDatabaseForOneDriveRoot();
|
checkDatabaseForOneDriveRoot();
|
||||||
|
|
||||||
|
@ -240,6 +249,18 @@ final class SyncEngine
|
||||||
noRemoteDelete = true;
|
noRemoteDelete = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Configure disableUploadValidation if function is called
|
||||||
|
// By default, disableUploadValidation = false;
|
||||||
|
// Meaning we will always validate our uploads
|
||||||
|
// However, when uploading a file that can contain metadata SharePoint will associate some
|
||||||
|
// metadata from the library the file is uploaded to directly in the file
|
||||||
|
// which breaks this validation. See https://github.com/abraunegg/onedrive/issues/205
|
||||||
|
void setDisableUploadValidation()
|
||||||
|
{
|
||||||
|
disableUploadValidation = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
// download all new changes from OneDrive
|
// download all new changes from OneDrive
|
||||||
void applyDifferences()
|
void applyDifferences()
|
||||||
{
|
{
|
||||||
|
@ -1410,29 +1431,35 @@ final class SyncEngine
|
||||||
// This has been seen with PNG / JPG files mainly, which then contributes to generating a 412 error when we attempt to update the metadata
|
// This has been seen with PNG / JPG files mainly, which then contributes to generating a 412 error when we attempt to update the metadata
|
||||||
// Validate here that the file uploaded, at least in size, matches in the response to what the size is on disk
|
// Validate here that the file uploaded, at least in size, matches in the response to what the size is on disk
|
||||||
if (thisFileSize != uploadFileSize){
|
if (thisFileSize != uploadFileSize){
|
||||||
// OK .. the uploaded file does not match
|
if(disableUploadValidation){
|
||||||
log.log("Uploaded file size does not match local file - upload failure - retrying");
|
// Print a warning message
|
||||||
// Delete uploaded bad file
|
log.log("WARNING: Uploaded file size does not match local file - skipping upload validation");
|
||||||
onedrive.deleteById(response["parentReference"]["driveId"].str, response["id"].str, response["eTag"].str);
|
|
||||||
// Re-upload
|
|
||||||
uploadNewFile(path);
|
|
||||||
return;
|
|
||||||
} else {
|
|
||||||
if ((accountType == "personal") || (thisFileSize == 0)){
|
|
||||||
// Update the item's metadata on OneDrive
|
|
||||||
string id = response["id"].str;
|
|
||||||
string cTag = response["cTag"].str;
|
|
||||||
SysTime mtime = timeLastModified(path).toUTC();
|
|
||||||
// use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded
|
|
||||||
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
|
|
||||||
return;
|
|
||||||
} else {
|
} else {
|
||||||
// OneDrive Business Account - always use a session to upload
|
// OK .. the uploaded file does not match and we did not disable this validation
|
||||||
// The session includes a Request Body element containing lastModifiedDateTime
|
log.log("Uploaded file size does not match local file - upload failure - retrying");
|
||||||
// which negates the need for a modify event against OneDrive
|
// Delete uploaded bad file
|
||||||
saveItem(response);
|
onedrive.deleteById(response["parentReference"]["driveId"].str, response["id"].str, response["eTag"].str);
|
||||||
|
// Re-upload
|
||||||
|
uploadNewFile(path);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// File validation is OK
|
||||||
|
if ((accountType == "personal") || (thisFileSize == 0)){
|
||||||
|
// Update the item's metadata on OneDrive
|
||||||
|
string id = response["id"].str;
|
||||||
|
string cTag = response["cTag"].str;
|
||||||
|
SysTime mtime = timeLastModified(path).toUTC();
|
||||||
|
// use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded
|
||||||
|
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
// OneDrive Business Account - always use a session to upload
|
||||||
|
// The session includes a Request Body element containing lastModifiedDateTime
|
||||||
|
// which negates the need for a modify event against OneDrive
|
||||||
|
saveItem(response);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue