mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-01 13:32:16 +02:00
support for files bigger than 100 MB and resumable uploads
This commit is contained in:
parent
88b11433a7
commit
9b80d99ad4
1
Makefile
1
Makefile
|
@ -13,6 +13,7 @@ SOURCES = \
|
||||||
src/onedrive.d \
|
src/onedrive.d \
|
||||||
src/sqlite.d \
|
src/sqlite.d \
|
||||||
src/sync.d \
|
src/sync.d \
|
||||||
|
src/upload.d \
|
||||||
src/util.d
|
src/util.d
|
||||||
|
|
||||||
onedrive: $(SOURCES)
|
onedrive: $(SOURCES)
|
||||||
|
|
25
src/main.d
25
src/main.d
|
@ -62,21 +62,22 @@ void main(string[] args)
|
||||||
if (verbose) writeln("Opening the item database ...");
|
if (verbose) writeln("Opening the item database ...");
|
||||||
auto itemdb = new ItemDatabase(databaseFilePath);
|
auto itemdb = new ItemDatabase(databaseFilePath);
|
||||||
|
|
||||||
if (verbose) writeln("Initializing the Synchronization Engine ...");
|
|
||||||
auto sync = new SyncEngine(cfg, onedrive, itemdb, verbose);
|
|
||||||
sync.onStatusToken = (string statusToken) {
|
|
||||||
std.file.write(statusTokenFilePath, statusToken);
|
|
||||||
};
|
|
||||||
try {
|
|
||||||
string statusToken = readText(statusTokenFilePath);
|
|
||||||
sync.setStatusToken(statusToken);
|
|
||||||
} catch (FileException e) {
|
|
||||||
// swallow exception
|
|
||||||
}
|
|
||||||
|
|
||||||
string syncDir = expandTilde(cfg.get("sync_dir"));
|
string syncDir = expandTilde(cfg.get("sync_dir"));
|
||||||
if (verbose) writeln("All operations will be performed in: ", syncDir);
|
if (verbose) writeln("All operations will be performed in: ", syncDir);
|
||||||
chdir(syncDir);
|
chdir(syncDir);
|
||||||
|
|
||||||
|
if (verbose) writeln("Initializing the Synchronization Engine ...");
|
||||||
|
auto sync = new SyncEngine(cfg, onedrive, itemdb, configDirName, verbose);
|
||||||
|
sync.onStatusToken = (string statusToken) {
|
||||||
|
std.file.write(statusTokenFilePath, statusToken);
|
||||||
|
};
|
||||||
|
string statusToken;
|
||||||
|
try {
|
||||||
|
statusToken = readText(statusTokenFilePath);
|
||||||
|
} catch (FileException e) {
|
||||||
|
// swallow exception
|
||||||
|
}
|
||||||
|
sync.init(statusToken);
|
||||||
performSync(sync);
|
performSync(sync);
|
||||||
|
|
||||||
if (monitor) {
|
if (monitor) {
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
import std.datetime, std.exception, std.json, std.net.curl, std.path;
|
import std.datetime, std.exception, std.json, std.net.curl, std.path;
|
||||||
import std.string, std.uni, std.uri;
|
import std.stdio, std.string, std.uni, std.uri;
|
||||||
import config;
|
import config;
|
||||||
|
|
||||||
private immutable {
|
private immutable {
|
||||||
|
@ -181,6 +181,49 @@ final class OneDriveApi
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// https://dev.onedrive.com/items/upload_large_files.htm
|
||||||
|
JSONValue createUploadSession(const(char)[] path, const(char)[] eTag = null)
|
||||||
|
{
|
||||||
|
checkAccessTokenExpired();
|
||||||
|
string url = itemByPathUrl ~ encodeComponent(path) ~ ":/upload.createSession";
|
||||||
|
if (eTag) http.addRequestHeader("If-Match", eTag);
|
||||||
|
auto result = post(url, null);
|
||||||
|
// remove the if-match header
|
||||||
|
if (eTag) setAccessToken(accessToken);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://dev.onedrive.com/items/upload_large_files.htm
|
||||||
|
JSONValue uploadFragment(const(char)[] uploadUrl, string filepath, long offset, long offsetSize, long fileSize)
|
||||||
|
{
|
||||||
|
checkAccessTokenExpired();
|
||||||
|
http.method = HTTP.Method.put;
|
||||||
|
http.url = uploadUrl;
|
||||||
|
ubyte[] content;
|
||||||
|
http.onReceive = (ubyte[] data) {
|
||||||
|
content ~= data;
|
||||||
|
return data.length;
|
||||||
|
};
|
||||||
|
auto file = File(filepath, "rb");
|
||||||
|
file.seek(offset);
|
||||||
|
http.onSend = data => file.rawRead(data).length;
|
||||||
|
http.contentLength = offsetSize;
|
||||||
|
import std.conv;
|
||||||
|
string contentRange = "bytes " ~ to!string(offset) ~ "-" ~ to!string(offset + offsetSize - 1) ~ "/" ~ to!string(fileSize);
|
||||||
|
http.addRequestHeader("Content-Range", contentRange);
|
||||||
|
http.perform();
|
||||||
|
checkHttpCode(); // TODO: retry on 5xx errors
|
||||||
|
// remove the content-range header
|
||||||
|
scope(exit) setAccessToken(accessToken);
|
||||||
|
return parseJSON(content);
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://dev.onedrive.com/items/upload_large_files.htm
|
||||||
|
JSONValue requestUploadStatus(const(char)[] uploadUrl)
|
||||||
|
{
|
||||||
|
return get(uploadUrl);
|
||||||
|
}
|
||||||
|
|
||||||
private void redeemToken(const(char)[] authCode)
|
private void redeemToken(const(char)[] authCode)
|
||||||
{
|
{
|
||||||
string postData = "client_id=" ~ clientId ~ "&redirect_url=" ~ redirectUrl ~ "&client_secret=" ~ clientSecret;
|
string postData = "client_id=" ~ clientId ~ "&redirect_url=" ~ redirectUrl ~ "&client_secret=" ~ clientSecret;
|
||||||
|
|
45
src/sync.d
45
src/sync.d
|
@ -1,7 +1,11 @@
|
||||||
import std.exception: ErrnoException;
|
import std.exception: ErrnoException;
|
||||||
import std.algorithm, std.datetime, std.file, std.json, std.path, std.regex;
|
import std.algorithm, std.datetime, std.file, std.json, std.path, std.regex;
|
||||||
import std.stdio, std.string;
|
import std.stdio, std.string;
|
||||||
import config, itemdb, onedrive, util;
|
import config, itemdb, onedrive, upload, util;
|
||||||
|
|
||||||
|
private string uploadStateFileName = "resume_upload";
|
||||||
|
// threshold after which files will be uploaded using an upload session
|
||||||
|
private long thresholdFileSize = 10 * 2^^20; // 10 Mib
|
||||||
|
|
||||||
private bool isItemFolder(const ref JSONValue item)
|
private bool isItemFolder(const ref JSONValue item)
|
||||||
{
|
{
|
||||||
|
@ -52,6 +56,7 @@ final class SyncEngine
|
||||||
private ItemDatabase itemdb;
|
private ItemDatabase itemdb;
|
||||||
private bool verbose;
|
private bool verbose;
|
||||||
private Regex!char skipDir, skipFile;
|
private Regex!char skipDir, skipFile;
|
||||||
|
private UploadSession session;
|
||||||
// token representing the last status correctly synced
|
// token representing the last status correctly synced
|
||||||
private string statusToken;
|
private string statusToken;
|
||||||
// list of items to skip while applying the changes downloaded
|
// list of items to skip while applying the changes downloaded
|
||||||
|
@ -61,20 +66,28 @@ final class SyncEngine
|
||||||
|
|
||||||
void delegate(string) onStatusToken;
|
void delegate(string) onStatusToken;
|
||||||
|
|
||||||
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, bool verbose)
|
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, string configDirName, bool verbose)
|
||||||
{
|
{
|
||||||
assert(onedrive && itemdb);
|
assert(onedrive && itemdb);
|
||||||
this.cfg = cfg;
|
this.cfg = cfg;
|
||||||
this.onedrive = onedrive;
|
this.onedrive = onedrive;
|
||||||
this.itemdb = itemdb;
|
this.itemdb = itemdb;
|
||||||
|
//this.configDirName = configDirName;
|
||||||
this.verbose = verbose;
|
this.verbose = verbose;
|
||||||
skipDir = wild2regex(cfg.get("skip_dir"));
|
skipDir = wild2regex(cfg.get("skip_dir"));
|
||||||
skipFile = wild2regex(cfg.get("skip_file"));
|
skipFile = wild2regex(cfg.get("skip_file"));
|
||||||
|
session = UploadSession(onedrive, configDirName ~ "/" ~ uploadStateFileName, verbose);
|
||||||
}
|
}
|
||||||
|
|
||||||
void setStatusToken(string statusToken)
|
void init(string statusToken = null)
|
||||||
{
|
{
|
||||||
this.statusToken = statusToken;
|
this.statusToken = statusToken;
|
||||||
|
// check if there is an interrupted upload session
|
||||||
|
if (session.restore()) {
|
||||||
|
writeln("Continuing the upload session ...");
|
||||||
|
auto item = session.upload();
|
||||||
|
saveItem(item);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void applyDifferences()
|
void applyDifferences()
|
||||||
|
@ -395,13 +408,18 @@ final class SyncEngine
|
||||||
if (!testCrc32(path, item.crc32)) {
|
if (!testCrc32(path, item.crc32)) {
|
||||||
if (verbose) writeln("The file content has changed");
|
if (verbose) writeln("The file content has changed");
|
||||||
writeln("Uploading: ", path);
|
writeln("Uploading: ", path);
|
||||||
auto res = onedrive.simpleUpload(path, path, item.eTag);
|
JSONValue response;
|
||||||
saveItem(res);
|
if (getSize(path) <= thresholdFileSize) {
|
||||||
id = res["id"].str;
|
response = onedrive.simpleUpload(path, path);
|
||||||
|
} else {
|
||||||
|
response = session.upload(path, path);
|
||||||
|
}
|
||||||
|
saveItem(response);
|
||||||
|
id = response["id"].str;
|
||||||
/* use the cTag instead of the eTag because Onedrive changes the
|
/* use the cTag instead of the eTag because Onedrive changes the
|
||||||
* metadata of some type of files (ex. images) AFTER they have been
|
* metadata of some type of files (ex. images) AFTER they have been
|
||||||
* uploaded */
|
* uploaded */
|
||||||
eTag = res["cTag"].str;
|
eTag = response["cTag"].str;
|
||||||
}
|
}
|
||||||
uploadLastModifiedTime(id, eTag, localModifiedTime.toUTC());
|
uploadLastModifiedTime(id, eTag, localModifiedTime.toUTC());
|
||||||
} else {
|
} else {
|
||||||
|
@ -453,10 +471,15 @@ final class SyncEngine
|
||||||
private void uploadNewFile(string path)
|
private void uploadNewFile(string path)
|
||||||
{
|
{
|
||||||
writeln("Uploading: ", path);
|
writeln("Uploading: ", path);
|
||||||
JSONValue res = onedrive.simpleUpload(path, path);
|
JSONValue response;
|
||||||
saveItem(res);
|
if (getSize(path) <= thresholdFileSize) {
|
||||||
string id = res["id"].str;
|
response = onedrive.simpleUpload(path, path);
|
||||||
string cTag = res["cTag"].str;
|
} else {
|
||||||
|
response = session.upload(path, path);
|
||||||
|
}
|
||||||
|
saveItem(response);
|
||||||
|
string id = response["id"].str;
|
||||||
|
string cTag = response["cTag"].str;
|
||||||
SysTime mtime = timeLastModified(path).toUTC();
|
SysTime mtime = timeLastModified(path).toUTC();
|
||||||
/* use the cTag instead of the eTag because Onedrive changes the
|
/* use the cTag instead of the eTag because Onedrive changes the
|
||||||
* metadata of some type of files (ex. images) AFTER they have been
|
* metadata of some type of files (ex. images) AFTER they have been
|
||||||
|
|
97
src/upload.d
Normal file
97
src/upload.d
Normal file
|
@ -0,0 +1,97 @@
|
||||||
|
import std.algorithm;
|
||||||
|
import std.conv;
|
||||||
|
import std.datetime;
|
||||||
|
import std.file;
|
||||||
|
import std.json;
|
||||||
|
import std.stdio;
|
||||||
|
import onedrive;
|
||||||
|
|
||||||
|
private long fragmentSize = 10 * 2^^20; // 10 Mib
|
||||||
|
|
||||||
|
struct UploadSession
|
||||||
|
{
|
||||||
|
private OneDriveApi onedrive;
|
||||||
|
private bool verbose;
|
||||||
|
// https://dev.onedrive.com/resources/uploadSession.htm
|
||||||
|
private JSONValue session;
|
||||||
|
// path where to save the session
|
||||||
|
private string sessionFilePath;
|
||||||
|
|
||||||
|
this(OneDriveApi onedrive, string sessionFilePath, bool verbose)
|
||||||
|
{
|
||||||
|
assert(onedrive);
|
||||||
|
this.onedrive = onedrive;
|
||||||
|
this.sessionFilePath = sessionFilePath;
|
||||||
|
this.verbose = verbose;
|
||||||
|
}
|
||||||
|
|
||||||
|
JSONValue upload(string localPath, string remotePath, const(char)[] eTag = null)
|
||||||
|
{
|
||||||
|
session = onedrive.createUploadSession(remotePath, eTag);
|
||||||
|
session["localPath"] = localPath;
|
||||||
|
save();
|
||||||
|
return upload();
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Restore the previous upload session.
|
||||||
|
* Returns true if the session is valid. Call upload() to resume it.
|
||||||
|
* Returns false if there is no session or the session is expired. */
|
||||||
|
bool restore()
|
||||||
|
{
|
||||||
|
if (exists(sessionFilePath)) {
|
||||||
|
if (verbose) writeln("Trying to restore the upload session ...");
|
||||||
|
session = readText(sessionFilePath).parseJSON();
|
||||||
|
auto expiration = SysTime.fromISOExtString(session["expirationDateTime"].str);
|
||||||
|
if (expiration < Clock.currTime()) {
|
||||||
|
if (verbose) writeln("The upload session is expired");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!exists(session["localPath"].str)) {
|
||||||
|
if (verbose) writeln("The file do not exist anymore");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// request the session status
|
||||||
|
auto response = onedrive.requestUploadStatus(session["uploadUrl"].str);
|
||||||
|
session["expirationDateTime"] = response["expirationDateTime"];
|
||||||
|
session["nextExpectedRanges"] = response["nextExpectedRanges"];
|
||||||
|
if (session["nextExpectedRanges"].array.length == 0) {
|
||||||
|
if (verbose) writeln("The upload session is completed");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
JSONValue upload()
|
||||||
|
{
|
||||||
|
long offset = session["nextExpectedRanges"][0].str.splitter('-').front.to!long;
|
||||||
|
long fileSize = getSize(session["localPath"].str);
|
||||||
|
JSONValue response;
|
||||||
|
while (true) {
|
||||||
|
long fragSize = fragmentSize < fileSize - offset ? fragmentSize : fileSize - offset;
|
||||||
|
if (verbose) writeln("Uploading fragment: ", offset, "-", offset + fragSize, "/", fileSize);
|
||||||
|
response = onedrive.uploadFragment(
|
||||||
|
session["uploadUrl"].str,
|
||||||
|
session["localPath"].str,
|
||||||
|
offset,
|
||||||
|
fragSize,
|
||||||
|
fileSize
|
||||||
|
);
|
||||||
|
offset += fragmentSize;
|
||||||
|
if (offset >= fileSize) break;
|
||||||
|
// update the session
|
||||||
|
session["expirationDateTime"] = response["expirationDateTime"];
|
||||||
|
session["nextExpectedRanges"] = response["nextExpectedRanges"];
|
||||||
|
save();
|
||||||
|
}
|
||||||
|
// upload complete
|
||||||
|
remove(sessionFilePath);
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void save()
|
||||||
|
{
|
||||||
|
std.file.write(sessionFilePath, session.toString());
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue