mirror of
https://github.com/abraunegg/onedrive
synced 2024-05-20 22:56:36 +02:00
* Implement feature request to add a progress bar for large file uploads & downloads
This commit is contained in:
parent
2bb5dce752
commit
bed2b6c75f
3
Makefile
3
Makefile
|
@ -14,7 +14,8 @@ SOURCES = \
|
||||||
src/sqlite.d \
|
src/sqlite.d \
|
||||||
src/sync.d \
|
src/sync.d \
|
||||||
src/upload.d \
|
src/upload.d \
|
||||||
src/util.d
|
src/util.d \
|
||||||
|
src/progress.d
|
||||||
|
|
||||||
all: onedrive onedrive.service
|
all: onedrive onedrive.service
|
||||||
|
|
||||||
|
|
|
@ -1,8 +1,10 @@
|
||||||
import std.net.curl: CurlException, HTTP;
|
import std.net.curl: CurlException, HTTP;
|
||||||
import std.datetime, std.exception, std.file, std.json, std.path;
|
import std.datetime, std.exception, std.file, std.json, std.path;
|
||||||
import std.stdio, std.string, std.uni, std.uri;
|
import std.stdio, std.string, std.uni, std.uri;
|
||||||
import config;
|
|
||||||
import core.stdc.stdlib;
|
import core.stdc.stdlib;
|
||||||
|
import core.thread, std.conv, std.math;
|
||||||
|
import progress;
|
||||||
|
import config;
|
||||||
static import log;
|
static import log;
|
||||||
shared bool debugResponse = false;
|
shared bool debugResponse = false;
|
||||||
|
|
||||||
|
@ -118,7 +120,7 @@ final class OneDriveApi
|
||||||
}
|
}
|
||||||
|
|
||||||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get_content
|
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get_content
|
||||||
void downloadById(const(char)[] driveId, const(char)[] id, string saveToPath)
|
void downloadById(const(char)[] driveId, const(char)[] id, string saveToPath, long fileSize)
|
||||||
{
|
{
|
||||||
checkAccessTokenExpired();
|
checkAccessTokenExpired();
|
||||||
scope(failure) {
|
scope(failure) {
|
||||||
|
@ -126,7 +128,7 @@ final class OneDriveApi
|
||||||
}
|
}
|
||||||
mkdirRecurse(dirName(saveToPath));
|
mkdirRecurse(dirName(saveToPath));
|
||||||
const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content?AVOverride=1";
|
const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content?AVOverride=1";
|
||||||
download(url, saveToPath);
|
download(url, saveToPath, fileSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
|
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
|
||||||
|
@ -202,6 +204,17 @@ final class OneDriveApi
|
||||||
return get(url);
|
return get(url);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Return the requested details of the specified id
|
||||||
|
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
|
||||||
|
JSONValue getFileSize(const(char)[] driveId, const(char)[] id)
|
||||||
|
{
|
||||||
|
checkAccessTokenExpired();
|
||||||
|
const(char)[] url;
|
||||||
|
// string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
|
||||||
|
url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
|
||||||
|
url ~= "?select=size";
|
||||||
|
return get(url);
|
||||||
|
}
|
||||||
|
|
||||||
// https://dev.onedrive.com/items/move.htm
|
// https://dev.onedrive.com/items/move.htm
|
||||||
JSONValue moveByPath(const(char)[] sourcePath, JSONValue moveData)
|
JSONValue moveByPath(const(char)[] sourcePath, JSONValue moveData)
|
||||||
|
@ -328,8 +341,11 @@ final class OneDriveApi
|
||||||
checkHttpCode(response);
|
checkHttpCode(response);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void download(const(char)[] url, string filename)
|
private void download(const(char)[] url, string filename, long fileSize)
|
||||||
{
|
{
|
||||||
|
// Threshold for displaying download bar
|
||||||
|
long thresholdFileSize = 4 * 2^^20; // 4 MiB
|
||||||
|
|
||||||
scope(exit) http.clearRequestHeaders();
|
scope(exit) http.clearRequestHeaders();
|
||||||
http.method = HTTP.Method.get;
|
http.method = HTTP.Method.get;
|
||||||
http.url = url;
|
http.url = url;
|
||||||
|
@ -339,7 +355,43 @@ final class OneDriveApi
|
||||||
f.rawWrite(data);
|
f.rawWrite(data);
|
||||||
return data.length;
|
return data.length;
|
||||||
};
|
};
|
||||||
http.perform();
|
|
||||||
|
if (fileSize >= thresholdFileSize){
|
||||||
|
// Download Progress Bar
|
||||||
|
size_t iteration = 20;
|
||||||
|
Progress p = new Progress(iteration);
|
||||||
|
p.title = "Downloading";
|
||||||
|
writeln();
|
||||||
|
|
||||||
|
real previousDLPercent = -1.0;
|
||||||
|
real percentCheck = 5.0;
|
||||||
|
// Setup progress bar to display
|
||||||
|
http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow)
|
||||||
|
{
|
||||||
|
// For each onProgress, what is the % of dlnow to dltotal
|
||||||
|
real currentDLPercent = round(double(dlnow)/dltotal*100);
|
||||||
|
// If matching 5% of download, increment progress bar
|
||||||
|
if ((isIdentical(fmod(currentDLPercent, percentCheck), 0.0)) && (previousDLPercent != currentDLPercent)) {
|
||||||
|
p.next();
|
||||||
|
previousDLPercent = currentDLPercent;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Perform download & display progress bar
|
||||||
|
http.perform();
|
||||||
|
writeln();
|
||||||
|
// Reset onProgress to not display anything for next download
|
||||||
|
http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
// No progress bar
|
||||||
|
http.perform();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check the HTTP response code
|
||||||
checkHttpCode();
|
checkHttpCode();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
153
src/progress.d
Normal file
153
src/progress.d
Normal file
|
@ -0,0 +1,153 @@
|
||||||
|
module progress;
|
||||||
|
|
||||||
|
import std.stdio;
|
||||||
|
import std.range;
|
||||||
|
import std.format;
|
||||||
|
import std.datetime;
|
||||||
|
import core.sys.posix.unistd;
|
||||||
|
import core.sys.posix.sys.ioctl;
|
||||||
|
|
||||||
|
class Progress
|
||||||
|
{
|
||||||
|
private:
|
||||||
|
|
||||||
|
immutable static size_t default_width = 80;
|
||||||
|
size_t max_width = 40;
|
||||||
|
size_t width = default_width;
|
||||||
|
|
||||||
|
ulong start_time;
|
||||||
|
string caption = "Progress";
|
||||||
|
size_t iterations;
|
||||||
|
size_t counter;
|
||||||
|
|
||||||
|
|
||||||
|
size_t getTerminalWidth() {
|
||||||
|
size_t column;
|
||||||
|
winsize ws;
|
||||||
|
if(ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) != -1) {
|
||||||
|
column = ws.ws_col;
|
||||||
|
}
|
||||||
|
if(column == 0) column = default_width;
|
||||||
|
|
||||||
|
return column;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void clear() {
|
||||||
|
write("\r");
|
||||||
|
for(auto i = 0; i < width; i++) write(" ");
|
||||||
|
write("\r");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
int calc_eta() {
|
||||||
|
immutable auto ratio = cast(double)counter / iterations;
|
||||||
|
auto current_time = Clock.currTime.toUnixTime();
|
||||||
|
auto duration = cast(int)(current_time - start_time);
|
||||||
|
int hours, minutes, seconds;
|
||||||
|
double elapsed = (current_time - start_time);
|
||||||
|
int eta_sec = cast(int)((elapsed / ratio) - elapsed);
|
||||||
|
|
||||||
|
// Return an ETA or Duration?
|
||||||
|
if (eta_sec != 0){
|
||||||
|
return eta_sec;
|
||||||
|
} else {
|
||||||
|
return duration;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
string progressbarText(string header_text, string footer_text) {
|
||||||
|
immutable auto ratio = cast(double)counter / iterations;
|
||||||
|
string result = "";
|
||||||
|
|
||||||
|
double bar_length = width - header_text.length - footer_text.length;
|
||||||
|
if(bar_length > max_width && max_width > 0) {
|
||||||
|
bar_length = max_width;
|
||||||
|
}
|
||||||
|
size_t i = 0;
|
||||||
|
for(; i < ratio * bar_length; i++) result ~= "o";
|
||||||
|
for(; i < bar_length; i++) result ~= " ";
|
||||||
|
|
||||||
|
return header_text ~ result ~ footer_text;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void print() {
|
||||||
|
immutable auto ratio = cast(double)counter / iterations;
|
||||||
|
auto header = appender!string();
|
||||||
|
auto footer = appender!string();
|
||||||
|
|
||||||
|
header.formattedWrite("%s %3d%% |", caption, cast(int)(ratio * 100));
|
||||||
|
|
||||||
|
if(counter <= 1 || ratio == 0.0) {
|
||||||
|
footer.formattedWrite("| ETA --:--:--:");
|
||||||
|
} else {
|
||||||
|
int h, m, s;
|
||||||
|
dur!"seconds"(calc_eta())
|
||||||
|
.split!("hours", "minutes", "seconds")(h, m, s);
|
||||||
|
if (counter != iterations){
|
||||||
|
footer.formattedWrite("| ETA %02d:%02d:%02d ", h, m, s);
|
||||||
|
} else {
|
||||||
|
footer.formattedWrite("| DONE IN %02d:%02d:%02d ", h, m, s);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
write(progressbarText(header.data, footer.data));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void update() {
|
||||||
|
width = getTerminalWidth();
|
||||||
|
|
||||||
|
clear();
|
||||||
|
|
||||||
|
print();
|
||||||
|
stdout.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public:
|
||||||
|
|
||||||
|
this(size_t iterations) {
|
||||||
|
if(iterations <= 0) iterations = 1;
|
||||||
|
|
||||||
|
counter = 0;
|
||||||
|
this.iterations = iterations;
|
||||||
|
start_time = Clock.currTime.toUnixTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
@property {
|
||||||
|
string title() { return caption; }
|
||||||
|
string title(string text) { return caption = text; }
|
||||||
|
}
|
||||||
|
|
||||||
|
@property {
|
||||||
|
size_t count() { return counter; }
|
||||||
|
size_t count(size_t val) {
|
||||||
|
if(val > iterations) val = iterations;
|
||||||
|
return counter = val;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@property {
|
||||||
|
size_t maxWidth() { return max_width; }
|
||||||
|
size_t maxWidth(size_t w) {
|
||||||
|
return max_width = w;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void reset() {
|
||||||
|
counter = 0;
|
||||||
|
start_time = Clock.currTime.toUnixTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
void next() {
|
||||||
|
counter++;
|
||||||
|
if(counter > iterations) counter = iterations;
|
||||||
|
|
||||||
|
update();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
10
src/sync.d
10
src/sync.d
|
@ -642,11 +642,13 @@ final class SyncEngine
|
||||||
private void downloadFileItem(Item item, string path)
|
private void downloadFileItem(Item item, string path)
|
||||||
{
|
{
|
||||||
assert(item.type == ItemType.file);
|
assert(item.type == ItemType.file);
|
||||||
write("Downloading ", path, "...");
|
write("Downloading file ", path, " ... ");
|
||||||
onedrive.downloadById(item.driveId, item.id, path);
|
JSONValue fileSizeDetails = onedrive.getFileSize(item.driveId, item.id);
|
||||||
|
auto fileSize = fileSizeDetails["size"].integer;
|
||||||
|
onedrive.downloadById(item.driveId, item.id, path, fileSize);
|
||||||
|
writeln("done.");
|
||||||
|
log.fileOnly("Downloading file ", path, " ... done.");
|
||||||
setTimes(path, item.mtime, item.mtime);
|
setTimes(path, item.mtime, item.mtime);
|
||||||
writeln(" done.");
|
|
||||||
log.fileOnly("Downloading ", path, "... done.");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// returns true if the given item corresponds to the local one
|
// returns true if the given item corresponds to the local one
|
||||||
|
|
12
src/upload.d
12
src/upload.d
|
@ -1,4 +1,6 @@
|
||||||
import std.algorithm, std.conv, std.datetime, std.file, std.json;
|
import std.algorithm, std.conv, std.datetime, std.file, std.json;
|
||||||
|
import std.stdio, core.thread;
|
||||||
|
import progress;
|
||||||
import onedrive;
|
import onedrive;
|
||||||
static import log;
|
static import log;
|
||||||
|
|
||||||
|
@ -88,10 +90,16 @@ struct UploadSession
|
||||||
{
|
{
|
||||||
long offset = session["nextExpectedRanges"][0].str.splitter('-').front.to!long;
|
long offset = session["nextExpectedRanges"][0].str.splitter('-').front.to!long;
|
||||||
long fileSize = getSize(session["localPath"].str);
|
long fileSize = getSize(session["localPath"].str);
|
||||||
|
|
||||||
|
// Upload Progress Bar
|
||||||
|
size_t iteration = (roundTo!int(double(fileSize)/double(fragmentSize)))+1;
|
||||||
|
Progress p = new Progress(iteration);
|
||||||
|
p.title = "Uploading";
|
||||||
|
|
||||||
JSONValue response;
|
JSONValue response;
|
||||||
while (true) {
|
while (true) {
|
||||||
|
p.next();
|
||||||
long fragSize = fragmentSize < fileSize - offset ? fragmentSize : fileSize - offset;
|
long fragSize = fragmentSize < fileSize - offset ? fragmentSize : fileSize - offset;
|
||||||
log.vlog("Uploading fragment: ", offset, "-", offset + fragSize, "/", fileSize);
|
|
||||||
response = onedrive.uploadFragment(
|
response = onedrive.uploadFragment(
|
||||||
session["uploadUrl"].str,
|
session["uploadUrl"].str,
|
||||||
session["localPath"].str,
|
session["localPath"].str,
|
||||||
|
@ -107,6 +115,8 @@ struct UploadSession
|
||||||
save();
|
save();
|
||||||
}
|
}
|
||||||
// upload complete
|
// upload complete
|
||||||
|
p.next();
|
||||||
|
writeln();
|
||||||
remove(sessionFilePath);
|
remove(sessionFilePath);
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue