mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-08 08:52:15 +02:00
* Implement feature request to add a progress bar for large file uploads & downloads
This commit is contained in:
parent
2bb5dce752
commit
bed2b6c75f
3
Makefile
3
Makefile
|
@ -14,7 +14,8 @@ SOURCES = \
|
|||
src/sqlite.d \
|
||||
src/sync.d \
|
||||
src/upload.d \
|
||||
src/util.d
|
||||
src/util.d \
|
||||
src/progress.d
|
||||
|
||||
all: onedrive onedrive.service
|
||||
|
||||
|
|
|
@ -1,8 +1,10 @@
|
|||
import std.net.curl: CurlException, HTTP;
|
||||
import std.datetime, std.exception, std.file, std.json, std.path;
|
||||
import std.stdio, std.string, std.uni, std.uri;
|
||||
import config;
|
||||
import core.stdc.stdlib;
|
||||
import core.thread, std.conv, std.math;
|
||||
import progress;
|
||||
import config;
|
||||
static import log;
|
||||
shared bool debugResponse = false;
|
||||
|
||||
|
@ -118,7 +120,7 @@ final class OneDriveApi
|
|||
}
|
||||
|
||||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get_content
|
||||
void downloadById(const(char)[] driveId, const(char)[] id, string saveToPath)
|
||||
void downloadById(const(char)[] driveId, const(char)[] id, string saveToPath, long fileSize)
|
||||
{
|
||||
checkAccessTokenExpired();
|
||||
scope(failure) {
|
||||
|
@ -126,7 +128,7 @@ final class OneDriveApi
|
|||
}
|
||||
mkdirRecurse(dirName(saveToPath));
|
||||
const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content?AVOverride=1";
|
||||
download(url, saveToPath);
|
||||
download(url, saveToPath, fileSize);
|
||||
}
|
||||
|
||||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
|
||||
|
@ -202,6 +204,17 @@ final class OneDriveApi
|
|||
return get(url);
|
||||
}
|
||||
|
||||
// Return the requested details of the specified id
|
||||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
|
||||
JSONValue getFileSize(const(char)[] driveId, const(char)[] id)
|
||||
{
|
||||
checkAccessTokenExpired();
|
||||
const(char)[] url;
|
||||
// string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
|
||||
url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
|
||||
url ~= "?select=size";
|
||||
return get(url);
|
||||
}
|
||||
|
||||
// https://dev.onedrive.com/items/move.htm
|
||||
JSONValue moveByPath(const(char)[] sourcePath, JSONValue moveData)
|
||||
|
@ -328,8 +341,11 @@ final class OneDriveApi
|
|||
checkHttpCode(response);
|
||||
}
|
||||
|
||||
private void download(const(char)[] url, string filename)
|
||||
private void download(const(char)[] url, string filename, long fileSize)
|
||||
{
|
||||
// Threshold for displaying download bar
|
||||
long thresholdFileSize = 4 * 2^^20; // 4 MiB
|
||||
|
||||
scope(exit) http.clearRequestHeaders();
|
||||
http.method = HTTP.Method.get;
|
||||
http.url = url;
|
||||
|
@ -339,7 +355,43 @@ final class OneDriveApi
|
|||
f.rawWrite(data);
|
||||
return data.length;
|
||||
};
|
||||
|
||||
if (fileSize >= thresholdFileSize){
|
||||
// Download Progress Bar
|
||||
size_t iteration = 20;
|
||||
Progress p = new Progress(iteration);
|
||||
p.title = "Downloading";
|
||||
writeln();
|
||||
|
||||
real previousDLPercent = -1.0;
|
||||
real percentCheck = 5.0;
|
||||
// Setup progress bar to display
|
||||
http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow)
|
||||
{
|
||||
// For each onProgress, what is the % of dlnow to dltotal
|
||||
real currentDLPercent = round(double(dlnow)/dltotal*100);
|
||||
// If matching 5% of download, increment progress bar
|
||||
if ((isIdentical(fmod(currentDLPercent, percentCheck), 0.0)) && (previousDLPercent != currentDLPercent)) {
|
||||
p.next();
|
||||
previousDLPercent = currentDLPercent;
|
||||
}
|
||||
return 0;
|
||||
};
|
||||
|
||||
// Perform download & display progress bar
|
||||
http.perform();
|
||||
writeln();
|
||||
// Reset onProgress to not display anything for next download
|
||||
http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow)
|
||||
{
|
||||
return 0;
|
||||
};
|
||||
} else {
|
||||
// No progress bar
|
||||
http.perform();
|
||||
}
|
||||
|
||||
// Check the HTTP response code
|
||||
checkHttpCode();
|
||||
}
|
||||
|
||||
|
|
153
src/progress.d
Normal file
153
src/progress.d
Normal file
|
@ -0,0 +1,153 @@
|
|||
module progress;
|
||||
|
||||
import std.stdio;
|
||||
import std.range;
|
||||
import std.format;
|
||||
import std.datetime;
|
||||
import core.sys.posix.unistd;
|
||||
import core.sys.posix.sys.ioctl;
|
||||
|
||||
class Progress
|
||||
{
|
||||
private:
|
||||
|
||||
immutable static size_t default_width = 80;
|
||||
size_t max_width = 40;
|
||||
size_t width = default_width;
|
||||
|
||||
ulong start_time;
|
||||
string caption = "Progress";
|
||||
size_t iterations;
|
||||
size_t counter;
|
||||
|
||||
|
||||
size_t getTerminalWidth() {
|
||||
size_t column;
|
||||
winsize ws;
|
||||
if(ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) != -1) {
|
||||
column = ws.ws_col;
|
||||
}
|
||||
if(column == 0) column = default_width;
|
||||
|
||||
return column;
|
||||
}
|
||||
|
||||
|
||||
void clear() {
|
||||
write("\r");
|
||||
for(auto i = 0; i < width; i++) write(" ");
|
||||
write("\r");
|
||||
}
|
||||
|
||||
|
||||
int calc_eta() {
|
||||
immutable auto ratio = cast(double)counter / iterations;
|
||||
auto current_time = Clock.currTime.toUnixTime();
|
||||
auto duration = cast(int)(current_time - start_time);
|
||||
int hours, minutes, seconds;
|
||||
double elapsed = (current_time - start_time);
|
||||
int eta_sec = cast(int)((elapsed / ratio) - elapsed);
|
||||
|
||||
// Return an ETA or Duration?
|
||||
if (eta_sec != 0){
|
||||
return eta_sec;
|
||||
} else {
|
||||
return duration;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
string progressbarText(string header_text, string footer_text) {
|
||||
immutable auto ratio = cast(double)counter / iterations;
|
||||
string result = "";
|
||||
|
||||
double bar_length = width - header_text.length - footer_text.length;
|
||||
if(bar_length > max_width && max_width > 0) {
|
||||
bar_length = max_width;
|
||||
}
|
||||
size_t i = 0;
|
||||
for(; i < ratio * bar_length; i++) result ~= "o";
|
||||
for(; i < bar_length; i++) result ~= " ";
|
||||
|
||||
return header_text ~ result ~ footer_text;
|
||||
}
|
||||
|
||||
|
||||
void print() {
|
||||
immutable auto ratio = cast(double)counter / iterations;
|
||||
auto header = appender!string();
|
||||
auto footer = appender!string();
|
||||
|
||||
header.formattedWrite("%s %3d%% |", caption, cast(int)(ratio * 100));
|
||||
|
||||
if(counter <= 1 || ratio == 0.0) {
|
||||
footer.formattedWrite("| ETA --:--:--:");
|
||||
} else {
|
||||
int h, m, s;
|
||||
dur!"seconds"(calc_eta())
|
||||
.split!("hours", "minutes", "seconds")(h, m, s);
|
||||
if (counter != iterations){
|
||||
footer.formattedWrite("| ETA %02d:%02d:%02d ", h, m, s);
|
||||
} else {
|
||||
footer.formattedWrite("| DONE IN %02d:%02d:%02d ", h, m, s);
|
||||
}
|
||||
}
|
||||
|
||||
write(progressbarText(header.data, footer.data));
|
||||
}
|
||||
|
||||
|
||||
void update() {
|
||||
width = getTerminalWidth();
|
||||
|
||||
clear();
|
||||
|
||||
print();
|
||||
stdout.flush();
|
||||
}
|
||||
|
||||
|
||||
public:
|
||||
|
||||
this(size_t iterations) {
|
||||
if(iterations <= 0) iterations = 1;
|
||||
|
||||
counter = 0;
|
||||
this.iterations = iterations;
|
||||
start_time = Clock.currTime.toUnixTime;
|
||||
}
|
||||
|
||||
@property {
|
||||
string title() { return caption; }
|
||||
string title(string text) { return caption = text; }
|
||||
}
|
||||
|
||||
@property {
|
||||
size_t count() { return counter; }
|
||||
size_t count(size_t val) {
|
||||
if(val > iterations) val = iterations;
|
||||
return counter = val;
|
||||
}
|
||||
}
|
||||
|
||||
@property {
|
||||
size_t maxWidth() { return max_width; }
|
||||
size_t maxWidth(size_t w) {
|
||||
return max_width = w;
|
||||
}
|
||||
}
|
||||
|
||||
void reset() {
|
||||
counter = 0;
|
||||
start_time = Clock.currTime.toUnixTime;
|
||||
}
|
||||
|
||||
void next() {
|
||||
counter++;
|
||||
if(counter > iterations) counter = iterations;
|
||||
|
||||
update();
|
||||
}
|
||||
|
||||
|
||||
}
|
10
src/sync.d
10
src/sync.d
|
@ -642,11 +642,13 @@ final class SyncEngine
|
|||
private void downloadFileItem(Item item, string path)
|
||||
{
|
||||
assert(item.type == ItemType.file);
|
||||
write("Downloading ", path, "...");
|
||||
onedrive.downloadById(item.driveId, item.id, path);
|
||||
write("Downloading file ", path, " ... ");
|
||||
JSONValue fileSizeDetails = onedrive.getFileSize(item.driveId, item.id);
|
||||
auto fileSize = fileSizeDetails["size"].integer;
|
||||
onedrive.downloadById(item.driveId, item.id, path, fileSize);
|
||||
writeln("done.");
|
||||
log.fileOnly("Downloading file ", path, " ... done.");
|
||||
setTimes(path, item.mtime, item.mtime);
|
||||
writeln(" done.");
|
||||
log.fileOnly("Downloading ", path, "... done.");
|
||||
}
|
||||
|
||||
// returns true if the given item corresponds to the local one
|
||||
|
|
12
src/upload.d
12
src/upload.d
|
@ -1,4 +1,6 @@
|
|||
import std.algorithm, std.conv, std.datetime, std.file, std.json;
|
||||
import std.stdio, core.thread;
|
||||
import progress;
|
||||
import onedrive;
|
||||
static import log;
|
||||
|
||||
|
@ -88,10 +90,16 @@ struct UploadSession
|
|||
{
|
||||
long offset = session["nextExpectedRanges"][0].str.splitter('-').front.to!long;
|
||||
long fileSize = getSize(session["localPath"].str);
|
||||
|
||||
// Upload Progress Bar
|
||||
size_t iteration = (roundTo!int(double(fileSize)/double(fragmentSize)))+1;
|
||||
Progress p = new Progress(iteration);
|
||||
p.title = "Uploading";
|
||||
|
||||
JSONValue response;
|
||||
while (true) {
|
||||
p.next();
|
||||
long fragSize = fragmentSize < fileSize - offset ? fragmentSize : fileSize - offset;
|
||||
log.vlog("Uploading fragment: ", offset, "-", offset + fragSize, "/", fileSize);
|
||||
response = onedrive.uploadFragment(
|
||||
session["uploadUrl"].str,
|
||||
session["localPath"].str,
|
||||
|
@ -107,6 +115,8 @@ struct UploadSession
|
|||
save();
|
||||
}
|
||||
// upload complete
|
||||
p.next();
|
||||
writeln();
|
||||
remove(sessionFilePath);
|
||||
return response;
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue