Implement Feature Request: Add a progress bar for large uploads & downloads (Issue #12) (#99)

* Implement feature request to add a progress bar for large file uploads & downloads

Makefile

@@ -14,7 +14,8 @@ SOURCES = \
 	src/sqlite.d \
 	src/sync.d \
 	src/upload.d \
-	src/util.d
+	src/util.d \
+	src/progress.d
 
 all: onedrive onedrive.service
 

src/onedrive.d

@@ -1,8 +1,10 @@
 import std.net.curl: CurlException, HTTP;
 import std.datetime, std.exception, std.file, std.json, std.path;
 import std.stdio, std.string, std.uni, std.uri;
-import config;
 import core.stdc.stdlib;
+import core.thread, std.conv, std.math;
+import progress;
+import config;
 static import log;
 shared bool debugResponse = false;
 
@@ -118,7 +120,7 @@ final class OneDriveApi
 	}
 
 	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get_content
-	void downloadById(const(char)[] driveId, const(char)[] id, string saveToPath)
+	void downloadById(const(char)[] driveId, const(char)[] id, string saveToPath, long fileSize)
 	{
 		checkAccessTokenExpired();
 		scope(failure) {
@@ -126,7 +128,7 @@ final class OneDriveApi
 		}
 		mkdirRecurse(dirName(saveToPath));
 		const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content?AVOverride=1";
-		download(url, saveToPath);
+		download(url, saveToPath, fileSize);
 	}
 
 	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
@@ -202,6 +204,17 @@ final class OneDriveApi
 		return get(url);
 	}
 	
+	// Return the requested details of the specified id
+	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
+	JSONValue getFileSize(const(char)[] driveId, const(char)[] id)
+	{
+		checkAccessTokenExpired();
+		const(char)[] url;
+		//		string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
+		url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
+		url ~= "?select=size";
+		return get(url);
+	}
 	
 	// https://dev.onedrive.com/items/move.htm
 	JSONValue moveByPath(const(char)[] sourcePath, JSONValue moveData)
@@ -328,8 +341,11 @@ final class OneDriveApi
 		checkHttpCode(response);
 	}
 
-	private void download(const(char)[] url, string filename)
+	private void download(const(char)[] url, string filename, long fileSize)
 	{
+		// Threshold for displaying download bar
+		long thresholdFileSize = 4 * 2^^20; // 4 MiB
+		
 		scope(exit) http.clearRequestHeaders();
 		http.method = HTTP.Method.get;
 		http.url = url;
@@ -339,7 +355,43 @@ final class OneDriveApi
 			f.rawWrite(data);
 			return data.length;
 		};
-		http.perform();
+		
+		if (fileSize >= thresholdFileSize){
+			// Download Progress Bar
+			size_t iteration = 20;
+			Progress p = new Progress(iteration);
+			p.title = "Downloading";
+			writeln();
+	
+			real previousDLPercent = -1.0;
+			real percentCheck = 5.0;
+			// Setup progress bar to display
+			http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow)
+			{
+				// For each onProgress, what is the % of dlnow to dltotal
+				real currentDLPercent = round(double(dlnow)/dltotal*100);
+				// If matching 5% of download, increment progress bar
+				if ((isIdentical(fmod(currentDLPercent, percentCheck), 0.0)) && (previousDLPercent != currentDLPercent)) {
+					p.next();
+					previousDLPercent = currentDLPercent;
+				}
+				return 0;
+			};
+		
+			// Perform download & display progress bar
+			http.perform();
+			writeln();
+			// Reset onProgress to not display anything for next download
+			http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow)
+			{
+				return 0;
+			};
+		} else {
+			// No progress bar
+			http.perform();
+		}
+		
+		// Check the HTTP response code
 		checkHttpCode();
 	}
 

src/progress.d

@@ -0,0 +1,153 @@
+module progress;
+
+import std.stdio;
+import std.range;
+import std.format;
+import std.datetime;
+import core.sys.posix.unistd;
+import core.sys.posix.sys.ioctl;
+
+class Progress
+{
+  private:
+
+    immutable static size_t default_width = 80;
+    size_t max_width = 40;
+    size_t width = default_width;
+
+    ulong start_time;
+    string caption = "Progress";
+    size_t iterations;
+    size_t counter;
+
+
+    size_t getTerminalWidth() {
+      size_t column;
+      winsize ws;
+      if(ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) != -1) {
+        column = ws.ws_col;
+      }
+      if(column == 0) column = default_width;
+
+      return column;
+    }
+
+
+    void clear() {
+      write("\r");
+      for(auto i = 0; i < width; i++) write(" ");
+      write("\r");
+    }
+
+
+    int calc_eta() {
+      immutable auto ratio = cast(double)counter / iterations;
+      auto current_time = Clock.currTime.toUnixTime();
+      auto duration = cast(int)(current_time - start_time);
+      int hours, minutes, seconds;
+      double elapsed = (current_time - start_time);
+      int eta_sec = cast(int)((elapsed / ratio) - elapsed);
+
+	  // Return an ETA or Duration?
+	  if (eta_sec != 0){
+		return eta_sec;
+	  } else {
+	    return duration;
+	  }
+    }
+
+
+    string progressbarText(string header_text, string footer_text) {
+      immutable auto ratio = cast(double)counter / iterations;
+      string result = "";
+
+      double bar_length = width - header_text.length - footer_text.length;
+      if(bar_length > max_width && max_width > 0) {
+        bar_length = max_width;
+      }
+      size_t i = 0;
+      for(; i < ratio * bar_length; i++) result ~= "o";
+      for(; i < bar_length; i++) result ~= " ";
+
+      return header_text ~ result ~ footer_text;
+    }
+
+
+    void print() {
+      immutable auto ratio = cast(double)counter / iterations;
+      auto header = appender!string();
+      auto footer = appender!string();
+
+      header.formattedWrite("%s %3d%% |", caption, cast(int)(ratio * 100));
+
+      if(counter <= 1 || ratio == 0.0) {
+        footer.formattedWrite("|   ETA   --:--:--:");
+      } else {
+        int h, m, s;
+        dur!"seconds"(calc_eta())
+          .split!("hours", "minutes", "seconds")(h, m, s);
+		if (counter != iterations){   
+          footer.formattedWrite("|   ETA   %02d:%02d:%02d ", h, m, s);
+		} else {
+		  footer.formattedWrite("| DONE IN %02d:%02d:%02d ", h, m, s);
+		}
+      }
+
+      write(progressbarText(header.data, footer.data));
+    }
+
+
+    void update() {
+      width = getTerminalWidth();
+ 
+      clear();
+
+      print();
+      stdout.flush();
+    }
+
+
+  public:
+
+    this(size_t iterations) {
+      if(iterations <= 0) iterations = 1;
+
+      counter = 0;
+      this.iterations = iterations;
+      start_time = Clock.currTime.toUnixTime;
+    }
+
+    @property {
+      string title() { return caption; }
+      string title(string text) { return caption = text; }
+    }
+
+    @property {
+      size_t count() { return counter; }
+      size_t count(size_t val) {
+        if(val > iterations) val = iterations;
+        return counter = val;
+      }
+    }
+
+    @property {
+      size_t maxWidth() { return max_width; }
+      size_t maxWidth(size_t w) {
+        return max_width = w;
+      }
+    }
+
+    void reset() {
+      counter = 0;
+      start_time = Clock.currTime.toUnixTime;
+    }
+
+    void next() {
+      counter++;
+      if(counter > iterations) counter = iterations;
+
+      update();
+    }
+
+
+}

src/sync.d

@@ -642,11 +642,13 @@ final class SyncEngine
 	private void downloadFileItem(Item item, string path)
 	{
 		assert(item.type == ItemType.file);
-		write("Downloading ", path, "...");
-		onedrive.downloadById(item.driveId, item.id, path);
+		write("Downloading file ", path, " ... ");
+		JSONValue fileSizeDetails = onedrive.getFileSize(item.driveId, item.id);
+		auto fileSize = fileSizeDetails["size"].integer;
+		onedrive.downloadById(item.driveId, item.id, path, fileSize);
+		writeln("done.");
+		log.fileOnly("Downloading file ", path, " ... done.");
 		setTimes(path, item.mtime, item.mtime);
-		writeln(" done.");
-		log.fileOnly("Downloading ", path, "... done.");
 	}
 
 	// returns true if the given item corresponds to the local one

src/upload.d

@@ -1,4 +1,6 @@
 import std.algorithm, std.conv, std.datetime, std.file, std.json;
+import std.stdio, core.thread;
+import progress;
 import onedrive;
 static import log;
 
@@ -88,10 +90,16 @@ struct UploadSession
 	{
 		long offset = session["nextExpectedRanges"][0].str.splitter('-').front.to!long;
 		long fileSize = getSize(session["localPath"].str);
+		
+		// Upload Progress Bar
+		size_t iteration = (roundTo!int(double(fileSize)/double(fragmentSize)))+1;
+		Progress p = new Progress(iteration);
+		p.title = "Uploading";
+				
 		JSONValue response;
 		while (true) {
+			p.next();
 			long fragSize = fragmentSize < fileSize - offset ? fragmentSize : fileSize - offset;
-			log.vlog("Uploading fragment: ", offset, "-", offset + fragSize, "/", fileSize);
 			response = onedrive.uploadFragment(
 				session["uploadUrl"].str,
 				session["localPath"].str,
@@ -107,6 +115,8 @@ struct UploadSession
 			save();
 		}
 		// upload complete
+		p.next();
+		writeln();
 		remove(sessionFilePath);
 		return response;
 	}