2017-05-28 19:37:51 +02:00
|
|
|
import std.base64;
|
2015-11-29 21:12:44 +01:00
|
|
|
import std.conv;
|
2017-05-28 22:13:19 +02:00
|
|
|
import std.digest.crc, std.digest.sha;
|
2016-12-25 22:28:00 +01:00
|
|
|
import std.net.curl;
|
2018-04-30 11:41:54 +02:00
|
|
|
import std.datetime;
|
|
|
|
import std.file;
|
2015-11-29 21:12:44 +01:00
|
|
|
import std.path;
|
|
|
|
import std.regex;
|
|
|
|
import std.socket;
|
|
|
|
import std.stdio;
|
|
|
|
import std.string;
|
2018-07-02 06:05:21 +02:00
|
|
|
import std.algorithm;
|
2018-07-02 11:40:54 +02:00
|
|
|
import std.uri;
|
2020-12-09 04:18:16 +01:00
|
|
|
import std.json;
|
|
|
|
import std.traits;
|
2017-05-28 19:37:51 +02:00
|
|
|
import qxor;
|
2018-08-14 10:30:13 +02:00
|
|
|
static import log;
|
2015-09-01 20:45:34 +02:00
|
|
|
|
2019-03-11 07:57:47 +01:00
|
|
|
shared string deviceName;
|
2015-09-01 20:45:34 +02:00
|
|
|
|
|
|
|
static this()
|
|
|
|
{
|
|
|
|
deviceName = Socket.hostName;
|
|
|
|
}
|
|
|
|
|
2017-05-28 19:49:55 +02:00
|
|
|
// gives a new name to the specified file or directory
|
2015-09-01 20:45:34 +02:00
|
|
|
void safeRename(const(char)[] path)
|
|
|
|
{
|
|
|
|
auto ext = extension(path);
|
|
|
|
auto newPath = path.chomp(ext) ~ "-" ~ deviceName;
|
|
|
|
if (exists(newPath ~ ext)) {
|
|
|
|
int n = 2;
|
|
|
|
char[] newPath2;
|
|
|
|
do {
|
|
|
|
newPath2 = newPath ~ "-" ~ n.to!string;
|
|
|
|
n++;
|
|
|
|
} while (exists(newPath2 ~ ext));
|
|
|
|
newPath = newPath2;
|
|
|
|
}
|
|
|
|
newPath ~= ext;
|
|
|
|
rename(path, newPath);
|
|
|
|
}
|
|
|
|
|
2017-05-28 19:49:55 +02:00
|
|
|
// deletes the specified file without throwing an exception if it does not exists
|
2016-08-05 00:12:58 +02:00
|
|
|
void safeRemove(const(char)[] path)
|
|
|
|
{
|
|
|
|
if (exists(path)) remove(path);
|
|
|
|
}
|
|
|
|
|
2017-05-28 19:49:55 +02:00
|
|
|
// returns the crc32 hex string of a file
|
2015-09-01 20:45:34 +02:00
|
|
|
string computeCrc32(string path)
|
|
|
|
{
|
|
|
|
CRC32 crc;
|
|
|
|
auto file = File(path, "rb");
|
|
|
|
foreach (ubyte[] data; chunks(file, 4096)) {
|
|
|
|
crc.put(data);
|
|
|
|
}
|
|
|
|
return crc.finish().toHexString().dup;
|
|
|
|
}
|
2015-09-19 09:45:45 +02:00
|
|
|
|
2017-05-28 22:13:19 +02:00
|
|
|
// returns the sha1 hash hex string of a file
|
|
|
|
string computeSha1Hash(string path)
|
|
|
|
{
|
|
|
|
SHA1 sha;
|
|
|
|
auto file = File(path, "rb");
|
|
|
|
foreach (ubyte[] data; chunks(file, 4096)) {
|
|
|
|
sha.put(data);
|
|
|
|
}
|
|
|
|
return sha.finish().toHexString().dup;
|
|
|
|
}
|
|
|
|
|
2017-05-28 19:49:55 +02:00
|
|
|
// returns the quickXorHash base64 string of a file
|
2017-05-28 19:37:51 +02:00
|
|
|
string computeQuickXorHash(string path)
|
|
|
|
{
|
|
|
|
QuickXor qxor;
|
|
|
|
auto file = File(path, "rb");
|
|
|
|
foreach (ubyte[] data; chunks(file, 4096)) {
|
|
|
|
qxor.put(data);
|
|
|
|
}
|
|
|
|
return Base64.encode(qxor.finish());
|
|
|
|
}
|
|
|
|
|
2017-05-28 19:49:55 +02:00
|
|
|
// converts wildcards (*, ?) to regex
|
2015-09-21 13:04:05 +02:00
|
|
|
Regex!char wild2regex(const(char)[] pattern)
|
2015-09-19 09:45:45 +02:00
|
|
|
{
|
2015-09-21 13:04:05 +02:00
|
|
|
string str;
|
|
|
|
str.reserve(pattern.length + 2);
|
2017-03-12 16:07:45 +01:00
|
|
|
str ~= "^";
|
2015-09-19 09:45:45 +02:00
|
|
|
foreach (c; pattern) {
|
|
|
|
switch (c) {
|
|
|
|
case '*':
|
2015-09-21 13:04:05 +02:00
|
|
|
str ~= "[^/]*";
|
2015-09-19 09:45:45 +02:00
|
|
|
break;
|
|
|
|
case '.':
|
2015-09-21 13:04:05 +02:00
|
|
|
str ~= "\\.";
|
2015-09-19 09:45:45 +02:00
|
|
|
break;
|
|
|
|
case '?':
|
2015-09-21 13:04:05 +02:00
|
|
|
str ~= "[^/]";
|
2015-09-19 09:45:45 +02:00
|
|
|
break;
|
|
|
|
case '|':
|
2017-03-12 16:07:45 +01:00
|
|
|
str ~= "$|^";
|
2015-09-19 09:45:45 +02:00
|
|
|
break;
|
2018-06-28 05:46:20 +02:00
|
|
|
case '+':
|
|
|
|
str ~= "\\+";
|
2018-07-10 05:08:17 +02:00
|
|
|
break;
|
|
|
|
case ' ':
|
|
|
|
str ~= "\\s+";
|
2018-06-28 05:46:20 +02:00
|
|
|
break;
|
2018-07-10 05:08:17 +02:00
|
|
|
case '/':
|
|
|
|
str ~= "\\/";
|
|
|
|
break;
|
2020-03-27 21:49:33 +01:00
|
|
|
case '(':
|
|
|
|
str ~= "\\(";
|
|
|
|
break;
|
|
|
|
case ')':
|
|
|
|
str ~= "\\)";
|
|
|
|
break;
|
2015-09-19 09:45:45 +02:00
|
|
|
default:
|
2015-09-21 13:04:05 +02:00
|
|
|
str ~= c;
|
2015-09-19 09:45:45 +02:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2015-09-21 13:04:05 +02:00
|
|
|
str ~= "$";
|
|
|
|
return regex(str, "i");
|
2015-09-19 09:45:45 +02:00
|
|
|
}
|
2015-11-29 21:12:44 +01:00
|
|
|
|
2017-05-28 19:49:55 +02:00
|
|
|
// returns true if the network connection is available
|
2015-11-29 21:12:44 +01:00
|
|
|
bool testNetwork()
|
|
|
|
{
|
2020-05-20 03:37:11 +02:00
|
|
|
// Use low level HTTP struct
|
|
|
|
auto http = HTTP();
|
|
|
|
http.url = "https://login.microsoftonline.com";
|
2020-12-17 20:17:52 +01:00
|
|
|
// DNS lookup timeout
|
2020-05-20 03:37:11 +02:00
|
|
|
http.dnsTimeout = (dur!"seconds"(5));
|
2020-12-17 20:17:52 +01:00
|
|
|
// Timeout for connecting
|
|
|
|
http.connectTimeout = (dur!"seconds"(5));
|
|
|
|
// HTTP connection test method
|
2020-05-20 03:37:11 +02:00
|
|
|
http.method = HTTP.Method.head;
|
|
|
|
// Attempt to contact the Microsoft Online Service
|
2020-12-17 20:17:52 +01:00
|
|
|
try {
|
|
|
|
log.vdebug("Attempting to contact online service");
|
2018-04-24 04:10:27 +02:00
|
|
|
http.perform();
|
2020-12-17 20:17:52 +01:00
|
|
|
log.vdebug("Shutting down HTTP engine as sucessfully reached OneDrive Online Service");
|
2020-05-20 03:37:11 +02:00
|
|
|
http.shutdown();
|
2018-04-24 04:10:27 +02:00
|
|
|
return true;
|
2020-12-17 20:17:52 +01:00
|
|
|
} catch (SocketException e) {
|
|
|
|
// Socket issue
|
|
|
|
log.vdebug("HTTP Socket Issue");
|
|
|
|
log.error("Cannot connect to Microsoft OneDrive Service - Socket Issue");
|
|
|
|
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
|
|
|
return false;
|
|
|
|
} catch (CurlException e) {
|
|
|
|
// No network connection to OneDrive Service
|
|
|
|
log.vdebug("No Network Connection");
|
|
|
|
log.error("Cannot connect to Microsoft OneDrive Service - Network Connection Issue");
|
|
|
|
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
2018-04-24 04:10:27 +02:00
|
|
|
return false;
|
|
|
|
}
|
2015-11-29 21:12:44 +01:00
|
|
|
}
|
2016-09-18 11:50:10 +02:00
|
|
|
|
2018-08-14 10:30:13 +02:00
|
|
|
// Can we read the file - as a permissions issue or file corruption will cause a failure
|
|
|
|
// https://github.com/abraunegg/onedrive/issues/113
|
|
|
|
// returns true if file can be accessed
|
|
|
|
bool readLocalFile(string path)
|
|
|
|
{
|
|
|
|
try {
|
2019-03-11 07:56:26 +01:00
|
|
|
// attempt to read up to the first 1 byte of the file
|
|
|
|
// validates we can 'read' the file based on file permissions
|
|
|
|
read(path,1);
|
2018-08-14 10:30:13 +02:00
|
|
|
} catch (std.file.FileException e) {
|
|
|
|
// unable to read the new local file
|
2020-12-11 19:58:07 +01:00
|
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
2018-08-14 10:30:13 +02:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2017-05-28 19:49:55 +02:00
|
|
|
// calls globMatch for each string in pattern separated by '|'
|
2016-09-18 11:50:10 +02:00
|
|
|
bool multiGlobMatch(const(char)[] path, const(char)[] pattern)
|
|
|
|
{
|
|
|
|
foreach (glob; pattern.split('|')) {
|
|
|
|
if (globMatch!(std.path.CaseSensitive.yes)(path, glob)) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2018-04-13 03:20:38 +02:00
|
|
|
bool isValidName(string path)
|
|
|
|
{
|
2018-07-02 06:05:21 +02:00
|
|
|
// Restriction and limitations about windows naming files
|
|
|
|
// https://msdn.microsoft.com/en-us/library/aa365247
|
|
|
|
// https://support.microsoft.com/en-us/help/3125202/restrictions-and-limitations-when-you-sync-files-and-folders
|
|
|
|
|
2018-04-13 03:20:38 +02:00
|
|
|
// allow root item
|
|
|
|
if (path == ".") {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2018-07-02 06:05:21 +02:00
|
|
|
bool matched = true;
|
2018-04-13 03:20:38 +02:00
|
|
|
string itemName = baseName(path);
|
|
|
|
|
|
|
|
auto invalidNameReg =
|
|
|
|
ctRegex!(
|
2018-07-02 06:05:21 +02:00
|
|
|
// Leading whitespace and trailing whitespace/dot
|
2018-04-13 03:20:38 +02:00
|
|
|
`^\s.*|^.*[\s\.]$|` ~
|
2018-07-02 06:05:21 +02:00
|
|
|
// Invalid characters
|
2018-05-25 06:12:39 +02:00
|
|
|
`.*[<>:"\|\?*/\\].*|` ~
|
2018-07-02 06:05:21 +02:00
|
|
|
// Reserved device name and trailing .~
|
|
|
|
`(?:^CON|^PRN|^AUX|^NUL|^COM[0-9]|^LPT[0-9])(?:[.].+)?$`
|
2018-04-13 03:20:38 +02:00
|
|
|
);
|
|
|
|
auto m = match(itemName, invalidNameReg);
|
2018-07-02 06:05:21 +02:00
|
|
|
matched = m.empty;
|
|
|
|
|
|
|
|
// Additional explicit validation checks
|
|
|
|
if (itemName == ".lock") {matched = false;}
|
|
|
|
if (itemName == "desktop.ini") {matched = false;}
|
|
|
|
// _vti_ cannot appear anywhere in a file or folder name
|
|
|
|
if(canFind(itemName, "_vti_")){matched = false;}
|
2019-11-14 01:53:07 +01:00
|
|
|
// Item name cannot equal '~'
|
|
|
|
if (itemName == "~") {matched = false;}
|
2018-07-02 06:05:21 +02:00
|
|
|
|
|
|
|
// return response
|
|
|
|
return matched;
|
2018-04-13 03:20:38 +02:00
|
|
|
}
|
|
|
|
|
2018-07-02 11:40:54 +02:00
|
|
|
bool containsBadWhiteSpace(string path)
|
|
|
|
{
|
|
|
|
// allow root item
|
|
|
|
if (path == ".") {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
// https://github.com/abraunegg/onedrive/issues/35
|
|
|
|
// Issue #35 presented an interesting issue where the filename contained a newline item
|
|
|
|
// 'State-of-the-art, challenges, and open issues in the integration of Internet of'$'\n''Things and Cloud Computing.pdf'
|
|
|
|
// When the check to see if this file was present the GET request queries as follows:
|
|
|
|
// /v1.0/me/drive/root:/.%2FState-of-the-art%2C%20challenges%2C%20and%20open%20issues%20in%20the%20integration%20of%20Internet%20of%0AThings%20and%20Cloud%20Computing.pdf
|
|
|
|
// The '$'\n'' is translated to %0A which causes the OneDrive query to fail
|
|
|
|
// Check for the presence of '%0A' via regex
|
|
|
|
|
|
|
|
string itemName = encodeComponent(baseName(path));
|
|
|
|
auto invalidWhitespaceReg =
|
|
|
|
ctRegex!(
|
|
|
|
// Check for \n which is %0A when encoded
|
|
|
|
`%0A`
|
|
|
|
);
|
|
|
|
auto m = match(itemName, invalidWhitespaceReg);
|
|
|
|
return m.empty;
|
|
|
|
}
|
|
|
|
|
2018-10-04 01:33:39 +02:00
|
|
|
bool containsASCIIHTMLCodes(string path)
|
|
|
|
{
|
|
|
|
// https://github.com/abraunegg/onedrive/issues/151
|
|
|
|
// If a filename contains ASCII HTML codes, regardless of if it gets encoded, it generates an error
|
|
|
|
// Check if the filename contains an ASCII HTML code sequence
|
|
|
|
|
|
|
|
auto invalidASCIICode =
|
|
|
|
ctRegex!(
|
|
|
|
// Check to see if &#XXXX is in the filename
|
|
|
|
`(?:&#|&#[0-9][0-9]|&#[0-9][0-9][0-9]|&#[0-9][0-9][0-9][0-9])`
|
|
|
|
);
|
|
|
|
|
|
|
|
auto m = match(path, invalidASCIICode);
|
|
|
|
return m.empty;
|
|
|
|
}
|
2018-07-02 11:40:54 +02:00
|
|
|
|
2020-12-09 04:18:16 +01:00
|
|
|
// Parse and display error message received from OneDrive
|
|
|
|
void displayOneDriveErrorMessage(string message, string callingFunction)
|
|
|
|
{
|
|
|
|
log.error("\nERROR: Microsoft OneDrive API returned an error with the following message:");
|
|
|
|
auto errorArray = splitLines(message);
|
|
|
|
log.error(" Error Message: ", errorArray[0]);
|
|
|
|
// Extract 'message' as the reason
|
|
|
|
JSONValue errorMessage = parseJSON(replace(message, errorArray[0], ""));
|
|
|
|
// extra debug
|
|
|
|
log.vdebug("Raw Error Data: ", message);
|
|
|
|
log.vdebug("JSON Message: ", errorMessage);
|
|
|
|
|
|
|
|
// What is the reason for the error
|
|
|
|
if (errorMessage.type() == JSONType.object) {
|
|
|
|
// configure the error reason
|
|
|
|
string errorReason;
|
|
|
|
string requestDate;
|
|
|
|
string requestId;
|
|
|
|
|
|
|
|
// set the reason for the error
|
|
|
|
try {
|
|
|
|
// Use error_description as reason
|
|
|
|
errorReason = errorMessage["error_description"].str;
|
|
|
|
} catch (JSONException e) {
|
|
|
|
// we dont want to do anything here
|
|
|
|
}
|
|
|
|
|
|
|
|
// set the reason for the error
|
|
|
|
try {
|
|
|
|
// Use ["error"]["message"] as reason
|
|
|
|
errorReason = errorMessage["error"]["message"].str;
|
|
|
|
} catch (JSONException e) {
|
|
|
|
// we dont want to do anything here
|
|
|
|
}
|
|
|
|
|
|
|
|
// Display the error reason
|
|
|
|
if (errorReason.startsWith("<!DOCTYPE")) {
|
|
|
|
// a HTML Error Reason was given
|
|
|
|
log.error(" Error Reason: A HTML Error response was provided. Use debug logging (--verbose --verbose) to view this error");
|
|
|
|
log.vdebug(errorReason);
|
|
|
|
} else {
|
|
|
|
// a non HTML Error Reason was given
|
|
|
|
log.error(" Error Reason: ", errorReason);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get the date of request if available
|
|
|
|
try {
|
|
|
|
// Use ["error"]["innerError"]["date"] as date
|
|
|
|
requestDate = errorMessage["error"]["innerError"]["date"].str;
|
|
|
|
} catch (JSONException e) {
|
|
|
|
// we dont want to do anything here
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get the request-id if available
|
|
|
|
try {
|
|
|
|
// Use ["error"]["innerError"]["request-id"] as request-id
|
|
|
|
requestId = errorMessage["error"]["innerError"]["request-id"].str;
|
|
|
|
} catch (JSONException e) {
|
|
|
|
// we dont want to do anything here
|
|
|
|
}
|
|
|
|
|
|
|
|
// Display the date and request id if available
|
|
|
|
if (requestDate != "") log.error(" Error Timestamp: ", requestDate);
|
|
|
|
if (requestId != "") log.error(" API Request ID: ", requestId);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Where in the code was this error generated
|
|
|
|
log.error(" Calling Function: ", callingFunction);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Parse and display error message received from the local file system
|
|
|
|
void displayFileSystemErrorMessage(string message, string callingFunction)
|
|
|
|
{
|
2020-12-11 19:58:07 +01:00
|
|
|
log.error("\nERROR: The local file system returned an error with the following message:");
|
2020-12-09 04:18:16 +01:00
|
|
|
auto errorArray = splitLines(message);
|
|
|
|
// What was the error message
|
|
|
|
log.error(" Error Message: ", errorArray[0]);
|
|
|
|
// Where in the code was this error generated
|
|
|
|
log.error(" Calling Function: ", callingFunction);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get the function name that is being called to assist with identifying where an error is being generated
|
|
|
|
string getFunctionName(alias func)() {
|
|
|
|
return __traits(identifier, __traits(parent, func)) ~ "()\n";
|
|
|
|
}
|
|
|
|
|
|
|
|
// Unit Tests
|
2018-04-13 03:20:38 +02:00
|
|
|
unittest
|
|
|
|
{
|
2018-04-24 04:10:27 +02:00
|
|
|
assert(multiGlobMatch(".hidden", ".*"));
|
|
|
|
assert(multiGlobMatch(".hidden", "file|.*"));
|
|
|
|
assert(!multiGlobMatch("foo.bar", "foo|bar"));
|
2018-04-13 03:20:38 +02:00
|
|
|
// that should detect invalid file/directory name.
|
|
|
|
assert(isValidName("."));
|
|
|
|
assert(isValidName("./general.file"));
|
|
|
|
assert(!isValidName("./ leading_white_space"));
|
|
|
|
assert(!isValidName("./trailing_white_space "));
|
|
|
|
assert(!isValidName("./trailing_dot."));
|
|
|
|
assert(!isValidName("./includes<in the path"));
|
|
|
|
assert(!isValidName("./includes>in the path"));
|
|
|
|
assert(!isValidName("./includes:in the path"));
|
|
|
|
assert(!isValidName(`./includes"in the path`));
|
|
|
|
assert(!isValidName("./includes|in the path"));
|
|
|
|
assert(!isValidName("./includes?in the path"));
|
|
|
|
assert(!isValidName("./includes*in the path"));
|
|
|
|
assert(!isValidName("./includes / in the path"));
|
|
|
|
assert(!isValidName(`./includes\ in the path`));
|
|
|
|
assert(!isValidName(`./includes\\ in the path`));
|
|
|
|
assert(!isValidName(`./includes\\\\ in the path`));
|
|
|
|
assert(!isValidName("./includes\\ in the path"));
|
|
|
|
assert(!isValidName("./includes\\\\ in the path"));
|
|
|
|
assert(!isValidName("./CON"));
|
|
|
|
assert(!isValidName("./CON.text"));
|
|
|
|
assert(!isValidName("./PRN"));
|
|
|
|
assert(!isValidName("./AUX"));
|
|
|
|
assert(!isValidName("./NUL"));
|
|
|
|
assert(!isValidName("./COM0"));
|
|
|
|
assert(!isValidName("./COM1"));
|
|
|
|
assert(!isValidName("./COM2"));
|
|
|
|
assert(!isValidName("./COM3"));
|
|
|
|
assert(!isValidName("./COM4"));
|
|
|
|
assert(!isValidName("./COM5"));
|
|
|
|
assert(!isValidName("./COM6"));
|
|
|
|
assert(!isValidName("./COM7"));
|
|
|
|
assert(!isValidName("./COM8"));
|
|
|
|
assert(!isValidName("./COM9"));
|
|
|
|
assert(!isValidName("./LPT0"));
|
|
|
|
assert(!isValidName("./LPT1"));
|
|
|
|
assert(!isValidName("./LPT2"));
|
|
|
|
assert(!isValidName("./LPT3"));
|
|
|
|
assert(!isValidName("./LPT4"));
|
|
|
|
assert(!isValidName("./LPT5"));
|
|
|
|
assert(!isValidName("./LPT6"));
|
|
|
|
assert(!isValidName("./LPT7"));
|
|
|
|
assert(!isValidName("./LPT8"));
|
|
|
|
assert(!isValidName("./LPT9"));
|
|
|
|
}
|