mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-17 05:06:17 +02:00
d169dfc642
* When enabling system logging to a log file, the actual ERROR line is forced to a new line in the application log. The reason for this is the \n prefix in the error message, which was in place so that when performing CLI logging or systemd logging, the error message would be displayed clearly. This change removes the \n from the actual error message, but inserts a newline before the error message is displayed (and also in some cases post error message) - thus keeping the application runtime look and feel, but improving the application log output.
1853 lines
66 KiB
D
1853 lines
66 KiB
D
import std.net.curl;
|
||
import etc.c.curl: CurlOption;
|
||
import std.datetime, std.datetime.systime, std.exception, std.file, std.json, std.path;
|
||
import std.stdio, std.string, std.uni, std.uri, std.file, std.uuid;
|
||
import std.array: split;
|
||
import core.atomic : atomicOp;
|
||
import core.stdc.stdlib;
|
||
import core.thread, std.conv, std.math;
|
||
import std.algorithm.searching;
|
||
import std.concurrency;
|
||
import progress;
|
||
import config;
|
||
import util;
|
||
import arsd.cgi;
|
||
import std.datetime;
|
||
static import log;
|
||
shared bool debugResponse = false;
|
||
private bool dryRun = false;
|
||
private bool simulateNoRefreshTokenFile = false;
|
||
private ulong retryAfterValue = 0;
|
||
|
||
private immutable {
|
||
// Client ID / Application ID (abraunegg)
|
||
string clientIdDefault = "d50ca740-c83f-4d1b-b616-12c519384f0c";
|
||
|
||
// Azure Active Directory & Graph Explorer Endpoints
|
||
// Global & Defaults
|
||
string globalAuthEndpoint = "https://login.microsoftonline.com";
|
||
string globalGraphEndpoint = "https://graph.microsoft.com";
|
||
|
||
// US Government L4
|
||
string usl4AuthEndpoint = "https://login.microsoftonline.us";
|
||
string usl4GraphEndpoint = "https://graph.microsoft.us";
|
||
|
||
// US Government L5
|
||
string usl5AuthEndpoint = "https://login.microsoftonline.us";
|
||
string usl5GraphEndpoint = "https://dod-graph.microsoft.us";
|
||
|
||
// Germany
|
||
string deAuthEndpoint = "https://login.microsoftonline.de";
|
||
string deGraphEndpoint = "https://graph.microsoft.de";
|
||
|
||
// China
|
||
string cnAuthEndpoint = "https://login.chinacloudapi.cn";
|
||
string cnGraphEndpoint = "https://microsoftgraph.chinacloudapi.cn";
|
||
}
|
||
|
||
private {
|
||
// Client ID / Application ID
|
||
string clientId = clientIdDefault;
|
||
|
||
// Default User Agent configuration
|
||
string isvTag = "ISV";
|
||
string companyName = "abraunegg";
|
||
// Application name as per Microsoft Azure application registration
|
||
string appTitle = "OneDrive Client for Linux";
|
||
|
||
// Default Drive ID
|
||
string driveId = "";
|
||
|
||
// API Query URL's, based on using defaults, but can be updated by config option 'azure_ad_endpoint'
|
||
// Authentication
|
||
string authUrl = globalAuthEndpoint ~ "/common/oauth2/v2.0/authorize";
|
||
string redirectUrl = globalAuthEndpoint ~ "/common/oauth2/nativeclient";
|
||
string tokenUrl = globalAuthEndpoint ~ "/common/oauth2/v2.0/token";
|
||
|
||
// Drive Queries
|
||
string driveUrl = globalGraphEndpoint ~ "/v1.0/me/drive";
|
||
string driveByIdUrl = globalGraphEndpoint ~ "/v1.0/drives/";
|
||
|
||
// What is 'shared with me' Query
|
||
string sharedWithMeUrl = globalGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
|
||
|
||
// Item Queries
|
||
string itemByIdUrl = globalGraphEndpoint ~ "/v1.0/me/drive/items/";
|
||
string itemByPathUrl = globalGraphEndpoint ~ "/v1.0/me/drive/root:/";
|
||
|
||
// Office 365 / SharePoint Queries
|
||
string siteSearchUrl = globalGraphEndpoint ~ "/v1.0/sites?search";
|
||
string siteDriveUrl = globalGraphEndpoint ~ "/v1.0/sites/";
|
||
|
||
// Subscriptions
|
||
string subscriptionUrl = globalGraphEndpoint ~ "/v1.0/subscriptions";
|
||
}
|
||
|
||
class OneDriveException: Exception
|
||
{
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/errors
|
||
int httpStatusCode;
|
||
JSONValue error;
|
||
|
||
@safe pure this(int httpStatusCode, string reason, string file = __FILE__, size_t line = __LINE__)
|
||
{
|
||
this.httpStatusCode = httpStatusCode;
|
||
this.error = error;
|
||
string msg = format("HTTP request returned status code %d (%s)", httpStatusCode, reason);
|
||
super(msg, file, line);
|
||
}
|
||
|
||
this(int httpStatusCode, string reason, ref const JSONValue error, string file = __FILE__, size_t line = __LINE__)
|
||
{
|
||
this.httpStatusCode = httpStatusCode;
|
||
this.error = error;
|
||
string msg = format("HTTP request returned status code %d (%s)\n%s", httpStatusCode, reason, toJSON(error, true));
|
||
super(msg, file, line);
|
||
}
|
||
}
|
||
|
||
class OneDriveWebhook {
|
||
// We need OneDriveWebhook.serve to be a static function, otherwise we would hit the member function
|
||
// "requires a dual-context, which is deprecated" warning. The root cause is described here:
|
||
// - https://issues.dlang.org/show_bug.cgi?id=5710
|
||
// - https://forum.dlang.org/post/fkyppfxzegenniyzztos@forum.dlang.org
|
||
// The problem is deemed a bug and should be fixed in the compilers eventually. The singleton stuff
|
||
// could be undone when it is fixed.
|
||
//
|
||
// Following the singleton pattern described here: https://wiki.dlang.org/Low-Lock_Singleton_Pattern
|
||
// Cache instantiation flag in thread-local bool
|
||
// Thread local
|
||
private static bool instantiated_;
|
||
|
||
// Thread global
|
||
private __gshared OneDriveWebhook instance_;
|
||
|
||
private string host;
|
||
private ushort port;
|
||
private Tid parentTid;
|
||
private shared uint count;
|
||
|
||
static OneDriveWebhook getOrCreate(string host, ushort port, Tid parentTid) {
|
||
if (!instantiated_) {
|
||
synchronized(OneDriveWebhook.classinfo) {
|
||
if (!instance_) {
|
||
instance_ = new OneDriveWebhook(host, port, parentTid);
|
||
}
|
||
|
||
instantiated_ = true;
|
||
}
|
||
}
|
||
|
||
return instance_;
|
||
}
|
||
|
||
private this(string host, ushort port, Tid parentTid) {
|
||
this.host = host;
|
||
this.port = port;
|
||
this.parentTid = parentTid;
|
||
this.count = 0;
|
||
}
|
||
|
||
// The static serve() is necessary because spawn() does not like instance methods
|
||
static serve() {
|
||
// we won't create the singleton instance if it hasn't been created already
|
||
// such case is a bug which should crash the program and gets fixed
|
||
instance_.serveImpl();
|
||
}
|
||
|
||
// The static handle() is necessary to work around the dual-context warning mentioned above
|
||
private static void handle(Cgi cgi) {
|
||
// we won't create the singleton instance if it hasn't been created already
|
||
// such case is a bug which should crash the program and gets fixed
|
||
instance_.handleImpl(cgi);
|
||
}
|
||
|
||
private void serveImpl() {
|
||
auto server = new RequestServer(host, port);
|
||
server.serveEmbeddedHttp!handle();
|
||
}
|
||
|
||
private void handleImpl(Cgi cgi) {
|
||
if (.debugResponse) {
|
||
log.log("Webhook request: ", cgi.requestMethod, " ", cgi.requestUri);
|
||
if (!cgi.postBody.empty) {
|
||
log.log("Webhook post body: ", cgi.postBody);
|
||
}
|
||
}
|
||
|
||
cgi.setResponseContentType("text/plain");
|
||
|
||
if ("validationToken" in cgi.get) {
|
||
// For validation requests, respond with the validation token passed in the query string
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/webhook-receiver-validation-request
|
||
cgi.write(cgi.get["validationToken"]);
|
||
log.log("Webhook: handled validation request");
|
||
} else {
|
||
// Notifications don't include any information about the changes that triggered them.
|
||
// Put a refresh signal in the queue and let the main monitor loop process it.
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/using-webhooks
|
||
count.atomicOp!"+="(1);
|
||
send(parentTid, to!ulong(count));
|
||
cgi.write("OK");
|
||
log.log("Webhook: sent refresh signal #", count);
|
||
}
|
||
}
|
||
}
|
||
|
||
final class OneDriveApi
|
||
{
|
||
private Config cfg;
|
||
private string refreshToken, accessToken, subscriptionId;
|
||
private SysTime accessTokenExpiration;
|
||
private HTTP http;
|
||
private OneDriveWebhook webhook;
|
||
private SysTime subscriptionExpiration;
|
||
private Duration subscriptionExpirationInterval, subscriptionRenewalInterval;
|
||
private string notificationUrl;
|
||
|
||
// if true, every new access token is printed
|
||
bool printAccessToken;
|
||
|
||
this(Config cfg)
|
||
{
|
||
this.cfg = cfg;
|
||
http = HTTP();
|
||
// Curl Timeout Handling
|
||
// libcurl dns_cache_timeout timeout
|
||
http.dnsTimeout = (dur!"seconds"(60));
|
||
// Timeout for HTTPS connections
|
||
http.connectTimeout = (dur!"seconds"(10));
|
||
// with the following settings we force
|
||
// - if there is no data flow for 10min, abort
|
||
// - if the download time for one item exceeds 1h, abort
|
||
//
|
||
// timeout for activity on connection
|
||
// this translates into Curl's CURLOPT_LOW_SPEED_TIME
|
||
// which says
|
||
// It contains the time in number seconds that the
|
||
// transfer speed should be below the CURLOPT_LOW_SPEED_LIMIT
|
||
// for the library to consider it too slow and abort.
|
||
http.dataTimeout = (dur!"seconds"(600));
|
||
// maximum time an operation is allowed to take
|
||
// This includes dns resolution, connecting, data transfer, etc.
|
||
http.operationTimeout = (dur!"seconds"(cfg.getValueLong("operation_timeout")));
|
||
// Specify how many redirects should be allowed
|
||
http.maxRedirects(5);
|
||
|
||
// Do we enable curl debugging?
|
||
if (cfg.getValueBool("debug_https")) {
|
||
http.verbose = true;
|
||
.debugResponse = true;
|
||
}
|
||
|
||
// Update clientId if application_id is set in config file
|
||
if (cfg.getValueString("application_id") != "") {
|
||
// an application_id is set in config file
|
||
log.vdebug("Setting custom application_id to: " , cfg.getValueString("application_id"));
|
||
clientId = cfg.getValueString("application_id");
|
||
companyName = "custom_application";
|
||
}
|
||
|
||
// Configure tenant id value, if 'azure_tenant_id' is configured,
|
||
// otherwise use the "common" multiplexer
|
||
string tenantId = "common";
|
||
if (cfg.getValueString("azure_tenant_id") != "") {
|
||
// Use the value entered by the user
|
||
tenantId = cfg.getValueString("azure_tenant_id");
|
||
}
|
||
|
||
// Configure Azure AD endpoints if 'azure_ad_endpoint' is configured
|
||
string azureConfigValue = cfg.getValueString("azure_ad_endpoint");
|
||
switch(azureConfigValue) {
|
||
case "":
|
||
if (tenantId == "common") {
|
||
log.log("Configuring Global Azure AD Endpoints");
|
||
} else {
|
||
log.log("Configuring Global Azure AD Endpoints - Single Tenant Application");
|
||
}
|
||
// Authentication
|
||
authUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
|
||
redirectUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||
tokenUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
|
||
break;
|
||
case "USL4":
|
||
log.log("Configuring Azure AD for US Government Endpoints");
|
||
// Authentication
|
||
authUrl = usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
|
||
tokenUrl = usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
|
||
if (clientId == clientIdDefault) {
|
||
// application_id == default
|
||
log.vdebug("USL4 AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint");
|
||
redirectUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||
} else {
|
||
// custom application_id
|
||
redirectUrl = usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||
}
|
||
|
||
// Drive Queries
|
||
driveUrl = usl4GraphEndpoint ~ "/v1.0/me/drive";
|
||
driveByIdUrl = usl4GraphEndpoint ~ "/v1.0/drives/";
|
||
// Item Queries
|
||
itemByIdUrl = usl4GraphEndpoint ~ "/v1.0/me/drive/items/";
|
||
itemByPathUrl = usl4GraphEndpoint ~ "/v1.0/me/drive/root:/";
|
||
// Office 365 / SharePoint Queries
|
||
siteSearchUrl = usl4GraphEndpoint ~ "/v1.0/sites?search";
|
||
siteDriveUrl = usl4GraphEndpoint ~ "/v1.0/sites/";
|
||
// Shared With Me
|
||
sharedWithMeUrl = usl4GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
|
||
// Subscriptions
|
||
subscriptionUrl = usl4GraphEndpoint ~ "/v1.0/subscriptions";
|
||
break;
|
||
case "USL5":
|
||
log.log("Configuring Azure AD for US Government Endpoints (DOD)");
|
||
// Authentication
|
||
authUrl = usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
|
||
tokenUrl = usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
|
||
if (clientId == clientIdDefault) {
|
||
// application_id == default
|
||
log.vdebug("USL5 AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint");
|
||
redirectUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||
} else {
|
||
// custom application_id
|
||
redirectUrl = usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||
}
|
||
|
||
// Drive Queries
|
||
driveUrl = usl5GraphEndpoint ~ "/v1.0/me/drive";
|
||
driveByIdUrl = usl5GraphEndpoint ~ "/v1.0/drives/";
|
||
// Item Queries
|
||
itemByIdUrl = usl5GraphEndpoint ~ "/v1.0/me/drive/items/";
|
||
itemByPathUrl = usl5GraphEndpoint ~ "/v1.0/me/drive/root:/";
|
||
// Office 365 / SharePoint Queries
|
||
siteSearchUrl = usl5GraphEndpoint ~ "/v1.0/sites?search";
|
||
siteDriveUrl = usl5GraphEndpoint ~ "/v1.0/sites/";
|
||
// Shared With Me
|
||
sharedWithMeUrl = usl5GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
|
||
// Subscriptions
|
||
subscriptionUrl = usl5GraphEndpoint ~ "/v1.0/subscriptions";
|
||
break;
|
||
case "DE":
|
||
log.log("Configuring Azure AD Germany");
|
||
// Authentication
|
||
authUrl = deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
|
||
tokenUrl = deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
|
||
if (clientId == clientIdDefault) {
|
||
// application_id == default
|
||
log.vdebug("DE AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint");
|
||
redirectUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||
} else {
|
||
// custom application_id
|
||
redirectUrl = deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||
}
|
||
|
||
// Drive Queries
|
||
driveUrl = deGraphEndpoint ~ "/v1.0/me/drive";
|
||
driveByIdUrl = deGraphEndpoint ~ "/v1.0/drives/";
|
||
// Item Queries
|
||
itemByIdUrl = deGraphEndpoint ~ "/v1.0/me/drive/items/";
|
||
itemByPathUrl = deGraphEndpoint ~ "/v1.0/me/drive/root:/";
|
||
// Office 365 / SharePoint Queries
|
||
siteSearchUrl = deGraphEndpoint ~ "/v1.0/sites?search";
|
||
siteDriveUrl = deGraphEndpoint ~ "/v1.0/sites/";
|
||
// Shared With Me
|
||
sharedWithMeUrl = deGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
|
||
// Subscriptions
|
||
subscriptionUrl = deGraphEndpoint ~ "/v1.0/subscriptions";
|
||
break;
|
||
case "CN":
|
||
log.log("Configuring AD China operated by 21Vianet");
|
||
// Authentication
|
||
authUrl = cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
|
||
tokenUrl = cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
|
||
if (clientId == clientIdDefault) {
|
||
// application_id == default
|
||
log.vdebug("CN AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint");
|
||
redirectUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||
} else {
|
||
// custom application_id
|
||
redirectUrl = cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||
}
|
||
|
||
// Drive Queries
|
||
driveUrl = cnGraphEndpoint ~ "/v1.0/me/drive";
|
||
driveByIdUrl = cnGraphEndpoint ~ "/v1.0/drives/";
|
||
// Item Queries
|
||
itemByIdUrl = cnGraphEndpoint ~ "/v1.0/me/drive/items/";
|
||
itemByPathUrl = cnGraphEndpoint ~ "/v1.0/me/drive/root:/";
|
||
// Office 365 / SharePoint Queries
|
||
siteSearchUrl = cnGraphEndpoint ~ "/v1.0/sites?search";
|
||
siteDriveUrl = cnGraphEndpoint ~ "/v1.0/sites/";
|
||
// Shared With Me
|
||
sharedWithMeUrl = cnGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
|
||
// Subscriptions
|
||
subscriptionUrl = cnGraphEndpoint ~ "/v1.0/subscriptions";
|
||
break;
|
||
// Default - all other entries
|
||
default:
|
||
log.log("Unknown Azure AD Endpoint request - using Global Azure AD Endpoints");
|
||
}
|
||
|
||
// Debug output of configured URL's
|
||
// Authentication
|
||
log.vdebug("Configured authUrl: ", authUrl);
|
||
log.vdebug("Configured redirectUrl: ", redirectUrl);
|
||
log.vdebug("Configured tokenUrl: ", tokenUrl);
|
||
|
||
// Drive Queries
|
||
log.vdebug("Configured driveUrl: ", driveUrl);
|
||
log.vdebug("Configured driveByIdUrl: ", driveByIdUrl);
|
||
|
||
// Shared With Me
|
||
log.vdebug("Configured sharedWithMeUrl: ", sharedWithMeUrl);
|
||
|
||
// Item Queries
|
||
log.vdebug("Configured itemByIdUrl: ", itemByIdUrl);
|
||
log.vdebug("Configured itemByPathUrl: ", itemByPathUrl);
|
||
|
||
// SharePoint Queries
|
||
log.vdebug("Configured siteSearchUrl: ", siteSearchUrl);
|
||
log.vdebug("Configured siteDriveUrl: ", siteDriveUrl);
|
||
|
||
// Configure the User Agent string
|
||
if (cfg.getValueString("user_agent") == "") {
|
||
// Application User Agent string defaults
|
||
// Comply with OneDrive traffic decoration requirements
|
||
// https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
|
||
// - Identify as ISV and include Company Name, App Name separated by a pipe character and then adding Version number separated with a slash character
|
||
// Note: If you've created an application, the recommendation is to register and use AppID and AppTitle
|
||
// The issue here is that currently the application is still using the 'skilion' application ID, thus no idea what the AppTitle used was.
|
||
http.setUserAgent = isvTag ~ "|" ~ companyName ~ "|" ~ appTitle ~ "/" ~ strip(import("version"));
|
||
} else {
|
||
// Use the value entered by the user
|
||
http.setUserAgent = cfg.getValueString("user_agent");
|
||
}
|
||
|
||
// What version of HTTP protocol do we use?
|
||
// Curl >= 7.62.0 defaults to http2 for a significant number of operations
|
||
if (cfg.getValueBool("force_http_11")) {
|
||
// Downgrade to curl to use HTTP 1.1 for all operations
|
||
log.vlog("Downgrading all HTTP operations to HTTP/1.1 due to user configuration");
|
||
// Downgrade to HTTP 1.1 - yes version = 2 is HTTP 1.1
|
||
http.handle.set(CurlOption.http_version,2);
|
||
} else {
|
||
// Use curl defaults
|
||
log.vlog("Using Curl defaults for all HTTP operations");
|
||
}
|
||
|
||
// Configure upload / download rate limits if configured
|
||
long userRateLimit = cfg.getValueLong("rate_limit");
|
||
// 131072 = 128 KB/s - minimum for basic application operations to prevent timeouts
|
||
// A 0 value means rate is unlimited, and is the curl default
|
||
|
||
if (userRateLimit > 0) {
|
||
// User configured rate limit
|
||
writeln("User Configured Rate Limit: ", userRateLimit);
|
||
|
||
// If user provided rate limit is < 131072, flag that this is too low, setting to the minimum of 131072
|
||
if (userRateLimit < 131072) {
|
||
// user provided limit too low
|
||
log.log("WARNING: User configured rate limit too low for normal application processing and preventing application timeouts. Overriding to default minimum of 131072 (128KB/s)");
|
||
userRateLimit = 131072;
|
||
}
|
||
|
||
// set rate limit
|
||
http.handle.set(CurlOption.max_send_speed_large,userRateLimit);
|
||
http.handle.set(CurlOption.max_recv_speed_large,userRateLimit);
|
||
}
|
||
|
||
// Explicitly set libcurl options
|
||
// https://curl.se/libcurl/c/CURLOPT_NOSIGNAL.html
|
||
// Ensure that nosignal is set to 0 - Setting CURLOPT_NOSIGNAL to 0 makes libcurl ask the system to ignore SIGPIPE signals
|
||
http.handle.set(CurlOption.nosignal,0);
|
||
// https://curl.se/libcurl/c/CURLOPT_TCP_NODELAY.html
|
||
// Ensure that TCP_NODELAY is set to 0 to ensure that TCP NAGLE is enabled
|
||
http.handle.set(CurlOption.tcp_nodelay,0);
|
||
// https://curl.se/libcurl/c/CURLOPT_FORBID_REUSE.html
|
||
// Ensure that we ARE reusing connections - setting to 0 ensures that we are reusing connections
|
||
http.handle.set(CurlOption.forbid_reuse,0);
|
||
|
||
// Do we set the dryRun handlers?
|
||
if (cfg.getValueBool("dry_run")) {
|
||
.dryRun = true;
|
||
if (cfg.getValueBool("logout")) {
|
||
.simulateNoRefreshTokenFile = true;
|
||
}
|
||
}
|
||
|
||
subscriptionExpiration = Clock.currTime(UTC());
|
||
subscriptionExpirationInterval = dur!"seconds"(cfg.getValueLong("webhook_expiration_interval"));
|
||
subscriptionRenewalInterval = dur!"seconds"(cfg.getValueLong("webhook_renewal_interval"));
|
||
notificationUrl = cfg.getValueString("webhook_public_url");
|
||
}
|
||
|
||
// Shutdown OneDrive HTTP construct
|
||
void shutdown()
|
||
{
|
||
// delete subscription if there exists any
|
||
deleteSubscription();
|
||
|
||
// reset any values to defaults, freeing any set objects
|
||
http.clearRequestHeaders();
|
||
http.onSend = null;
|
||
http.onReceive = null;
|
||
http.onReceiveHeader = null;
|
||
http.onReceiveStatusLine = null;
|
||
http.contentLength = 0;
|
||
// shut down the curl instance
|
||
http.shutdown();
|
||
}
|
||
|
||
bool init()
|
||
{
|
||
static import std.utf;
|
||
// detail what we are using for applicaion identification
|
||
log.vdebug("clientId = ", clientId);
|
||
log.vdebug("companyName = ", companyName);
|
||
log.vdebug("appTitle = ", appTitle);
|
||
|
||
try {
|
||
driveId = cfg.getValueString("drive_id");
|
||
if (driveId.length) {
|
||
driveUrl = driveByIdUrl ~ driveId;
|
||
itemByIdUrl = driveUrl ~ "/items";
|
||
itemByPathUrl = driveUrl ~ "/root:/";
|
||
}
|
||
} catch (Exception e) {}
|
||
|
||
if (!.dryRun) {
|
||
// original code
|
||
try {
|
||
refreshToken = readText(cfg.refreshTokenFilePath);
|
||
} catch (FileException e) {
|
||
try {
|
||
return authorize();
|
||
} catch (CurlException e) {
|
||
log.error("Cannot authorize with Microsoft OneDrive Service");
|
||
return false;
|
||
}
|
||
} catch (std.utf.UTFException e) {
|
||
// path contains characters which generate a UTF exception
|
||
log.error("Cannot read refreshToken from: ", cfg.refreshTokenFilePath);
|
||
log.error(" Error Reason:", e.msg);
|
||
return false;
|
||
}
|
||
return true;
|
||
} else {
|
||
// --dry-run
|
||
if (!.simulateNoRefreshTokenFile) {
|
||
try {
|
||
refreshToken = readText(cfg.refreshTokenFilePath);
|
||
} catch (FileException e) {
|
||
return authorize();
|
||
} catch (std.utf.UTFException e) {
|
||
// path contains characters which generate a UTF exception
|
||
log.error("Cannot read refreshToken from: ", cfg.refreshTokenFilePath);
|
||
log.error(" Error Reason:", e.msg);
|
||
return false;
|
||
}
|
||
return true;
|
||
} else {
|
||
// --dry-run & --reauth
|
||
return authorize();
|
||
}
|
||
}
|
||
}
|
||
|
||
bool authorize()
|
||
{
|
||
import std.stdio, std.regex;
|
||
char[] response;
|
||
string authScope;
|
||
// What authentication scope to use?
|
||
if (cfg.getValueBool("read_only_auth_scope")) {
|
||
// read-only authentication scopes has been requested
|
||
authScope = "&scope=Files.Read%20Files.Read.All%20Sites.Read.All%20offline_access&response_type=code&prompt=login&redirect_uri=";
|
||
} else {
|
||
// read-write authentication scopes will be used (default)
|
||
authScope = "&scope=Files.ReadWrite%20Files.ReadWrite.All%20Sites.ReadWrite.All%20offline_access&response_type=code&prompt=login&redirect_uri=";
|
||
}
|
||
|
||
string url = authUrl ~ "?client_id=" ~ clientId ~ authScope ~ redirectUrl;
|
||
string authFilesString = cfg.getValueString("auth_files");
|
||
string authResponseString = cfg.getValueString("auth_response");
|
||
if (authResponseString != "") {
|
||
response = cast(char[]) authResponseString;
|
||
} else if (authFilesString != "") {
|
||
string[] authFiles = authFilesString.split(":");
|
||
string authUrl = authFiles[0];
|
||
string responseUrl = authFiles[1];
|
||
auto authUrlFile = File(authUrl, "w");
|
||
authUrlFile.write(url);
|
||
authUrlFile.close();
|
||
while (!exists(responseUrl)) {
|
||
Thread.sleep(dur!("msecs")(100));
|
||
}
|
||
|
||
// read response from OneDrive
|
||
try {
|
||
response = cast(char[]) read(responseUrl);
|
||
} catch (OneDriveException e) {
|
||
// exception generated
|
||
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
||
return false;
|
||
}
|
||
|
||
// try to remove old files
|
||
try {
|
||
std.file.remove(authUrl);
|
||
std.file.remove(responseUrl);
|
||
} catch (FileException e) {
|
||
log.error("Cannot remove files ", authUrl, " ", responseUrl);
|
||
return false;
|
||
}
|
||
} else {
|
||
log.log("Authorize this app visiting:\n");
|
||
write(url, "\n\n", "Enter the response uri: ");
|
||
readln(response);
|
||
cfg.applicationAuthorizeResponseUri = true;
|
||
}
|
||
// match the authorization code
|
||
auto c = matchFirst(response, r"(?:[\?&]code=)([\w\d-.]+)");
|
||
if (c.empty) {
|
||
log.log("Invalid response uri entered");
|
||
return false;
|
||
}
|
||
c.popFront(); // skip the whole match
|
||
redeemToken(c.front);
|
||
return true;
|
||
}
|
||
|
||
ulong getRetryAfterValue()
|
||
{
|
||
// Return the current value of retryAfterValue if it has been set to something other than 0
|
||
return .retryAfterValue;
|
||
}
|
||
|
||
void resetRetryAfterValue()
|
||
{
|
||
// Reset the current value of retryAfterValue to 0 after it has been used
|
||
.retryAfterValue = 0;
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get
|
||
JSONValue getDefaultDrive()
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
url = driveUrl;
|
||
return get(driveUrl);
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
|
||
JSONValue getDefaultRoot()
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
url = driveUrl ~ "/root";
|
||
return get(url);
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
|
||
JSONValue getDriveIdRoot(const(char)[] driveId)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
url = driveByIdUrl ~ driveId ~ "/root";
|
||
return get(url);
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/graph/api/drive-sharedwithme
|
||
JSONValue getSharedWithMe()
|
||
{
|
||
checkAccessTokenExpired();
|
||
return get(sharedWithMeUrl);
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get
|
||
JSONValue getDriveQuota(const(char)[] driveId)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
url = driveByIdUrl ~ driveId ~ "/";
|
||
url ~= "?select=quota";
|
||
return get(url);
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta
|
||
JSONValue viewChangesByItemId(const(char)[] driveId, const(char)[] id, const(char)[] deltaLink)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
// configure deltaLink to query
|
||
if (deltaLink.empty) {
|
||
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/delta";
|
||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
|
||
} else {
|
||
url = deltaLink;
|
||
}
|
||
return get(url);
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta
|
||
JSONValue viewChangesByDriveId(const(char)[] driveId, const(char)[] deltaLink)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url = deltaLink;
|
||
if (url == null) {
|
||
url = driveByIdUrl ~ driveId ~ "/root/delta";
|
||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
|
||
}
|
||
return get(url);
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_list_children
|
||
JSONValue listChildren(const(char)[] driveId, const(char)[] id, const(char)[] nextLink)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
// configure URL to query
|
||
if (nextLink.empty) {
|
||
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/children";
|
||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
|
||
} else {
|
||
url = nextLink;
|
||
}
|
||
return get(url);
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get_content
|
||
void downloadById(const(char)[] driveId, const(char)[] id, string saveToPath, long fileSize)
|
||
{
|
||
checkAccessTokenExpired();
|
||
scope(failure) {
|
||
if (exists(saveToPath)) {
|
||
// try and remove the file, catch error
|
||
try {
|
||
remove(saveToPath);
|
||
} catch (FileException e) {
|
||
// display the error message
|
||
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
||
}
|
||
}
|
||
}
|
||
|
||
// Create the required local directory
|
||
string newPath = dirName(saveToPath);
|
||
|
||
// Does the path exist locally?
|
||
if (!exists(newPath)) {
|
||
try {
|
||
log.vdebug("Requested path does not exist, creating directory structure: ", newPath);
|
||
mkdirRecurse(newPath);
|
||
// Configure the applicable permissions for the folder
|
||
log.vdebug("Setting directory permissions for: ", newPath);
|
||
newPath.setAttributes(cfg.returnRequiredDirectoryPermisions());
|
||
} catch (FileException e) {
|
||
// display the error message
|
||
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
||
}
|
||
}
|
||
|
||
const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content?AVOverride=1";
|
||
// Download file
|
||
download(url, saveToPath, fileSize);
|
||
// Does path exist?
|
||
if (exists(saveToPath)) {
|
||
// File was downloaded successfully - configure the applicable permissions for the file
|
||
log.vdebug("Setting file permissions for: ", saveToPath);
|
||
saveToPath.setAttributes(cfg.returnRequiredFilePermisions());
|
||
}
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
|
||
JSONValue simpleUpload(string localPath, string parentDriveId, string parentId, string filename, const(char)[] eTag = null)
|
||
{
|
||
checkAccessTokenExpired();
|
||
string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/content";
|
||
// TODO: investigate why this fails for remote folders
|
||
//if (eTag) http.addRequestHeader("If-Match", eTag);
|
||
/*else http.addRequestHeader("If-None-Match", "*");*/
|
||
return upload(localPath, url);
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
|
||
JSONValue simpleUploadReplace(string localPath, string driveId, string id, const(char)[] eTag = null)
|
||
{
|
||
checkAccessTokenExpired();
|
||
string url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content";
|
||
if (eTag) http.addRequestHeader("If-Match", eTag);
|
||
return upload(localPath, url);
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_update
|
||
JSONValue updateById(const(char)[] driveId, const(char)[] id, JSONValue data, const(char)[] eTag = null)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
|
||
if (eTag) http.addRequestHeader("If-Match", eTag);
|
||
http.addRequestHeader("Content-Type", "application/json");
|
||
return patch(url, data.toString());
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delete
|
||
void deleteById(const(char)[] driveId, const(char)[] id, const(char)[] eTag = null)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
|
||
//TODO: investigate why this always fail with 412 (Precondition Failed)
|
||
//if (eTag) http.addRequestHeader("If-Match", eTag);
|
||
del(url);
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_post_children
|
||
JSONValue createById(const(char)[] parentDriveId, const(char)[] parentId, JSONValue item)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ "/children";
|
||
http.addRequestHeader("Content-Type", "application/json");
|
||
return post(url, item.toString());
|
||
}
|
||
|
||
// Return the details of the specified path
|
||
JSONValue getPathDetails(const(string) path)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
if ((path == ".")||(path == "/")) url = driveUrl ~ "/root/";
|
||
else url = itemByPathUrl ~ encodeComponent(path) ~ ":/";
|
||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
|
||
return get(url);
|
||
}
|
||
|
||
// Return the details of the specified id
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
|
||
JSONValue getPathDetailsById(const(char)[] driveId, const(char)[] id)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
|
||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
|
||
return get(url);
|
||
}
|
||
|
||
// Return the requested details of the specified path on the specified drive id and path
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get?view=odsp-graph-online
|
||
JSONValue getPathDetailsByDriveId(const(char)[] driveId, const(string) path)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
// string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
|
||
// Required format: /drives/{drive-id}/root:/{item-path}
|
||
url = driveByIdUrl ~ driveId ~ "/root:/" ~ encodeComponent(path);
|
||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
|
||
return get(url);
|
||
}
|
||
|
||
// Return the requested details of the specified path on the specified drive id and item id
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get?view=odsp-graph-online
|
||
JSONValue getPathDetailsByDriveIdAndItemId(const(char)[] driveId, const(char)[] itemId)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
// string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
|
||
// Required format: /drives/{drive-id}/items/{item-id}
|
||
url = driveByIdUrl ~ driveId ~ "/items/" ~ itemId;
|
||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
|
||
return get(url);
|
||
}
|
||
|
||
// Return the requested details of the specified id
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
|
||
JSONValue getFileDetails(const(char)[] driveId, const(char)[] id)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
|
||
url ~= "?select=size,malware,file,webUrl,lastModifiedBy,lastModifiedDateTime";
|
||
return get(url);
|
||
}
|
||
|
||
// Create an anonymous read-only shareable link for an existing file on OneDrive
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createlink
|
||
JSONValue createShareableLink(const(char)[] driveId, const(char)[] id, JSONValue accessScope)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/createLink";
|
||
http.addRequestHeader("Content-Type", "application/json");
|
||
return post(url, accessScope.toString());
|
||
}
|
||
|
||
// https://dev.onedrive.com/items/move.htm
|
||
JSONValue moveByPath(const(char)[] sourcePath, JSONValue moveData)
|
||
{
|
||
// Need to use itemByPathUrl
|
||
checkAccessTokenExpired();
|
||
string url = itemByPathUrl ~ encodeComponent(sourcePath);
|
||
http.addRequestHeader("Content-Type", "application/json");
|
||
return move(url, moveData.toString());
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
|
||
JSONValue createUploadSession(const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null, JSONValue item = null)
|
||
{
|
||
checkAccessTokenExpired();
|
||
const(char)[] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/createUploadSession";
|
||
if (eTag) http.addRequestHeader("If-Match", eTag);
|
||
http.addRequestHeader("Content-Type", "application/json");
|
||
return post(url, item.toString());
|
||
}
|
||
|
||
// https://dev.onedrive.com/items/upload_large_files.htm
|
||
JSONValue uploadFragment(const(char)[] uploadUrl, string filepath, long offset, long offsetSize, long fileSize)
|
||
{
|
||
checkAccessTokenExpired();
|
||
// open file as read-only in binary mode
|
||
auto file = File(filepath, "rb");
|
||
file.seek(offset);
|
||
string contentRange = "bytes " ~ to!string(offset) ~ "-" ~ to!string(offset + offsetSize - 1) ~ "/" ~ to!string(fileSize);
|
||
log.vdebugNewLine("contentRange: ", contentRange);
|
||
|
||
// function scopes
|
||
scope(exit) {
|
||
http.clearRequestHeaders();
|
||
http.onSend = null;
|
||
http.onReceive = null;
|
||
http.onReceiveHeader = null;
|
||
http.onReceiveStatusLine = null;
|
||
http.contentLength = 0;
|
||
// close file if open
|
||
if (file.isOpen()){
|
||
// close open file
|
||
file.close();
|
||
}
|
||
}
|
||
|
||
http.method = HTTP.Method.put;
|
||
http.url = uploadUrl;
|
||
http.addRequestHeader("Content-Range", contentRange);
|
||
http.onSend = data => file.rawRead(data).length;
|
||
// convert offsetSize to ulong
|
||
http.contentLength = to!ulong(offsetSize);
|
||
auto response = perform();
|
||
// TODO: retry on 5xx errors
|
||
checkHttpCode(response);
|
||
return response;
|
||
}
|
||
|
||
// https://dev.onedrive.com/items/upload_large_files.htm
|
||
JSONValue requestUploadStatus(const(char)[] uploadUrl)
|
||
{
|
||
checkAccessTokenExpired();
|
||
// when using microsoft graph the auth code is different
|
||
return get(uploadUrl, true);
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/site_search?view=odsp-graph-online
|
||
JSONValue o365SiteSearch(const(char)[] nextLink){
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
// configure URL to query
|
||
if (nextLink.empty) {
|
||
url = siteSearchUrl ~ "=*";
|
||
} else {
|
||
url = nextLink;
|
||
}
|
||
return get(url);
|
||
}
|
||
|
||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_list?view=odsp-graph-online
|
||
JSONValue o365SiteDrives(string site_id){
|
||
checkAccessTokenExpired();
|
||
const(char)[] url;
|
||
url = siteDriveUrl ~ site_id ~ "/drives";
|
||
return get(url);
|
||
}
|
||
|
||
// Create a new subscription or renew the existing subscription
|
||
void createOrRenewSubscription() {
|
||
checkAccessTokenExpired();
|
||
|
||
// Kick off the webhook server first
|
||
if (webhook is null) {
|
||
webhook = OneDriveWebhook.getOrCreate(
|
||
cfg.getValueString("webhook_listening_host"),
|
||
to!ushort(cfg.getValueLong("webhook_listening_port")),
|
||
thisTid
|
||
);
|
||
spawn(&OneDriveWebhook.serve);
|
||
}
|
||
|
||
if (!hasValidSubscription()) {
|
||
createSubscription();
|
||
} else if (isSubscriptionUpForRenewal()) {
|
||
try {
|
||
renewSubscription();
|
||
} catch (OneDriveException e) {
|
||
if (e.httpStatusCode == 404) {
|
||
log.log("The subscription is not found on the server. Recreating subscription ...");
|
||
createSubscription();
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
private bool hasValidSubscription() {
|
||
return !subscriptionId.empty && subscriptionExpiration > Clock.currTime(UTC());
|
||
}
|
||
|
||
private bool isSubscriptionUpForRenewal() {
|
||
return subscriptionExpiration < Clock.currTime(UTC()) + subscriptionRenewalInterval;
|
||
}
|
||
|
||
private void createSubscription() {
|
||
log.log("Initializing subscription for updates ...");
|
||
|
||
auto expirationDateTime = Clock.currTime(UTC()) + subscriptionExpirationInterval;
|
||
const(char)[] url;
|
||
url = subscriptionUrl;
|
||
const JSONValue request = [
|
||
"changeType": "updated",
|
||
"notificationUrl": notificationUrl,
|
||
"resource": "/me/drive/root",
|
||
"expirationDateTime": expirationDateTime.toISOExtString(),
|
||
"clientState": randomUUID().toString()
|
||
];
|
||
http.addRequestHeader("Content-Type", "application/json");
|
||
JSONValue response;
|
||
|
||
try {
|
||
response = post(url, request.toString());
|
||
} catch (OneDriveException e) {
|
||
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
||
|
||
// We need to exit here, user needs to fix issue
|
||
log.error("ERROR: Unable to initialize subscriptions for updates. Please fix this issue.");
|
||
shutdown();
|
||
exit(-1);
|
||
}
|
||
|
||
// Save important subscription metadata including id and expiration
|
||
subscriptionId = response["id"].str;
|
||
subscriptionExpiration = SysTime.fromISOExtString(response["expirationDateTime"].str);
|
||
}
|
||
|
||
private void renewSubscription() {
|
||
log.log("Renewing subscription for updates ...");
|
||
|
||
auto expirationDateTime = Clock.currTime(UTC()) + subscriptionExpirationInterval;
|
||
const(char)[] url;
|
||
url = subscriptionUrl ~ "/" ~ subscriptionId;
|
||
const JSONValue request = [
|
||
"expirationDateTime": expirationDateTime.toISOExtString()
|
||
];
|
||
http.addRequestHeader("Content-Type", "application/json");
|
||
JSONValue response = patch(url, request.toString());
|
||
|
||
// Update subscription expiration from the response
|
||
subscriptionExpiration = SysTime.fromISOExtString(response["expirationDateTime"].str);
|
||
}
|
||
|
||
private void deleteSubscription() {
|
||
if (!hasValidSubscription()) {
|
||
return;
|
||
}
|
||
|
||
const(char)[] url;
|
||
url = subscriptionUrl ~ "/" ~ subscriptionId;
|
||
del(url);
|
||
log.log("Deleted subscription");
|
||
}
|
||
|
||
private void redeemToken(const(char)[] authCode)
|
||
{
|
||
const(char)[] postData =
|
||
"client_id=" ~ clientId ~
|
||
"&redirect_uri=" ~ redirectUrl ~
|
||
"&code=" ~ authCode ~
|
||
"&grant_type=authorization_code";
|
||
acquireToken(postData);
|
||
}
|
||
|
||
private void newToken()
|
||
{
|
||
string postData =
|
||
"client_id=" ~ clientId ~
|
||
"&redirect_uri=" ~ redirectUrl ~
|
||
"&refresh_token=" ~ refreshToken ~
|
||
"&grant_type=refresh_token";
|
||
acquireToken(postData);
|
||
}
|
||
|
||
private void acquireToken(const(char)[] postData)
|
||
{
|
||
JSONValue response;
|
||
|
||
try {
|
||
response = post(tokenUrl, postData);
|
||
} catch (OneDriveException e) {
|
||
// an error was generated
|
||
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
||
}
|
||
|
||
if (response.type() == JSONType.object) {
|
||
// Has the client been configured to use read_only_auth_scope
|
||
if (cfg.getValueBool("read_only_auth_scope")) {
|
||
// read_only_auth_scope has been configured
|
||
if ("scope" in response){
|
||
string effectiveScopes = response["scope"].str();
|
||
// Display the effective authentication scopes
|
||
writeln();
|
||
writeln("Effective API Authentication Scopes: ", effectiveScopes);
|
||
// if we have any write scopes, we need to tell the user to update an remove online prior authentication and exit application
|
||
if (canFind(effectiveScopes, "Write")) {
|
||
// effective scopes contain write scopes .. so not a read-only configuration
|
||
writeln();
|
||
writeln("ERROR: You have authentication scopes that allow write operations. You need to remove your existing application access consent");
|
||
writeln();
|
||
writeln("Please login to https://account.live.com/consent/Manage and remove your existing application access consent");
|
||
writeln();
|
||
// force exit
|
||
shutdown();
|
||
exit(-1);
|
||
}
|
||
}
|
||
}
|
||
|
||
if ("access_token" in response){
|
||
accessToken = "bearer " ~ response["access_token"].str();
|
||
refreshToken = response["refresh_token"].str();
|
||
accessTokenExpiration = Clock.currTime() + dur!"seconds"(response["expires_in"].integer());
|
||
if (!.dryRun) {
|
||
try {
|
||
// try and update the refresh_token file
|
||
std.file.write(cfg.refreshTokenFilePath, refreshToken);
|
||
log.vdebug("Setting file permissions for: ", cfg.refreshTokenFilePath);
|
||
cfg.refreshTokenFilePath.setAttributes(cfg.returnRequiredFilePermisions());
|
||
} catch (FileException e) {
|
||
// display the error message
|
||
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
||
}
|
||
}
|
||
if (printAccessToken) writeln("New access token: ", accessToken);
|
||
} else {
|
||
log.error("\nInvalid authentication response from OneDrive. Please check the response uri\n");
|
||
// re-authorize
|
||
authorize();
|
||
}
|
||
} else {
|
||
log.vdebug("Invalid JSON response from OneDrive unable to initialize application");
|
||
}
|
||
}
|
||
|
||
private void checkAccessTokenExpired()
|
||
{
|
||
try {
|
||
if (Clock.currTime() >= accessTokenExpiration) {
|
||
newToken();
|
||
}
|
||
} catch (OneDriveException e) {
|
||
if (e.httpStatusCode == 400 || e.httpStatusCode == 401) {
|
||
// flag error and notify
|
||
writeln();
|
||
log.errorAndNotify("ERROR: Refresh token invalid, use --reauth to authorize the client again.");
|
||
writeln();
|
||
// set error message
|
||
e.msg ~= "\nRefresh token invalid, use --reauth to authorize the client again";
|
||
}
|
||
}
|
||
}
|
||
|
||
private void addAccessTokenHeader()
|
||
{
|
||
http.addRequestHeader("Authorization", accessToken);
|
||
}
|
||
|
||
private JSONValue get(const(char)[] url, bool skipToken = false)
|
||
{
|
||
scope(exit) http.clearRequestHeaders();
|
||
log.vdebug("Request URL = ", url);
|
||
http.method = HTTP.Method.get;
|
||
http.url = url;
|
||
if (!skipToken) addAccessTokenHeader(); // HACK: requestUploadStatus
|
||
JSONValue response;
|
||
response = perform();
|
||
checkHttpCode(response);
|
||
// OneDrive API Response Debugging if --https-debug is being used
|
||
if (.debugResponse){
|
||
log.vdebug("OneDrive API Response: ", response);
|
||
}
|
||
return response;
|
||
}
|
||
|
||
private void del(const(char)[] url)
|
||
{
|
||
scope(exit) http.clearRequestHeaders();
|
||
http.method = HTTP.Method.del;
|
||
http.url = url;
|
||
addAccessTokenHeader();
|
||
auto response = perform();
|
||
checkHttpCode(response);
|
||
}
|
||
|
||
private void download(const(char)[] url, string filename, long fileSize)
|
||
{
|
||
// Threshold for displaying download bar
|
||
long thresholdFileSize = 4 * 2^^20; // 4 MiB
|
||
|
||
// To support marking of partially-downloaded files,
|
||
string originalFilename = filename;
|
||
string downloadFilename = filename ~ ".partial";
|
||
|
||
// open downloadFilename as write in binary mode
|
||
auto file = File(downloadFilename, "wb");
|
||
|
||
// function scopes
|
||
scope(exit) {
|
||
http.clearRequestHeaders();
|
||
http.onSend = null;
|
||
http.onReceive = null;
|
||
http.onReceiveHeader = null;
|
||
http.onReceiveStatusLine = null;
|
||
http.contentLength = 0;
|
||
// Reset onProgress to not display anything for next download
|
||
http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow)
|
||
{
|
||
return 0;
|
||
};
|
||
// close file if open
|
||
if (file.isOpen()){
|
||
// close open file
|
||
file.close();
|
||
}
|
||
}
|
||
|
||
http.method = HTTP.Method.get;
|
||
http.url = url;
|
||
addAccessTokenHeader();
|
||
|
||
http.onReceive = (ubyte[] data) {
|
||
file.rawWrite(data);
|
||
return data.length;
|
||
};
|
||
|
||
if (fileSize >= thresholdFileSize){
|
||
// Download Progress Bar
|
||
size_t iteration = 20;
|
||
Progress p = new Progress(iteration);
|
||
p.title = "Downloading";
|
||
writeln();
|
||
bool barInit = false;
|
||
real previousProgressPercent = -1.0;
|
||
real percentCheck = 5.0;
|
||
long segmentCount = 1;
|
||
// Setup progress bar to display
|
||
http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow)
|
||
{
|
||
// For each onProgress, what is the % of dlnow to dltotal
|
||
// floor - rounds down to nearest whole number
|
||
real currentDLPercent = floor(double(dlnow)/dltotal*100);
|
||
// Have we started downloading?
|
||
if (currentDLPercent > 0){
|
||
// We have started downloading
|
||
log.vdebugNewLine("Data Received = ", dlnow);
|
||
log.vdebug("Expected Total = ", dltotal);
|
||
log.vdebug("Percent Complete = ", currentDLPercent);
|
||
// Every 5% download we need to increment the download bar
|
||
|
||
// Has the user set a data rate limit?
|
||
// when using rate_limit, we will get odd download rates, for example:
|
||
// Percent Complete = 24
|
||
// Data Received = 13080163
|
||
// Expected Total = 52428800
|
||
// Percent Complete = 24
|
||
// Data Received = 13685777
|
||
// Expected Total = 52428800
|
||
// Percent Complete = 26 <---- jumps to 26% missing 25%, thus fmod misses incrementing progress bar
|
||
// Data Received = 13685777
|
||
// Expected Total = 52428800
|
||
// Percent Complete = 26
|
||
|
||
if (cfg.getValueLong("rate_limit") > 0) {
|
||
// User configured rate limit
|
||
// How much data should be in each segment to qualify for 5%
|
||
long dataPerSegment = to!long(floor(double(dltotal)/iteration));
|
||
// How much data received do we need to validate against
|
||
long thisSegmentData = dataPerSegment * segmentCount;
|
||
long nextSegmentData = dataPerSegment * (segmentCount + 1);
|
||
// Has the data that has been received in a 5% window that we need to increment the progress bar at
|
||
if ((dlnow > thisSegmentData) && (dlnow < nextSegmentData) && (previousProgressPercent != currentDLPercent) || (dlnow == dltotal)) {
|
||
// Downloaded data equals approx 5%
|
||
log.vdebug("Incrementing Progress Bar using calculated 5% of data received");
|
||
// Downloading 50% |oooooooooooooooooooo | ETA 00:01:40
|
||
// increment progress bar
|
||
p.next();
|
||
// update values
|
||
log.vdebug("Setting previousProgressPercent to ", currentDLPercent);
|
||
previousProgressPercent = currentDLPercent;
|
||
log.vdebug("Incrementing segmentCount");
|
||
segmentCount++;
|
||
}
|
||
} else {
|
||
// Is currentDLPercent divisible by 5 leaving remainder 0 and does previousProgressPercent not equal currentDLPercent
|
||
if ((isIdentical(fmod(currentDLPercent, percentCheck), 0.0)) && (previousProgressPercent != currentDLPercent)) {
|
||
// currentDLPercent matches a new increment
|
||
log.vdebug("Incrementing Progress Bar using fmod match");
|
||
// Downloading 50% |oooooooooooooooooooo | ETA 00:01:40
|
||
// increment progress bar
|
||
p.next();
|
||
// update values
|
||
previousProgressPercent = currentDLPercent;
|
||
}
|
||
}
|
||
} else {
|
||
if ((currentDLPercent == 0) && (!barInit)) {
|
||
// Initialise the download bar at 0%
|
||
// Downloading 0% | | ETA --:--:--:
|
||
p.next();
|
||
barInit = true;
|
||
}
|
||
}
|
||
return 0;
|
||
};
|
||
|
||
// Perform download & display progress bar
|
||
try {
|
||
// try and catch any curl error
|
||
http.perform();
|
||
// Check the HTTP Response headers - needed for correct 429 handling
|
||
// check will be performed in checkHttpCode()
|
||
writeln();
|
||
// Reset onProgress to not display anything for next download done using exit scope
|
||
} catch (CurlException e) {
|
||
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
||
}
|
||
// free progress bar memory
|
||
p = null;
|
||
} else {
|
||
// No progress bar
|
||
try {
|
||
// try and catch any curl error
|
||
http.perform();
|
||
// Check the HTTP Response headers - needed for correct 429 handling
|
||
// check will be performed in checkHttpCode()
|
||
} catch (CurlException e) {
|
||
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
||
}
|
||
}
|
||
|
||
// Rename downloaded file
|
||
rename(downloadFilename, originalFilename);
|
||
|
||
// Check the HTTP response code, which, if a 429, will also check response headers
|
||
checkHttpCode();
|
||
}
|
||
|
||
private auto patch(T)(const(char)[] url, const(T)[] patchData)
|
||
{
|
||
scope(exit) http.clearRequestHeaders();
|
||
http.method = HTTP.Method.patch;
|
||
http.url = url;
|
||
addAccessTokenHeader();
|
||
auto response = perform(patchData);
|
||
checkHttpCode(response);
|
||
return response;
|
||
}
|
||
|
||
private auto post(T)(const(char)[] url, const(T)[] postData)
|
||
{
|
||
scope(exit) http.clearRequestHeaders();
|
||
http.method = HTTP.Method.post;
|
||
http.url = url;
|
||
addAccessTokenHeader();
|
||
auto response = perform(postData);
|
||
checkHttpCode(response);
|
||
return response;
|
||
}
|
||
|
||
private auto move(T)(const(char)[] url, const(T)[] postData)
|
||
{
|
||
scope(exit) http.clearRequestHeaders();
|
||
http.method = HTTP.Method.patch;
|
||
http.url = url;
|
||
addAccessTokenHeader();
|
||
auto response = perform(postData);
|
||
// Check the HTTP response code, which, if a 429, will also check response headers
|
||
checkHttpCode();
|
||
return response;
|
||
}
|
||
|
||
private JSONValue upload(string filepath, string url)
|
||
{
|
||
checkAccessTokenExpired();
|
||
// open file as read-only in binary mode
|
||
auto file = File(filepath, "rb");
|
||
|
||
// function scopes
|
||
scope(exit) {
|
||
http.clearRequestHeaders();
|
||
http.onSend = null;
|
||
http.onReceive = null;
|
||
http.onReceiveHeader = null;
|
||
http.onReceiveStatusLine = null;
|
||
http.contentLength = 0;
|
||
// close file if open
|
||
if (file.isOpen()){
|
||
// close open file
|
||
file.close();
|
||
}
|
||
}
|
||
|
||
http.method = HTTP.Method.put;
|
||
http.url = url;
|
||
addAccessTokenHeader();
|
||
http.addRequestHeader("Content-Type", "application/octet-stream");
|
||
http.onSend = data => file.rawRead(data).length;
|
||
http.contentLength = file.size;
|
||
auto response = perform();
|
||
checkHttpCode(response);
|
||
return response;
|
||
}
|
||
|
||
private JSONValue perform(const(void)[] sendData)
|
||
{
|
||
scope(exit) {
|
||
http.onSend = null;
|
||
http.contentLength = 0;
|
||
}
|
||
if (sendData) {
|
||
http.contentLength = sendData.length;
|
||
http.onSend = (void[] buf) {
|
||
import std.algorithm: min;
|
||
size_t minLen = min(buf.length, sendData.length);
|
||
if (minLen == 0) return 0;
|
||
buf[0 .. minLen] = sendData[0 .. minLen];
|
||
sendData = sendData[minLen .. $];
|
||
return minLen;
|
||
};
|
||
} else {
|
||
http.onSend = buf => 0;
|
||
}
|
||
auto response = perform();
|
||
return response;
|
||
}
|
||
|
||
private JSONValue perform()
|
||
{
|
||
scope(exit) http.onReceive = null;
|
||
char[] content;
|
||
JSONValue json;
|
||
|
||
http.onReceive = (ubyte[] data) {
|
||
content ~= data;
|
||
// HTTP Server Response Code Debugging if --https-debug is being used
|
||
if (.debugResponse){
|
||
log.vdebug("onedrive.perform() => OneDrive HTTP Server Response: ", http.statusLine.code);
|
||
}
|
||
return data.length;
|
||
};
|
||
|
||
try {
|
||
http.perform();
|
||
// Check the HTTP Response headers - needed for correct 429 handling
|
||
checkHTTPResponseHeaders();
|
||
} catch (CurlException e) {
|
||
// Parse and display error message received from OneDrive
|
||
log.vdebug("onedrive.perform() Generated a OneDrive CurlException");
|
||
auto errorArray = splitLines(e.msg);
|
||
string errorMessage = errorArray[0];
|
||
|
||
// what is contained in the curl error message?
|
||
if (canFind(errorMessage, "Couldn't connect to server on handle") || canFind(errorMessage, "Couldn't resolve host name on handle") || canFind(errorMessage, "Timeout was reached on handle")) {
|
||
// This is a curl timeout
|
||
// or is this a 408 request timeout
|
||
// https://github.com/abraunegg/onedrive/issues/694
|
||
// Back off & retry with incremental delay
|
||
int retryCount = 10000;
|
||
int retryAttempts = 0;
|
||
int backoffInterval = 0;
|
||
int maxBackoffInterval = 3600;
|
||
int timestampAlign = 0;
|
||
bool retrySuccess = false;
|
||
SysTime currentTime;
|
||
|
||
// what caused the initial curl exception?
|
||
if (canFind(errorMessage, "Couldn't connect to server on handle")) log.vdebug("Unable to connect to server - HTTPS access blocked?");
|
||
if (canFind(errorMessage, "Couldn't resolve host name on handle")) log.vdebug("Unable to resolve server - DNS access blocked?");
|
||
if (canFind(errorMessage, "Timeout was reached on handle")) log.vdebug("A timeout was triggered - data too slow, no response ... use --debug-https to diagnose further");
|
||
|
||
while (!retrySuccess){
|
||
try {
|
||
// configure libcurl to perform a fresh connection
|
||
log.vdebug("Configuring libcurl to use a fresh connection for re-try");
|
||
http.handle.set(CurlOption.fresh_connect,1);
|
||
// try the access
|
||
http.perform();
|
||
// Check the HTTP Response headers - needed for correct 429 handling
|
||
checkHTTPResponseHeaders();
|
||
// no error from http.perform() on re-try
|
||
log.log("Internet connectivity to Microsoft OneDrive service has been restored");
|
||
// unset the fresh connect option as this then creates performance issues if left enabled
|
||
log.vdebug("Unsetting libcurl to use a fresh connection as this causes a performance impact if left enabled");
|
||
http.handle.set(CurlOption.fresh_connect,0);
|
||
// connectivity restored
|
||
retrySuccess = true;
|
||
} catch (CurlException e) {
|
||
// when was the exception generated
|
||
currentTime = Clock.currTime();
|
||
// Increment retry attempts
|
||
retryAttempts++;
|
||
if (canFind(e.msg, "Couldn't connect to server on handle") || canFind(e.msg, "Couldn't resolve host name on handle") || canFind(errorMessage, "Timeout was reached on handle")) {
|
||
// no access to Internet
|
||
writeln();
|
||
log.error("ERROR: There was a timeout in accessing the Microsoft OneDrive service - Internet connectivity issue?");
|
||
// what is the error reason to assis the user as what to check
|
||
if (canFind(e.msg, "Couldn't connect to server on handle")) {
|
||
log.log(" - Check HTTPS access or Firewall Rules");
|
||
timestampAlign = 9;
|
||
}
|
||
if (canFind(e.msg, "Couldn't resolve host name on handle")) {
|
||
log.log(" - Check DNS resolution or Firewall Rules");
|
||
timestampAlign = 0;
|
||
}
|
||
|
||
// increment backoff interval
|
||
backoffInterval++;
|
||
int thisBackOffInterval = retryAttempts*backoffInterval;
|
||
|
||
// display retry information
|
||
currentTime.fracSecs = Duration.zero;
|
||
auto timeString = currentTime.toString();
|
||
log.vlog(" Retry attempt: ", retryAttempts);
|
||
log.vlog(" This attempt timestamp: ", timeString);
|
||
if (thisBackOffInterval > maxBackoffInterval) {
|
||
thisBackOffInterval = maxBackoffInterval;
|
||
}
|
||
|
||
// detail when the next attempt will be tried
|
||
// factor in the delay for curl to generate the exception - otherwise the next timestamp appears to be 'out' even though technically correct
|
||
auto nextRetry = currentTime + dur!"seconds"(thisBackOffInterval) + dur!"seconds"(timestampAlign);
|
||
log.vlog(" Next retry in approx: ", (thisBackOffInterval + timestampAlign), " seconds");
|
||
log.vlog(" Next retry approx: ", nextRetry);
|
||
|
||
// thread sleep
|
||
Thread.sleep(dur!"seconds"(thisBackOffInterval));
|
||
}
|
||
if (retryAttempts == retryCount) {
|
||
// we have attempted to re-connect X number of times
|
||
// false set this to true to break out of while loop
|
||
retrySuccess = true;
|
||
}
|
||
}
|
||
}
|
||
if (retryAttempts >= retryCount) {
|
||
log.error(" ERROR: Unable to reconnect to the Microsoft OneDrive service after ", retryCount, " attempts lasting over 1.2 years!");
|
||
throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?");
|
||
}
|
||
} else {
|
||
// Log that an error was returned
|
||
log.error("ERROR: OneDrive returned an error with the following message:");
|
||
// Some other error was returned
|
||
log.error(" Error Message: ", errorMessage);
|
||
log.error(" Calling Function: ", getFunctionName!({}));
|
||
|
||
// Was this a curl initialization error?
|
||
if (canFind(errorMessage, "Failed initialization on handle")) {
|
||
// initialization error ... prevent a run-away process if we have zero disk space
|
||
ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace("."));
|
||
if (localActualFreeSpace == 0) {
|
||
// force exit
|
||
shutdown();
|
||
exit(-1);
|
||
}
|
||
}
|
||
}
|
||
// return an empty JSON for handling
|
||
return json;
|
||
}
|
||
|
||
try {
|
||
json = content.parseJSON();
|
||
} catch (JSONException e) {
|
||
// Log that a JSON Exception was caught, dont output the HTML response from OneDrive
|
||
log.vdebug("JSON Exception caught when performing HTTP operations - use --debug-https to diagnose further");
|
||
}
|
||
return json;
|
||
}
|
||
|
||
private void checkHTTPResponseHeaders()
|
||
{
|
||
// Get the HTTP Response headers - needed for correct 429 handling
|
||
auto responseHeaders = http.responseHeaders();
|
||
if (.debugResponse){
|
||
log.vdebug("http.perform() => HTTP Response Headers: ", responseHeaders);
|
||
}
|
||
|
||
// is retry-after in the response headers
|
||
if ("retry-after" in http.responseHeaders) {
|
||
// Set the retry-after value
|
||
log.vdebug("http.perform() => Received a 'Retry-After' Header Response with the following value: ", http.responseHeaders["retry-after"]);
|
||
log.vdebug("http.perform() => Setting retryAfterValue to: ", http.responseHeaders["retry-after"]);
|
||
.retryAfterValue = to!ulong(http.responseHeaders["retry-after"]);
|
||
}
|
||
}
|
||
|
||
private void checkHttpCode()
|
||
{
|
||
// https://dev.onedrive.com/misc/errors.htm
|
||
// https://developer.overdrive.com/docs/reference-guide
|
||
|
||
/*
|
||
HTTP/1.1 Response handling
|
||
|
||
Errors in the OneDrive API are returned using standard HTTP status codes, as well as a JSON error response object. The following HTTP status codes should be expected.
|
||
|
||
Status code Status message Description
|
||
100 Continue Continue
|
||
200 OK Request was handled OK
|
||
201 Created This means you've made a successful POST to checkout, lock in a format, or place a hold
|
||
204 No Content This means you've made a successful DELETE to remove a hold or return a title
|
||
|
||
400 Bad Request Cannot process the request because it is malformed or incorrect.
|
||
401 Unauthorized Required authentication information is either missing or not valid for the resource.
|
||
403 Forbidden Access is denied to the requested resource. The user might not have enough permission.
|
||
404 Not Found The requested resource doesn’t exist.
|
||
405 Method Not Allowed The HTTP method in the request is not allowed on the resource.
|
||
406 Not Acceptable This service doesn’t support the format requested in the Accept header.
|
||
408 Request Time out Not expected from OneDrive, but can be used to handle Internet connection failures the same (fallback and try again)
|
||
409 Conflict The current state conflicts with what the request expects. For example, the specified parent folder might not exist.
|
||
410 Gone The requested resource is no longer available at the server.
|
||
411 Length Required A Content-Length header is required on the request.
|
||
412 Precondition Failed A precondition provided in the request (such as an if-match header) does not match the resource's current state.
|
||
413 Request Entity Too Large The request size exceeds the maximum limit.
|
||
415 Unsupported Media Type The content type of the request is a format that is not supported by the service.
|
||
416 Requested Range Not Satisfiable The specified byte range is invalid or unavailable.
|
||
422 Unprocessable Entity Cannot process the request because it is semantically incorrect.
|
||
429 Too Many Requests Client application has been throttled and should not attempt to repeat the request until an amount of time has elapsed.
|
||
|
||
500 Internal Server Error There was an internal server error while processing the request.
|
||
501 Not Implemented The requested feature isn’t implemented.
|
||
502 Bad Gateway The service was unreachable
|
||
503 Service Unavailable The service is temporarily unavailable. You may repeat the request after a delay. There may be a Retry-After header.
|
||
507 Insufficient Storage The maximum storage quota has been reached.
|
||
509 Bandwidth Limit Exceeded Your app has been throttled for exceeding the maximum bandwidth cap. Your app can retry the request again after more time has elapsed.
|
||
|
||
HTTP/2 Response handling
|
||
|
||
0 OK
|
||
|
||
*/
|
||
|
||
switch(http.statusLine.code)
|
||
{
|
||
// 0 - OK ... HTTP2 version of 200 OK
|
||
case 0:
|
||
break;
|
||
// 100 - Continue
|
||
case 100:
|
||
break;
|
||
// 200 - OK
|
||
case 200:
|
||
// No Log ..
|
||
break;
|
||
// 201 - Created OK
|
||
// 202 - Accepted
|
||
// 204 - Deleted OK
|
||
case 201,202,204:
|
||
// No actions, but log if verbose logging
|
||
//log.vlog("OneDrive Response: '", http.statusLine.code, " - ", http.statusLine.reason, "'");
|
||
break;
|
||
|
||
// 302 - resource found and available at another location, redirect
|
||
case 302:
|
||
break;
|
||
|
||
// 400 - Bad Request
|
||
case 400:
|
||
// Bad Request .. how should we act?
|
||
log.vlog("OneDrive returned a 'HTTP 400 - Bad Request' - gracefully handling error");
|
||
break;
|
||
|
||
// 403 - Forbidden
|
||
case 403:
|
||
// OneDrive responded that the user is forbidden
|
||
log.vlog("OneDrive returned a 'HTTP 403 - Forbidden' - gracefully handling error");
|
||
break;
|
||
|
||
// 404 - Item not found
|
||
case 404:
|
||
// Item was not found - do not throw an exception
|
||
log.vlog("OneDrive returned a 'HTTP 404 - Item not found' - gracefully handling error");
|
||
break;
|
||
|
||
// 408 - Request Timeout
|
||
case 408:
|
||
// Request to connect to OneDrive service timed out
|
||
log.vlog("Request Timeout - gracefully handling error");
|
||
throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?");
|
||
|
||
// 409 - Conflict
|
||
case 409:
|
||
// Conflict handling .. how should we act? This only really gets triggered if we are using --local-first & we remove items.db as the DB thinks the file is not uploaded but it is
|
||
log.vlog("OneDrive returned a 'HTTP 409 - Conflict' - gracefully handling error");
|
||
break;
|
||
|
||
// 412 - Precondition Failed
|
||
case 412:
|
||
// A precondition provided in the request (such as an if-match header) does not match the resource's current state.
|
||
log.vlog("OneDrive returned a 'HTTP 412 - Precondition Failed' - gracefully handling error");
|
||
break;
|
||
|
||
// 415 - Unsupported Media Type
|
||
case 415:
|
||
// Unsupported Media Type ... sometimes triggered on image files, especially PNG
|
||
log.vlog("OneDrive returned a 'HTTP 415 - Unsupported Media Type' - gracefully handling error");
|
||
break;
|
||
|
||
// 429 - Too Many Requests
|
||
case 429:
|
||
// Too many requests in a certain time window
|
||
// Check the HTTP Response headers - needed for correct 429 handling
|
||
checkHTTPResponseHeaders();
|
||
// https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
|
||
log.vlog("OneDrive returned a 'HTTP 429 - Too Many Requests' - gracefully handling error");
|
||
throw new OneDriveException(http.statusLine.code, http.statusLine.reason);
|
||
|
||
// Server side (OneDrive) Errors
|
||
// 500 - Internal Server Error
|
||
// 502 - Bad Gateway
|
||
// 503 - Service Unavailable
|
||
// 504 - Gateway Timeout (Issue #320)
|
||
case 500:
|
||
// No actions
|
||
log.vlog("OneDrive returned a 'HTTP 500 Internal Server Error' - gracefully handling error");
|
||
break;
|
||
|
||
case 502:
|
||
// No actions
|
||
log.vlog("OneDrive returned a 'HTTP 502 Bad Gateway Error' - gracefully handling error");
|
||
break;
|
||
|
||
case 503:
|
||
// No actions
|
||
log.vlog("OneDrive returned a 'HTTP 503 Service Unavailable Error' - gracefully handling error");
|
||
break;
|
||
|
||
case 504:
|
||
// No actions
|
||
log.vlog("OneDrive returned a 'HTTP 504 Gateway Timeout Error' - gracefully handling error");
|
||
break;
|
||
|
||
// "else"
|
||
default:
|
||
throw new OneDriveException(http.statusLine.code, http.statusLine.reason);
|
||
}
|
||
}
|
||
|
||
private void checkHttpCode(ref const JSONValue response)
|
||
{
|
||
switch(http.statusLine.code)
|
||
{
|
||
// 0 - OK ... HTTP2 version of 200 OK
|
||
case 0:
|
||
break;
|
||
// 100 - Continue
|
||
case 100:
|
||
break;
|
||
// 200 - OK
|
||
case 200:
|
||
// No Log ..
|
||
break;
|
||
// 201 - Created OK
|
||
// 202 - Accepted
|
||
// 204 - Deleted OK
|
||
case 201,202,204:
|
||
// No actions, but log if verbose logging
|
||
//log.vlog("OneDrive Response: '", http.statusLine.code, " - ", http.statusLine.reason, "'");
|
||
break;
|
||
|
||
// 302 - resource found and available at another location, redirect
|
||
case 302:
|
||
break;
|
||
|
||
// 400 - Bad Request
|
||
case 400:
|
||
// Bad Request .. how should we act?
|
||
// make sure this is thrown so that it is caught
|
||
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
|
||
|
||
// 403 - Forbidden
|
||
case 403:
|
||
// OneDrive responded that the user is forbidden
|
||
log.vlog("OneDrive returned a 'HTTP 403 - Forbidden' - gracefully handling error");
|
||
// Throw this as a specific exception so this is caught when performing sync.o365SiteSearch
|
||
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
|
||
|
||
// 412 - Precondition Failed
|
||
case 412:
|
||
// Throw this as a specific exception so this is caught when performing sync.uploadLastModifiedTime
|
||
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
|
||
|
||
// Server side (OneDrive) Errors
|
||
// 500 - Internal Server Error
|
||
// 502 - Bad Gateway
|
||
// 503 - Service Unavailable
|
||
// 504 - Gateway Timeout (Issue #320)
|
||
case 500:
|
||
// Throw this as a specific exception so this is caught
|
||
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
|
||
|
||
case 502:
|
||
// Throw this as a specific exception so this is caught
|
||
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
|
||
|
||
case 503:
|
||
// Throw this as a specific exception so this is caught
|
||
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
|
||
|
||
case 504:
|
||
// Throw this as a specific exception so this is caught
|
||
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
|
||
|
||
// Default - all other errors that are not a 2xx or a 302
|
||
default:
|
||
if (http.statusLine.code / 100 != 2 && http.statusLine.code != 302) {
|
||
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
unittest
|
||
{
|
||
string configDirName = expandTilde("~/.config/onedrive");
|
||
auto cfg = new config.Config(configDirName);
|
||
cfg.init();
|
||
OneDriveApi onedrive = new OneDriveApi(cfg);
|
||
onedrive.init();
|
||
std.file.write("/tmp/test", "test");
|
||
|
||
// simpleUpload
|
||
auto item = onedrive.simpleUpload("/tmp/test", "/test");
|
||
try {
|
||
item = onedrive.simpleUpload("/tmp/test", "/test");
|
||
} catch (OneDriveException e) {
|
||
assert(e.httpStatusCode == 409);
|
||
}
|
||
try {
|
||
item = onedrive.simpleUpload("/tmp/test", "/test", "123");
|
||
} catch (OneDriveException e) {
|
||
assert(e.httpStatusCode == 412);
|
||
}
|
||
item = onedrive.simpleUpload("/tmp/test", "/test", item["eTag"].str);
|
||
|
||
// deleteById
|
||
try {
|
||
onedrive.deleteById(item["id"].str, "123");
|
||
} catch (OneDriveException e) {
|
||
assert(e.httpStatusCode == 412);
|
||
}
|
||
onedrive.deleteById(item["id"].str, item["eTag"].str);
|
||
onedrive.http.shutdown();
|
||
}
|