mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-04 23:12:18 +02:00
Merge branch 'master' of github.com:skilion/onedrive
This commit is contained in:
commit
8f5f54f6a1
27
Makefile
27
Makefile
|
@ -1,6 +1,5 @@
|
||||||
DC = dmd
|
|
||||||
DFLAGS = -ofonedrive -L-lcurl -L-lsqlite3 -L-ldl
|
DFLAGS = -ofonedrive -L-lcurl -L-lsqlite3 -L-ldl
|
||||||
DESTDIR = /usr/local/bin
|
PREFIX = /usr/local
|
||||||
|
|
||||||
SOURCES = \
|
SOURCES = \
|
||||||
src/config.d \
|
src/config.d \
|
||||||
|
@ -9,28 +8,34 @@ SOURCES = \
|
||||||
src/main.d \
|
src/main.d \
|
||||||
src/monitor.d \
|
src/monitor.d \
|
||||||
src/onedrive.d \
|
src/onedrive.d \
|
||||||
|
src/qxor.d \
|
||||||
src/selective.d \
|
src/selective.d \
|
||||||
src/sqlite.d \
|
src/sqlite.d \
|
||||||
src/sync.d \
|
src/sync.d \
|
||||||
src/upload.d \
|
src/upload.d \
|
||||||
src/util.d
|
src/util.d
|
||||||
|
|
||||||
|
all: onedrive onedrive.service
|
||||||
|
|
||||||
onedrive: $(SOURCES)
|
onedrive: $(SOURCES)
|
||||||
$(DC) -O -release -inline -boundscheck=off $(DFLAGS) $(SOURCES)
|
dmd -g -inline -O -release $(DFLAGS) $(SOURCES)
|
||||||
|
|
||||||
|
onedrive.service:
|
||||||
|
sed "s|@PREFIX@|$(PREFIX)|g" onedrive.service.in > onedrive.service
|
||||||
|
|
||||||
debug: $(SOURCES)
|
debug: $(SOURCES)
|
||||||
$(DC) -debug -g -gs $(DFLAGS) $(SOURCES)
|
dmd -debug -g -gs $(DFLAGS) $(SOURCES)
|
||||||
|
|
||||||
unittest: $(SOURCES)
|
unittest: $(SOURCES)
|
||||||
$(DC) -unittest -debug -g -gs $(DFLAGS) $(SOURCES)
|
dmd -debug -g -gs -unittest $(DFLAGS) $(SOURCES)
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -f onedrive.o onedrive
|
rm -f onedrive onedrive.o onedrive.service
|
||||||
|
|
||||||
install: onedrive onedrive.service
|
install: all
|
||||||
install onedrive $(DESTDIR)/onedrive
|
install -D onedrive $(DESTDIR)$(PREFIX)/bin/onedrive
|
||||||
install -m 644 onedrive.service /usr/lib/systemd/user
|
install -D -m 644 onedrive.service $(DESTDIR)/usr/lib/systemd/user/onedrive.service
|
||||||
|
|
||||||
uninstall:
|
uninstall:
|
||||||
rm -f $(DESTDIR)/onedrive
|
rm -f $(DESTDIR)$(PREFIX)/bin/onedrive
|
||||||
rm -f /usr/lib/systemd/user/onedrive.service
|
rm -f $(DESTDIR)/usr/lib/systemd/user/onedrive.service
|
||||||
|
|
115
README.md
115
README.md
|
@ -1,97 +1,136 @@
|
||||||
OneDrive Free Client
|
# OneDrive Free Client
|
||||||
====================
|
###### A complete tool to interact with OneDrive on Linux. Built following the UNIX philosophy.
|
||||||
|
|
||||||
### Features:
|
### Features:
|
||||||
* State caching
|
* State caching
|
||||||
* Real-Time file monitoring with Inotify
|
* Real-Time file monitoring with Inotify
|
||||||
* Resumable uploads
|
* Resumable uploads
|
||||||
|
* Support OneDrive for Business (part of Office 365)
|
||||||
|
|
||||||
### What's missing:
|
### What's missing:
|
||||||
* OneDrive for business is not supported
|
* Shared folders are not supported
|
||||||
* While local changes are uploaded right away, remote changes are delayed.
|
* While local changes are uploaded right away, remote changes are delayed
|
||||||
* No GUI
|
* No GUI
|
||||||
|
|
||||||
|
## Setup
|
||||||
|
|
||||||
### Dependencies
|
### Dependencies
|
||||||
* [libcurl](http://curl.haxx.se/libcurl/)
|
* [libcurl](http://curl.haxx.se/libcurl/)
|
||||||
* [SQLite 3](https://www.sqlite.org/)
|
* [SQLite 3](https://www.sqlite.org/)
|
||||||
* [Digital Mars D Compiler (DMD)](http://dlang.org/download.html)
|
* [Digital Mars D Compiler (DMD)](http://dlang.org/download.html)
|
||||||
|
|
||||||
### Dependencies: Ubuntu
|
### Dependencies: Ubuntu/Debian
|
||||||
```
|
```sh
|
||||||
sudo apt-get install libcurl-dev
|
sudo apt-get install libcurl4-openssl-dev
|
||||||
sudo apt-get install libsqlite3-dev
|
sudo apt-get install libsqlite3-dev
|
||||||
sudo wget http://master.dl.sourceforge.net/project/d-apt/files/d-apt.list -O /etc/apt/sources.list.d/d-apt.list
|
curl -fsS https://dlang.org/install.sh | bash -s dmd
|
||||||
wget -qO - http://dlang.org/d-keyring.gpg | sudo apt-key add -
|
```
|
||||||
sudo apt-get update && sudo apt-get install dmd-bin
|
|
||||||
|
### Dependencies: Fedora/CentOS
|
||||||
|
```sh
|
||||||
|
sudo apt-get install libcurl-devel
|
||||||
|
sudo apt-get install sqlite-devel
|
||||||
|
curl -fsS https://dlang.org/install.sh | bash -s dmd
|
||||||
```
|
```
|
||||||
|
|
||||||
### Installation
|
### Installation
|
||||||
```
|
```sh
|
||||||
git clone git@github.com:skilion/onedrive.git
|
git clone https://github.com/skilion/onedrive.git
|
||||||
cd onedrive
|
cd onedrive
|
||||||
make
|
make
|
||||||
sudo make install
|
sudo make install
|
||||||
```
|
```
|
||||||
|
|
||||||
### Configuration:
|
### First run :zap:
|
||||||
You should copy the default config file into your home directory before making changes:
|
After installing the application you must run it at least one time from the terminal to authorize it. The procedure requires a web browser.
|
||||||
|
You will be asked to open a specific link where you will have to login into your Microsoft Account and give the application the permission to access your files. After giving the permission, you will be redirected to a blank page. Copy the URI of the blank page into the application.
|
||||||
|
|
||||||
|
### Uninstall
|
||||||
|
```sh
|
||||||
|
sudo make uninstall
|
||||||
|
# delete the application state
|
||||||
|
rm -rf .config/onedrive
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
Configuration is optional. By default all files are downloaded in `~/OneDrive` and only hidden files are skipped.
|
||||||
|
If you want to change the defaults, you can copy and edit the included config file into your `~/.config/onedrive` directory:
|
||||||
|
```sh
|
||||||
mkdir -p ~/.config/onedrive
|
mkdir -p ~/.config/onedrive
|
||||||
cp ./config ~/.config/onedrive/config
|
cp ./config ~/.config/onedrive/config
|
||||||
|
nano ~/.config/onedrive/config
|
||||||
```
|
```
|
||||||
|
|
||||||
Available options:
|
Available options:
|
||||||
* `sync_dir`: directory where the files will be synced
|
* `sync_dir`: directory where the files will be synced
|
||||||
* `skip_file`: any files or directories that match this pattern will be skipped during sync
|
* `skip_file`: any files or directories that match this pattern will be skipped during sync.
|
||||||
|
|
||||||
Pattern are case insensitive.
|
Patterns are case insensitive. `*` and `?` [wildcards characters](https://technet.microsoft.com/en-us/library/bb490639.aspx) are supported. Use `|` to separate multiple patterns.
|
||||||
`*` and `?` [wildcards characters][1] are supported.
|
|
||||||
Use `|` to separate multiple patterns.
|
|
||||||
|
|
||||||
[1]: https://technet.microsoft.com/en-us/library/bb490639.aspx
|
Note: after changing `skip_file`, you must perform a full synchronization by executing `onedrive --resync`
|
||||||
|
|
||||||
### Selective sync
|
### Selective sync :zap:
|
||||||
Selective sync allows you to sync only specific files and directories.
|
Selective sync allows you to sync only specific files and directories.
|
||||||
To enable selective sync create a file named `sync_list` in `~/.config/onedrive`.
|
To enable selective sync create a file named `sync_list` in `~/.config/onedrive`.
|
||||||
Each line represents a path to a file or directory relative from your `sync_dir`.
|
Each line of the file represents a path to a file or directory relative from your `sync_dir`.
|
||||||
```
|
Here is an example:
|
||||||
$ cat ~/.config/onedrive/sync_list
|
```text
|
||||||
Backup
|
Backup
|
||||||
Documents/report.odt
|
Documents/latest_report.docx
|
||||||
Work/ProjectX
|
Work/ProjectX
|
||||||
notes.txt
|
notes.txt
|
||||||
```
|
```
|
||||||
|
Note: after changing the sync list, you must perform a full synchronization by executing `onedrive --resync`
|
||||||
|
|
||||||
### First run
|
### OneDrive service
|
||||||
The first time you run the program you will be asked to sign in. The procedure requires a web browser.
|
|
||||||
|
|
||||||
### Service
|
|
||||||
If you want to sync your files automatically, enable and start the systemd service:
|
If you want to sync your files automatically, enable and start the systemd service:
|
||||||
```
|
```sh
|
||||||
systemctl --user enable onedrive
|
systemctl --user enable onedrive
|
||||||
systemctl --user start onedrive
|
systemctl --user start onedrive
|
||||||
```
|
```
|
||||||
|
|
||||||
To see the logs run:
|
To see the logs run:
|
||||||
```
|
```sh
|
||||||
journalctl --user-unit onedrive -f
|
journalctl --user-unit onedrive -f
|
||||||
```
|
```
|
||||||
|
|
||||||
### Usage:
|
### Using multiple accounts
|
||||||
|
You can run multiple instances of the application specifying a different config directory in order to handle multiple OneDrive accounts.
|
||||||
|
To do this you can use the `--confdir` parameter.
|
||||||
|
Here is an example:
|
||||||
|
```sh
|
||||||
|
onedrive --monitor --confdir="~/.config/onedrivePersonal" &
|
||||||
|
onedrive --monitor --confdir="~/.config/onedriveWork" &
|
||||||
```
|
```
|
||||||
onedrive [OPTION]...
|
|
||||||
|
|
||||||
|
`--monitor` keeps the application running and monitoring for changes
|
||||||
|
|
||||||
|
`&` puts the application in background and leaves the terminal interactive
|
||||||
|
|
||||||
|
## Extra
|
||||||
|
|
||||||
|
### Reporting issues
|
||||||
|
If you encounter any bugs you can report them here on Github. Before filing an issue be sure to:
|
||||||
|
|
||||||
|
1. Have compiled the application in debug mode with `make debug`
|
||||||
|
2. Run the application in verbose mode `onedrive --verbose`
|
||||||
|
3. Have the log of the error (preferably uploaded on an external website such as [pastebin](https://pastebin.com/))
|
||||||
|
4. Collect any information that you may think it is relevant to the error (such as the steps to trigger it)
|
||||||
|
|
||||||
|
### All available commands:
|
||||||
|
```text
|
||||||
|
Usage: onedrive [OPTION]...
|
||||||
|
|
||||||
|
no option Sync and exit.
|
||||||
-m --monitor Keep monitoring for local and remote changes.
|
-m --monitor Keep monitoring for local and remote changes.
|
||||||
--resync Forget the last saved state, perform a full sync.
|
--resync Forget the last saved state, perform a full sync.
|
||||||
--logout Logout the current user.
|
--logout Logout the current user.
|
||||||
--confdir Set the directory to use to store the configuration files.
|
--confdir Set the directory to use to store the configuration files.
|
||||||
-v --verbose Print more details, useful for debugging.
|
-v --verbose Print more details, useful for debugging.
|
||||||
|
--print-token Print the access token, useful for debugging.
|
||||||
-h --help This help information.
|
-h --help This help information.
|
||||||
```
|
```
|
||||||
|
|
||||||
### Notes:
|
### File naming
|
||||||
* After changing `skip_file` in your configs or the sync list, you must execute `onedrive --resync`
|
The files and directories in the synchronization directory must follow the [Windows naming conventions](https://msdn.microsoft.com/en-us/library/aa365247).
|
||||||
* [Windows naming conventions][2] apply
|
The application will crash for example if you have two files with the same name but different case. This is expected behavior and won't be fixed.
|
||||||
* Use `make debug` to generate an executable for debugging
|
|
||||||
|
|
||||||
[2]: https://msdn.microsoft.com/en-us/library/aa365247
|
|
||||||
|
|
|
@ -3,7 +3,7 @@ Description=OneDrive Free Client
|
||||||
Documentation=https://github.com/skilion/onedrive
|
Documentation=https://github.com/skilion/onedrive
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
ExecStart=/usr/local/bin/onedrive -m
|
ExecStart=@PREFIX@/bin/onedrive -m
|
||||||
Restart=no
|
Restart=no
|
||||||
|
|
||||||
[Install]
|
[Install]
|
|
@ -5,7 +5,7 @@ static import log;
|
||||||
final class Config
|
final class Config
|
||||||
{
|
{
|
||||||
public string refreshTokenFilePath;
|
public string refreshTokenFilePath;
|
||||||
public string statusTokenFilePath;
|
public string deltaLinkFilePath;
|
||||||
public string databaseFilePath;
|
public string databaseFilePath;
|
||||||
public string uploadStateFilePath;
|
public string uploadStateFilePath;
|
||||||
public string syncListFilePath;
|
public string syncListFilePath;
|
||||||
|
@ -17,7 +17,7 @@ final class Config
|
||||||
this(string configDirName)
|
this(string configDirName)
|
||||||
{
|
{
|
||||||
refreshTokenFilePath = configDirName ~ "/refresh_token";
|
refreshTokenFilePath = configDirName ~ "/refresh_token";
|
||||||
statusTokenFilePath = configDirName ~ "/status_token";
|
deltaLinkFilePath = configDirName ~ "/delta_link";
|
||||||
databaseFilePath = configDirName ~ "/items.sqlite3";
|
databaseFilePath = configDirName ~ "/items.sqlite3";
|
||||||
uploadStateFilePath = configDirName ~ "/resume_upload";
|
uploadStateFilePath = configDirName ~ "/resume_upload";
|
||||||
userConfigFilePath = configDirName ~ "/config";
|
userConfigFilePath = configDirName ~ "/config";
|
||||||
|
|
219
src/itemdb.d
219
src/itemdb.d
|
@ -4,46 +4,60 @@ import sqlite;
|
||||||
enum ItemType
|
enum ItemType
|
||||||
{
|
{
|
||||||
file,
|
file,
|
||||||
dir
|
dir,
|
||||||
|
remote
|
||||||
}
|
}
|
||||||
|
|
||||||
struct Item
|
struct Item
|
||||||
{
|
{
|
||||||
|
string driveId;
|
||||||
string id;
|
string id;
|
||||||
string name;
|
string name;
|
||||||
ItemType type;
|
ItemType type;
|
||||||
string eTag;
|
string eTag;
|
||||||
string cTag;
|
string cTag;
|
||||||
SysTime mtime;
|
SysTime mtime;
|
||||||
|
string parentDriveId;
|
||||||
string parentId;
|
string parentId;
|
||||||
string crc32;
|
string crc32Hash;
|
||||||
|
string sha1Hash;
|
||||||
|
string quickXorHash;
|
||||||
}
|
}
|
||||||
|
|
||||||
final class ItemDatabase
|
final class ItemDatabase
|
||||||
{
|
{
|
||||||
// increment this for every change in the db schema
|
// increment this for every change in the db schema
|
||||||
immutable int itemDatabaseVersion = 3;
|
immutable int itemDatabaseVersion = 5;
|
||||||
|
|
||||||
Database db;
|
Database db;
|
||||||
Statement insertItemStmt;
|
Statement insertItemStmt;
|
||||||
Statement updateItemStmt;
|
Statement updateItemStmt;
|
||||||
Statement selectItemByIdStmt;
|
Statement selectItemByIdStmt;
|
||||||
Statement selectItemByParentIdStmt;
|
Statement selectItemByParentIdStmt;
|
||||||
|
Statement deleteItemByIdStmt;
|
||||||
|
|
||||||
this(const(char)[] filename)
|
this(const(char)[] filename)
|
||||||
{
|
{
|
||||||
db = Database(filename);
|
db = Database(filename);
|
||||||
if (db.getVersion() == 0) {
|
if (db.getVersion() == 0) {
|
||||||
db.exec("CREATE TABLE item (
|
db.exec("CREATE TABLE item (
|
||||||
id TEXT NOT NULL PRIMARY KEY,
|
driveId TEXT NOT NULL,
|
||||||
|
id TEXT NOT NULL,
|
||||||
name TEXT NOT NULL,
|
name TEXT NOT NULL,
|
||||||
type TEXT NOT NULL,
|
type TEXT NOT NULL,
|
||||||
eTag TEXT,
|
eTag TEXT,
|
||||||
cTag TEXT,
|
cTag TEXT,
|
||||||
mtime TEXT NOT NULL,
|
mtime TEXT NOT NULL,
|
||||||
|
parentDriveId TEXT,
|
||||||
parentId TEXT,
|
parentId TEXT,
|
||||||
crc32 TEXT,
|
crc32Hash TEXT,
|
||||||
FOREIGN KEY (parentId) REFERENCES item (id) ON DELETE CASCADE
|
sha1Hash TEXT,
|
||||||
|
quickXorHash TEXT,
|
||||||
|
PRIMARY KEY (driveId, id),
|
||||||
|
FOREIGN KEY (parentDriveId, parentId)
|
||||||
|
REFERENCES item (driveId, id)
|
||||||
|
ON DELETE CASCADE
|
||||||
|
ON UPDATE RESTRICT
|
||||||
)");
|
)");
|
||||||
db.exec("CREATE INDEX name_idx ON item (name)");
|
db.exec("CREATE INDEX name_idx ON item (name)");
|
||||||
db.setVersion(itemDatabaseVersion);
|
db.setVersion(itemDatabaseVersion);
|
||||||
|
@ -52,15 +66,22 @@ final class ItemDatabase
|
||||||
}
|
}
|
||||||
db.exec("PRAGMA foreign_keys = ON");
|
db.exec("PRAGMA foreign_keys = ON");
|
||||||
db.exec("PRAGMA recursive_triggers = ON");
|
db.exec("PRAGMA recursive_triggers = ON");
|
||||||
|
insertItemStmt = db.prepare("
|
||||||
insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32) VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
|
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentDriveId, parentId, crc32Hash, sha1Hash, quickXorHash)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
");
|
||||||
updateItemStmt = db.prepare("
|
updateItemStmt = db.prepare("
|
||||||
UPDATE item
|
UPDATE item
|
||||||
SET name = ?2, type = ?3, eTag = ?4, cTag = ?5, mtime = ?6, parentId = ?7, crc32 = ?8
|
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentDriveId = ?8, parentId = ?9, crc32Hash = ?10, sha1Hash = ?11, quickXorHash = ?12
|
||||||
WHERE id = ?1
|
WHERE driveId = ?1 AND id = ?2
|
||||||
");
|
");
|
||||||
selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32 FROM item WHERE id = ?");
|
selectItemByIdStmt = db.prepare("
|
||||||
selectItemByParentIdStmt = db.prepare("SELECT id FROM item WHERE parentId = ?");
|
SELECT *
|
||||||
|
FROM item
|
||||||
|
WHERE driveId = ?1 AND id = ?2
|
||||||
|
");
|
||||||
|
selectItemByParentIdStmt = db.prepare("SELECT driveId, id FROM item WHERE parentId = ? AND id = ?");
|
||||||
|
deleteItemByIdStmt = db.prepare("DELETE FROM item WHERE driveId = ? AND id = ?");
|
||||||
}
|
}
|
||||||
|
|
||||||
void insert(const ref Item item)
|
void insert(const ref Item item)
|
||||||
|
@ -77,8 +98,9 @@ final class ItemDatabase
|
||||||
|
|
||||||
void upsert(const ref Item item)
|
void upsert(const ref Item item)
|
||||||
{
|
{
|
||||||
auto s = db.prepare("SELECT COUNT(*) FROM item WHERE id = ?");
|
auto s = db.prepare("SELECT COUNT(*) FROM item WHERE driveId = ? AND id = ?");
|
||||||
s.bind(1, item.id);
|
s.bind(1, item.driveId);
|
||||||
|
s.bind(2, item.id);
|
||||||
auto r = s.exec();
|
auto r = s.exec();
|
||||||
Statement* stmt;
|
Statement* stmt;
|
||||||
if (r.front[0] == "0") stmt = &insertItemStmt;
|
if (r.front[0] == "0") stmt = &insertItemStmt;
|
||||||
|
@ -87,23 +109,25 @@ final class ItemDatabase
|
||||||
stmt.exec();
|
stmt.exec();
|
||||||
}
|
}
|
||||||
|
|
||||||
Item[] selectChildren(const(char)[] id)
|
Item[] selectChildren(const(char)[] driveId, const(char)[] id)
|
||||||
{
|
{
|
||||||
selectItemByParentIdStmt.bind(1, id);
|
selectItemByParentIdStmt.bind(1, driveId);
|
||||||
|
selectItemByParentIdStmt.bind(2, id);
|
||||||
auto res = selectItemByParentIdStmt.exec();
|
auto res = selectItemByParentIdStmt.exec();
|
||||||
Item[] items;
|
Item[] items;
|
||||||
foreach (row; res) {
|
foreach (row; res) {
|
||||||
Item item;
|
Item item;
|
||||||
bool found = selectById(row[0], item);
|
bool found = selectById(row[0], row[1], item);
|
||||||
assert(found);
|
assert(found, "Could not select the child of the item");
|
||||||
items ~= item;
|
items ~= item;
|
||||||
}
|
}
|
||||||
return items;
|
return items;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool selectById(const(char)[] id, out Item item)
|
bool selectById(const(char)[] driveId, const(char)[] id, out Item item)
|
||||||
{
|
{
|
||||||
selectItemByIdStmt.bind(1, id);
|
selectItemByIdStmt.bind(1, driveId);
|
||||||
|
selectItemByIdStmt.bind(2, id);
|
||||||
auto r = selectItemByIdStmt.exec();
|
auto r = selectItemByIdStmt.exec();
|
||||||
if (!r.empty) {
|
if (!r.empty) {
|
||||||
item = buildItem(r);
|
item = buildItem(r);
|
||||||
|
@ -112,139 +136,112 @@ final class ItemDatabase
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// returns the item with the given path
|
||||||
|
// the path is relative to the sync directory ex: "./Music/Turbo Killer.mp3"
|
||||||
bool selectByPath(const(char)[] path, out Item item)
|
bool selectByPath(const(char)[] path, out Item item)
|
||||||
{
|
{
|
||||||
// prefix with the root dir
|
Item currItem;
|
||||||
path = "root/" ~ path.chompPrefix(".");
|
path = "root/" ~ path.chompPrefix(".");
|
||||||
|
auto s = db.prepare("SELECT * FROM item WHERE name IS ?1 AND parentDriveId IS ?2 AND parentId IS ?3");
|
||||||
// initialize the search
|
foreach (name; pathSplitter(path)) {
|
||||||
string[2][] candidates; // [id, parentId]
|
s.bind(1, name);
|
||||||
auto s = db.prepare("SELECT id, parentId FROM item WHERE name = ?");
|
s.bind(2, currItem.driveId);
|
||||||
s.bind(1, baseName(path));
|
s.bind(3, currItem.id);
|
||||||
auto r = s.exec();
|
auto r = s.exec();
|
||||||
foreach (row; r) candidates ~= [row[0].dup, row[1].dup];
|
if (r.empty) return false;
|
||||||
path = dirName(path);
|
currItem = buildItem(r);
|
||||||
|
// if the item is of type remote substitute it with the child
|
||||||
if (path != ".") {
|
if (currItem.type == ItemType.remote) {
|
||||||
s = db.prepare("SELECT parentId FROM item WHERE id = ? AND name = ?");
|
auto children = selectChildren(currItem.driveId, currItem.id);
|
||||||
// discard the candidates that do not have the correct parent
|
enforce(children.length == 1, "The remote item does not have exactly 1 child");
|
||||||
do {
|
// keep the name of the remote item
|
||||||
s.bind(2, baseName(path));
|
children[0].name = currItem.name;
|
||||||
string[2][] newCandidates;
|
currItem = children[0];
|
||||||
newCandidates.reserve(candidates.length);
|
|
||||||
foreach (candidate; candidates) {
|
|
||||||
s.bind(1, candidate[1]);
|
|
||||||
r = s.exec();
|
|
||||||
if (!r.empty) {
|
|
||||||
string[2] c = [candidate[0], r.front[0].idup];
|
|
||||||
newCandidates ~= c;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
candidates = newCandidates;
|
item = currItem;
|
||||||
path = dirName(path);
|
return true;
|
||||||
} while (path != ".");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// reached the root
|
void deleteById(const(char)[] driveId, const(char)[] id)
|
||||||
string[2][] newCandidates;
|
|
||||||
foreach (candidate; candidates) {
|
|
||||||
if (!candidate[1]) {
|
|
||||||
newCandidates ~= candidate;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
candidates = newCandidates;
|
|
||||||
assert(candidates.length <= 1);
|
|
||||||
|
|
||||||
if (candidates.length == 1) return selectById(candidates[0][0], item);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
void deleteById(const(char)[] id)
|
|
||||||
{
|
{
|
||||||
auto s = db.prepare("DELETE FROM item WHERE id = ?");
|
deleteItemByIdStmt.bind(1, driveId);
|
||||||
s.bind(1, id);
|
deleteItemByIdStmt.bind(2, id);
|
||||||
s.exec();
|
deleteItemByIdStmt.exec();
|
||||||
}
|
|
||||||
|
|
||||||
// returns true if the item has the specified parent
|
|
||||||
bool hasParent(T)(const(char)[] itemId, T parentId)
|
|
||||||
if (is(T : const(char)[]) || is(T : const(char[])[]))
|
|
||||||
{
|
|
||||||
auto s = db.prepare("SELECT parentId FROM item WHERE id = ?");
|
|
||||||
while (true) {
|
|
||||||
s.bind(1, itemId);
|
|
||||||
auto r = s.exec();
|
|
||||||
if (r.empty) break;
|
|
||||||
auto currParentId = r.front[0];
|
|
||||||
static if (is(T : const(char)[])) {
|
|
||||||
if (currParentId == parentId) return true;
|
|
||||||
} else {
|
|
||||||
foreach (id; parentId) if (currParentId == id) return true;
|
|
||||||
}
|
|
||||||
itemId = currParentId.dup;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void bindItem(const ref Item item, ref Statement stmt)
|
private void bindItem(const ref Item item, ref Statement stmt)
|
||||||
{
|
{
|
||||||
with (stmt) with (item) {
|
with (stmt) with (item) {
|
||||||
bind(1, id);
|
bind(1, driveId);
|
||||||
bind(2, name);
|
bind(2, id);
|
||||||
|
bind(3, name);
|
||||||
string typeStr = null;
|
string typeStr = null;
|
||||||
final switch (type) with (ItemType) {
|
final switch (type) with (ItemType) {
|
||||||
case file: typeStr = "file"; break;
|
case file: typeStr = "file"; break;
|
||||||
case dir: typeStr = "dir"; break;
|
case dir: typeStr = "dir"; break;
|
||||||
|
case remote: typeStr = "remote"; break;
|
||||||
}
|
}
|
||||||
bind(3, typeStr);
|
bind(4, typeStr);
|
||||||
bind(4, eTag);
|
bind(5, eTag);
|
||||||
bind(5, cTag);
|
bind(6, cTag);
|
||||||
bind(6, mtime.toISOExtString());
|
bind(7, mtime.toISOExtString());
|
||||||
bind(7, parentId);
|
bind(8, parentDriveId);
|
||||||
bind(8, crc32);
|
bind(9, parentId);
|
||||||
|
bind(10, crc32Hash);
|
||||||
|
bind(11, sha1Hash);
|
||||||
|
bind(12, quickXorHash);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Item buildItem(Statement.Result result)
|
private Item buildItem(Statement.Result result)
|
||||||
{
|
{
|
||||||
assert(!result.empty && result.front.length == 8);
|
assert(!result.empty, "The result must not be empty");
|
||||||
|
assert(result.front.length == 12, "The result must have 12 columns");
|
||||||
Item item = {
|
Item item = {
|
||||||
id: result.front[0].dup,
|
driveId: result.front[0].dup,
|
||||||
name: result.front[1].dup,
|
id: result.front[1].dup,
|
||||||
eTag: result.front[3].dup,
|
name: result.front[2].dup,
|
||||||
cTag: result.front[4].dup,
|
eTag: result.front[4].dup,
|
||||||
mtime: SysTime.fromISOExtString(result.front[5]),
|
cTag: result.front[5].dup,
|
||||||
parentId: result.front[6].dup,
|
mtime: SysTime.fromISOExtString(result.front[6]),
|
||||||
crc32: result.front[7].dup
|
parentDriveId: result.front[7].dup,
|
||||||
|
parentId: result.front[8].dup,
|
||||||
|
crc32Hash: result.front[9].dup,
|
||||||
|
sha1Hash: result.front[10].dup,
|
||||||
|
quickXorHash: result.front[11].dup
|
||||||
};
|
};
|
||||||
switch (result.front[2]) {
|
switch (result.front[3]) {
|
||||||
case "file": item.type = ItemType.file; break;
|
case "file": item.type = ItemType.file; break;
|
||||||
case "dir": item.type = ItemType.dir; break;
|
case "dir": item.type = ItemType.dir; break;
|
||||||
|
case "remote": item.type = ItemType.remote; break;
|
||||||
default: assert(0);
|
default: assert(0);
|
||||||
}
|
}
|
||||||
return item;
|
return item;
|
||||||
}
|
}
|
||||||
|
|
||||||
// computes the path of the given item id
|
// computes the path of the given item id
|
||||||
// the path is relative to the sync directory ex: "./Music/Turbo Killer.mp3"
|
// the path is relative to the sync directory ex: "Music/Turbo Killer.mp3"
|
||||||
// a trailing slash is never added
|
// a trailing slash is not added if the item is a directory
|
||||||
string computePath(const(char)[] id)
|
string computePath(const(char)[] driveId, const(char)[] id)
|
||||||
{
|
{
|
||||||
string path;
|
string path;
|
||||||
auto s = db.prepare("SELECT name, parentId FROM item WHERE id = ?");
|
Item item;
|
||||||
while (true) {
|
while (true) {
|
||||||
s.bind(1, id);
|
enforce(selectById(driveId, id, item), "Unknow item id");
|
||||||
auto r = s.exec();
|
if (item.type == ItemType.remote) {
|
||||||
enforce(!r.empty, "Unknow item id");
|
// substitute the last name with the current
|
||||||
if (r.front[1]) {
|
path = item.name ~ path[indexOf(path, '/') .. $];
|
||||||
if (path) path = r.front[0].idup ~ "/" ~ path;
|
} else if (item.parentId) {
|
||||||
else path = r.front[0].idup;
|
if (path) path = item.name ~ "/" ~ path;
|
||||||
|
else path = item.name;
|
||||||
} else {
|
} else {
|
||||||
// root
|
// root
|
||||||
if (!path) path = ".";
|
if (!path) path = ".";
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
id = r.front[1].dup;
|
driveId = item.parentDriveId;
|
||||||
|
id = item.parentId;
|
||||||
}
|
}
|
||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
|
|
14
src/main.d
14
src/main.d
|
@ -7,7 +7,7 @@ static import log;
|
||||||
int main(string[] args)
|
int main(string[] args)
|
||||||
{
|
{
|
||||||
// configuration directory
|
// configuration directory
|
||||||
string configDirName = expandTilde(environment.get("XDG_CONFIG_HOME", "~/.config")) ~ "/onedrive";
|
string configDirName = environment.get("XDG_CONFIG_HOME", "~/.config") ~ "/onedrive";
|
||||||
// enable monitor mode
|
// enable monitor mode
|
||||||
bool monitor;
|
bool monitor;
|
||||||
// force a full resync
|
// force a full resync
|
||||||
|
@ -16,6 +16,8 @@ int main(string[] args)
|
||||||
bool logout;
|
bool logout;
|
||||||
// enable verbose logging
|
// enable verbose logging
|
||||||
bool verbose;
|
bool verbose;
|
||||||
|
// print the access token
|
||||||
|
bool printAccessToken;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
auto opt = getopt(
|
auto opt = getopt(
|
||||||
|
@ -25,7 +27,8 @@ int main(string[] args)
|
||||||
"resync", "Forget the last saved state, perform a full sync.", &resync,
|
"resync", "Forget the last saved state, perform a full sync.", &resync,
|
||||||
"logout", "Logout the current user.", &logout,
|
"logout", "Logout the current user.", &logout,
|
||||||
"confdir", "Set the directory to use to store the configuration files.", &configDirName,
|
"confdir", "Set the directory to use to store the configuration files.", &configDirName,
|
||||||
"verbose|v", "Print more details, useful for debugging.", &log.verbose
|
"verbose|v", "Print more details, useful for debugging.", &log.verbose,
|
||||||
|
"print-token", "Print the access token, useful for debugging.", &printAccessToken
|
||||||
);
|
);
|
||||||
if (opt.helpWanted) {
|
if (opt.helpWanted) {
|
||||||
defaultGetoptPrinter(
|
defaultGetoptPrinter(
|
||||||
|
@ -42,7 +45,7 @@ int main(string[] args)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.vlog("Loading config ...");
|
log.vlog("Loading config ...");
|
||||||
configDirName = expandTilde(configDirName);
|
configDirName = configDirName.expandTilde().absolutePath();
|
||||||
if (!exists(configDirName)) mkdir(configDirName);
|
if (!exists(configDirName)) mkdir(configDirName);
|
||||||
auto cfg = new config.Config(configDirName);
|
auto cfg = new config.Config(configDirName);
|
||||||
cfg.init();
|
cfg.init();
|
||||||
|
@ -55,9 +58,9 @@ int main(string[] args)
|
||||||
}
|
}
|
||||||
|
|
||||||
if (resync || logout) {
|
if (resync || logout) {
|
||||||
log.log("Deleting the saved status ...");
|
log.vlog("Deleting the saved status ...");
|
||||||
safeRemove(cfg.databaseFilePath);
|
safeRemove(cfg.databaseFilePath);
|
||||||
safeRemove(cfg.statusTokenFilePath);
|
safeRemove(cfg.deltaLinkFilePath);
|
||||||
safeRemove(cfg.uploadStateFilePath);
|
safeRemove(cfg.uploadStateFilePath);
|
||||||
if (logout) {
|
if (logout) {
|
||||||
safeRemove(cfg.refreshTokenFilePath);
|
safeRemove(cfg.refreshTokenFilePath);
|
||||||
|
@ -71,6 +74,7 @@ int main(string[] args)
|
||||||
return EXIT_FAILURE;
|
return EXIT_FAILURE;
|
||||||
}
|
}
|
||||||
auto onedrive = new OneDriveApi(cfg);
|
auto onedrive = new OneDriveApi(cfg);
|
||||||
|
onedrive.printAccessToken = printAccessToken;
|
||||||
if (!onedrive.init()) {
|
if (!onedrive.init()) {
|
||||||
log.log("Could not initialize the OneDrive API");
|
log.log("Could not initialize the OneDrive API");
|
||||||
// workaround for segfault in std.net.curl.Curl.shutdown() on exit
|
// workaround for segfault in std.net.curl.Curl.shutdown() on exit
|
||||||
|
|
|
@ -13,6 +13,7 @@ private immutable {
|
||||||
string driveUrl = "https://graph.microsoft.com/v1.0/me/drive";
|
string driveUrl = "https://graph.microsoft.com/v1.0/me/drive";
|
||||||
string itemByIdUrl = "https://graph.microsoft.com/v1.0/me/drive/items/";
|
string itemByIdUrl = "https://graph.microsoft.com/v1.0/me/drive/items/";
|
||||||
string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
|
string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
|
||||||
|
string driveByIdUrl = "https://graph.microsoft.com/v1.0/me/drives/";
|
||||||
}
|
}
|
||||||
|
|
||||||
class OneDriveException: Exception
|
class OneDriveException: Exception
|
||||||
|
@ -50,6 +51,9 @@ final class OneDriveApi
|
||||||
private SysTime accessTokenExpiration;
|
private SysTime accessTokenExpiration;
|
||||||
/* private */ HTTP http;
|
/* private */ HTTP http;
|
||||||
|
|
||||||
|
// if true, every new access token is printed
|
||||||
|
bool printAccessToken;
|
||||||
|
|
||||||
this(Config cfg)
|
this(Config cfg)
|
||||||
{
|
{
|
||||||
this.cfg = cfg;
|
this.cfg = cfg;
|
||||||
|
@ -61,15 +65,8 @@ final class OneDriveApi
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
refreshToken = readText(cfg.refreshTokenFilePath);
|
refreshToken = readText(cfg.refreshTokenFilePath);
|
||||||
getDefaultDrive();
|
|
||||||
} catch (FileException e) {
|
} catch (FileException e) {
|
||||||
return authorize();
|
return authorize();
|
||||||
} catch (OneDriveException e) {
|
|
||||||
if (e.httpStatusCode == 400 || e.httpStatusCode == 401) {
|
|
||||||
log.log("Refresh token invalid");
|
|
||||||
return authorize();
|
|
||||||
}
|
|
||||||
throw e;
|
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -101,24 +98,24 @@ final class OneDriveApi
|
||||||
}
|
}
|
||||||
|
|
||||||
// https://dev.onedrive.com/items/view_delta.htm
|
// https://dev.onedrive.com/items/view_delta.htm
|
||||||
JSONValue viewChangesById(const(char)[] id, const(char)[] statusToken)
|
JSONValue viewChangesById(const(char)[] driveId, const(char)[] id, const(char)[] deltaLink)
|
||||||
{
|
{
|
||||||
checkAccessTokenExpired();
|
checkAccessTokenExpired();
|
||||||
const(char)[] url = itemByIdUrl ~ id ~ "/delta";
|
if (deltaLink) return get(deltaLink);
|
||||||
|
const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/delta";
|
||||||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
|
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
|
||||||
if (statusToken) url ~= "&token=" ~ statusToken;
|
|
||||||
return get(url);
|
return get(url);
|
||||||
}
|
}
|
||||||
|
|
||||||
// https://dev.onedrive.com/items/view_delta.htm
|
// https://dev.onedrive.com/items/view_delta.htm
|
||||||
JSONValue viewChangesByPath(const(char)[] path, const(char)[] statusToken)
|
JSONValue viewChangesByPath(const(char)[] path, const(char)[] deltaLink)
|
||||||
{
|
{
|
||||||
checkAccessTokenExpired();
|
checkAccessTokenExpired();
|
||||||
|
if (deltaLink) return get(deltaLink);
|
||||||
string url = itemByPathUrl ~ encodeComponent(path) ~ ":/delta";
|
string url = itemByPathUrl ~ encodeComponent(path) ~ ":/delta";
|
||||||
// HACK
|
// HACK
|
||||||
if (path == ".") url = driveUrl ~ "/root/delta";
|
if (path == ".") url = driveUrl ~ "/root/delta";
|
||||||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
|
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
|
||||||
if (statusToken) url ~= "&token=" ~ statusToken;
|
|
||||||
return get(url);
|
return get(url);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -248,13 +245,21 @@ final class OneDriveApi
|
||||||
refreshToken = response["refresh_token"].str();
|
refreshToken = response["refresh_token"].str();
|
||||||
accessTokenExpiration = Clock.currTime() + dur!"seconds"(response["expires_in"].integer());
|
accessTokenExpiration = Clock.currTime() + dur!"seconds"(response["expires_in"].integer());
|
||||||
std.file.write(cfg.refreshTokenFilePath, refreshToken);
|
std.file.write(cfg.refreshTokenFilePath, refreshToken);
|
||||||
|
if (printAccessToken) writeln("New access token: ", accessToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void checkAccessTokenExpired()
|
private void checkAccessTokenExpired()
|
||||||
{
|
{
|
||||||
|
try {
|
||||||
if (Clock.currTime() >= accessTokenExpiration) {
|
if (Clock.currTime() >= accessTokenExpiration) {
|
||||||
newToken();
|
newToken();
|
||||||
}
|
}
|
||||||
|
} catch (OneDriveException e) {
|
||||||
|
if (e.httpStatusCode == 400 || e.httpStatusCode == 401) {
|
||||||
|
e.msg ~= "\nRefresh token invalid, use --logout to authorize the client again";
|
||||||
|
}
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addAccessTokenHeader()
|
private void addAccessTokenHeader()
|
||||||
|
|
88
src/qxor.d
Normal file
88
src/qxor.d
Normal file
|
@ -0,0 +1,88 @@
|
||||||
|
import std.algorithm;
|
||||||
|
import std.digest.digest;
|
||||||
|
|
||||||
|
// implementation of the QuickXorHash algorithm in D
|
||||||
|
// https://github.com/OneDrive/onedrive-api-docs/blob/master/snippets/quickxorhash.md
|
||||||
|
struct QuickXor
|
||||||
|
{
|
||||||
|
private immutable int widthInBits = 160;
|
||||||
|
private immutable size_t lengthInBytes = (widthInBits - 1) / 8 + 1;
|
||||||
|
private immutable size_t lengthInQWords = (widthInBits - 1) / 64 + 1;
|
||||||
|
private immutable int bitsInLastCell = widthInBits % 64; // 32
|
||||||
|
private immutable int shift = 11;
|
||||||
|
|
||||||
|
private ulong[lengthInQWords] _data;
|
||||||
|
private ulong _lengthSoFar;
|
||||||
|
private int _shiftSoFar;
|
||||||
|
|
||||||
|
nothrow @safe void put(scope const(ubyte)[] array...)
|
||||||
|
{
|
||||||
|
int vectorArrayIndex = _shiftSoFar / 64;
|
||||||
|
int vectorOffset = _shiftSoFar % 64;
|
||||||
|
immutable size_t iterations = min(array.length, widthInBits);
|
||||||
|
|
||||||
|
for (size_t i = 0; i < iterations; i++) {
|
||||||
|
immutable bool isLastCell = vectorArrayIndex == _data.length - 1;
|
||||||
|
immutable int bitsInVectorCell = isLastCell ? bitsInLastCell : 64;
|
||||||
|
|
||||||
|
if (vectorOffset <= bitsInVectorCell - 8) {
|
||||||
|
for (size_t j = i; j < array.length; j += widthInBits) {
|
||||||
|
_data[vectorArrayIndex] ^= cast(ulong) array[j] << vectorOffset;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
int index1 = vectorArrayIndex;
|
||||||
|
int index2 = isLastCell ? 0 : (vectorArrayIndex + 1);
|
||||||
|
byte low = cast(byte) (bitsInVectorCell - vectorOffset);
|
||||||
|
|
||||||
|
byte xoredByte = 0;
|
||||||
|
for (size_t j = i; j < array.length; j += widthInBits) {
|
||||||
|
xoredByte ^= array[j];
|
||||||
|
}
|
||||||
|
|
||||||
|
_data[index1] ^= cast(ulong) xoredByte << vectorOffset;
|
||||||
|
_data[index2] ^= cast(ulong) xoredByte >> low;
|
||||||
|
}
|
||||||
|
|
||||||
|
vectorOffset += shift;
|
||||||
|
if (vectorOffset >= bitsInVectorCell) {
|
||||||
|
vectorArrayIndex = isLastCell ? 0 : vectorArrayIndex + 1;
|
||||||
|
vectorOffset -= bitsInVectorCell;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_shiftSoFar = cast(int) (_shiftSoFar + shift * (array.length % widthInBits)) % widthInBits;
|
||||||
|
_lengthSoFar += array.length;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
nothrow @safe void start()
|
||||||
|
{
|
||||||
|
_data = _data.init;
|
||||||
|
_shiftSoFar = 0;
|
||||||
|
_lengthSoFar = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
nothrow @trusted ubyte[lengthInBytes] finish()
|
||||||
|
{
|
||||||
|
ubyte[lengthInBytes] tmp;
|
||||||
|
tmp[0 .. lengthInBytes] = (cast(ubyte*) _data)[0 .. lengthInBytes];
|
||||||
|
for (size_t i = 0; i < 8; i++) {
|
||||||
|
tmp[lengthInBytes - 8 + i] ^= (cast(ubyte*) &_lengthSoFar)[i];
|
||||||
|
}
|
||||||
|
return tmp;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unittest
|
||||||
|
{
|
||||||
|
assert(isDigest!QuickXor);
|
||||||
|
}
|
||||||
|
|
||||||
|
unittest
|
||||||
|
{
|
||||||
|
QuickXor qxor;
|
||||||
|
qxor.put(cast(ubyte[]) "The quick brown fox jumps over the lazy dog");
|
||||||
|
assert(qxor.finish().toHexString() == "6CC4A56F2B26C492FA4BBE57C1F31C4193A972BE");
|
||||||
|
}
|
||||||
|
|
||||||
|
alias QuickXorDigest = WrapperDigest!(QuickXor);
|
|
@ -197,17 +197,17 @@ unittest
|
||||||
|
|
||||||
auto s = db.prepare("INSERT INTO test VALUES (?, ?)");
|
auto s = db.prepare("INSERT INTO test VALUES (?, ?)");
|
||||||
s.bind(1, "key1");
|
s.bind(1, "key1");
|
||||||
s.bind(2, "value1");
|
s.bind(2, "value");
|
||||||
s.exec();
|
s.exec();
|
||||||
s.bind(1, "key2");
|
s.bind(1, "key2");
|
||||||
s.bind(2, "value2");
|
s.bind(2, null);
|
||||||
s.exec();
|
s.exec();
|
||||||
|
|
||||||
s = db.prepare("SELECT * FROM test ORDER BY id ASC");
|
s = db.prepare("SELECT * FROM test ORDER BY id ASC");
|
||||||
auto r = s.exec();
|
auto r = s.exec();
|
||||||
assert(r.front[0] == "key1");
|
assert(r.front[0] == "key1");
|
||||||
r.popFront();
|
r.popFront();
|
||||||
assert(r.front[1] == "value2");
|
assert(r.front[1] == null);
|
||||||
r.popFront();
|
r.popFront();
|
||||||
assert(r.empty);
|
assert(r.empty);
|
||||||
}
|
}
|
||||||
|
|
269
src/sync.d
269
src/sync.d
|
@ -23,7 +23,7 @@ private bool isItemFile(const ref JSONValue item)
|
||||||
private bool isItemDeleted(const ref JSONValue item)
|
private bool isItemDeleted(const ref JSONValue item)
|
||||||
{
|
{
|
||||||
// HACK: fix for https://github.com/skilion/onedrive/issues/157
|
// HACK: fix for https://github.com/skilion/onedrive/issues/157
|
||||||
return ("deleted" in item) || ("fileSystemInfo" !in item);
|
return ("deleted" in item) || ("fileSystemInfo" !in item && "remoteItem" !in item);
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool isItemRoot(const ref JSONValue item)
|
private bool isItemRoot(const ref JSONValue item)
|
||||||
|
@ -31,11 +31,63 @@ private bool isItemRoot(const ref JSONValue item)
|
||||||
return ("root" in item) != null;
|
return ("root" in item) != null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool testCrc32(string path, const(char)[] crc32)
|
private bool isItemRemote(const ref JSONValue item)
|
||||||
{
|
{
|
||||||
if (crc32) {
|
return ("remoteItem" in item) != null;
|
||||||
string localCrc32 = computeCrc32(path);
|
}
|
||||||
if (crc32 == localCrc32) return true;
|
|
||||||
|
// HACK: OneDrive Biz does not return parentReference for the root
|
||||||
|
string defaultDriveId;
|
||||||
|
|
||||||
|
private Item makeItem(const ref JSONValue jsonItem)
|
||||||
|
{
|
||||||
|
ItemType type;
|
||||||
|
if (isItemFile(jsonItem)) {
|
||||||
|
type = ItemType.file;
|
||||||
|
} else if (isItemFolder(jsonItem)) {
|
||||||
|
type = ItemType.dir;
|
||||||
|
} else if (isItemRemote(jsonItem)) {
|
||||||
|
type = ItemType.remote;
|
||||||
|
}
|
||||||
|
|
||||||
|
Item item = {
|
||||||
|
driveId: isItemRoot(jsonItem) ? defaultDriveId : jsonItem["parentReference"]["driveId"].str,
|
||||||
|
id: jsonItem["id"].str,
|
||||||
|
name: "name" in jsonItem ? jsonItem["name"].str : null, // name may be missing for deleted files in OneDrive Biz
|
||||||
|
type: type,
|
||||||
|
eTag: "eTag" in jsonItem ? jsonItem["eTag"].str : null, // eTag is not returned for the root in OneDrive Biz
|
||||||
|
cTag: "cTag" in jsonItem ? jsonItem["cTag"].str : null, // cTag is missing in old files (and all folders)
|
||||||
|
mtime: "fileSystemInfo" in jsonItem ? SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str) : SysTime(0),
|
||||||
|
parentDriveId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["driveId"].str,
|
||||||
|
parentId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["id"].str
|
||||||
|
};
|
||||||
|
|
||||||
|
// extract the file hash
|
||||||
|
if (isItemFile(jsonItem)) {
|
||||||
|
if ("hashes" in jsonItem["file"]) {
|
||||||
|
if ("crc32Hash" in jsonItem["file"]["hashes"]) {
|
||||||
|
item.crc32Hash = jsonItem["file"]["hashes"]["crc32Hash"].str;
|
||||||
|
} else if ("sha1Hash" in jsonItem["file"]["hashes"]) {
|
||||||
|
item.sha1Hash = jsonItem["file"]["hashes"]["sha1Hash"].str;
|
||||||
|
} else if ("quickXorHash" in jsonItem["file"]["hashes"]) {
|
||||||
|
item.quickXorHash = jsonItem["file"]["hashes"]["quickXorHash"].str;
|
||||||
|
} else {
|
||||||
|
log.vlog("The file does not have any hash");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool testFileHash(string path, const ref Item item)
|
||||||
|
{
|
||||||
|
if (item.crc32Hash) {
|
||||||
|
if (item.crc32Hash == computeCrc32(path)) return true;
|
||||||
|
} else if (item.sha1Hash) {
|
||||||
|
if (item.sha1Hash == computeSha1Hash(path)) return true;
|
||||||
|
} else if (item.quickXorHash) {
|
||||||
|
if (item.quickXorHash == computeQuickXorHash(path)) return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -60,12 +112,10 @@ final class SyncEngine
|
||||||
private ItemDatabase itemdb;
|
private ItemDatabase itemdb;
|
||||||
private UploadSession session;
|
private UploadSession session;
|
||||||
private SelectiveSync selectiveSync;
|
private SelectiveSync selectiveSync;
|
||||||
// token representing the last status correctly synced
|
|
||||||
private string statusToken;
|
|
||||||
// list of items to skip while applying the changes
|
// list of items to skip while applying the changes
|
||||||
private string[] skippedItems;
|
private string[] skippedItems;
|
||||||
// list of items to delete after the changes has been downloaded
|
// list of items to delete after the changes has been downloaded
|
||||||
private string[] idsToDelete;
|
private string[2][] idsToDelete;
|
||||||
|
|
||||||
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync)
|
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync)
|
||||||
{
|
{
|
||||||
|
@ -79,12 +129,6 @@ final class SyncEngine
|
||||||
|
|
||||||
void init()
|
void init()
|
||||||
{
|
{
|
||||||
// restore the previous status token
|
|
||||||
try {
|
|
||||||
statusToken = readText(cfg.statusTokenFilePath);
|
|
||||||
} catch (FileException e) {
|
|
||||||
// swallow exception
|
|
||||||
}
|
|
||||||
// check if there is an interrupted upload session
|
// check if there is an interrupted upload session
|
||||||
if (session.restore()) {
|
if (session.restore()) {
|
||||||
log.log("Continuing the upload session ...");
|
log.log("Continuing the upload session ...");
|
||||||
|
@ -96,33 +140,37 @@ final class SyncEngine
|
||||||
void applyDifferences()
|
void applyDifferences()
|
||||||
{
|
{
|
||||||
log.vlog("Applying differences ...");
|
log.vlog("Applying differences ...");
|
||||||
|
|
||||||
|
// restore the last known state
|
||||||
|
string deltaLink;
|
||||||
try {
|
try {
|
||||||
|
deltaLink = readText(cfg.deltaLinkFilePath);
|
||||||
|
} catch (FileException e) {
|
||||||
|
// swallow exception
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
defaultDriveId = onedrive.getDefaultDrive()["id"].str;
|
||||||
JSONValue changes;
|
JSONValue changes;
|
||||||
do {
|
do {
|
||||||
// get changes from the server
|
// get changes from the server
|
||||||
try {
|
try {
|
||||||
changes = onedrive.viewChangesByPath(".", statusToken);
|
changes = onedrive.viewChangesByPath(".", deltaLink);
|
||||||
} catch (OneDriveException e) {
|
} catch (OneDriveException e) {
|
||||||
if (e.httpStatusCode == 410) {
|
if (e.httpStatusCode == 410) {
|
||||||
log.log("Status token expired, resyncing");
|
log.log("Delta link expired, resyncing");
|
||||||
statusToken = null;
|
deltaLink = null;
|
||||||
continue;
|
continue;
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
foreach (item; changes["value"].array) {
|
foreach (item; changes["value"].array) {
|
||||||
applyDifference(item);
|
applyDifference(item);
|
||||||
}
|
}
|
||||||
// hack to reuse old code
|
if ("@odata.nextLink" in changes) deltaLink = changes["@odata.nextLink"].str;
|
||||||
string url;
|
if ("@odata.deltaLink" in changes) deltaLink = changes["@odata.deltaLink"].str;
|
||||||
if ("@odata.nextLink" in changes) url = changes["@odata.nextLink"].str;
|
std.file.write(cfg.deltaLinkFilePath, deltaLink);
|
||||||
if ("@odata.deltaLink" in changes) url = changes["@odata.deltaLink"].str;
|
|
||||||
auto c = matchFirst(url, r"(?:token=)([\w\d]+)");
|
|
||||||
c.popFront(); // skip the whole match
|
|
||||||
statusToken = c.front;
|
|
||||||
std.file.write(cfg.statusTokenFilePath, statusToken);
|
|
||||||
} while ("@odata.nextLink" in changes);
|
} while ("@odata.nextLink" in changes);
|
||||||
} catch (ErrnoException e) {
|
} catch (ErrnoException e) {
|
||||||
throw new SyncException(e.msg, e);
|
throw new SyncException(e.msg, e);
|
||||||
|
@ -140,38 +188,58 @@ final class SyncEngine
|
||||||
assumeSafeAppend(skippedItems);
|
assumeSafeAppend(skippedItems);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void applyDifference(JSONValue item)
|
private void applyDifference(JSONValue jsonItem)
|
||||||
{
|
{
|
||||||
string id = item["id"].str;
|
log.vlog(jsonItem["id"].str, " ", "name" in jsonItem ? jsonItem["name"].str : null);
|
||||||
string name = item["name"].str;
|
Item item = makeItem(jsonItem);
|
||||||
|
|
||||||
log.vlog(id, " ", name);
|
string path = ".";
|
||||||
|
bool unwanted;
|
||||||
|
unwanted |= skippedItems.find(item.parentId).length != 0;
|
||||||
|
unwanted |= selectiveSync.isNameExcluded(item.name);
|
||||||
|
|
||||||
// eTag and parentId do not exists for the root in OneDrive Biz
|
if (!unwanted && !isItemRoot(jsonItem)) {
|
||||||
string eTag, parentId;
|
// delay path computation after assuring the item parent is not excluded
|
||||||
if (!isItemRoot(item)) {
|
path = itemdb.computePath(item.parentDriveId, item.parentId) ~ "/" ~ item.name;
|
||||||
eTag = item["eTag"].str;
|
// selective sync
|
||||||
parentId = item["parentReference"]["id"].str;
|
unwanted |= selectiveSync.isPathExcluded(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
// skip unwanted items early
|
// skip unwanted items early
|
||||||
if (skippedItems.find(parentId).length != 0) {
|
if (unwanted) {
|
||||||
log.vlog("Filtered out");
|
log.vlog("Filtered out");
|
||||||
skippedItems ~= id;
|
skippedItems ~= item.id;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (selectiveSync.isNameExcluded(name)) {
|
|
||||||
log.vlog("Filtered out");
|
// check if the item is going to be deleted
|
||||||
skippedItems ~= id;
|
if (isItemDeleted(jsonItem)) {
|
||||||
|
log.vlog("The item is marked for deletion");
|
||||||
|
idsToDelete ~= [item.driveId, item.id];
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// check the item type
|
||||||
|
if (isItemRemote(jsonItem)) {
|
||||||
|
// TODO
|
||||||
|
// check name change
|
||||||
|
// scan the children later
|
||||||
|
// fix child references
|
||||||
|
log.vlog("Remote items are not supported yet");
|
||||||
|
skippedItems ~= item.id;
|
||||||
|
return;
|
||||||
|
} else if (!isItemFile(jsonItem) && !isItemFolder(jsonItem)) {
|
||||||
|
log.vlog("The item is neither a file nor a directory, skipping");
|
||||||
|
skippedItems ~= item.id;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// rename the local item if it is unsynced and there is a new version of it
|
// rename the local item if it is unsynced and there is a new version of it
|
||||||
Item oldItem;
|
Item oldItem;
|
||||||
string oldPath;
|
string oldPath;
|
||||||
bool cached = itemdb.selectById(id, oldItem);
|
bool cached = itemdb.selectById(item.driveId, item.id, oldItem);
|
||||||
if (cached && eTag != oldItem.eTag) {
|
if (cached && item.eTag != oldItem.eTag) {
|
||||||
oldPath = itemdb.computePath(id);
|
oldPath = itemdb.computePath(item.driveId, item.id);
|
||||||
if (!isItemSynced(oldItem, oldPath)) {
|
if (!isItemSynced(oldItem, oldPath)) {
|
||||||
log.vlog("The local item is unsynced, renaming");
|
log.vlog("The local item is unsynced, renaming");
|
||||||
if (exists(oldPath)) safeRename(oldPath);
|
if (exists(oldPath)) safeRename(oldPath);
|
||||||
|
@ -179,67 +247,17 @@ final class SyncEngine
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// check if the item is to be deleted
|
|
||||||
if (isItemDeleted(item)) {
|
|
||||||
log.vlog("The item is marked for deletion");
|
|
||||||
if (cached) idsToDelete ~= id;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// compute the path of the item
|
|
||||||
string path = ".";
|
|
||||||
if (parentId) {
|
|
||||||
path = itemdb.computePath(parentId) ~ "/" ~ name;
|
|
||||||
// selective sync
|
|
||||||
if (selectiveSync.isPathExcluded(path)) {
|
|
||||||
log.vlog("Filtered out: ", path);
|
|
||||||
skippedItems ~= id;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ItemType type;
|
|
||||||
if (isItemFile(item)) {
|
|
||||||
type = ItemType.file;
|
|
||||||
} else if (isItemFolder(item)) {
|
|
||||||
type = ItemType.dir;
|
|
||||||
} else {
|
|
||||||
log.vlog("The item is neither a file nor a directory, skipping");
|
|
||||||
skippedItems ~= id;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
string crc32;
|
|
||||||
if (type == ItemType.file) {
|
|
||||||
try {
|
|
||||||
crc32 = item["file"]["hashes"]["crc32Hash"].str;
|
|
||||||
} catch (JSONException e) {
|
|
||||||
log.vlog("The hash is not available");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Item newItem = {
|
|
||||||
id: id,
|
|
||||||
name: name,
|
|
||||||
type: type,
|
|
||||||
eTag: eTag,
|
|
||||||
cTag: "cTag" in item ? item["cTag"].str : null,
|
|
||||||
mtime: SysTime.fromISOExtString(item["fileSystemInfo"]["lastModifiedDateTime"].str),
|
|
||||||
parentId: parentId,
|
|
||||||
crc32: crc32
|
|
||||||
};
|
|
||||||
|
|
||||||
if (!cached) {
|
if (!cached) {
|
||||||
applyNewItem(newItem, path);
|
applyNewItem(item, path);
|
||||||
} else {
|
} else {
|
||||||
applyChangedItem(oldItem, newItem, path);
|
applyChangedItem(oldItem, oldPath, item, path);
|
||||||
}
|
}
|
||||||
|
|
||||||
// save the item in the db
|
// save the item in the db
|
||||||
if (oldItem.id) {
|
if (oldItem.id) {
|
||||||
itemdb.update(newItem);
|
itemdb.update(item);
|
||||||
} else {
|
} else {
|
||||||
itemdb.insert(newItem);
|
itemdb.insert(item);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -266,21 +284,25 @@ final class SyncEngine
|
||||||
//Use mkdirRecuse to deal nested dir
|
//Use mkdirRecuse to deal nested dir
|
||||||
mkdirRecurse(path);
|
mkdirRecurse(path);
|
||||||
break;
|
break;
|
||||||
|
case ItemType.remote:
|
||||||
|
assert(0);
|
||||||
}
|
}
|
||||||
setTimes(path, item.mtime, item.mtime);
|
setTimes(path, item.mtime, item.mtime);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void applyChangedItem(Item oldItem, Item newItem, string newPath)
|
// update a local item
|
||||||
|
// the local item is assumed to be in sync with the local db
|
||||||
|
private void applyChangedItem(Item oldItem, string oldPath, Item newItem, string newPath)
|
||||||
{
|
{
|
||||||
|
assert(oldItem.driveId == newItem.driveId);
|
||||||
assert(oldItem.id == newItem.id);
|
assert(oldItem.id == newItem.id);
|
||||||
assert(oldItem.type == newItem.type);
|
assert(oldItem.type == newItem.type);
|
||||||
|
|
||||||
if (oldItem.eTag != newItem.eTag) {
|
if (oldItem.eTag != newItem.eTag) {
|
||||||
string oldPath = itemdb.computePath(oldItem.id);
|
|
||||||
if (oldPath != newPath) {
|
if (oldPath != newPath) {
|
||||||
log.log("Moving: ", oldPath, " -> ", newPath);
|
log.log("Moving: ", oldPath, " -> ", newPath);
|
||||||
if (exists(newPath)) {
|
if (exists(newPath)) {
|
||||||
log.vlog("The destination is occupied, renaming ...");
|
log.vlog("The destination is occupied, renaming the conflicting file...");
|
||||||
safeRename(newPath);
|
safeRename(newPath);
|
||||||
}
|
}
|
||||||
rename(oldPath, newPath);
|
rename(oldPath, newPath);
|
||||||
|
@ -311,7 +333,7 @@ final class SyncEngine
|
||||||
} else {
|
} else {
|
||||||
log.vlog("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime);
|
log.vlog("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime);
|
||||||
}
|
}
|
||||||
if (testCrc32(path, item.crc32)) {
|
if (testFileHash(path, item)) {
|
||||||
return true;
|
return true;
|
||||||
} else {
|
} else {
|
||||||
log.vlog("The local item has a different hash");
|
log.vlog("The local item has a different hash");
|
||||||
|
@ -327,6 +349,8 @@ final class SyncEngine
|
||||||
log.vlog("The local item is a file but should be a directory");
|
log.vlog("The local item is a file but should be a directory");
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
case ItemType.remote:
|
||||||
|
assert(0);
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -334,9 +358,9 @@ final class SyncEngine
|
||||||
private void deleteItems()
|
private void deleteItems()
|
||||||
{
|
{
|
||||||
log.vlog("Deleting files ...");
|
log.vlog("Deleting files ...");
|
||||||
foreach_reverse (id; idsToDelete) {
|
foreach_reverse (i; idsToDelete) {
|
||||||
string path = itemdb.computePath(id);
|
string path = itemdb.computePath(i[0], i[1]);
|
||||||
itemdb.deleteById(id);
|
itemdb.deleteById(i[0], i[1]);
|
||||||
if (exists(path)) {
|
if (exists(path)) {
|
||||||
if (isFile(path)) {
|
if (isFile(path)) {
|
||||||
remove(path);
|
remove(path);
|
||||||
|
@ -384,7 +408,7 @@ final class SyncEngine
|
||||||
log.vlog("Filtered out");
|
log.vlog("Filtered out");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
string path = itemdb.computePath(item.id);
|
string path = itemdb.computePath(item.driveId, item.id);
|
||||||
if (selectiveSync.isPathExcluded(path)) {
|
if (selectiveSync.isPathExcluded(path)) {
|
||||||
log.vlog("Filtered out: ", path);
|
log.vlog("Filtered out: ", path);
|
||||||
return;
|
return;
|
||||||
|
@ -397,6 +421,8 @@ final class SyncEngine
|
||||||
case ItemType.file:
|
case ItemType.file:
|
||||||
uploadFileDifferences(item, path);
|
uploadFileDifferences(item, path);
|
||||||
break;
|
break;
|
||||||
|
case ItemType.remote:
|
||||||
|
assert(0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -411,7 +437,7 @@ final class SyncEngine
|
||||||
} else {
|
} else {
|
||||||
log.vlog("The directory has not changed");
|
log.vlog("The directory has not changed");
|
||||||
// loop trough the children
|
// loop trough the children
|
||||||
foreach (Item child; itemdb.selectChildren(item.id)) {
|
foreach (Item child; itemdb.selectChildren(item.driveId, item.id)) {
|
||||||
uploadDifferences(child);
|
uploadDifferences(child);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -434,7 +460,7 @@ final class SyncEngine
|
||||||
log.vlog("The file last modified time has changed");
|
log.vlog("The file last modified time has changed");
|
||||||
string id = item.id;
|
string id = item.id;
|
||||||
string eTag = item.eTag;
|
string eTag = item.eTag;
|
||||||
if (!testCrc32(path, item.crc32)) {
|
if (!testFileHash(path, item)) {
|
||||||
log.vlog("The file content has changed");
|
log.vlog("The file content has changed");
|
||||||
log.log("Uploading: ", path);
|
log.log("Uploading: ", path);
|
||||||
JSONValue response;
|
JSONValue response;
|
||||||
|
@ -536,7 +562,7 @@ final class SyncEngine
|
||||||
if (e.httpStatusCode == 404) log.log(e.msg);
|
if (e.httpStatusCode == 404) log.log(e.msg);
|
||||||
else throw e;
|
else throw e;
|
||||||
}
|
}
|
||||||
itemdb.deleteById(item.id);
|
itemdb.deleteById(item.driveId, item.id);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void uploadLastModifiedTime(const(char)[] id, const(char)[] eTag, SysTime mtime)
|
private void uploadLastModifiedTime(const(char)[] id, const(char)[] eTag, SysTime mtime)
|
||||||
|
@ -552,30 +578,7 @@ final class SyncEngine
|
||||||
|
|
||||||
private void saveItem(JSONValue jsonItem)
|
private void saveItem(JSONValue jsonItem)
|
||||||
{
|
{
|
||||||
ItemType type;
|
Item item = makeItem(jsonItem);
|
||||||
if (isItemFile(jsonItem)) {
|
|
||||||
type = ItemType.file;
|
|
||||||
} else if (isItemFolder(jsonItem)) {
|
|
||||||
type = ItemType.dir;
|
|
||||||
} else {
|
|
||||||
assert(0);
|
|
||||||
}
|
|
||||||
Item item = {
|
|
||||||
id: jsonItem["id"].str,
|
|
||||||
name: jsonItem["name"].str,
|
|
||||||
type: type,
|
|
||||||
eTag: jsonItem["eTag"].str,
|
|
||||||
cTag: "cTag" in jsonItem ? jsonItem["cTag"].str : null,
|
|
||||||
mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str),
|
|
||||||
parentId: jsonItem["parentReference"]["id"].str
|
|
||||||
};
|
|
||||||
if (type == ItemType.file) {
|
|
||||||
try {
|
|
||||||
item.crc32 = jsonItem["file"]["hashes"]["crc32Hash"].str;
|
|
||||||
} catch (JSONException e) {
|
|
||||||
log.vlog("The hash is not available");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
itemdb.upsert(item);
|
itemdb.upsert(item);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -43,7 +43,7 @@ struct UploadSession
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!exists(session["localPath"].str)) {
|
if (!exists(session["localPath"].str)) {
|
||||||
log.vlog("The file do not exist anymore");
|
log.vlog("The file does not exist anymore");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
// request the session status
|
// request the session status
|
||||||
|
|
38
src/util.d
38
src/util.d
|
@ -1,5 +1,6 @@
|
||||||
|
import std.base64;
|
||||||
import std.conv;
|
import std.conv;
|
||||||
import std.digest.crc;
|
import std.digest.crc, std.digest.sha;
|
||||||
import std.file;
|
import std.file;
|
||||||
import std.net.curl;
|
import std.net.curl;
|
||||||
import std.path;
|
import std.path;
|
||||||
|
@ -7,6 +8,7 @@ import std.regex;
|
||||||
import std.socket;
|
import std.socket;
|
||||||
import std.stdio;
|
import std.stdio;
|
||||||
import std.string;
|
import std.string;
|
||||||
|
import qxor;
|
||||||
|
|
||||||
private string deviceName;
|
private string deviceName;
|
||||||
|
|
||||||
|
@ -15,7 +17,7 @@ static this()
|
||||||
deviceName = Socket.hostName;
|
deviceName = Socket.hostName;
|
||||||
}
|
}
|
||||||
|
|
||||||
// give a new name to the specified file or directory
|
// gives a new name to the specified file or directory
|
||||||
void safeRename(const(char)[] path)
|
void safeRename(const(char)[] path)
|
||||||
{
|
{
|
||||||
auto ext = extension(path);
|
auto ext = extension(path);
|
||||||
|
@ -33,13 +35,13 @@ void safeRename(const(char)[] path)
|
||||||
rename(path, newPath);
|
rename(path, newPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
// delete the specified file without throwing an exception if it does not exists
|
// deletes the specified file without throwing an exception if it does not exists
|
||||||
void safeRemove(const(char)[] path)
|
void safeRemove(const(char)[] path)
|
||||||
{
|
{
|
||||||
if (exists(path)) remove(path);
|
if (exists(path)) remove(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
// return the crc32 hex string of a file
|
// returns the crc32 hex string of a file
|
||||||
string computeCrc32(string path)
|
string computeCrc32(string path)
|
||||||
{
|
{
|
||||||
CRC32 crc;
|
CRC32 crc;
|
||||||
|
@ -50,7 +52,29 @@ string computeCrc32(string path)
|
||||||
return crc.finish().toHexString().dup;
|
return crc.finish().toHexString().dup;
|
||||||
}
|
}
|
||||||
|
|
||||||
// convert wildcards (*, ?) to regex
|
// returns the sha1 hash hex string of a file
|
||||||
|
string computeSha1Hash(string path)
|
||||||
|
{
|
||||||
|
SHA1 sha;
|
||||||
|
auto file = File(path, "rb");
|
||||||
|
foreach (ubyte[] data; chunks(file, 4096)) {
|
||||||
|
sha.put(data);
|
||||||
|
}
|
||||||
|
return sha.finish().toHexString().dup;
|
||||||
|
}
|
||||||
|
|
||||||
|
// returns the quickXorHash base64 string of a file
|
||||||
|
string computeQuickXorHash(string path)
|
||||||
|
{
|
||||||
|
QuickXor qxor;
|
||||||
|
auto file = File(path, "rb");
|
||||||
|
foreach (ubyte[] data; chunks(file, 4096)) {
|
||||||
|
qxor.put(data);
|
||||||
|
}
|
||||||
|
return Base64.encode(qxor.finish());
|
||||||
|
}
|
||||||
|
|
||||||
|
// converts wildcards (*, ?) to regex
|
||||||
Regex!char wild2regex(const(char)[] pattern)
|
Regex!char wild2regex(const(char)[] pattern)
|
||||||
{
|
{
|
||||||
string str;
|
string str;
|
||||||
|
@ -79,7 +103,7 @@ Regex!char wild2regex(const(char)[] pattern)
|
||||||
return regex(str, "i");
|
return regex(str, "i");
|
||||||
}
|
}
|
||||||
|
|
||||||
// return true if the network connection is available
|
// returns true if the network connection is available
|
||||||
bool testNetwork()
|
bool testNetwork()
|
||||||
{
|
{
|
||||||
HTTP http = HTTP("https://login.microsoftonline.com");
|
HTTP http = HTTP("https://login.microsoftonline.com");
|
||||||
|
@ -87,7 +111,7 @@ bool testNetwork()
|
||||||
return http.perform(ThrowOnError.no) == 0;
|
return http.perform(ThrowOnError.no) == 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
// call globMatch for each string in pattern separated by '|'
|
// calls globMatch for each string in pattern separated by '|'
|
||||||
bool multiGlobMatch(const(char)[] path, const(char)[] pattern)
|
bool multiGlobMatch(const(char)[] path, const(char)[] pattern)
|
||||||
{
|
{
|
||||||
foreach (glob; pattern.split('|')) {
|
foreach (glob; pattern.split('|')) {
|
||||||
|
|
Loading…
Reference in a new issue