deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-18 05:36:41 +02:00
Code Issues Projects Releases Wiki Activity

Resolve another itemdb.d(307): Assertion

Browse source 
* Clean up --single-directory logging as we should not be spilling out the actual item details
* Fix some spelling errors
* Fix another core.exception.AssertError@src/itemdb.d(307): Assertion failure when attempting to compute the path
* Fix logging when items are being filtered out as non relevant
* Add upload validation that file uploaded equals the size of the file locally as this is a contributing factor to 412 errors
This commit is contained in:
abraunegg 2018-05-15 06:59:17 +10:00
parent 9520fe7716
commit 8b0e9513e3
1 changed files with 48 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
src/sync.d
View file

@ -49,7 +49,6 @@ private Item makeItem(const ref JSONValue driveItem)
name: "name" in driveItem ? driveItem["name"].str : null, // name may be missing for deleted files in OneDrive Biz name: "name" in driveItem ? driveItem["name"].str : null, // name may be missing for deleted files in OneDrive Biz
eTag: "eTag" in driveItem ? driveItem["eTag"].str : null, // eTag is not returned for the root in OneDrive Biz eTag: "eTag" in driveItem ? driveItem["eTag"].str : null, // eTag is not returned for the root in OneDrive Biz
cTag: "cTag" in driveItem ? driveItem["cTag"].str : null, // cTag is missing in old files (and all folders in OneDrive Biz) cTag: "cTag" in driveItem ? driveItem["cTag"].str : null, // cTag is missing in old files (and all folders in OneDrive Biz)
//mtime: "fileSystemInfo" in driveItem ? SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str) : SysTime(0),
}; };
// OneDrive API Change: https://github.com/OneDrive/onedrive-api-docs/issues/834 // OneDrive API Change: https://github.com/OneDrive/onedrive-api-docs/issues/834
@ -334,6 +333,7 @@ final class SyncEngine
// Exit Application // Exit Application
log.log("\n\nOneDrive returned a 'HTTP 500 - Internal Server Error'"); log.log("\n\nOneDrive returned a 'HTTP 500 - Internal Server Error'");
log.log("This is a OneDrive API Bug - https://github.com/OneDrive/onedrive-api-docs/issues/844\n\n"); log.log("This is a OneDrive API Bug - https://github.com/OneDrive/onedrive-api-docs/issues/844\n\n");
log.log("Remove your 'items.sqlite3' file and try to sync again\n\n");
return; return;
} }
@ -399,7 +399,7 @@ final class SyncEngine
idsToDelete ~= [driveId, item["id"].str]; idsToDelete ~= [driveId, item["id"].str];
} }
} else { } else {
log.vlog("Remote Change Discarded: ", item); log.vlog("Remote change discarded - not in --single-directory scope");
} }
} }
} }
@ -426,7 +426,7 @@ final class SyncEngine
if (isItemRoot(driveItem) || !item.parentId || isRoot) { if (isItemRoot(driveItem) || !item.parentId || isRoot) {
item.parentId = null; // ensures that it has no parent item.parentId = null; // ensures that it has no parent
item.driveId = driveId; // HACK: makeItem() cannot set the driveId propery of the root item.driveId = driveId; // HACK: makeItem() cannot set the driveId property of the root
itemdb.upsert(item); itemdb.upsert(item);
return; return;
} }
@ -453,14 +453,19 @@ final class SyncEngine
// check for selective sync // check for selective sync
string path; string path;
if (!unwanted) { if (!unwanted) {
path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name; // Is the item in the local database
path = buildNormalizedPath(path); if (itemdb.idInLocalDatabase(item.driveId, item.parentId)){
unwanted = selectiveSync.isPathExcluded(path); path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name;
path = buildNormalizedPath(path);
unwanted = selectiveSync.isPathExcluded(path);
} else {
unwanted = true;
}
} }
// skip unwanted items early // skip unwanted items early
if (unwanted) { if (unwanted) {
log.vlog("Filtered out"); //log.vlog("Filtered out");
skippedItems ~= item.id; skippedItems ~= item.id;
return; return;
} }
@ -471,7 +476,7 @@ final class SyncEngine
// check if the item is going to be deleted // check if the item is going to be deleted
if (isItemDeleted(driveItem)) { if (isItemDeleted(driveItem)) {
log.vlog("The item is marked for deletion"); log.vlog("This item is marked for deletion:", item.name);
if (cached) { if (cached) {
// flag to delete // flag to delete
idsToDelete ~= [item.driveId, item.id]; idsToDelete ~= [item.driveId, item.id];
@ -482,14 +487,17 @@ final class SyncEngine
return; return;
} }
// rename the local item if it is unsynced and there is a new version of it // rename the local item if it is unsynced and there is a new version of it on OneDrive
string oldPath; string oldPath;
if (cached && item.eTag != oldItem.eTag) { if (cached && item.eTag != oldItem.eTag) {
oldPath = itemdb.computePath(item.driveId, item.id); // Is the item in the local database
if (!isItemSynced(oldItem, oldPath)) { if (itemdb.idInLocalDatabase(item.driveId, item.parentId)){
log.vlog("The local item is unsynced, renaming"); oldPath = itemdb.computePath(item.driveId, item.id);
if (exists(oldPath)) safeRename(oldPath); if (!isItemSynced(oldItem, oldPath)) {
cached = false; log.vlog("The local item is unsynced, renaming");
if (exists(oldPath)) safeRename(oldPath);
cached = false;
}
} }
} }
@ -686,7 +694,7 @@ final class SyncEngine
// skip unwanted items // skip unwanted items
if (unwanted) { if (unwanted) {
log.vlog("Filtered out"); //log.vlog("Filtered out");
return; return;
} }
@ -802,7 +810,7 @@ final class SyncEngine
uploadCreateDir(path); uploadCreateDir(path);
} }
} else { } else {
log.vlog("The file has been deleted"); log.vlog("The file has been deleted locally");
uploadDeleteItem(item, path); uploadDeleteItem(item, path);
} }
} }
@ -929,7 +937,7 @@ final class SyncEngine
// Submit the creation request // Submit the creation request
auto res = onedrive.createById(parent.driveId, parent.id, driveItem); auto res = onedrive.createById(parent.driveId, parent.id, driveItem);
saveItem(res); saveItem(res);
log.vlog("Sucessfully created the remote directory ", path, " on OneDrive"); log.vlog("Successfully created the remote directory ", path, " on OneDrive");
return; return;
} }
} }
@ -1003,13 +1011,29 @@ final class SyncEngine
} }
log.fileOnly("Uploading file ", path, " ... done."); log.fileOnly("Uploading file ", path, " ... done.");
// Update the item's metadata on OneDrive // The file was uploaded
string id = response["id"].str; ulong uploadFileSize = response["size"].integer;
string cTag = response["cTag"].str;
SysTime mtime = timeLastModified(path).toUTC(); // In some cases the file that was uploaded was not complete, but 'completed' without errors on OneDrive
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded // This has been seen with PNG / JPG files mainly, which then contributes to generating a 412 error when we attempt to update the metadata
uploadLastModifiedTime(parent.driveId, id, cTag, mtime); // Validate here that the file uploaded, at least in size, matches in the response to what the size is on disk
return; if (thisFileSize != uploadFileSize){
// OK .. the uploaded file does not match
log.log("Uploaded file size does not match local file - upload failure - retrying");
// Delete uploaded bad file
onedrive.deleteById(response["parentReference"]["driveId"].str, response["id"].str, response["eTag"].str);
// Re-upload
uploadNewFile(path);
return;
} else {
// Update the item's metadata on OneDrive
string id = response["id"].str;
string cTag = response["cTag"].str;
SysTime mtime = timeLastModified(path).toUTC();
// use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
return;
}
} }
} }