diff --git a/Makefile.in b/Makefile.in index 5f0ad31c..ead92c9a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -55,7 +55,7 @@ endif system_unit_files = contrib/systemd/onedrive@.service user_unit_files = contrib/systemd/onedrive.service -DOCFILES = README.md config LICENSE CHANGELOG.md docs/Docker.md docs/INSTALL.md docs/SharePoint-Shared-Libraries.md docs/USAGE.md docs/BusinessSharedFolders.md docs/advanced-usage.md docs/application-security.md +DOCFILES = readme.md config LICENSE changelog.md docs/advanced-usage.md docs/application-config-options.md docs/application-security.md docs/business-shared-folders.md docs/docker.md docs/install.md docs/national-cloud-deployments.md docs/podman.md docs/privacy-policy.md docs/sharepoint-libraries.md docs/terms-of-service.md docs/ubuntu-package-install.md docs/usage.md ifneq ("$(wildcard /etc/redhat-release)","") RHEL = $(shell cat /etc/redhat-release | grep -E "(Red Hat Enterprise Linux|CentOS)" | wc -l) @@ -66,20 +66,19 @@ RHEL_VERSION = 0 endif SOURCES = \ - src/config.d \ - src/itemdb.d \ - src/log.d \ src/main.d \ - src/monitor.d \ - src/onedrive.d \ - src/qxor.d \ - src/selective.d \ - src/sqlite.d \ - src/sync.d \ - src/upload.d \ + src/config.d \ + src/log.d \ src/util.d \ + src/qxor.d \ + src/curlEngine.d \ + src/onedrive.d \ + src/sync.d \ + src/itemdb.d \ + src/sqlite.d \ + src/clientSideFiltering.d \ src/progress.d \ - src/arsd/cgi.d + src/monitor.d ifeq ($(NOTIFICATIONS),yes) SOURCES += src/notifications/notify.d src/notifications/dnotify.d diff --git a/changelog.md b/changelog.md new file mode 100644 index 00000000..8f7f357a --- /dev/null +++ b/changelog.md @@ -0,0 +1,1041 @@ +# Changelog +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +## 2.5.0 - TBA + + +### Changed +* Renamed various documentation files to align with document content + + +## 2.4.25 - 2023-06-21 +### Fixed +* Fixed that the application was reporting as v2.2.24 when in fact it was v2.4.24 (release tagging issue) +* Fixed that the running version obsolete flag (due to above issue) was causing a false flag as being obsolete +* Fixed that zero-byte files do not have a hash as reported by the OneDrive API thus should not generate an error message + +### Updated +* Update to Debian Docker file to resolve Docker image Operating System reported vulnerabilities +* Update to Alpine Docker file to resolve Docker image Operating System reported vulnerabilities +* Update to Fedora Docker file to resolve Docker image Operating System reported vulnerabilities +* Updated documentation (various) + +## 2.4.24 - 2023-06-20 +### Fixed +* Fix for extra encoded quotation marks surrounding Docker environment variables +* Fix webhook subscription creation for SharePoint Libraries +* Fix that a HTTP 504 - Gateway Timeout causes local files to be deleted when using --download-only & --cleanup-local-files mode +* Fix that folders are renamed despite using --dry-run +* Fix deprecation warnings with dmd 2.103.0 +* Fix error that the application is unable to perform a database vacuum: out of memory when exiting + +### Removed +* Remove sha1 from being used by the client as this is being depreciated by Microsoft in July 2023 +* Complete the removal of crc32 elements + +### Added +* Added ONEDRIVE_SINGLE_DIRECTORY configuration capability to Docker +* Added --get-file-link shell completion +* Added configuration to allow HTTP session timeout(s) tuning via config (taken from v2.5.x) + +### Updated +* Update to Debian Docker file to resolve Docker image Operating System reported vulnerabilities +* Update to Alpine Docker file to resolve Docker image Operating System reported vulnerabilities +* Update to Fedora Docker file to resolve Docker image Operating System reported vulnerabilities +* Updated cgi.d to commit 680003a - last upstream change before requiring `core.d` dependency requirement +* Updated documentation (various) + +## 2.4.23 - 2023-01-06 +### Fixed +* Fixed RHEL7, RHEL8 and RHEL9 Makefile and SPEC file compatibility + +### Removed +* Disable systemd 'PrivateUsers' due to issues with systemd running processes when option is enabled, causes local file deletes on RHEL based systems + +### Updated +* Update --get-O365-drive-id error handling to display a more a more appropriate error message if the API cannot be found +* Update the GitHub version check to utilise the date a release was done, to allow 1 month grace period before generating obsolete version message +* Update Alpine Dockerfile to use Alpine 3.17 and Golang 1.19 +* Update handling of --source-directory and --destination-directory if one is empty or missing and if used with --synchronize or --monitor +* Updated documentation (various) + +## 2.4.22 - 2022-12-06 +### Fixed +* Fix application crash when local file is changed to a symbolic link with non-existent target +* Fix build error with dmd-2.101.0 +* Fix build error with LDC 1.28.1 on Alpine +* Fix issue of silent exit when unable to delete local files when using --cleanup-local-files +* Fix application crash due to access permissions on configured path for sync_dir +* Fix potential application crash when exiting due to failure state and unable to cleanly shutdown the database +* Fix creation of parent empty directories when parent is excluded by sync_list + +### Added +* Added performance output details for key functions + +### Changed +* Switch Docker 'latest' to point at Debian builds rather than Fedora due to ongoing Fedora build failures +* Align application logging events to actual application defaults for --monitor operations +* Performance Improvement: Avoid duplicate costly path calculations and DB operations if not required +* Disable non-working remaining sandboxing options within systemd service files +* Performance Improvement: Only check 'sync_list' if this has been enabled and configured +* Display 'Sync with OneDrive is complete' when using --synchronize +* Change the order of processing between Microsoft OneDrive restrictions and limitations check and skip_file|skip_dir check + +### Removed +* Remove building Fedora ARMv7 builds due to ongoing build failures + +### Updated +* Update config change detection handling +* Updated documentation (various) + +## 2.4.21 - 2022-09-27 +### Fixed +* Fix that the download progress bar doesn't always reach 100% when rate_limit is set +* Fix --resync handling of database file removal +* Fix Makefile to be consistent with permissions that are being used +* Fix that logging output for skipped uploaded files is missing +* Fix to allow non-sync tasks while sync is running +* Fix where --resync is enforced for non-sync operations +* Fix to resolve segfault when running 'onedrive --display-sync-status' when run as 2nd process +* Fix DMD 2.100.2 depreciation warning + +### Added +* Add GitHub Action Test Build Workflow (replacing Travis CI) +* Add option --display-running-config to display the running configuration as used at application startup +* Add 'config' option to request readonly access in oauth authorization step +* Add option --cleanup-local-files to cleanup local files regardless of sync state when using --download-only +* Add option --with-editing-perms to create a read-write shareable link when used with --create-share-link + +### Changed +* Change the exit code of the application to 126 when a --resync is required + +### Updated +* Updated --get-O365-drive-id implementation for data access +* Update what application options require an argument +* Update application logging output for error messages to remove certain \n prefix when logging to a file +* Update onedrive.spec.in to fix error building RPM +* Update GUI notification handling for specific skipped scenarios +* Updated documentation (various) + +## 2.4.20 - 2022-07-20 +### Fixed +* Fix 'foreign key constraint failed' when using OneDrive Business Shared Folders due to change to using /delta query +* Fix various little spelling fixes (check with lintian during Debian packaging) +* Fix handling of a custom configuration directory when using --confdir +* Fix to ensure that any active http instance is shutdown before any application exit +* Fix to enforce that --confdir must be a directory + +### Added +* Added 'force_http_11' configuration option to allow forcing HTTP/1.1 operations + +### Changed +* Increased thread sleep for better process I/O wait handling +* Removed 'force_http_2' configuration option + +### Updated +* Update OneDrive API response handling for National Cloud Deployments +* Updated to switch to using curl defaults for HTTP/2 operations +* Updated documentation (various) + +## 2.4.19 - 2022-06-15 +### Fixed +* Update Business Shared Folders to use a /delta query +* Update when DB is updated by OneDrive API data and update when file hash is required to be generated + +### Added +* Added ONEDRIVE_UPLOADONLY flag for Docker + +### Updated +* Updated GitHub workflows +* Updated documentation (various) + +## 2.4.18 - 2022-06-02 +### Fixed +* Fixed various database related access issues steming from running multiple instances of the application at the same time using the same configuration data +* Fixed --display-config being impacted by --resync flag +* Fixed installation permissions for onedrive man-pages file +* Fixed that in some situations that users try --upload-only and --download-only together which is not possible +* Fixed application crash if unable to read required hash files + +### Added +* Added Feature Request to add an override for skip_dir|skip_file through flag to force sync +* Added a check to validate local filesystem available space before attempting file download +* Added GitHub Actions to build Docker containers and push to DockerHub + +### Updated +* Updated all Docker build files to current distributions, using updated distribution LDC version +* Updated logging output to logfiles when an actual sync process is occuring +* Updated output of --display-config to be more relevant +* Updated manpage to align with application configuration +* Updated documentation and Docker files based on minimum compiler versions to dmd-2.088.0 and ldc-1.18.0 +* Updated documentation (various) + +## 2.4.17 - 2022-04-30 +### Fixed +* Fix docker build, by add missing git package for Fedora builds +* Fix application crash when attempting to sync a broken symbolic link +* Fix Internet connect disruption retry handling and logging output +* Fix local folder creation timestamp with timestamp from OneDrive +* Fix logging output when download failed + +### Added +* Add additional logging specifically for delete event to denote in log output the source of a deletion event when running in --monitor mode + +### Changed +* Improve when the local database integrity check is performed and on what frequency the database integrity check is performed + +### Updated +* Remove application output ambiguity on how to access 'help' for the client +* Update logging output when running in --monitor --verbose mode in regards to the inotify events +* Updated documentation (various) + +## 2.4.16 - 2022-03-10 +### Fixed +* Update application file logging error handling +* Explicitly set libcurl options +* Fix that when a sync_list exclusion is matched, the item needs to be excluded when using --resync +* Fix so that application can be compiled correctly on Android hosts +* Fix the handling of 429 and 5xx responses when they are generated by OneDrive in a self-referencing circular pattern +* Fix applying permissions to volume directories when running in rootless podman +* Fix unhandled errors from OneDrive when initialising subscriptions fail + +### Added +* Enable GitHub Sponsors +* Implement --resync-auth to enable CLI passing in of --rsync approval +* Add function to check client version vs latest GitHub release +* Add --reauth to allow easy re-authentication of the client +* Implement --modified-by to display who last modified a file and when the modification was done +* Implement feature request to mark partially-downloaded files as .partial during download +* Add documentation for Podman support + +### Changed +* Document risk regarding using --resync and force user acceptance of usage risk to proceed +* Use YAML for Bug Reports and Feature Requests +* Update Dockerfiles to use more modern base Linux distribution + +### Updated +* Updated documentation (various) + +## 2.4.15 - 2021-12-31 +### Fixed +* Fix unable to upload to OneDrive Business Shared Folders due to OneDrive API restricting quota information +* Update fixing edge case with OneDrive Personal Shared Folders and --resync --upload-only + +### Added +* Add SystemD hardening +* Add --operation-timeout argument + +### Changed +* Updated minimum compiler versions to dmd-2.087.0 and ldc-1.17.0 + +### Updated +* Updated Dockerfile-alpine to use Apline 3.14 +* Updated documentation (various) + +## 2.4.14 - 2021-11-24 +### Fixed +* Support DMD 2.097.0 as compiler for Docker Builds +* Fix getPathDetailsByDriveId query when using --dry-run and a nested path with --single-directory +* Fix edge case when syncing OneDrive Personal Shared Folders +* Catch unhandled API response errors when querying OneDrive Business Shared Folders +* Catch unhandled API response errors when listing OneDrive Business Shared Folders +* Fix error 'Key not found: remaining' with Business Shared Folders (OneDrive API change) +* Fix overwriting local files with older versions from OneDrive when items.sqlite3 does not exist and --resync is not used + +### Added +* Added operation_timeout as a new configuration to assist in cases where operations take longer that 1h to complete +* Add Real-Time syncing of remote updates via webhooks +* Add --auth-response option and expose through entrypoint.sh for Docker +* Add --disable-download-validation + +### Changed +* Always prompt for credentials for authentication rather than re-using cached browser details +* Do not re-auth on --logout + +### Updated +* Updated documentation (various) + +## 2.4.13 - 2021-7-14 +### Fixed +* Support DMD 2.097.0 as compiler +* Fix to handle OneDrive API Bad Request response when querying if file exists +* Fix application crash and incorrect handling of --single-directory when syncing a OneDrive Business Shared Folder due to using 'Add Shortcut to My Files' +* Fix application crash due to invalid UTF-8 sequence in the pathname for the application configuration +* Fix error message when deleting a large number of files +* Fix Docker build process to source GOSU keys from updated GPG key location +* Fix application crash due to a conversion overflow when calculating file offset for session uploads +* Fix Docker Alpine build failing due to filesystem permissions issue due to Docker build system and Alpine Linux 3.14 incompatibility +* Fix that Business Shared Folders with parentheses are ignored + +### Updated +* Updated Lock Bot to run daily +* Updated documentation (various) + +## 2.4.12 - 2021-5-28 +### Fixed +* Fix an unhandled Error 412 when uploading modified files to OneDrive Business Accounts +* Fix 'sync_list' handling of inclusions when name is included in another folders name +* Fix that options --upload-only & --remove-source-files are ignored on an upload session restore +* Fix to add file check when adding item to database if using --upload-only --remove-source-files +* Fix application crash when SharePoint displayName is being withheld + +### Updated +* Updated Lock Bot to use GitHub Actions +* Updated documentation (various) + +## 2.4.11 - 2021-4-07 +### Fixed +* Fix support for '/*' regardless of location within sync_list file +* Fix 429 response handling correctly check for 'retry-after' response header and use set value +* Fix 'sync_list' path handling for sub item matching, so that items in parent are not implicitly matched when there is no wildcard present +* Fix --get-O365-drive-id to use 'nextLink' value if present when searching for specific SharePoint site names +* Fix OneDrive Business Shared Folder existing name conflict check +* Fix incorrect error message 'Item cannot be deleted from OneDrive because it was not found in the local database' when item is actually present +* Fix application crash when unable to rename folder structure due to unhandled file-system issue +* Fix uploading documents to Shared Business Folders when the shared folder exists on a SharePoint site due to Microsoft Sharepoint 'enrichment' of files +* Fix that a file record is kept in database when using --no-remote-delete & --remove-source-files + +### Added +* Added support in --get-O365-drive-id to provide the 'drive_id' for multiple 'document libraries' within a single Shared Library Site + +### Removed +* Removed the depreciated config option 'force_http_11' which was flagged as depreciated by PR #549 in v2.3.6 (June 2019) + +### Updated +* Updated error output of --get-O365-drive-id to provide more details why an error occurred if a SharePoint site lacks the details we need to perform the match +* Updated Docker build files for Raspberry Pi to dedicated armhf & aarch64 Dockerfiles +* Updated logging output when in --monitor mode, avoid outputting misleading logging when the new or modified item is a file, not a directory +* Updated documentation (various) + +## 2.4.10 - 2021-2-19 +### Fixed +* Catch database assertion when item path cannot be calculated +* Fix alpine Docker build so it uses the same golang alpine version +* Search all distinct drive id's rather than just default drive id for --get-file-link +* Use correct driveId value to query for changes when using --single-directory +* Improve upload handling of files for SharePoint sites and detecting when SharePoint modifies the file post upload +* Correctly handle '~' when present in 'log_dir' configuration option +* Fix logging output when handing downloaded new files +* Fix to use correct path offset for sync_list exclusion matching + +### Added +* Add upload speed metrics when files are uploaded and clarify that 'data to transfer' is what is needed to be downloaded from OneDrive +* Add new config option to rate limit connection to OneDrive +* Support new file maximum upload size of 250GB +* Support sync_list matching full path root wildcard with exclusions to simplify sync_list configuration + +### Updated +* Rename Office365.md --> SharePoint-Shared-Libraries.md which better describes this document +* Updated Dockerfile config for arm64 +* Updated documentation (various) + +## 2.4.9 - 2020-12-27 +### Fixed +* Fix to handle case where API provided deltaLink generates a further API error +* Fix application crash when unable to read a local file due to local file permissions +* Fix application crash when calculating the path length due to invalid UTF characters in local path +* Fix Docker build on Alpine due missing symbols due to using the edge version of ldc and ldc-runtime +* Fix application crash with --get-O365-drive-id when API response is restricted + +### Added +* Add debug log output of the configured URL's which will be used throughout the application to remove any ambiguity as to using incorrect URL's when making API calls +* Improve application startup when using --monitor when there is no network connection to the OneDrive API and only initialise application once OneDrive API is reachable +* Add Docker environment variable to allow --logout for re-authentication + +### Updated +* Remove duplicate code for error output functions and enhance error logging output +* Updated documentation + +## 2.4.8 - 2020-11-30 +### Fixed +* Fix to use config set option for 'remove_source_files' and 'skip_dir_strict_match' rather than ignore if set +* Fix download failure and crash due to incorrect local filesystem permissions when using mounted external devices +* Fix to not change permissions on pre-existing local directories +* Fix logging output when authentication authorisation fails to not say authorisation was successful +* Fix to check application_id before setting redirect URL when using specific Azure endpoints +* Fix application crash in --monitor mode due to 'Failed to stat file' when setgid is used on a directory and data cannot be read + +### Added +* Added advanced-usage.md to document advaced client usage such as multi account configurations and Windows dual-boot + +### Updated +* Updated --verbose logging output for config options when set +* Updated documentation (man page, USAGE.md, Office365.md, BusinessSharedFolders.md) + +## 2.4.7 - 2020-11-09 +### Fixed +* Fix debugging output for /delta changes available queries +* Fix logging output for modification comparison source data +* Fix Business Shared Folder handling to process only Shared Folders, not individually shared files +* Fix cleanup dryrun shm and wal files if they exist +* Fix --list-shared-folders to only show folders +* Fix to check for the presence of .nosync when processing DB entries +* Fix skip_dir matching when using --resync +* Fix uploading data to shared business folders when using --upload-only +* Fix to merge contents of SQLite WAL file into main database file on sync completion +* Fix to check if localModifiedTime is >= than item.mtime to avoid re-upload for equal modified time +* Fix to correctly set config directory permissions at first start + +### Added +* Added environment variable to allow easy HTTPS debug in docker +* Added environment variable to allow download-only mode in Docker +* Implement Feature: Allow config to specify a tenant id for non-multi-tenant applications +* Implement Feature: Adding support for authentication with single tenant custom applications +* Implement Feature: Configure specific File and Folder Permissions + +### Updated +* Updated documentation (readme.md, install.md, usage.md, bug_report.md) + +## 2.4.6 - 2020-10-04 +### Fixed +* Fix flagging of remaining free space when value is being restricted +* Fix --single-directory path handling when path does not exist locally +* Fix checking for 'Icon' path as no longer listed by Microsoft as an invalid file or folder name +* Fix removing child items on OneDrive when parent item responds with access denied +* Fix to handle deletion events for files when inotify events are missing +* Fix uninitialised value error as reported by valgrind +* Fix to handle deletion events for directories when inotify events are missing + +### Added +* Implement Feature: Create shareable link +* Implement Feature: Support wildcard within sync_list entries +* Implement Feature: Support negative patterns in sync_list for fine grained exclusions +* Implement Feature: Multiple skip_dir & skip_file configuration rules +* Add GUI notification to advise users when the client needs to be reauthenticated + +### Updated +* Updated documentation (readme.md, install.md, usage.md, bug_report.md) + +## 2.4.5 - 2020-08-13 +### Fixed +* Fixed fish auto completions installation destination + +## 2.4.4 - 2020-08-11 +### Fixed +* Fix 'skip_dir' & 'skip_file' pattern matching to ensure correct matching is performed +* Fix 'skip_dir' & 'skip_file' so that each directive is only used against directories or files as requried in --monitor +* Fix client hand when attempting to sync a Unix pipe file +* Fix --single-directory & 'sync_list' performance +* Fix erroneous 'return' statements which could prematurely end processing all changes returned from OneDrive +* Fix segfault when attempting to perform a comparison on an inotify event when determining if event path is directory or file +* Fix handling of Shared Folders to ensure these are checked against 'skip_dir' entries +* Fix 'Skipping uploading this new file as parent path is not in the database' when uploading to a Personal Shared Folder +* Fix how available free space is tracked when uploading files to OneDrive and Shared Folders +* Fix --single-directory handling of parent path matching if path is being seen for first time + +### Added +* Added Fish auto completions + +### Updated +* Increase maximum individual file size to 100GB due to Microsoft file limit increase +* Update Docker build files and align version of compiler across all Docker builds +* Update Docker documentation +* Update NixOS build information +* Update the 'Processing XXXX' output to display the full path +* Update logging output when a sync starts and completes when using --monitor +* Update Office 365 / SharePoint site search query and response if query return zero match + +## 2.4.3 - 2020-06-29 +### Fixed +* Check if symbolic link is relative to location path +* When using output logfile, fix inconsistent output spacing +* Perform initial sync at startup in monitor mode +* Handle a 'race' condition to process inotify events generated whilst performing DB or filesystem walk +* Fix segfault when moving folder outside the sync directory when using --monitor on Arch Linux + +### Added +* Added additional inotify event debugging +* Added support for loading system configs if there's no user config +* Added Ubuntu installation details to include installing the client from a PPA +* Added openSUSE installation details to include installing the client from a package +* Added support for comments in sync_list file +* Implement recursive deletion when Retention Policy is enabled on OneDrive Business Accounts +* Implement support for National cloud deployments +* Implement OneDrive Business Shared Folders Support + +### Updated +* Updated documentation files (various) +* Updated log output messaging when a full scan has been set or triggered +* Updated buildNormalizedPath complexity to simplify code +* Updated to only process OneDrive Personal Shared Folders only if account type is 'personal' + +## 2.4.2 - 2020-05-27 +### Fixed +* Fixed the catching of an unhandled exception when inotify throws an error +* Fixed an uncaught '100 Continue' response when files are being uploaded +* Fixed progress bar for uploads to be more accurate regarding percentage complete +* Fixed handling of database query enforcement if item is from a shared folder +* Fixed compiler depreciation of std.digest.digest +* Fixed checking & loading of configuration file sequence +* Fixed multiple issues reported by Valgrind +* Fixed double scan at application startup when using --monitor & --resync together +* Fixed when renaming a file locally, ensure that the target filename is valid before attempting to upload to OneDrive +* Fixed so that if a file is modified locally and --resync is used, rename the local file for data preservation to prevent local data loss + +### Added +* Implement 'bypass_data_preservation' enhancement + +### Changed +* Changed the monitor interval default to 300 seconds + +### Updated +* Updated the handling of out-of-space message when OneDrive is out of space +* Updated debug logging for retry wait times + +## 2.4.1 - 2020-05-02 +### Fixed +* Fixed the handling of renaming files to a name starting with a dot when skip_dotfiles = true +* Fixed the handling of parentheses from path or file names, when doing comparison with regex +* Fixed the handling of renaming dotfiles to another dotfile when skip_dotfile=true in monitor mode +* Fixed the handling of --dry-run and --resync together correctly as current database may be corrupt +* Fixed building on Alpine Linux under Docker +* Fixed the handling of --single-directory for --dry-run and --resync scenarios +* Fixed the handling of .nosync directive when downloading new files into existing directories that is (was) in sync +* Fixed the handling of zero-byte modified files for OneDrive Business +* Fixed skip_dotfiles handling of .folders when in monitor mode to prevent monitoring +* Fixed the handling of '.folder' -> 'folder' move when skip_dotfiles is enabled +* Fixed the handling of folders that cannot be read (permission error) if parent should be skipped +* Fixed the handling of moving folders from skipped directory to non-skipped directory via OneDrive web interface +* Fixed building on CentOS Linux under Docker +* Fixed Codacy reported issues: double quote to prevent globbing and word splitting +* Fixed an assertion when attempting to compute complex path comparison from shared folders +* Fixed the handling of .folders when being skipped via skip_dir + +### Added +* Implement Feature: Implement the ability to set --resync as a config option, default is false + +### Updated +* Update error logging to be consistent when initialising fails +* Update error logging output to handle HTML error response reasoning if present +* Update link to new Microsoft documentation +* Update logging output to differentiate between OneNote objects and other unsupported objects +* Update RHEL/CentOS spec file example +* Update known-issues.md regarding 'SSL_ERROR_SYSCALL, errno 104' +* Update progress bar to be more accurate when downloading large files +* Updated #658 and #865 handling of when to trigger a directory walk when changes occur on OneDrive +* Updated handling of when a full scan is requried due to utilising sync_list +* Updated handling of when OneDrive service throws a 429 or 504 response to retry original request after a delay + +## 2.4.0 - 2020-03-22 +### Fixed +* Fixed how the application handles 429 response codes from OneDrive (critical update) +* Fixed building on Alpine Linux under Docker +* Fixed how the 'username' is determined from the running process for logfile naming +* Fixed file handling when a failed download has occured due to exiting via CTRL-C +* Fixed an unhandled exception when OneDrive throws an error response on initialising +* Fixed the handling of moving files into a skipped .folder when skip_dotfiles = true +* Fixed the regex parsing of response URI to avoid potentially generating a bad request to OneDrive, leading to a 'AADSTS9002313: Invalid request. Request is malformed or invalid.' response. + +### Added +* Added a Dockerfile for building on Rasberry Pi / ARM platforms +* Implement Feature: warning on big deletes to safeguard data on OneDrive +* Implement Feature: delete local files after sync +* Implement Feature: perform skip_dir explicit match only +* Implement Feature: provide config file option for specifying the Client Identifier + +### Changed +* Updated the 'Client Identifier' to a new Application ID + +### Updated +* Updated relevant documentation (README.md, USAGE.md) to add new feature details and clarify existing information +* Update completions to include the --force-http-2 option +* Update to always log when a file is skipped due to the item being invalid +* Update application output when just authorising application to make information clearer +* Update logging output when using sync_list to be clearer as to what is actually being processed and why + +## 2.3.13 - 2019-12-31 +### Fixed +* Change the sync list override flag to false as default when not using sync_list +* Fix --dry-run output when using --upload-only & --no-remote-delete and deleting local files + +### Added +* Add a verbose log entry when a monitor sync loop with OneDrive starts & completes + +### Changed +* Remove logAndNotify for 'processing X changes' as it is excessive for each change bundle to inform the desktop of the number of changes the client is processing + +### Updated +* Updated INSTALL.md with Ubuntu 16.x i386 build instructions to reflect working configuration on legacy hardware +* Updated INSTALL.md with details of Linux packages +* Updated INSTALL.md build instructions for CentOS platforms + +## 2.3.12 - 2019-12-04 +### Fixed +* Retry session upload fragment when transient errors occur to prevent silent upload failure +* Update Microsoft restriction and limitations about windows naming files to include '~' for folder names +* Docker guide fixes, add multiple account setup instructions +* Check database for excluded sync_list items previously in scope +* Catch DNS resolution error +* Fix where an item now out of scope should be flagged for local delete +* Fix rebuilding of onedrive, but ensure version is properly updated +* Update Ubuntu i386 build instructions to use DMD using preferred method + +### Added +* Add debug message to when a message is sent to dbus or notification daemon +* Add i386 instructions for legacy low memory platforms using LDC + +## 2.3.11 - 2019-11-05 +### Fixed +* Fix typo in the documentation regarding invalid config when upgrading from 'skilion' codebase +* Fix handling of skip_dir, skip_file & sync_list config options +* Fix typo in the documentation regarding sync_list +* Fix log output to be consistent with sync_list exclusion +* Fix 'Processing X changes' output to be more reflective of actual activity when using sync_list +* Remove unused and unexported SED variable in Makefile.in +* Handle curl exceptions and timeouts better with backoff/retry logic +* Update skip_dir pattern matching when using wildcards +* Fix when a full rescan is performed when using sync_list +* Fix 'Key not found: name' when computing skip_dir path +* Fix call from --monitor to observe --no-remote-delete +* Fix unhandled exception when monitor initialisation failure occurs due to too many open local files +* Fix unhandled 412 error response from OneDrive API when moving files right after upload +* Fix --monitor when used with --download-only. This fixes a regression introduced in 12947d1. +* Fix if --single-directory is being used, and we are using --monitor, only set inotify watches on the single directory + +### Changed +* Move JSON logging output from error messages to debug output + +## 2.3.10 - 2019-10-01 +### Fixed +* Fix searching for 'name' when deleting a synced item, if the OneDrive API does not return the expected details in the API call +* Fix abnormal termination when no Internet connection +* Fix downloading of files from OneDrive Personal Shared Folders when the OneDrive API responds with unexpected additional path data +* Fix logging of 'initialisation' of client to actually when the attempt to initialise is performed +* Fix when using a sync_list file, using deltaLink will actually 'miss' changes (moves & deletes) on OneDrive as using sync_list discards changes +* Fix OneDrive API status code 500 handling when uploading files as error message is not correct +* Fix crash when resume_upload file is not a valid JSON +* Fix crash when a file system exception is generated when attempting to update the file date & time and this fails + +### Added +* If there is a case-insensitive match error, also return the remote name from the response +* Make user-agent string a configuration option & add to config file +* Set default User-Agent to 'OneDrive Client for Linux v{version}' + +### Changed +* Make verbose logging output optional on Docker +* Enable --resync & debug client output via environment variables on Docker + +## 2.3.9 - 2019-09-01 +### Fixed +* Catch a 403 Forbidden exception when querying Sharepoint Library Names +* Fix unhandled error exceptions that cause application to exit / crash when uploading files +* Fix JSON object validation for queries made against OneDrive where a JSON response is expected and where that response is to be used and expected to be valid +* Fix handling of 5xx responses from OneDrive when uploading via a session + +### Added +* Detect the need for --resync when config changes either via config file or cli override + +### Changed +* Change minimum required version of LDC to v1.12.0 + +### Removed +* Remove redundant logging output due to change in how errors are reported from OneDrive + +## 2.3.8 - 2019-08-04 +### Fixed +* Fix unable to download all files when OneDrive fails to return file level details used to validate file integrity +* Included the flag "-m" to create the home directory when creating the user +* Fix entrypoint.sh to work with "sudo docker run" +* Fix docker build error on stretch +* Fix hidden directories in 'root' from having prefix removed +* Fix Sharepoint Document Library handling for .txt & .csv files +* Fix logging for init.d service +* Fix OneDrive response missing required 'id' element when uploading images +* Fix 'Unexpected character '<'. (Line 1:1)' when OneDrive has an exception error +* Fix error when creating the sync dir fails when there is no permission to create the sync dir + +### Added +* Add explicit check for hashes to be returned in cases where OneDrive API fails to provide them despite requested to do so +* Add comparison with sha1 if OneDrive provides that rather than quickXor +* Add selinux configuration details for a sync folder outside of the home folder +* Add date tag on docker.hub +* Add back CentOS 6 install & uninstall to Makefile +* Add a check to handle moving items out of sync_list sync scope & delete locally if true +* Implement --get-file-link which will return the weburl of a file which has been synced to OneDrive + +### Changed +* Change unauthorized-api exit code to 3 +* Update LDC to v1.16.0 for Travis CI testing +* Use replace function for modified Sharepoint Document Library files rather than delete and upload as new file, preserving file history +* Update Sharepoint modified file handling for files > 4Mb in size + +### Removed +* Remove -d shorthand for --download-only to avoid confusion with other GNU applications where -d stands for 'debug' + +## 2.3.7 - 2019-07-03 +### Fixed +* Fix not all files being downloaded due to OneDrive query failure +* False DB update which potentially could had lead to false data loss on OneDrive + +## 2.3.6 - 2019-07-03 (DO NOT USE) +### Fixed +* Fix JSONValue object validation +* Fix building without git being available +* Fix some spelling/grammatical errors +* Fix OneDrive error response on creating upload session + +### Added +* Add download size & hash check to ensure downloaded files are valid and not corrupt +* Added --force-http-2 to use HTTP/2 if desired + +### Changed +* Depreciated --force-http-1.1 (enabled by default) due to OneDrive inconsistent behavior with HTTP/2 protocol + +## 2.3.5 - 2019-06-19 +### Fixed +* Handle a directory in the sync_dir when no permission to access +* Get rid of forced root necessity during installation +* Fix broken autoconf code for --enable-XXX options +* Fix so that skip_size check should only be used if configured +* Fix a OneDrive Internal Error exception occurring before attempting to download a file + +### Added +* Check for supported version of D compiler + +## 2.3.4 - 2019-06-13 +### Fixed +* Fix 'Local files not deleted' when using bad 'skip_file' entry +* Fix --dry-run logging output for faking downloading new files +* Fix install unit files to correct location on RHEL/CentOS 7 +* Fix up unit file removal on all platforms +* Fix setting times on a file by adding a check to see if the file was actually downloaded before attempting to set the times on the file +* Fix an unhandled curl exception when OneDrive throws an internal timeout error +* Check timestamp to ensure that latest timestamp is used when comparing OneDrive changes +* Fix handling responses where cTag JSON elements are missing +* Fix Docker entrypoint.sh failures when GID is defined but not UID + +### Added +* Add autoconf based build system +* Add an encoding validation check before any path length checks are performed as if the path contains any invalid UTF-8 sequences +* Implement --sync-root-files to sync all files in the OneDrive root when using a sync_list file that would normally exclude these files from being synced +* Implement skip_size feature request +* Implement feature request to support file based OneDrive authorization (request | response) + +### Updated +* Better handle initialisation issues when OneDrive / MS Graph is experiencing problems that generate 401 & 5xx error codes +* Enhance error message when unable to connect to Microsoft OneDrive service when the local CA SSL certificate(s) have issues +* Update Dockerfile to correctly build on Docker Hub +* Rework directory layout and re-factor MD files for readability + +## 2.3.3 - 2019-04-16 +### Fixed +* Fix --upload-only check for Sharepoint uploads +* Fix check to ensure item root we flag as 'root' actually is OneDrive account 'root' +* Handle object error response from OneDrive when uploading to OneDrive Business +* Fix handling of some OneDrive accounts not providing 'quota' details +* Fix 'resume_upload' handling in the event of bad OneDrive response + +### Added +* Add debugging for --get-O365-drive-id function +* Add shell (bash,zsh) completion support +* Add config options for command line switches to allow for better config handling in docker containers + +### Updated +* Implement more meaningful 5xx error responses +* Update onedrive.logrotate indentations and comments +* Update 'min_notif_changes' to 'min_notify_changes' + +## 2.3.2 - 2019-04-02 +### Fixed +* Reduce scanning the entire local system in monitor mode for local changes +* Resolve file creation loop when working directly in the synced folder and Microsoft Sharepoint + +### Added +* Add 'monitor_fullscan_frequency' config option to set the frequency of performing a full disk scan when in monitor mode + +### Updated +* Update default 'skip_file' to include tmp and lock files generated by LibreOffice +* Update database version due to changing defaults of 'skip_file' which will force a rebuild and use of new skip_file default regex + +## 2.3.1 - 2019-03-26 +### Fixed +* Resolve 'make install' issue where rebuild of application would occur due to 'version' being flagged as .PHONY +* Update readme build instructions to include 'make clean;' before build to ensure that 'version' is cleanly removed and can be updated correctly +* Update Debian Travis CI build URL's + +## 2.3.0 - 2019-03-25 +### Fixed +* Resolve application crash if no 'size' value is returned when uploading a new file +* Resolve application crash if a 5xx error is returned when uploading a new file +* Resolve not 'refreshing' version file when rebuilding +* Resolve unexpected application processing by preventing use of --synchronize & --monitor together +* Resolve high CPU usage when performing DB reads +* Update error logging around directory case-insensitive match +* Update Travis CI and ARM dependencies for LDC 1.14.0 +* Update Makefile due to build failure if building from release archive file +* Update logging as to why a OneDrive object was skipped + +### Added +* Implement config option 'skip_dir' + +## 2.2.6 - 2019-03-12 +### Fixed +* Resolve application crash when unable to delete remote folders when business retention policies are enabled +* Resolve deprecation warning: loop index implicitly converted from size_t to int +* Resolve warnings regarding 'bashisms' +* Resolve handling of notification failure is dbus server has not started or available +* Resolve handling of response JSON to ensure that 'id' key element is always checked for +* Resolve excessive & needless logging in monitor mode +* Resolve compiling with LDC on Alpine as musl lacks some standard interfaces +* Resolve notification issues when offline and cannot act on changes +* Resolve Docker entrypoint.sh to accept command line arguments +* Resolve to create a new upload session on reinit +* Resolve where on OneDrive query failure, default root and drive id is used if a response is not returned +* Resolve Key not found: nextExpectedRanges when attempting session uploads and incorrect response is returned +* Resolve application crash when re-using an authentication URI twice after previous --logout +* Resolve creating a folder on a shared personal folder appears successful but returns a JSON error +* Resolve to treat mv of new file as upload of mv target +* Update Debian i386 build dependencies +* Update handling of --get-O365-drive-id to print out all 'site names' that match the explicit search entry rather than just the last match +* Update Docker readme & documentation +* Update handling of validating local file permissions for new file uploads +### Added +* Add support for install & uninstall on RHEL / CentOS 6.x +* Add support for when notifications are enabled, display the number of OneDrive changes to process if any are found +* Add 'config' option 'min_notif_changes' for minimum number of changes to notify on, default = 5 +* Add additional Docker container builds utilising a smaller OS footprint +* Add configurable interval of logging in monitor mode +* Implement new CLI option --skip-dot-files to skip .files and .folders if option is used +* Implement new CLI option --check-for-nosync to ignore folder when special file (.nosync) present +* Implement new CLI option --dry-run + +## 2.2.5 - 2019-01-16 +### Fixed +* Update handling of HTTP 412 - Precondition Failed errors +* Update --display-config to display sync_list if configured +* Add a check for 'id' key on metadata update to prevent 'std.json.JSONException@std/json.d(494): Key not found: id' +* Update handling of 'remote' folder designation as 'root' items +* Ensure that remote deletes are handled correctly +* Handle 'Item not found' exception when unable to query OneDrive 'root' for changes +* Add handling for JSON response error when OneDrive API returns a 404 due to OneDrive API regression +* Fix items highlighted by codacy review +### Added +* Add --force-http-1.1 flag to downgrade any HTTP/2 curl operations to HTTP 1.1 protocol +* Support building with ldc2 and usage of pkg-config for lib finding + +## 2.2.4 - 2018-12-28 +### Fixed +* Resolve JSONException when supplying --get-O365-drive-id option with a string containing spaces +* Resolve 'sync_dir' not read from 'config' file when run in Docker container +* Resolve logic where potentially a 'default' ~/OneDrive sync_dir could be set despite 'config' file configured for an alternate +* Make sure sqlite checkpointing works by properly finalizing statements +* Update logic handling of --single-directory to prevent inadvertent local data loss +* Resolve signal handling and database shutdown on SIGINT and SIGTERM +* Update man page +* Implement better help output formatting +### Added +* Add debug handling for sync_dir operations +* Add debug handling for homePath calculation +* Add debug handling for configDirBase calculation +* Add debug handling if syncDir is created +* Implement Feature Request: Add status command or switch + +## 2.2.3 - 2018-12-20 +### Fixed +* Fix syncdir option is ignored + +## 2.2.2 - 2018-12-20 +### Fixed +* Handle short lived files in monitor mode +* Provide better log messages, less noise on temporary timeouts +* Deal with items that disappear during upload +* Deal with deleted move targets +* Reinitialize sync engine after three failed attempts +* Fix activation of dmd for docker builds +* Fix to check displayName rather than description for --get-O365-drive-id +* Fix checking of config file keys for validity +* Fix exception handling when missing parameter from usage option +### Added +* Notification support via libnotify +* Add very verbose (debug) mode by double -v -v +* Implement option --display-config + +## 2.2.1 - 2018-12-04 +### Fixed +* Gracefully handle connection errors in monitor mode +* Fix renaming of files when syncing +* Installation of doc files, addition of man page +* Adjust timeout values for libcurl +* Continue in monitor mode when sync timed out +* Fix unreachable statements +* Update Makefile to better support packaging +* Allow starting offline in monitor mode +### Added +* Implement --get-O365-drive-id to get correct SharePoint Shared Library (#248) +* Docker buildfiles for onedrive service (#262) + +## 2.2.0 - 2018-11-24 +### Fixed +* Updated client to output additional logging when debugging +* Resolve database assertion failure due to authentication +* Resolve unable to create folders on shared OneDrive Personal accounts +### Added +* Implement feature request to Sync from Microsoft SharePoint +* Implement feature request to specify a logging directory if logging is enabled +### Changed +* Change '--download' to '--download-only' to align with '--upload-only' +* Change logging so that logging to a separate file is no longer the default + +## 2.1.6 - 2018-11-15 +### Fixed +* Updated HTTP/2 transport handling when using curl 7.62.0 for session uploads +### Added +* Added PKGBUILD for makepkg for building packages under Arch Linux + +## 2.1.5 - 2018-11-11 +### Fixed +* Resolve 'Key not found: path' when syncing from some shared folders due to OneDrive API change +* Resolve to only upload changes on remote folder if the item is in the database - dont assert if false +* Resolve files will not download or upload when using curl 7.62.0 due to HTTP/2 being set as default for all curl operations +* Resolve to handle HTTP request returned status code 412 (Precondition Failed) for session uploads to OneDrive Personal Accounts +* Resolve unable to remove '~/.config/onedrive/resume_upload: No such file or directory' if there is a session upload error and the resume file does not get created +* Resolve handling of response codes when using 2 different systems when using '--upload-only' but the same OneDrive account and uploading the same filename to the same location +### Updated +* Updated Travis CI building on LDC v1.11.0 for ARMHF builds +* Updated Makefile to use 'install -D -m 644' rather than 'cp -raf' +* Updated default config to be aligned to code defaults + +## 2.1.4 - 2018-10-10 +### Fixed +* Resolve syncing of OneDrive Personal Shared Folders due to OneDrive API change +* Resolve incorrect systemd installation location(s) in Makefile + +## 2.1.3 - 2018-10-04 +### Fixed +* Resolve File download fails if the file is marked as malware in OneDrive +* Resolve high CPU usage when running in monitor mode +* Resolve how default path is set when running under systemd on headless systems +* Resolve incorrectly nested configDir in X11 systems +* Resolve Key not found: driveType +* Resolve to validate filename length before download to conform with Linux FS limits +* Resolve file handling to look for HTML ASCII codes which will cause uploads to fail +* Resolve Key not found: expirationDateTime on session resume +### Added +* Update Travis CI building to test build on ARM64 + +## 2.1.2 - 2018-08-27 +### Fixed +* Resolve skipping of symlinks in monitor mode +* Resolve Gateway Timeout - JSONValue is not an object +* Resolve systemd/user is not supported on CentOS / RHEL +* Resolve HTTP request returned status code 429 (Too Many Requests) +* Resolve handling of maximum path length calculation +* Resolve 'The parent item is not in the local database' +* Resolve Correctly handle file case sensitivity issues in same folder +* Update unit files documentation link + +## 2.1.1 - 2018-08-14 +### Fixed +* Fix handling no remote delete of remote directories when using --no-remote-delete +* Fix handling of no permission to access a local file / corrupt local file +* Fix application crash when unable to access login.microsoft.com upon application startup +### Added +* Build instructions for openSUSE Leap 15.0 + +## 2.1.0 - 2018-08-10 +### Fixed +* Fix handling of database exit scenarios when there is zero disk space left on drive where the items database resides +* Fix handling of incorrect database permissions +* Fix handling of different database versions to automatically re-create tables if version mis-match +* Fix handling timeout when accessing the Microsoft OneDrive Service +* Fix localFileModifiedTime to not use fraction seconds +### Added +* Implement Feature: Add a progress bar for large uploads & downloads +* Implement Feature: Make checkinterval for monitor configurable +* Implement Feature: Upload Only Option that does not perform remote delete +* Implement Feature: Add ability to skip symlinks +* Add dependency, ebuild and build instructions for Gentoo distributions +### Changed +* Build instructions for x86, x86_64 and ARM32 platforms +* Travis CI files to automate building on x32, x64 and ARM32 architectures +* Travis CI files to test built application against valid, invalid and problem files from previous issues + +## 2.0.2 - 2018-07-18 +### Fixed +* Fix systemd service install for builds with DESTDIR defined +* Fix 'HTTP 412 - Precondition Failed' error handling +* Gracefully handle OneDrive account password change +* Update logic handling of --upload-only and --local-first + +## 2.0.1 - 2018-07-11 +### Fixed +* Resolve computeQuickXorHash generates a different hash when files are > 64Kb + +## 2.0.0 - 2018-07-10 +### Fixed +* Resolve conflict resolution issue during syncing - the client does not handle conflicts very well & keeps on adding the hostname to files +* Resolve skilion #356 by adding additional check for 409 response from OneDrive +* Resolve multiple versions of file shown on website after single upload +* Resolve to gracefully fail when 'onedrive' process cannot get exclusive database lock +* Resolve 'Key not found: fileSystemInfo' when then item is a remote item (OneDrive Personal) +* Resolve skip_file config entry needs to be checked for any characters to escape +* Resolve Microsoft Naming Convention not being followed correctly +* Resolve Error when trying to upload a file with weird non printable characters present +* Resolve Crash if file is locked by online editing (status code 423) +* Resolve Resolve compilation issue with dmd-2.081.0 +* Resolve skip_file configuration doesn't handle spaces or specified directory paths +### Added +* Implement Feature: Add a flag to detect when the sync-folder is missing +* Implement Travis CI for code testing +### Changed +* Update Makefile to use DESTDIR variables +* Update OneDrive Business maximum path length from 256 to 400 +* Update OneDrive Business allowed characters for files and folders +* Update sync_dir handling to use the absolute path for setting parameter to something other than ~/OneDrive via config file or command line +* Update Fedora build instructions + +## 1.1.2 - 2018-05-17 +### Fixed +* Fix 4xx errors including (412 pre-condition, 409 conflict) +* Fix Key not found: lastModifiedDateTime (OneDrive API change) +* Fix configuration directory not found when run via init.d +* Fix skilion Issues #73, #121, #132, #224, #257, #294, #295, #297, #298, #300, #306, #315, #320, #329, #334, #337, #341 +### Added +* Add logging - log client activities to a file (/var/log/onedrive/%username%.onedrive.log or ~/onedrive.log) +* Add https debugging as a flag +* Add `--synchronize` to prevent from syncing when just blindly running the application +* Add individual folder sync +* Add sync from local directory first rather than download first then upload +* Add upload long path check +* Add upload only +* Add check for max upload file size before attempting upload +* Add systemd unit files for single & multi user configuration +* Add init.d file for older init.d based services +* Add Microsoft naming conventions and namespace validation for items that will be uploaded +* Add remaining free space counter at client initialisation to avoid out of space upload issue +* Add large file upload size check to align to OneDrive file size limitations +* Add upload file size validation & retry if does not match +* Add graceful handling of some fatal errors (OneDrive 5xx error handling) + +## Unreleased - 2018-02-19 +### Fixed +* Crash when the delta link is expired +### Changed +* Disabled buffering on stdout + +## 1.1.1 - 2018-01-20 +### Fixed +* Wrong regex for parsing authentication uri + +## 1.1.0 - 2018-01-19 +### Added +* Support for shared folders (OneDrive Personal only) +* `--download` option to only download changes +* `DC` variable in Makefile to chose the compiler +### Changed +* Print logs on stdout instead of stderr +* Improve log messages + +## 1.0.1 - 2017-08-01 +### Added +* `--syncdir` option +### Changed +* `--version` output simplified +* Updated README +### Fixed +* Fix crash caused by remotely deleted and recreated directories + +## 1.0.0 - 2017-07-14 +### Added +* `--version` option diff --git a/configure b/configure index f68a775c..ca4f2f21 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for onedrive v2.4.25. +# Generated by GNU Autoconf 2.69 for onedrive v2.5.0-alpha-2. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='onedrive' PACKAGE_TARNAME='onedrive' -PACKAGE_VERSION='v2.4.25' -PACKAGE_STRING='onedrive v2.4.25' +PACKAGE_VERSION='v2.5.0-alpha-2' +PACKAGE_STRING='onedrive v2.5.0-alpha-2' PACKAGE_BUGREPORT='https://github.com/abraunegg/onedrive' PACKAGE_URL='' @@ -1219,7 +1219,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures onedrive v2.4.25 to adapt to many kinds of systems. +\`configure' configures onedrive v2.5.0-alpha-2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1280,7 +1280,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of onedrive v2.4.25:";; + short | recursive ) echo "Configuration of onedrive v2.5.0-alpha-2:";; esac cat <<\_ACEOF @@ -1393,7 +1393,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -onedrive configure v2.4.25 +onedrive configure v2.5.0-alpha-2 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1410,7 +1410,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by onedrive $as_me v2.4.25, which was +It was created by onedrive $as_me v2.5.0-alpha-2, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2162,7 +2162,7 @@ fi -PACKAGE_DATE="June 2023" +PACKAGE_DATE="October 2023" @@ -3159,7 +3159,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by onedrive $as_me v2.4.25, which was +This file was extended by onedrive $as_me v2.5.0-alpha-2, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -3212,7 +3212,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -onedrive config.status v2.4.25 +onedrive config.status v2.5.0-alpha-2 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 9c2c0db2..65c7abf5 100644 --- a/configure.ac +++ b/configure.ac @@ -9,7 +9,7 @@ dnl - commit the changed files (configure.ac, configure) dnl - tag the release AC_PREREQ([2.69]) -AC_INIT([onedrive],[v2.4.25], [https://github.com/abraunegg/onedrive], [onedrive]) +AC_INIT([onedrive],[v2.5.0-alpha-2], [https://github.com/abraunegg/onedrive], [onedrive]) AC_CONFIG_SRCDIR([src/main.d]) diff --git a/docs/advanced-usage.md b/docs/advanced-usage.md new file mode 100644 index 00000000..2701909d --- /dev/null +++ b/docs/advanced-usage.md @@ -0,0 +1,302 @@ +# Advanced Configuration of the OneDrive Free Client +This document covers the following scenarios: +* [Configuring the client to use multiple OneDrive accounts / configurations](#configuring-the-client-to-use-multiple-onedrive-accounts--configurations) +* [Configuring the client to use multiple OneDrive accounts / configurations using Docker](#configuring-the-client-to-use-multiple-onedrive-accounts--configurations-using-docker) +* [Configuring the client for use in dual-boot (Windows / Linux) situations](#configuring-the-client-for-use-in-dual-boot-windows--linux-situations) +* [Configuring the client for use when 'sync_dir' is a mounted directory](#configuring-the-client-for-use-when-sync_dir-is-a-mounted-directory) +* [Upload data from the local ~/OneDrive folder to a specific location on OneDrive](#upload-data-from-the-local-onedrive-folder-to-a-specific-location-on-onedrive) + +## Configuring the client to use multiple OneDrive accounts / configurations +Essentially, each OneDrive account or SharePoint Shared Library which you require to be synced needs to have its own and unique configuration, local sync directory and service files. To do this, the following steps are needed: +1. Create a unique configuration folder for each onedrive client configuration that you need +2. Copy to this folder a copy of the default configuration file +3. Update the default configuration file as required, changing the required minimum config options and any additional options as needed to support your multi-account configuration +4. Authenticate the client using the new configuration directory +5. Test the configuration using '--display-config' and '--dry-run' +6. Sync the OneDrive account data as required using `--synchronize` or `--monitor` +7. Configure a unique systemd service file for this account configuration + +### 1. Create a unique configuration folder for each onedrive client configuration that you need +Make the configuration folder as required for this new configuration, for example: +```text +mkdir ~/.config/my-new-config +``` + +### 2. Copy to this folder a copy of the default configuration file +Copy to this folder a copy of the default configuration file by downloading this file from GitHub and saving this file in the directory created above: +```text +wget https://raw.githubusercontent.com/abraunegg/onedrive/master/config -O ~/.config/my-new-config/config +``` + +### 3. Update the default configuration file +The following config options *must* be updated to ensure that individual account data is not cross populated with other OneDrive accounts or other configurations: +* sync_dir + +Other options that may require to be updated, depending on the OneDrive account that is being configured: +* drive_id +* application_id +* sync_business_shared_folders +* skip_dir +* skip_file +* Creation of a 'sync_list' file if required +* Creation of a 'business_shared_folders' file if required + +### 4. Authenticate the client +Authenticate the client using the specific configuration file: +```text +onedrive --confdir="~/.config/my-new-config" +``` +You will be asked to open a specific URL by using your web browser where you will have to login into your Microsoft Account and give the application the permission to access your files. After giving permission to the application, you will be redirected to a blank page. Copy the URI of the blank page into the application. +```text +[user@hostname ~]$ onedrive --confdir="~/.config/my-new-config" +Configuration file successfully loaded +Configuring Global Azure AD Endpoints +Authorize this app visiting: + +https://..... + +Enter the response uri: + +``` + +### 5. Display and Test the configuration +Test the configuration using '--display-config' and '--dry-run'. By doing so, this allows you to test any configuration that you have currently made, enabling you to fix this configuration before using the configuration. + +#### Display the configuration +```text +onedrive --confdir="~/.config/my-new-config" --display-config +``` + +#### Test the configuration by performing a dry-run +```text +onedrive --confdir="~/.config/my-new-config" --synchronize --verbose --dry-run +``` + +If both of these operate as per your expectation, the configuration of this client setup is complete and validated. If not, amend your configuration as required. + +### 6. Sync the OneDrive account data as required +Sync the data for the new account configuration as required: +```text +onedrive --confdir="~/.config/my-new-config" --synchronize --verbose +``` +or +```text +onedrive --confdir="~/.config/my-new-config" --monitor --verbose +``` + +* `--synchronize` does a one-time sync +* `--monitor` keeps the application running and monitoring for changes both local and remote + +### 7. Automatic syncing of new OneDrive configuration +In order to automatically start syncing your OneDrive accounts, you will need to create a service file for each account. From the applicable 'systemd folder' where the applicable systemd service file exists: +* RHEL / CentOS: `/usr/lib/systemd/system` +* Others: `/usr/lib/systemd/user` and `/lib/systemd/system` + +### Step1: Create a new systemd service file +#### Red Hat Enterprise Linux, CentOS Linux +Copy the required service file to a new name: +```text +sudo cp /usr/lib/systemd/system/onedrive.service /usr/lib/systemd/system/onedrive-my-new-config +``` +or +```text +sudo cp /usr/lib/systemd/system/onedrive@.service /usr/lib/systemd/system/onedrive-my-new-config@.service +``` + +#### Others such as Arch, Ubuntu, Debian, OpenSuSE, Fedora +Copy the required service file to a new name: +```text +sudo cp /usr/lib/systemd/user/onedrive.service /usr/lib/systemd/user/onedrive-my-new-config.service +``` +or +```text +sudo cp /lib/systemd/system/onedrive@.service /lib/systemd/system/onedrive-my-new-config@.service +``` + +### Step 2: Edit new systemd service file +Edit the new systemd file, updating the line beginning with `ExecStart` so that the confdir mirrors the one you used above: +```text +ExecStart=/usr/local/bin/onedrive --monitor --confdir="/full/path/to/config/dir" +``` + +Example: +```text +ExecStart=/usr/local/bin/onedrive --monitor --confdir="/home/myusername/.config/my-new-config" +``` + +**Note:** When running the client manually, `--confdir="~/.config/......` is acceptable. In a systemd configuration file, the full path must be used. The `~` must be expanded. + +### Step 3: Enable the new systemd service +Once the file is correctly editied, you can enable the new systemd service using the following commands. + +#### Red Hat Enterprise Linux, CentOS Linux +```text +systemctl enable onedrive-my-new-config +systemctl start onedrive-my-new-config +``` + +#### Others such as Arch, Ubuntu, Debian, OpenSuSE, Fedora +```text +systemctl --user enable onedrive-my-new-config +systemctl --user start onedrive-my-new-config +``` +or +```text +systemctl --user enable onedrive-my-new-config@myusername.service +systemctl --user start onedrive-my-new-config@myusername.service +``` + +### Step 4: Viewing systemd status and logs for the custom service +#### Viewing systemd service status - Red Hat Enterprise Linux, CentOS Linux +```text +systemctl status onedrive-my-new-config +``` + +#### Viewing systemd service status - Others such as Arch, Ubuntu, Debian, OpenSuSE, Fedora +```text +systemctl --user status onedrive-my-new-config +``` + +#### Viewing journalctl systemd logs - Red Hat Enterprise Linux, CentOS Linux +```text +journalctl --unit=onedrive-my-new-config -f +``` + +#### Viewing journalctl systemd logs - Others such as Arch, Ubuntu, Debian, OpenSuSE, Fedora +```text +journalctl --user --unit=onedrive-my-new-config -f +``` + +### Step 5: (Optional) Run custom systemd service at boot without user login +In some cases it may be desirable for the systemd service to start without having to login as your 'user' + +All the systemd steps above that utilise the `--user` option, will run the systemd service as your particular user. As such, the systemd service will not start unless you actually login to your system. + +To avoid this issue, you need to reconfigure your 'user' account so that the systemd services you have created will startup without you having to login to your system: +```text +loginctl enable-linger +``` + +Example: +```text +alex@ubuntu-headless:~$ loginctl enable-linger alex +``` + +Repeat these steps for each OneDrive new account that you wish to use. + +## Configuring the client to use multiple OneDrive accounts / configurations using Docker +In some situations it may be desirable to run multiple Docker containers at the same time, each with their own configuration. + +To run the Docker container successfully, it needs two unique Docker volumes to operate: +* Your configuration Docker volumes +* Your data Docker volume + +When running multiple Docker containers, this is no different - each Docker container must have it's own configuration and data volume. + +### High level steps: +1. Create the required unique Docker volumes for the configuration volume +2. Create the required unique local path used for the Docker data volume +3. Start the multiple Docker containers with the required configuration for each container + +#### Create the required unique Docker volumes for the configuration volume +Create the required unique Docker volumes for the configuration volume(s): +```text +docker volume create onedrive_conf_sharepoint_site1 +docker volume create onedrive_conf_sharepoint_site2 +docker volume create onedrive_conf_sharepoint_site3 +... +docker volume create onedrive_conf_sharepoint_site50 +``` + +#### Create the required unique local path used for the Docker data volume +Create the required unique local path used for the Docker data volume +```text +mkdir -p /use/full/local/path/no/tilda/SharePointSite1 +mkdir -p /use/full/local/path/no/tilda/SharePointSite2 +mkdir -p /use/full/local/path/no/tilda/SharePointSite3 +... +mkdir -p /use/full/local/path/no/tilda/SharePointSite50 +``` + +#### Start the Docker container with the required configuration (example) +```text +docker run -it --name onedrive -v onedrive_conf_sharepoint_site1:/onedrive/conf -v "/use/full/local/path/no/tilda/SharePointSite1:/onedrive/data" driveone/onedrive:latest +docker run -it --name onedrive -v onedrive_conf_sharepoint_site2:/onedrive/conf -v "/use/full/local/path/no/tilda/SharePointSite2:/onedrive/data" driveone/onedrive:latest +docker run -it --name onedrive -v onedrive_conf_sharepoint_site3:/onedrive/conf -v "/use/full/local/path/no/tilda/SharePointSite3:/onedrive/data" driveone/onedrive:latest +... +docker run -it --name onedrive -v onedrive_conf_sharepoint_site50:/onedrive/conf -v "/use/full/local/path/no/tilda/SharePointSite50:/onedrive/data" driveone/onedrive:latest +``` + +#### TIP +To avoid 're-authenticating' and 'authorising' each individual Docker container, if all the Docker containers are using the 'same' OneDrive credentials, you can re-use the 'refresh_token' from one Docker container to another by copying this file to the configuration Docker volume of each Docker container. + +If the account credentials are different .. you will need to re-authenticate each Docker container individually. + +## Configuring the client for use in dual-boot (Windows / Linux) situations +When dual booting Windows and Linux, depending on the Windows OneDrive account configuration, the 'Files On-Demand' option may be enabled when running OneDrive within your Windows environment. + +When this option is enabled in Windows, if you are sharing this location between your Windows and Linux systems, all files will be a 0 byte link, and cannot be used under Linux. + +To fix the problem of windows turning all files (that should be kept offline) into links, you have to uncheck a specific option in the onedrive settings window. The option in question is `Save space and download files as you use them`. + +To find this setting, open the onedrive pop-up window from the taskbar, click "Help & Settings" > "Settings". This opens a new window. Go to the tab "Settings" and look for the section "Files On-Demand". + +After unchecking the option and clicking "OK", the Windows OneDrive client should restart itself and start actually downloading your files so they will truely be available on your disk when offline. These files will then be fully accessible under Linux and the Linux OneDrive client. + +| OneDrive Personal | Onedrive Business
SharePoint | +|---|---| +| ![Uncheck-Personal](./images/personal-files-on-demand.png) | ![Uncheck-Business](./images/business-files-on-demand.png) | + +## Configuring the client for use when 'sync_dir' is a mounted directory +In some environments, your setup might be that your configured 'sync_dir' is pointing to another mounted file system - a NFS|CIFS location, an external drive (USB stuc, eSATA etc). As such, you configure your 'sync_dir' as follows: +```text +sync_dir = "/path/to/mountpoint/OneDrive" +``` + +The issue here is - how does the client react if the mount point gets removed - network loss, device removal? + +The client has zero knowledge of any event that causes a mountpoint to become unavailable, thus, the client (if you are running as a service) will assume that you deleted the files, thus, will go ahead and delete all your files on OneDrive. This is most certainly an undesirable action. + +There are a few options here which you can configure in your 'config' file to assist you to prevent this sort of item from occuring: +1. classify_as_big_delete +2. check_nomount +3. check_nosync + +**Note:** Before making any change to your configuration, stop any sync process & stop any onedrive systemd service from running. + +### classify_as_big_delete +By default, this uses a value of 1000 files|folders. An undesirable unmount if you have more than 1000 files, this default level will prevent the client from executing the online delete. Modify this value up or down as desired + +### check_nomount & check_nosync +These two options are really the right safe guards to use. + +In your 'mount point', *before* you mount your external folder|device, create empty `.nosync` file, so that this is the *only* file present in the mount location before you mount your data to your mount point. When you mount your data, this '.nosync' file will not be visible, but, if the device you are mounting goes away - this '.nosync' file is the only file visible. + +Next, in your 'config' file, configure the following options: `check_nomount = "true"` and `check_nosync = "true"` + +What this will do is tell the client, if at *any* point you see this file - stop syncing - thus, protecting your online data from being deleted by the mounted device being suddenly unavailable. + +After making this sort of change - test with `--dry-run` so you can see the impacts of your mount point being unavailable, and how the client is now reacting. Once you are happy with how the system will react, restart your sync processes. + + +## Upload data from the local ~/OneDrive folder to a specific location on OneDrive +In some environments, you may not want your local ~/OneDrive folder to be uploaded directly to the root of your OneDrive account online. + +Unfortunatly, the OneDrive API lacks any facility to perform a re-direction of data during upload. + +The workaround for this is to structure your local filesystem and reconfigure your client to achieve the desired goal. + +### High level steps: +1. Create a new folder, for example `/opt/OneDrive` +2. Configure your application config 'sync_dir' to look at this folder +3. Inside `/opt/OneDrive` create the folder you wish to sync the data online to, for example: `/opt/OneDrive/RemoteOnlineDestination` +4. Configure the application to only sync `/opt/OneDrive/RemoteDestination` via 'sync_list' +5. Symbolically link `~/OneDrive` -> `/opt/OneDrive/RemoteOnlineDestination` + +### Outcome: +* Your `~/OneDrive` will look / feel as per normal +* The data will be stored online under `/RemoteOnlineDestination` + +### Testing: +* Validate your configuration with `onedrive --display-config` +* Test your configuration with `onedrive --dry-run` diff --git a/docs/application-config-options.md b/docs/application-config-options.md new file mode 100644 index 00000000..2f2df44b --- /dev/null +++ b/docs/application-config-options.md @@ -0,0 +1,871 @@ +# Application Configuration Options for the OneDrive Client for Linux +## Application Version +Before reading this document, please ensure you are running application version [![Version](https://img.shields.io/github/v/release/abraunegg/onedrive)](https://github.com/abraunegg/onedrive/releases) or greater. Use `onedrive --version` to determine what application version you are using and upgrade your client if required. + +## Table of Contents +TABLE OF CONTENTS GOES HERE + +## Configuration File Options + +### application_id +_**Description:**_ This is the config option for application id that used used to identify itself to Microsoft OneDrive. In some circumstances, it may be desirable to use your own application id. To do this, you must register a new application with Microsoft Azure via https://portal.azure.com/, then use your new application id with this config option. + +_**Value Type:**_ String + +_**Default Value:**_ d50ca740-c83f-4d1b-b616-12c519384f0c + +_**Config Example:**_ `application_id = "d50ca740-c83f-4d1b-b616-12c519384f0c"` + +### azure_ad_endpoint +_**Description:**_ This is the config option to change the Microsoft Azure Authentication Endpoint that the client uses to conform with data and security requirements that requires data to reside within the geographic borders of that country. + +_**Value Type:**_ String + +_**Default Value:**_ *Empty* - not required for normal operation + +_**Valid Values:**_ USL4, USL5, DE, CN + +_**Config Example:**_ `azure_ad_endpoint = "DE"` + +### azure_tenant_id +_**Description:**_ This config option allows the locking of the client to a specific single tenant and will configure your client to use the specified tenant id in its Azure AD and Graph endpoint URIs, instead of "common". The tenant id may be the GUID Directory ID or the fully qualified tenant name. + +_**Value Type:**_ String + +_**Default Value:**_ *Empty* - not required for normal operation + +_**Config Example:**_ `azure_tenant_id = "example.onmicrosoft.us"` or `azure_tenant_id = "0c4be462-a1ab-499b-99e0-da08ce52a2cc"` + +_**Additional Usage Requirement:**_ Must be configured if 'azure_ad_endpoint' is configured. + +### bypass_data_preservation +_**Description:**_ This config option allows the disabling of preserving local data by renaming the local file in the event of data conflict. If this is enabled, you will experience data loss on your local data as the local file will be over-written with data from OneDrive online. Use with care and caution. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `bypass_data_preservation = "false"` or `bypass_data_preservation = "true"` + +### check_nomount +_**Description:**_ This config option is useful to prevent application startup & ongoing use in 'Monitor Mode' if the configured 'sync_dir' is a separate disk that is being mounted by your system. This option will check for the presence of a `.nosync` file in your mount point, and if present, abort any sync process to preserve data. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `check_nomount = "false"` or `check_nomount = "true"` + +_**CLI Option:**_ `--check-for-nomount` + +_**Additional Usage Requirement:**_ Create a `.nosync` file in your mount point *before* you mount your disk so that this is visible, in your mount point if your disk is unmounted. + +### check_nosync +_**Description:**_ This config option is useful to prevent the sync of a *local* directory to Microsoft OneDrive. It will *not* check for this file online to prevent the download of directories to your local system. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `check_nosync = "false"` or `check_nosync = "true"` + +_**CLI Option Use:**_ `--check-for-nosync` + +_**Additional Usage Requirement:**_ Create a `.nosync` file in any *local* directory that you wish to not sync to Microsoft OneDrive when you enable this option. + +### classify_as_big_delete +_**Description:**_ This config option defines the number of children in a path that is locally removed which will be classified as a 'big data delete' to safeguard large data removals - which are typically accidental local delete events. + +_**Value Type:**_ Integer + +_**Default Value:**_ 1000 + +_**Config Example:**_ `classify_as_big_delete = "2000"` + +_**CLI Option Use:**_ `--classify-as-big-delete 2000` + +_**Additional Usage Requirement:**_ If this option is triggered, you will need to add `--force` to force a sync to occur. + +### cleanup_local_files +_**Description:**_ This config option provides the capability to cleanup local files and folders if they are removed online. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `cleanup_local_files = "false"` or `cleanup_local_files = "true"` + +_**CLI Option Use:**_ `--cleanup-local-files` + +_**Additional Usage Requirement:**_ This configuration option can only be used with 'download_only'. It cannot be used with any other application option. + +### connect_timeout +_**Description:**_ This configuration setting manages the TCP connection timeout duration in seconds for HTTPS connections to Microsoft OneDrive when using the curl library. + +_**Value Type:**_ Integer + +_**Default Value:**_ 30 + +_**Config Example:**_ `connect_timeout = "20"` + +### data_timeout +_**Description:**_ This setting controls the timeout duration, in seconds, for when data is not received on an active connection to Microsoft OneDrive over HTTPS when using the curl library, before that connection is timeout out. + +_**Value Type:**_ Integer + +_**Default Value:**_ 240 + +_**Config Example:**_ `data_timeout = "300"` + +### debug_https +_**Description:**_ This setting controls whether the curl library is configured to output additional data to assist with diagnosing HTTPS issues and problems. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `debug_https = "false"` or `debug_https = "true"` + +_**CLI Option Use:**_ `--debug-https` + +_**Additional Usage Notes:**_ Whilst this option can be used at any time, it is advisable that you only use this option when advised as this will output your `Authorization: bearer` - which is your authentication token to Microsoft OneDrive. + +### disable_download_validation +_**Description:**_ This option determines whether the client will conduct integrity validation on files downloaded from Microsoft OneDrive. Sometimes, when downloading files, particularly from SharePoint, there is a discrepancy between the file size reported by the OneDrive API and the byte count received from the SharePoint HTTP Server for the same file. Enable this option to disable the integrity checks performed by this client. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `disable_download_validation = "false"` or `disable_download_validation = "true"` + +_**CLI Option Use:**_ `--disable-download-validation` + +_**Additional Usage Notes:**_ If you're downloading data from SharePoint or OneDrive Business Shared Folders, you might find it necessary to activate this option. It's important to note that any issues encountered aren't due to a problem with this client; instead, they should be regarded as issues with the Microsoft OneDrive technology stack. + +### disable_notifications +_**Description:**_ This setting controls whether GUI notifications are sent from the client to your display manager session. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `disable_notifications = "false"` or `disable_notifications = "true"` + +_**CLI Option Use:**_ `--disable-notifications` + +### disable_upload_validation +_**Description:**_ This option determines whether the client will conduct integrity validation on files uploaded to Microsoft OneDrive. Sometimes, when uploading files, particularly to SharePoint, SharePoint will modify your file post upload by adding new data to your file which breaks the integrity checking of the upload performed by this client. Enable this option to disable the integrity checks performed by this client. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `disable_upload_validation = "false"` or `disable_upload_validation = "true"` + +_**CLI Option Use:**_ `--disable-upload-validation` + +_**Additional Usage Notes:**_ If you're uploading data to SharePoint or OneDrive Business Shared Folders, you might find it necessary to activate this option. It's important to note that any issues encountered aren't due to a problem with this client; instead, they should be regarded as issues with the Microsoft OneDrive technology stack. + +### display_running_config +_**Description:**_ This option will include the running config of the application at application startup. This may be desirable to enable when running in containerised environments so that any application logging that is occuring, will have the application configuration being consumed at startup, written out to any applicable log file. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `display_running_config = "false"` or `display_running_config = "true"` + +_**CLI Option Use:**_ `--display-running-config` + +### dns_timeout +_**Description:**_ This setting controls the libcurl DNS cache value. By default, libcurl caches this info for 60 seconds. This libcurl DNS cache timeout is entirely speculative that a name resolves to the same address for a small amount of time into the future as libcurl does not use DNS TTL properties. We recommend users not to tamper with this option unless strictly necessary. + +_**Value Type:**_ Integer + +_**Default Value:**_ 60 + +_**Config Example:**_ `dns_timeout = "90"` + +### download_only +_**Description:**_ This setting forces the client to only download data from Microsoft OneDrive and replicate that data locally. No changes made locally will be uploaded to Microsoft OneDrive when using this option. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `download_only = "false"` or `download_only = "true"` + +_**CLI Option Use:**_ `--download-only` + +### drive_id +_**Description:**_ This setting controls the specific drive identifier the client will use when syncing with Microsoft OneDrive. + +_**Value Type:**_ String + +_**Default Value:**_ *None* + +_**Config Example:**_ `drive_id = "b!bO8V6s9SSk9R7mWhpIjUrotN73WlW3tEv3OxP_QfIdQimEdOHR-1So6CqeG1MfDB"` + +_**Additional Usage Notes:**_ This option is typically only used when configuring the client to sync a specific SharePoint Library. If this configuration option is specified in your config file, a value must be specified otherwise the application will exit citing a fatal error has occured. + +### dry_run +_**Description:**_ This setting controls the application capability to test your application configuration without actually performing any actual activity (download, upload, move, delete, folder creation). + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `dry_run = "false"` or `dry_run = "true"` + +_**CLI Option Use:**_ `--dry-run` + +### enable_logging +_**Description:**_ This setting controls the application logging all actions to a separate file. By default, all log files will be written to `/var/log/onedrive`, however this can changed by using the 'log_dir' config option + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `enable_logging = "false"` or `enable_logging = "true"` + +_**CLI Option Use:**_ `--enable-logging` + +_**Additional Usage Notes:**_ Additional configuration is potentially required to configure the default log directory. Refer to usage.md for details (ADD LINK) + +### force_http_11 +_**Description:**_ This setting controls the application HTTP protocol version. By default, the application will use libcurl defaults for which HTTP prodocol version will be used to interact with Microsoft OneDrive. Use this setting to downgrade libcurl to only use HTTP/1.1. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `force_http_11 = "false"` or `force_http_11 = "true"` + +_**CLI Option Use:**_ `--force-http-11` + +### ip_protocol_version +_**Description:**_ This setting controls the application IP protocol that should be used when communicating with Microsoft OneDrive. The default is to use IPv4 and IPv6 networks for communicating to Microsoft OneDrive. + +_**Value Type:**_ Integer + +_**Default Value:**_ 0 + +_**Valid Values:**_ 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only + +_**Config Example:**_ `ip_protocol_version = "0"` or `ip_protocol_version = "1"` or `ip_protocol_version = "2"` + +_**Additional Usage Notes:**_ In some environments where IPv4 and IPv6 are configured at the same time, this causes resolution and routing issues to Microsoft OneDrive. If this is the case, it is advisable to change 'ip_protocol_version' to match your environment. + +### local_first +_**Description:**_ This setting controls what the application considers the 'source of truth' for your data. By default, what is stored online will be considered as the 'source of truth' when syncing to your local machine. When using this option, your local data will be considered the 'source of truth'. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `local_first = "false"` or `local_first = "true"` + +_**CLI Option Use:**_ `--local-first` + +### log_dir +_**Description:**_ This setting controls the custom application log path when 'enable_logging' has been enabled. By default, all log files will be written to `/var/log/onedrive`. + +_**Value Type:**_ String + +_**Default Value:**_ *None* + +_**Config Example:**_ `log_dir = "~/logs/"` + +_**CLI Option Use:**_ `--log-dir "~/logs/"` + +### monitor_fullscan_frequency +_**Description:**_ This configuration option controls the number of 'monitor_interval' iterations between when a full scan of your data is performed to ensure data integrity and consistency. + +_**Value Type:**_ Integer + +_**Default Value:**_ 12 + +_**Config Example:**_ `monitor_fullscan_frequency = "24"` + +_**CLI Option Use:**_ `--monitor-fullscan-frequency '24'` + +_**Additional Usage Notes:**_ By default without configuration, 'monitor_fullscan_frequency' is set to 12. In this default state, this means that a full scan is performed every 'monitor_interval' x 'monitor_fullscan_frequency' = 3600 seconds. This setting is only applicable when running in `--monitor` mode. + +### monitor_interval +_**Description:**_ This configuration setting determines how often the synchronisation loops run in --monitor mode, measured in seconds. When this time period elapses, the client will check for online changes in Microsoft OneDrive, conduct integrity checks on local data and scan the local 'sync_dir' to identify any new content that hasn't been uploaded yet. + +_**Value Type:**_ Integer + +_**Default Value:**_ 300 + +_**Config Example:**_ `monitor_interval = "600"` + +_**CLI Option Use:**_ `--monitor-interval '600'` + +_**Additional Usage Notes:**_ A minimum value of 300 is enforced for this configuration setting. + +### monitor_log_frequency +_**Description:**_ This configuration option controls the suppression of frequently printed log items to the system console when using `--monitor` mode. The aim of this configuration item is to reduce the log output when near zero sync activity is occuring. + +_**Value Type:**_ Integer + +_**Default Value:**_ 12 + +_**Config Example:**_ `monitor_log_frequency = "24"` + +_**CLI Option Use:**_ `--monitor-log-frequency '24'` + +_**Additional Usage Notes:**_ + +By default, at application start-up when using `--monitor` mode, the following will be logged to indicate that the application has correctly started and has performed all the initial processing steps: +```text +Reading configuration file: /home/user/.config/onedrive/config +Configuration file successfully loaded +Configuring Global Azure AD Endpoints +Sync Engine Initialised with new Onedrive API instance +All application operations will be performed in: /home/user/OneDrive +OneDrive synchronisation interval (seconds): 300 +Initialising filesystem inotify monitoring ... +Performing initial syncronisation to ensure consistent local state ... +Starting a sync with Microsoft OneDrive +Fetching items from the OneDrive API for Drive ID: b!bO8V6s9SSk9R7mWhpIjUrotN73WlW3tEv3OxP_QfIdQimEdOHR-1So6CqeG1MfDB .. +Processing changes and items received from Microsoft OneDrive ... +Performing a database consistency and integrity check on locally stored data ... +Scanning the local file system '~/OneDrive' for new data to upload ... +Performing a final true-up scan of online data from Microsoft OneDrive +Fetching items from the OneDrive API for Drive ID: b!bO8V6s9SSk9R7mWhpIjUrotN73WlW3tEv3OxP_QfIdQimEdOHR-1So6CqeG1MfDB .. +Processing changes and items received from Microsoft OneDrive ... +Sync with Microsoft OneDrive is complete +``` +Then, based on 'monitor_log_frequency', the following output will be logged until the suppression loop value is reached: +```text +Starting a sync with Microsoft OneDrive +Syncing changes from Microsoft OneDrive ... +Sync with Microsoft OneDrive is complete +``` +**Note:** The additional log output `Performing a database consistency and integrity check on locally stored data ...` will only be displayed when this activity is occuring which is triggered by 'monitor_fullscan_frequency'. + +**Note:** If verbose application output is being used (`--verbose`), then this configuration setting has zero effect, as application verbose output takes priority over application output surpression. + +### no_remote_delete +_**Description:**_ This configuration option controls whether local file and folder deletes are actioned on Microsoft OneDrive. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `local_first = "false"` or `local_first = "true"` + +_**CLI Option Use:**_ `--no-remote-delete` + +_**Additional Usage Notes:**_ This configuration option can *only* be used in conjunction with `--upload-only` + +### operation_timeout +_**Description:**_ This configuration option controls the maximum amount of time (seconds) a file operation is allowed to take. This includes DNS resolution, connecting, data transfer, etc. We recommend users not to tamper with this option unless strictly necessary. + +_**Value Type:**_ Integer + +_**Default Value:**_ 3600 + +_**Config Example:**_ `operation_timeout = "3600"` + +### rate_limit +_**Description:**_ This configuration option controls the bandwidth used by the application, per thread, when interacting with Microsoft OneDrive. + +_**Value Type:**_ Integer + +_**Default Value:**_ 0 (unlimited, use available bandwidth per thread) + +_**Valid Values:**_ Valid tested values for this configuration option are as follows: + +* 131072 = 128 KB/s - absolute minimum for basic application operations to prevent timeouts +* 262144 = 256 KB/s +* 524288 = 512 KB/s +* 1048576 = 1 MB/s +* 10485760 = 10 MB/s +* 104857600 = 100 MB/s + +_**Config Example:**_ `rate_limit = "131072"` + +### read_only_auth_scope +_**Description:**_ This configuration option controls whether the OneDrive Client for Linux operates in a totally in read-only operation. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `read_only_auth_scope = "false"` or `read_only_auth_scope = "true"` + +_**Additional Usage Notes:**_ When using 'read_only_auth_scope' you also will need to remove your existing application access consent otherwise old authentication consent will be valid and will be used. This will mean the application will technically have the consent to upload data until you revoke this consent. + +### remove_source_files +_**Description:**_ This configuration option controls whether the OneDrive Client for Linux removes the local file post successful transfer to Microsoft OneDrive. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `remove_source_files = "false"` or `remove_source_files = "true"` + +_**CLI Option Use:**_ `--remove-source-files` + +_**Additional Usage Notes:**_ This configuration option can *only* be used in conjunction with `--upload-only` + +### resync +_**Description:**_ This configuration option controls whether the known local sync state with Microsoft OneDrive is removed at application startup. When this option is used, a full scan of your data online is performed to ensure that the local sync state is correctly built back up. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `resync = "false"` or `resync = "true"` + +_**CLI Option Use:**_ `--resync` + +_**Additional Usage Notes:**_ It's highly recommended to use this option only if the application prompts you to do so. Don't blindly use this option as a default option. If you alter any of the subsequent configuration items, you will be required to execute a `--resync` to make sure your client is syncing your data with the updated configuration: +* drive_id +* sync_dir +* skip_file +* skip_dir +* skip_dotfiles +* skip_symlinks +* sync_business_shared_items +* Creating, Modifying or Deleting the 'sync_list' file + +### resync_auth +_**Description:**_ This configuration option controls the approval of performing a 'resync' which can be beneficial in automated environments. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `resync_auth = "false"` or `resync_auth = "true"` + +_**CLI Option Use:**_ `--resync-auth` + +_**Additional Usage Notes:**_ In certain automated environments (assuming you know what you're doing due to automation), to avoid the 'proceed with acknowledgement' resync requirement, this option allows you to automatically acknowledge the resync prompt. + +### skip_dir +_**Description:**_ This configuration option controls whether the application skips certain directories from being synced. + +_**Value Type:**_ String + +_**Default Value:**_ *Empty* - not required for normal operation + +_**Config Example:**_ + +Patterns are case insensitive. `*` and `?` [wildcards characters](https://technet.microsoft.com/en-us/library/bb490639.aspx) are supported. Use `|` to separate multiple patterns. Entries for 'skip_dir' are *relative* to your 'sync_dir' path. +```text +# When changing a config option below, remove the '#' from the start of the line +# For explanations of all config options below see docs/USAGE.md or the man page. +# +# sync_dir = "~/OneDrive" +# skip_file = "~*|.~*|*.tmp" +# monitor_interval = "300" +skip_dir = "Desktop|Documents/IISExpress|Documents/SQL Server Management Studio|Documents/Visual Studio*|Documents/WindowsPowerShell" +# log_dir = "/var/log/onedrive/" +``` + +The 'skip_dir' option can be specified multiple times within your config file, for example: +```text +skip_dir = "SomeDir|OtherDir|ThisDir|ThatDir" +skip_dir = "/Path/To/A/Directory" +skip_dir = "/Another/Path/To/Different/Directory" +``` + +This will be interpreted the same as: +```text +skip_dir = "SomeDir|OtherDir|ThisDir|ThatDir|/Path/To/A/Directory|/Another/Path/To/Different/Directory" +``` + +_**CLI Option Use:**_ `--skip-dir 'SomeDir|OtherDir|ThisDir|ThatDir|/Path/To/A/Directory|/Another/Path/To/Different/Directory'` + +_**Additional Usage Notes:**_ This option is considered a 'Client Side Filtering Rule' and if configured, is utilised for all sync operations. If using the config file and CLI option is used, the CLI option will *replace* the config file entries. After changing or modifying this option, you will be required to perform a resync. + +### skip_dir_strict_match +_**Description:**_ This configuration option controls whether the application performs strict directory matching when checking 'skip_dir' items. When enabled, the 'skip_dir' item must be a full path match to the path to be skipped. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `skip_dir_strict_match = "false"` or `skip_dir_strict_match = "true"` + +_**CLI Option Use:**_ `--skip-dir-strict-match` + +### skip_dotfiles +_**Description:**_ This configuration option controls whether the application will skip all .files and .folders when performing sync operations. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `skip_dotfiles = "false"` or `skip_dotfiles = "true"` + +_**CLI Option Use:**_ `--skip-dot-files` + +_**Additional Usage Notes:**_ This option is considered a 'Client Side Filtering Rule' and if configured, is utilised for all sync operations. After changing this option, you will be required to perform a resync. + +### skip_file +_**Description:**_ This configuration option controls whether the application skips certain files from being synced. + +_**Value Type:**_ String + +_**Default Value:**_ `~*|.~*|*.tmp|*.swp|*.partial` + +_**Config Example:**_ + +Patterns are case insensitive. `*` and `?` [wildcards characters](https://technet.microsoft.com/en-us/library/bb490639.aspx) are supported. Use `|` to separate multiple patterns. + +By default, the following files will be skipped: +* Files that start with ~ +* Files that start with .~ (like .~lock.* files generated by LibreOffice) +* Files that end in .tmp, .swp and .partial + +Files can be skipped in the following fashion: +* Specify a wildcard, eg: '*.txt' (skip all txt files) +* Explicitly specify the filename and it's full path relative to your sync_dir, eg: '/path/to/file/filename.ext' +* Explicitly specify the filename only and skip every instance of this filename, eg: 'filename.ext' + +```text +# When changing a config option below, remove the '#' from the start of the line +# For explanations of all config options below see docs/USAGE.md or the man page. +# +# sync_dir = "~/OneDrive" +skip_file = "~*|/Documents/OneNote*|/Documents/config.xlaunch|myfile.ext|/Documents/keepass.kdbx" +# monitor_interval = "300" +# skip_dir = "" +# log_dir = "/var/log/onedrive/" +``` +The 'skip_file' option can be specified multiple times within your config file, for example: +```text +skip_file = "~*|.~*|*.tmp|*.swp" +skip_file = "*.blah" +skip_file = "never_sync.file" +skip_file = "/Documents/keepass.kdbx" +``` +This will be interpreted the same as: +```text +skip_file = "~*|.~*|*.tmp|*.swp|*.blah|never_sync.file|/Documents/keepass.kdbx" +``` + +_**CLI Option Use:**_ `--skip-file '~*|.~*|*.tmp|*.swp|*.blah|never_sync.file|/Documents/keepass.kdbx'` + +_**Additional Usage Notes:**_ This option is considered a 'Client Side Filtering Rule' and if configured, is utilised for all sync operations. If using the config file and CLI option is used, the CLI option will *replace* the config file entries. After changing or modifying this option, you will be required to perform a resync. + +### skip_size +_**Description:**_ This configuration option controls whether the application skips syncing certain files larger than the specified size. The value specified is in MB. + +_**Value Type:**_ Integer + +_**Default Value:**_ 0 (all files, regardless of size, are synced) + +_**Config Example:**_ `skip_size = "50"` + +_**CLI Option Use:**_ `--skip-size '50'` + +### skip_symlinks +_**Description:**_ This configuration option controls whether the application will skip all symbolic links when performing sync operations. Microsoft OneDrive has no concept or understanding of symbolic links, and attempting to upload a symbolic link to Microsoft OneDrive generates a platform API error. All data (files and folders) that are uploaded to OneDrive must be whole files or actual directories. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `skip_symlinks = "false"` or `skip_symlinks = "true"` + +_**CLI Option Use:**_ `--skip-symlinks` + +_**Additional Usage Notes:**_ This option is considered a 'Client Side Filtering Rule' and if configured, is utilised for all sync operations. After changing this option, you will be required to perform a resync. + +### space_reservation +_**Description:**_ This configuration option controls how much local disk space should be reserved, to prevent the application from filling up your entire disk due to misconfiguration + +_**Value Type:**_ Integer + +_**Default Value:**_ 50 MB (expressesed as Bytes when using `--display-config`) + +_**Config Example:**_ `space_reservation = "100"` + +_**CLI Option Use:**_ `--space-reservation '100'` + +### sync_business_shared_items +_**Description:**_ This configuration option controls whether OneDrive Business | Office 365 Shared Folders, when added as a 'shortcut' to your 'My Files' will be synced to your local system. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `sync_business_shared_items = "false"` or `sync_business_shared_items = "true"` + +_**CLI Option Use:**_ *none* - this is a config file option only + +_**Additional Usage Notes:**_ This option is considered a 'Client Side Filtering Rule' and if configured, is utilised for all sync operations. After changing this option, you will be required to perform a resync. + +### sync_dir +_**Description:**_ This configuration option determines the location on your local filesystem where your data from Microsoft OneDrive will be saved. + +_**Value Type:**_ String + +_**Default Value:**_ `~/OneDrive` + +_**Config Example:**_ `sync_dir = "~/MyDirToSync"` + +_**CLI Option Use:**_ `--syncdir '~/MyDirToSync'` + +_**Additional Usage Notes:**_ After changing this option, you will be required to perform a resync. + +### sync_dir_permissions +_**Description:**_ This configuration option defines the directory permissions applied when a new directory is created locally during the process of syncing your data from Microsoft OneDrive. + +_**Value Type:**_ Integer + +_**Default Value:**_ `700` - This provides the following permissions: `drwx------` + +_**Config Example:**_ `sync_dir_permissions = "700"` + +_**Additional Usage Notes:**_ Use the [Unix Permissions Calculator](https://chmod-calculator.com/) to help you determine the necessary new permissions. You will need to manually update all existing directory permissions if you modify this value. + +### sync_file_permissions +_**Description:**_ This configuration option defines the file permissions applied when a new file is created locally during the process of syncing your data from Microsoft OneDrive. + +_**Value Type:**_ Integer + +_**Default Value:**_ `600` - This provides the following permissions: `-rw-------` + +_**Config Example:**_ `sync_file_permissions = "600"` + +_**Additional Usage Notes:**_ Use the [Unix Permissions Calculator](https://chmod-calculator.com/) to help you determine the necessary new permissions. You will need to manually update all existing directory permissions if you modify this value. + +### sync_root_files +_**Description:**_ This configuration option manages the synchronisation of files located in the 'sync_dir' root when using a 'sync_list.' It enables you to sync all these files by default, eliminating the need to repeatedly modify your 'sync_list' and initiate resynchronisation. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `sync_root_files = "false"` or `sync_root_files = "true"` + +_**CLI Option Use:**_ `--sync-root-files` + +_**Additional Usage Notes:**_ Although it's not mandatory, it's recommended that after enabling this option, you perform a `--resync`. This ensures that any previously excluded content is now included in your sync process. + +### upload_only +_**Description:**_ This setting forces the client to only upload data to Microsoft OneDrive and replicate the locate state online. By default, this will also remove content online, that has been removed locally. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `upload_only = "false"` or `upload_only = "true"` + +_**CLI Option Use:**_ `--upload-only` + +_**Additional Usage Notes:**_ To ensure that data deleted locally remains accessible online, you can use the 'no_remote_delete' option. If you want to delete the data from your local storage after a successful upload to Microsoft OneDrive, you can use the 'remove_source_files' option. + +### user_agent +_**Description:**_ This configuration option controls the 'User-Agent' request header that is presented to Microsoft Graph API when accessing the Microsoft OneDrive service. This string lets servers and network peers identify the application, operating system, vendor, and/or version of the application making the request. We recommend users not to tamper with this option unless strictly necessary. + +_**Value Type:**_ String + +_**Default Value:**_ `ISV|abraunegg|OneDrive Client for Linux/vX.Y.Z-A-bcdefghi` + +_**Config Example:**_ `user_agent = "ISV|CompanyName|AppName/Version"` + +_**Additional Usage Notes:**_ The current value conforms the the Microsoft Graph API documentation for presenting an appropriate 'User-Agent' header and aligns to the registered 'application_id' that this application uses. + +### webhook_enabled +_**Description:**_ + +_**Value Type:**_ + +_**Default Value:**_ + +_**Config Example:**_ + +### webhook_expiration_interval +_**Description:**_ + +_**Value Type:**_ + +_**Default Value:**_ + +_**Config Example:**_ + +### webhook_listening_host +_**Description:**_ + +_**Value Type:**_ + +_**Default Value:**_ + +_**Config Example:**_ + +### webhook_listening_port +_**Description:**_ + +_**Value Type:**_ + +_**Default Value:**_ + +_**Config Example:**_ + +### webhook_public_url +_**Description:**_ + +_**Value Type:**_ + +_**Default Value:**_ + +_**Config Example:**_ + +### webhook_renewal_interval +_**Description:**_ + +_**Value Type:**_ + +_**Default Value:**_ + +_**Config Example:**_ + + + +## Command Line Interface (CLI) Only Options + +### CLI Option: --auth-files +_**Description:**_ + +_**Usage Example:**_ + +### CLI Option: --auth-response +_**Description:**_ + +_**Usage Example:**_ + +### CLI Option: --confdir +_**Description:**_ This CLI option allows the user to specify where all the application configuration and relevant components are stored. + +_**Usage Example:**_ `onedrive --confdir '~/.config/onedrive-business/'` + +_**Additional Usage Notes:**_ If using this option, it must be specified each and every time the application is used. If this is ommited, the application default configuration directory will be used. + +### CLI Option: --create-directory +_**Description:**_ + +_**Usage Example:**_ + +### CLI Option: --create-share-link +_**Description:**_ + +_**Usage Example:**_ + +### CLI Option: --destination-directory +_**Description:**_ + +_**Usage Example:**_ + +### CLI Option: --display-config +_**Description:**_ This CLI option will display the effective application configuration + +_**Usage Example:**_ `onedrive --display-config` + +### CLI Option: --display-sync-status +_**Description:**_ + +_**Usage Example:**_ + +### CLI Option: --force +_**Description:**_ This CLI option enables the force the deletion of data when a 'big delete' is detected + +_**Usage Example:**_ + +### CLI Option: --force-sync +_**Description:**_ + +_**Usage Example:**_ + +### CLI Option: --get-file-link +_**Description:**_ + +_**Usage Example:**_ + +### CLI Option: --get-sharepoint-drive-id +_**Description:**_ This CLI option queries the OneDrive API and return's the Office 365 Drive ID for a given Office 365 SharePoint Shared Library that can then be used with 'drive_id' to sync a specific SharePoint Library. + +_**Usage Example:**_ `onedrive --get-sharepoint-drive-id '*'` or `onedrive --get-sharepoint-drive-id 'PointPublishing Hub Site'` + +### CLI Option: --logout +_**Description:**_ This CLI option removes this clients authentictaion status with Microsoft OneDrive. Any further application use will requrie the application to be re-authenticated with Microsoft OneDrive. + +_**Usage Example:**_ `onedrive --logout` + +### CLI Option: --modified-by +_**Description:**_ + +_**Usage Example:**_ + +### CLI Option: --monitor | -m +_**Description:**_ This CLI option controls the 'Monitor Mode' operational aspect of the client. When this option is used, the client will perform on-going syncs of data between Microsoft OneDrive and your local system. Local changes will be uploaded in near-realtime, whilst online changes will be downloaded on the next sync process. The frequency of these checks is governed by the 'monitor_interval' value. + +_**Usage Example:**_ `onedrive --monitor` or `onedrive -m` + +### CLI Option: --print-access-token +_**Description:**_ Print the current access token being used to access Microsoft OneDrive. + +_**Usage Example:**_ `onedrive --verbose --verbose --debug-https --print-access-token` + +_**Additional Usage Notes:**_ Do not use this option if you do not know why you are wanting to use it. Be highly cautious of exposing this object. Change your password if you feel that you have inadvertantly exposed this token. + +### CLI Option: --reauth +_**Description:**_ This CLI option controls the ability to re-authenticate your client with Microsoft OneDrive. + +_**Usage Example:**_ `onedrive --reauth` + +### CLI Option: --remove-directory +_**Description:**_ + +_**Usage Example:**_ + +### CLI Option: --single-directory +_**Description:**_ + +_**Usage Example:**_ + +### CLI Option: --source-directory +_**Description:**_ + +_**Usage Example:**_ + +### CLI Option: --sync | -s +_**Description:**_ This CLI option controls the 'Standalone Mode' operational aspect of the client. When this option is used, the client will perform a one-time sync of data between Microsoft OneDrive and your local system. + +_**Usage Example:**_ `onedrive --sync` or `onedrive -s` + +### CLI Option: --verbose | -v+ +_**Description:**_ + +_**Usage Example:**_ + +### CLI Option: --with-editing-perms +_**Description:**_ + +_**Usage Example:**_ + +## Depreciated Configuration File and CLI Options +The following configuration options are no longer supported + +### min_notify_changes +_**Description:**_ Minimum number of pending incoming changes necessary to trigger a GUI desktop notification. + +_**Depreciated Config Example:**_ `min_notify_changes = "50"` + +_**Depreciated CLI Option:**_ `--min-notify-changes '50'` + +_**Reason for depreciation:**_ Application has been totally re-written. When this item was introduced, it was done so to reduce spamming of all events to the GUI desktop. + +### CLI Option: --synchronize +_**Description:**_ Perform a synchronisation with Microsoft OneDrive + +_**Depreciated CLI Option:**_ `--synchronize` + +_**Reason for depreciation:**_ `--synchronize` has been depreciated in favour of `--sync` or `-s` diff --git a/docs/application-security.md b/docs/application-security.md new file mode 100644 index 00000000..7c22c4f1 --- /dev/null +++ b/docs/application-security.md @@ -0,0 +1,97 @@ +# OneDrive Client for Linux Application Security +This document details the following information: + +* Why is this application an 'unverified publisher'? +* Application Security and Permission Scopes +* How to change Permission Scopes +* How to review your existing application access consent + +## Why is this application an 'unverified publisher'? +Publisher Verification, as per the Microsoft [process](https://learn.microsoft.com/en-us/azure/active-directory/develop/publisher-verification-overview) has actually been configured, and, actually has been verified! + +### Verified Publisher Configuration Evidence +As per the image below, the Azure portal shows that the 'Publisher Domain' has actually been verified: +![confirmed_verified_publisher](./images/confirmed_verified_publisher.jpg) + +* The 'Publisher Domain' is: https://abraunegg.github.io/ +* The required 'Microsoft Identity Association' is: https://abraunegg.github.io/.well-known/microsoft-identity-association.json + +## Application Security and Permission Scopes +There are 2 main components regarding security for this application: +* Azure Application Permissions +* User Authentication Permissions + +Keeping this in mind, security options should follow the security principal of 'least privilege': +> The principle that a security architecture should be designed so that each entity +> is granted the minimum system resources and authorizations that the entity needs +> to perform its function. + +Reference: [https://csrc.nist.gov/glossary/term/least_privilege](https://csrc.nist.gov/glossary/term/least_privilege) + +As such, the following API permissions are used by default: + +### Default Azure Application Permissions + +| API / Permissions name | Type | Description | Admin consent required | +|---|---|---|---| +| Files.Read | Delegated | Have read-only access to user files | No | +| Files.Read.All | Delegated | Have read-only access to all files user can access | No | +| Sites.Read.All | Delegated | Have read-only access to all items in all site collections | No | +| offline_access | Delegated | Maintain access to data you have given it access to | No | + +![default_authentication_scopes](./images/default_authentication_scopes.jpg) + +### Default User Authentication Permissions + +When a user authenticates with Microsoft OneDrive, additional account permissions are provided by service to give the user specific access to their data. These are delegated permissions provided by the platform: + +| API / Permissions name | Type | Description | Admin consent required | +|---|---|---|---| +| Files.ReadWrite | Delegated | Have full access to user files | No | +| Files.ReadWrite.All | Delegated | Have full access to all files user can access | No | +| Sites.ReadWrite.All | Delegated | Have full access to all items in all site collections | No | +| offline_access | Delegated | Maintain access to data you have given it access to | No | + +When these delegated API permissions are combined, these provide the effective authentication scope for the OneDrive Client for Linux to access your data. The resulting effective 'default' permissions will be: + +| API / Permissions name | Type | Description | Admin consent required | +|---|---|---|---| +| Files.ReadWrite | Delegated | Have full access to user files | No | +| Files.ReadWrite.All | Delegated | Have full access to all files user can access | No | +| Sites.ReadWrite.All | Delegated | Have full access to all items in all site collections | No | +| offline_access | Delegated | Maintain access to data you have given it access to | No | + +These 'default' permissions will allow the OneDrive Client for Linux to read, write and delete data associated with your OneDrive Account. + +## Configuring read-only access to your OneDrive data +In some situations, it may be desirable to configure the OneDrive Client for Linux totally in read-only operation. + +To change the application to 'read-only' access, add the following to your configuration file: +```text +read_only_auth_scope = "true" +``` +This will change the user authentication scope request to use read-only access. + +**Note:** When changing this value, you *must* re-authenticate the client using the `--reauth` option to utilise the change in authentication scopes. + +When using read-only authentication scopes, the uploading of any data or local change to OneDrive will fail with the following error: +``` +2022-Aug-06 13:16:45.3349625 ERROR: Microsoft OneDrive API returned an error with the following message: +2022-Aug-06 13:16:45.3351661 Error Message: HTTP request returned status code 403 (Forbidden) +2022-Aug-06 13:16:45.3352467 Error Reason: Access denied +2022-Aug-06 13:16:45.3352838 Error Timestamp: 2022-06-12T13:16:45 +2022-Aug-06 13:16:45.3353171 API Request ID: +``` + +As such, it is also advisable for you to add the following to your configuration file so that 'uploads' are prevented: +```text +download_only = "true" +``` + +**Important:** Additionally when using 'read_only_auth_scope' you also will need to remove your existing application access consent otherwise old authentication consent will be valid and will be used. This will mean the application will technically have the consent to upload data. See below on how to remove your prior application consent. + +## Reviewing your existing application access consent + +To review your existing application access consent, you need to access the following URL: https://account.live.com/consent/Manage + +From here, you are able to review what applications have been given what access to your data, and remove application access as required. diff --git a/docs/build-rpm-howto.md b/docs/build-rpm-howto.md new file mode 100644 index 00000000..5439c366 --- /dev/null +++ b/docs/build-rpm-howto.md @@ -0,0 +1,379 @@ +# RPM Package Build Process +The instuctions below have been tested on the following systems: +* CentOS 7 x86_64 +* CentOS 8 x86_64 + +These instructions should also be applicable for RedHat & Fedora platforms, or any other RedHat RPM based distribution. + +## Prepare Package Development Environment (CentOS 7, 8) +Install the following dependencies on your build system: +```text +sudo yum groupinstall -y 'Development Tools' +sudo yum install -y libcurl-devel +sudo yum install -y sqlite-devel +sudo yum install -y libnotify-devel +sudo yum install -y wget +sudo yum install -y http://downloads.dlang.org/releases/2.x/2.088.0/dmd-2.088.0-0.fedora.x86_64.rpm +mkdir -p ~/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS} +``` + +## Build RPM from spec file +Build the RPM from the provided spec file: +```text +wget https://github.com/abraunegg/onedrive/archive/refs/tags/v2.4.22.tar.gz -O ~/rpmbuild/SOURCES/v2.4.22.tar.gz +wget https://raw.githubusercontent.com/abraunegg/onedrive/master/contrib/spec/onedrive.spec.in -O ~/rpmbuild/SPECS/onedrive.spec +rpmbuild -ba ~/rpmbuild/SPECS/onedrive.spec +``` + +## RPM Build Example Results +Below are example output results of building, installing and running the RPM package on the respective platforms: + +### CentOS 7 +```text +[alex@localhost ~]$ rpmbuild -ba ~/rpmbuild/SPECS/onedrive.spec +Executing(%prep): /bin/sh -e /var/tmp/rpm-tmp.wi6Tdz ++ umask 022 ++ cd /home/alex/rpmbuild/BUILD ++ cd /home/alex/rpmbuild/BUILD ++ rm -rf onedrive-2.4.15 ++ /usr/bin/tar -xf - ++ /usr/bin/gzip -dc /home/alex/rpmbuild/SOURCES/v2.4.15.tar.gz ++ STATUS=0 ++ '[' 0 -ne 0 ']' ++ cd onedrive-2.4.15 ++ /usr/bin/chmod -Rf a+rX,u+w,g-w,o-w . ++ exit 0 +Executing(%build): /bin/sh -e /var/tmp/rpm-tmp.dyeEuM ++ umask 022 ++ cd /home/alex/rpmbuild/BUILD ++ cd onedrive-2.4.15 ++ CFLAGS='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic' ++ export CFLAGS ++ CXXFLAGS='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic' ++ export CXXFLAGS ++ FFLAGS='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -I/usr/lib64/gfortran/modules' ++ export FFLAGS ++ FCFLAGS='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -I/usr/lib64/gfortran/modules' ++ export FCFLAGS ++ LDFLAGS='-Wl,-z,relro ' ++ export LDFLAGS ++ '[' 1 == 1 ']' ++ '[' x86_64 == ppc64le ']' +++ find . -name config.guess -o -name config.sub ++ ./configure --build=x86_64-redhat-linux-gnu --host=x86_64-redhat-linux-gnu --program-prefix= --disable-dependency-tracking --prefix=/usr --exec-prefix=/usr --bindir=/usr/bin --sbindir=/usr/sbin --sysconfdir=/etc --datadir=/usr/share --includedir=/usr/include --libdir=/usr/lib64 --libexecdir=/usr/libexec --localstatedir=/var --sharedstatedir=/var/lib --mandir=/usr/share/man --infodir=/usr/share/info +configure: WARNING: unrecognized options: --disable-dependency-tracking +checking for a BSD-compatible install... /usr/bin/install -c +checking for x86_64-redhat-linux-gnu-pkg-config... no +checking for pkg-config... /usr/bin/pkg-config +checking pkg-config is at least version 0.9.0... yes +checking for dmd... dmd +checking version of D compiler... 2.087.0 +checking for curl... yes +checking for sqlite... yes +configure: creating ./config.status +config.status: creating Makefile +config.status: creating contrib/pacman/PKGBUILD +config.status: creating contrib/spec/onedrive.spec +config.status: creating onedrive.1 +config.status: creating contrib/systemd/onedrive.service +config.status: creating contrib/systemd/onedrive@.service +configure: WARNING: unrecognized options: --disable-dependency-tracking ++ make +if [ -f .git/HEAD ] ; then \ + git describe --tags > version ; \ +else \ + echo v2.4.15 > version ; \ +fi +dmd -w -g -O -J. -L-lcurl -L-lsqlite3 -L-ldl src/config.d src/itemdb.d src/log.d src/main.d src/monitor.d src/onedrive.d src/qxor.d src/selective.d src/sqlite.d src/sync.d src/upload.d src/util.d src/progress.d src/arsd/cgi.d -ofonedrive ++ exit 0 +Executing(%install): /bin/sh -e /var/tmp/rpm-tmp.L3JbHy ++ umask 022 ++ cd /home/alex/rpmbuild/BUILD ++ '[' /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64 '!=' / ']' ++ rm -rf /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64 +++ dirname /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64 ++ mkdir -p /home/alex/rpmbuild/BUILDROOT ++ mkdir /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64 ++ cd onedrive-2.4.15 ++ /usr/bin/make install DESTDIR=/home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64 PREFIX=/home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64 +/usr/bin/install -c -D onedrive /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/usr/bin/onedrive +/usr/bin/install -c -D onedrive.1 /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/usr/share/man/man1/onedrive.1 +/usr/bin/install -c -D -m 644 contrib/logrotate/onedrive.logrotate /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/etc/logrotate.d/onedrive +mkdir -p /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/usr/share/doc/onedrive +/usr/bin/install -c -D -m 644 README.md config LICENSE CHANGELOG.md docs/Docker.md docs/INSTALL.md docs/SharePoint-Shared-Libraries.md docs/USAGE.md docs/BusinessSharedFolders.md docs/advanced-usage.md /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/usr/share/doc/onedrive +/usr/bin/install -c -d -m 0755 /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/usr/lib/systemd/user /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/usr/lib/systemd/system +/usr/bin/install -c -m 0644 contrib/systemd/onedrive@.service /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/usr/lib/systemd/system +/usr/bin/install -c -m 0644 contrib/systemd/onedrive.service /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/usr/lib/systemd/system ++ /usr/lib/rpm/check-buildroot ++ /usr/lib/rpm/redhat/brp-compress ++ /usr/lib/rpm/redhat/brp-strip /usr/bin/strip ++ /usr/lib/rpm/redhat/brp-strip-comment-note /usr/bin/strip /usr/bin/objdump ++ /usr/lib/rpm/redhat/brp-strip-static-archive /usr/bin/strip ++ /usr/lib/rpm/brp-python-bytecompile /usr/bin/python 1 ++ /usr/lib/rpm/redhat/brp-python-hardlink ++ /usr/lib/rpm/redhat/brp-java-repack-jars +Processing files: onedrive-2.4.15-1.el7.x86_64 +Executing(%doc): /bin/sh -e /var/tmp/rpm-tmp.cpSXho ++ umask 022 ++ cd /home/alex/rpmbuild/BUILD ++ cd onedrive-2.4.15 ++ DOCDIR=/home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/usr/share/doc/onedrive-2.4.15 ++ export DOCDIR ++ /usr/bin/mkdir -p /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/usr/share/doc/onedrive-2.4.15 ++ cp -pr README.md /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/usr/share/doc/onedrive-2.4.15 ++ cp -pr LICENSE /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/usr/share/doc/onedrive-2.4.15 ++ cp -pr CHANGELOG.md /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64/usr/share/doc/onedrive-2.4.15 ++ exit 0 +Provides: config(onedrive) = 2.4.15-1.el7 onedrive = 2.4.15-1.el7 onedrive(x86-64) = 2.4.15-1.el7 +Requires(rpmlib): rpmlib(CompressedFileNames) <= 3.0.4-1 rpmlib(FileDigests) <= 4.6.0-1 rpmlib(PayloadFilesHavePrefix) <= 4.0-1 +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd +Requires: ld-linux-x86-64.so.2()(64bit) ld-linux-x86-64.so.2(GLIBC_2.3)(64bit) libc.so.6()(64bit) libc.so.6(GLIBC_2.14)(64bit) libc.so.6(GLIBC_2.15)(64bit) libc.so.6(GLIBC_2.2.5)(64bit) libc.so.6(GLIBC_2.3.2)(64bit) libc.so.6(GLIBC_2.3.4)(64bit) libc.so.6(GLIBC_2.4)(64bit) libc.so.6(GLIBC_2.6)(64bit) libc.so.6(GLIBC_2.8)(64bit) libc.so.6(GLIBC_2.9)(64bit) libcurl.so.4()(64bit) libdl.so.2()(64bit) libdl.so.2(GLIBC_2.2.5)(64bit) libgcc_s.so.1()(64bit) libgcc_s.so.1(GCC_3.0)(64bit) libgcc_s.so.1(GCC_4.2.0)(64bit) libm.so.6()(64bit) libm.so.6(GLIBC_2.2.5)(64bit) libpthread.so.0()(64bit) libpthread.so.0(GLIBC_2.2.5)(64bit) libpthread.so.0(GLIBC_2.3.2)(64bit) libpthread.so.0(GLIBC_2.3.4)(64bit) librt.so.1()(64bit) librt.so.1(GLIBC_2.2.5)(64bit) libsqlite3.so.0()(64bit) rtld(GNU_HASH) +Checking for unpackaged file(s): /usr/lib/rpm/check-files /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el7.x86_64 +Wrote: /home/alex/rpmbuild/SRPMS/onedrive-2.4.15-1.el7.src.rpm +Wrote: /home/alex/rpmbuild/RPMS/x86_64/onedrive-2.4.15-1.el7.x86_64.rpm +Executing(%clean): /bin/sh -e /var/tmp/rpm-tmp.nWoW33 ++ umask 022 ++ cd /home/alex/rpmbuild/BUILD ++ cd onedrive-2.4.15 ++ exit 0 +[alex@localhost ~]$ sudo yum -y install /home/alex/rpmbuild/RPMS/x86_64/onedrive-2.4.15-1.el7.x86_64.rpm +Loaded plugins: fastestmirror +Examining /home/alex/rpmbuild/RPMS/x86_64/onedrive-2.4.15-1.el7.x86_64.rpm: onedrive-2.4.15-1.el7.x86_64 +Marking /home/alex/rpmbuild/RPMS/x86_64/onedrive-2.4.15-1.el7.x86_64.rpm to be installed +Resolving Dependencies +--> Running transaction check +---> Package onedrive.x86_64 0:2.4.15-1.el7 will be installed +--> Finished Dependency Resolution + +Dependencies Resolved + +============================================================================================================================================================================================== + Package Arch Version Repository Size +============================================================================================================================================================================================== +Installing: + onedrive x86_64 2.4.15-1.el7 /onedrive-2.4.15-1.el7.x86_64 7.2 M + +Transaction Summary +============================================================================================================================================================================================== +Install 1 Package + +Total size: 7.2 M +Installed size: 7.2 M +Downloading packages: +Running transaction check +Running transaction test +Transaction test succeeded +Running transaction + Installing : onedrive-2.4.15-1.el7.x86_64 1/1 + Verifying : onedrive-2.4.15-1.el7.x86_64 1/1 + +Installed: + onedrive.x86_64 0:2.4.15-1.el7 + +Complete! +[alex@localhost ~]$ which onedrive +/usr/bin/onedrive +[alex@localhost ~]$ onedrive --version +onedrive v2.4.15 +[alex@localhost ~]$ onedrive --display-config +onedrive version = v2.4.15 +Config path = /home/alex/.config/onedrive +Config file found in config path = false +Config option 'check_nosync' = false +Config option 'sync_dir' = /home/alex/OneDrive +Config option 'skip_dir' = +Config option 'skip_file' = ~*|.~*|*.tmp +Config option 'skip_dotfiles' = false +Config option 'skip_symlinks' = false +Config option 'monitor_interval' = 300 +Config option 'min_notify_changes' = 5 +Config option 'log_dir' = /var/log/onedrive/ +Config option 'classify_as_big_delete' = 1000 +Config option 'upload_only' = false +Config option 'no_remote_delete' = false +Config option 'remove_source_files' = false +Config option 'sync_root_files' = false +Selective sync 'sync_list' configured = false +Business Shared Folders configured = false +[alex@localhost ~]$ +``` + +### CentOS 8 +```text +[alex@localhost ~]$ rpmbuild -ba ~/rpmbuild/SPECS/onedrive.spec +Executing(%prep): /bin/sh -e /var/tmp/rpm-tmp.UINFyE ++ umask 022 ++ cd /home/alex/rpmbuild/BUILD ++ cd /home/alex/rpmbuild/BUILD ++ rm -rf onedrive-2.4.15 ++ /usr/bin/gzip -dc /home/alex/rpmbuild/SOURCES/v2.4.15.tar.gz ++ /usr/bin/tar -xof - ++ STATUS=0 ++ '[' 0 -ne 0 ']' ++ cd onedrive-2.4.15 ++ /usr/bin/chmod -Rf a+rX,u+w,g-w,o-w . ++ exit 0 +Executing(%build): /bin/sh -e /var/tmp/rpm-tmp.cX1WQa ++ umask 022 ++ cd /home/alex/rpmbuild/BUILD ++ cd onedrive-2.4.15 ++ CFLAGS='-O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection' ++ export CFLAGS ++ CXXFLAGS='-O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection' ++ export CXXFLAGS ++ FFLAGS='-O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -I/usr/lib64/gfortran/modules' ++ export FFLAGS ++ FCFLAGS='-O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -I/usr/lib64/gfortran/modules' ++ export FCFLAGS ++ LDFLAGS='-Wl,-z,relro -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld' ++ export LDFLAGS ++ '[' 1 = 1 ']' ++++ dirname ./configure +++ find . -name config.guess -o -name config.sub ++ '[' 1 = 1 ']' ++ '[' x '!=' 'x-Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld' ']' +++ find . -name ltmain.sh ++ ./configure --build=x86_64-redhat-linux-gnu --host=x86_64-redhat-linux-gnu --program-prefix= --disable-dependency-tracking --prefix=/usr --exec-prefix=/usr --bindir=/usr/bin --sbindir=/usr/sbin --sysconfdir=/etc --datadir=/usr/share --includedir=/usr/include --libdir=/usr/lib64 --libexecdir=/usr/libexec --localstatedir=/var --sharedstatedir=/var/lib --mandir=/usr/share/man --infodir=/usr/share/info +configure: WARNING: unrecognized options: --disable-dependency-tracking +checking for a BSD-compatible install... /usr/bin/install -c +checking for x86_64-redhat-linux-gnu-pkg-config... /usr/bin/x86_64-redhat-linux-gnu-pkg-config +checking pkg-config is at least version 0.9.0... yes +checking for dmd... dmd +checking version of D compiler... 2.087.0 +checking for curl... yes +checking for sqlite... yes +configure: creating ./config.status +config.status: creating Makefile +config.status: creating contrib/pacman/PKGBUILD +config.status: creating contrib/spec/onedrive.spec +config.status: creating onedrive.1 +config.status: creating contrib/systemd/onedrive.service +config.status: creating contrib/systemd/onedrive@.service +configure: WARNING: unrecognized options: --disable-dependency-tracking ++ make +if [ -f .git/HEAD ] ; then \ + git describe --tags > version ; \ +else \ + echo v2.4.15 > version ; \ +fi +dmd -w -g -O -J. -L-lcurl -L-lsqlite3 -L-ldl src/config.d src/itemdb.d src/log.d src/main.d src/monitor.d src/onedrive.d src/qxor.d src/selective.d src/sqlite.d src/sync.d src/upload.d src/util.d src/progress.d src/arsd/cgi.d -ofonedrive ++ exit 0 +Executing(%install): /bin/sh -e /var/tmp/rpm-tmp.dNFPdx ++ umask 022 ++ cd /home/alex/rpmbuild/BUILD ++ '[' /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64 '!=' / ']' ++ rm -rf /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64 +++ dirname /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64 ++ mkdir -p /home/alex/rpmbuild/BUILDROOT ++ mkdir /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64 ++ cd onedrive-2.4.15 ++ /usr/bin/make install DESTDIR=/home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64 'INSTALL=/usr/bin/install -p' PREFIX=/home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64 +/usr/bin/install -p -D onedrive /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/usr/bin/onedrive +/usr/bin/install -p -D onedrive.1 /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/usr/share/man/man1/onedrive.1 +/usr/bin/install -p -D -m 644 contrib/logrotate/onedrive.logrotate /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/etc/logrotate.d/onedrive +mkdir -p /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/usr/share/doc/onedrive +/usr/bin/install -p -D -m 644 README.md config LICENSE CHANGELOG.md docs/Docker.md docs/INSTALL.md docs/SharePoint-Shared-Libraries.md docs/USAGE.md docs/BusinessSharedFolders.md docs/advanced-usage.md /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/usr/share/doc/onedrive +/usr/bin/install -p -d -m 0755 /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/usr/lib/systemd/user /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/usr/lib/systemd/system +/usr/bin/install -p -m 0644 contrib/systemd/onedrive@.service /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/usr/lib/systemd/system +/usr/bin/install -p -m 0644 contrib/systemd/onedrive.service /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/usr/lib/systemd/system ++ /usr/lib/rpm/check-buildroot ++ /usr/lib/rpm/redhat/brp-ldconfig +/sbin/ldconfig: Warning: ignoring configuration file that cannot be opened: /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/etc/ld.so.conf: No such file or directory ++ /usr/lib/rpm/brp-compress ++ /usr/lib/rpm/brp-strip /usr/bin/strip ++ /usr/lib/rpm/brp-strip-comment-note /usr/bin/strip /usr/bin/objdump ++ /usr/lib/rpm/brp-strip-static-archive /usr/bin/strip ++ /usr/lib/rpm/brp-python-bytecompile 1 ++ /usr/lib/rpm/brp-python-hardlink ++ PYTHON3=/usr/libexec/platform-python ++ /usr/lib/rpm/redhat/brp-mangle-shebangs +Processing files: onedrive-2.4.15-1.el8.x86_64 +Executing(%doc): /bin/sh -e /var/tmp/rpm-tmp.TnFKbZ ++ umask 022 ++ cd /home/alex/rpmbuild/BUILD ++ cd onedrive-2.4.15 ++ DOCDIR=/home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/usr/share/doc/onedrive ++ export LC_ALL=C ++ LC_ALL=C ++ export DOCDIR ++ /usr/bin/mkdir -p /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/usr/share/doc/onedrive ++ cp -pr README.md /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/usr/share/doc/onedrive ++ cp -pr LICENSE /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/usr/share/doc/onedrive ++ cp -pr CHANGELOG.md /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64/usr/share/doc/onedrive ++ exit 0 +warning: File listed twice: /usr/share/doc/onedrive +warning: File listed twice: /usr/share/doc/onedrive/CHANGELOG.md +warning: File listed twice: /usr/share/doc/onedrive/LICENSE +warning: File listed twice: /usr/share/doc/onedrive/README.md +Provides: config(onedrive) = 2.4.15-1.el8 onedrive = 2.4.15-1.el8 onedrive(x86-64) = 2.4.15-1.el8 +Requires(rpmlib): rpmlib(CompressedFileNames) <= 3.0.4-1 rpmlib(FileDigests) <= 4.6.0-1 rpmlib(PayloadFilesHavePrefix) <= 4.0-1 +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd +Requires: ld-linux-x86-64.so.2()(64bit) ld-linux-x86-64.so.2(GLIBC_2.3)(64bit) libc.so.6()(64bit) libc.so.6(GLIBC_2.14)(64bit) libc.so.6(GLIBC_2.15)(64bit) libc.so.6(GLIBC_2.2.5)(64bit) libc.so.6(GLIBC_2.3.2)(64bit) libc.so.6(GLIBC_2.3.4)(64bit) libc.so.6(GLIBC_2.4)(64bit) libc.so.6(GLIBC_2.6)(64bit) libc.so.6(GLIBC_2.8)(64bit) libc.so.6(GLIBC_2.9)(64bit) libcurl.so.4()(64bit) libdl.so.2()(64bit) libdl.so.2(GLIBC_2.2.5)(64bit) libgcc_s.so.1()(64bit) libgcc_s.so.1(GCC_3.0)(64bit) libgcc_s.so.1(GCC_4.2.0)(64bit) libm.so.6()(64bit) libm.so.6(GLIBC_2.2.5)(64bit) libpthread.so.0()(64bit) libpthread.so.0(GLIBC_2.2.5)(64bit) libpthread.so.0(GLIBC_2.3.2)(64bit) libpthread.so.0(GLIBC_2.3.4)(64bit) librt.so.1()(64bit) librt.so.1(GLIBC_2.2.5)(64bit) libsqlite3.so.0()(64bit) rtld(GNU_HASH) +Checking for unpackaged file(s): /usr/lib/rpm/check-files /home/alex/rpmbuild/BUILDROOT/onedrive-2.4.15-1.el8.x86_64 +Wrote: /home/alex/rpmbuild/SRPMS/onedrive-2.4.15-1.el8.src.rpm +Wrote: /home/alex/rpmbuild/RPMS/x86_64/onedrive-2.4.15-1.el8.x86_64.rpm +Executing(%clean): /bin/sh -e /var/tmp/rpm-tmp.FAMTFz ++ umask 022 ++ cd /home/alex/rpmbuild/BUILD ++ cd onedrive-2.4.15 ++ exit 0 +[alex@localhost ~]$ sudo yum -y install /home/alex/rpmbuild/RPMS/x86_64/onedrive-2.4.15-1.el8.x86_64.rpm +Last metadata expiration check: 0:04:07 ago on Fri 14 Jan 2022 14:22:13 EST. +Dependencies resolved. +============================================================================================================================================================================================== + Package Architecture Version Repository Size +============================================================================================================================================================================================== +Installing: + onedrive x86_64 2.4.15-1.el8 @commandline 1.5 M + +Transaction Summary +============================================================================================================================================================================================== +Install 1 Package + +Total size: 1.5 M +Installed size: 7.1 M +Downloading Packages: +Running transaction check +Transaction check succeeded. +Running transaction test +Transaction test succeeded. +Running transaction + Preparing : 1/1 + Installing : onedrive-2.4.15-1.el8.x86_64 1/1 + Running scriptlet: onedrive-2.4.15-1.el8.x86_64 1/1 + Verifying : onedrive-2.4.15-1.el8.x86_64 1/1 + +Installed: + onedrive-2.4.15-1.el8.x86_64 + +Complete! +[alex@localhost ~]$ which onedrive +/usr/bin/onedrive +[alex@localhost ~]$ onedrive --version +onedrive v2.4.15 +[alex@localhost ~]$ onedrive --display-config +onedrive version = v2.4.15 +Config path = /home/alex/.config/onedrive +Config file found in config path = false +Config option 'check_nosync' = false +Config option 'sync_dir' = /home/alex/OneDrive +Config option 'skip_dir' = +Config option 'skip_file' = ~*|.~*|*.tmp +Config option 'skip_dotfiles' = false +Config option 'skip_symlinks' = false +Config option 'monitor_interval' = 300 +Config option 'min_notify_changes' = 5 +Config option 'log_dir' = /var/log/onedrive/ +Config option 'classify_as_big_delete' = 1000 +Config option 'upload_only' = false +Config option 'no_remote_delete' = false +Config option 'remove_source_files' = false +Config option 'sync_root_files' = false +Selective sync 'sync_list' configured = false +Business Shared Folders configured = false +[alex@localhost ~]$ +``` diff --git a/docs/business-shared-folders.md b/docs/business-shared-folders.md new file mode 100644 index 00000000..4282f4ac --- /dev/null +++ b/docs/business-shared-folders.md @@ -0,0 +1,40 @@ +# How to configure OneDrive Business Shared Folder Sync +## Application Version +Before reading this document, please ensure you are running application version [![Version](https://img.shields.io/github/v/release/abraunegg/onedrive)](https://github.com/abraunegg/onedrive/releases) or greater. Use `onedrive --version` to determine what application version you are using and upgrade your client if required. + +## Important Note +This feature has been 100% re-written from v2.5.0 onwards. A pre-requesite before using this capability in v2.5.0 and above is for you to revert any Shared Business Folder configuration you may be currently using, including, but not limited to: +* Removing `sync_business_shared_folders = "true|false"` from your 'config' file +* Removing the 'business_shared_folders' file +* Removing any local data | shared folder data from your configured 'sync_dir' to ensure that there are no conflicts or issues. + +## Process Overview +Syncing OneDrive Business Shared Folders requires additional configuration for your 'onedrive' client: +1. From the OneDrive web interface, review the 'Shared' objects that have been shared with you. +2. Select the applicable folder, and click the 'Add shortcut to My files', which will then add this to your 'My files' folder +3. Update your OneDrive Client for Linux 'config' file to enable the feature by adding `sync_business_shared_items = "true"`. Adding this option will trigger a `--resync` requirement. +4. Test the configuration using '--dry-run' +5. Remove the use of '--dry-run' and sync the OneDrive Business Shared folders as required + + +**NOTE:** This documentation will be updated as this feature progresses. + + +### Enable syncing of OneDrive Business Shared Folders via config file +```text +sync_business_shared_items = "true" +``` + +### Disable syncing of OneDrive Business Shared Folders via config file +```text +sync_business_shared_items = "false" +``` + +## Known Issues +Shared folders, shared with you from people outside of your 'organisation' are unable to be synced. This is due to the Microsoft Graph API not presenting these folders. + +Shared folders that match this scenario, when you view 'Shared' via OneDrive online, will have a 'world' symbol as per below: + +![shared_with_me](./images/shared_with_me.JPG) + +This issue is being tracked by: [#966](https://github.com/abraunegg/onedrive/issues/966) diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 00000000..d142fef8 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,320 @@ +# Run the OneDrive Client for Linux under Docker +This client can be run as a Docker container, with 3 available container base options for you to choose from: + +| Container Base | Docker Tag | Description | i686 | x86_64 | ARMHF | AARCH64 | +|----------------|-------------|----------------------------------------------------------------|:------:|:------:|:-----:|:-------:| +| Alpine Linux | edge-alpine | Docker container based on Alpine 3.18 using 'master' |❌|✔|❌|✔| +| Alpine Linux | alpine | Docker container based on Alpine 3.18 using latest release |❌|✔|❌|✔| +| Debian | debian | Docker container based on Debian Stable using latest release |✔|✔|✔|✔| +| Debian | edge | Docker container based on Debian Stable using 'master' |✔|✔|✔|✔| +| Debian | edge-debian | Docker container based on Debian Stable using 'master' |✔|✔|✔|✔| +| Debian | latest | Docker container based on Debian Stable using latest release |✔|✔|✔|✔| +| Fedora | edge-fedora | Docker container based on Fedora 38 using 'master' |❌|✔|❌|✔| +| Fedora | fedora | Docker container based on Fedora 38 using latest release |❌|✔|❌|✔| + +These containers offer a simple monitoring-mode service for the OneDrive Client for Linux. + +The instructions below have been validated on: +* Red Hat Enterprise Linux 8.x +* Ubuntu Server 22.04 + +The instructions below will utilise the 'edge' tag, however this can be substituted for any of the other docker tags such as 'latest' from the table above if desired. + +The 'edge' Docker Container will align closer to all documentation and features, where as 'latest' is the release version from a static point in time. The 'latest' tag however may contain bugs and/or issues that will have been fixed, and those fixes are contained in 'edge'. + +Additionally there are specific version release tags for each release. Refer to https://hub.docker.com/r/driveone/onedrive/tags for any other Docker tags you may be interested in. + +## Basic Setup +### 0. Install docker using your distribution platform's instructions +1. Ensure that SELinux has been disabled on your system. A reboot may be required to ensure that this is correctly disabled. +2. Install Docker as per required for your platform. Refer to https://docs.docker.com/engine/install/ for assistance. +3. Obtain your normal, non-root user UID and GID by using the `id` command +4. As your normal, non-root user, ensure that you can run `docker run hello-world` *without* using `sudo` + +Once the above 4 steps are complete and you can successfully run `docker run hello-world` without sudo, only then proceed to 'Pulling and Running the Docker Image' + +## Pulling and Running the Docker Image +### 1. Pull the image +```bash +docker pull driveone/onedrive:edge +``` + +**NOTE:** SELinux context needs to be configured or disabled for Docker to be able to write to OneDrive host directory. + +### 2. Prepare config volume +The Docker container requries 2 Docker volumes: +* Config Volume +* Data Volume + +Create the config volume with the following command: +```bash +docker volume create onedrive_conf +``` + +This will create a docker volume labeled `onedrive_conf`, where all configuration of your onedrive account will be stored. You can add a custom config file and other things later. + +The second docker volume is for your data folder and is created in the next step. This volume needs to be a path to a directory on your local filesystem, and this is where your data will be stored from OneDrive. Keep in mind that: + +* The owner of this specified folder must not be root +* The owner of this specified folder must have permissions for its parent directory + +**NOTE:** Issues occur when this target folder is a mounted folder of an external system (NAS, SMB mount, USB Drive etc) as the 'mount' itself is owned by 'root'. If this is your use case, you *must* ensure your normal user can mount your desired target without having the target mounted by 'root'. If you do not fix this, your Docker container will fail to start with the following error message: +```bash +ROOT level privileges prohibited! +``` + +### 3. First run +The 'onedrive' client within the Docker container needs to be authorized with your Microsoft account. This is achieved by initially running docker in interactive mode. + +Run the docker image with the commands below and make sure to change the value of `ONEDRIVE_DATA_DIR` to the actual onedrive data directory on your filesystem that you wish to use (e.g. `export ONEDRIVE_DATA_DIR="/home/abraunegg/OneDrive"`). +```bash +export ONEDRIVE_DATA_DIR="${HOME}/OneDrive" +mkdir -p ${ONEDRIVE_DATA_DIR} +docker run -it --name onedrive -v onedrive_conf:/onedrive/conf \ + -v "${ONEDRIVE_DATA_DIR}:/onedrive/data" \ + -e "ONEDRIVE_UID=${ONEDRIVE_UID}" \ + -e "ONEDRIVE_GID=${ONEDRIVE_GID}" \ + driveone/onedrive:edge +``` +**Important:** The 'target' folder of `ONEDRIVE_DATA_DIR` must exist before running the Docker container, otherwise, Docker will create the target folder, and the folder will be given 'root' permissions, which then causes the Docker container to fail upon startup with the following error message: +```bash +ROOT level privileges prohibited! +``` +**NOTE:** It is also highly advisable for you to replace `${ONEDRIVE_UID}` and `${ONEDRIVE_GID}` with your actual UID and GID as specified by your `id` command output to avoid any any potential user or group conflicts. + +**Example:** +```bash +export ONEDRIVE_UID=`id -u` +export ONEDRIVE_GID=`id -g` +export ONEDRIVE_DATA_DIR="${HOME}/OneDrive" +mkdir -p ${ONEDRIVE_DATA_DIR} +docker run -it --name onedrive -v onedrive_conf:/onedrive/conf \ + -v "${ONEDRIVE_DATA_DIR}:/onedrive/data" \ + -e "ONEDRIVE_UID=${ONEDRIVE_UID}" \ + -e "ONEDRIVE_GID=${ONEDRIVE_GID}" \ + driveone/onedrive:edge +``` + +When the Docker container successfully starts: +* You will be asked to open a specific link using your web browser +* Login to your Microsoft Account and give the application the permission +* After giving the permission, you will be redirected to a blank page +* Copy the URI of the blank page into the application prompt to authorise the application + +Once the 'onedrive' application is authorised, the client will automatically start monitoring your `ONEDRIVE_DATA_DIR` for data changes to be uploaded to OneDrive. Files stored on OneDrive will be downloaded to this location. + +If the client is working as expected, you can detach from the container with Ctrl+p, Ctrl+q. + +### 4. Docker Container Status, stop, and restart +Check if the monitor service is running + +```bash +docker ps -f name=onedrive +``` + +Show monitor run logs + +```bash +docker logs onedrive +``` + +Stop running monitor + +```bash +docker stop onedrive +``` + +Resume monitor + +```bash +docker start onedrive +``` + +Remove onedrive Docker container + +```bash +docker rm -f onedrive +``` +## Advanced Setup + +### 5. Docker-compose +Also supports docker-compose schemas > 3. +In the following example it is assumed you have a `ONEDRIVE_DATA_DIR` environment variable and a `onedrive_conf` volume. +However, you can also use bind mounts for the configuration folder, e.g. `export ONEDRIVE_CONF="${HOME}/OneDriveConfig"`. + +``` +version: "3" +services: + onedrive: + image: driveone/onedrive:edge + restart: unless-stopped + environment: + - ONEDRIVE_UID=${PUID} + - ONEDRIVE_GID=${PGID} + volumes: + - onedrive_conf:/onedrive/conf + - ${ONEDRIVE_DATA_DIR}:/onedrive/data +``` + +Note that you still have to perform step 3: First Run. + +### 6. Edit the config +The 'onedrive' client should run in default configuration, however you can change this default configuration by placing a custom config file in the `onedrive_conf` docker volume. First download the default config from [here](https://raw.githubusercontent.com/abraunegg/onedrive/master/config) +Then put it into your onedrive_conf volume path, which can be found with: + +```bash +docker volume inspect onedrive_conf +``` + +Or you can map your own config folder to the config volume. Make sure to copy all files from the docker volume into your mapped folder first. + +The detailed document for the config can be found here: [Configuration](https://github.com/abraunegg/onedrive/blob/master/docs/usage.md#configuration) + +### 7. Sync multiple accounts +There are many ways to do this, the easiest is probably to +1. Create a second docker config volume (replace `Work` with your desired name): `docker volume create onedrive_conf_Work` +2. And start a second docker monitor container (again replace `Work` with your desired name): +``` +export ONEDRIVE_DATA_DIR_WORK="/home/abraunegg/OneDriveWork" +mkdir -p ${ONEDRIVE_DATA_DIR_WORK} +docker run -it --restart unless-stopped --name onedrive_Work -v onedrive_conf_Work:/onedrive/conf -v "${ONEDRIVE_DATA_DIR_WORK}:/onedrive/data" driveone/onedrive:edge +``` + +## Run or update with one script +If you are experienced with docker and onedrive, you can use the following script: + +```bash +# Update ONEDRIVE_DATA_DIR with correct OneDrive directory path +ONEDRIVE_DATA_DIR="${HOME}/OneDrive" +# Create directory if non-existant +mkdir -p ${ONEDRIVE_DATA_DIR} + +firstRun='-d' +docker pull driveone/onedrive:edge +docker inspect onedrive_conf > /dev/null 2>&1 || { docker volume create onedrive_conf; firstRun='-it'; } +docker inspect onedrive > /dev/null 2>&1 && docker rm -f onedrive +docker run $firstRun --restart unless-stopped --name onedrive -v onedrive_conf:/onedrive/conf -v "${ONEDRIVE_DATA_DIR}:/onedrive/data" driveone/onedrive:edge +``` + +## Environment Variables +| Variable | Purpose | Sample Value | +| ---------------- | --------------------------------------------------- |:--------------------------------------------------------------------------------------------------------------------------------:| +| ONEDRIVE_UID | UserID (UID) to run as | 1000 | +| ONEDRIVE_GID | GroupID (GID) to run as | 1000 | +| ONEDRIVE_VERBOSE | Controls "--verbose" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_DEBUG | Controls "--verbose --verbose" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_DEBUG_HTTPS | Controls "--debug-https" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_RESYNC | Controls "--resync" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_DOWNLOADONLY | Controls "--download-only" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_UPLOADONLY | Controls "--upload-only" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_NOREMOTEDELETE | Controls "--no-remote-delete" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_LOGOUT | Controls "--logout" switch. Default is 0 | 1 | +| ONEDRIVE_REAUTH | Controls "--reauth" switch. Default is 0 | 1 | +| ONEDRIVE_AUTHFILES | Controls "--auth-files" option. Default is "" | "authUrl:responseUrl" | +| ONEDRIVE_AUTHRESPONSE | Controls "--auth-response" option. Default is "" | See [here](https://github.com/abraunegg/onedrive/blob/master/docs/usage.md#authorize-the-application-with-your-onedrive-account) | +| ONEDRIVE_DISPLAY_CONFIG | Controls "--display-running-config" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_SINGLE_DIRECTORY | Controls "--single-directory" option. Default = "" | "mydir" | + +### Usage Examples +**Verbose Output:** +```bash +docker container run -e ONEDRIVE_VERBOSE=1 -v onedrive_conf:/onedrive/conf -v "${ONEDRIVE_DATA_DIR}:/onedrive/data" driveone/onedrive:edge +``` +**Debug Output:** +```bash +docker container run -e ONEDRIVE_DEBUG=1 -v onedrive_conf:/onedrive/conf -v "${ONEDRIVE_DATA_DIR}:/onedrive/data" driveone/onedrive:edge +``` +**Perform a --resync:** +```bash +docker container run -e ONEDRIVE_RESYNC=1 -v onedrive_conf:/onedrive/conf -v "${ONEDRIVE_DATA_DIR}:/onedrive/data" driveone/onedrive:edge +``` +**Perform a --resync and --verbose:** +```bash +docker container run -e ONEDRIVE_RESYNC=1 -e ONEDRIVE_VERBOSE=1 -v onedrive_conf:/onedrive/conf -v "${ONEDRIVE_DATA_DIR}:/onedrive/data" driveone/onedrive:edge +``` +**Perform a --logout and re-authenticate:** +```bash +docker container run -it -e ONEDRIVE_LOGOUT=1 -v onedrive_conf:/onedrive/conf -v "${ONEDRIVE_DATA_DIR}:/onedrive/data" driveone/onedrive:edge +``` + +## Build instructions +### Build Environment Requirements +* Build environment must have at least 1GB of memory & 2GB swap space + +There are 2 ways to validate this requirement: +* Modify the file `/etc/dphys-swapfile` and edit the `CONF_SWAPSIZE`, for example: `CONF_SWAPSIZE=2048`. A reboot is required to make this change effective. +* Dynamically allocate a swapfile for building: +```bash +cd /var +sudo fallocate -l 1.5G swapfile +sudo chmod 600 swapfile +sudo mkswap swapfile +sudo swapon swapfile +# make swap permanent +sudo nano /etc/fstab +# add "/swapfile swap swap defaults 0 0" at the end of file +# check it has been assigned +swapon -s +free -h +``` + +### Building a custom Docker image +You can also build your own image instead of pulling the one from [hub.docker.com](https://hub.docker.com/r/driveone/onedrive): +```bash +git clone https://github.com/abraunegg/onedrive +cd onedrive +docker build . -t local-onedrive -f contrib/docker/Dockerfile +``` + +There are alternate, smaller images available by building +Dockerfile-debian or Dockerfile-alpine. These [multi-stage builder pattern](https://docs.docker.com/develop/develop-images/multistage-build/) +Dockerfiles require Docker version at least 17.05. + +#### How to build and run a custom Docker image based on Debian +``` bash +docker build . -t local-ondrive-debian -f contrib/docker/Dockerfile-debian +docker container run -v onedrive_conf:/onedrive/conf -v "${ONEDRIVE_DATA_DIR}:/onedrive/data" local-ondrive-debian:latest +``` + +#### How to build and run a custom Docker image based on Alpine Linux +``` bash +docker build . -t local-ondrive-alpine -f contrib/docker/Dockerfile-alpine +docker container run -v onedrive_conf:/onedrive/conf -v "${ONEDRIVE_DATA_DIR}:/onedrive/data" local-ondrive-alpine:latest +``` + +#### How to build and run a custom Docker image for ARMHF (Raspberry Pi) +Compatible with: +* Raspberry Pi +* Raspberry Pi 2 +* Raspberry Pi Zero +* Raspberry Pi 3 +* Raspberry Pi 4 +``` bash +docker build . -t local-onedrive-armhf -f contrib/docker/Dockerfile-debian +docker container run -v onedrive_conf:/onedrive/conf -v "${ONEDRIVE_DATA_DIR}:/onedrive/data" local-onedrive-armhf:latest +``` + +#### How to build and run a custom Docker image for AARCH64 Platforms +``` bash +docker build . -t local-onedrive-aarch64 -f contrib/docker/Dockerfile-debian +docker container run -v onedrive_conf:/onedrive/conf -v "${ONEDRIVE_DATA_DIR}:/onedrive/data" local-onedrive-aarch64:latest +``` + +#### How to support double-byte languages +In some geographic regions, you may need to change and/or update the locale specification of the Docker container to better support the local language used for your local filesystem. To do this, follow the example below: +``` +FROM driveone/onedrive + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update +RUN apt-get install -y locales + +RUN echo "ja_JP.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen ja_JP.UTF-8 && \ + dpkg-reconfigure locales && \ + /usr/sbin/update-locale LANG=ja_JP.UTF-8 + +ENV LC_ALL ja_JP.UTF-8 +``` +The above example changes the Docker container to support Japanese. To support your local language, change `ja_JP.UTF-8` to the required entry. diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 00000000..c3e126c3 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,281 @@ +# Installing or Upgrading using Distribution Packages or Building the OneDrive Client for Linux from source + +## Installing or Upgrading using Distribution Packages +This project has been packaged for the following Linux distributions as per below. The current client release is: [![Version](https://img.shields.io/github/v/release/abraunegg/onedrive)](https://github.com/abraunegg/onedrive/releases) + +Only the current release version or greater is supported. Earlier versions are not supported and should not be installed or used. + +#### Important Note: +Distribution packages may be of an older release when compared to the latest release that is [available](https://github.com/abraunegg/onedrive/releases). If any package version indicator below is 'red' for your distribution, it is recommended that you build from source. Do not install the software from the available distribution package. If a package is out of date, please contact the package maintainer for resolution. + +| Distribution | Package Name & Package Link |   PKG_Version   |  i686  | x86_64 | ARMHF | AARCH64 | Extra Details | +|---------------------------------|------------------------------------------------------------------------------|:---------------:|:----:|:------:|:-----:|:-------:|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Alpine Linux | [onedrive](https://pkgs.alpinelinux.org/packages?name=onedrive&branch=edge) |Alpine Linux Edge package|❌|✔|❌|✔ | | +| Arch Linux

Manjaro Linux | [onedrive-abraunegg](https://aur.archlinux.org/packages/onedrive-abraunegg/) |AUR package|✔|✔|✔|✔ | Install via: `pamac build onedrive-abraunegg` from the Arch Linux User Repository (AUR)

**Note:** If asked regarding a provider for 'd-runtime' and 'd-compiler', select 'liblphobos' and 'ldc'

**Note:** System must have at least 1GB of memory & 1GB swap space +| Debian 11 | [onedrive](https://packages.debian.org/bullseye/source/onedrive) |Debian 11 package|✔|✔|✔|✔| **Note:** Do not install from Debian Package Repositories

It is recommended that for Debian 11 that you install from OpenSuSE Build Service using the Debian Package Install [Instructions](ubuntu-package-install.md) | +| Debian 12 | [onedrive](https://packages.debian.org/bookworm/source/onedrive) |Debian 12 package|✔|✔|✔|✔| **Note:** Do not install from Debian Package Repositories

It is recommended that for Debian 12 that you install from OpenSuSE Build Service using the Debian Package Install [Instructions](ubuntu-package-install.md) | +| Fedora | [onedrive](https://koji.fedoraproject.org/koji/packageinfo?packageID=26044) |Fedora Rawhide package|✔|✔|✔|✔| | +| Gentoo | [onedrive](https://gpo.zugaina.org/net-misc/onedrive) | No API Available |✔|✔|❌|❌| | +| Homebrew | [onedrive](https://formulae.brew.sh/formula/onedrive) | Homebrew package |❌|✔|❌|❌| | +| Linux Mint 20.x | [onedrive](https://community.linuxmint.com/software/view/onedrive) |Ubuntu 20.04 package |❌|✔|✔|✔| **Note:** Do not install from Linux Mint Repositories

It is recommended that for Linux Mint that you install from OpenSuSE Build Service using the Ubuntu Package Install [Instructions](ubuntu-package-install.md) | +| Linux Mint 21.x | [onedrive](https://community.linuxmint.com/software/view/onedrive) |Ubuntu 22.04 package |❌|✔|✔|✔| **Note:** Do not install from Linux Mint Repositories

It is recommended that for Linux Mint that you install from OpenSuSE Build Service using the Ubuntu Package Install [Instructions](ubuntu-package-install.md) | +| NixOS | [onedrive](https://search.nixos.org/packages?channel=20.09&from=0&size=50&sort=relevance&query=onedrive)|nixpkgs unstable package|❌|✔|❌|❌| Use package `onedrive` either by adding it to `configuration.nix` or by using the command `nix-env -iA .onedrive`. This does not install a service. To install a service, use unstable channel (will stabilize in 20.09) and add `services.onedrive.enable=true` in `configuration.nix`. You can also add a custom package using the `services.onedrive.package` option (recommended since package lags upstream). Enabling the service installs a default package too (based on the channel). You can also add multiple onedrive accounts trivially, see [documentation](https://github.com/NixOS/nixpkgs/pull/77734#issuecomment-575874225). | +| OpenSuSE | [onedrive](https://software.opensuse.org/package/onedrive) |openSUSE Tumbleweed package|✔|✔|❌|❌| | +| OpenSuSE Build Service | [onedrive](https://build.opensuse.org/package/show/home:npreining:debian-ubuntu-onedrive/onedrive) | No API Available |✔|✔|✔|✔| Package Build Service for Debian and Ubuntu | +| Raspbian | [onedrive](https://archive.raspbian.org/raspbian/pool/main/o/onedrive/) |Raspbian Stable package |❌|❌|✔|✔| **Note:** Do not install from Raspbian Package Repositories

It is recommended that for Raspbian that you install from OpenSuSE Build Service using the Debian Package Install [Instructions](ubuntu-package-install.md) | +| Slackware | [onedrive](https://slackbuilds.org/result/?search=onedrive&sv=) |SlackBuilds package|✔|✔|❌|❌| | +| Solus | [onedrive](https://dev.getsol.us/search/query/FB7PIf1jG9Z9/#R) |Solus package|✔|✔|❌|❌| | +| Ubuntu 20.04 | [onedrive](https://packages.ubuntu.com/focal/onedrive) |Ubuntu 20.04 package |❌|✔|✔|✔| **Note:** Do not install from Ubuntu Universe

It is recommended that for Ubuntu that you install from OpenSuSE Build Service using the Ubuntu Package Install [Instructions](ubuntu-package-install.md) | +| Ubuntu 22.04 | [onedrive](https://packages.ubuntu.com/jammy/onedrive) |Ubuntu 22.04 package |❌|✔|✔|✔| **Note:** Do not install from Ubuntu Universe

It is recommended that for Ubuntu that you install from OpenSuSE Build Service using the Ubuntu Package Install [Instructions](ubuntu-package-install.md) | +| Ubuntu 23.04 | [onedrive](https://packages.ubuntu.com/lunar/onedrive) |Ubuntu 23.04 package |❌|✔|✔|✔| **Note:** Do not install from Ubuntu Universe

It is recommended that for Ubuntu that you install from OpenSuSE Build Service using the Ubuntu Package Install [Instructions](ubuntu-package-install.md) | +| Void Linux | [onedrive](https://voidlinux.org/packages/?arch=x86_64&q=onedrive) |Void Linux x86_64 package|✔|✔|❌|❌| | + +#### Important information for all Ubuntu and Ubuntu based distribution users: +This information is specifically for the following platforms and distributions: +* Ubuntu +* Lubuntu +* Linux Mint +* POP OS +* Peppermint OS + +Whilst there are [onedrive](https://packages.ubuntu.com/search?keywords=onedrive&searchon=names&suite=all§ion=all) Universe packages available for Ubuntu, do not install 'onedrive' from these Universe packages. The default Universe packages are out-of-date and are not supported and should not be used. If you wish to use a package, it is highly recommended that you utilise the [OpenSuSE Build Service](ubuntu-package-install.md) to install packages for these platforms. If the OpenSuSE Build Service does not cater for your version, your only option is to build from source. + +If you wish to change this situation so that you can just use the Universe packages via 'apt install onedrive', consider becoming the Ubuntu package maintainer and contribute back to your community. + +## Building from Source - High Level Requirements +* Build environment must have at least 1GB of memory & 1GB swap space +* Install the required distribution package dependencies +* [libcurl](http://curl.haxx.se/libcurl/) +* [SQLite 3](https://www.sqlite.org/) >= 3.7.15 +* [Digital Mars D Compiler (DMD)](http://dlang.org/download.html) or [LDC – the LLVM-based D Compiler](https://github.com/ldc-developers/ldc) + +**Note:** DMD version >= 2.088.0 or LDC version >= 1.18.0 is required to compile this application + +### Example for installing DMD Compiler +```text +curl -fsS https://dlang.org/install.sh | bash -s dmd +``` + +### Example for installing LDC Compiler +```text +curl -fsS https://dlang.org/install.sh | bash -s ldc +``` + +## Distribution Package Dependencies +### Dependencies: Ubuntu 16.x +Ubuntu Linux 16.x LTS reached the end of its five-year LTS window on April 30th 2021 and is no longer supported. + +### Dependencies: Ubuntu 18.x / Lubuntu 18.x +Ubuntu Linux 18.x LTS reached the end of its five-year LTS window on May 31th 2023 and is no longer supported. + +### Dependencies: Debian 9 +Debian 9 reached the end of its five-year support window on June 30th 2022 and is no longer supported. + +### Dependencies: Ubuntu 20.x -> Ubuntu 23.x / Debian 10 -> Debian 12 - x86_64 +These dependencies are also applicable for all Ubuntu based distributions such as: +* Lubuntu +* Linux Mint +* POP OS +* Peppermint OS +```text +sudo apt install build-essential +sudo apt install libcurl4-openssl-dev libsqlite3-dev pkg-config git curl +curl -fsS https://dlang.org/install.sh | bash -s dmd +``` +For notifications the following is also necessary: +```text +sudo apt install libnotify-dev +``` + +### Dependencies: CentOS 6.x / RHEL 6.x +CentOS 6.x and RHEL 6.x reached End of Life status on November 30th 2020 and is no longer supported. + +### Dependencies: Fedora < Version 18 / CentOS 7.x / RHEL 7.x +```text +sudo yum groupinstall 'Development Tools' +sudo yum install libcurl-devel sqlite-devel +curl -fsS https://dlang.org/install.sh | bash -s dmd-2.099.0 +``` +For notifications the following is also necessary: +```text +sudo yum install libnotify-devel +``` + +### Dependencies: Fedora > Version 18 / CentOS 8.x / RHEL 8.x / RHEL 9.x +```text +sudo dnf groupinstall 'Development Tools' +sudo dnf install libcurl-devel sqlite-devel +curl -fsS https://dlang.org/install.sh | bash -s dmd +``` +For notifications the following is also necessary: +```text +sudo dnf install libnotify-devel +``` + +### Dependencies: Arch Linux & Manjaro Linux +```text +sudo pacman -S make pkg-config curl sqlite ldc +``` +For notifications the following is also necessary: +```text +sudo pacman -S libnotify +``` + +### Dependencies: Raspbian (ARMHF) and Ubuntu 22.x / Debian 11 / Debian 12 / Raspbian (ARM64) +**Note:** The minimum LDC compiler version required to compile this application is now 1.18.0, which is not available for Debian Buster or distributions based on Debian Buster. You are advised to first upgrade your platform distribution to one that is based on Debian Bullseye (Debian 11) or later. + +These instructions were validated using: +* `Linux raspberrypi 5.10.92-v8+ #1514 SMP PREEMPT Mon Jan 17 17:39:38 GMT 2022 aarch64` (2022-01-28-raspios-bullseye-armhf-lite) using Raspberry Pi 3B (revision 1.2) +* `Linux raspberrypi 5.10.92-v8+ #1514 SMP PREEMPT Mon Jan 17 17:39:38 GMT 2022 aarch64` (2022-01-28-raspios-bullseye-arm64-lite) using Raspberry Pi 3B (revision 1.2) +* `Linux ubuntu 5.15.0-1005-raspi #5-Ubuntu SMP PREEMPT Mon Apr 4 12:21:48 UTC 2022 aarch64 aarch64 aarch64 GNU/Linux` (ubuntu-22.04-preinstalled-server-arm64+raspi) using Raspberry Pi 3B (revision 1.2) + +**Note:** Build environment must have at least 1GB of memory & 1GB swap space. Check with `swapon`. + +```text +sudo apt install build-essential +sudo apt install libcurl4-openssl-dev libsqlite3-dev pkg-config git curl ldc +``` +For notifications the following is also necessary: +```text +sudo apt install libnotify-dev +``` + +### Dependencies: Gentoo +```text +sudo emerge app-portage/layman +sudo layman -a dlang +``` +Add ebuild from contrib/gentoo to a local overlay to use. + +For notifications the following is also necessary: +```text +sudo emerge x11-libs/libnotify +``` + +### Dependencies: OpenSuSE Leap 15.0 +```text +sudo zypper addrepo https://download.opensuse.org/repositories/devel:languages:D/openSUSE_Leap_15.0/devel:languages:D.repo +sudo zypper refresh +sudo zypper install gcc git libcurl-devel sqlite3-devel dmd phobos-devel phobos-devel-static +``` +For notifications the following is also necessary: +```text +sudo zypper install libnotify-devel +``` + +### Dependencies: OpenSuSE Leap 15.1 +```text +sudo zypper addrepo https://download.opensuse.org/repositories/devel:languages:D/openSUSE_Leap_15.1/devel:languages:D.repo +sudo zypper refresh +sudo zypper install gcc git libcurl-devel sqlite3-devel dmd phobos-devel phobos-devel-static +``` +For notifications the following is also necessary: +```text +sudo zypper install libnotify-devel +``` + +### Dependencies: OpenSuSE Leap 15.2 +```text +sudo zypper refresh +sudo zypper install gcc git libcurl-devel sqlite3-devel dmd phobos-devel phobos-devel-static +``` +For notifications the following is also necessary: +```text +sudo zypper install libnotify-devel +``` + +## Compilation & Installation +### High Level Steps +1. Install the platform dependencies for your Linux OS +2. Activate your DMD or LDC compiler +3. Clone the GitHub repository, run configure and make, then install +4. Deactivate your DMD or LDC compiler + +### Building using DMD Reference Compiler +Before cloning and compiling, if you have installed DMD via curl for your OS, you will need to activate DMD as per example below: +```text +Run `source ~/dlang/dmd-2.088.0/activate` in your shell to use dmd-2.088.0. +This will setup PATH, LIBRARY_PATH, LD_LIBRARY_PATH, DMD, DC, and PS1. +Run `deactivate` later on to restore your environment. +``` +Without performing this step, the compilation process will fail. + +**Note:** Depending on your DMD version, substitute `2.088.0` above with your DMD version that is installed. + +```text +git clone https://github.com/abraunegg/onedrive.git +cd onedrive +./configure +make clean; make; +sudo make install +``` + +### Build options +#### GUI Notification Support +GUI notification support can be enabled using the `configure` switch `--enable-notifications`. + +#### systemd service directory customisation support +Systemd service files are installed in the appropriate directories on the system, +as provided by `pkg-config systemd` settings. If the need for overriding the +deduced path are necessary, the two options `--with-systemdsystemunitdir` (for +the Systemd system unit location), and `--with-systemduserunitdir` (for the +Systemd user unit location) can be specified. Passing in `no` to one of these +options disabled service file installation. + +#### Additional Compiler Debug +By passing `--enable-debug` to the `configure` call, `onedrive` gets built with additional debug +information, useful (for example) to get `perf`-issued figures. + +#### Shell Completion Support +By passing `--enable-completions` to the `configure` call, shell completion functions are +installed for `bash`, `zsh` and `fish`. The installation directories are determined +as far as possible automatically, but can be overridden by passing +`--with-bash-completion-dir=`, `--with-zsh-completion-dir=`, and +`--with-fish-completion-dir=` to `configure`. + +### Building using a different compiler (for example [LDC](https://wiki.dlang.org/LDC)) +#### ARMHF Architecture (Raspbian) and ARM64 Architecture (Ubuntu 22.x / Debian 11 / Raspbian) +**Note:** The minimum LDC compiler version required to compile this application is now 1.18.0, which is not available for Debian Buster or distributions based on Debian Buster. You are advised to first upgrade your platform distribution to one that is based on Debian Bullseye (Debian 11) or later. + +**Note:** Build environment must have at least 1GB of memory & 1GB swap space. Check with `swapon`. +```text +git clone https://github.com/abraunegg/onedrive.git +cd onedrive +./configure DC=/usr/bin/ldmd2 +make clean; make +sudo make install +``` + +## Upgrading the client +If you have installed the client from a distribution package, the client will be updated when the distribution package is updated by the package maintainer and will be updated to the new application version when you perform your package update. + +If you have built the client from source, to upgrade your client, it is recommended that you first uninstall your existing 'onedrive' binary (see below), then re-install the client by re-cloning, re-compiling and re-installing the client again to install the new version. + +**Note:** Following the uninstall process will remove all client components including *all* systemd files, including any custom files created for specific access such as SharePoint Libraries. + +You can optionally choose to not perform this uninstallation step, and simply re-install the client by re-cloning, re-compiling and re-installing the client again - however the risk here is that you end up with two onedrive client binaries on your system, and depending on your system search path preferences, this will determine which binary is used. + +**Important:** Before performing any upgrade, it is highly recommended for you to stop any running systemd service if applicable to ensure that these services are restarted using the updated client version. + +Post re-install, to confirm that you have the new version of the client installed, use `onedrive --version` to determine the client version that is now installed. + +## Uninstalling the client +### Uninstalling the client if installed from distribution package +Follow your distribution documentation to uninstall the package that you installed + +### Uninstalling the client if installed and built from source +From within your GitHub repository clone, perform the following to remove the 'onedrive' binary: +```text +sudo make uninstall +``` + +If you are not upgrading your client, to remove your application state and configuration, perform the following additional step: +``` +rm -rf ~/.config/onedrive +``` +**Note:** If you are using the `--confdir option`, substitute `~/.config/onedrive` for the correct directory storing your client configuration. + +If you want to just delete the application key, but keep the items database: +```text +rm -f ~/.config/onedrive/refresh_token +``` diff --git a/docs/known-issues.md b/docs/known-issues.md new file mode 100644 index 00000000..6d970ff9 --- /dev/null +++ b/docs/known-issues.md @@ -0,0 +1,54 @@ +# Known Issues +The below are known issues with this client: + +## Moving files into different folders should not cause data to delete and be re-uploaded +**Issue Tracker:** [#876](https://github.com/abraunegg/onedrive/issues/876) + +**Description:** + +When running the client in standalone mode (`--synchronize`) moving folders that are successfully synced around between subsequent standalone syncs causes a deletion & re-upload of data to occur. + +**Explanation:** + +Technically, the client is 'working' correctly, as, when moving files, you are 'deleting' them from the current location, but copying them to the 'new location'. As the client is running in standalone sync mode, there is no way to track what OS operations have been done when the client is not running - thus, this is why the 'delete and upload' is occurring. + +**Workaround:** + +If the tracking of moving data to new local directories is requried, it is better to run the client in service mode (`--monitor`) rather than in standalone mode, as the 'move' of files can then be handled at the point when it occurs, so that the data is moved to the new location on OneDrive without the need to be deleted and re-uploaded. + +## Application 'stops' running without any visible reason +**Issue Tracker:** [#494](https://github.com/abraunegg/onedrive/issues/494), [#753](https://github.com/abraunegg/onedrive/issues/753), [#792](https://github.com/abraunegg/onedrive/issues/792), [#884](https://github.com/abraunegg/onedrive/issues/884), [#1162](https://github.com/abraunegg/onedrive/issues/1162), [#1408](https://github.com/abraunegg/onedrive/issues/1408), [#1520](https://github.com/abraunegg/onedrive/issues/1520), [#1526](https://github.com/abraunegg/onedrive/issues/1526) + +**Description:** + +When running the client and performing an upload or download operation, the application just stops working without any reason or explanation. If `echo $?` is used after the application has exited without visible reason, an error level of 141 may be provided. + +Additionally, this issue has mainly been seen when the client is operating against Microsoft's Europe Data Centre's. + +**Explanation:** + +The client is heavily dependant on Curl and OpenSSL to perform the activities with the Microsoft OneDrive service. Generally, when this issue occurs, the following is found in the HTTPS Debug Log: +``` +OpenSSL SSL_read: SSL_ERROR_SYSCALL, errno 104 +``` +The only way to determine that this is the cause of the application ceasing to work is to generate a HTTPS debug log using the following additional flags: +``` +--verbose --verbose --debug-https +``` + +This is indicative of the following: +* Some sort of flaky Internet connection somewhere between you and the OneDrive service +* Some sort of 'broken' HTTPS transparent inspection service inspecting your traffic somewhere between you and the OneDrive service + +**How to resolve:** + +The best avenue of action here are: +* Ensure your OS is as up-to-date as possible +* Get support from your OS vendor +* Speak to your ISP or Help Desk for assistance +* Open a ticket with OpenSSL and/or Curl teams to better handle this sort of connection failure +* Generate a HTTPS Debug Log for this application and open a new support request with Microsoft and provide the debug log file for their analysis. + +If you wish to diagnose this issue further, refer to the following: + +https://maulwuff.de/research/ssl-debugging.html diff --git a/docs/national-cloud-deployments.md b/docs/national-cloud-deployments.md new file mode 100644 index 00000000..6b348388 --- /dev/null +++ b/docs/national-cloud-deployments.md @@ -0,0 +1,145 @@ +# How to configure access to specific Microsoft Azure deployments +## Application Version +Before reading this document, please ensure you are running application version [![Version](https://img.shields.io/github/v/release/abraunegg/onedrive)](https://github.com/abraunegg/onedrive/releases) or greater. Use `onedrive --version` to determine what application version you are using and upgrade your client if required. + +## Process Overview +In some cases it is a requirement to utilise specific Microsoft Azure cloud deployments to conform with data and security reuqirements that requires data to reside within the geographic borders of that country. +Current national clouds that are supported are: +* Microsoft Cloud for US Government +* Microsoft Cloud Germany +* Azure and Office365 operated by 21Vianet in China + +In order to successfully use these specific Microsoft Azure deployments, the following steps are required: +1. Register an application with the Microsoft identity platform using the Azure portal +2. Configure the new application with the appropriate authentication scopes +3. Validate that the authentication / redirect URI is correct for your application registration +4. Configure the onedrive client to use the new application id as provided during application registration +5. Configure the onedrive client to use the right Microsoft Azure deployment region that your application was registered with +6. Authenticate the client + +## Step 1: Register a new application with Microsoft Azure +1. Log into your applicable Microsoft Azure Portal with your applicable Office365 identity: + +| National Cloud Environment | Microsoft Azure Portal | +|---|---| +| Microsoft Cloud for US Government | https://portal.azure.com/ | +| Microsoft Cloud Germany | https://portal.azure.com/ | +| Azure and Office365 operated by 21Vianet | https://portal.azure.cn/ | + +2. Select 'Azure Active Directory' as the service you wish to configure +3. Under 'Manage', select 'App registrations' to register a new application +4. Click 'New registration' +5. Type in the appropriate details required as per below: + +![application_registration](./images/application_registration.jpg) + +6. To save the application registration, click 'Register' and something similar to the following will be displayed: + +![application_registration_done](./images/application_registration_done.jpg) + +**Note:** The Application (client) ID UUID as displayed after client registration, is what is required as the 'application_id' for Step 4 below. + +## Step 2: Configure application authentication scopes +Configure the API permissions as per the following: + +| API / Permissions name | Type | Description | Admin consent required | +|---|---|---|---| +| Files.ReadWrite | Delegated | Have full access to user files | No | +| Files.ReadWrite.All | Delegated | Have full access to all files user can access | No | +| Sites.ReadWrite.All | Delegated | Have full access to all items in all site collections | No | +| offline_access | Delegated | Maintain access to data you have given it access to | No | + +![authentication_scopes](./images/authentication_scopes.jpg) + +## Step 3: Validate that the authentication / redirect URI is correct +Add the appropriate redirect URI for your Azure deployment: + +![authentication_response_uri](./images/authentication_response_uri.jpg) + +A valid entry for the response URI should be one of: +* https://login.microsoftonline.us/common/oauth2/nativeclient (Microsoft Cloud for US Government) +* https://login.microsoftonline.de/common/oauth2/nativeclient (Microsoft Cloud Germany) +* https://login.chinacloudapi.cn/common/oauth2/nativeclient (Azure and Office365 operated by 21Vianet in China) + +For a single-tenant application, it may be necessary to use your specific tenant id instead of "common": +* https://login.microsoftonline.us/example.onmicrosoft.us/oauth2/nativeclient (Microsoft Cloud for US Government) +* https://login.microsoftonline.de/example.onmicrosoft.de/oauth2/nativeclient (Microsoft Cloud Germany) +* https://login.chinacloudapi.cn/example.onmicrosoft.cn/oauth2/nativeclient (Azure and Office365 operated by 21Vianet in China) + +## Step 4: Configure the onedrive client to use new application registration +Update to your 'onedrive' configuration file (`~/.config/onedrive/config`) the following: +```text +application_id = "insert valid entry here" +``` + +This will reconfigure the client to use the new application registration you have created. + +**Example:** +```text +application_id = "22c49a0d-d21c-4792-aed1-8f163c982546" +``` + +## Step 5: Configure the onedrive client to use the specific Microsoft Azure deployment +Update to your 'onedrive' configuration file (`~/.config/onedrive/config`) the following: +```text +azure_ad_endpoint = "insert valid entry here" +``` + +Valid entries are: +* USL4 (Microsoft Cloud for US Government) +* USL5 (Microsoft Cloud for US Government - DOD) +* DE (Microsoft Cloud Germany) +* CN (Azure and Office365 operated by 21Vianet in China) + +This will configure your client to use the correct Azure AD and Graph endpoints as per [https://docs.microsoft.com/en-us/graph/deployments](https://docs.microsoft.com/en-us/graph/deployments) + +**Example:** +```text +azure_ad_endpoint = "USL4" +``` + +If the Microsoft Azure deployment does not support multi-tenant applications, update to your 'onedrive' configuration file (`~/.config/onedrive/config`) the following: +```text +azure_tenant_id = "insert valid entry here" +``` + +This will configure your client to use the specified tenant id in its Azure AD and Graph endpoint URIs, instead of "common". +The tenant id may be the GUID Directory ID (formatted "00000000-0000-0000-0000-000000000000"), or the fully qualified tenant name (e.g. "example.onmicrosoft.us"). +The GUID Directory ID may be located in the Azure administation page as per [https://docs.microsoft.com/en-us/onedrive/find-your-office-365-tenant-id](https://docs.microsoft.com/en-us/onedrive/find-your-office-365-tenant-id). Note that you may need to go to your national-deployment-specific administration page, rather than following the links within that document. +The tenant name may be obtained by following the PowerShell instructions on [https://docs.microsoft.com/en-us/onedrive/find-your-office-365-tenant-id](https://docs.microsoft.com/en-us/onedrive/find-your-office-365-tenant-id); it is shown as the "TenantDomain" upon completion of the "Connect-AzureAD" command. + +**Example:** +```text +azure_tenant_id = "example.onmicrosoft.us" +# or +azure_tenant_id = "0c4be462-a1ab-499b-99e0-da08ce52a2cc" +``` + +## Step 6: Authenticate the client +Run the application without any additional command switches. + +You will be asked to open a specific URL by using your web browser where you will have to login into your Microsoft Account and give the application the permission to access your files. After giving permission to the application, you will be redirected to a blank page. Copy the URI of the blank page into the application. +```text +[user@hostname ~]$ onedrive + +Authorize this app visiting: + +https://..... + +Enter the response uri: + +``` + +**Example:** +``` +[user@hostname ~]$ onedrive +Authorize this app visiting: + +https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=22c49a0d-d21c-4792-aed1-8f163c982546&scope=Files.ReadWrite%20Files.ReadWrite.all%20Sites.ReadWrite.All%20offline_access&response_type=code&redirect_uri=https://login.microsoftonline.com/common/oauth2/nativeclient + +Enter the response uri: https://login.microsoftonline.com/common/oauth2/nativeclient?code= + +Application has been successfully authorised, however no additional command switches were provided. + +Please use --help for further assistance in regards to running this application. +``` diff --git a/docs/podman.md b/docs/podman.md new file mode 100644 index 00000000..62b3af0b --- /dev/null +++ b/docs/podman.md @@ -0,0 +1,289 @@ +# Run the OneDrive Client for Linux under Podman +This client can be run as a Podman container, with 3 available container base options for you to choose from: + +| Container Base | Docker Tag | Description | i686 | x86_64 | ARMHF | AARCH64 | +|----------------|-------------|----------------------------------------------------------------|:------:|:------:|:-----:|:-------:| +| Alpine Linux | edge-alpine | Podman container based on Alpine 3.18 using 'master' |❌|✔|❌|✔| +| Alpine Linux | alpine | Podman container based on Alpine 3.18 using latest release |❌|✔|❌|✔| +| Debian | debian | Podman container based on Debian Stable using latest release |✔|✔|✔|✔| +| Debian | edge | Podman container based on Debian Stable using 'master' |✔|✔|✔|✔| +| Debian | edge-debian | Podman container based on Debian Stable using 'master' |✔|✔|✔|✔| +| Debian | latest | Podman container based on Debian Stable using latest release |✔|✔|✔|✔| +| Fedora | edge-fedora | Podman container based on Fedora 38 using 'master' |❌|✔|❌|✔| +| Fedora | fedora | Podman container based on Fedora 38 using latest release |❌|✔|❌|✔| + +These containers offer a simple monitoring-mode service for the OneDrive Client for Linux. + +The instructions below have been validated on: +* Fedora 35 + +The instructions below will utilise the 'latest' tag, however this can be substituted for any of the other docker tags from the table above if desired. + +Additionally there are specific version release tags for each release. Refer to https://hub.docker.com/r/driveone/onedrive/tags for any other Docker tags you may be interested in. + +**Note:** The below instructions for podman have only been tested as the root user while running the containers themselves as non-root users. + +## Basic Setup +### 0. Install podman using your distribution platform's instructions if not already installed +1. Ensure that SELinux has been disabled on your system. A reboot may be required to ensure that this is correctly disabled. +2. Install Podman as per requried for your platform +3. Obtain your normal, non-root user UID and GID by using the `id` command or select another non-root id to run the container as + +**NOTE:** SELinux context needs to be configured or disabled for Podman to be able to write to OneDrive host directory. + +### 1.1 Prepare data volume +The container requries 2 Podman volumes: +* Config Volume +* Data Volume + +The first volume is for your data folder and is created in the next step. This volume needs to be a path to a directory on your local filesystem, and this is where your data will be stored from OneDrive. Keep in mind that: + +* The owner of this specified folder must not be root +* Podman will attempt to change the permissions of the volume to the user the container is configured to run as + +**NOTE:** Issues occur when this target folder is a mounted folder of an external system (NAS, SMB mount, USB Drive etc) as the 'mount' itself is owed by 'root'. If this is your use case, you *must* ensure your normal user can mount your desired target without having the target mounted by 'root'. If you do not fix this, your Podman container will fail to start with the following error message: +```bash +ROOT level privileges prohibited! +``` + +### 1.2 Prepare config volume +Although not required, you can prepare the config volume before starting the container. Otherwise it will be created automatically during initial startup of the container. + +Create the config volume with the following command: +```bash +podman volume create onedrive_conf +``` + +This will create a podman volume labeled `onedrive_conf`, where all configuration of your onedrive account will be stored. You can add a custom config file and other things later. + +### 2. First run +The 'onedrive' client within the container needs to be authorized with your Microsoft account. This is achieved by initially running podman in interactive mode. + +Run the podman image with the commands below and make sure to change the value of `ONEDRIVE_DATA_DIR` to the actual onedrive data directory on your filesystem that you wish to use (e.g. `export ONEDRIVE_DATA_DIR="/home/abraunegg/OneDrive"`). + +It is a requirement that the container be run using a non-root uid and gid, you must insert a non-root UID and GID (e.g.` export ONEDRIVE_UID=1000` and export `ONEDRIVE_GID=1000`). + +```bash +export ONEDRIVE_DATA_DIR="${HOME}/OneDrive" +export ONEDRIVE_UID=1000 +export ONEDRIVE_GID=1000 +mkdir -p ${ONEDRIVE_DATA_DIR} +podman run -it --name onedrive --user "${ONEDRIVE_UID}:${ONEDRIVE_GID}" \ + -v onedrive_conf:/onedrive/conf:U,Z \ + -v "${ONEDRIVE_DATA_DIR}:/onedrive/data:U,Z" \ + driveone/onedrive:latest +``` +**Important:** The 'target' folder of `ONEDRIVE_DATA_DIR` must exist before running the podman container + +**If you plan to use podmans built in auto-updating of container images described in step 5, you must pass an additional argument to set a label during the first run.** + +**Important:** In some scenarios, 'podman' sets the configuration and data directories to a different UID & GID as specified. To resolve this situation, you must run 'podman' with the `--userns=keep-id` flag to ensure 'podman' uses the UID and GID as specified. + +The run command would look instead look like as follows: +``` +podman run -it --name onedrive --user "${ONEDRIVE_UID}:${ONEDRIVE_GID}" \ + -v onedrive_conf:/onedrive/conf:U,Z \ + -v "onedrive-test-data:/onedrive/data:U,Z" \ + -e PODMAN=1 \ + --label "io.containers.autoupdate=image" + driveone/onedrive:latest +``` + +When the Podman container successfully starts: +* You will be asked to open a specific link using your web browser +* Login to your Microsoft Account and give the application the permission +* After giving the permission, you will be redirected to a blank page +* Copy the URI of the blank page into the application prompt to authorise the application + +Once the 'onedrive' application is authorised, the client will automatically start monitoring your `ONEDRIVE_DATA_DIR` for data changes to be uploaded to OneDrive. Files stored on OneDrive will be downloaded to this location. + +If the client is working as expected, you can detach from the container with Ctrl+p, Ctrl+q. + +### 4. Podman Container Status, stop, and restart +Check if the monitor service is running + +```bash +podman ps -f name=onedrive +``` + +Show monitor run logs + +```bash +podman logs onedrive +``` + +Stop running monitor + +```bash +podman stop onedrive +``` + +Resume monitor + +```bash +podman start onedrive +``` + +Remove onedrive container + +```bash +podman rm -f onedrive +``` +## Advanced Setup + +### 5. Systemd Service & Auto Updating + +Podman supports running containers as a systemd service and also auto updating of the container images. Using the existing running container you can generate a systemd unit file to be installed by the **root** user. To have your container image auto-update with podman, it must first be created with the label `"io.containers.autoupdate=image"` mentioned in step 2. + +``` +cd /tmp +podman generate systemd --new --restart-policy on-failure --name -f onedrive +/tmp/container-onedrive.service + +# copy the generated systemd unit file to the systemd path and reload the daemon + +cp -Z ~/container-onedrive.service /usr/lib/systemd/system +systemctl daemon-reload + +#optionally enable it to startup on boot + +systemctl enable container-onedrive.service + +#check status + +systemctl status container-onedrive + +#start/stop/restart container as a systemd service + +systemctl stop container-onedrive +systemctl start container-onedrive +``` + +To update the image using podman (Ad-hoc) +``` +podman auto-update +``` + +To update the image using systemd (Automatic/Scheduled) +``` +# Enable the podman-auto-update.timer service at system start: + +systemctl enable podman-auto-update.timer + +# Start the service + +systemctl start podman-auto-update.timer + +# Containers with the autoupdate label will be updated on the next scheduled timer + +systemctl list-timers --all +``` + +### 6. Edit the config +The 'onedrive' client should run in default configuration, however you can change this default configuration by placing a custom config file in the `onedrive_conf` podman volume. First download the default config from [here](https://raw.githubusercontent.com/abraunegg/onedrive/master/config) +Then put it into your onedrive_conf volume path, which can be found with: + +```bash +podman volume inspect onedrive_conf +``` +Or you can map your own config folder to the config volume. Make sure to copy all files from the volume into your mapped folder first. + +The detailed document for the config can be found here: [Configuration](https://github.com/abraunegg/onedrive/blob/master/docs/usage.md#configuration) + +### 7. Sync multiple accounts +There are many ways to do this, the easiest is probably to +1. Create a second podman config volume (replace `Work` with your desired name): `podman volume create onedrive_conf_Work` +2. And start a second podman monitor container (again replace `Work` with your desired name): +``` +export ONEDRIVE_DATA_DIR_WORK="/home/abraunegg/OneDriveWork" +mkdir -p ${ONEDRIVE_DATA_DIR_WORK} +podman run -it --restart unless-stopped --name onedrive_work \ + -v onedrive_conf_Work:/onedrive/conf \ + -v "${ONEDRIVE_DATA_DIR_WORK}:/onedrive/data" \ + --user "${ONEDRIVE_UID}:${ONEDRIVE_GID}" \ + driveone/onedrive:latest +``` + +## Environment Variables +| Variable | Purpose | Sample Value | +| ---------------- | --------------------------------------------------- |:-------------:| +| ONEDRIVE_UID | UserID (UID) to run as | 1000 | +| ONEDRIVE_GID | GroupID (GID) to run as | 1000 | +| ONEDRIVE_VERBOSE | Controls "--verbose" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_DEBUG | Controls "--verbose --verbose" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_DEBUG_HTTPS | Controls "--debug-https" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_RESYNC | Controls "--resync" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_DOWNLOADONLY | Controls "--download-only" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_UPLOADONLY | Controls "--upload-only" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_NOREMOTEDELETE | Controls "--no-remote-delete" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_LOGOUT | Controls "--logout" switch. Default is 0 | 1 | +| ONEDRIVE_REAUTH | Controls "--reauth" switch. Default is 0 | 1 | +| ONEDRIVE_AUTHFILES | Controls "--auth-files" option. Default is "" | "authUrl:responseUrl" | +| ONEDRIVE_AUTHRESPONSE | Controls "--auth-response" option. Default is "" | See [here](https://github.com/abraunegg/onedrive/blob/master/docs/usage.md#authorize-the-application-with-your-onedrive-account) | +| ONEDRIVE_DISPLAY_CONFIG | Controls "--display-running-config" switch on onedrive sync. Default is 0 | 1 | +| ONEDRIVE_SINGLE_DIRECTORY | Controls "--single-directory" option. Default = "" | "mydir" | + +### Usage Examples +**Verbose Output:** +```bash +podman run -e ONEDRIVE_VERBOSE=1 -v onedrive_conf:/onedrive/conf:U,Z -v "${ONEDRIVE_DATA_DIR}:/onedrive/data:U,Z" --user "${ONEDRIVE_UID}:${ONEDRIVE_GID}" driveone/onedrive:latest +``` +**Debug Output:** +```bash +podman run -e ONEDRIVE_DEBUG=1 -v onedrive_conf:/onedrive/conf:U,Z -v "${ONEDRIVE_DATA_DIR}:/onedrive/data:U,Z" --user "${ONEDRIVE_UID}:${ONEDRIVE_GID}" driveone/onedrive:latest +``` +**Perform a --resync:** +```bash +podman run -e ONEDRIVE_RESYNC=1 -v onedrive_conf:/onedrive/conf:U,Z -v "${ONEDRIVE_DATA_DIR}:/onedrive/data:U,Z" --user "${ONEDRIVE_UID}:${ONEDRIVE_GID}" driveone/onedrive:latest +``` +**Perform a --resync and --verbose:** +```bash +podman run -e ONEDRIVE_RESYNC=1 -e ONEDRIVE_VERBOSE=1 -v onedrive_conf:/onedrive/conf:U,Z -v "${ONEDRIVE_DATA_DIR}:/onedrive/data:U,Z" --user "${ONEDRIVE_UID}:${ONEDRIVE_GID}" driveone/onedrive:latest +``` +**Perform a --logout and re-authenticate:** +```bash +podman run -it -e ONEDRIVE_LOGOUT=1 -v onedrive_conf:/onedrive/conf:U,Z -v "${ONEDRIVE_DATA_DIR}:/onedrive/data:U,Z" --user "${ONEDRIVE_UID}:${ONEDRIVE_GID}" driveone/onedrive:latest +``` + +## Build instructions +### Building a custom Podman image +You can also build your own image instead of pulling the one from [hub.docker.com](https://hub.docker.com/r/driveone/onedrive): +```bash +git clone https://github.com/abraunegg/onedrive +cd onedrive +podman build . -t local-onedrive -f contrib/docker/Dockerfile +``` + +There are alternate, smaller images available by building +Dockerfile-debian or Dockerfile-alpine. These [multi-stage builder pattern](https://docs.docker.com/develop/develop-images/multistage-build/) +Dockerfiles require Docker version at least 17.05. + +#### How to build and run a custom Podman image based on Debian +``` bash +podman build . -t local-ondrive-debian -f contrib/docker/Dockerfile-debian +podman run -v onedrive_conf:/onedrive/conf:U,Z -v "${ONEDRIVE_DATA_DIR}:/onedrive/data:U,Z" --user "${ONEDRIVE_UID}:${ONEDRIVE_GID}" local-ondrive-debian:latest +``` + +#### How to build and run a custom Podman image based on Alpine Linux +``` bash +podman build . -t local-ondrive-alpine -f contrib/docker/Dockerfile-alpine +podman run -v onedrive_conf:/onedrive/conf:U,Z -v "${ONEDRIVE_DATA_DIR}:/onedrive/data:U,Z" --user "${ONEDRIVE_UID}:${ONEDRIVE_GID}" local-ondrive-alpine:latest +``` + +#### How to build and run a custom Podman image for ARMHF (Raspberry Pi) +Compatible with: +* Raspberry Pi +* Raspberry Pi 2 +* Raspberry Pi Zero +* Raspberry Pi 3 +* Raspberry Pi 4 +``` bash +podman build . -t local-onedrive-armhf -f contrib/docker/Dockerfile-debian +podman run -v onedrive_conf:/onedrive/conf:U,Z -v "${ONEDRIVE_DATA_DIR}:/onedrive/data:U,Z" --user "${ONEDRIVE_UID}:${ONEDRIVE_GID}" local-onedrive-armhf:latest +``` + +#### How to build and run a custom Podman image for AARCH64 Platforms +``` bash +podman build . -t local-onedrive-aarch64 -f contrib/docker/Dockerfile-debian +podman run -v onedrive_conf:/onedrive/conf:U,Z -v "${ONEDRIVE_DATA_DIR}:/onedrive/data:U,Z" --user "${ONEDRIVE_UID}:${ONEDRIVE_GID}" local-onedrive-aarch64:latest +``` diff --git a/docs/privacy-policy.md b/docs/privacy-policy.md new file mode 100644 index 00000000..64fe1dd3 --- /dev/null +++ b/docs/privacy-policy.md @@ -0,0 +1,65 @@ +# Privacy Policy +Effective Date: May 16 2018 + +## Introduction + +This Privacy Policy outlines how OneDrive Client for Linux ("we," "our," or "us") collects, uses, and protects information when you use our software ("OneDrive Client for Linux"). We respect your privacy and are committed to ensuring the confidentiality and security of any information you provide while using the Software. + +## Information We Do Not Collect + +We want to be transparent about the fact that we do not collect any personal data, usage data, or tracking data through the Software. This means: + +1. **No Personal Data**: We do not collect any information that can be used to personally identify you, such as your name, email address, phone number, or physical address. + +2. **No Usage Data**: We do not collect data about how you use the Software, such as the features you use, the duration of your sessions, or any interactions within the Software. + +3. **No Tracking Data**: We do not use cookies or similar tracking technologies to monitor your online behavior or track your activities across websites or apps. + +## How We Use Your Information + +Since we do not collect any personal, usage, or tracking data, there is no information for us to use for any purpose. + +## Third-Party Services + +The Software may include links to third-party websites or services, but we do not have control over the privacy practices or content of these third-party services. We encourage you to review the privacy policies of any third-party services you access through the Software. + +## Children's Privacy + +Since we do not collect any personal, usage, or tracking data, there is no restriction on the use of this application by anyone under the age of 18. + +## Information You Choose to Share + +While we do not collect personal data, usage data, or tracking data through the Software, there may be instances where you voluntarily choose to share information with us, particularly when submitting bug reports. These bug reports may contain sensitive information such as account details, file names, and directory names. It's important to note that these details are included in the logs and debug logs solely for the purpose of diagnosing and resolving technical issues with the Software. + +We want to emphasize that, even in these cases, we do not have access to your actual data. The logs and debug logs provided in bug reports are used exclusively for technical troubleshooting and debugging purposes. We take measures to treat this information with the utmost care, and it is only accessible to our technical support and development teams. We do not use this information for any other purpose, and we have strict security measures in place to protect it. + +## Protecting Your Sensitive Data + +We are committed to safeguarding your sensitive data and maintaining its confidentiality. To ensure its protection: + +1. **Limited Access**: Only authorized personnel within our technical support and development teams have access to the logs and debug logs containing sensitive data, and they are trained in handling this information securely. + +2. **Data Encryption**: We use industry-standard encryption protocols to protect the transmission and storage of sensitive data. + +3. **Data Retention**: We retain bug report data for a limited time necessary for resolving the reported issue. Once the issue is resolved, we promptly delete or anonymize the data. + +4. **Security Measures**: We employ robust security measures to prevent unauthorized access, disclosure, or alteration of sensitive data. + +By submitting a bug report, you acknowledge and consent to the inclusion of sensitive information in logs and debug logs for the sole purpose of addressing technical issues with the Software. + +## Your Responsibilities + +While we take measures to protect your sensitive data, it is essential for you to exercise caution when submitting bug reports. Please refrain from including any sensitive or personally identifiable information that is not directly related to the technical issue you are reporting. You have the option to redact or obfuscate sensitive details in bug reports to further protect your data. + +## Changes to this Privacy Policy + +We may update this Privacy Policy from time to time to reflect changes in our practices or for other operational, legal, or regulatory reasons. We will notify you of any material changes by posting the updated Privacy Policy on our website or through the Software. We encourage you to review this Privacy Policy periodically. + +## Contact Us + +If you have any questions or concerns about this Privacy Policy or our privacy practices, please contact us at support@mynas.com.au or via GitHub (https://github.com/abraunegg/onedrive) + +## Conclusion + +By using the Software, you agree to the terms outlined in this Privacy Policy. If you do not agree with any part of this policy, please discontinue the use of the Software. + diff --git a/docs/sharepoint-libraries.md b/docs/sharepoint-libraries.md new file mode 100644 index 00000000..d1714d4e --- /dev/null +++ b/docs/sharepoint-libraries.md @@ -0,0 +1,228 @@ +# How to configure OneDrive SharePoint Shared Library sync +**WARNING:** Several users have reported files being overwritten causing data loss as a result of using this client with SharePoint Libraries when running as a systemd service. + +When this has been investigated, the following has been noted as potential root causes: +* File indexing application such as Baloo File Indexer or Tracker3 constantly indexing your OneDrive data +* The use of WPS Office and how it 'saves' files by deleting the existing item and replaces it with the saved data + +Additionally there could be a yet unknown bug with the client, however all debugging and data provided previously shows that an 'external' process to the 'onedrive' application modifies the files triggering the undesirable upload to occur. + +**Possible Preventative Actions:** +* Disable all File Indexing for your SharePoint Library data. It is out of scope to detail on how you should do this. +* Disable using a systemd service for syncing your SharePoint Library data. +* Do not use WPS Office to edit your documents. Use OpenOffice or LibreOffice as these do not exhibit the same 'delete to save' action that WPS Office has. + +Additionally, please use caution when using this client with SharePoint. + +## Application Version +Before reading this document, please ensure you are running application version [![Version](https://img.shields.io/github/v/release/abraunegg/onedrive)](https://github.com/abraunegg/onedrive/releases) or greater. Use `onedrive --version` to determine what application version you are using and upgrade your client if required. + +## Process Overview +Syncing a OneDrive SharePoint library requires additional configuration for your 'onedrive' client: +1. Login to OneDrive and under 'Shared Libraries' obtain the shared library name +2. Query that shared library name using the client to obtain the required configuration details +3. Create a unique local folder which will be the SharePoint Library 'root' +4. Configure the client's config file with the required 'drive_id' +5. Test the configuration using '--dry-run' +6. Sync the SharePoint Library as required + +**Note:** The `--get-O365-drive-id` process below requires a fully configured 'onedrive' configuration so that the applicable Drive ID for the given Office 365 SharePoint Shared Library can be determined. It is highly recommended that you do not use the application 'default' configuration directory for any SharePoint Site, and configure separate items for each site you wish to use. + +## 1. Listing available OneDrive SharePoint Libraries +Login to the OneDrive web interface and determine which shared library you wish to configure the client for: +![shared_libraries](./images/SharedLibraries.jpg) + +## 2. Query OneDrive API to obtain required configuration details +Run the following command using the 'onedrive' client to query the OneDrive API to obtain the required 'drive_id' of the SharePoint Library that you wish to sync: +```text +onedrive --get-O365-drive-id '' +``` +This will return something similar to the following: +```text +Configuration file successfully loaded +Configuring Global Azure AD Endpoints +Initializing the Synchronization Engine ... +Office 365 Library Name Query: +----------------------------------------------- +Site Name: +Library Name: +drive_id: b!6H_y8B...xU5 +Library URL: +----------------------------------------------- +``` +If there are no matches to the site you are attempting to search, the following will be displayed: +```text +Configuration file successfully loaded +Configuring Global Azure AD Endpoints +Initializing the Synchronization Engine ... +Office 365 Library Name Query: blah + +ERROR: The requested SharePoint site could not be found. Please check it's name and your permissions to access the site. + +The following SharePoint site names were returned: + * + * + ... + * +``` +This list of site names can be used as a basis to search for the correct site for which you are searching + +## 3. Create a new configuration directory and sync location for this SharePoint Library +Create a new configuration directory for this SharePoint Library in the following manner: +```text +mkdir ~/.config/SharePoint_My_Library_Name +``` + +Create a new local folder to store the SharePoint Library data in: +```text +mkdir ~/SharePoint_My_Library_Name +``` + +**Note:** Do not use spaces in the directory name, use '_' as a replacement + +## 4. Configure SharePoint Library config file with the required 'drive_id' & 'sync_dir' options +Download a copy of the default configuration file by downloading this file from GitHub and saving this file in the directory created above: +```text +wget https://raw.githubusercontent.com/abraunegg/onedrive/master/config -O ~/.config/SharePoint_My_Library_Name/config +``` + +Update your 'onedrive' configuration file (`~/.config/SharePoint_My_Library_Name/config`) with the local folder where you will store your data: +```text +sync_dir = "~/SharePoint_My_Library_Name" +``` + +Update your 'onedrive' configuration file(`~/.config/SharePoint_My_Library_Name/config`) with the 'drive_id' value obtained in the steps above: +```text +drive_id = "insert the drive_id value from above here" +``` +The OneDrive client will now be configured to sync this SharePoint shared library to your local system and the location you have configured. + +**Note:** After changing `drive_id`, you must perform a full re-synchronization by adding `--resync` to your existing command line. + +## 5. Validate and Test the configuration +Validate your new configuration using the `--display-config` option to validate you have configured the application correctly: +```text +onedrive --confdir="~/.config/SharePoint_My_Library_Name" --display-config +``` + +Test your new configuration using the `--dry-run` option to validate the application configuration: +```text +onedrive --confdir="~/.config/SharePoint_My_Library_Name" --synchronize --verbose --dry-run +``` + +**Note:** As this is a *new* configuration, the application will be required to be re-authorised the first time this command is run with the new configuration. + +## 6. Sync the SharePoint Library as required +Sync the SharePoint Library to your system with either `--synchronize` or `--monitor` operations: +```text +onedrive --confdir="~/.config/SharePoint_My_Library_Name" --synchronize --verbose +``` + +```text +onedrive --confdir="~/.config/SharePoint_My_Library_Name" --monitor --verbose +``` + +**Note:** As this is a *new* configuration, the application will be required to be re-authorised the first time this command is run with the new configuration. + +## 7. Enable custom systemd service for SharePoint Library +Systemd can be used to automatically run this configuration in the background, however, a unique systemd service will need to be setup for this SharePoint Library instance + +In order to automatically start syncing each SharePoint Library, you will need to create a service file for each SharePoint Library. From the applicable 'systemd folder' where the applicable systemd service file exists: +* RHEL / CentOS: `/usr/lib/systemd/system` +* Others: `/usr/lib/systemd/user` and `/lib/systemd/system` + +### Step1: Create a new systemd service file +#### Red Hat Enterprise Linux, CentOS Linux +Copy the required service file to a new name: +```text +sudo cp /usr/lib/systemd/system/onedrive.service /usr/lib/systemd/system/onedrive-SharePoint_My_Library_Name.service +``` +or +```text +sudo cp /usr/lib/systemd/system/onedrive@.service /usr/lib/systemd/system/onedrive-SharePoint_My_Library_Name@.service +``` + +#### Others such as Arch, Ubuntu, Debian, OpenSuSE, Fedora +Copy the required service file to a new name: +```text +sudo cp /usr/lib/systemd/user/onedrive.service /usr/lib/systemd/user/onedrive-SharePoint_My_Library_Name.service +``` +or +```text +sudo cp /lib/systemd/system/onedrive@.service /lib/systemd/system/onedrive-SharePoint_My_Library_Name@.service +``` + +### Step 2: Edit new systemd service file +Edit the new systemd file, updating the line beginning with `ExecStart` so that the confdir mirrors the one you used above: +```text +ExecStart=/usr/local/bin/onedrive --monitor --confdir="/full/path/to/config/dir" +``` + +Example: +```text +ExecStart=/usr/local/bin/onedrive --monitor --confdir="/home/myusername/.config/SharePoint_My_Library_Name" +``` + +**Note:** When running the client manually, `--confdir="~/.config/......` is acceptable. In a systemd configuration file, the full path must be used. The `~` must be expanded. + +### Step 3: Enable the new systemd service +Once the file is correctly editied, you can enable the new systemd service using the following commands. + +#### Red Hat Enterprise Linux, CentOS Linux +```text +systemctl enable onedrive-SharePoint_My_Library_Name +systemctl start onedrive-SharePoint_My_Library_Name +``` + +#### Others such as Arch, Ubuntu, Debian, OpenSuSE, Fedora +```text +systemctl --user enable onedrive-SharePoint_My_Library_Name +systemctl --user start onedrive-SharePoint_My_Library_Name +``` +or +```text +systemctl --user enable onedrive-SharePoint_My_Library_Name@myusername.service +systemctl --user start onedrive-SharePoint_My_Library_Name@myusername.service +``` + +### Step 4: Viewing systemd status and logs for the custom service +#### Viewing systemd service status - Red Hat Enterprise Linux, CentOS Linux +```text +systemctl status onedrive-SharePoint_My_Library_Name +``` + +#### Viewing systemd service status - Others such as Arch, Ubuntu, Debian, OpenSuSE, Fedora +```text +systemctl --user status onedrive-SharePoint_My_Library_Name +``` + +#### Viewing journalctl systemd logs - Red Hat Enterprise Linux, CentOS Linux +```text +journalctl --unit=onedrive-SharePoint_My_Library_Name -f +``` + +#### Viewing journalctl systemd logs - Others such as Arch, Ubuntu, Debian, OpenSuSE, Fedora +```text +journalctl --user --unit=onedrive-SharePoint_My_Library_Name -f +``` + +### Step 5: (Optional) Run custom systemd service at boot without user login +In some cases it may be desirable for the systemd service to start without having to login as your 'user' + +All the systemd steps above that utilise the `--user` option, will run the systemd service as your particular user. As such, the systemd service will not start unless you actually login to your system. + +To avoid this issue, you need to reconfigure your 'user' account so that the systemd services you have created will startup without you having to login to your system: +```text +loginctl enable-linger +``` + +Example: +```text +alex@ubuntu-headless:~$ loginctl enable-linger alex +``` + +## 8. Configuration for a SharePoint Library is complete +The 'onedrive' client configuration for this particular SharePoint Library is now complete. + +# How to configure multiple OneDrive SharePoint Shared Library sync +Create a new configuration as per the process above. Repeat these steps for each SharePoint Library that you wish to use. diff --git a/docs/terms-of-service.md b/docs/terms-of-service.md new file mode 100644 index 00000000..cdf7c432 --- /dev/null +++ b/docs/terms-of-service.md @@ -0,0 +1,54 @@ +# OneDrive Client for Linux - Software Service Terms of Service + +## 1. Introduction + +These Terms of Service ("Terms") govern your use of the OneDrive Client for Linux ("Application") software and related Microsoft OneDrive services ("Service") provided by Microsoft. By accessing or using the Service, you agree to comply with and be bound by these Terms. If you do not agree to these Terms, please do not use the Service. + +## 2. License Compliance + +The OneDrive Client for Linux software is licensed under the GNU General Public License, version 3.0 (the "GPLv3"). Your use of the software must comply with the terms and conditions of the GPLv3. A copy of the GPLv3 can be found here: https://www.gnu.org/licenses/gpl-3.0.en.html + +## 3. Use of the Service + +### 3.1. Access and Accounts + +You may need to create an account or provide personal information to access certain features of the Service. You are responsible for maintaining the confidentiality of your account information and are solely responsible for all activities that occur under your account. + +### 3.2. Prohibited Activities + +You agree not to: + +- Use the Service in any way that violates applicable laws or regulations. +- Use the Service to engage in any unlawful, harmful, or fraudulent activity. +- Use the Service in any manner that disrupts, damages, or impairs the Service. + +## 4. Intellectual Property + +The OneDrive Client for Linux software is subject to the GPLv3, and you must respect all copyrights, trademarks, and other intellectual property rights associated with the software. Any contributions you make to the software must also comply with the GPLv3. + +## 5. Disclaimer of Warranties + +The OneDrive Client for Linux software is provided "as is" without any warranties, either expressed or implied. We do not guarantee that the use of the Application will be error-free or uninterrupted. + +Microsoft is not responsible for OneDrive Client for Linux. Any issues or problems with OneDrive Client for Linux should be raised on GitHub at https://github.com/abraunegg/onedrive or email support@mynas.com.au + +OneDrive Client for Linux is not responsible for the Microsoft OneDrive Service or the Microsoft Graph API Service that this Application utilizes. Any issue with either Microsoft OneDrive or Microsoft Graph API should be raised with Microsoft via their support channel in your country. + +## 6. Limitation of Liability + +To the fullest extent permitted by law, we shall not be liable for any direct, indirect, incidental, special, consequential, or punitive damages, or any loss of profits or revenues, whether incurred directly or indirectly, or any loss of data, use, goodwill, or other intangible losses, resulting from (a) your use or inability to use the Service, or (b) any other matter relating to the Service. + +This limitiation of liability explicitly relates to the use of the OneDrive Client for Linux software and does not affect your rights under the GPLv3. + +## 7. Changes to Terms + +We reserve the right to update or modify these Terms at any time without prior notice. Any changes will be effective immediately upon posting on GitHub. Your continued use of the Service after the posting of changes constitutes your acceptance of such changes. Changes can be reviewed on GitHub. + +## 8. Governing Law + +These Terms shall be governed by and construed in accordance with the laws of Australia, without regard to its conflict of law principles. + +## 9. Contact Us + +If you have any questions or concerns about these Terms, please contact us at https://github.com/abraunegg/onedrive or email support@mynas.com.au + diff --git a/docs/ubuntu-package-install.md b/docs/ubuntu-package-install.md new file mode 100644 index 00000000..abdcace9 --- /dev/null +++ b/docs/ubuntu-package-install.md @@ -0,0 +1,383 @@ +# Installation of 'onedrive' package on Debian and Ubuntu + +This document covers the appropriate steps to install the 'onedrive' client using the provided packages for Debian and Ubuntu. + +#### Important information for all Ubuntu and Ubuntu based distribution users: +This information is specifically for the following platforms and distributions: + +* Lubuntu +* Linux Mint +* POP OS +* Peppermint OS +* Raspbian +* Ubuntu + +Whilst there are [onedrive](https://packages.ubuntu.com/search?keywords=onedrive&searchon=names&suite=all§ion=all) Universe packages available for Ubuntu, do not install 'onedrive' from these Universe packages. The default Ubuntu Universe packages are out-of-date and are not supported and should not be used. + +## Determine which instructions to use +Ubuntu and its clones are based on various different releases, thus, you must use the correct instructions below, otherwise you may run into package dependancy issues and will be unable to install the client. + +### Step 1: Remove any configured PPA and associated 'onedrive' package and systemd service files +Many Internet 'help' pages provide inconsistent details on how to install the OneDrive Client for Linux. A number of these websites continue to point users to install the client via the yann1ck PPA repository however this PPA no longer exists and should not be used. + +To remove the PPA repository and the older client, perform the following actions: +```text +sudo apt remove onedrive +sudo add-apt-repository --remove ppa:yann1ck/onedrive +``` + +Additionally, Ubuntu and its clones have a bad habit of creating a 'default' systemd service file when installing the 'onedrive' package so that the client will automatically run the client post being authenticated. This systemd entry is erroneous and needs to be removed. +``` +Created symlink /etc/systemd/user/default.target.wants/onedrive.service → /usr/lib/systemd/user/onedrive.service. +``` +To remove this symbolic link, run the following command: +``` +sudo rm /etc/systemd/user/default.target.wants/onedrive.service +``` + +### Step 2: Ensure your system is up-to-date +Use a script, similar to the following to ensure your system is updated correctly: +```text +#!/bin/bash +rm -rf /var/lib/dpkg/lock-frontend +rm -rf /var/lib/dpkg/lock +apt-get update +apt-get upgrade -y +apt-get dist-upgrade -y +apt-get autoremove -y +apt-get autoclean -y +``` + +Run this script as 'root' by using `su -` to elevate to 'root'. Example below: +```text +Welcome to Ubuntu 20.04.1 LTS (GNU/Linux 5.4.0-48-generic x86_64) + + * Documentation: https://help.ubuntu.com + * Management: https://landscape.canonical.com + * Support: https://ubuntu.com/advantage + +425 updates can be installed immediately. +208 of these updates are security updates. +To see these additional updates run: apt list --upgradable + +Your Hardware Enablement Stack (HWE) is supported until April 2025. +Last login: Thu Jan 20 14:21:48 2022 from my.ip.address +alex@ubuntu-20-LTS:~$ su - +Password: +root@ubuntu-20-LTS:~# ls -la +total 28 +drwx------ 3 root root 4096 Oct 10 2020 . +drwxr-xr-x 20 root root 4096 Oct 10 2020 .. +-rw------- 1 root root 175 Jan 20 14:23 .bash_history +-rw-r--r-- 1 root root 3106 Dec 6 2019 .bashrc +drwx------ 2 root root 4096 Apr 23 2020 .cache +-rw-r--r-- 1 root root 161 Dec 6 2019 .profile +-rwxr-xr-x 1 root root 174 Oct 10 2020 update-os.sh +root@ubuntu-20-LTS:~# cat update-os.sh +#!/bin/bash +rm -rf /var/lib/dpkg/lock-frontend +rm -rf /var/lib/dpkg/lock +apt-get update +apt-get upgrade -y +apt-get dist-upgrade -y +apt-get autoremove -y +apt-get autoclean -y +root@ubuntu-20-LTS:~# ./update-os.sh +Hit:1 http://au.archive.ubuntu.com/ubuntu focal InRelease +Hit:2 http://au.archive.ubuntu.com/ubuntu focal-updates InRelease +Hit:3 http://au.archive.ubuntu.com/ubuntu focal-backports InRelease +Hit:4 http://security.ubuntu.com/ubuntu focal-security InRelease +Reading package lists... 96% +... +Sourcing file `/etc/default/grub' +Sourcing file `/etc/default/grub.d/init-select.cfg' +Generating grub configuration file ... +Found linux image: /boot/vmlinuz-5.13.0-27-generic +Found initrd image: /boot/initrd.img-5.13.0-27-generic +Found linux image: /boot/vmlinuz-5.4.0-48-generic +Found initrd image: /boot/initrd.img-5.4.0-48-generic +Found memtest86+ image: /boot/memtest86+.elf +Found memtest86+ image: /boot/memtest86+.bin +done +Removing linux-modules-5.4.0-26-generic (5.4.0-26.30) ... +Processing triggers for libc-bin (2.31-0ubuntu9.2) ... +Reading package lists... Done +Building dependency tree +Reading state information... Done +root@ubuntu-20-LTS:~# +``` + +Reboot your system after running this process before continuing with Step 3. +```text +reboot +``` + +### Step 3: Determine what your OS is based on +Determine what your OS is based on. To do this, run the following command: +```text +lsb_release -a +``` +**Example:** +```text +alex@ubuntu-system:~$ lsb_release -a +No LSB modules are available. +Distributor ID: Ubuntu +Description: Ubuntu 22.04 LTS +Release: 22.04 +Codename: jammy +``` + +### Step 4: Pick the correct instructions to use +If required, review the table below based on your 'lsb_release' information to pick the appropriate instructions to use: + +| Release & Codename | Instructions to use | +|--------------------|---------------------| +| Linux Mint 19.x | This platform is End-of-Life (EOL) and no longer supported. You must upgrade to Linux Mint 21.x | +| Linux Mint 20.x | Use [Ubuntu 20.04](#distribution-ubuntu-2004) instructions below | +| Linux Mint 21.x | Use [Ubuntu 22.04](#distribution-ubuntu-2204) instructions below | +| Debian 9 | This platform is End-of-Life (EOL) and no longer supported. You must upgrade to Debian 11 | +| Debian 10 | You must build from source or upgrade your Operating System to Debian 11 | +| Debian 11 | Use [Debian 11](#distribution-debian-11) instructions below | +| Debian 12 | Use [Debian 12](#distribution-debian-12) instructions below | +| Raspbian GNU/Linux 10 | You must build from source or upgrade your Operating System to Raspbian GNU/Linux 11 | +| Raspbian GNU/Linux 11 | Use [Debian 11](#distribution-debian-11) instructions below | +| Raspbian GNU/Linux 12 | Use [Debian 12](#distribution-debian-12) instructions below | +| Ubuntu 18.04 / Bionic | This platform is End-of-Life (EOL) and no longer supported. You must upgrade to Ubuntu 22.x | +| Ubuntu 20.04 / Focal | Use [Ubuntu 20.04](#distribution-ubuntu-2004) instructions below | +| Ubuntu 21.04 / Hirsute | Use [Ubuntu 21.04](#distribution-ubuntu-2104) instructions below | +| Ubuntu 21.10 / Impish | Use [Ubuntu 21.10](#distribution-ubuntu-2110) instructions below | +| Ubuntu 22.04 / Jammy | Use [Ubuntu 22.04](#distribution-ubuntu-2204) instructions below | +| Ubuntu 22.10 / Kinetic | Use [Ubuntu 22.10](#distribution-ubuntu-2210) instructions below | +| Ubuntu 23.04 / Lunar | Use [Ubuntu 23.04](#distribution-ubuntu-2304) instructions below | + +## Distribution Package Install Instructions + +### Distribution: Debian 11 +The packages support the following platform architectures: +|  i686  | x86_64 | ARMHF | AARCH64 | +|:----:|:------:|:-----:|:-------:| +|✔|✔|✔|✔| | + +#### Step 1: Add the OpenSuSE Build Service repository release key +Add the OpenSuSE Build Service repository release key using the following command: +```text +wget -qO - https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/Debian_11/Release.key | gpg --dearmor | sudo tee /usr/share/keyrings/obs-onedrive.gpg > /dev/null +``` + +#### Step 2: Add the OpenSuSE Build Service repository +Add the OpenSuSE Build Service repository using the following command: +```text +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/obs-onedrive.gpg] https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/Debian_11/ ./" | sudo tee /etc/apt/sources.list.d/onedrive.list +``` + +#### Step 3: Update your apt package cache +Run: `sudo apt-get update` + +#### Step 4: Install 'onedrive' +Run: `sudo apt install --no-install-recommends --no-install-suggests onedrive` + +#### Step 5: Read 'Known Issues' with these packages +Read and understand the [known issues](#known-issues-with-installing-from-the-above-packages) with these packages below, taking any action that is needed. + +### Distribution: Debian 12 +The packages support the following platform architectures: +|  i686  | x86_64 | ARMHF | AARCH64 | +|:----:|:------:|:-----:|:-------:| +|✔|✔|✔|✔| | + +#### Step 1: Add the OpenSuSE Build Service repository release key +Add the OpenSuSE Build Service repository release key using the following command: +```text +wget -qO - https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/Debian_12/Release.key | gpg --dearmor | sudo tee /usr/share/keyrings/obs-onedrive.gpg > /dev/null +``` + +#### Step 2: Add the OpenSuSE Build Service repository +Add the OpenSuSE Build Service repository using the following command: +```text +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/obs-onedrive.gpg] https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/Debian_12/ ./" | sudo tee /etc/apt/sources.list.d/onedrive.list +``` + +#### Step 3: Update your apt package cache +Run: `sudo apt-get update` + +#### Step 4: Install 'onedrive' +Run: `sudo apt install --no-install-recommends --no-install-suggests onedrive` + +#### Step 5: Read 'Known Issues' with these packages +Read and understand the [known issues](#known-issues-with-installing-from-the-above-packages) with these packages below, taking any action that is needed. + +### Distribution: Ubuntu 20.04 +The packages support the following platform architectures: +|  i686  | x86_64 | ARMHF | AARCH64 | +|:----:|:------:|:-----:|:-------:| +❌|✔|✔|✔| | + +#### Step 1: Add the OpenSuSE Build Service repository release key +Add the OpenSuSE Build Service repository release key using the following command: +```text +wget -qO - https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_20.04/Release.key | sudo apt-key add - +``` + +#### Step 2: Add the OpenSuSE Build Service repository +Add the OpenSuSE Build Service repository using the following command: +```text +echo 'deb https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_20.04/ ./' | sudo tee /etc/apt/sources.list.d/onedrive.list +``` + +#### Step 3: Update your apt package cache +Run: `sudo apt-get update` + +#### Step 4: Install 'onedrive' +Run: `sudo apt install --no-install-recommends --no-install-suggests onedrive` + +#### Step 5: Read 'Known Issues' with these packages +Read and understand the [known issues](#known-issues-with-installing-from-the-above-packages) with these packages below, taking any action that is needed. + +### Distribution: Ubuntu 21.04 +The packages support the following platform architectures: +|  i686  | x86_64 | ARMHF | AARCH64 | +|:----:|:------:|:-----:|:-------:| +❌|✔|✔|✔| | + +#### Step 1: Add the OpenSuSE Build Service repository release key +Add the OpenSuSE Build Service repository release key using the following command: +```text +wget -qO - https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_21.04/Release.key | gpg --dearmor | sudo tee /usr/share/keyrings/obs-onedrive.gpg > /dev/null +``` + +#### Step 2: Add the OpenSuSE Build Service repository +Add the OpenSuSE Build Service repository using the following command: +```text +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/obs-onedrive.gpg] https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_21.04/ ./" | sudo tee /etc/apt/sources.list.d/onedrive.list +``` + +#### Step 3: Update your apt package cache +Run: `sudo apt-get update` + +#### Step 4: Install 'onedrive' +Run: `sudo apt install --no-install-recommends --no-install-suggests onedrive` + +#### Step 5: Read 'Known Issues' with these packages +Read and understand the [known issues](#known-issues-with-installing-from-the-above-packages) with these packages below, taking any action that is needed. + +### Distribution: Ubuntu 21.10 +The packages support the following platform architectures: +|  i686  | x86_64 | ARMHF | AARCH64 | +|:----:|:------:|:-----:|:-------:| +❌|✔|✔|✔| | + +#### Step 1: Add the OpenSuSE Build Service repository release key +Add the OpenSuSE Build Service repository release key using the following command: +```text +wget -qO - https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_21.10/Release.key | gpg --dearmor | sudo tee /usr/share/keyrings/obs-onedrive.gpg > /dev/null +``` + +#### Step 2: Add the OpenSuSE Build Service repository +Add the OpenSuSE Build Service repository using the following command: +```text +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/obs-onedrive.gpg] https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_21.10/ ./" | sudo tee /etc/apt/sources.list.d/onedrive.list +``` + +#### Step 3: Update your apt package cache +Run: `sudo apt-get update` + +#### Step 4: Install 'onedrive' +Run: `sudo apt install --no-install-recommends --no-install-suggests onedrive` + +#### Step 5: Read 'Known Issues' with these packages +Read and understand the [known issues](#known-issues-with-installing-from-the-above-packages) with these packages below, taking any action that is needed. + +### Distribution: Ubuntu 22.04 +The packages support the following platform architectures: +|  i686  | x86_64 | ARMHF | AARCH64 | +|:----:|:------:|:-----:|:-------:| +❌|✔|✔|✔| | + +#### Step 1: Add the OpenSuSE Build Service repository release key +Add the OpenSuSE Build Service repository release key using the following command: +```text +wget -qO - https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_22.04/Release.key | gpg --dearmor | sudo tee /usr/share/keyrings/obs-onedrive.gpg > /dev/null +``` + +#### Step 2: Add the OpenSuSE Build Service repository +Add the OpenSuSE Build Service repository using the following command: +```text +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/obs-onedrive.gpg] https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_22.04/ ./" | sudo tee /etc/apt/sources.list.d/onedrive.list +``` + +#### Step 3: Update your apt package cache +Run: `sudo apt-get update` + +#### Step 4: Install 'onedrive' +Run: `sudo apt install --no-install-recommends --no-install-suggests onedrive` + +#### Step 5: Read 'Known Issues' with these packages +Read and understand the [known issues](#known-issues-with-installing-from-the-above-packages) with these packages below, taking any action that is needed. + +### Distribution: Ubuntu 22.10 +The packages support the following platform architectures: +|  i686  | x86_64 | ARMHF | AARCH64 | +|:----:|:------:|:-----:|:-------:| +❌|✔|✔|✔| | + +#### Step 1: Add the OpenSuSE Build Service repository release key +Add the OpenSuSE Build Service repository release key using the following command: +```text +wget -qO - https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_22.10/Release.key | gpg --dearmor | sudo tee /usr/share/keyrings/obs-onedrive.gpg > /dev/null +``` + +#### Step 2: Add the OpenSuSE Build Service repository +Add the OpenSuSE Build Service repository using the following command: +```text +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/obs-onedrive.gpg] https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_22.10/ ./" | sudo tee /etc/apt/sources.list.d/onedrive.list +``` + +#### Step 3: Update your apt package cache +Run: `sudo apt-get update` + +#### Step 4: Install 'onedrive' +Run: `sudo apt install --no-install-recommends --no-install-suggests onedrive` + +#### Step 5: Read 'Known Issues' with these packages +Read and understand the [known issues](#known-issues-with-installing-from-the-above-packages) with these packages below, taking any action that is needed. + +### Distribution: Ubuntu 23.04 +The packages support the following platform architectures: +|  i686  | x86_64 | ARMHF | AARCH64 | +|:----:|:------:|:-----:|:-------:| +❌|✔|✔|✔| | + +#### Step 1: Add the OpenSuSE Build Service repository release key +Add the OpenSuSE Build Service repository release key using the following command: +```text +wget -qO - https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_23.04/Release.key | gpg --dearmor | sudo tee /usr/share/keyrings/obs-onedrive.gpg > /dev/null +``` + +#### Step 2: Add the OpenSuSE Build Service repository +Add the OpenSuSE Build Service repository using the following command: +```text +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/obs-onedrive.gpg] https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_23.04/ ./" | sudo tee /etc/apt/sources.list.d/onedrive.list +``` + +#### Step 3: Update your apt package cache +Run: `sudo apt-get update` + +#### Step 4: Install 'onedrive' +Run: `sudo apt install --no-install-recommends --no-install-suggests onedrive` + +#### Step 5: Read 'Known Issues' with these packages +Read and understand the [known issues](#known-issues-with-installing-from-the-above-packages) with these packages below, taking any action that is needed. + +## Known Issues with Installing from the above packages + +### 1. The client may segfault | core-dump when exiting +When the client is run in `--monitor` mode manually, or when using the systemd service, the client may segfault on exit. + +This issue is caused by the way the 'onedrive' packages are built using the distribution LDC package & the default distribution compiler options which is the root cause for this issue. Refer to: https://bugs.launchpad.net/ubuntu/+source/ldc/+bug/1895969 + +**Additional references:** +* https://github.com/abraunegg/onedrive/issues/1053 +* https://github.com/abraunegg/onedrive/issues/1609 + +**Resolution Options:** +* Uninstall the package and build client from source diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 00000000..8fc1a063 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,848 @@ +# Using the OneDrive Client for Linux +## Application Version +Before reading this document, please ensure you are running application version [![Version](https://img.shields.io/github/v/release/abraunegg/onedrive)](https://github.com/abraunegg/onedrive/releases) or greater. Use `onedrive --version` to determine what application version you are using and upgrade your client if required. + +## Table of Contents +TABLE OF CONTENTS GOES HERE + +## Important Notes +### Upgrading from the 'skilion' Client +The 'skilion' version has a significant number of issues in how it manages the local sync state. When upgrading from the 'skilion' client to this client, it's recommended to stop any service or OneDrive process that may be running. Once all OneDrive services are stopped, make sure to remove any old client binaries from your system. + +Furthermore, if you're using a 'config' file within your configuration directory (`~/.config/onedrive/`), please ensure that you update the `skip_file = ` option as shown below: + +**Invalid 'skilion' configuration:** +```text +skip_file = ".*|~*" +``` +**Minimum valid configuration:** +```text +skip_file = "~*" +``` +**Default valid configuration:** +```text +skip_file = "~*|.~*|*.tmp|*.swp|*.partial" +``` + +Avoid using a 'skip_file' entry of `.*` as it may prevent the correct detection of local changes to process. The configuration values for 'skip_file' will be checked for validity, and if there is an issue, the following error message will be displayed: +```text +ERROR: Invalid skip_file entry '.*' detected +``` + +### Naming Conventions for Local Files and Folders +In the synchronisation directory, it's crucial to adhere to the [Windows naming conventions](https://docs.microsoft.com/windows/win32/fileio/naming-a-file) for your files and folders. + +If you happen to have two files with the same names but different capitalisation, our application will make an effort to handle it. However, when there's a clash within the namespace, the file causing the conflict won't be synchronised. Please note that this behaviour is intentional and won't be addressed. + +### Compatibility with curl +If your system uses curl < 7.47.0, curl will default to HTTP/1.1 for HTTPS operations, and the client will follow suit, using HTTP/1.1. + +For systems running curl >= 7.47.0 and < 7.62.0, curl will prefer HTTP/2 for HTTPS, but it will still use HTTP/1.1 as the default for these operations. The client will employ HTTP/1.1 for HTTPS operations as well. + +However, if your system employs curl >= 7.62.0, curl will, by default, prioritise HTTP/2 over HTTP/1.1. In this case, the client will utilise HTTP/2 for most HTTPS operations and stick with HTTP/1.1 for others. Please note that this distinction is governed by the OneDrive platform, not our client. + +If you explicitly want to use HTTP/1.1, you can do so by using the `--force-http-11` flag or setting the configuration option `force_http_11 = "true"`. This will compel the application to exclusively use HTTP/1.1. Otherwise, all client operations will align with the curl default settings for your distribution. + +## First Steps +### Authorise the Application with Your Microsoft OneDrive Account +Once you've installed the application, you'll need to authorise it using your Microsoft OneDrive Account. This can be done by simply running the application without any additional command switches. + +Please be aware that some companies may require you to explicitly add this app to the [Microsoft MyApps portal](https://myapps.microsoft.com/). To add an approved app to your apps, click on the ellipsis in the top-right corner and select "Request new apps." On the next page, you can add this app. If it's not listed, you should make a request through your IT department. + +When you run the application for the first time, you'll be prompted to open a specific URL using your web browser, where you'll need to log in to your Microsoft Account and grant the application permission to access your files. After granting permission to the application, you'll be redirected to a blank page. Simply copy the URI from the blank page and paste it into the application. + +**Example:** +```text +[user@hostname ~]$ onedrive +Authorise this app by visiting: + +https://login.microsoftonline.com/common/oauth2/v2.0/authorise?client_id=22c49a0d-d21c-4792-aed1-8f163c982546&scope=Files.ReadWrite%20Files.ReadWrite.all%20Sites.ReadWrite.All%20offline_access&response_type=code&redirect_uri=https://login.microsoftonline.com/common/oauth2/nativeclient + +Enter the response URI from your browser: https://login.microsoftonline.com/common/oauth2/nativeclient?code= + +The application has been successfully authorised, but no additional command switches were provided. + +Please use 'onedrive --help' for further assistance on how to run this application. +``` + +### Display Your Applicable Runtime Configuration +To verify the configuration that the application will use, use the following command: +```text +onedrive --display-config +``` +This command will display all the relevant runtime interpretations of the options and configurations you are using. An example output is as follows: +```text +Reading configuration file: /home/user/.config/onedrive/config +Configuration file successfully loaded +onedrive version = vX.Y.Z-A-bcdefghi +Config path = /home/user/.config/onedrive +Config file found in config path = true +Config option 'drive_id' = +Config option 'sync_dir' = ~/OneDrive +... +Config option 'webhook_enabled' = false +``` + +### Understanding OneDrive Client for Linux Operational Modes +There are two modes of operation when using the client: +1. Standalone sync mode that performs a single sync action against Microsoft OneDrive. +2. Ongoing sync mode that continuously syncs your data with Microsoft OneDrive. + +#### Standalone Synchronisation Operational Mode (Standalone Mode) +This method of use can be employed by issuing the following option to the client: +```text +onedrive --sync +``` +For simplicity, this can be reduced to the following: +```text +onedrive -s +``` + +#### Ongoing Synchronisation Operational Mode (Monitor Mode) +This method of use can be utilised by issuing the following option to the client: +```text +onedrive --monitor +``` +For simplicity, this can be shortened to the following: +```text +onedrive -m +``` +**Note:** This method of use is typically employed when enabling a systemd service to run the application in the background. + +Two common errors can occur when using monitor mode: +* Initialisation failure +* Unable to add a new inotify watch + +Both of these errors are local environment issues, where the following system variables need to be increased as the current system values are potentially too low: +* `fs.file-max` +* `fs.inotify.max_user_watches` + +To determine what the existing values are on your system, use the following commands: +```text +sysctl fs.file-max +sysctl fs.inotify.max_user_watches +``` +Alternatively, when running the client with increased verbosity (see below), the client will display what the current configured system maximum values are: +```text +... +All application operations will be performed in: /home/user/OneDrive +OneDrive synchronisation interval (seconds): 300 +Maximum allowed open files: 393370 <-- This is the fs.file-max value +Maximum allowed inotify watches: 29374 <-- This is the fs.inotify.max_user_watches value +Initialising filesystem inotify monitoring ... +... +``` +To determine what value to change to, you need to count all the files and folders in your configured 'sync_dir': +```text +cd /path/to/your/sync/dir +ls -laR | wc -l +``` + +To make a change to these variables using your file and folder count, use the following process: +```text +sudo sysctl fs.file-max= +sudo sysctl fs.inotify.max_user_watches= +``` +Once these values are changed, you will need to restart your client so that the new values are detected and used. + +To make these changes permanent on your system, refer to your OS reference documentation. + +### Increasing application logging level +When running a sync (`--sync`) or using monitor mode (`--monitor`), it may be desirable to see additional information regarding the progress and operation of the client. For example, for a `--sync` command, this would be: +```text +onedrive --sync --verbose +``` +Furthermore, for simplicity, this can be simplified to the following: +``` +onedrive -s -v +``` +Adding `--verbose` twice will enable debug logging output. This is generally required when raising a bug report or needing to understand a problem. + +### Testing your configuration +You can test your configuration by utilising the `--dry-run` CLI option. No files will be downloaded, uploaded, or removed; however, the application will display what 'would' have occurred. For example: +```text +onedrive --sync --verbose --dry-run +Reading configuration file: /home/user/.config/onedrive/config +Configuration file successfully loaded +Using 'user' Config Dir: /home/user/.config/onedrive +DRY-RUN Configured. Output below shows what 'would' have occurred. +DRY-RUN: Copying items.sqlite3 to items-dryrun.sqlite3 to use for dry run operations +DRY RUN: Not creating backup config file as --dry-run has been used +DRY RUN: Not updating hash files as --dry-run has been used +Checking Application Version ... +Attempting to initialise the OneDrive API ... +Configuring Global Azure AD Endpoints +The OneDrive API was initialised successfully +Opening the item database ... +Sync Engine Initialised with new Onedrive API instance +Application version: vX.Y.Z-A-bcdefghi +Account Type: +Default Drive ID: +Default Root ID: +Remaining Free Space: 1058488129 KB +All application operations will be performed in: /home/user/OneDrive +Fetching items from the OneDrive API for Drive ID: .. +... +Performing a database consistency and integrity check on locally stored data ... +Processing DB entries for this Drive ID: +Processing ~/OneDrive +The directory has not changed +... +Scanning local filesystem '~/OneDrive' for new data to upload ... +... +Performing a final true-up scan of online data from Microsoft OneDrive +Fetching items from the OneDrive API for Drive ID: .. + +Sync with Microsoft OneDrive is complete +``` + +### Performing a sync with Microsoft OneDrive +By default, all files are downloaded in `~/OneDrive`. This download location is controlled by the 'sync_dir' config option. + +After authorising the application, a sync of your data can be performed by running: +```text +onedrive --sync +``` +This will synchronise files from your Microsoft OneDrive account to your `~/OneDrive` local directory or to your specified 'sync_dir' location. + +If you prefer to use your local files as stored in `~/OneDrive` as your 'source of truth,' use the following sync command: +```text +onedrive --sync --local-first +``` + +### Performing a single directory synchronisation with Microsoft OneDrive +In some cases, it may be desirable to synchronise a single directory under ~/OneDrive without having to change your client configuration. To do this, use the following command: +```text +onedrive --sync --single-directory '' +``` + +**Example:** If the full path is `~/OneDrive/mydir`, the command would be `onedrive --sync --single-directory 'mydir'` + +### Performing a 'one-way' download synchronisation with Microsoft OneDrive +In some cases, it may be desirable to 'download only' from Microsoft OneDrive. To do this, use the following command: +```text +onedrive --sync --download-only +``` +This will download all the content from Microsoft OneDrive to your `~/OneDrive` location. Any files that are deleted online remain locally and will not be removed. + +However, in some circumstances, it may be desirable to clean up local files that have been removed online. To do this, use the following command: + +```text +onedrive --sync --download-only --cleanup-local-files +``` + +### Performing a 'one-way' upload synchronisation with Microsoft OneDrive +In some cases, it may be desirable to 'upload only' to Microsoft OneDrive. To do this, use the following command: +```text +onedrive --sync --upload-only +``` +**Note:** If a file or folder is present on Microsoft OneDrive, which was previously synchronised and now does not exist locally, that item will be removed from Microsoft OneDrive online. If the data on Microsoft OneDrive should be kept, the following should be used: +```text +onedrive --sync --upload-only --no-remote-delete +``` +**Note:** The operation of 'upload only' does not request data from Microsoft OneDrive about what 'other' data exists online. The client only knows about the data that 'this' client uploaded, thus any files or folders created or uploaded outside of this client will remain untouched online. + +### Performing a selective synchronisation via 'sync_list' file +Selective synchronisation allows you to sync only specific files and directories. +To enable selective synchronisation, create a file named `sync_list` in your application configuration directory (default is `~/.config/onedrive`). + +Important points to understand before using 'sync_list'. +* 'sync_list' excludes _everything_ by default on OneDrive. +* 'sync_list' follows an _"exclude overrides include"_ rule, and requires **explicit inclusion**. +* Order exclusions before inclusions, so that anything _specifically included_ is included. +* How and where you place your `/` matters for excludes and includes in subdirectories. + +Each line of the file represents a relative path from your `sync_dir`. All files and directories not matching any line of the file will be skipped during all operations. + +Additionally, the use of `/` is critically important to determine how a rule is interpreted. It is very similar to `**` wildcards, for those that are familiar with globbing patterns. +Here is an example of `sync_list`: +```text +# sync_list supports comments +# +# The ordering of entries is highly recommended - exclusions before inclusions +# +# Exclude temp folder(s) or file(s) under Documents folder(s), anywhere in OneDrive +!Documents/temp* +# +# Exclude secret data folder in root directory only +!/Secret_data/* +# +# Include everything else in root directory +/* +# +# Include my Backup folder(s) or file(s) anywhere on OneDrive +Backup +# +# Include my Backup folder in root +/Backup/ +# +# Include Documents folder(s) anywhere in OneDrive +Documents/ +# +# Include all PDF files in Documents folder(s), anywhere in OneDrive +Documents/*.pdf +# +# Include this single document in Documents folder(s), anywhere in OneDrive +Documents/latest_report.docx +# +# Include all Work/Project directories or files, inside 'Work' folder(s), anywhere in OneDrive +Work/Project* +# +# Include all "notes.txt" files, anywhere in OneDrive +notes.txt +# +# Include /Blender in the ~OneDrive root but not if elsewhere in OneDrive +/Blender +# +# Include these directories(or files) in 'Pictures' folder(s), that have a space in their name +Pictures/Camera Roll +Pictures/Saved Pictures +# +# Include these names if they match any file or folder +Cinema Soc +Codes +Textbooks +Year 2 +``` +The following are supported for pattern matching and exclusion rules: +* Use the `*` to wildcard select any characters to match for the item to be included +* Use either `!` or `-` characters at the start of the line to exclude an otherwise included item + +**Note:** When enabling the use of 'sync_list,' utilise the `--display-config` option to validate that your configuration will be used by the application, and test your configuration by adding `--dry-run` to ensure the client will operate as per your requirement. + +**Note:** After changing the sync_list, you must perform a full re-synchronisation by adding `--resync` to your existing command line - for example: `onedrive --sync --resync` + +**Note:** In some circumstances, it may be required to sync all the individual files within the 'sync_dir', but due to frequent name change / addition / deletion of these files, it is not desirable to constantly change the 'sync_list' file to include / exclude these files and force a resync. To assist with this, enable the following in your configuration file: +```text +sync_root_files = "true" +``` +This will tell the application to sync any file that it finds in your 'sync_dir' root by default, negating the need to constantly update your 'sync_list' file. + +### Performing a --resync +If you alter any of the subsequent configuration items, you will be required to execute a `--resync` to make sure your client is syncing your data with the updated configuration: +* drive_id +* sync_dir +* skip_file +* skip_dir +* skip_dotfiles +* skip_symlinks +* sync_business_shared_items +* Creating, Modifying or Deleting the 'sync_list' file + +Additionally, you might opt for a `--resync` if you think it's necessary to ensure your data remains in sync. If you're using this switch simply because you're unsure of the sync status, you can check the actual sync status using `--display-sync-status`. + +When you use `--resync`, you'll encounter the following warning and advice: +```text +Using --resync will delete your local 'onedrive' client state, so there won't be a record of your current 'sync status.' +This may potentially overwrite local versions of files with older versions downloaded from OneDrive, leading to local data loss. +If in doubt, back up your local data before using --resync. + +Are you sure you want to proceed with --resync? [Y/N] +``` + +To proceed with `--resync`, you must type 'y' or 'Y' to allow the application to continue. + +**Note:** It's highly recommended to use `--resync` only if the application prompts you to do so. Don't blindly set the application to start with `--resync` as the default option. + +**Note:** In certain automated environments (assuming you know what you're doing due to automation), to avoid the 'proceed with acknowledgement' requirement, add `--resync-auth` to automatically acknowledge the prompt. + +### Performing a --force-sync without a --resync or changing your configuration +In some cases and situations, you may have configured the application to skip certain files and folders using 'skip_file' and 'skip_dir' configuration. You then may have a requirement to actually sync one of these items, but do not wish to modify your configuration, nor perform an entire `--resync` twice. + +The `--force-sync` option allows you to sync a specific directory, ignoring your 'skip_file' and 'skip_dir' configuration and negating the requirement to perform a `--resync`. + +To use this option, you must run the application manually in the following manner: +```text +onedrive --sync --single-directory '' --force-sync +``` + +When using `--force-sync`, you'll encounter the following warning and advice: +```text +WARNING: Overriding application configuration to use application defaults for skip_dir and skip_file due to --sync --single-directory --force-sync being used + +Using --force-sync will reconfigure the application to use defaults. This may have unknown future impacts. +By proceeding with this option, you accept any impacts, including potential data loss resulting from using --force-sync. + +Are you sure you want to proceed with --force-sync [Y/N] +``` + +To proceed with `--force-sync`, you must type 'y' or 'Y' to allow the application to continue. + +### Enabling the Client Activity Log +When running onedrive, all actions can be logged to a separate log file. This can be enabled by using the `--enable-logging` flag. By default, log files will be written to `/var/log/onedrive/` and will be in the format of `%username%.onedrive.log`, where `%username%` represents the user who ran the client to allow easy sorting of user to client activity log. + +**Note:** You will need to ensure the existence of this directory and that your user has the applicable permissions to write to this directory; otherwise, the following error message will be printed: +```text +ERROR: Unable to access /var/log/onedrive +ERROR: Please manually create '/var/log/onedrive' and set appropriate permissions to allow write access +ERROR: The requested client activity log will instead be located in your user's home directory +``` + +On many systems, this can be achieved by performing the following: +```text +sudo mkdir /var/log/onedrive +sudo chown root:users /var/log/onedrive +sudo chmod 0775 /var/log/onedrive +``` + +Additionally, you need to ensure that your user account is part of the 'users' group: +``` +cat /etc/group | grep users +``` + +If your user is not part of this group, then you need to add your user to this group: +``` +sudo usermod -a -G users +``` + +If you need to make a group modification, you will need to 'logout' of all sessions / SSH sessions to log in again to have the new group access applied. + +If the client is unable to write the client activity log, the following error message will be printed: +```text +ERROR: Unable to write the activity log to /var/log/onedrive/%username%.onedrive.log +ERROR: Please set appropriate permissions to allow write access to the logging directory for your user account +ERROR: The requested client activity log will instead be located in your user's home directory +``` + +If you receive this error message, you will need to diagnose why your system cannot write to the specified file location. + +#### Client Activity Log Example: +An example of a client activity log for the command `onedrive --sync --enable-logging` is below: +```text +2023-Sep-27 08:16:00.1128806 Configuring Global Azure AD Endpoints +2023-Sep-27 08:16:00.1160620 Sync Engine Initialised with new Onedrive API instance +2023-Sep-27 08:16:00.5227122 All application operations will be performed in: /home/user/OneDrive +2023-Sep-27 08:16:00.5227977 Fetching items from the OneDrive API for Drive ID: +2023-Sep-27 08:16:00.7780979 Processing changes and items received from Microsoft OneDrive ... +2023-Sep-27 08:16:00.7781548 Performing a database consistency and integrity check on locally stored data ... +2023-Sep-27 08:16:00.7785889 Scanning the local file system '~/OneDrive' for new data to upload ... +2023-Sep-27 08:16:00.7813710 Performing a final true-up scan of online data from Microsoft OneDrive +2023-Sep-27 08:16:00.7814668 Fetching items from the OneDrive API for Drive ID: +2023-Sep-27 08:16:01.0141776 Processing changes and items received from Microsoft OneDrive ... +2023-Sep-27 08:16:01.0142454 Sync with Microsoft OneDrive is complete +``` +An example of a client activity log for the command `onedrive --sync --verbose --enable-logging` is below: +```text +2023-Sep-27 08:20:05.4600464 Checking Application Version ... +2023-Sep-27 08:20:05.5235017 Attempting to initialise the OneDrive API ... +2023-Sep-27 08:20:05.5237207 Configuring Global Azure AD Endpoints +2023-Sep-27 08:20:05.5238087 The OneDrive API was initialised successfully +2023-Sep-27 08:20:05.5238536 Opening the item database ... +2023-Sep-27 08:20:05.5270612 Sync Engine Initialised with new Onedrive API instance +2023-Sep-27 08:20:05.9226535 Application version: vX.Y.Z-A-bcdefghi +2023-Sep-27 08:20:05.9227079 Account Type: +2023-Sep-27 08:20:05.9227360 Default Drive ID: +2023-Sep-27 08:20:05.9227550 Default Root ID: +2023-Sep-27 08:20:05.9227862 Remaining Free Space: +2023-Sep-27 08:20:05.9228296 All application operations will be performed in: /home/user/OneDrive +2023-Sep-27 08:20:05.9228989 Fetching items from the OneDrive API for Drive ID: +2023-Sep-27 08:20:06.2076569 Performing a database consistency and integrity check on locally stored data ... +2023-Sep-27 08:20:06.2077121 Processing DB entries for this Drive ID: +2023-Sep-27 08:20:06.2078408 Processing ~/OneDrive +2023-Sep-27 08:20:06.2078739 The directory has not changed +2023-Sep-27 08:20:06.2079783 Processing Attachments +2023-Sep-27 08:20:06.2080071 The directory has not changed +2023-Sep-27 08:20:06.2081585 Processing Attachments/file.docx +2023-Sep-27 08:20:06.2082079 The file has not changed +2023-Sep-27 08:20:06.2082760 Processing Documents +2023-Sep-27 08:20:06.2083225 The directory has not changed +2023-Sep-27 08:20:06.2084284 Processing Documents/file.log +2023-Sep-27 08:20:06.2084886 The file has not changed +2023-Sep-27 08:20:06.2085150 Scanning the local file system '~/OneDrive' for new data to upload ... +2023-Sep-27 08:20:06.2087133 Skipping item - excluded by sync_list config: ./random_25k_files +2023-Sep-27 08:20:06.2116235 Performing a final true-up scan of online data from Microsoft OneDrive +2023-Sep-27 08:20:06.2117190 Fetching items from the OneDrive API for Drive ID: +2023-Sep-27 08:20:06.5049743 Sync with Microsoft OneDrive is complete +``` + +#### Client Activity Log Differences +Despite application logging being enabled as early as possible, the following log entries will be missing from the client activity log when compared to console output: + +**No user configuration file:** +```text +No user or system config file found, using application defaults +Using 'user' configuration path for application state data: /home/user/.config/onedrive +Using the following path to store the runtime application log: /var/log/onedrive +``` +**User configuration file:** +```text +Reading configuration file: /home/user/.config/onedrive/config +Configuration file successfully loaded +Using 'user' configuration path for application state data: /home/user/.config/onedrive +Using the following path to store the runtime application log: /var/log/onedrive +``` + +### GUI Notifications +If notification support has been compiled in (refer to GUI Notification Support in install.md .. ADD LINK LATER), the following events will trigger a GUI notification within the display manager session: +* Aborting a sync if .nosync file is found +* Skipping a particular item due to an invalid name +* Skipping a particular item due to an invalid symbolic link +* Skipping a particular item due to an invalid UTF sequence +* Skipping a particular item due to an invalid character encoding sequence +* Cannot create remote directory +* Cannot upload file changes (free space issue, breaches maximum allowed size, breaches maximum OneDrive Account path length) +* Cannot delete remote file / folder +* Cannot move remote file / folder +* When a re-authentication is required +* When a new client version is available +* Files that fail to upload +* Files that fail to download + +### Handling a Microsoft OneDrive Account Password Change +If you change your Microsoft OneDrive Account Password, the client will no longer be authorised to sync, and will generate the following error upon next application run: +```text +AADSTS50173: The provided grant has expired due to it being revoked, a fresh auth token is needed. The user might have changed or reset their password. The grant was issued on '' and the TokensValidFrom date (before which tokens are not valid) for this user is ''. + +ERROR: You will need to issue a --reauth and re-authorise this client to obtain a fresh auth token. +``` + +To re-authorise the client, follow the steps below: +1. If running the client as a system service (init.d or systemd), stop the applicable system service +2. Run the command `onedrive --reauth`. This will clean up the previous authorisation, and will prompt you to re-authorise the client as per initial configuration. Please note, if you are using `--confdir` as part of your application runtime configuration, you must include this when telling the client to re-authenticate. +3. Restart the client if running as a system service or perform the standalone sync operation again + +The application will now sync with OneDrive with the new credentials. + +### Determining the synchronisation result +When the client has finished syncing without errors, the following will be displayed: +``` +Sync with Microsoft OneDrive is complete +``` + +If any items failed to sync, the following will be displayed: +``` +Sync with Microsoft OneDrive has completed, however there are items that failed to sync. +``` +A file list of either upload or download items will be then listed to allow you to determine your next steps. + +In order to fix the upload or download failures, you may need to re-try your command and perform a resync to ensure your system is correctly synced with your Microsoft OneDrive Account. + +## Frequently Asked Configuration Questions + +### How to change the default configuration of the client? +Configuration is determined by three layers, and applied in the following order: +* Application default values +* Values that are set in the configuration file +* Values that are passed in via the command line at application runtime. These values will override any configuration file set value. + +The default application values provide a reasonable operational default, and additional configuration is entirely optional. + +If you want to change the application defaults, you can download a copy of the config file into your application configuration directory. Valid default directories for the config file are: +* `~/.config/onedrive` +* `/etc/onedrive` + +**Example:** To download a copy of the config file, use the following: +```text +mkdir -p ~/.config/onedrive +wget https://raw.githubusercontent.com/abraunegg/onedrive/master/config -O ~/.config/onedrive/config +``` + +For full configuration options and CLI switches, please refer to application-config-options.md + +### How to change where my data from Microsoft OneDrive is stored? +By default, the location where your Microsoft OneDrive data is stored, is within your Home Directory under a directory called 'OneDrive'. This replicates as close as possible where the Microsoft Windows OneDrive client stores data. + +To change this location, the application configuration option 'sync_dir' is used to specify a new local directory where your Microsoft OneDrive data should be stored. + +**Important Note:** If your `sync_dir` is pointing to a network mount point (a network share via NFS, Windows Network Share, Samba Network Share) these types of network mount points do not support 'inotify', thus tracking real-time changes via inotify of local files is not possible when using 'Monitor Mode'. Local filesystem changes will be replicated between the local filesystem and Microsoft OneDrive based on the `monitor_interval` value. This is not something (inotify support for NFS, Samba) that this client can fix. + +### How to change what file and directory permissions are assigned to data that is downloaded from Microsoft OneDrive? +The following are the application default permissions for any new directory or file that is created locally when downloaded from Microsoft OneDrive: +* Directories: 700 - This provides the following permissions: `drwx------` +* Files: 600 - This provides the following permissions: `-rw-------` + +These default permissions align to the security principal of 'least privilege' so that only you should have access to your data that you download from Microsoft OneDrive. + +To alter these default permissions, you can adjust the values of two configuration options as follows. You can also use the [Unix Permissions Calculator](https://chmod-calculator.com/) to help you determine the necessary new permissions. +```text +sync_dir_permissions = "700" +sync_file_permissions = "600" +``` + +**Important:** Please note that special permission bits such as setuid, setgid, and the sticky bit are not supported. Valid permission values range from `000` to `777` only. + +### How to only sync a specific directory? +There are two methods to achieve this: +* Employ the '--single-directory' option to only sync this specific path +* Employ 'sync_list' as part of your 'config' file to configure what files and directories to sync, and what should be excluded + +### How to 'skip' files from syncing? +There are two methods to achieve this: +* Employ 'skip_file' as part of your 'config' file to configure what files to skip +* Employ 'sync_list' to configure what files and directories to sync, and what should be excluded + +### How to 'skip' directories from syncing? +There are three methods available to 'skip' a directory from the sync process: +* Employ 'skip_dir' as part of your 'config' file to configure what directories to skip +* Employ 'sync_list' to configure what files and directories to sync, and what should be excluded +* Employ 'check_nosync' as part of your 'config' file and a '.nosync' empty file within the directory to exclude to skip that directory + +### How to 'skip' dot files and folders from syncing? +There are three methods to achieve this: +* Employ 'skip_file' or 'skip_dir' to configure what files or folders to skip +* Employ 'sync_list' to configure what files and directories to sync, and what should be excluded +* Employ 'skip_dotfiles' as part of your 'config' file to skip any dot file (for example: `.Trash-1000` or `.xdg-volume-info`) from syncing to OneDrive + +### How to 'skip' files larger than a certain size from syncing? +Use `skip_size = "value"` as part of your 'config' file where files larger than this size (in MB) will be skipped. + +### How to 'rate limit' the application to control bandwidth consumed for upload & download operations? +To minimise the Internet bandwidth for upload and download operations, you can add the 'rate_limit' configuration option as part of your 'config' file. + +The default value is '0' which means use all available bandwidth for the application. + +The value being used can be reviewed when using `--display-config`. + +### How can I prevent my local disk from filling up? +By default, the application will reserve 50MB of disk space to prevent your filesystem from running out of disk space. + +This default value can be modified by adding the 'space_reservation' configuration option and the applicable value as part of your 'config' file. + +You can review the value being used when using `--display-config`. + +### How does the client handle symbolic links? +Microsoft OneDrive has no concept or understanding of symbolic links, and attempting to upload a symbolic link to Microsoft OneDrive generates a platform API error. All data (files and folders) that are uploaded to OneDrive must be whole files or actual directories. + +As such, there are only two methods to support symbolic links with this client: +1. Follow the Linux symbolic link and upload whatever the local symbolic link is pointing to to Microsoft OneDrive. This is the default behaviour. +2. Skip symbolic links by configuring the application to do so. When skipping, no data, no link, no reference is uploaded to OneDrive. + +Use 'skip_symlinks' as part of your 'config' file to configure the skipping of all symbolic links while syncing. + +### How to synchronise shared folders (OneDrive Personal)? +Folders shared with you can be synchronised by adding them to your OneDrive online. To do that, open your OneDrive account online, go to the Shared files list, right-click on the folder you want to synchronise, and then click on "Add to my OneDrive". + +### How to synchronise shared folders (OneDrive Business or Office 365)? +Folders shared with you can be synchronised by adding them to your OneDrive online. To do that, open your OneDrive account online, go to the Shared files list, right-click on the folder you want to synchronise, and then click on "Add to my OneDrive". + +Refer to [./business-shared-folders.md](business-shared-folders.md) for further details. + +### How to synchronise SharePoint / Office 365 Shared Libraries? +There are two methods to achieve this: +* SharePoint library can be directly added to your OneDrive online. To do that, open your OneDrive account online, go to the Shared files list, right-click on the SharePoint Library you want to synchronise, and then click on "Add to my OneDrive". +* Configure a separate application instance to only synchronise that specific SharePoint Library. Refer to [./sharepoint-libraries.md](sharepoint-libraries.md) for configuration assistance. + +### How to Create a Shareable Link? +In certain situations, you might want to generate a shareable file link and provide this link to other users for accessing a specific file. + +To accomplish this, employ the following command: +```text +onedrive --create-share-link +``` +**Note:** By default, this access permissions for the file link will be read-only. + +To make it a read-write link, execute the following command: +```text +onedrive --create-share-link --with-editing-perms +``` +**Note:** The order of the file path and option flag is crucial. + +### How to Synchronise Both Personal and Business Accounts at once? +You need to set up separate instances of the application configuration for each account. + +Refer to [./advanced-usage.md](advanced-usage.md) for guidance on configuration. + +### How to Synchronise Multiple SharePoint Libraries simultaneously? +For each SharePoint Library, configure a separate instance of the application configuration. + +Refer to [./advanced-usage.md](advanced-usage.md) for configuration instructions. + +### How to Receive Real-time Changes from Microsoft OneDrive Service, instead of waiting for the next sync period? +When operating in 'Monitor Mode,' it may be advantageous to receive real-time updates to online data. A 'webhook' is the method to achieve this, so that when in 'Monitor Mode,' the client subscribes to remote updates. + +Remote changes can then be promptly synchronised to your local file system, without waiting for the next synchronisation cycle. + +This is accomplished by: +* Using 'webhook_enabled' as part of your 'config' file to enable this feature +* Using 'webhook_public_url' as part of your 'config' file to configure the URL the webhook will use for subscription updates + +### How to initiate the client as a background service? +There are a few ways to employ onedrive as a service: +* via init.d +* via systemd +* via runit + +#### OneDrive service running as root user via init.d +```text +chkconfig onedrive on +service onedrive start +``` +To view the logs, execute: +```text +tail -f /var/log/onedrive/.onedrive.log +``` +To alter the 'user' under which the client operates (typically root by default), manually modify the init.d service file and adjust `daemon --user root onedrive_service.sh` to match the correct user. + +#### OneDrive service running as root user via systemd (Arch, Ubuntu, Debian, OpenSuSE, Fedora) +Initially, switch to the root user with `su - root`, then activate the systemd service: +```text +systemctl --user enable onedrive +systemctl --user start onedrive +``` +**Note:** The `systemctl --user` command is not applicable to Red Hat Enterprise Linux (RHEL) or CentOS Linux platforms - see below. + +**Note:** This will execute the 'onedrive' process with a UID/GID of '0', which means any files or folders created will be owned by 'root'. + +To monitor the service's status, use the following: +```text +systemctl --user status onedrive.service +``` + +To observe the systemd application logs, use: +```text +journalctl --user-unit=onedrive -f +``` + +**Note:** For systemd to function correctly, it requires the presence of XDG environment variables. If you encounter the following error while enabling the systemd service: +```text +Failed to connect to bus: No such file or directory +``` +The most likely cause is missing XDG environment variables. To resolve this, add the following lines to `.bashrc` or another file executed upon user login: +```text +export XDG_RUNTIME_DIR="/run/user/$UID" +export DBUS_SESSION_BUS_ADDRESS="unix:path=${XDG_RUNTIME_DIR}/bus" +``` + +To apply this change, you must log out of all user accounts where it has been made. + +**Note:** On certain systems (e.g., Raspbian / Ubuntu / Debian on Raspberry Pi), the XDG fix above may not persist after system reboots. An alternative to starting the client via systemd as root is as follows: +1. Create a symbolic link from `/home/root/.config/onedrive` to `/root/.config/onedrive/`. +2. Establish a systemd service using the '@' service file: `systemctl enable onedrive@root.service`. +3. Start the root@service: `systemctl start onedrive@root.service`. + +This ensures that the service correctly restarts upon system reboot. + +To examine the systemd application logs, run: +```text +journalctl --unit=onedrive@ -f +``` + +#### OneDrive service running as root user via systemd (Red Hat Enterprise Linux, CentOS Linux) +```text +systemctl enable onedrive +systemctl start onedrive +``` +**Note:** This will execute the 'onedrive' process with a UID/GID of '0', meaning any files or folders created will be owned by 'root'. + +To view the systemd application logs, execute: +```text +journalctl --unit=onedrive -f +``` + +#### OneDrive service running as a non-root user via systemd (All Linux Distributions) +In some instances, it is preferable to run the OneDrive client as a service without the 'root' user. Follow the instructions below to configure the service for your regular user login. + +1. As the user who will run the service, launch the application in standalone mode, authorize it for use, and verify that synchronization is functioning as expected: +```text +onedrive --sync --verbose +``` +2. After validating the application for your user, switch to the 'root' user, where is your username from step 1 above. +```text +systemctl enable onedrive@.service +systemctl start onedrive@.service +``` +3. To check the service's status for the user, use the following: +```text +systemctl status onedrive@.service +``` + +To observe the systemd application logs, use: +```text +journalctl --unit=onedrive@ -f +``` + +#### OneDrive service running as a non-root user via systemd (with notifications enabled) (Arch, Ubuntu, Debian, OpenSuSE, Fedora) +In some scenarios, you may want to receive GUI notifications when using the client as a non-root user. In this case, follow these steps: + +1. Log in via the graphical UI as the user you want to enable the service for. +2. Disable any `onedrive@` service files for your username, e.g.: +```text +sudo systemctl stop onedrive@alex.service +sudo systemctl disable onedrive@alex.service +``` +3. Enable the service as follows: +```text +systemctl --user enable onedrive +systemctl --user start onedrive +``` + +To check the service's status for the user, use the following: +```text +systemctl --user status onedrive.service +``` + +To view the systemd application logs, execute: +```text +journalctl --user-unit=onedrive -f +``` + +**Note:** The `systemctl --user` command is not applicable to Red Hat Enterprise Linux (RHEL) or CentOS Linux platforms. + +#### OneDrive service running as a non-root user via runit (antiX, Devuan, Artix, Void) + +1. Create the following folder if it doesn't already exist: `/etc/sv/runsvdir-` + + - where `` is the `USER` targeted for the service + - e.g., `# mkdir /etc/sv/runsvdir-nolan` + +2. Create a file called `run` under the previously created folder with executable permissions + + - `# touch /etc/sv/runsvdir-/run` + - `# chmod 0755 /etc/sv/runsvdir-/run` + +3. Edit the `run` file with the following contents (permissions needed): + + ```sh + #!/bin/sh + export USER="" + export HOME="/home/" + + groups="$(id -Gn "${USER}" | tr ' ' ':')" + svdir="${HOME}/service" + + exec chpst -u "${USER}:${groups}" runsvdir "${svdir}" + ``` + + - Ensure you replace `` with the `USER` set in step #1. + +4. Enable the previously created folder as a service + + - `# ln -fs /etc/sv/runsvdir- /var/service/` + +5. Create a subfolder in the `USER`'s `HOME` directory to store the services (or symlinks) + + - `$ mkdir ~/service` + +6. Create a subfolder specifically for OneDrive + + - `$ mkdir ~/service/onedrive/` + +7. Create a file called `run` under the previously created folder with executable permissions + + - `$ touch ~/service/onedrive/run` + - `$ chmod 0755 ~/service/onedrive/run` + +8. Append the following contents to the `run` file + + ```sh + #!/usr/bin/env sh + exec /usr/bin/onedrive --monitor + ``` + + - In some scenarios, the path to the `onedrive` binary may vary. You can obtain it by running `$ command -v onedrive`. + +9. Reboot to apply the changes + +10. Check the status of user-defined services + + - `$ sv status ~/service/*` + +For additional details, you can refer to Void's documentation on [Per-User Services](https://docs.voidlinux.org/config/services/user-services.html). + +### How to start a user systemd service at boot without user login? +In some situations, it may be necessary for the systemd service to start without requiring your 'user' to log in. + +To address this issue, you need to reconfigure your 'user' account so that the systemd services you've created launch without the need for you to log in to your system: +```text +loginctl enable-linger +``` \ No newline at end of file diff --git a/readme.md b/readme.md new file mode 100644 index 00000000..7cf3697c --- /dev/null +++ b/readme.md @@ -0,0 +1,84 @@ +# OneDrive Client for Linux +[![Version](https://img.shields.io/github/v/release/abraunegg/onedrive)](https://github.com/abraunegg/onedrive/releases) +[![Release Date](https://img.shields.io/github/release-date/abraunegg/onedrive)](https://github.com/abraunegg/onedrive/releases) +[![Test Build](https://github.com/abraunegg/onedrive/actions/workflows/testbuild.yaml/badge.svg)](https://github.com/abraunegg/onedrive/actions/workflows/testbuild.yaml) +[![Build Docker Images](https://github.com/abraunegg/onedrive/actions/workflows/docker.yaml/badge.svg)](https://github.com/abraunegg/onedrive/actions/workflows/docker.yaml) +[![Docker Pulls](https://img.shields.io/docker/pulls/driveone/onedrive)](https://hub.docker.com/r/driveone/onedrive) + +Introducing a free Microsoft OneDrive Client that seamlessly supports OneDrive Personal, OneDrive for Business, OneDrive for Office365, and SharePoint Libraries. + +This robust and highly customisable client is compatible with all major Linux distributions and FreeBSD, and can also be deployed as a container using Docker or Podman. It offers both one-way and two-way synchronisation capabilities while ensuring a secure connection to Microsoft OneDrive services. + +Originally derived as a 'fork' from the [skilion](https://github.com/skilion/onedrive) client, it's worth noting that the developer of the original client has explicitly stated they have no intention of maintaining or supporting their work ([reference](https://github.com/skilion/onedrive/issues/518#issuecomment-717604726)). + +This client represents a 100% re-imagining of the original work, addressing numerous notable bugs and issues while incorporating a significant array of new features. This client has been under active development since mid-2018. + +## Features +* State caching +* Real-Time local file monitoring with inotify +* Real-Time syncing of remote updates via webhooks +* File upload / download validation to ensure data integrity +* Resumable uploads +* Support OneDrive for Business (part of Office 365) +* Shared Folder support for OneDrive Personal and OneDrive Business accounts +* SharePoint / Office365 Shared Libraries +* Desktop notifications via libnotify +* Dry-run capability to test configuration changes +* Prevent major OneDrive accidental data deletion after configuration change +* Support for National cloud deployments (Microsoft Cloud for US Government, Microsoft Cloud Germany, Azure and Office 365 operated by 21Vianet in China) +* Supports single & multi-tenanted applications +* Supports rate limiting of traffic +* Supports multi-threaded uploads and downloads + +## What's missing +* Ability to encrypt/decrypt files on-the-fly when uploading/downloading files from OneDrive +* Support for Windows 'On-Demand' functionality so file is only downloaded when accessed locally + +## External Enhancements +* A GUI for configuration management: [OneDrive Client for Linux GUI](https://github.com/bpozdena/OneDriveGUI) +* Colorful log output terminal modification: [OneDrive Client for Linux Colorful log Output](https://github.com/zzzdeb/dotfiles/blob/master/scripts/tools/onedrive_log) +* System Tray Icon: [OneDrive Client for Linux System Tray Icon](https://github.com/DanielBorgesOliveira/onedrive_tray) + +## Frequently Asked Questions +Refer to [Frequently Asked Questions](https://github.com/abraunegg/onedrive/wiki/Frequently-Asked-Questions) + +## Have a question +If you have a question or need something clarified, please raise a new disscussion post [here](https://github.com/abraunegg/onedrive/discussions) + +## Reporting an Issue or Bug +If you encounter any bugs you can report them here on Github. Before filing an issue be sure to: + +1. Check the version of the application you are using `onedrive --version` and ensure that you are running either the latest [release](https://github.com/abraunegg/onedrive/releases) or built from master. +2. Fill in a new bug report using the [issue template](https://github.com/abraunegg/onedrive/issues/new?template=bug_report.md) +3. Generate a debug log for support using the following [process](https://github.com/abraunegg/onedrive/wiki/Generate-debug-log-for-support) + * If you are in *any* way concerned regarding the sensitivity of the data contained with in the verbose debug log file, create a new OneDrive account, configure the client to use that, use *dummy* data to simulate your environment and then replicate your original issue + * If you are still concerned, provide an NDA or confidentiality document to sign +4. Upload the debug log to [pastebin](https://pastebin.com/) or archive and email to support@mynas.com.au + * If you are concerned regarding the sensitivity of your debug data, encrypt + password protect the archive file and provide the decryption password via an out-of-band (OOB) mechanism. Email support@mynas.com.au for an OOB method for the password to be sent. + * If you are still concerned, provide an NDA or confidentiality document to sign + +## Known issues +Refer to [docs/known-issues.md](https://github.com/abraunegg/onedrive/blob/master/docs/known-issues.md) + +## Documentation and Configuration Assistance +### Installing from Distribution Packages or Building the OneDrive Client for Linux from source +Refer to [docs/install.md](https://github.com/abraunegg/onedrive/blob/master/docs/install.md) + +### Configuration and Usage +Refer to [docs/usage.md](https://github.com/abraunegg/onedrive/blob/master/docs/usage.md) + +### Configure OneDrive Business Shared Folders +Refer to [docs/business-shared-folders.md](https://github.com/abraunegg/onedrive/blob/master/docs/business-shared-folders.md) + +### Configure SharePoint / Office 365 Shared Libraries (Business or Education) +Refer to [docs/sharepoint-libraries.md](https://github.com/abraunegg/onedrive/blob/master/docs/sharepoint-libraries.md) + +### Configure National Cloud support +Refer to [docs/national-cloud-deployments.md](https://github.com/abraunegg/onedrive/blob/master/docs/national-cloud-deployments.md) + +### Docker support +Refer to [docs/docker.md](https://github.com/abraunegg/onedrive/blob/master/docs/docker.md) + +### Podman support +Refer to [docs/podman.md](https://github.com/abraunegg/onedrive/blob/master/docs/podman.md) + diff --git a/src/arsd/README.md b/src/arsd/README.md new file mode 100644 index 00000000..f4f1d0d5 --- /dev/null +++ b/src/arsd/README.md @@ -0,0 +1,8 @@ +The files in this directory have been obtained form the following places: + +cgi.d + https://github.com/adamdruppe/arsd/blob/a870179988b8881b04126856105f0fad2cc0018d/cgi.d + License: Boost Software License - Version 1.0 + + Copyright 2008-2021, Adam D. Ruppe + see https://github.com/adamdruppe/arsd/blob/a870179988b8881b04126856105f0fad2cc0018d/LICENSE diff --git a/src/arsd/cgi.d b/src/arsd/cgi.d new file mode 100644 index 00000000..79f5feaa --- /dev/null +++ b/src/arsd/cgi.d @@ -0,0 +1,11810 @@ +// FIXME: if an exception is thrown, we shouldn't necessarily cache... +// FIXME: there's some annoying duplication of code in the various versioned mains + +// add the Range header in there too. should return 206 + +// FIXME: cgi per-request arena allocator + +// i need to add a bunch of type templates for validations... mayne @NotNull or NotNull! + +// FIXME: I might make a cgi proxy class which can change things; the underlying one is still immutable +// but the later one can edit and simplify the api. You'd have to use the subclass tho! + +/* +void foo(int f, @("test") string s) {} + +void main() { + static if(is(typeof(foo) Params == __parameters)) + //pragma(msg, __traits(getAttributes, Params[0])); + pragma(msg, __traits(getAttributes, Params[1..2])); + else + pragma(msg, "fail"); +} +*/ + +// Note: spawn-fcgi can help with fastcgi on nginx + +// FIXME: to do: add openssl optionally +// make sure embedded_httpd doesn't send two answers if one writes() then dies + +// future direction: websocket as a separate process that you can sendfile to for an async passoff of those long-lived connections + +/* + Session manager process: it spawns a new process, passing a + command line argument, to just be a little key/value store + of some serializable struct. On Windows, it CreateProcess. + On Linux, it can just fork or maybe fork/exec. The session + key is in a cookie. + + Server-side event process: spawns an async manager. You can + push stuff out to channel ids and the clients listen to it. + + websocket process: spawns an async handler. They can talk to + each other or get info from a cgi request. + + Tempting to put web.d 2.0 in here. It would: + * map urls and form generation to functions + * have data presentation magic + * do the skeleton stuff like 1.0 + * auto-cache generated stuff in files (at least if pure?) + * introspect functions in json for consumers + + + https://linux.die.net/man/3/posix_spawn +*/ + +/++ + Provides a uniform server-side API for CGI, FastCGI, SCGI, and HTTP web applications. Offers both lower- and higher- level api options among other common (optional) things like websocket and event source serving support, session management, and job scheduling. + + --- + import arsd.cgi; + + // Instead of writing your own main(), you should write a function + // that takes a Cgi param, and use mixin GenericMain + // for maximum compatibility with different web servers. + void hello(Cgi cgi) { + cgi.setResponseContentType("text/plain"); + + if("name" in cgi.get) + cgi.write("Hello, " ~ cgi.get["name"]); + else + cgi.write("Hello, world!"); + } + + mixin GenericMain!hello; + --- + + Or: + --- + import arsd.cgi; + + class MyApi : WebObject { + @UrlName("") + string hello(string name = null) { + if(name is null) + return "Hello, world!"; + else + return "Hello, " ~ name; + } + } + mixin DispatcherMain!( + "/".serveApi!MyApi + ); + --- + + $(NOTE + Please note that using the higher-level api will add a dependency on arsd.dom and arsd.jsvar to your application. + If you use `dmd -i` or `ldc2 -i` to build, it will just work, but with dub, you will have do `dub add arsd-official:jsvar` + and `dub add arsd-official:dom` yourself. + ) + + Test on console (works in any interface mode): + $(CONSOLE + $ ./cgi_hello GET / name=whatever + ) + + If using http version (default on `dub` builds, or on custom builds when passing `-version=embedded_httpd` to dmd): + $(CONSOLE + $ ./cgi_hello --port 8080 + # now you can go to http://localhost:8080/?name=whatever + ) + + Please note: the default port for http is 8085 and for scgi is 4000. I recommend you set your own by the command line argument in a startup script instead of relying on any hard coded defaults. It is possible though to code your own with [RequestServer], however. + + + Build_Configurations: + + cgi.d tries to be flexible to meet your needs. It is possible to configure it both at runtime (by writing your own `main` function and constructing a [RequestServer] object) or at compile time using the `version` switch to the compiler or a dub `subConfiguration`. + + If you are using `dub`, use: + + ```sdlang + subConfiguration "arsd-official:cgi" "VALUE_HERE" + ``` + + or to dub.json: + + ```json + "subConfigurations": {"arsd-official:cgi": "VALUE_HERE"} + ``` + + to change versions. The possible options for `VALUE_HERE` are: + + $(LIST + * `embedded_httpd` for the embedded httpd version (built-in web server). This is the default for dub builds. You can run the program then connect directly to it from your browser. + * `cgi` for traditional cgi binaries. These are run by an outside web server as-needed to handle requests. + * `fastcgi` for FastCGI builds. FastCGI is managed from an outside helper, there's one built into Microsoft IIS, Apache httpd, and Lighttpd, and a generic program you can use with nginx called `spawn-fcgi`. If you don't already know how to use it, I suggest you use one of the other modes. + * `scgi` for SCGI builds. SCGI is a simplified form of FastCGI, where you run the server as an application service which is proxied by your outside webserver. + * `stdio_http` for speaking raw http over stdin and stdout. This is made for systemd services. See [RequestServer.serveSingleHttpConnectionOnStdio] for more information. + ) + + With dmd, use: + + $(TABLE_ROWS + + * + Interfaces + + (mutually exclusive) + + * - `-version=plain_cgi` + - The default building the module alone without dub - a traditional, plain CGI executable will be generated. + * - `-version=embedded_httpd` + - A HTTP server will be embedded in the generated executable. This is default when building with dub. + * - `-version=fastcgi` + - A FastCGI executable will be generated. + * - `-version=scgi` + - A SCGI (SimpleCGI) executable will be generated. + * - `-version=embedded_httpd_hybrid` + - A HTTP server that uses a combination of processes, threads, and fibers to better handle large numbers of idle connections. Recommended if you are going to serve websockets in a non-local application. + * - `-version=embedded_httpd_threads` + - The embedded HTTP server will use a single process with a thread pool. (use instead of plain `embedded_httpd` if you want this specific implementation) + * - `-version=embedded_httpd_processes` + - The embedded HTTP server will use a prefork style process pool. (use instead of plain `embedded_httpd` if you want this specific implementation) + * - `-version=embedded_httpd_processes_accept_after_fork` + - It will call accept() in each child process, after forking. This is currently the only option, though I am experimenting with other ideas. You probably should NOT specify this right now. + * - `-version=stdio_http` + - The embedded HTTP server will be spoken over stdin and stdout. + + * + Tweaks + + (can be used together with others) + + * - `-version=cgi_with_websocket` + - The CGI class has websocket server support. (This is on by default now.) + + * - `-version=with_openssl` + - not currently used + * - `-version=cgi_embedded_sessions` + - The session server will be embedded in the cgi.d server process + * - `-version=cgi_session_server_process` + - The session will be provided in a separate process, provided by cgi.d. + ) + + For example, + + For CGI, `dmd yourfile.d cgi.d` then put the executable in your cgi-bin directory. + + For FastCGI: `dmd yourfile.d cgi.d -version=fastcgi` and run it. spawn-fcgi helps on nginx. You can put the file in the directory for Apache. On IIS, run it with a port on the command line (this causes it to call FCGX_OpenSocket, which can work on nginx too). + + For SCGI: `dmd yourfile.d cgi.d -version=scgi` and run the executable, providing a port number on the command line. + + For an embedded HTTP server, run `dmd yourfile.d cgi.d -version=embedded_httpd` and run the generated program. It listens on port 8085 by default. You can change this on the command line with the --port option when running your program. + + Simulating_requests: + + If you are using one of the [GenericMain] or [DispatcherMain] mixins, or main with your own call to [RequestServer.trySimulatedRequest], you can simulate requests from your command-ine shell. Call the program like this: + + $(CONSOLE + ./yourprogram GET / name=adr + ) + + And it will print the result to stdout instead of running a server, regardless of build more.. + + CGI_Setup_tips: + + On Apache, you may do `SetHandler cgi-script` in your `.htaccess` file to set a particular file to be run through the cgi program. Note that all "subdirectories" of it also run the program; if you configure `/foo` to be a cgi script, then going to `/foo/bar` will call your cgi handler function with `cgi.pathInfo == "/bar"`. + + Overview_Of_Basic_Concepts: + + cgi.d offers both lower-level handler apis as well as higher-level auto-dispatcher apis. For a lower-level handler function, you'll probably want to review the following functions: + + Input: [Cgi.get], [Cgi.post], [Cgi.request], [Cgi.files], [Cgi.cookies], [Cgi.pathInfo], [Cgi.requestMethod], + and HTTP headers ([Cgi.headers], [Cgi.userAgent], [Cgi.referrer], [Cgi.accept], [Cgi.authorization], [Cgi.lastEventId]) + + Output: [Cgi.write], [Cgi.header], [Cgi.setResponseStatus], [Cgi.setResponseContentType], [Cgi.gzipResponse] + + Cookies: [Cgi.setCookie], [Cgi.clearCookie], [Cgi.cookie], [Cgi.cookies] + + Caching: [Cgi.setResponseExpires], [Cgi.updateResponseExpires], [Cgi.setCache] + + Redirections: [Cgi.setResponseLocation] + + Other Information: [Cgi.remoteAddress], [Cgi.https], [Cgi.port], [Cgi.scriptName], [Cgi.requestUri], [Cgi.getCurrentCompleteUri], [Cgi.onRequestBodyDataReceived] + + Websockets: [Websocket], [websocketRequested], [acceptWebsocket]. For websockets, use the `embedded_httpd_hybrid` build mode for best results, because it is optimized for handling large numbers of idle connections compared to the other build modes. + + Overriding behavior for special cases streaming input data: see the virtual functions [Cgi.handleIncomingDataChunk], [Cgi.prepareForIncomingDataChunks], [Cgi.cleanUpPostDataState] + + A basic program using the lower-level api might look like: + + --- + import arsd.cgi; + + // you write a request handler which always takes a Cgi object + void handler(Cgi cgi) { + /+ + when the user goes to your site, suppose you are being hosted at http://example.com/yourapp + + If the user goes to http://example.com/yourapp/test?name=value + then the url will be parsed out into the following pieces: + + cgi.pathInfo == "/test". This is everything after yourapp's name. (If you are doing an embedded http server, your app's name is blank, so pathInfo will be the whole path of the url.) + + cgi.scriptName == "yourapp". With an embedded http server, this will be blank. + + cgi.host == "example.com" + + cgi.https == false + + cgi.queryString == "name=value" (there's also cgi.search, which will be "?name=value", including the ?) + + The query string is further parsed into the `get` and `getArray` members, so: + + cgi.get == ["name": "value"], meaning you can do `cgi.get["name"] == "value"` + + And + + cgi.getArray == ["name": ["value"]]. + + Why is there both `get` and `getArray`? The standard allows names to be repeated. This can be very useful, + it is how http forms naturally pass multiple items like a set of checkboxes. So `getArray` is the complete data + if you need it. But since so often you only care about one value, the `get` member provides more convenient access. + + We can use these members to process the request and build link urls. Other info from the request are in other members, we'll look at them later. + +/ + switch(cgi.pathInfo) { + // the home page will be a small html form that can set a cookie. + case "/": + cgi.write(` + + +
+ + +
+ + + `, true); // the , true tells it that this is the one, complete response i want to send, allowing some optimizations. + break; + // POSTing to this will set a cookie with our submitted name + case "/set-cookie": + // HTTP has a number of request methods (also called "verbs") to tell + // what you should do with the given resource. + // The most common are GET and POST, the ones used in html forms. + // You can check which one was used with the `cgi.requestMethod` property. + if(cgi.requestMethod == Cgi.RequestMethod.POST) { + + // headers like redirections need to be set before we call `write` + cgi.setResponseLocation("read-cookie"); + + // just like how url params go into cgi.get/getArray, form data submitted in a POST + // body go to cgi.post/postArray. Please note that a POST request can also have get + // params in addition to post params. + // + // There's also a convenience function `cgi.request("name")` which checks post first, + // then get if it isn't found there, and then returns a default value if it is in neither. + if("name" in cgi.post) { + // we can set cookies with a method too + // again, cookies need to be set before calling `cgi.write`, since they + // are a kind of header. + cgi.setCookie("name" , cgi.post["name"]); + } + + // the user will probably never see this, since the response location + // is an automatic redirect, but it is still best to say something anyway + cgi.write("Redirecting you to see the cookie...", true); + } else { + // you can write out response codes and headers + // as well as response bodies + // + // But always check the cgi docs before using the generic + // `header` method - if there is a specific method for your + // header, use it before resorting to the generic one to avoid + // a header value from being sent twice. + cgi.setResponseLocation("405 Method Not Allowed"); + // there is no special accept member, so you can use the generic header function + cgi.header("Accept: POST"); + // but content type does have a method, so prefer to use it: + cgi.setResponseContentType("text/plain"); + + // all the headers are buffered, and will be sent upon the first body + // write. you can actually modify some of them before sending if need be. + cgi.write("You must use the POST http verb on this resource.", true); + } + break; + // and GETting this will read the cookie back out + case "/read-cookie": + // I did NOT pass `,true` here because this is writing a partial response. + // It is possible to stream data to the user in chunks by writing partial + // responses the calling `cgi.flush();` to send the partial response immediately. + // normally, you'd only send partial chunks if you have to - it is better to build + // a response as a whole and send it as a whole whenever possible - but here I want + // to demo that you can. + cgi.write("Hello, "); + if("name" in cgi.cookies) { + import arsd.dom; // dom.d provides a lot of helpers for html + // since the cookie is set, we need to write it out properly to + // avoid cross-site scripting attacks. + // + // Getting this stuff right automatically is a benefit of using the higher + // level apis, but this demo is to show the fundamental building blocks, so + // we're responsible to take care of it. + cgi.write(htmlEntitiesEncode(cgi.cookies["name"])); + } else { + cgi.write("friend"); + } + + // note that I never called cgi.setResponseContentType, since the default is text/html. + // it doesn't hurt to do it explicitly though, just remember to do it before any cgi.write + // calls. + break; + default: + // no path matched + cgi.setResponseStatus("404 Not Found"); + cgi.write("Resource not found.", true); + } + } + + // and this adds the boilerplate to set up a server according to the + // compile version configuration and call your handler as requests come in + mixin GenericMain!handler; // the `handler` here is the name of your function + --- + + Even if you plan to always use the higher-level apis, I still recommend you at least familiarize yourself with the lower level functions, since they provide the lightest weight, most flexible options to get down to business if you ever need them. + + In the lower-level api, the [Cgi] object represents your HTTP transaction. It has functions to describe the request and for you to send your response. It leaves the details of how you o it up to you. The general guideline though is to avoid depending any variables outside your handler function, since there's no guarantee they will survive to another handler. You can use global vars as a lazy initialized cache, but you should always be ready in case it is empty. (One exception: if you use `-version=embedded_httpd_threads -version=cgi_no_fork`, then you can rely on it more, but you should still really write things assuming your function won't have anything survive beyond its return for max scalability and compatibility.) + + A basic program using the higher-level apis might look like: + + --- + /+ + import arsd.cgi; + + struct LoginData { + string currentUser; + } + + class AppClass : WebObject { + string foo() {} + } + + mixin DispatcherMain!( + "/assets/.serveStaticFileDirectory("assets/", true), // serve the files in the assets subdirectory + "/".serveApi!AppClass, + "/thing/".serveRestObject, + ); + +/ + --- + + Guide_for_PHP_users: + (Please note: I wrote this section in 2008. A lot of PHP hosts still ran 4.x back then, so it was common to avoid using classes - introduced in php 5 - to maintain compatibility! If you're coming from php more recently, this may not be relevant anymore, but still might help you.) + + If you are coming from old-style PHP, here's a quick guide to help you get started: + + $(SIDE_BY_SIDE + $(COLUMN + ```php + + ``` + ) + $(COLUMN + --- + import arsd.cgi; + void app(Cgi cgi) { + string foo = cgi.post["foo"]; + string bar = cgi.get["bar"]; + string baz = cgi.cookies["baz"]; + + string user_ip = cgi.remoteAddress; + string host = cgi.host; + string path = cgi.pathInfo; + + cgi.setCookie("baz", "some value"); + + cgi.write("hello!"); + } + + mixin GenericMain!app + --- + ) + ) + + $(H3 Array elements) + + + In PHP, you can give a form element a name like `"something[]"`, and then + `$_POST["something"]` gives an array. In D, you can use whatever name + you want, and access an array of values with the `cgi.getArray["name"]` and + `cgi.postArray["name"]` members. + + $(H3 Databases) + + PHP has a lot of stuff in its standard library. cgi.d doesn't include most + of these, but the rest of my arsd repository has much of it. For example, + to access a MySQL database, download `database.d` and `mysql.d` from my + github repo, and try this code (assuming, of course, your database is + set up): + + --- + import arsd.cgi; + import arsd.mysql; + + void app(Cgi cgi) { + auto database = new MySql("localhost", "username", "password", "database_name"); + foreach(row; mysql.query("SELECT count(id) FROM people")) + cgi.write(row[0] ~ " people in database"); + } + + mixin GenericMain!app; + --- + + Similar modules are available for PostgreSQL, Microsoft SQL Server, and SQLite databases, + implementing the same basic interface. + + See_Also: + + You may also want to see [arsd.dom], [arsd.webtemplate], and maybe some functions from my old [arsd.html] for more code for making + web applications. dom and webtemplate are used by the higher-level api here in cgi.d. + + For working with json, try [arsd.jsvar]. + + [arsd.database], [arsd.mysql], [arsd.postgres], [arsd.mssql], and [arsd.sqlite] can help in + accessing databases. + + If you are looking to access a web application via HTTP, try [arsd.http2]. + + Copyright: + + cgi.d copyright 2008-2023, Adam D. Ruppe. Provided under the Boost Software License. + + Yes, this file is old, and yes, it is still actively maintained and used. ++/ +module arsd.cgi; + +// FIXME: Nullable!T can be a checkbox that enables/disables the T on the automatic form +// and a SumType!(T, R) can be a radio box to pick between T and R to disclose the extra boxes on the automatic form + +/++ + This micro-example uses the [dispatcher] api to act as a simple http file server, serving files found in the current directory and its children. ++/ +unittest { + import arsd.cgi; + + mixin DispatcherMain!( + "/".serveStaticFileDirectory(null, true) + ); +} + +/++ + Same as the previous example, but written out long-form without the use of [DispatcherMain] nor [GenericMain]. ++/ +unittest { + import arsd.cgi; + + void requestHandler(Cgi cgi) { + cgi.dispatcher!( + "/".serveStaticFileDirectory(null, true) + ); + } + + // mixin GenericMain!requestHandler would add this function: + void main(string[] args) { + // this is all the content of [cgiMainImpl] which you can also call + + // cgi.d embeds a few add on functions like real time event forwarders + // and session servers it can run in other processes. this spawns them, if needed. + if(tryAddonServers(args)) + return; + + // cgi.d allows you to easily simulate http requests from the command line, + // without actually starting a server. this function will do that. + if(trySimulatedRequest!(requestHandler, Cgi)(args)) + return; + + RequestServer server; + // you can change the default port here if you like + // server.listeningPort = 9000; + + // then call this to let the command line args override your default + server.configureFromCommandLine(args); + + // here is where you could print out the listeningPort to the user if you wanted + + // and serve the request(s) according to the compile configuration + server.serve!(requestHandler)(); + + // or you could explicitly choose a serve mode like this: + // server.serveEmbeddedHttp!requestHandler(); + } +} + +/++ + cgi.d has built-in testing helpers too. These will provide mock requests and mock sessions that + otherwise run through the rest of the internal mechanisms to call your functions without actually + spinning up a server. ++/ +unittest { + import arsd.cgi; + + void requestHandler(Cgi cgi) { + + } + + // D doesn't let me embed a unittest inside an example unittest + // so this is a function, but you can do it however in your real program + /* unittest */ void runTests() { + auto tester = new CgiTester(&requestHandler); + + auto response = tester.GET("/"); + assert(response.code == 200); + } +} + +static import std.file; + +// for a single thread, linear request thing, use: +// -version=embedded_httpd_threads -version=cgi_no_threads + +version(Posix) { + version(CRuntime_Musl) { + + } else version(minimal) { + + } else { + version(GNU) { + // GDC doesn't support static foreach so I had to cheat on it :( + } else version(FreeBSD) { + // I never implemented the fancy stuff there either + } else { + version=with_breaking_cgi_features; + version=with_sendfd; + version=with_addon_servers; + } + } +} + +version(Windows) { + version(minimal) { + + } else { + // not too concerned about gdc here since the mingw version is fairly new as well + version=with_breaking_cgi_features; + } +} + +void cloexec(int fd) { + version(Posix) { + import core.sys.posix.fcntl; + fcntl(fd, F_SETFD, FD_CLOEXEC); + } +} + +void cloexec(Socket s) { + version(Posix) { + import core.sys.posix.fcntl; + fcntl(s.handle, F_SETFD, FD_CLOEXEC); + } +} + +version(embedded_httpd_hybrid) { + version=embedded_httpd_threads; + version(cgi_no_fork) {} else version(Posix) + version=cgi_use_fork; + version=cgi_use_fiber; +} + +version(cgi_use_fork) + enum cgi_use_fork_default = true; +else + enum cgi_use_fork_default = false; + +// the servers must know about the connections to talk to them; the interfaces are vital +version(with_addon_servers) + version=with_addon_servers_connections; + +version(embedded_httpd) { + version(linux) + version=embedded_httpd_processes; + else { + version=embedded_httpd_threads; + } + + /* + version(with_openssl) { + pragma(lib, "crypto"); + pragma(lib, "ssl"); + } + */ +} + +version(embedded_httpd_processes) + version=embedded_httpd_processes_accept_after_fork; // I am getting much better average performance on this, so just keeping it. But the other way MIGHT help keep the variation down so i wanna keep the code to play with later + +version(embedded_httpd_threads) { + // unless the user overrides the default.. + version(cgi_session_server_process) + {} + else + version=cgi_embedded_sessions; +} +version(scgi) { + // unless the user overrides the default.. + version(cgi_session_server_process) + {} + else + version=cgi_embedded_sessions; +} + +// fall back if the other is not defined so we can cleanly version it below +version(cgi_embedded_sessions) {} +else version=cgi_session_server_process; + + +version=cgi_with_websocket; + +enum long defaultMaxContentLength = 5_000_000; + +/* + + To do a file download offer in the browser: + + cgi.setResponseContentType("text/csv"); + cgi.header("Content-Disposition: attachment; filename=\"customers.csv\""); +*/ + +// FIXME: the location header is supposed to be an absolute url I guess. + +// FIXME: would be cool to flush part of a dom document before complete +// somehow in here and dom.d. + + +// these are public so you can mixin GenericMain. +// FIXME: use a function level import instead! +public import std.string; +public import std.stdio; +public import std.conv; +import std.uri; +import std.uni; +import std.algorithm.comparison; +import std.algorithm.searching; +import std.exception; +import std.base64; +static import std.algorithm; +import std.datetime; +import std.range; + +import std.process; + +import std.zlib; + + +T[] consume(T)(T[] range, int count) { + if(count > range.length) + count = range.length; + return range[count..$]; +} + +int locationOf(T)(T[] data, string item) { + const(ubyte[]) d = cast(const(ubyte[])) data; + const(ubyte[]) i = cast(const(ubyte[])) item; + + // this is a vague sanity check to ensure we aren't getting insanely + // sized input that will infinite loop below. it should never happen; + // even huge file uploads ought to come in smaller individual pieces. + if(d.length > (int.max/2)) + throw new Exception("excessive block of input"); + + for(int a = 0; a < d.length; a++) { + if(a + i.length > d.length) + return -1; + if(d[a..a+i.length] == i) + return a; + } + + return -1; +} + +/// If you are doing a custom cgi class, mixing this in can take care of +/// the required constructors for you +mixin template ForwardCgiConstructors() { + this(long maxContentLength = defaultMaxContentLength, + string[string] env = null, + const(ubyte)[] delegate() readdata = null, + void delegate(const(ubyte)[]) _rawDataOutput = null, + void delegate() _flush = null + ) { super(maxContentLength, env, readdata, _rawDataOutput, _flush); } + + this(string[] args) { super(args); } + + this( + BufferedInputRange inputData, + string address, ushort _port, + int pathInfoStarts = 0, + bool _https = false, + void delegate(const(ubyte)[]) _rawDataOutput = null, + void delegate() _flush = null, + // this pointer tells if the connection is supposed to be closed after we handle this + bool* closeConnection = null) + { + super(inputData, address, _port, pathInfoStarts, _https, _rawDataOutput, _flush, closeConnection); + } + + this(BufferedInputRange ir, bool* closeConnection) { super(ir, closeConnection); } +} + +/// thrown when a connection is closed remotely while we waiting on data from it +class ConnectionClosedException : Exception { + this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + super(message, file, line, next); + } +} + + +version(Windows) { +// FIXME: ugly hack to solve stdin exception problems on Windows: +// reading stdin results in StdioException (Bad file descriptor) +// this is probably due to http://d.puremagic.com/issues/show_bug.cgi?id=3425 +private struct stdin { + struct ByChunk { // Replicates std.stdio.ByChunk + private: + ubyte[] chunk_; + public: + this(size_t size) + in { + assert(size, "size must be larger than 0"); + } + do { + chunk_ = new ubyte[](size); + popFront(); + } + + @property bool empty() const { + return !std.stdio.stdin.isOpen || std.stdio.stdin.eof; // Ugly, but seems to do the job + } + @property nothrow ubyte[] front() { return chunk_; } + void popFront() { + enforce(!empty, "Cannot call popFront on empty range"); + chunk_ = stdin.rawRead(chunk_); + } + } + + import core.sys.windows.windows; +static: + + T[] rawRead(T)(T[] buf) { + uint bytesRead; + auto result = ReadFile(GetStdHandle(STD_INPUT_HANDLE), buf.ptr, cast(int) (buf.length * T.sizeof), &bytesRead, null); + + if (!result) { + auto err = GetLastError(); + if (err == 38/*ERROR_HANDLE_EOF*/ || err == 109/*ERROR_BROKEN_PIPE*/) // 'good' errors meaning end of input + return buf[0..0]; + // Some other error, throw it + + char* buffer; + scope(exit) LocalFree(buffer); + + // FORMAT_MESSAGE_ALLOCATE_BUFFER = 0x00000100 + // FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000 + FormatMessageA(0x1100, null, err, 0, cast(char*)&buffer, 256, null); + throw new Exception(to!string(buffer)); + } + enforce(!(bytesRead % T.sizeof), "I/O error"); + return buf[0..bytesRead / T.sizeof]; + } + + auto byChunk(size_t sz) { return ByChunk(sz); } + + void close() { + std.stdio.stdin.close; + } +} +} + +/// The main interface with the web request +class Cgi { + public: + /// the methods a request can be + enum RequestMethod { GET, HEAD, POST, PUT, DELETE, // GET and POST are the ones that really work + // these are defined in the standard, but idk if they are useful for anything + OPTIONS, TRACE, CONNECT, + // These seem new, I have only recently seen them + PATCH, MERGE, + // this is an extension for when the method is not specified and you want to assume + CommandLine } + + + /+ + /++ + Cgi provides a per-request memory pool + + +/ + void[] allocateMemory(size_t nBytes) { + + } + + /// ditto + void[] reallocateMemory(void[] old, size_t nBytes) { + + } + + /// ditto + void freeMemory(void[] memory) { + + } + +/ + + +/* + import core.runtime; + auto args = Runtime.args(); + + we can call the app a few ways: + + 1) set up the environment variables and call the app (manually simulating CGI) + 2) simulate a call automatically: + ./app method 'uri' + + for example: + ./app get /path?arg arg2=something + + Anything on the uri is treated as query string etc + + on get method, further args are appended to the query string (encoded automatically) + on post method, further args are done as post + + + @name means import from file "name". if name == -, it uses stdin + (so info=@- means set info to the value of stdin) + + + Other arguments include: + --cookie name=value (these are all concated together) + --header 'X-Something: cool' + --referrer 'something' + --port 80 + --remote-address some.ip.address.here + --https yes + --user-agent 'something' + --userpass 'user:pass' + --authorization 'Basic base64encoded_user:pass' + --accept 'content' // FIXME: better example + --last-event-id 'something' + --host 'something.com' + + Non-simulation arguments: + --port xxx listening port for non-cgi things (valid for the cgi interfaces) + --listening-host the ip address the application should listen on, or if you want to use unix domain sockets, it is here you can set them: `--listening-host unix:filename` or, on Linux, `--listening-host abstract:name`. + +*/ + + /** Initializes it with command line arguments (for easy testing) */ + this(string[] args, void delegate(const(ubyte)[]) _rawDataOutput = null) { + rawDataOutput = _rawDataOutput; + // these are all set locally so the loop works + // without triggering errors in dmd 2.064 + // we go ahead and set them at the end of it to the this version + int port; + string referrer; + string remoteAddress; + string userAgent; + string authorization; + string origin; + string accept; + string lastEventId; + bool https; + string host; + RequestMethod requestMethod; + string requestUri; + string pathInfo; + string queryString; + + bool lookingForMethod; + bool lookingForUri; + string nextArgIs; + + string _cookie; + string _queryString; + string[][string] _post; + string[string] _headers; + + string[] breakUp(string s) { + string k, v; + auto idx = s.indexOf("="); + if(idx == -1) { + k = s; + } else { + k = s[0 .. idx]; + v = s[idx + 1 .. $]; + } + + return [k, v]; + } + + lookingForMethod = true; + + scriptName = args[0]; + scriptFileName = args[0]; + + environmentVariables = cast(const) environment.toAA; + + foreach(arg; args[1 .. $]) { + if(arg.startsWith("--")) { + nextArgIs = arg[2 .. $]; + } else if(nextArgIs.length) { + if (nextArgIs == "cookie") { + auto info = breakUp(arg); + if(_cookie.length) + _cookie ~= "; "; + _cookie ~= std.uri.encodeComponent(info[0]) ~ "=" ~ std.uri.encodeComponent(info[1]); + } + else if (nextArgIs == "port") { + port = to!int(arg); + } + else if (nextArgIs == "referrer") { + referrer = arg; + } + else if (nextArgIs == "remote-address") { + remoteAddress = arg; + } + else if (nextArgIs == "user-agent") { + userAgent = arg; + } + else if (nextArgIs == "authorization") { + authorization = arg; + } + else if (nextArgIs == "userpass") { + authorization = "Basic " ~ Base64.encode(cast(immutable(ubyte)[]) (arg)).idup; + } + else if (nextArgIs == "origin") { + origin = arg; + } + else if (nextArgIs == "accept") { + accept = arg; + } + else if (nextArgIs == "last-event-id") { + lastEventId = arg; + } + else if (nextArgIs == "https") { + if(arg == "yes") + https = true; + } + else if (nextArgIs == "header") { + string thing, other; + auto idx = arg.indexOf(":"); + if(idx == -1) + throw new Exception("need a colon in a http header"); + thing = arg[0 .. idx]; + other = arg[idx + 1.. $]; + _headers[thing.strip.toLower()] = other.strip; + } + else if (nextArgIs == "host") { + host = arg; + } + // else + // skip, we don't know it but that's ok, it might be used elsewhere so no error + + nextArgIs = null; + } else if(lookingForMethod) { + lookingForMethod = false; + lookingForUri = true; + + if(arg.asLowerCase().equal("commandline")) + requestMethod = RequestMethod.CommandLine; + else + requestMethod = to!RequestMethod(arg.toUpper()); + } else if(lookingForUri) { + lookingForUri = false; + + requestUri = arg; + + auto idx = arg.indexOf("?"); + if(idx == -1) + pathInfo = arg; + else { + pathInfo = arg[0 .. idx]; + _queryString = arg[idx + 1 .. $]; + } + } else { + // it is an argument of some sort + if(requestMethod == Cgi.RequestMethod.POST || requestMethod == Cgi.RequestMethod.PATCH || requestMethod == Cgi.RequestMethod.PUT || requestMethod == Cgi.RequestMethod.CommandLine) { + auto parts = breakUp(arg); + _post[parts[0]] ~= parts[1]; + allPostNamesInOrder ~= parts[0]; + allPostValuesInOrder ~= parts[1]; + } else { + if(_queryString.length) + _queryString ~= "&"; + auto parts = breakUp(arg); + _queryString ~= std.uri.encodeComponent(parts[0]) ~ "=" ~ std.uri.encodeComponent(parts[1]); + } + } + } + + acceptsGzip = false; + keepAliveRequested = false; + requestHeaders = cast(immutable) _headers; + + cookie = _cookie; + cookiesArray = getCookieArray(); + cookies = keepLastOf(cookiesArray); + + queryString = _queryString; + getArray = cast(immutable) decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder); + get = keepLastOf(getArray); + + postArray = cast(immutable) _post; + post = keepLastOf(_post); + + // FIXME + filesArray = null; + files = null; + + isCalledWithCommandLineArguments = true; + + this.port = port; + this.referrer = referrer; + this.remoteAddress = remoteAddress; + this.userAgent = userAgent; + this.authorization = authorization; + this.origin = origin; + this.accept = accept; + this.lastEventId = lastEventId; + this.https = https; + this.host = host; + this.requestMethod = requestMethod; + this.requestUri = requestUri; + this.pathInfo = pathInfo; + this.queryString = queryString; + this.postBody = null; + } + + private { + string[] allPostNamesInOrder; + string[] allPostValuesInOrder; + string[] allGetNamesInOrder; + string[] allGetValuesInOrder; + } + + CgiConnectionHandle getOutputFileHandle() { + return _outputFileHandle; + } + + CgiConnectionHandle _outputFileHandle = INVALID_CGI_CONNECTION_HANDLE; + + /** Initializes it using a CGI or CGI-like interface */ + this(long maxContentLength = defaultMaxContentLength, + // use this to override the environment variable listing + in string[string] env = null, + // and this should return a chunk of data. return empty when done + const(ubyte)[] delegate() readdata = null, + // finally, use this to do custom output if needed + void delegate(const(ubyte)[]) _rawDataOutput = null, + // to flush teh custom output + void delegate() _flush = null + ) + { + + // these are all set locally so the loop works + // without triggering errors in dmd 2.064 + // we go ahead and set them at the end of it to the this version + int port; + string referrer; + string remoteAddress; + string userAgent; + string authorization; + string origin; + string accept; + string lastEventId; + bool https; + string host; + RequestMethod requestMethod; + string requestUri; + string pathInfo; + string queryString; + + + + isCalledWithCommandLineArguments = false; + rawDataOutput = _rawDataOutput; + flushDelegate = _flush; + auto getenv = delegate string(string var) { + if(env is null) + return std.process.environment.get(var); + auto e = var in env; + if(e is null) + return null; + return *e; + }; + + environmentVariables = env is null ? + cast(const) environment.toAA : + env; + + // fetching all the request headers + string[string] requestHeadersHere; + foreach(k, v; env is null ? cast(const) environment.toAA() : env) { + if(k.startsWith("HTTP_")) { + requestHeadersHere[replace(k["HTTP_".length .. $].toLower(), "_", "-")] = v; + } + } + + this.requestHeaders = assumeUnique(requestHeadersHere); + + requestUri = getenv("REQUEST_URI"); + + cookie = getenv("HTTP_COOKIE"); + cookiesArray = getCookieArray(); + cookies = keepLastOf(cookiesArray); + + referrer = getenv("HTTP_REFERER"); + userAgent = getenv("HTTP_USER_AGENT"); + remoteAddress = getenv("REMOTE_ADDR"); + host = getenv("HTTP_HOST"); + pathInfo = getenv("PATH_INFO"); + + queryString = getenv("QUERY_STRING"); + scriptName = getenv("SCRIPT_NAME"); + { + import core.runtime; + auto sfn = getenv("SCRIPT_FILENAME"); + scriptFileName = sfn.length ? sfn : (Runtime.args.length ? Runtime.args[0] : null); + } + + bool iis = false; + + // Because IIS doesn't pass requestUri, we simulate it here if it's empty. + if(requestUri.length == 0) { + // IIS sometimes includes the script name as part of the path info - we don't want that + if(pathInfo.length >= scriptName.length && (pathInfo[0 .. scriptName.length] == scriptName)) + pathInfo = pathInfo[scriptName.length .. $]; + + requestUri = scriptName ~ pathInfo ~ (queryString.length ? ("?" ~ queryString) : ""); + + iis = true; // FIXME HACK - used in byChunk below - see bugzilla 6339 + + // FIXME: this works for apache and iis... but what about others? + } + + + auto ugh = decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder); + getArray = assumeUnique(ugh); + get = keepLastOf(getArray); + + + // NOTE: on shitpache, you need to specifically forward this + authorization = getenv("HTTP_AUTHORIZATION"); + // this is a hack because Apache is a shitload of fuck and + // refuses to send the real header to us. Compatible + // programs should send both the standard and X- versions + + // NOTE: if you have access to .htaccess or httpd.conf, you can make this + // unnecessary with mod_rewrite, so it is commented + + //if(authorization.length == 0) // if the std is there, use it + // authorization = getenv("HTTP_X_AUTHORIZATION"); + + // the REDIRECT_HTTPS check is here because with an Apache hack, the port can become wrong + if(getenv("SERVER_PORT").length && getenv("REDIRECT_HTTPS") != "on") + port = to!int(getenv("SERVER_PORT")); + else + port = 0; // this was probably called from the command line + + auto ae = getenv("HTTP_ACCEPT_ENCODING"); + if(ae.length && ae.indexOf("gzip") != -1) + acceptsGzip = true; + + accept = getenv("HTTP_ACCEPT"); + lastEventId = getenv("HTTP_LAST_EVENT_ID"); + + auto ka = getenv("HTTP_CONNECTION"); + if(ka.length && ka.asLowerCase().canFind("keep-alive")) + keepAliveRequested = true; + + auto or = getenv("HTTP_ORIGIN"); + origin = or; + + auto rm = getenv("REQUEST_METHOD"); + if(rm.length) + requestMethod = to!RequestMethod(getenv("REQUEST_METHOD")); + else + requestMethod = RequestMethod.CommandLine; + + // FIXME: hack on REDIRECT_HTTPS; this is there because the work app uses mod_rewrite which loses the https flag! So I set it with [E=HTTPS=%HTTPS] or whatever but then it gets translated to here so i want it to still work. This is arguably wrong but meh. + https = (getenv("HTTPS") == "on" || getenv("REDIRECT_HTTPS") == "on"); + + // FIXME: DOCUMENT_ROOT? + + // FIXME: what about PUT? + if(requestMethod == RequestMethod.POST || requestMethod == Cgi.RequestMethod.PATCH || requestMethod == Cgi.RequestMethod.PUT || requestMethod == Cgi.RequestMethod.CommandLine) { + version(preserveData) // a hack to make forwarding simpler + immutable(ubyte)[] data; + size_t amountReceived = 0; + auto contentType = getenv("CONTENT_TYPE"); + + // FIXME: is this ever not going to be set? I guess it depends + // on if the server de-chunks and buffers... seems like it has potential + // to be slow if they did that. The spec says it is always there though. + // And it has worked reliably for me all year in the live environment, + // but some servers might be different. + auto cls = getenv("CONTENT_LENGTH"); + auto contentLength = to!size_t(cls.length ? cls : "0"); + + immutable originalContentLength = contentLength; + if(contentLength) { + if(maxContentLength > 0 && contentLength > maxContentLength) { + setResponseStatus("413 Request entity too large"); + write("You tried to upload a file that is too large."); + close(); + throw new Exception("POST too large"); + } + prepareForIncomingDataChunks(contentType, contentLength); + + + int processChunk(in ubyte[] chunk) { + if(chunk.length > contentLength) { + handleIncomingDataChunk(chunk[0..contentLength]); + amountReceived += contentLength; + contentLength = 0; + return 1; + } else { + handleIncomingDataChunk(chunk); + contentLength -= chunk.length; + amountReceived += chunk.length; + } + if(contentLength == 0) + return 1; + + onRequestBodyDataReceived(amountReceived, originalContentLength); + return 0; + } + + + if(readdata is null) { + foreach(ubyte[] chunk; stdin.byChunk(iis ? contentLength : 4096)) + if(processChunk(chunk)) + break; + } else { + // we have a custom data source.. + auto chunk = readdata(); + while(chunk.length) { + if(processChunk(chunk)) + break; + chunk = readdata(); + } + } + + onRequestBodyDataReceived(amountReceived, originalContentLength); + postArray = assumeUnique(pps._post); + filesArray = assumeUnique(pps._files); + files = keepLastOf(filesArray); + post = keepLastOf(postArray); + this.postBody = pps.postBody; + cleanUpPostDataState(); + } + + version(preserveData) + originalPostData = data; + } + // fixme: remote_user script name + + + this.port = port; + this.referrer = referrer; + this.remoteAddress = remoteAddress; + this.userAgent = userAgent; + this.authorization = authorization; + this.origin = origin; + this.accept = accept; + this.lastEventId = lastEventId; + this.https = https; + this.host = host; + this.requestMethod = requestMethod; + this.requestUri = requestUri; + this.pathInfo = pathInfo; + this.queryString = queryString; + } + + /// Cleans up any temporary files. Do not use the object + /// after calling this. + /// + /// NOTE: it is called automatically by GenericMain + // FIXME: this should be called if the constructor fails too, if it has created some garbage... + void dispose() { + foreach(file; files) { + if(!file.contentInMemory) + if(std.file.exists(file.contentFilename)) + std.file.remove(file.contentFilename); + } + } + + private { + struct PostParserState { + string contentType; + string boundary; + string localBoundary; // the ones used at the end or something lol + bool isMultipart; + bool needsSavedBody; + + ulong expectedLength; + ulong contentConsumed; + immutable(ubyte)[] buffer; + + // multipart parsing state + int whatDoWeWant; + bool weHaveAPart; + string[] thisOnesHeaders; + immutable(ubyte)[] thisOnesData; + + string postBody; + + UploadedFile piece; + bool isFile = false; + + size_t memoryCommitted; + + // do NOT keep mutable references to these anywhere! + // I assume they are unique in the constructor once we're all done getting data. + string[][string] _post; + UploadedFile[][string] _files; + } + + PostParserState pps; + } + + /// This represents a file the user uploaded via a POST request. + static struct UploadedFile { + /// If you want to create one of these structs for yourself from some data, + /// use this function. + static UploadedFile fromData(immutable(void)[] data, string name = null) { + Cgi.UploadedFile f; + f.filename = name; + f.content = cast(immutable(ubyte)[]) data; + f.contentInMemory = true; + return f; + } + + string name; /// The name of the form element. + string filename; /// The filename the user set. + string contentType; /// The MIME type the user's browser reported. (Not reliable.) + + /** + For small files, cgi.d will buffer the uploaded file in memory, and make it + directly accessible to you through the content member. I find this very convenient + and somewhat efficient, since it can avoid hitting the disk entirely. (I + often want to inspect and modify the file anyway!) + + I find the file is very large, it is undesirable to eat that much memory just + for a file buffer. In those cases, if you pass a large enough value for maxContentLength + to the constructor so they are accepted, cgi.d will write the content to a temporary + file that you can re-read later. + + You can override this behavior by subclassing Cgi and overriding the protected + handlePostChunk method. Note that the object is not initialized when you + write that method - the http headers are available, but the cgi.post method + is not. You may parse the file as it streams in using this method. + + + Anyway, if the file is small enough to be in memory, contentInMemory will be + set to true, and the content is available in the content member. + + If not, contentInMemory will be set to false, and the content saved in a file, + whose name will be available in the contentFilename member. + + + Tip: if you know you are always dealing with small files, and want the convenience + of ignoring this member, construct Cgi with a small maxContentLength. Then, if + a large file comes in, it simply throws an exception (and HTTP error response) + instead of trying to handle it. + + The default value of maxContentLength in the constructor is for small files. + */ + bool contentInMemory = true; // the default ought to always be true + immutable(ubyte)[] content; /// The actual content of the file, if contentInMemory == true + string contentFilename; /// the file where we dumped the content, if contentInMemory == false. Note that if you want to keep it, you MUST move the file, since otherwise it is considered garbage when cgi is disposed. + + /// + ulong fileSize() { + if(contentInMemory) + return content.length; + import std.file; + return std.file.getSize(contentFilename); + + } + + /// + void writeToFile(string filenameToSaveTo) const { + import std.file; + if(contentInMemory) + std.file.write(filenameToSaveTo, content); + else + std.file.rename(contentFilename, filenameToSaveTo); + } + } + + // given a content type and length, decide what we're going to do with the data.. + protected void prepareForIncomingDataChunks(string contentType, ulong contentLength) { + pps.expectedLength = contentLength; + + auto terminator = contentType.indexOf(";"); + if(terminator == -1) + terminator = contentType.length; + + pps.contentType = contentType[0 .. terminator]; + auto b = contentType[terminator .. $]; + if(b.length) { + auto idx = b.indexOf("boundary="); + if(idx != -1) { + pps.boundary = b[idx + "boundary=".length .. $]; + pps.localBoundary = "\r\n--" ~ pps.boundary; + } + } + + // while a content type SHOULD be sent according to the RFC, it is + // not required. We're told we SHOULD guess by looking at the content + // but it seems to me that this only happens when it is urlencoded. + if(pps.contentType == "application/x-www-form-urlencoded" || pps.contentType == "") { + pps.isMultipart = false; + pps.needsSavedBody = false; + } else if(pps.contentType == "multipart/form-data") { + pps.isMultipart = true; + enforce(pps.boundary.length, "no boundary"); + } else if(pps.contentType == "text/xml") { // FIXME: could this be special and load the post params + // save the body so the application can handle it + pps.isMultipart = false; + pps.needsSavedBody = true; + } else if(pps.contentType == "application/json") { // FIXME: this could prolly try to load post params too + // save the body so the application can handle it + pps.needsSavedBody = true; + pps.isMultipart = false; + } else { + // the rest is 100% handled by the application. just save the body and send it to them + pps.needsSavedBody = true; + pps.isMultipart = false; + } + } + + // handles streaming POST data. If you handle some other content type, you should + // override this. If the data isn't the content type you want, you ought to call + // super.handleIncomingDataChunk so regular forms and files still work. + + // FIXME: I do some copying in here that I'm pretty sure is unnecessary, and the + // file stuff I'm sure is inefficient. But, my guess is the real bottleneck is network + // input anyway, so I'm not going to get too worked up about it right now. + protected void handleIncomingDataChunk(const(ubyte)[] chunk) { + if(chunk.length == 0) + return; + assert(chunk.length <= 32 * 1024 * 1024); // we use chunk size as a memory constraint thing, so + // if we're passed big chunks, it might throw unnecessarily. + // just pass it smaller chunks at a time. + if(pps.isMultipart) { + // multipart/form-data + + + // FIXME: this might want to be factored out and factorized + // need to make sure the stream hooks actually work. + void pieceHasNewContent() { + // we just grew the piece's buffer. Do we have to switch to file backing? + if(pps.piece.contentInMemory) { + if(pps.piece.content.length <= 10 * 1024 * 1024) + // meh, I'm ok with it. + return; + else { + // this is too big. + if(!pps.isFile) + throw new Exception("Request entity too large"); // a variable this big is kinda ridiculous, just reject it. + else { + // a file this large is probably acceptable though... let's use a backing file. + pps.piece.contentInMemory = false; + // FIXME: say... how do we intend to delete these things? cgi.dispose perhaps. + + int count = 0; + pps.piece.contentFilename = getTempDirectory() ~ "arsd_cgi_uploaded_file_" ~ to!string(getUtcTime()) ~ "-" ~ to!string(count); + // odds are this loop will never be entered, but we want it just in case. + while(std.file.exists(pps.piece.contentFilename)) { + count++; + pps.piece.contentFilename = getTempDirectory() ~ "arsd_cgi_uploaded_file_" ~ to!string(getUtcTime()) ~ "-" ~ to!string(count); + } + // I hope this creates the file pretty quickly, or the loop might be useless... + // FIXME: maybe I should write some kind of custom transaction here. + std.file.write(pps.piece.contentFilename, pps.piece.content); + + pps.piece.content = null; + } + } + } else { + // it's already in a file, so just append it to what we have + if(pps.piece.content.length) { + // FIXME: this is surely very inefficient... we'll be calling this by 4kb chunk... + std.file.append(pps.piece.contentFilename, pps.piece.content); + pps.piece.content = null; + } + } + } + + + void commitPart() { + if(!pps.weHaveAPart) + return; + + pieceHasNewContent(); // be sure the new content is handled every time + + if(pps.isFile) { + // I'm not sure if other environments put files in post or not... + // I used to not do it, but I think I should, since it is there... + pps._post[pps.piece.name] ~= pps.piece.filename; + pps._files[pps.piece.name] ~= pps.piece; + + allPostNamesInOrder ~= pps.piece.name; + allPostValuesInOrder ~= pps.piece.filename; + } else { + pps._post[pps.piece.name] ~= cast(string) pps.piece.content; + + allPostNamesInOrder ~= pps.piece.name; + allPostValuesInOrder ~= cast(string) pps.piece.content; + } + + /* + stderr.writeln("RECEIVED: ", pps.piece.name, "=", + pps.piece.content.length < 1000 + ? + to!string(pps.piece.content) + : + "too long"); + */ + + // FIXME: the limit here + pps.memoryCommitted += pps.piece.content.length; + + pps.weHaveAPart = false; + pps.whatDoWeWant = 1; + pps.thisOnesHeaders = null; + pps.thisOnesData = null; + + pps.piece = UploadedFile.init; + pps.isFile = false; + } + + void acceptChunk() { + pps.buffer ~= chunk; + chunk = null; // we've consumed it into the buffer, so keeping it just brings confusion + } + + immutable(ubyte)[] consume(size_t howMuch) { + pps.contentConsumed += howMuch; + auto ret = pps.buffer[0 .. howMuch]; + pps.buffer = pps.buffer[howMuch .. $]; + return ret; + } + + dataConsumptionLoop: do { + switch(pps.whatDoWeWant) { + default: assert(0); + case 0: + acceptChunk(); + // the format begins with two extra leading dashes, then we should be at the boundary + if(pps.buffer.length < 2) + return; + assert(pps.buffer[0] == '-', "no leading dash"); + consume(1); + assert(pps.buffer[0] == '-', "no second leading dash"); + consume(1); + + pps.whatDoWeWant = 1; + goto case 1; + /* fallthrough */ + case 1: // looking for headers + // here, we should be lined up right at the boundary, which is followed by a \r\n + + // want to keep the buffer under control in case we're under attack + //stderr.writeln("here once"); + //if(pps.buffer.length + chunk.length > 70 * 1024) // they should be < 1 kb really.... + // throw new Exception("wtf is up with the huge mime part headers"); + + acceptChunk(); + + if(pps.buffer.length < pps.boundary.length) + return; // not enough data, since there should always be a boundary here at least + + if(pps.contentConsumed + pps.boundary.length + 6 == pps.expectedLength) { + assert(pps.buffer.length == pps.boundary.length + 4 + 2); // --, --, and \r\n + // we *should* be at the end here! + assert(pps.buffer[0] == '-'); + consume(1); + assert(pps.buffer[0] == '-'); + consume(1); + + // the message is terminated by --BOUNDARY--\r\n (after a \r\n leading to the boundary) + assert(pps.buffer[0 .. pps.boundary.length] == cast(const(ubyte[])) pps.boundary, + "not lined up on boundary " ~ pps.boundary); + consume(pps.boundary.length); + + assert(pps.buffer[0] == '-'); + consume(1); + assert(pps.buffer[0] == '-'); + consume(1); + + assert(pps.buffer[0] == '\r'); + consume(1); + assert(pps.buffer[0] == '\n'); + consume(1); + + assert(pps.buffer.length == 0); + assert(pps.contentConsumed == pps.expectedLength); + break dataConsumptionLoop; // we're done! + } else { + // we're not done yet. We should be lined up on a boundary. + + // But, we want to ensure the headers are here before we consume anything! + auto headerEndLocation = locationOf(pps.buffer, "\r\n\r\n"); + if(headerEndLocation == -1) + return; // they *should* all be here, so we can handle them all at once. + + assert(pps.buffer[0 .. pps.boundary.length] == cast(const(ubyte[])) pps.boundary, + "not lined up on boundary " ~ pps.boundary); + + consume(pps.boundary.length); + // the boundary is always followed by a \r\n + assert(pps.buffer[0] == '\r'); + consume(1); + assert(pps.buffer[0] == '\n'); + consume(1); + } + + // re-running since by consuming the boundary, we invalidate the old index. + auto headerEndLocation = locationOf(pps.buffer, "\r\n\r\n"); + assert(headerEndLocation >= 0, "no header"); + auto thisOnesHeaders = pps.buffer[0..headerEndLocation]; + + consume(headerEndLocation + 4); // The +4 is the \r\n\r\n that caps it off + + pps.thisOnesHeaders = split(cast(string) thisOnesHeaders, "\r\n"); + + // now we'll parse the headers + foreach(h; pps.thisOnesHeaders) { + auto p = h.indexOf(":"); + assert(p != -1, "no colon in header, got " ~ to!string(pps.thisOnesHeaders)); + string hn = h[0..p]; + string hv = h[p+2..$]; + + switch(hn.toLower) { + default: assert(0); + case "content-disposition": + auto info = hv.split("; "); + foreach(i; info[1..$]) { // skipping the form-data + auto o = i.split("="); // FIXME + string pn = o[0]; + string pv = o[1][1..$-1]; + + if(pn == "name") { + pps.piece.name = pv; + } else if (pn == "filename") { + pps.piece.filename = pv; + pps.isFile = true; + } + } + break; + case "content-type": + pps.piece.contentType = hv; + break; + } + } + + pps.whatDoWeWant++; // move to the next step - the data + break; + case 2: + // when we get here, pps.buffer should contain our first chunk of data + + if(pps.buffer.length + chunk.length > 8 * 1024 * 1024) // we might buffer quite a bit but not much + throw new Exception("wtf is up with the huge mime part buffer"); + + acceptChunk(); + + // so the trick is, we want to process all the data up to the boundary, + // but what if the chunk's end cuts the boundary off? If we're unsure, we + // want to wait for the next chunk. We start by looking for the whole boundary + // in the buffer somewhere. + + auto boundaryLocation = locationOf(pps.buffer, pps.localBoundary); + // assert(boundaryLocation != -1, "should have seen "~to!string(cast(ubyte[]) pps.localBoundary)~" in " ~ to!string(pps.buffer)); + if(boundaryLocation != -1) { + // this is easy - we can see it in it's entirety! + + pps.piece.content ~= consume(boundaryLocation); + + assert(pps.buffer[0] == '\r'); + consume(1); + assert(pps.buffer[0] == '\n'); + consume(1); + assert(pps.buffer[0] == '-'); + consume(1); + assert(pps.buffer[0] == '-'); + consume(1); + // the boundary here is always preceded by \r\n--, which is why we used localBoundary instead of boundary to locate it. Cut that off. + pps.weHaveAPart = true; + pps.whatDoWeWant = 1; // back to getting headers for the next part + + commitPart(); // we're done here + } else { + // we can't see the whole thing, but what if there's a partial boundary? + + enforce(pps.localBoundary.length < 128); // the boundary ought to be less than a line... + assert(pps.localBoundary.length > 1); // should already be sane but just in case + bool potentialBoundaryFound = false; + + boundaryCheck: for(int a = 1; a < pps.localBoundary.length; a++) { + // we grow the boundary a bit each time. If we think it looks the + // same, better pull another chunk to be sure it's not the end. + // Starting small because exiting the loop early is desirable, since + // we're not keeping any ambiguity and 1 / 256 chance of exiting is + // the best we can do. + if(a > pps.buffer.length) + break; // FIXME: is this right? + assert(a <= pps.buffer.length); + assert(a > 0); + if(std.algorithm.endsWith(pps.buffer, pps.localBoundary[0 .. a])) { + // ok, there *might* be a boundary here, so let's + // not treat the end as data yet. The rest is good to + // use though, since if there was a boundary there, we'd + // have handled it up above after locationOf. + + pps.piece.content ~= pps.buffer[0 .. $ - a]; + consume(pps.buffer.length - a); + pieceHasNewContent(); + potentialBoundaryFound = true; + break boundaryCheck; + } + } + + if(!potentialBoundaryFound) { + // we can consume the whole thing + pps.piece.content ~= pps.buffer; + pieceHasNewContent(); + consume(pps.buffer.length); + } else { + // we found a possible boundary, but there was + // insufficient data to be sure. + assert(pps.buffer == cast(const(ubyte[])) pps.localBoundary[0 .. pps.buffer.length]); + + return; // wait for the next chunk. + } + } + } + } while(pps.buffer.length); + + // btw all boundaries except the first should have a \r\n before them + } else { + // application/x-www-form-urlencoded and application/json + + // not using maxContentLength because that might be cranked up to allow + // large file uploads. We can handle them, but a huge post[] isn't any good. + if(pps.buffer.length + chunk.length > 8 * 1024 * 1024) // surely this is plenty big enough + throw new Exception("wtf is up with such a gigantic form submission????"); + + pps.buffer ~= chunk; + + // simple handling, but it works... until someone bombs us with gigabytes of crap at least... + if(pps.buffer.length == pps.expectedLength) { + if(pps.needsSavedBody) + pps.postBody = cast(string) pps.buffer; + else + pps._post = decodeVariables(cast(string) pps.buffer, "&", &allPostNamesInOrder, &allPostValuesInOrder); + version(preserveData) + originalPostData = pps.buffer; + } else { + // just for debugging + } + } + } + + protected void cleanUpPostDataState() { + pps = PostParserState.init; + } + + /// you can override this function to somehow react + /// to an upload in progress. + /// + /// Take note that parts of the CGI object is not yet + /// initialized! Stuff from HTTP headers, including get[], is usable. + /// But, none of post[] is usable, and you cannot write here. That's + /// why this method is const - mutating the object won't do much anyway. + /// + /// My idea here was so you can output a progress bar or + /// something to a cooperative client (see arsd.rtud for a potential helper) + /// + /// The default is to do nothing. Subclass cgi and use the + /// CustomCgiMain mixin to do something here. + void onRequestBodyDataReceived(size_t receivedSoFar, size_t totalExpected) const { + // This space intentionally left blank. + } + + /// Initializes the cgi from completely raw HTTP data. The ir must have a Socket source. + /// *closeConnection will be set to true if you should close the connection after handling this request + this(BufferedInputRange ir, bool* closeConnection) { + isCalledWithCommandLineArguments = false; + import al = std.algorithm; + + immutable(ubyte)[] data; + + void rdo(const(ubyte)[] d) { + //import std.stdio; writeln(d); + sendAll(ir.source, d); + } + + auto ira = ir.source.remoteAddress(); + auto irLocalAddress = ir.source.localAddress(); + + ushort port = 80; + if(auto ia = cast(InternetAddress) irLocalAddress) { + port = ia.port; + } else if(auto ia = cast(Internet6Address) irLocalAddress) { + port = ia.port; + } + + // that check for UnixAddress is to work around a Phobos bug + // see: https://github.com/dlang/phobos/pull/7383 + // but this might be more useful anyway tbh for this case + version(Posix) + this(ir, ira is null ? null : cast(UnixAddress) ira ? "unix:" : ira.toString(), port, 0, false, &rdo, null, closeConnection); + else + this(ir, ira is null ? null : ira.toString(), port, 0, false, &rdo, null, closeConnection); + } + + /** + Initializes it from raw HTTP request data. GenericMain uses this when you compile with -version=embedded_httpd. + + NOTE: If you are behind a reverse proxy, the values here might not be what you expect.... it will use X-Forwarded-For for remote IP and X-Forwarded-Host for host + + Params: + inputData = the incoming data, including headers and other raw http data. + When the constructor exits, it will leave this range exactly at the start of + the next request on the connection (if there is one). + + address = the IP address of the remote user + _port = the port number of the connection + pathInfoStarts = the offset into the path component of the http header where the SCRIPT_NAME ends and the PATH_INFO begins. + _https = if this connection is encrypted (note that the input data must not actually be encrypted) + _rawDataOutput = delegate to accept response data. It should write to the socket or whatever; Cgi does all the needed processing to speak http. + _flush = if _rawDataOutput buffers, this delegate should flush the buffer down the wire + closeConnection = if the request asks to close the connection, *closeConnection == true. + */ + this( + BufferedInputRange inputData, +// string[] headers, immutable(ubyte)[] data, + string address, ushort _port, + int pathInfoStarts = 0, // use this if you know the script name, like if this is in a folder in a bigger web environment + bool _https = false, + void delegate(const(ubyte)[]) _rawDataOutput = null, + void delegate() _flush = null, + // this pointer tells if the connection is supposed to be closed after we handle this + bool* closeConnection = null) + { + // these are all set locally so the loop works + // without triggering errors in dmd 2.064 + // we go ahead and set them at the end of it to the this version + int port; + string referrer; + string remoteAddress; + string userAgent; + string authorization; + string origin; + string accept; + string lastEventId; + bool https; + string host; + RequestMethod requestMethod; + string requestUri; + string pathInfo; + string queryString; + string scriptName; + string[string] get; + string[][string] getArray; + bool keepAliveRequested; + bool acceptsGzip; + string cookie; + + + + environmentVariables = cast(const) environment.toAA; + + idlol = inputData; + + isCalledWithCommandLineArguments = false; + + https = _https; + port = _port; + + rawDataOutput = _rawDataOutput; + flushDelegate = _flush; + nph = true; + + remoteAddress = address; + + // streaming parser + import al = std.algorithm; + + // FIXME: tis cast is technically wrong, but Phobos deprecated al.indexOf... for some reason. + auto idx = indexOf(cast(string) inputData.front(), "\r\n\r\n"); + while(idx == -1) { + inputData.popFront(0); + idx = indexOf(cast(string) inputData.front(), "\r\n\r\n"); + } + + assert(idx != -1); + + + string contentType = ""; + string[string] requestHeadersHere; + + size_t contentLength; + + bool isChunked; + + { + import core.runtime; + scriptFileName = Runtime.args.length ? Runtime.args[0] : null; + } + + + int headerNumber = 0; + foreach(line; al.splitter(inputData.front()[0 .. idx], "\r\n")) + if(line.length) { + headerNumber++; + auto header = cast(string) line.idup; + if(headerNumber == 1) { + // request line + auto parts = al.splitter(header, " "); + requestMethod = to!RequestMethod(parts.front); + parts.popFront(); + requestUri = parts.front; + + // FIXME: the requestUri could be an absolute path!!! should I rename it or something? + scriptName = requestUri[0 .. pathInfoStarts]; + + auto question = requestUri.indexOf("?"); + if(question == -1) { + queryString = ""; + // FIXME: double check, this might be wrong since it could be url encoded + pathInfo = requestUri[pathInfoStarts..$]; + } else { + queryString = requestUri[question+1..$]; + pathInfo = requestUri[pathInfoStarts..question]; + } + + auto ugh = decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder); + getArray = cast(string[][string]) assumeUnique(ugh); + + if(header.indexOf("HTTP/1.0") != -1) { + http10 = true; + autoBuffer = true; + if(closeConnection) { + // on http 1.0, close is assumed (unlike http/1.1 where we assume keep alive) + *closeConnection = true; + } + } + } else { + // other header + auto colon = header.indexOf(":"); + if(colon == -1) + throw new Exception("HTTP headers should have a colon!"); + string name = header[0..colon].toLower; + string value = header[colon+2..$]; // skip the colon and the space + + requestHeadersHere[name] = value; + + if (name == "accept") { + accept = value; + } + else if (name == "origin") { + origin = value; + } + else if (name == "connection") { + if(value == "close" && closeConnection) + *closeConnection = true; + if(value.asLowerCase().canFind("keep-alive")) { + keepAliveRequested = true; + + // on http 1.0, the connection is closed by default, + // but not if they request keep-alive. then we don't close + // anymore - undoing the set above + if(http10 && closeConnection) { + *closeConnection = false; + } + } + } + else if (name == "transfer-encoding") { + if(value == "chunked") + isChunked = true; + } + else if (name == "last-event-id") { + lastEventId = value; + } + else if (name == "authorization") { + authorization = value; + } + else if (name == "content-type") { + contentType = value; + } + else if (name == "content-length") { + contentLength = to!size_t(value); + } + else if (name == "x-forwarded-for") { + remoteAddress = value; + } + else if (name == "x-forwarded-host" || name == "host") { + if(name != "host" || host is null) + host = value; + } + // FIXME: https://tools.ietf.org/html/rfc7239 + else if (name == "accept-encoding") { + if(value.indexOf("gzip") != -1) + acceptsGzip = true; + } + else if (name == "user-agent") { + userAgent = value; + } + else if (name == "referer") { + referrer = value; + } + else if (name == "cookie") { + cookie ~= value; + } else if(name == "expect") { + if(value == "100-continue") { + // FIXME we should probably give user code a chance + // to process and reject but that needs to be virtual, + // perhaps part of the CGI redesign. + + // FIXME: if size is > max content length it should + // also fail at this point. + _rawDataOutput(cast(ubyte[]) "HTTP/1.1 100 Continue\r\n\r\n"); + + // FIXME: let the user write out 103 early hints too + } + } + // else + // ignore it + + } + } + + inputData.consume(idx + 4); + // done + + requestHeaders = assumeUnique(requestHeadersHere); + + ByChunkRange dataByChunk; + + // reading Content-Length type data + // We need to read up the data we have, and write it out as a chunk. + if(!isChunked) { + dataByChunk = byChunk(inputData, contentLength); + } else { + // chunked requests happen, but not every day. Since we need to know + // the content length (for now, maybe that should change), we'll buffer + // the whole thing here instead of parse streaming. (I think this is what Apache does anyway in cgi modes) + auto data = dechunk(inputData); + + // set the range here + dataByChunk = byChunk(data); + contentLength = data.length; + } + + assert(dataByChunk !is null); + + if(contentLength) { + prepareForIncomingDataChunks(contentType, contentLength); + foreach(dataChunk; dataByChunk) { + handleIncomingDataChunk(dataChunk); + } + postArray = assumeUnique(pps._post); + filesArray = assumeUnique(pps._files); + files = keepLastOf(filesArray); + post = keepLastOf(postArray); + postBody = pps.postBody; + cleanUpPostDataState(); + } + + this.port = port; + this.referrer = referrer; + this.remoteAddress = remoteAddress; + this.userAgent = userAgent; + this.authorization = authorization; + this.origin = origin; + this.accept = accept; + this.lastEventId = lastEventId; + this.https = https; + this.host = host; + this.requestMethod = requestMethod; + this.requestUri = requestUri; + this.pathInfo = pathInfo; + this.queryString = queryString; + + this.scriptName = scriptName; + this.get = keepLastOf(getArray); + this.getArray = cast(immutable) getArray; + this.keepAliveRequested = keepAliveRequested; + this.acceptsGzip = acceptsGzip; + this.cookie = cookie; + + cookiesArray = getCookieArray(); + cookies = keepLastOf(cookiesArray); + + } + BufferedInputRange idlol; + + private immutable(string[string]) keepLastOf(in string[][string] arr) { + string[string] ca; + foreach(k, v; arr) + ca[k] = v[$-1]; + + return assumeUnique(ca); + } + + // FIXME duplication + private immutable(UploadedFile[string]) keepLastOf(in UploadedFile[][string] arr) { + UploadedFile[string] ca; + foreach(k, v; arr) + ca[k] = v[$-1]; + + return assumeUnique(ca); + } + + + private immutable(string[][string]) getCookieArray() { + auto forTheLoveOfGod = decodeVariables(cookie, "; "); + return assumeUnique(forTheLoveOfGod); + } + + /// Very simple method to require a basic auth username and password. + /// If the http request doesn't include the required credentials, it throws a + /// HTTP 401 error, and an exception. + /// + /// Note: basic auth does not provide great security, especially over unencrypted HTTP; + /// the user's credentials are sent in plain text on every request. + /// + /// If you are using Apache, the HTTP_AUTHORIZATION variable may not be sent to the + /// application. Either use Apache's built in methods for basic authentication, or add + /// something along these lines to your server configuration: + /// + /// RewriteEngine On + /// RewriteCond %{HTTP:Authorization} ^(.*) + /// RewriteRule ^(.*) - [E=HTTP_AUTHORIZATION:%1] + /// + /// To ensure the necessary data is available to cgi.d. + void requireBasicAuth(string user, string pass, string message = null) { + if(authorization != "Basic " ~ Base64.encode(cast(immutable(ubyte)[]) (user ~ ":" ~ pass))) { + setResponseStatus("401 Authorization Required"); + header ("WWW-Authenticate: Basic realm=\""~message~"\""); + close(); + throw new Exception("Not authorized; got " ~ authorization); + } + } + + /// Very simple caching controls - setCache(false) means it will never be cached. Good for rapidly updated or sensitive sites. + /// setCache(true) means it will always be cached for as long as possible. Best for static content. + /// Use setResponseExpires and updateResponseExpires for more control + void setCache(bool allowCaching) { + noCache = !allowCaching; + } + + /// Set to true and use cgi.write(data, true); to send a gzipped response to browsers + /// who can accept it + bool gzipResponse; + + immutable bool acceptsGzip; + immutable bool keepAliveRequested; + + /// Set to true if and only if this was initialized with command line arguments + immutable bool isCalledWithCommandLineArguments; + + /// This gets a full url for the current request, including port, protocol, host, path, and query + string getCurrentCompleteUri() const { + ushort defaultPort = https ? 443 : 80; + + string uri = "http"; + if(https) + uri ~= "s"; + uri ~= "://"; + uri ~= host; + /+ // the host has the port so p sure this never needed, cgi on apache and embedded http all do the right hting now + version(none) + if(!(!port || port == defaultPort)) { + uri ~= ":"; + uri ~= to!string(port); + } + +/ + uri ~= requestUri; + return uri; + } + + /// You can override this if your site base url isn't the same as the script name + string logicalScriptName() const { + return scriptName; + } + + /++ + Sets the HTTP status of the response. For example, "404 File Not Found" or "500 Internal Server Error". + It assumes "200 OK", and automatically changes to "302 Found" if you call setResponseLocation(). + Note setResponseStatus() must be called *before* you write() any data to the output. + + History: + The `int` overload was added on January 11, 2021. + +/ + void setResponseStatus(string status) { + assert(!outputtedResponseData); + responseStatus = status; + } + /// ditto + void setResponseStatus(int statusCode) { + setResponseStatus(getHttpCodeText(statusCode)); + } + private string responseStatus = null; + + /// Returns true if it is still possible to output headers + bool canOutputHeaders() { + return !isClosed && !outputtedResponseData; + } + + /// Sets the location header, which the browser will redirect the user to automatically. + /// Note setResponseLocation() must be called *before* you write() any data to the output. + /// The optional important argument is used if it's a default suggestion rather than something to insist upon. + void setResponseLocation(string uri, bool important = true, string status = null) { + if(!important && isCurrentResponseLocationImportant) + return; // important redirects always override unimportant ones + + if(uri is null) { + responseStatus = "200 OK"; + responseLocation = null; + isCurrentResponseLocationImportant = important; + return; // this just cancels the redirect + } + + assert(!outputtedResponseData); + if(status is null) + responseStatus = "302 Found"; + else + responseStatus = status; + + responseLocation = uri.strip; + isCurrentResponseLocationImportant = important; + } + protected string responseLocation = null; + private bool isCurrentResponseLocationImportant = false; + + /// Sets the Expires: http header. See also: updateResponseExpires, setPublicCaching + /// The parameter is in unix_timestamp * 1000. Try setResponseExpires(getUTCtime() + SOME AMOUNT) for normal use. + /// Note: the when parameter is different than setCookie's expire parameter. + void setResponseExpires(long when, bool isPublic = false) { + responseExpires = when; + setCache(true); // need to enable caching so the date has meaning + + responseIsPublic = isPublic; + responseExpiresRelative = false; + } + + /// Sets a cache-control max-age header for whenFromNow, in seconds. + void setResponseExpiresRelative(int whenFromNow, bool isPublic = false) { + responseExpires = whenFromNow; + setCache(true); // need to enable caching so the date has meaning + + responseIsPublic = isPublic; + responseExpiresRelative = true; + } + private long responseExpires = long.min; + private bool responseIsPublic = false; + private bool responseExpiresRelative = false; + + /// This is like setResponseExpires, but it can be called multiple times. The setting most in the past is the one kept. + /// If you have multiple functions, they all might call updateResponseExpires about their own return value. The program + /// output as a whole is as cacheable as the least cachable part in the chain. + + /// setCache(false) always overrides this - it is, by definition, the strictest anti-cache statement available. If your site outputs sensitive user data, you should probably call setCache(false) when you do, to ensure no other functions will cache the content, as it may be a privacy risk. + /// Conversely, setting here overrides setCache(true), since any expiration date is in the past of infinity. + void updateResponseExpires(long when, bool isPublic) { + if(responseExpires == long.min) + setResponseExpires(when, isPublic); + else if(when < responseExpires) + setResponseExpires(when, responseIsPublic && isPublic); // if any part of it is private, it all is + } + + /* + /// Set to true if you want the result to be cached publically - that is, is the content shared? + /// Should generally be false if the user is logged in. It assumes private cache only. + /// setCache(true) also turns on public caching, and setCache(false) sets to private. + void setPublicCaching(bool allowPublicCaches) { + publicCaching = allowPublicCaches; + } + private bool publicCaching = false; + */ + + /++ + History: + Added January 11, 2021 + +/ + enum SameSitePolicy { + Lax, + Strict, + None + } + + /++ + Sets an HTTP cookie, automatically encoding the data to the correct string. + expiresIn is how many milliseconds in the future the cookie will expire. + TIP: to make a cookie accessible from subdomains, set the domain to .yourdomain.com. + Note setCookie() must be called *before* you write() any data to the output. + + History: + Parameter `sameSitePolicy` was added on January 11, 2021. + +/ + void setCookie(string name, string data, long expiresIn = 0, string path = null, string domain = null, bool httpOnly = false, bool secure = false, SameSitePolicy sameSitePolicy = SameSitePolicy.Lax) { + assert(!outputtedResponseData); + string cookie = std.uri.encodeComponent(name) ~ "="; + cookie ~= std.uri.encodeComponent(data); + if(path !is null) + cookie ~= "; path=" ~ path; + // FIXME: should I just be using max-age here? (also in cache below) + if(expiresIn != 0) + cookie ~= "; expires=" ~ printDate(cast(DateTime) Clock.currTime(UTC()) + dur!"msecs"(expiresIn)); + if(domain !is null) + cookie ~= "; domain=" ~ domain; + if(secure == true) + cookie ~= "; Secure"; + if(httpOnly == true ) + cookie ~= "; HttpOnly"; + final switch(sameSitePolicy) { + case SameSitePolicy.Lax: + cookie ~= "; SameSite=Lax"; + break; + case SameSitePolicy.Strict: + cookie ~= "; SameSite=Strict"; + break; + case SameSitePolicy.None: + cookie ~= "; SameSite=None"; + assert(secure); // cookie spec requires this now, see: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite + break; + } + + if(auto idx = name in cookieIndexes) { + responseCookies[*idx] = cookie; + } else { + cookieIndexes[name] = responseCookies.length; + responseCookies ~= cookie; + } + } + private string[] responseCookies; + private size_t[string] cookieIndexes; + + /// Clears a previously set cookie with the given name, path, and domain. + void clearCookie(string name, string path = null, string domain = null) { + assert(!outputtedResponseData); + setCookie(name, "", 1, path, domain); + } + + /// Sets the content type of the response, for example "text/html" (the default) for HTML, or "image/png" for a PNG image + void setResponseContentType(string ct) { + assert(!outputtedResponseData); + responseContentType = ct; + } + private string responseContentType = null; + + /// Adds a custom header. It should be the name: value, but without any line terminator. + /// For example: header("X-My-Header: Some value"); + /// Note you should use the specialized functions in this object if possible to avoid + /// duplicates in the output. + void header(string h) { + customHeaders ~= h; + } + + /++ + I named the original function `header` after PHP, but this pattern more fits + the rest of the Cgi object. + + Either name are allowed. + + History: + Alias added June 17, 2022. + +/ + alias setResponseHeader = header; + + private string[] customHeaders; + private bool websocketMode; + + void flushHeaders(const(void)[] t, bool isAll = false) { + StackBuffer buffer = StackBuffer(0); + + prepHeaders(t, isAll, &buffer); + + if(rawDataOutput !is null) + rawDataOutput(cast(const(ubyte)[]) buffer.get()); + else { + stdout.rawWrite(buffer.get()); + } + } + + private void prepHeaders(const(void)[] t, bool isAll, StackBuffer* buffer) { + string terminator = "\n"; + if(rawDataOutput !is null) + terminator = "\r\n"; + + if(responseStatus !is null) { + if(nph) { + if(http10) + buffer.add("HTTP/1.0 ", responseStatus, terminator); + else + buffer.add("HTTP/1.1 ", responseStatus, terminator); + } else + buffer.add("Status: ", responseStatus, terminator); + } else if (nph) { + if(http10) + buffer.add("HTTP/1.0 200 OK", terminator); + else + buffer.add("HTTP/1.1 200 OK", terminator); + } + + if(websocketMode) + goto websocket; + + if(nph) { // we're responsible for setting the date too according to http 1.1 + char[29] db = void; + printDateToBuffer(cast(DateTime) Clock.currTime(UTC()), db[]); + buffer.add("Date: ", db[], terminator); + } + + // FIXME: what if the user wants to set his own content-length? + // The custom header function can do it, so maybe that's best. + // Or we could reuse the isAll param. + if(responseLocation !is null) { + buffer.add("Location: ", responseLocation, terminator); + } + if(!noCache && responseExpires != long.min) { // an explicit expiration date is set + if(responseExpiresRelative) { + buffer.add("Cache-Control: ", responseIsPublic ? "public" : "private", ", max-age="); + buffer.add(responseExpires); + buffer.add(", no-cache=\"set-cookie, set-cookie2\"", terminator); + } else { + auto expires = SysTime(unixTimeToStdTime(cast(int)(responseExpires / 1000)), UTC()); + char[29] db = void; + printDateToBuffer(cast(DateTime) expires, db[]); + buffer.add("Expires: ", db[], terminator); + // FIXME: assuming everything is private unless you use nocache - generally right for dynamic pages, but not necessarily + buffer.add("Cache-Control: ", (responseIsPublic ? "public" : "private"), ", no-cache=\"set-cookie, set-cookie2\""); + buffer.add(terminator); + } + } + if(responseCookies !is null && responseCookies.length > 0) { + foreach(c; responseCookies) + buffer.add("Set-Cookie: ", c, terminator); + } + if(noCache) { // we specifically do not want caching (this is actually the default) + buffer.add("Cache-Control: private, no-cache=\"set-cookie\"", terminator); + buffer.add("Expires: 0", terminator); + buffer.add("Pragma: no-cache", terminator); + } else { + if(responseExpires == long.min) { // caching was enabled, but without a date set - that means assume cache forever + buffer.add("Cache-Control: public", terminator); + buffer.add("Expires: Tue, 31 Dec 2030 14:00:00 GMT", terminator); // FIXME: should not be more than one year in the future + } + } + if(responseContentType !is null) { + buffer.add("Content-Type: ", responseContentType, terminator); + } else + buffer.add("Content-Type: text/html; charset=utf-8", terminator); + + if(gzipResponse && acceptsGzip && isAll) { // FIXME: isAll really shouldn't be necessary + buffer.add("Content-Encoding: gzip", terminator); + } + + + if(!isAll) { + if(nph && !http10) { + buffer.add("Transfer-Encoding: chunked", terminator); + responseChunked = true; + } + } else { + buffer.add("Content-Length: "); + buffer.add(t.length); + buffer.add(terminator); + if(nph && keepAliveRequested) { + buffer.add("Connection: Keep-Alive", terminator); + } + } + + websocket: + + foreach(hd; customHeaders) + buffer.add(hd, terminator); + + // FIXME: what about duplicated headers? + + // end of header indicator + buffer.add(terminator); + + outputtedResponseData = true; + } + + /// Writes the data to the output, flushing headers if they have not yet been sent. + void write(const(void)[] t, bool isAll = false, bool maybeAutoClose = true) { + assert(!closed, "Output has already been closed"); + + StackBuffer buffer = StackBuffer(0); + + if(gzipResponse && acceptsGzip && isAll) { // FIXME: isAll really shouldn't be necessary + // actually gzip the data here + + auto c = new Compress(HeaderFormat.gzip); // want gzip + + auto data = c.compress(t); + data ~= c.flush(); + + // std.file.write("/tmp/last-item", data); + + t = data; + } + + if(!outputtedResponseData && (!autoBuffer || isAll)) { + prepHeaders(t, isAll, &buffer); + } + + if(requestMethod != RequestMethod.HEAD && t.length > 0) { + if (autoBuffer && !isAll) { + outputBuffer ~= cast(ubyte[]) t; + } + if(!autoBuffer || isAll) { + if(rawDataOutput !is null) + if(nph && responseChunked) { + //rawDataOutput(makeChunk(cast(const(ubyte)[]) t)); + // we're making the chunk here instead of in a function + // to avoid unneeded gc pressure + buffer.add(toHex(t.length)); + buffer.add("\r\n"); + buffer.add(cast(char[]) t, "\r\n"); + } else { + buffer.add(cast(char[]) t); + } + else + buffer.add(cast(char[]) t); + } + } + + if(rawDataOutput !is null) + rawDataOutput(cast(const(ubyte)[]) buffer.get()); + else + stdout.rawWrite(buffer.get()); + + if(maybeAutoClose && isAll) + close(); // if you say it is all, that means we're definitely done + // maybeAutoClose can be false though to avoid this (important if you call from inside close()! + } + + /++ + Convenience method to set content type to json and write the string as the complete response. + + History: + Added January 16, 2020 + +/ + void writeJson(string json) { + this.setResponseContentType("application/json"); + this.write(json, true); + } + + /// Flushes the pending buffer, leaving the connection open so you can send more. + void flush() { + if(rawDataOutput is null) + stdout.flush(); + else if(flushDelegate !is null) + flushDelegate(); + } + + version(autoBuffer) + bool autoBuffer = true; + else + bool autoBuffer = false; + ubyte[] outputBuffer; + + /// Flushes the buffers to the network, signifying that you are done. + /// You should always call this explicitly when you are done outputting data. + void close() { + if(closed) + return; // don't double close + + if(!outputtedResponseData) + write("", true, false); + + // writing auto buffered data + if(requestMethod != RequestMethod.HEAD && autoBuffer) { + if(!nph) + stdout.rawWrite(outputBuffer); + else + write(outputBuffer, true, false); // tell it this is everything + } + + // closing the last chunk... + if(nph && rawDataOutput !is null && responseChunked) + rawDataOutput(cast(const(ubyte)[]) "0\r\n\r\n"); + + if(flushDelegate) + flushDelegate(); + + closed = true; + } + + // Closes without doing anything, shouldn't be used often + void rawClose() { + closed = true; + } + + /++ + Gets a request variable as a specific type, or the default value of it isn't there + or isn't convertible to the request type. + + Checks both GET and POST variables, preferring the POST variable, if available. + + A nice trick is using the default value to choose the type: + + --- + /* + The return value will match the type of the default. + Here, I gave 10 as a default, so the return value will + be an int. + + If the user-supplied value cannot be converted to the + requested type, you will get the default value back. + */ + int a = cgi.request("number", 10); + + if(cgi.get["number"] == "11") + assert(a == 11); // conversion succeeds + + if("number" !in cgi.get) + assert(a == 10); // no value means you can't convert - give the default + + if(cgi.get["number"] == "twelve") + assert(a == 10); // conversion from string to int would fail, so we get the default + --- + + You can use an enum as an easy whitelist, too: + + --- + enum Operations { + add, remove, query + } + + auto op = cgi.request("op", Operations.query); + + if(cgi.get["op"] == "add") + assert(op == Operations.add); + if(cgi.get["op"] == "remove") + assert(op == Operations.remove); + if(cgi.get["op"] == "query") + assert(op == Operations.query); + + if(cgi.get["op"] == "random string") + assert(op == Operations.query); // the value can't be converted to the enum, so we get the default + --- + +/ + T request(T = string)(in string name, in T def = T.init) const nothrow { + try { + return + (name in post) ? to!T(post[name]) : + (name in get) ? to!T(get[name]) : + def; + } catch(Exception e) { return def; } + } + + /// Is the output already closed? + bool isClosed() const { + return closed; + } + + /++ + Gets a session object associated with the `cgi` request. You can use different type throughout your application. + +/ + Session!Data getSessionObject(Data)() { + if(testInProcess !is null) { + // test mode + auto obj = testInProcess.getSessionOverride(typeid(typeof(return))); + if(obj !is null) + return cast(typeof(return)) obj; + else { + auto o = new MockSession!Data(); + testInProcess.setSessionOverride(typeid(typeof(return)), o); + return o; + } + } else { + // normal operation + return new BasicDataServerSession!Data(this); + } + } + + // if it is in test mode; triggers mock sessions. Used by CgiTester + version(with_breaking_cgi_features) + private CgiTester testInProcess; + + /* Hooks for redirecting input and output */ + private void delegate(const(ubyte)[]) rawDataOutput = null; + private void delegate() flushDelegate = null; + + /* This info is used when handling a more raw HTTP protocol */ + private bool nph; + private bool http10; + private bool closed; + private bool responseChunked = false; + + version(preserveData) // note: this can eat lots of memory; don't use unless you're sure you need it. + immutable(ubyte)[] originalPostData; + + /++ + This holds the posted body data if it has not been parsed into [post] and [postArray]. + + It is intended to be used for JSON and XML request content types, but also may be used + for other content types your application can handle. But it will NOT be populated + for content types application/x-www-form-urlencoded or multipart/form-data, since those are + parsed into the post and postArray members. + + Remember that anything beyond your `maxContentLength` param when setting up [GenericMain], etc., + will be discarded to the client with an error. This helps keep this array from being exploded in size + and consuming all your server's memory (though it may still be possible to eat excess ram from a concurrent + client in certain build modes.) + + History: + Added January 5, 2021 + Documented February 21, 2023 (dub v11.0) + +/ + public immutable string postBody; + alias postJson = postBody; // old name + + /* Internal state flags */ + private bool outputtedResponseData; + private bool noCache = true; + + const(string[string]) environmentVariables; + + /** What follows is data gotten from the HTTP request. It is all fully immutable, + partially because it logically is (your code doesn't change what the user requested...) + and partially because I hate how bad programs in PHP change those superglobals to do + all kinds of hard to follow ugliness. I don't want that to ever happen in D. + + For some of these, you'll want to refer to the http or cgi specs for more details. + */ + immutable(string[string]) requestHeaders; /// All the raw headers in the request as name/value pairs. The name is stored as all lower case, but otherwise the same as it is in HTTP; words separated by dashes. For example, "cookie" or "accept-encoding". Many HTTP headers have specialized variables below for more convenience and static name checking; you should generally try to use them. + + immutable(char[]) host; /// The hostname in the request. If one program serves multiple domains, you can use this to differentiate between them. + immutable(char[]) origin; /// The origin header in the request, if present. Some HTML5 cross-domain apis set this and you should check it on those cross domain requests and websockets. + immutable(char[]) userAgent; /// The browser's user-agent string. Can be used to identify the browser. + immutable(char[]) pathInfo; /// This is any stuff sent after your program's name on the url, but before the query string. For example, suppose your program is named "app". If the user goes to site.com/app, pathInfo is empty. But, he can also go to site.com/app/some/sub/path; treating your program like a virtual folder. In this case, pathInfo == "/some/sub/path". + immutable(char[]) scriptName; /// The full base path of your program, as seen by the user. If your program is located at site.com/programs/apps, scriptName == "/programs/apps". + immutable(char[]) scriptFileName; /// The physical filename of your script + immutable(char[]) authorization; /// The full authorization string from the header, undigested. Useful for implementing auth schemes such as OAuth 1.0. Note that some web servers do not forward this to the app without taking extra steps. See requireBasicAuth's comment for more info. + immutable(char[]) accept; /// The HTTP accept header is the user agent telling what content types it is willing to accept. This is often */*; they accept everything, so it's not terribly useful. (The similar sounding Accept-Encoding header is handled automatically for chunking and gzipping. Simply set gzipResponse = true and cgi.d handles the details, zipping if the user's browser is willing to accept it.) + immutable(char[]) lastEventId; /// The HTML 5 draft includes an EventSource() object that connects to the server, and remains open to take a stream of events. My arsd.rtud module can help with the server side part of that. The Last-Event-Id http header is defined in the draft to help handle loss of connection. When the browser reconnects to you, it sets this header to the last event id it saw, so you can catch it up. This member has the contents of that header. + + immutable(RequestMethod) requestMethod; /// The HTTP request verb: GET, POST, etc. It is represented as an enum in cgi.d (which, like many enums, you can convert back to string with std.conv.to()). A HTTP GET is supposed to, according to the spec, not have side effects; a user can GET something over and over again and always have the same result. On all requests, the get[] and getArray[] members may be filled in. The post[] and postArray[] members are only filled in on POST methods. + immutable(char[]) queryString; /// The unparsed content of the request query string - the stuff after the ? in your URL. See get[] and getArray[] for a parse view of it. Sometimes, the unparsed string is useful though if you want a custom format of data up there (probably not a good idea, unless it is really simple, like "?username" perhaps.) + immutable(char[]) cookie; /// The unparsed content of the Cookie: header in the request. See also the cookies[string] member for a parsed view of the data. + /** The Referer header from the request. (It is misspelled in the HTTP spec, and thus the actual request and cgi specs too, but I spelled the word correctly here because that's sane. The spec's misspelling is an implementation detail.) It contains the site url that referred the user to your program; the site that linked to you, or if you're serving images, the site that has you as an image. Also, if you're in an iframe, the referrer is the site that is framing you. + + Important note: if the user copy/pastes your url, this is blank, and, just like with all other user data, their browsers can also lie to you. Don't rely on it for real security. + */ + immutable(char[]) referrer; + immutable(char[]) requestUri; /// The full url if the current request, excluding the protocol and host. requestUri == scriptName ~ pathInfo ~ (queryString.length ? "?" ~ queryString : ""); + + immutable(char[]) remoteAddress; /// The IP address of the user, as we see it. (Might not match the IP of the user's computer due to things like proxies and NAT.) + + immutable bool https; /// Was the request encrypted via https? + immutable int port; /// On what TCP port number did the server receive the request? + + /** Here come the parsed request variables - the things that come close to PHP's _GET, _POST, etc. superglobals in content. */ + + immutable(string[string]) get; /// The data from your query string in the url, only showing the last string of each name. If you want to handle multiple values with the same name, use getArray. This only works right if the query string is x-www-form-urlencoded; the default you see on the web with name=value pairs separated by the & character. + immutable(string[string]) post; /// The data from the request's body, on POST requests. It parses application/x-www-form-urlencoded data (used by most web requests, including typical forms), and multipart/form-data requests (used by file uploads on web forms) into the same container, so you can always access them the same way. It makes no attempt to parse other content types. If you want to accept an XML Post body (for a web api perhaps), you'll need to handle the raw data yourself. + immutable(string[string]) cookies; /// Separates out the cookie header into individual name/value pairs (which is how you set them!) + + /** + Represents user uploaded files. + + When making a file upload form, be sure to follow the standard: set method="POST" and enctype="multipart/form-data" in your html
tag attributes. The key into this array is the name attribute on your input tag, just like with other post variables. See the comments on the UploadedFile struct for more information about the data inside, including important notes on max size and content location. + */ + immutable(UploadedFile[][string]) filesArray; + immutable(UploadedFile[string]) files; + + /// Use these if you expect multiple items submitted with the same name. btw, assert(get[name] is getArray[name][$-1); should pass. Same for post and cookies. + /// the order of the arrays is the order the data arrives + immutable(string[][string]) getArray; /// like get, but an array of values per name + immutable(string[][string]) postArray; /// ditto for post + immutable(string[][string]) cookiesArray; /// ditto for cookies + + // convenience function for appending to a uri without extra ? + // matches the name and effect of javascript's location.search property + string search() const { + if(queryString.length) + return "?" ~ queryString; + return ""; + } + + // FIXME: what about multiple files with the same name? + private: + //RequestMethod _requestMethod; +} + +/// use this for testing or other isolated things when you want it to be no-ops +Cgi dummyCgi(Cgi.RequestMethod method = Cgi.RequestMethod.GET, string url = null, in ubyte[] data = null, void delegate(const(ubyte)[]) outputSink = null) { + // we want to ignore, not use stdout + if(outputSink is null) + outputSink = delegate void(const(ubyte)[]) { }; + + string[string] env; + env["REQUEST_METHOD"] = to!string(method); + env["CONTENT_LENGTH"] = to!string(data.length); + + auto cgi = new Cgi( + 0, + env, + { return data; }, + outputSink, + null); + + return cgi; +} + +/++ + A helper test class for request handler unittests. ++/ +version(with_breaking_cgi_features) +class CgiTester { + private { + SessionObject[TypeInfo] mockSessions; + SessionObject getSessionOverride(TypeInfo ti) { + if(auto o = ti in mockSessions) + return *o; + else + return null; + } + void setSessionOverride(TypeInfo ti, SessionObject so) { + mockSessions[ti] = so; + } + } + + /++ + Gets (and creates if necessary) a mock session object for this test. Note + it will be the same one used for any test operations through this CgiTester instance. + +/ + Session!Data getSessionObject(Data)() { + auto obj = getSessionOverride(typeid(typeof(return))); + if(obj !is null) + return cast(typeof(return)) obj; + else { + auto o = new MockSession!Data(); + setSessionOverride(typeid(typeof(return)), o); + return o; + } + } + + /++ + Pass a reference to your request handler when creating the tester. + +/ + this(void function(Cgi) requestHandler) { + this.requestHandler = requestHandler; + } + + /++ + You can check response information with these methods after you call the request handler. + +/ + struct Response { + int code; + string[string] headers; + string responseText; + ubyte[] responseBody; + } + + /++ + Executes a test request on your request handler, and returns the response. + + Params: + url = The URL to test. Should be an absolute path, but excluding domain. e.g. `"/test"`. + args = additional arguments. Same format as cgi's command line handler. + +/ + Response GET(string url, string[] args = null) { + return executeTest("GET", url, args); + } + /// ditto + Response POST(string url, string[] args = null) { + return executeTest("POST", url, args); + } + + /// ditto + Response executeTest(string method, string url, string[] args) { + ubyte[] outputtedRawData; + void outputSink(const(ubyte)[] data) { + outputtedRawData ~= data; + } + auto cgi = new Cgi(["test", method, url] ~ args, &outputSink); + cgi.testInProcess = this; + scope(exit) cgi.dispose(); + + requestHandler(cgi); + + cgi.close(); + + Response response; + + if(outputtedRawData.length) { + enum LINE = "\r\n"; + + auto idx = outputtedRawData.locationOf(LINE ~ LINE); + assert(idx != -1, to!string(outputtedRawData)); + auto headers = cast(string) outputtedRawData[0 .. idx]; + response.code = 200; + while(headers.length) { + auto i = headers.locationOf(LINE); + if(i == -1) i = cast(int) headers.length; + + auto header = headers[0 .. i]; + + auto c = header.locationOf(":"); + if(c != -1) { + auto name = header[0 .. c]; + auto value = header[c + 2 ..$]; + + if(name == "Status") + response.code = value[0 .. value.locationOf(" ")].to!int; + + response.headers[name] = value; + } else { + assert(0); + } + + if(i != headers.length) + i += 2; + headers = headers[i .. $]; + } + response.responseBody = outputtedRawData[idx + 4 .. $]; + response.responseText = cast(string) response.responseBody; + } + + return response; + } + + private void function(Cgi) requestHandler; +} + + +// should this be a separate module? Probably, but that's a hassle. + +/// Makes a data:// uri that can be used as links in most newer browsers (IE8+). +string makeDataUrl(string mimeType, in void[] data) { + auto data64 = Base64.encode(cast(const(ubyte[])) data); + return "data:" ~ mimeType ~ ";base64," ~ assumeUnique(data64); +} + +// FIXME: I don't think this class correctly decodes/encodes the individual parts +/// Represents a url that can be broken down or built up through properties +struct Uri { + alias toString this; // blargh idk a url really is a string, but should it be implicit? + + // scheme//userinfo@host:port/path?query#fragment + + string scheme; /// e.g. "http" in "http://example.com/" + string userinfo; /// the username (and possibly a password) in the uri + string host; /// the domain name + int port; /// port number, if given. Will be zero if a port was not explicitly given + string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" + string query; /// the stuff after the ? in a uri + string fragment; /// the stuff after the # in a uri. + + // idk if i want to keep these, since the functions they wrap are used many, many, many times in existing code, so this is either an unnecessary alias or a gratuitous break of compatibility + // the decode ones need to keep different names anyway because we can't overload on return values... + static string encode(string s) { return std.uri.encodeComponent(s); } + static string encode(string[string] s) { return encodeVariables(s); } + static string encode(string[][string] s) { return encodeVariables(s); } + + /// Breaks down a uri string to its components + this(string uri) { + reparse(uri); + } + + private void reparse(string uri) { + // from RFC 3986 + // the ctRegex triples the compile time and makes ugly errors for no real benefit + // it was a nice experiment but just not worth it. + // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; + /* + Captures: + 0 = whole url + 1 = scheme, with : + 2 = scheme, no : + 3 = authority, with // + 4 = authority, no // + 5 = path + 6 = query string, with ? + 7 = query string, no ? + 8 = anchor, with # + 9 = anchor, no # + */ + // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! + // instead, I will DIY and cut that down to 0.6s on the same computer. + /* + + Note that authority is + user:password@domain:port + where the user:password@ part is optional, and the :port is optional. + + Regex translation: + + Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. + Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. + Path cannot have any ? or # in it. It is optional. + Query must start with ? and must not have # in it. It is optional. + Anchor must start with # and can have anything else in it to end of string. It is optional. + */ + + this = Uri.init; // reset all state + + // empty uri = nothing special + if(uri.length == 0) { + return; + } + + size_t idx; + + scheme_loop: foreach(char c; uri[idx .. $]) { + switch(c) { + case ':': + case '/': + case '?': + case '#': + break scheme_loop; + default: + } + idx++; + } + + if(idx == 0 && uri[idx] == ':') { + // this is actually a path! we skip way ahead + goto path_loop; + } + + if(idx == uri.length) { + // the whole thing is a path, apparently + path = uri; + return; + } + + if(idx > 0 && uri[idx] == ':') { + scheme = uri[0 .. idx]; + idx++; + } else { + // we need to rewind; it found a / but no :, so the whole thing is prolly a path... + idx = 0; + } + + if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { + // we have an authority.... + idx += 2; + + auto authority_start = idx; + authority_loop: foreach(char c; uri[idx .. $]) { + switch(c) { + case '/': + case '?': + case '#': + break authority_loop; + default: + } + idx++; + } + + auto authority = uri[authority_start .. idx]; + + auto idx2 = authority.indexOf("@"); + if(idx2 != -1) { + userinfo = authority[0 .. idx2]; + authority = authority[idx2 + 1 .. $]; + } + + if(authority.length && authority[0] == '[') { + // ipv6 address special casing + idx2 = authority.indexOf(']'); + if(idx2 != -1) { + auto end = authority[idx2 + 1 .. $]; + if(end.length && end[0] == ':') + idx2 = idx2 + 1; + else + idx2 = -1; + } + } else { + idx2 = authority.indexOf(":"); + } + + if(idx2 == -1) { + port = 0; // 0 means not specified; we should use the default for the scheme + host = authority; + } else { + host = authority[0 .. idx2]; + if(idx2 + 1 < authority.length) + port = to!int(authority[idx2 + 1 .. $]); + else + port = 0; + } + } + + path_loop: + auto path_start = idx; + + foreach(char c; uri[idx .. $]) { + if(c == '?' || c == '#') + break; + idx++; + } + + path = uri[path_start .. idx]; + + if(idx == uri.length) + return; // nothing more to examine... + + if(uri[idx] == '?') { + idx++; + auto query_start = idx; + foreach(char c; uri[idx .. $]) { + if(c == '#') + break; + idx++; + } + query = uri[query_start .. idx]; + } + + if(idx < uri.length && uri[idx] == '#') { + idx++; + fragment = uri[idx .. $]; + } + + // uriInvalidated = false; + } + + private string rebuildUri() const { + string ret; + if(scheme.length) + ret ~= scheme ~ ":"; + if(userinfo.length || host.length) + ret ~= "//"; + if(userinfo.length) + ret ~= userinfo ~ "@"; + if(host.length) + ret ~= host; + if(port) + ret ~= ":" ~ to!string(port); + + ret ~= path; + + if(query.length) + ret ~= "?" ~ query; + + if(fragment.length) + ret ~= "#" ~ fragment; + + // uri = ret; + // uriInvalidated = false; + return ret; + } + + /// Converts the broken down parts back into a complete string + string toString() const { + // if(uriInvalidated) + return rebuildUri(); + } + + /// Returns a new absolute Uri given a base. It treats this one as + /// relative where possible, but absolute if not. (If protocol, domain, or + /// other info is not set, the new one inherits it from the base.) + /// + /// Browsers use a function like this to figure out links in html. + Uri basedOn(in Uri baseUrl) const { + Uri n = this; // copies + if(n.scheme == "data") + return n; + // n.uriInvalidated = true; // make sure we regenerate... + + // userinfo is not inherited... is this wrong? + + // if anything is given in the existing url, we don't use the base anymore. + if(n.scheme.empty) { + n.scheme = baseUrl.scheme; + if(n.host.empty) { + n.host = baseUrl.host; + if(n.port == 0) { + n.port = baseUrl.port; + if(n.path.length > 0 && n.path[0] != '/') { + auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1]; + if(b.length == 0) + b = "/"; + n.path = b ~ n.path; + } else if(n.path.length == 0) { + n.path = baseUrl.path; + } + } + } + } + + n.removeDots(); + + return n; + } + + void removeDots() { + auto parts = this.path.split("/"); + string[] toKeep; + foreach(part; parts) { + if(part == ".") { + continue; + } else if(part == "..") { + //if(toKeep.length > 1) + toKeep = toKeep[0 .. $-1]; + //else + //toKeep = [""]; + continue; + } else { + //if(toKeep.length && toKeep[$-1].length == 0 && part.length == 0) + //continue; // skip a `//` situation + toKeep ~= part; + } + } + + auto path = toKeep.join("/"); + if(path.length && path[0] != '/') + path = "/" ~ path; + + this.path = path; + } + + unittest { + auto uri = Uri("test.html"); + assert(uri.path == "test.html"); + uri = Uri("path/1/lol"); + assert(uri.path == "path/1/lol"); + uri = Uri("http://me@example.com"); + assert(uri.scheme == "http"); + assert(uri.userinfo == "me"); + assert(uri.host == "example.com"); + uri = Uri("http://example.com/#a"); + assert(uri.scheme == "http"); + assert(uri.host == "example.com"); + assert(uri.fragment == "a"); + uri = Uri("#foo"); + assert(uri.fragment == "foo"); + uri = Uri("?lol"); + assert(uri.query == "lol"); + uri = Uri("#foo?lol"); + assert(uri.fragment == "foo?lol"); + uri = Uri("?lol#foo"); + assert(uri.fragment == "foo"); + assert(uri.query == "lol"); + + uri = Uri("http://127.0.0.1/"); + assert(uri.host == "127.0.0.1"); + assert(uri.port == 0); + + uri = Uri("http://127.0.0.1:123/"); + assert(uri.host == "127.0.0.1"); + assert(uri.port == 123); + + uri = Uri("http://[ff:ff::0]/"); + assert(uri.host == "[ff:ff::0]"); + + uri = Uri("http://[ff:ff::0]:123/"); + assert(uri.host == "[ff:ff::0]"); + assert(uri.port == 123); + } + + // This can sometimes be a big pain in the butt for me, so lots of copy/paste here to cover + // the possibilities. + unittest { + auto url = Uri("cool.html"); // checking relative links + + assert(url.basedOn(Uri("http://test.com/what/test.html")) == "http://test.com/what/cool.html"); + assert(url.basedOn(Uri("https://test.com/what/test.html")) == "https://test.com/what/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/")) == "http://test.com/what/cool.html"); + assert(url.basedOn(Uri("http://test.com/")) == "http://test.com/cool.html"); + assert(url.basedOn(Uri("http://test.com")) == "http://test.com/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b")) == "http://test.com/what/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d")) == "http://test.com/what/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d#what")) == "http://test.com/what/cool.html"); + assert(url.basedOn(Uri("http://test.com")) == "http://test.com/cool.html"); + + url = Uri("/something/cool.html"); // same server, different path + assert(url.basedOn(Uri("http://test.com/what/test.html")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("https://test.com/what/test.html")) == "https://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com/")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d#what")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com")) == "http://test.com/something/cool.html"); + + url = Uri("?query=answer"); // same path. server, protocol, and port, just different query string and fragment + assert(url.basedOn(Uri("http://test.com/what/test.html")) == "http://test.com/what/test.html?query=answer"); + assert(url.basedOn(Uri("https://test.com/what/test.html")) == "https://test.com/what/test.html?query=answer"); + assert(url.basedOn(Uri("http://test.com/what/")) == "http://test.com/what/?query=answer"); + assert(url.basedOn(Uri("http://test.com/")) == "http://test.com/?query=answer"); + assert(url.basedOn(Uri("http://test.com")) == "http://test.com?query=answer"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b")) == "http://test.com/what/test.html?query=answer"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d")) == "http://test.com/what/test.html?query=answer"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d#what")) == "http://test.com/what/test.html?query=answer"); + assert(url.basedOn(Uri("http://test.com")) == "http://test.com?query=answer"); + + url = Uri("/test/bar"); + assert(Uri("./").basedOn(url) == "/test/", Uri("./").basedOn(url)); + assert(Uri("../").basedOn(url) == "/"); + + url = Uri("http://example.com/"); + assert(Uri("../foo").basedOn(url) == "http://example.com/foo"); + + //auto uriBefore = url; + url = Uri("#anchor"); // everything should remain the same except the anchor + //uriBefore.anchor = "anchor"); + //assert(url == uriBefore); + + url = Uri("//example.com"); // same protocol, but different server. the path here should be blank. + + url = Uri("//example.com/example.html"); // same protocol, but different server and path + + url = Uri("http://example.com/test.html"); // completely absolute link should never be modified + + url = Uri("http://example.com"); // completely absolute link should never be modified, even if it has no path + + // FIXME: add something for port too + } + + // these are like javascript's location.search and location.hash + string search() const { + return query.length ? ("?" ~ query) : ""; + } + string hash() const { + return fragment.length ? ("#" ~ fragment) : ""; + } +} + + +/* + for session, see web.d +*/ + +/// breaks down a url encoded string +string[][string] decodeVariables(string data, string separator = "&", string[]* namesInOrder = null, string[]* valuesInOrder = null) { + auto vars = data.split(separator); + string[][string] _get; + foreach(var; vars) { + auto equal = var.indexOf("="); + string name; + string value; + if(equal == -1) { + name = decodeComponent(var); + value = ""; + } else { + //_get[decodeComponent(var[0..equal])] ~= decodeComponent(var[equal + 1 .. $].replace("+", " ")); + // stupid + -> space conversion. + name = decodeComponent(var[0..equal].replace("+", " ")); + value = decodeComponent(var[equal + 1 .. $].replace("+", " ")); + } + + _get[name] ~= value; + if(namesInOrder) + (*namesInOrder) ~= name; + if(valuesInOrder) + (*valuesInOrder) ~= value; + } + return _get; +} + +/// breaks down a url encoded string, but only returns the last value of any array +string[string] decodeVariablesSingle(string data) { + string[string] va; + auto varArray = decodeVariables(data); + foreach(k, v; varArray) + va[k] = v[$-1]; + + return va; +} + +/// url encodes the whole string +string encodeVariables(in string[string] data) { + string ret; + + bool outputted = false; + foreach(k, v; data) { + if(outputted) + ret ~= "&"; + else + outputted = true; + + ret ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v); + } + + return ret; +} + +/// url encodes a whole string +string encodeVariables(in string[][string] data) { + string ret; + + bool outputted = false; + foreach(k, arr; data) { + foreach(v; arr) { + if(outputted) + ret ~= "&"; + else + outputted = true; + ret ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v); + } + } + + return ret; +} + +/// Encodes all but the explicitly unreserved characters per rfc 3986 +/// Alphanumeric and -_.~ are the only ones left unencoded +/// name is borrowed from php +string rawurlencode(in char[] data) { + string ret; + ret.reserve(data.length * 2); + foreach(char c; data) { + if( + (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '-' || c == '_' || c == '.' || c == '~') + { + ret ~= c; + } else { + ret ~= '%'; + // since we iterate on char, this should give us the octets of the full utf8 string + ret ~= toHexUpper(c); + } + } + + return ret; +} + + +// http helper functions + +// for chunked responses (which embedded http does whenever possible) +version(none) // this is moved up above to avoid making a copy of the data +const(ubyte)[] makeChunk(const(ubyte)[] data) { + const(ubyte)[] ret; + + ret = cast(const(ubyte)[]) toHex(data.length); + ret ~= cast(const(ubyte)[]) "\r\n"; + ret ~= data; + ret ~= cast(const(ubyte)[]) "\r\n"; + + return ret; +} + +string toHex(long num) { + string ret; + while(num) { + int v = num % 16; + num /= 16; + char d = cast(char) ((v < 10) ? v + '0' : (v-10) + 'a'); + ret ~= d; + } + + return to!string(array(ret.retro)); +} + +string toHexUpper(long num) { + string ret; + while(num) { + int v = num % 16; + num /= 16; + char d = cast(char) ((v < 10) ? v + '0' : (v-10) + 'A'); + ret ~= d; + } + + if(ret.length == 1) + ret ~= "0"; // url encoding requires two digits and that's what this function is used for... + + return to!string(array(ret.retro)); +} + + +// the generic mixins + +/++ + Use this instead of writing your own main + + It ultimately calls [cgiMainImpl] which creates a [RequestServer] for you. ++/ +mixin template GenericMain(alias fun, long maxContentLength = defaultMaxContentLength) { + mixin CustomCgiMain!(Cgi, fun, maxContentLength); +} + +/++ + Boilerplate mixin for a main function that uses the [dispatcher] function. + + You can send `typeof(null)` as the `Presenter` argument to use a generic one. + + History: + Added July 9, 2021 ++/ +mixin template DispatcherMain(Presenter, DispatcherArgs...) { + /++ + Handler to the generated presenter you can use from your objects, etc. + +/ + Presenter activePresenter; + + /++ + Request handler that creates the presenter then forwards to the [dispatcher] function. + Renders 404 if the dispatcher did not handle the request. + + Will automatically serve the presenter.style and presenter.script as "style.css" and "script.js" + +/ + void handler(Cgi cgi) { + auto presenter = new Presenter; + activePresenter = presenter; + scope(exit) activePresenter = null; + + if(cgi.dispatcher!DispatcherArgs(presenter)) + return; + + switch(cgi.pathInfo) { + case "/style.css": + cgi.setCache(true); + cgi.setResponseContentType("text/css"); + cgi.write(presenter.style(), true); + break; + case "/script.js": + cgi.setCache(true); + cgi.setResponseContentType("application/javascript"); + cgi.write(presenter.script(), true); + break; + default: + presenter.renderBasicError(cgi, 404); + } + } + mixin GenericMain!handler; +} + +mixin template DispatcherMain(DispatcherArgs...) if(!is(DispatcherArgs[0] : WebPresenter!T, T)) { + class GenericPresenter : WebPresenter!GenericPresenter {} + mixin DispatcherMain!(GenericPresenter, DispatcherArgs); +} + +private string simpleHtmlEncode(string s) { + return s.replace("&", "&").replace("<", "<").replace(">", ">").replace("\n", "
\n"); +} + +string messageFromException(Throwable t) { + string message; + if(t !is null) { + debug message = t.toString(); + else message = "An unexpected error has occurred."; + } else { + message = "Unknown error"; + } + return message; +} + +string plainHttpError(bool isCgi, string type, Throwable t) { + auto message = messageFromException(t); + message = simpleHtmlEncode(message); + + return format("%s %s\r\nContent-Length: %s\r\n\r\n%s", + isCgi ? "Status:" : "HTTP/1.0", + type, message.length, message); +} + +// returns true if we were able to recover reasonably +bool handleException(Cgi cgi, Throwable t) { + if(cgi.isClosed) { + // if the channel has been explicitly closed, we can't handle it here + return true; + } + + if(cgi.outputtedResponseData) { + // the headers are sent, but the channel is open... since it closes if all was sent, we can append an error message here. + return false; // but I don't want to, since I don't know what condition the output is in; I don't want to inject something (nor check the content-type for that matter. So we say it was not a clean handling. + } else { + // no headers are sent, we can send a full blown error and recover + cgi.setCache(false); + cgi.setResponseContentType("text/html"); + cgi.setResponseLocation(null); // cancel the redirect + cgi.setResponseStatus("500 Internal Server Error"); + cgi.write(simpleHtmlEncode(messageFromException(t))); + cgi.close(); + return true; + } +} + +bool isCgiRequestMethod(string s) { + s = s.toUpper(); + if(s == "COMMANDLINE") + return true; + foreach(member; __traits(allMembers, Cgi.RequestMethod)) + if(s == member) + return true; + return false; +} + +/// If you want to use a subclass of Cgi with generic main, use this mixin. +mixin template CustomCgiMain(CustomCgi, alias fun, long maxContentLength = defaultMaxContentLength) if(is(CustomCgi : Cgi)) { + // kinda hacky - the T... is passed to Cgi's constructor in standard cgi mode, and ignored elsewhere + void main(string[] args) { + cgiMainImpl!(fun, CustomCgi, maxContentLength)(args); + } +} + +version(embedded_httpd_processes) + __gshared int processPoolSize = 8; + +// Returns true if run. You should exit the program after that. +bool tryAddonServers(string[] args) { + if(args.length > 1) { + // run the special separate processes if needed + switch(args[1]) { + case "--websocket-server": + version(with_addon_servers) + websocketServers[args[2]](args[3 .. $]); + else + printf("Add-on servers not compiled in.\n"); + return true; + case "--websocket-servers": + import core.demangle; + version(with_addon_servers_connections) + foreach(k, v; websocketServers) + writeln(k, "\t", demangle(k)); + return true; + case "--session-server": + version(with_addon_servers) + runSessionServer(); + else + printf("Add-on servers not compiled in.\n"); + return true; + case "--event-server": + version(with_addon_servers) + runEventServer(); + else + printf("Add-on servers not compiled in.\n"); + return true; + case "--timer-server": + version(with_addon_servers) + runTimerServer(); + else + printf("Add-on servers not compiled in.\n"); + return true; + case "--timed-jobs": + import core.demangle; + version(with_addon_servers_connections) + foreach(k, v; scheduledJobHandlers) + writeln(k, "\t", demangle(k)); + return true; + case "--timed-job": + scheduledJobHandlers[args[2]](args[3 .. $]); + return true; + default: + // intentionally blank - do nothing and carry on to run normally + } + } + return false; +} + +/// Tries to simulate a request from the command line. Returns true if it does, false if it didn't find the args. +bool trySimulatedRequest(alias fun, CustomCgi = Cgi)(string[] args) if(is(CustomCgi : Cgi)) { + // we support command line thing for easy testing everywhere + // it needs to be called ./app method uri [other args...] + if(args.length >= 3 && isCgiRequestMethod(args[1])) { + Cgi cgi = new CustomCgi(args); + scope(exit) cgi.dispose(); + fun(cgi); + cgi.close(); + return true; + } + return false; +} + +/++ + A server control and configuration struct, as a potential alternative to calling [GenericMain] or [cgiMainImpl]. See the source of [cgiMainImpl] to an example of how you can use it. + + History: + Added Sept 26, 2020 (release version 8.5). ++/ +struct RequestServer { + /// + string listeningHost = defaultListeningHost(); + /// + ushort listeningPort = defaultListeningPort(); + + /++ + Uses a fork() call, if available, to provide additional crash resiliency and possibly improved performance. On the + other hand, if you fork, you must not assume any memory is shared between requests (you shouldn't be anyway though! But + if you have to, you probably want to set this to false and use an explicit threaded server with [serveEmbeddedHttp]) and + [stop] may not work as well. + + History: + Added August 12, 2022 (dub v10.9). Previously, this was only configurable through the `-version=cgi_no_fork` + argument to dmd. That version still defines the value of `cgi_use_fork_default`, used to initialize this, for + compatibility. + +/ + bool useFork = cgi_use_fork_default; + + /++ + Determines the number of worker threads to spawn per process, for server modes that use worker threads. 0 will use a + default based on the number of cpus modified by the server mode. + + History: + Added August 12, 2022 (dub v10.9) + +/ + int numberOfThreads = 0; + + /// + this(string defaultHost, ushort defaultPort) { + this.listeningHost = defaultHost; + this.listeningPort = defaultPort; + } + + /// + this(ushort defaultPort) { + listeningPort = defaultPort; + } + + /++ + Reads the command line arguments into the values here. + + Possible arguments are `--listening-host`, `--listening-port` (or `--port`), `--uid`, and `--gid`. + +/ + void configureFromCommandLine(string[] args) { + bool foundPort = false; + bool foundHost = false; + bool foundUid = false; + bool foundGid = false; + foreach(arg; args) { + if(foundPort) { + listeningPort = to!ushort(arg); + foundPort = false; + } + if(foundHost) { + listeningHost = arg; + foundHost = false; + } + if(foundUid) { + privilegesDropToUid = to!uid_t(arg); + foundUid = false; + } + if(foundGid) { + privilegesDropToGid = to!gid_t(arg); + foundGid = false; + } + if(arg == "--listening-host" || arg == "-h" || arg == "/listening-host") + foundHost = true; + else if(arg == "--port" || arg == "-p" || arg == "/port" || arg == "--listening-port") + foundPort = true; + else if(arg == "--uid") + foundUid = true; + else if(arg == "--gid") + foundGid = true; + } + } + + version(Windows) { + private alias uid_t = int; + private alias gid_t = int; + } + + /// user (uid) to drop privileges to + /// 0 … do nothing + uid_t privilegesDropToUid = 0; + /// group (gid) to drop privileges to + /// 0 … do nothing + gid_t privilegesDropToGid = 0; + + private void dropPrivileges() { + version(Posix) { + import core.sys.posix.unistd; + + if (privilegesDropToGid != 0 && setgid(privilegesDropToGid) != 0) + throw new Exception("Dropping privileges via setgid() failed."); + + if (privilegesDropToUid != 0 && setuid(privilegesDropToUid) != 0) + throw new Exception("Dropping privileges via setuid() failed."); + } + else { + // FIXME: Windows? + //pragma(msg, "Dropping privileges is not implemented for this platform"); + } + + // done, set zero + privilegesDropToGid = 0; + privilegesDropToUid = 0; + } + + /++ + Serves a single HTTP request on this thread, with an embedded server, then stops. Designed for cases like embedded oauth responders + + History: + Added Oct 10, 2020. + Example: + + --- + import arsd.cgi; + void main() { + RequestServer server = RequestServer("127.0.0.1", 6789); + string oauthCode; + string oauthScope; + server.serveHttpOnce!((cgi) { + oauthCode = cgi.request("code"); + oauthScope = cgi.request("scope"); + cgi.write("Thank you, please return to the application."); + }); + // use the code and scope given + } + --- + +/ + void serveHttpOnce(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() { + import std.socket; + + bool tcp; + void delegate() cleanup; + auto socket = startListening(listeningHost, listeningPort, tcp, cleanup, 1, &dropPrivileges); + auto connection = socket.accept(); + doThreadHttpConnectionGuts!(CustomCgi, fun, true)(connection); + + if(cleanup) + cleanup(); + } + + /++ + Starts serving requests according to the current configuration. + +/ + void serve(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() { + version(netman_httpd) { + // Obsolete! + + import arsd.httpd; + // what about forwarding the other constructor args? + // this probably needs a whole redoing... + serveHttp!CustomCgi(&fun, listeningPort);//5005); + return; + } else + version(embedded_httpd_processes) { + serveEmbeddedHttpdProcesses!(fun, CustomCgi)(this); + } else + version(embedded_httpd_threads) { + serveEmbeddedHttp!(fun, CustomCgi, maxContentLength)(); + } else + version(scgi) { + serveScgi!(fun, CustomCgi, maxContentLength)(); + } else + version(fastcgi) { + serveFastCgi!(fun, CustomCgi, maxContentLength)(this); + } else + version(stdio_http) { + serveSingleHttpConnectionOnStdio!(fun, CustomCgi, maxContentLength)(); + } else { + //version=plain_cgi; + handleCgiRequest!(fun, CustomCgi, maxContentLength)(); + } + } + + /++ + Runs the embedded HTTP thread server specifically, regardless of which build configuration you have. + + If you want the forking worker process server, you do need to compile with the embedded_httpd_processes config though. + +/ + void serveEmbeddedHttp(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)(ThisFor!fun _this) { + globalStopFlag = false; + static if(__traits(isStaticFunction, fun)) + alias funToUse = fun; + else + void funToUse(CustomCgi cgi) { + static if(__VERSION__ > 2097) + __traits(child, _this, fun)(cgi); + else static assert(0, "Not implemented in your compiler version!"); + } + auto manager = new ListeningConnectionManager(listeningHost, listeningPort, &doThreadHttpConnection!(CustomCgi, funToUse), null, useFork, numberOfThreads); + manager.listen(); + } + + /++ + Runs the embedded SCGI server specifically, regardless of which build configuration you have. + +/ + void serveScgi(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() { + globalStopFlag = false; + auto manager = new ListeningConnectionManager(listeningHost, listeningPort, &doThreadScgiConnection!(CustomCgi, fun, maxContentLength), null, useFork, numberOfThreads); + manager.listen(); + } + + /++ + Serves a single "connection", but the connection is spoken on stdin and stdout instead of on a socket. + + Intended for cases like working from systemd, like discussed here: [https://forum.dlang.org/post/avmkfdiitirnrenzljwc@forum.dlang.org] + + History: + Added May 29, 2021 + +/ + void serveSingleHttpConnectionOnStdio(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() { + doThreadHttpConnectionGuts!(CustomCgi, fun, true)(new FakeSocketForStdin()); + } + + /++ + The [stop] function sets a flag that request handlers can (and should) check periodically. If a handler doesn't + respond to this flag, the library will force the issue. This determines when and how the issue will be forced. + +/ + enum ForceStop { + /++ + Stops accepting new requests, but lets ones already in the queue start and complete before exiting. + +/ + afterQueuedRequestsComplete, + /++ + Finishes requests already started their handlers, but drops any others in the queue. Streaming handlers + should cooperate and exit gracefully, but if they don't, it will continue waiting for them. + +/ + afterCurrentRequestsComplete, + /++ + Partial response writes will throw an exception, cancelling any streaming response, but complete + writes will continue to process. Request handlers that respect the stop token will also gracefully cancel. + +/ + cancelStreamingRequestsEarly, + /++ + All writes will throw. + +/ + cancelAllRequestsEarly, + /++ + Use OS facilities to forcibly kill running threads. The server process will be in an undefined state after this call (if this call ever returns). + +/ + forciblyTerminate, + } + + version(embedded_httpd_processes) {} else + /++ + Stops serving after the current requests are completed. + + Bugs: + Not implemented on version=embedded_httpd_processes, version=fastcgi on any system, or embedded_httpd on Windows (it does work on embedded_httpd_hybrid + on Windows however). Only partially implemented on non-Linux posix systems. + + You might also try SIGINT perhaps. + + The stopPriority is not yet fully implemented. + +/ + static void stop(ForceStop stopPriority = ForceStop.afterCurrentRequestsComplete) { + globalStopFlag = true; + + version(Posix) { + if(cancelfd > 0) { + ulong a = 1; + core.sys.posix.unistd.write(cancelfd, &a, a.sizeof); + } + } + version(Windows) { + if(iocp) { + foreach(i; 0 .. 16) // FIXME + PostQueuedCompletionStatus(iocp, 0, cast(ULONG_PTR) null, null); + } + } + } +} + +private alias AliasSeq(T...) = T; + +version(with_breaking_cgi_features) +mixin(q{ + template ThisFor(alias t) { + static if(__traits(isStaticFunction, t)) { + alias ThisFor = AliasSeq!(); + } else { + alias ThisFor = __traits(parent, t); + } + } +}); +else + alias ThisFor(alias t) = AliasSeq!(); + +private __gshared bool globalStopFlag = false; + +version(embedded_httpd_processes) +void serveEmbeddedHttpdProcesses(alias fun, CustomCgi = Cgi)(RequestServer params) { + import core.sys.posix.unistd; + import core.sys.posix.sys.socket; + import core.sys.posix.netinet.in_; + //import std.c.linux.socket; + + int sock = socket(AF_INET, SOCK_STREAM, 0); + if(sock == -1) + throw new Exception("socket"); + + cloexec(sock); + + { + + sockaddr_in addr; + addr.sin_family = AF_INET; + addr.sin_port = htons(params.listeningPort); + auto lh = params.listeningHost; + if(lh.length) { + if(inet_pton(AF_INET, lh.toStringz(), &addr.sin_addr.s_addr) != 1) + throw new Exception("bad listening host given, please use an IP address.\nExample: --listening-host 127.0.0.1 means listen only on Localhost.\nExample: --listening-host 0.0.0.0 means listen on all interfaces.\nOr you can pass any other single numeric IPv4 address."); + } else + addr.sin_addr.s_addr = INADDR_ANY; + + // HACKISH + int on = 1; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &on, on.sizeof); + // end hack + + + if(bind(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) { + close(sock); + throw new Exception("bind"); + } + + // FIXME: if this queue is full, it will just ignore it + // and wait for the client to retransmit it. This is an + // obnoxious timeout condition there. + if(sock.listen(128) == -1) { + close(sock); + throw new Exception("listen"); + } + params.dropPrivileges(); + } + + version(embedded_httpd_processes_accept_after_fork) {} else { + int pipeReadFd; + int pipeWriteFd; + + { + int[2] pipeFd; + if(socketpair(AF_UNIX, SOCK_DGRAM, 0, pipeFd)) { + import core.stdc.errno; + throw new Exception("pipe failed " ~ to!string(errno)); + } + + pipeReadFd = pipeFd[0]; + pipeWriteFd = pipeFd[1]; + } + } + + + int processCount; + pid_t newPid; + reopen: + while(processCount < processPoolSize) { + newPid = fork(); + if(newPid == 0) { + // start serving on the socket + //ubyte[4096] backingBuffer; + for(;;) { + bool closeConnection; + uint i; + sockaddr addr; + i = addr.sizeof; + version(embedded_httpd_processes_accept_after_fork) { + int s = accept(sock, &addr, &i); + int opt = 1; + import core.sys.posix.netinet.tcp; + // the Cgi class does internal buffering, so disabling this + // helps with latency in many cases... + setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &opt, opt.sizeof); + cloexec(s); + } else { + int s; + auto readret = read_fd(pipeReadFd, &s, s.sizeof, &s); + if(readret != s.sizeof) { + import core.stdc.errno; + throw new Exception("pipe read failed " ~ to!string(errno)); + } + + //writeln("process ", getpid(), " got socket ", s); + } + + try { + + if(s == -1) + throw new Exception("accept"); + + scope(failure) close(s); + //ubyte[__traits(classInstanceSize, BufferedInputRange)] bufferedRangeContainer; + auto ir = new BufferedInputRange(s); + //auto ir = emplace!BufferedInputRange(bufferedRangeContainer, s, backingBuffer); + + while(!ir.empty) { + //ubyte[__traits(classInstanceSize, CustomCgi)] cgiContainer; + + Cgi cgi; + try { + cgi = new CustomCgi(ir, &closeConnection); + cgi._outputFileHandle = cast(CgiConnectionHandle) s; + // if we have a single process and the browser tries to leave the connection open while concurrently requesting another, it will block everything an deadlock since there's no other server to accept it. By closing after each request in this situation, it tells the browser to serialize for us. + if(processPoolSize <= 1) + closeConnection = true; + //cgi = emplace!CustomCgi(cgiContainer, ir, &closeConnection); + } catch(Throwable t) { + // a construction error is either bad code or bad request; bad request is what it should be since this is bug free :P + // anyway let's kill the connection + version(CRuntime_Musl) { + // LockingTextWriter fails here + // so working around it + auto estr = t.toString(); + stderr.rawWrite(estr); + stderr.rawWrite("\n"); + } else + stderr.writeln(t.toString()); + sendAll(ir.source, plainHttpError(false, "400 Bad Request", t)); + closeConnection = true; + break; + } + assert(cgi !is null); + scope(exit) + cgi.dispose(); + + try { + fun(cgi); + cgi.close(); + if(cgi.websocketMode) + closeConnection = true; + } catch(ConnectionException ce) { + closeConnection = true; + } catch(Throwable t) { + // a processing error can be recovered from + version(CRuntime_Musl) { + // LockingTextWriter fails here + // so working around it + auto estr = t.toString(); + stderr.rawWrite(estr); + } else { + stderr.writeln(t.toString); + } + if(!handleException(cgi, t)) + closeConnection = true; + } + + if(closeConnection) { + ir.source.close(); + break; + } else { + if(!ir.empty) + ir.popFront(); // get the next + else if(ir.sourceClosed) { + ir.source.close(); + } + } + } + + ir.source.close(); + } catch(Throwable t) { + version(CRuntime_Musl) {} else + debug writeln(t); + // most likely cause is a timeout + } + } + } else if(newPid < 0) { + throw new Exception("fork failed"); + } else { + processCount++; + } + } + + // the parent should wait for its children... + if(newPid) { + import core.sys.posix.sys.wait; + + version(embedded_httpd_processes_accept_after_fork) {} else { + import core.sys.posix.sys.select; + int[] fdQueue; + while(true) { + // writeln("select call"); + int nfds = pipeWriteFd; + if(sock > pipeWriteFd) + nfds = sock; + nfds += 1; + fd_set read_fds; + fd_set write_fds; + FD_ZERO(&read_fds); + FD_ZERO(&write_fds); + FD_SET(sock, &read_fds); + if(fdQueue.length) + FD_SET(pipeWriteFd, &write_fds); + auto ret = select(nfds, &read_fds, &write_fds, null, null); + if(ret == -1) { + import core.stdc.errno; + if(errno == EINTR) + goto try_wait; + else + throw new Exception("wtf select"); + } + + int s = -1; + if(FD_ISSET(sock, &read_fds)) { + uint i; + sockaddr addr; + i = addr.sizeof; + s = accept(sock, &addr, &i); + cloexec(s); + import core.sys.posix.netinet.tcp; + int opt = 1; + setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &opt, opt.sizeof); + } + + if(FD_ISSET(pipeWriteFd, &write_fds)) { + if(s == -1 && fdQueue.length) { + s = fdQueue[0]; + fdQueue = fdQueue[1 .. $]; // FIXME reuse buffer + } + write_fd(pipeWriteFd, &s, s.sizeof, s); + close(s); // we are done with it, let the other process take ownership + } else + fdQueue ~= s; + } + } + + try_wait: + + int status; + while(-1 != wait(&status)) { + version(CRuntime_Musl) {} else { + import std.stdio; writeln("Process died ", status); + } + processCount--; + goto reopen; + } + close(sock); + } +} + +version(fastcgi) +void serveFastCgi(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)(RequestServer params) { + // SetHandler fcgid-script + FCGX_Stream* input, output, error; + FCGX_ParamArray env; + + + + const(ubyte)[] getFcgiChunk() { + const(ubyte)[] ret; + while(FCGX_HasSeenEOF(input) != -1) + ret ~= cast(ubyte) FCGX_GetChar(input); + return ret; + } + + void writeFcgi(const(ubyte)[] data) { + FCGX_PutStr(data.ptr, data.length, output); + } + + void doARequest() { + string[string] fcgienv; + + for(auto e = env; e !is null && *e !is null; e++) { + string cur = to!string(*e); + auto idx = cur.indexOf("="); + string name, value; + if(idx == -1) + name = cur; + else { + name = cur[0 .. idx]; + value = cur[idx + 1 .. $]; + } + + fcgienv[name] = value; + } + + void flushFcgi() { + FCGX_FFlush(output); + } + + Cgi cgi; + try { + cgi = new CustomCgi(maxContentLength, fcgienv, &getFcgiChunk, &writeFcgi, &flushFcgi); + } catch(Throwable t) { + FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error); + writeFcgi(cast(const(ubyte)[]) plainHttpError(true, "400 Bad Request", t)); + return; //continue; + } + assert(cgi !is null); + scope(exit) cgi.dispose(); + try { + fun(cgi); + cgi.close(); + } catch(Throwable t) { + // log it to the error stream + FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error); + // handle it for the user, if we can + if(!handleException(cgi, t)) + return; // continue; + } + } + + auto lp = params.listeningPort; + auto host = params.listeningHost; + + FCGX_Request request; + if(lp || !host.empty) { + // if a listening port was specified on the command line, we want to spawn ourself + // (needed for nginx without spawn-fcgi, e.g. on Windows) + FCGX_Init(); + + int sock; + + if(host.startsWith("unix:")) { + sock = FCGX_OpenSocket(toStringz(params.listeningHost["unix:".length .. $]), 12); + } else if(host.startsWith("abstract:")) { + sock = FCGX_OpenSocket(toStringz("\0" ~ params.listeningHost["abstract:".length .. $]), 12); + } else { + sock = FCGX_OpenSocket(toStringz(params.listeningHost ~ ":" ~ to!string(lp)), 12); + } + + if(sock < 0) + throw new Exception("Couldn't listen on the port"); + FCGX_InitRequest(&request, sock, 0); + while(FCGX_Accept_r(&request) >= 0) { + input = request.inStream; + output = request.outStream; + error = request.errStream; + env = request.envp; + doARequest(); + } + } else { + // otherwise, assume the httpd is doing it (the case for Apache, IIS, and Lighttpd) + // using the version with a global variable since we are separate processes anyway + while(FCGX_Accept(&input, &output, &error, &env) >= 0) { + doARequest(); + } + } +} + +/// Returns the default listening port for the current cgi configuration. 8085 for embedded httpd, 4000 for scgi, irrelevant for others. +ushort defaultListeningPort() { + version(netman_httpd) + return 8080; + else version(embedded_httpd_processes) + return 8085; + else version(embedded_httpd_threads) + return 8085; + else version(scgi) + return 4000; + else + return 0; +} + +/// Default host for listening. 127.0.0.1 for scgi, null (aka all interfaces) for all others. If you want the server directly accessible from other computers on the network, normally use null. If not, 127.0.0.1 is a bit better. Settable with default handlers with --listening-host command line argument. +string defaultListeningHost() { + version(netman_httpd) + return null; + else version(embedded_httpd_processes) + return null; + else version(embedded_httpd_threads) + return null; + else version(scgi) + return "127.0.0.1"; + else + return null; + +} + +/++ + This is the function [GenericMain] calls. View its source for some simple boilerplate you can copy/paste and modify, or you can call it yourself from your `main`. + + Please note that this may spawn other helper processes that will call `main` again. It does this currently for the timer server and event source server (and the quasi-deprecated web socket server). + + Params: + fun = Your request handler + CustomCgi = a subclass of Cgi, if you wise to customize it further + maxContentLength = max POST size you want to allow + args = command-line arguments + + History: + Documented Sept 26, 2020. ++/ +void cgiMainImpl(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)(string[] args) if(is(CustomCgi : Cgi)) { + if(tryAddonServers(args)) + return; + + if(trySimulatedRequest!(fun, CustomCgi)(args)) + return; + + RequestServer server; + // you can change the port here if you like + // server.listeningPort = 9000; + + // then call this to let the command line args override your default + server.configureFromCommandLine(args); + + // and serve the request(s). + server.serve!(fun, CustomCgi, maxContentLength)(); +} + +//version(plain_cgi) +void handleCgiRequest(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() { + // standard CGI is the default version + + + // Set stdin to binary mode if necessary to avoid mangled newlines + // the fact that stdin is global means this could be trouble but standard cgi request + // handling is one per process anyway so it shouldn't actually be threaded here or anything. + version(Windows) { + version(Win64) + _setmode(std.stdio.stdin.fileno(), 0x8000); + else + setmode(std.stdio.stdin.fileno(), 0x8000); + } + + Cgi cgi; + try { + cgi = new CustomCgi(maxContentLength); + version(Posix) + cgi._outputFileHandle = cast(CgiConnectionHandle) 1; // stdout + else version(Windows) + cgi._outputFileHandle = cast(CgiConnectionHandle) GetStdHandle(STD_OUTPUT_HANDLE); + else static assert(0); + } catch(Throwable t) { + version(CRuntime_Musl) { + // LockingTextWriter fails here + // so working around it + auto s = t.toString(); + stderr.rawWrite(s); + stdout.rawWrite(plainHttpError(true, "400 Bad Request", t)); + } else { + stderr.writeln(t.msg); + // the real http server will probably handle this; + // most likely, this is a bug in Cgi. But, oh well. + stdout.write(plainHttpError(true, "400 Bad Request", t)); + } + return; + } + assert(cgi !is null); + scope(exit) cgi.dispose(); + + try { + fun(cgi); + cgi.close(); + } catch (Throwable t) { + version(CRuntime_Musl) { + // LockingTextWriter fails here + // so working around it + auto s = t.msg; + stderr.rawWrite(s); + } else { + stderr.writeln(t.msg); + } + if(!handleException(cgi, t)) + return; + } +} + +private __gshared int cancelfd = -1; + +/+ + The event loop for embedded_httpd_threads will prolly fiber dispatch + cgi constructors too, so slow posts will not monopolize a worker thread. + + May want to provide the worker task system just need to ensure all the fibers + has a big enough stack for real work... would also ideally like to reuse them. + + + So prolly bir would switch it to nonblocking. If it would block, it epoll + registers one shot with this existing fiber to take it over. + + new connection comes in. it picks a fiber off the free list, + or if there is none, it creates a new one. this fiber handles + this connection the whole time. + + epoll triggers the fiber when something comes in. it is called by + a random worker thread, it might change at any time. at least during + the constructor. maybe into the main body it will stay tied to a thread + just so TLS stuff doesn't randomly change in the middle. but I could + specify if you yield all bets are off. + + when the request is finished, if there's more data buffered, it just + keeps going. if there is no more data buffered, it epoll ctls to + get triggered when more data comes in. all one shot. + + when a connection is closed, the fiber returns and is then reset + and added to the free list. if the free list is full, the fiber is + just freed, this means it will balloon to a certain size but not generally + grow beyond that unless the activity keeps going. + + 256 KB stack i thnk per fiber. 4,000 active fibers per gigabyte of memory. + + So the fiber has its own magic methods to read and write. if they would block, it registers + for epoll and yields. when it returns, it read/writes and then returns back normal control. + + basically you issue the command and it tells you when it is done + + it needs to DEL the epoll thing when it is closed. add it when opened. mod it when anther thing issued + ++/ + +/++ + The stack size when a fiber is created. You can set this from your main or from a shared static constructor + to optimize your memory use if you know you don't need this much space. Be careful though, some functions use + more stack space than you realize and a recursive function (including ones like in dom.d) can easily grow fast! + + History: + Added July 10, 2021. Previously, it used the druntime default of 16 KB. ++/ +version(cgi_use_fiber) +__gshared size_t fiberStackSize = 4096 * 100; + +version(cgi_use_fiber) +class CgiFiber : Fiber { + private void function(Socket) f_handler; + private void f_handler_dg(Socket s) { // to avoid extra allocation w/ function + f_handler(s); + } + this(void function(Socket) handler) { + this.f_handler = handler; + this(&f_handler_dg); + } + + this(void delegate(Socket) handler) { + this.handler = handler; + super(&run, fiberStackSize); + } + + Socket connection; + void delegate(Socket) handler; + + void run() { + handler(connection); + } + + void delegate() postYield; + + private void setPostYield(scope void delegate() py) @nogc { + postYield = cast(void delegate()) py; + } + + void proceed() { + try { + call(); + auto py = postYield; + postYield = null; + if(py !is null) + py(); + } catch(Exception e) { + if(connection) + connection.close(); + goto terminate; + } + + if(state == State.TERM) { + terminate: + import core.memory; + GC.removeRoot(cast(void*) this); + } + } +} + +version(cgi_use_fiber) +version(Windows) { + +extern(Windows) private { + + import core.sys.windows.mswsock; + + alias GROUP=uint; + alias LPWSAPROTOCOL_INFOW = void*; + SOCKET WSASocketW(int af, int type, int protocol, LPWSAPROTOCOL_INFOW lpProtocolInfo, GROUP g, DWORD dwFlags); + int WSASend(SOCKET s, LPWSABUF lpBuffers, DWORD dwBufferCount, LPDWORD lpNumberOfBytesSent, DWORD dwFlags, LPWSAOVERLAPPED lpOverlapped, LPWSAOVERLAPPED_COMPLETION_ROUTINE lpCompletionRoutine); + int WSARecv(SOCKET s, LPWSABUF lpBuffers, DWORD dwBufferCount, LPDWORD lpNumberOfBytesRecvd, LPDWORD lpFlags, LPWSAOVERLAPPED lpOverlapped, LPWSAOVERLAPPED_COMPLETION_ROUTINE lpCompletionRoutine); + + struct WSABUF { + ULONG len; + CHAR *buf; + } + alias LPWSABUF = WSABUF*; + + alias WSAOVERLAPPED = OVERLAPPED; + alias LPWSAOVERLAPPED = LPOVERLAPPED; + /+ + + alias LPFN_ACCEPTEX = + BOOL + function( + SOCKET sListenSocket, + SOCKET sAcceptSocket, + //_Out_writes_bytes_(dwReceiveDataLength+dwLocalAddressLength+dwRemoteAddressLength) PVOID lpOutputBuffer, + void* lpOutputBuffer, + WORD dwReceiveDataLength, + WORD dwLocalAddressLength, + WORD dwRemoteAddressLength, + LPDWORD lpdwBytesReceived, + LPOVERLAPPED lpOverlapped + ); + + enum WSAID_ACCEPTEX = GUID([0xb5367df1,0xcbac,0x11cf,[0x95,0xca,0x00,0x80,0x5f,0x48,0xa1,0x92]]); + +/ + + enum WSAID_GETACCEPTEXSOCKADDRS = GUID(0xb5367df2,0xcbac,0x11cf,[0x95,0xca,0x00,0x80,0x5f,0x48,0xa1,0x92]); +} + +private class PseudoblockingOverlappedSocket : Socket { + SOCKET handle; + + CgiFiber fiber; + + this(AddressFamily af, SocketType st) { + auto handle = WSASocketW(af, st, 0, null, 0, 1 /*WSA_FLAG_OVERLAPPED*/); + if(!handle) + throw new Exception("WSASocketW"); + this.handle = handle; + + iocp = CreateIoCompletionPort(cast(HANDLE) handle, iocp, cast(ULONG_PTR) cast(void*) this, 0); + + if(iocp is null) { + writeln(GetLastError()); + throw new Exception("CreateIoCompletionPort"); + } + + super(cast(socket_t) handle, af); + } + this() pure nothrow @trusted { assert(0); } + + override void blocking(bool) {} // meaningless to us, just ignore it. + + protected override Socket accepting() pure nothrow { + assert(0); + } + + bool addressesParsed; + Address la; + Address ra; + + private void populateAddresses() { + if(addressesParsed) + return; + addressesParsed = true; + + int lalen, ralen; + + sockaddr_in* la; + sockaddr_in* ra; + + lpfnGetAcceptExSockaddrs( + scratchBuffer.ptr, + 0, // same as in the AcceptEx call! + sockaddr_in.sizeof + 16, + sockaddr_in.sizeof + 16, + cast(sockaddr**) &la, + &lalen, + cast(sockaddr**) &ra, + &ralen + ); + + if(la) + this.la = new InternetAddress(*la); + if(ra) + this.ra = new InternetAddress(*ra); + + } + + override @property @trusted Address localAddress() { + populateAddresses(); + return la; + } + override @property @trusted Address remoteAddress() { + populateAddresses(); + return ra; + } + + PseudoblockingOverlappedSocket accepted; + + __gshared static LPFN_ACCEPTEX lpfnAcceptEx; + __gshared static typeof(&GetAcceptExSockaddrs) lpfnGetAcceptExSockaddrs; + + override Socket accept() @trusted { + __gshared static LPFN_ACCEPTEX lpfnAcceptEx; + + if(lpfnAcceptEx is null) { + DWORD dwBytes; + GUID GuidAcceptEx = WSAID_ACCEPTEX; + + auto iResult = WSAIoctl(handle, 0xc8000006 /*SIO_GET_EXTENSION_FUNCTION_POINTER*/, + &GuidAcceptEx, GuidAcceptEx.sizeof, + &lpfnAcceptEx, lpfnAcceptEx.sizeof, + &dwBytes, null, null); + + GuidAcceptEx = WSAID_GETACCEPTEXSOCKADDRS; + iResult = WSAIoctl(handle, 0xc8000006 /*SIO_GET_EXTENSION_FUNCTION_POINTER*/, + &GuidAcceptEx, GuidAcceptEx.sizeof, + &lpfnGetAcceptExSockaddrs, lpfnGetAcceptExSockaddrs.sizeof, + &dwBytes, null, null); + + } + + auto pfa = new PseudoblockingOverlappedSocket(AddressFamily.INET, SocketType.STREAM); + accepted = pfa; + + SOCKET pendingForAccept = pfa.handle; + DWORD ignored; + + auto ret = lpfnAcceptEx(handle, + pendingForAccept, + // buffer to receive up front + pfa.scratchBuffer.ptr, + 0, + // size of local and remote addresses. normally + 16. + sockaddr_in.sizeof + 16, + sockaddr_in.sizeof + 16, + &ignored, // bytes would be given through the iocp instead but im not even requesting the thing + &overlapped + ); + + return pfa; + } + + override void connect(Address to) { assert(0); } + + DWORD lastAnswer; + ubyte[1024] scratchBuffer; + static assert(scratchBuffer.length > sockaddr_in.sizeof * 2 + 32); + + WSABUF[1] buffer; + OVERLAPPED overlapped; + override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) @trusted { + overlapped = overlapped.init; + buffer[0].len = cast(DWORD) buf.length; + buffer[0].buf = cast(CHAR*) buf.ptr; + fiber.setPostYield( () { + if(!WSASend(handle, buffer.ptr, cast(DWORD) buffer.length, null, 0, &overlapped, null)) { + if(GetLastError() != 997) { + //throw new Exception("WSASend fail"); + } + } + }); + + Fiber.yield(); + return lastAnswer; + } + override ptrdiff_t receive(scope void[] buf, SocketFlags flags) @trusted { + overlapped = overlapped.init; + buffer[0].len = cast(DWORD) buf.length; + buffer[0].buf = cast(CHAR*) buf.ptr; + + DWORD flags2 = 0; + + fiber.setPostYield(() { + if(!WSARecv(handle, buffer.ptr, cast(DWORD) buffer.length, null, &flags2 /* flags */, &overlapped, null)) { + if(GetLastError() != 997) { + //writeln("WSARecv ", WSAGetLastError()); + //throw new Exception("WSARecv fail"); + } + } + }); + + Fiber.yield(); + return lastAnswer; + } + + // I might go back and implement these for udp things. + override ptrdiff_t receiveFrom(scope void[] buf, SocketFlags flags, ref Address from) @trusted { + assert(0); + } + override ptrdiff_t receiveFrom(scope void[] buf, SocketFlags flags) @trusted { + assert(0); + } + override ptrdiff_t sendTo(scope const(void)[] buf, SocketFlags flags, Address to) @trusted { + assert(0); + } + override ptrdiff_t sendTo(scope const(void)[] buf, SocketFlags flags) @trusted { + assert(0); + } + + // lol overload sets + alias send = typeof(super).send; + alias receive = typeof(super).receive; + alias sendTo = typeof(super).sendTo; + alias receiveFrom = typeof(super).receiveFrom; + +} +} + +void doThreadHttpConnection(CustomCgi, alias fun)(Socket connection) { + assert(connection !is null); + version(cgi_use_fiber) { + auto fiber = new CgiFiber(&doThreadHttpConnectionGuts!(CustomCgi, fun)); + + version(Windows) { + (cast(PseudoblockingOverlappedSocket) connection).fiber = fiber; + } + + import core.memory; + GC.addRoot(cast(void*) fiber); + fiber.connection = connection; + fiber.proceed(); + } else { + doThreadHttpConnectionGuts!(CustomCgi, fun)(connection); + } +} + +void doThreadHttpConnectionGuts(CustomCgi, alias fun, bool alwaysCloseConnection = false)(Socket connection) { + scope(failure) { + // catch all for other errors + try { + sendAll(connection, plainHttpError(false, "500 Internal Server Error", null)); + connection.close(); + } catch(Exception e) {} // swallow it, we're aborting anyway. + } + + bool closeConnection = alwaysCloseConnection; + + /+ + ubyte[4096] inputBuffer = void; + ubyte[__traits(classInstanceSize, BufferedInputRange)] birBuffer = void; + ubyte[__traits(classInstanceSize, CustomCgi)] cgiBuffer = void; + + birBuffer[] = cast(ubyte[]) typeid(BufferedInputRange).initializer()[]; + BufferedInputRange ir = cast(BufferedInputRange) cast(void*) birBuffer.ptr; + ir.__ctor(connection, inputBuffer[], true); + +/ + + auto ir = new BufferedInputRange(connection); + + while(!ir.empty) { + + if(ir.view.length == 0) { + ir.popFront(); + if(ir.sourceClosed) { + connection.close(); + closeConnection = true; + break; + } + } + + Cgi cgi; + try { + cgi = new CustomCgi(ir, &closeConnection); + // There's a bunch of these casts around because the type matches up with + // the -version=.... specifiers, just you can also create a RequestServer + // and instantiate the things where the types don't match up. It isn't exactly + // correct but I also don't care rn. Might FIXME and either remove it later or something. + cgi._outputFileHandle = cast(CgiConnectionHandle) connection.handle; + } catch(ConnectionClosedException ce) { + closeConnection = true; + break; + } catch(ConnectionException ce) { + // broken pipe or something, just abort the connection + closeConnection = true; + break; + } catch(Throwable t) { + // a construction error is either bad code or bad request; bad request is what it should be since this is bug free :P + // anyway let's kill the connection + version(CRuntime_Musl) { + stderr.rawWrite(t.toString()); + stderr.rawWrite("\n"); + } else { + stderr.writeln(t.toString()); + } + sendAll(connection, plainHttpError(false, "400 Bad Request", t)); + closeConnection = true; + break; + } + assert(cgi !is null); + scope(exit) + cgi.dispose(); + + try { + fun(cgi); + cgi.close(); + if(cgi.websocketMode) + closeConnection = true; + } catch(ConnectionException ce) { + // broken pipe or something, just abort the connection + closeConnection = true; + } catch(ConnectionClosedException ce) { + // broken pipe or something, just abort the connection + closeConnection = true; + } catch(Throwable t) { + // a processing error can be recovered from + version(CRuntime_Musl) {} else + stderr.writeln(t.toString); + if(!handleException(cgi, t)) + closeConnection = true; + } + + if(globalStopFlag) + closeConnection = true; + + if(closeConnection || alwaysCloseConnection) { + connection.shutdown(SocketShutdown.BOTH); + connection.close(); + ir.dispose(); + closeConnection = false; // don't reclose after loop + break; + } else { + if(ir.front.length) { + ir.popFront(); // we can't just discard the buffer, so get the next bit and keep chugging along + } else if(ir.sourceClosed) { + ir.source.shutdown(SocketShutdown.BOTH); + ir.source.close(); + ir.dispose(); + closeConnection = false; + } else { + continue; + // break; // this was for a keepalive experiment + } + } + } + + if(closeConnection) { + connection.shutdown(SocketShutdown.BOTH); + connection.close(); + ir.dispose(); + } + + // I am otherwise NOT closing it here because the parent thread might still be able to make use of the keep-alive connection! +} + +void doThreadScgiConnection(CustomCgi, alias fun, long maxContentLength)(Socket connection) { + // and now we can buffer + scope(failure) + connection.close(); + + import al = std.algorithm; + + size_t size; + + string[string] headers; + + auto range = new BufferedInputRange(connection); + more_data: + auto chunk = range.front(); + // waiting for colon for header length + auto idx = indexOf(cast(string) chunk, ':'); + if(idx == -1) { + try { + range.popFront(); + } catch(Exception e) { + // it is just closed, no big deal + connection.close(); + return; + } + goto more_data; + } + + size = to!size_t(cast(string) chunk[0 .. idx]); + chunk = range.consume(idx + 1); + // reading headers + if(chunk.length < size) + range.popFront(0, size + 1); + // we are now guaranteed to have enough + chunk = range.front(); + assert(chunk.length > size); + + idx = 0; + string key; + string value; + foreach(part; al.splitter(chunk, '\0')) { + if(idx & 1) { // odd is value + value = cast(string)(part.idup); + headers[key] = value; // commit + } else + key = cast(string)(part.idup); + idx++; + } + + enforce(chunk[size] == ','); // the terminator + + range.consume(size + 1); + // reading data + // this will be done by Cgi + + const(ubyte)[] getScgiChunk() { + // we are already primed + auto data = range.front(); + if(data.length == 0 && !range.sourceClosed) { + range.popFront(0); + data = range.front(); + } else if (range.sourceClosed) + range.source.close(); + + return data; + } + + void writeScgi(const(ubyte)[] data) { + sendAll(connection, data); + } + + void flushScgi() { + // I don't *think* I have to do anything.... + } + + Cgi cgi; + try { + cgi = new CustomCgi(maxContentLength, headers, &getScgiChunk, &writeScgi, &flushScgi); + cgi._outputFileHandle = cast(CgiConnectionHandle) connection.handle; + } catch(Throwable t) { + sendAll(connection, plainHttpError(true, "400 Bad Request", t)); + connection.close(); + return; // this connection is dead + } + assert(cgi !is null); + scope(exit) cgi.dispose(); + try { + fun(cgi); + cgi.close(); + connection.close(); + } catch(Throwable t) { + // no std err + if(!handleException(cgi, t)) { + connection.close(); + return; + } else { + connection.close(); + return; + } + } +} + +string printDate(DateTime date) { + char[29] buffer = void; + printDateToBuffer(date, buffer[]); + return buffer.idup; +} + +int printDateToBuffer(DateTime date, char[] buffer) @nogc { + assert(buffer.length >= 29); + // 29 static length ? + + static immutable daysOfWeek = [ + "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" + ]; + + static immutable months = [ + null, "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" + ]; + + buffer[0 .. 3] = daysOfWeek[date.dayOfWeek]; + buffer[3 .. 5] = ", "; + buffer[5] = date.day / 10 + '0'; + buffer[6] = date.day % 10 + '0'; + buffer[7] = ' '; + buffer[8 .. 11] = months[date.month]; + buffer[11] = ' '; + auto y = date.year; + buffer[12] = cast(char) (y / 1000 + '0'); y %= 1000; + buffer[13] = cast(char) (y / 100 + '0'); y %= 100; + buffer[14] = cast(char) (y / 10 + '0'); y %= 10; + buffer[15] = cast(char) (y + '0'); + buffer[16] = ' '; + buffer[17] = date.hour / 10 + '0'; + buffer[18] = date.hour % 10 + '0'; + buffer[19] = ':'; + buffer[20] = date.minute / 10 + '0'; + buffer[21] = date.minute % 10 + '0'; + buffer[22] = ':'; + buffer[23] = date.second / 10 + '0'; + buffer[24] = date.second % 10 + '0'; + buffer[25 .. $] = " GMT"; + + return 29; +} + + +// Referencing this gigantic typeid seems to remind the compiler +// to actually put the symbol in the object file. I guess the immutable +// assoc array array isn't actually included in druntime +void hackAroundLinkerError() { + stdout.rawWrite(typeid(const(immutable(char)[][])[immutable(char)[]]).toString()); + stdout.rawWrite(typeid(immutable(char)[][][immutable(char)[]]).toString()); + stdout.rawWrite(typeid(Cgi.UploadedFile[immutable(char)[]]).toString()); + stdout.rawWrite(typeid(Cgi.UploadedFile[][immutable(char)[]]).toString()); + stdout.rawWrite(typeid(immutable(Cgi.UploadedFile)[immutable(char)[]]).toString()); + stdout.rawWrite(typeid(immutable(Cgi.UploadedFile[])[immutable(char)[]]).toString()); + stdout.rawWrite(typeid(immutable(char[])[immutable(char)[]]).toString()); + // this is getting kinda ridiculous btw. Moving assoc arrays + // to the library is the pain that keeps on coming. + + // eh this broke the build on the work server + // stdout.rawWrite(typeid(immutable(char)[][immutable(string[])])); + stdout.rawWrite(typeid(immutable(string[])[immutable(char)[]]).toString()); +} + + + + + +version(fastcgi) { + pragma(lib, "fcgi"); + + static if(size_t.sizeof == 8) // 64 bit + alias long c_int; + else + alias int c_int; + + extern(C) { + struct FCGX_Stream { + ubyte* rdNext; + ubyte* wrNext; + ubyte* stop; + ubyte* stopUnget; + c_int isReader; + c_int isClosed; + c_int wasFCloseCalled; + c_int FCGI_errno; + void* function(FCGX_Stream* stream) fillBuffProc; + void* function(FCGX_Stream* stream, c_int doClose) emptyBuffProc; + void* data; + } + + // note: this is meant to be opaque, so don't access it directly + struct FCGX_Request { + int requestId; + int role; + FCGX_Stream* inStream; + FCGX_Stream* outStream; + FCGX_Stream* errStream; + char** envp; + void* paramsPtr; + int ipcFd; + int isBeginProcessed; + int keepConnection; + int appStatus; + int nWriters; + int flags; + int listen_sock; + } + + int FCGX_InitRequest(FCGX_Request *request, int sock, int flags); + void FCGX_Init(); + + int FCGX_Accept_r(FCGX_Request *request); + + + alias char** FCGX_ParamArray; + + c_int FCGX_Accept(FCGX_Stream** stdin, FCGX_Stream** stdout, FCGX_Stream** stderr, FCGX_ParamArray* envp); + c_int FCGX_GetChar(FCGX_Stream* stream); + c_int FCGX_PutStr(const ubyte* str, c_int n, FCGX_Stream* stream); + int FCGX_HasSeenEOF(FCGX_Stream* stream); + c_int FCGX_FFlush(FCGX_Stream *stream); + + int FCGX_OpenSocket(in char*, int); + } +} + + +/* This might go int a separate module eventually. It is a network input helper class. */ + +import std.socket; + +version(cgi_use_fiber) { + import core.thread; + + version(linux) { + import core.sys.linux.epoll; + + int epfd = -1; // thread local because EPOLLEXCLUSIVE works much better this way... weirdly. + } else version(Windows) { + // declaring the iocp thing below... + } else static assert(0, "The hybrid fiber server is not implemented on your OS."); +} + +version(Windows) + __gshared HANDLE iocp; + +version(cgi_use_fiber) { + version(linux) + private enum WakeupEvent { + Read = EPOLLIN, + Write = EPOLLOUT + } + else version(Windows) + private enum WakeupEvent { + Read, Write + } + else static assert(0); +} + +version(cgi_use_fiber) +private void registerEventWakeup(bool* registered, Socket source, WakeupEvent e) @nogc { + + // static cast since I know what i have in here and don't want to pay for dynamic cast + auto f = cast(CgiFiber) cast(void*) Fiber.getThis(); + + version(linux) { + f.setPostYield = () { + if(*registered) { + // rearm + epoll_event evt; + evt.events = e | EPOLLONESHOT; + evt.data.ptr = cast(void*) f; + if(epoll_ctl(epfd, EPOLL_CTL_MOD, source.handle, &evt) == -1) + throw new Exception("epoll_ctl"); + } else { + // initial registration + *registered = true ; + int fd = source.handle; + epoll_event evt; + evt.events = e | EPOLLONESHOT; + evt.data.ptr = cast(void*) f; + if(epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &evt) == -1) + throw new Exception("epoll_ctl"); + } + }; + + Fiber.yield(); + + f.setPostYield(null); + } else version(Windows) { + Fiber.yield(); + } + else static assert(0); +} + +version(cgi_use_fiber) +void unregisterSource(Socket s) { + version(linux) { + epoll_event evt; + epoll_ctl(epfd, EPOLL_CTL_DEL, s.handle(), &evt); + } else version(Windows) { + // intentionally blank + } + else static assert(0); +} + +// it is a class primarily for reference semantics +// I might change this interface +/// This is NOT ACTUALLY an input range! It is too different. Historical mistake kinda. +class BufferedInputRange { + version(Posix) + this(int source, ubyte[] buffer = null) { + this(new Socket(cast(socket_t) source, AddressFamily.INET), buffer); + } + + this(Socket source, ubyte[] buffer = null, bool allowGrowth = true) { + // if they connect but never send stuff to us, we don't want it wasting the process + // so setting a time out + version(cgi_use_fiber) + source.blocking = false; + else + source.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"seconds"(3)); + + this.source = source; + if(buffer is null) { + underlyingBuffer = new ubyte[4096]; + this.allowGrowth = true; + } else { + underlyingBuffer = buffer; + this.allowGrowth = allowGrowth; + } + + assert(underlyingBuffer.length); + + // we assume view.ptr is always inside underlyingBuffer + view = underlyingBuffer[0 .. 0]; + + popFront(); // prime + } + + version(cgi_use_fiber) { + bool registered; + } + + void dispose() { + version(cgi_use_fiber) { + if(registered) + unregisterSource(source); + } + } + + /** + A slight difference from regular ranges is you can give it the maximum + number of bytes to consume. + + IMPORTANT NOTE: the default is to consume nothing, so if you don't call + consume() yourself and use a regular foreach, it will infinitely loop! + + The default is to do what a normal range does, and consume the whole buffer + and wait for additional input. + + You can also specify 0, to append to the buffer, or any other number + to remove the front n bytes and wait for more. + */ + void popFront(size_t maxBytesToConsume = 0 /*size_t.max*/, size_t minBytesToSettleFor = 0, bool skipConsume = false) { + if(sourceClosed) + throw new ConnectionClosedException("can't get any more data from a closed source"); + if(!skipConsume) + consume(maxBytesToConsume); + + // we might have to grow the buffer + if(minBytesToSettleFor > underlyingBuffer.length || view.length == underlyingBuffer.length) { + if(allowGrowth) { + //import std.stdio; writeln("growth"); + auto viewStart = view.ptr - underlyingBuffer.ptr; + size_t growth = 4096; + // make sure we have enough for what we're being asked for + if(minBytesToSettleFor > 0 && minBytesToSettleFor - underlyingBuffer.length > growth) + growth = minBytesToSettleFor - underlyingBuffer.length; + //import std.stdio; writeln(underlyingBuffer.length, " ", viewStart, " ", view.length, " ", growth, " ", minBytesToSettleFor, " ", minBytesToSettleFor - underlyingBuffer.length); + underlyingBuffer.length += growth; + view = underlyingBuffer[viewStart .. view.length]; + } else + throw new Exception("No room left in the buffer"); + } + + do { + auto freeSpace = underlyingBuffer[view.ptr - underlyingBuffer.ptr + view.length .. $]; + try_again: + auto ret = source.receive(freeSpace); + if(ret == Socket.ERROR) { + if(wouldHaveBlocked()) { + version(cgi_use_fiber) { + registerEventWakeup(®istered, source, WakeupEvent.Read); + goto try_again; + } else { + // gonna treat a timeout here as a close + sourceClosed = true; + return; + } + } + version(Posix) { + import core.stdc.errno; + if(errno == EINTR || errno == EAGAIN) { + goto try_again; + } + if(errno == ECONNRESET) { + sourceClosed = true; + return; + } + } + throw new Exception(lastSocketError); // FIXME + } + if(ret == 0) { + sourceClosed = true; + return; + } + + //import std.stdio; writeln(view.ptr); writeln(underlyingBuffer.ptr); writeln(view.length, " ", ret, " = ", view.length + ret); + view = underlyingBuffer[view.ptr - underlyingBuffer.ptr .. view.length + ret]; + //import std.stdio; writeln(cast(string) view); + } while(view.length < minBytesToSettleFor); + } + + /// Removes n bytes from the front of the buffer, and returns the new buffer slice. + /// You might want to idup the data you are consuming if you store it, since it may + /// be overwritten on the new popFront. + /// + /// You do not need to call this if you always want to wait for more data when you + /// consume some. + ubyte[] consume(size_t bytes) { + //import std.stdio; writeln("consuime ", bytes, "/", view.length); + view = view[bytes > $ ? $ : bytes .. $]; + if(view.length == 0) { + view = underlyingBuffer[0 .. 0]; // go ahead and reuse the beginning + /* + writeln("HERE"); + popFront(0, 0, true); // try to load more if we can, checks if the source is closed + writeln(cast(string)front); + writeln("DONE"); + */ + } + return front; + } + + bool empty() { + return sourceClosed && view.length == 0; + } + + ubyte[] front() { + return view; + } + + invariant() { + assert(view.ptr >= underlyingBuffer.ptr); + // it should never be equal, since if that happens view ought to be empty, and thus reusing the buffer + assert(view.ptr < underlyingBuffer.ptr + underlyingBuffer.length); + } + + ubyte[] underlyingBuffer; + bool allowGrowth; + ubyte[] view; + Socket source; + bool sourceClosed; +} + +private class FakeSocketForStdin : Socket { + import std.stdio; + + this() { + + } + + private bool closed; + + override ptrdiff_t receive(scope void[] buffer, std.socket.SocketFlags) @trusted { + if(closed) + throw new Exception("Closed"); + return stdin.rawRead(buffer).length; + } + + override ptrdiff_t send(const scope void[] buffer, std.socket.SocketFlags) @trusted { + if(closed) + throw new Exception("Closed"); + stdout.rawWrite(buffer); + return buffer.length; + } + + override void close() @trusted scope { + (cast(void delegate() @nogc nothrow) &realClose)(); + } + + override void shutdown(SocketShutdown s) { + // FIXME + } + + override void setOption(SocketOptionLevel, SocketOption, scope void[]) {} + override void setOption(SocketOptionLevel, SocketOption, Duration) {} + + override @property @trusted Address remoteAddress() { return null; } + override @property @trusted Address localAddress() { return null; } + + void realClose() { + closed = true; + try { + stdin.close(); + stdout.close(); + } catch(Exception e) { + + } + } +} + +import core.sync.semaphore; +import core.atomic; + +/** + To use this thing: + + --- + void handler(Socket s) { do something... } + auto manager = new ListeningConnectionManager("127.0.0.1", 80, &handler, &delegateThatDropsPrivileges); + manager.listen(); + --- + + The 4th parameter is optional. + + I suggest you use BufferedInputRange(connection) to handle the input. As a packet + comes in, you will get control. You can just continue; though to fetch more. + + + FIXME: should I offer an event based async thing like netman did too? Yeah, probably. +*/ +class ListeningConnectionManager { + Semaphore semaphore; + Socket[256] queue; + shared(ubyte) nextIndexFront; + ubyte nextIndexBack; + shared(int) queueLength; + + Socket acceptCancelable() { + version(Posix) { + import core.sys.posix.sys.select; + fd_set read_fds; + FD_ZERO(&read_fds); + FD_SET(listener.handle, &read_fds); + if(cancelfd != -1) + FD_SET(cancelfd, &read_fds); + auto max = listener.handle > cancelfd ? listener.handle : cancelfd; + auto ret = select(max + 1, &read_fds, null, null, null); + if(ret == -1) { + import core.stdc.errno; + if(errno == EINTR) + return null; + else + throw new Exception("wtf select"); + } + + if(cancelfd != -1 && FD_ISSET(cancelfd, &read_fds)) { + return null; + } + + if(FD_ISSET(listener.handle, &read_fds)) + return listener.accept(); + + return null; + } else { + + Socket socket = listener; + + auto check = new SocketSet(); + + keep_looping: + check.reset(); + check.add(socket); + + // just to check the stop flag on a kinda busy loop. i hate this FIXME + auto got = Socket.select(check, null, null, 3.seconds); + if(got > 0) + return listener.accept(); + if(globalStopFlag) + return null; + else + goto keep_looping; + } + } + + int defaultNumberOfThreads() { + import std.parallelism; + version(cgi_use_fiber) { + return totalCPUs * 1 + 1; + } else { + // I times 4 here because there's a good chance some will be blocked on i/o. + return totalCPUs * 4; + } + + } + + void listen() { + shared(int) loopBroken; + + version(Posix) { + import core.sys.posix.signal; + signal(SIGPIPE, SIG_IGN); + } + + version(linux) { + if(cancelfd == -1) + cancelfd = eventfd(0, 0); + } + + version(cgi_no_threads) { + // NEVER USE THIS + // it exists only for debugging and other special occasions + + // the thread mode is faster and less likely to stall the whole + // thing when a request is slow + while(!loopBroken && !globalStopFlag) { + auto sn = acceptCancelable(); + if(sn is null) continue; + cloexec(sn); + try { + handler(sn); + } catch(Exception e) { + // if a connection goes wrong, we want to just say no, but try to carry on unless it is an Error of some sort (in which case, we'll die. You might want an external helper program to revive the server when it dies) + sn.close(); + } + } + } else { + + if(useFork) { + version(linux) { + //asm { int 3; } + fork(); + } + } + + version(cgi_use_fiber) { + + version(Windows) { + listener.accept(); + } + + WorkerThread[] threads = new WorkerThread[](numberOfThreads); + foreach(i, ref thread; threads) { + thread = new WorkerThread(this, handler, cast(int) i); + thread.start(); + } + + bool fiber_crash_check() { + bool hasAnyRunning; + foreach(thread; threads) { + if(!thread.isRunning) { + thread.join(); + } else hasAnyRunning = true; + } + + return (!hasAnyRunning); + } + + + while(!globalStopFlag) { + Thread.sleep(1.seconds); + if(fiber_crash_check()) + break; + } + + } else { + semaphore = new Semaphore(); + + ConnectionThread[] threads = new ConnectionThread[](numberOfThreads); + foreach(i, ref thread; threads) { + thread = new ConnectionThread(this, handler, cast(int) i); + thread.start(); + } + + while(!loopBroken && !globalStopFlag) { + Socket sn; + + bool crash_check() { + bool hasAnyRunning; + foreach(thread; threads) { + if(!thread.isRunning) { + thread.join(); + } else hasAnyRunning = true; + } + + return (!hasAnyRunning); + } + + + void accept_new_connection() { + sn = acceptCancelable(); + if(sn is null) return; + cloexec(sn); + if(tcp) { + // disable Nagle's algorithm to avoid a 40ms delay when we send/recv + // on the socket because we do some buffering internally. I think this helps, + // certainly does for small requests, and I think it does for larger ones too + sn.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); + + sn.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"seconds"(10)); + } + } + + void existing_connection_new_data() { + // wait until a slot opens up + //int waited = 0; + while(queueLength >= queue.length) { + Thread.sleep(1.msecs); + //waited ++; + } + //if(waited) {import std.stdio; writeln(waited);} + synchronized(this) { + queue[nextIndexBack] = sn; + nextIndexBack++; + atomicOp!"+="(queueLength, 1); + } + semaphore.notify(); + } + + + accept_new_connection(); + if(sn !is null) + existing_connection_new_data(); + else if(sn is null && globalStopFlag) { + foreach(thread; threads) { + semaphore.notify(); + } + Thread.sleep(50.msecs); + } + + if(crash_check()) + break; + } + } + + // FIXME: i typically stop this with ctrl+c which never + // actually gets here. i need to do a sigint handler. + if(cleanup) + cleanup(); + } + } + + //version(linux) + //int epoll_fd; + + bool tcp; + void delegate() cleanup; + + private void function(Socket) fhandler; + private void dg_handler(Socket s) { + fhandler(s); + } + this(string host, ushort port, void function(Socket) handler, void delegate() dropPrivs = null, bool useFork = cgi_use_fork_default, int numberOfThreads = 0) { + fhandler = handler; + this(host, port, &dg_handler, dropPrivs, useFork, numberOfThreads); + } + + this(string host, ushort port, void delegate(Socket) handler, void delegate() dropPrivs = null, bool useFork = cgi_use_fork_default, int numberOfThreads = 0) { + this.handler = handler; + this.useFork = useFork; + this.numberOfThreads = numberOfThreads ? numberOfThreads : defaultNumberOfThreads(); + + listener = startListening(host, port, tcp, cleanup, 128, dropPrivs); + + version(cgi_use_fiber) + if(useFork) + listener.blocking = false; + + // this is the UI control thread and thus gets more priority + Thread.getThis.priority = Thread.PRIORITY_MAX; + } + + Socket listener; + void delegate(Socket) handler; + + immutable bool useFork; + int numberOfThreads; +} + +Socket startListening(string host, ushort port, ref bool tcp, ref void delegate() cleanup, int backQueue, void delegate() dropPrivs) { + Socket listener; + if(host.startsWith("unix:")) { + version(Posix) { + listener = new Socket(AddressFamily.UNIX, SocketType.STREAM); + cloexec(listener); + string filename = host["unix:".length .. $].idup; + listener.bind(new UnixAddress(filename)); + cleanup = delegate() { + listener.close(); + import std.file; + remove(filename); + }; + tcp = false; + } else { + throw new Exception("unix sockets not supported on this system"); + } + } else if(host.startsWith("abstract:")) { + version(linux) { + listener = new Socket(AddressFamily.UNIX, SocketType.STREAM); + cloexec(listener); + string filename = "\0" ~ host["abstract:".length .. $]; + import std.stdio; stderr.writeln("Listening to abstract unix domain socket: ", host["abstract:".length .. $]); + listener.bind(new UnixAddress(filename)); + tcp = false; + } else { + throw new Exception("abstract unix sockets not supported on this system"); + } + } else { + version(cgi_use_fiber) { + version(Windows) + listener = new PseudoblockingOverlappedSocket(AddressFamily.INET, SocketType.STREAM); + else + listener = new TcpSocket(); + } else { + listener = new TcpSocket(); + } + cloexec(listener); + listener.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true); + listener.bind(host.length ? parseAddress(host, port) : new InternetAddress(port)); + cleanup = delegate() { + listener.close(); + }; + tcp = true; + } + + listener.listen(backQueue); + + if (dropPrivs !is null) // can be null, backwards compatibility + dropPrivs(); + + return listener; +} + +// helper function to send a lot to a socket. Since this blocks for the buffer (possibly several times), you should probably call it in a separate thread or something. +void sendAll(Socket s, const(void)[] data, string file = __FILE__, size_t line = __LINE__) { + if(data.length == 0) return; + ptrdiff_t amount; + //import std.stdio; writeln("***",cast(string) data,"///"); + do { + amount = s.send(data); + if(amount == Socket.ERROR) { + version(cgi_use_fiber) { + if(wouldHaveBlocked()) { + bool registered = true; + registerEventWakeup(®istered, s, WakeupEvent.Write); + continue; + } + } + throw new ConnectionException(s, lastSocketError, file, line); + } + assert(amount > 0); + + data = data[amount .. $]; + } while(data.length); +} + +class ConnectionException : Exception { + Socket socket; + this(Socket s, string msg, string file = __FILE__, size_t line = __LINE__) { + this.socket = s; + super(msg, file, line); + } +} + +alias void delegate(Socket) CMT; + +import core.thread; +/+ + cgi.d now uses a hybrid of event i/o and threads at the top level. + + Top level thread is responsible for accepting sockets and selecting on them. + + It then indicates to a child that a request is pending, and any random worker + thread that is free handles it. It goes into blocking mode and handles that + http request to completion. + + At that point, it goes back into the waiting queue. + + + This concept is only implemented on Linux. On all other systems, it still + uses the worker threads and semaphores (which is perfectly fine for a lot of + things! Just having a great number of keep-alive connections will break that.) + + + So the algorithm is: + + select(accept, event, pending) + if accept -> send socket to free thread, if any. if not, add socket to queue + if event -> send the signaling thread a socket from the queue, if not, mark it free + - event might block until it can be *written* to. it is a fifo sending socket fds! + + A worker only does one http request at a time, then signals its availability back to the boss. + + The socket the worker was just doing should be added to the one-off epoll read. If it is closed, + great, we can get rid of it. Otherwise, it is considered `pending`. The *kernel* manages that; the + actual FD will not be kept out here. + + So: + queue = sockets we know are ready to read now, but no worker thread is available + idle list = worker threads not doing anything else. they signal back and forth + + the workers all read off the event fd. This is the semaphore wait + + the boss waits on accept or other sockets read events (one off! and level triggered). If anything happens wrt ready read, + it puts it in the queue and writes to the event fd. + + The child could put the socket back in the epoll thing itself. + + The child needs to be able to gracefully handle being given a socket that just closed with no work. ++/ +class ConnectionThread : Thread { + this(ListeningConnectionManager lcm, CMT dg, int myThreadNumber) { + this.lcm = lcm; + this.dg = dg; + this.myThreadNumber = myThreadNumber; + super(&run); + } + + void run() { + while(true) { + // so if there's a bunch of idle keep-alive connections, it can + // consume all the worker threads... just sitting there. + lcm.semaphore.wait(); + if(globalStopFlag) + return; + Socket socket; + synchronized(lcm) { + auto idx = lcm.nextIndexFront; + socket = lcm.queue[idx]; + lcm.queue[idx] = null; + atomicOp!"+="(lcm.nextIndexFront, 1); + atomicOp!"-="(lcm.queueLength, 1); + } + try { + //import std.stdio; writeln(myThreadNumber, " taking it"); + dg(socket); + /+ + if(socket.isAlive) { + // process it more later + version(linux) { + import core.sys.linux.epoll; + epoll_event ev; + ev.events = EPOLLIN | EPOLLONESHOT | EPOLLET; + ev.data.fd = socket.handle; + import std.stdio; writeln("adding"); + if(epoll_ctl(lcm.epoll_fd, EPOLL_CTL_ADD, socket.handle, &ev) == -1) { + if(errno == EEXIST) { + ev.events = EPOLLIN | EPOLLONESHOT | EPOLLET; + ev.data.fd = socket.handle; + if(epoll_ctl(lcm.epoll_fd, EPOLL_CTL_MOD, socket.handle, &ev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + } else + throw new Exception("epoll_ctl " ~ to!string(errno)); + } + //import std.stdio; writeln("keep alive"); + // writing to this private member is to prevent the GC from closing my precious socket when I'm trying to use it later + __traits(getMember, socket, "sock") = cast(socket_t) -1; + } else { + continue; // hope it times out in a reasonable amount of time... + } + } + +/ + } catch(ConnectionClosedException e) { + // can just ignore this, it is fairly normal + socket.close(); + } catch(Throwable e) { + import std.stdio; stderr.rawWrite(e.toString); stderr.rawWrite("\n"); + socket.close(); + } + } + } + + ListeningConnectionManager lcm; + CMT dg; + int myThreadNumber; +} + +version(cgi_use_fiber) +class WorkerThread : Thread { + this(ListeningConnectionManager lcm, CMT dg, int myThreadNumber) { + this.lcm = lcm; + this.dg = dg; + this.myThreadNumber = myThreadNumber; + super(&run); + } + + version(Windows) + void run() { + auto timeout = INFINITE; + PseudoblockingOverlappedSocket key; + OVERLAPPED* overlapped; + DWORD bytes; + while(!globalStopFlag && GetQueuedCompletionStatus(iocp, &bytes, cast(PULONG_PTR) &key, &overlapped, timeout)) { + if(key is null) + continue; + key.lastAnswer = bytes; + if(key.fiber) { + key.fiber.proceed(); + } else { + // we have a new connection, issue the first receive on it and issue the next accept + + auto sn = key.accepted; + + key.accept(); + + cloexec(sn); + if(lcm.tcp) { + // disable Nagle's algorithm to avoid a 40ms delay when we send/recv + // on the socket because we do some buffering internally. I think this helps, + // certainly does for small requests, and I think it does for larger ones too + sn.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); + + sn.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"seconds"(10)); + } + + dg(sn); + } + } + //SleepEx(INFINITE, TRUE); + } + + version(linux) + void run() { + + import core.sys.linux.epoll; + epfd = epoll_create1(EPOLL_CLOEXEC); + if(epfd == -1) + throw new Exception("epoll_create1 " ~ to!string(errno)); + scope(exit) { + import core.sys.posix.unistd; + close(epfd); + } + + { + epoll_event ev; + ev.events = EPOLLIN; + ev.data.fd = cancelfd; + epoll_ctl(epfd, EPOLL_CTL_ADD, cancelfd, &ev); + } + + epoll_event ev; + ev.events = EPOLLIN | EPOLLEXCLUSIVE; // EPOLLEXCLUSIVE is only available on kernels since like 2017 but that's prolly good enough. + ev.data.fd = lcm.listener.handle; + if(epoll_ctl(epfd, EPOLL_CTL_ADD, lcm.listener.handle, &ev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + + + + while(!globalStopFlag) { + Socket sn; + + epoll_event[64] events; + auto nfds = epoll_wait(epfd, events.ptr, events.length, -1); + if(nfds == -1) { + if(errno == EINTR) + continue; + throw new Exception("epoll_wait " ~ to!string(errno)); + } + + foreach(idx; 0 .. nfds) { + auto flags = events[idx].events; + + if(cast(size_t) events[idx].data.ptr == cast(size_t) cancelfd) { + globalStopFlag = true; + //import std.stdio; writeln("exit heard"); + break; + } else if(cast(size_t) events[idx].data.ptr == cast(size_t) lcm.listener.handle) { + //import std.stdio; writeln(myThreadNumber, " woken up ", flags); + // this try/catch is because it is set to non-blocking mode + // and Phobos' stupid api throws an exception instead of returning + // if it would block. Why would it block? because a forked process + // might have beat us to it, but the wakeup event thundered our herds. + try + sn = lcm.listener.accept(); // don't need to do the acceptCancelable here since the epoll checks it better + catch(SocketAcceptException e) { continue; } + + cloexec(sn); + if(lcm.tcp) { + // disable Nagle's algorithm to avoid a 40ms delay when we send/recv + // on the socket because we do some buffering internally. I think this helps, + // certainly does for small requests, and I think it does for larger ones too + sn.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); + + sn.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"seconds"(10)); + } + + dg(sn); + } else { + if(cast(size_t) events[idx].data.ptr < 1024) { + throw new Exception("this doesn't look like a fiber pointer..."); + } + auto fiber = cast(CgiFiber) events[idx].data.ptr; + fiber.proceed(); + } + } + } + } + + ListeningConnectionManager lcm; + CMT dg; + int myThreadNumber; +} + + +/* Done with network helper */ + +/* Helpers for doing temporary files. Used both here and in web.d */ + +version(Windows) { + import core.sys.windows.windows; + extern(Windows) DWORD GetTempPathW(DWORD, LPWSTR); + alias GetTempPathW GetTempPath; +} + +version(Posix) { + static import linux = core.sys.posix.unistd; +} + +string getTempDirectory() { + string path; + version(Windows) { + wchar[1024] buffer; + auto len = GetTempPath(1024, buffer.ptr); + if(len == 0) + throw new Exception("couldn't find a temporary path"); + + auto b = buffer[0 .. len]; + + path = to!string(b); + } else + path = "/tmp/"; + + return path; +} + + +// I like std.date. These functions help keep my old code and data working with phobos changing. + +long sysTimeToDTime(in SysTime sysTime) { + return convert!("hnsecs", "msecs")(sysTime.stdTime - 621355968000000000L); +} + +long dateTimeToDTime(in DateTime dt) { + return sysTimeToDTime(cast(SysTime) dt); +} + +long getUtcTime() { // renamed primarily to avoid conflict with std.date itself + return sysTimeToDTime(Clock.currTime(UTC())); +} + +// NOTE: new SimpleTimeZone(minutes); can perhaps work with the getTimezoneOffset() JS trick +SysTime dTimeToSysTime(long dTime, immutable TimeZone tz = null) { + immutable hnsecs = convert!("msecs", "hnsecs")(dTime) + 621355968000000000L; + return SysTime(hnsecs, tz); +} + + + +// this is a helper to read HTTP transfer-encoding: chunked responses +immutable(ubyte[]) dechunk(BufferedInputRange ir) { + immutable(ubyte)[] ret; + + another_chunk: + // If here, we are at the beginning of a chunk. + auto a = ir.front(); + int chunkSize; + int loc = locationOf(a, "\r\n"); + while(loc == -1) { + ir.popFront(); + a = ir.front(); + loc = locationOf(a, "\r\n"); + } + + string hex; + hex = ""; + for(int i = 0; i < loc; i++) { + char c = a[i]; + if(c >= 'A' && c <= 'Z') + c += 0x20; + if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z')) { + hex ~= c; + } else { + break; + } + } + + assert(hex.length); + + int power = 1; + int size = 0; + foreach(cc1; retro(hex)) { + dchar cc = cc1; + if(cc >= 'a' && cc <= 'z') + cc -= 0x20; + int val = 0; + if(cc >= '0' && cc <= '9') + val = cc - '0'; + else + val = cc - 'A' + 10; + + size += power * val; + power *= 16; + } + + chunkSize = size; + assert(size >= 0); + + if(loc + 2 > a.length) { + ir.popFront(0, a.length + loc + 2); + a = ir.front(); + } + + a = ir.consume(loc + 2); + + if(chunkSize == 0) { // we're done with the response + // if we got here, will change must be true.... + more_footers: + loc = locationOf(a, "\r\n"); + if(loc == -1) { + ir.popFront(); + a = ir.front; + goto more_footers; + } else { + assert(loc == 0); + ir.consume(loc + 2); + goto finish; + } + } else { + // if we got here, will change must be true.... + if(a.length < chunkSize + 2) { + ir.popFront(0, chunkSize + 2); + a = ir.front(); + } + + ret ~= (a[0..chunkSize]); + + if(!(a.length > chunkSize + 2)) { + ir.popFront(0, chunkSize + 2); + a = ir.front(); + } + assert(a[chunkSize] == 13); + assert(a[chunkSize+1] == 10); + a = ir.consume(chunkSize + 2); + chunkSize = 0; + goto another_chunk; + } + + finish: + return ret; +} + +// I want to be able to get data from multiple sources the same way... +interface ByChunkRange { + bool empty(); + void popFront(); + const(ubyte)[] front(); +} + +ByChunkRange byChunk(const(ubyte)[] data) { + return new class ByChunkRange { + override bool empty() { + return !data.length; + } + + override void popFront() { + if(data.length > 4096) + data = data[4096 .. $]; + else + data = null; + } + + override const(ubyte)[] front() { + return data[0 .. $ > 4096 ? 4096 : $]; + } + }; +} + +ByChunkRange byChunk(BufferedInputRange ir, size_t atMost) { + const(ubyte)[] f; + + f = ir.front; + if(f.length > atMost) + f = f[0 .. atMost]; + + return new class ByChunkRange { + override bool empty() { + return atMost == 0; + } + + override const(ubyte)[] front() { + return f; + } + + override void popFront() { + ir.consume(f.length); + atMost -= f.length; + auto a = ir.front(); + + if(a.length <= atMost) { + f = a; + atMost -= a.length; + a = ir.consume(a.length); + if(atMost != 0) + ir.popFront(); + if(f.length == 0) { + f = ir.front(); + } + } else { + // we actually have *more* here than we need.... + f = a[0..atMost]; + atMost = 0; + ir.consume(atMost); + } + } + }; +} + +version(cgi_with_websocket) { + // http://tools.ietf.org/html/rfc6455 + + /** + WEBSOCKET SUPPORT: + + Full example: + --- + import arsd.cgi; + + void websocketEcho(Cgi cgi) { + if(cgi.websocketRequested()) { + if(cgi.origin != "http://arsdnet.net") + throw new Exception("bad origin"); + auto websocket = cgi.acceptWebsocket(); + + websocket.send("hello"); + websocket.send(" world!"); + + auto msg = websocket.recv(); + while(msg.opcode != WebSocketOpcode.close) { + if(msg.opcode == WebSocketOpcode.text) { + websocket.send(msg.textData); + } else if(msg.opcode == WebSocketOpcode.binary) { + websocket.send(msg.data); + } + + msg = websocket.recv(); + } + + websocket.close(); + } else assert(0, "i want a web socket!"); + } + + mixin GenericMain!websocketEcho; + --- + */ + + class WebSocket { + Cgi cgi; + + private this(Cgi cgi) { + this.cgi = cgi; + + Socket socket = cgi.idlol.source; + socket.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"minutes"(5)); + } + + // returns true if data available, false if it timed out + bool recvAvailable(Duration timeout = dur!"msecs"(0)) { + if(!waitForNextMessageWouldBlock()) + return true; + if(isDataPending(timeout)) + return true; // this is kinda a lie. + + return false; + } + + public bool lowLevelReceive() { + auto bfr = cgi.idlol; + top: + auto got = bfr.front; + if(got.length) { + if(receiveBuffer.length < receiveBufferUsedLength + got.length) + receiveBuffer.length += receiveBufferUsedLength + got.length; + + receiveBuffer[receiveBufferUsedLength .. receiveBufferUsedLength + got.length] = got[]; + receiveBufferUsedLength += got.length; + bfr.consume(got.length); + + return true; + } + + if(bfr.sourceClosed) + return false; + + bfr.popFront(0); + if(bfr.sourceClosed) + return false; + goto top; + } + + + bool isDataPending(Duration timeout = 0.seconds) { + Socket socket = cgi.idlol.source; + + auto check = new SocketSet(); + check.add(socket); + + auto got = Socket.select(check, null, null, timeout); + if(got > 0) + return true; + return false; + } + + // note: this blocks + WebSocketFrame recv() { + return waitForNextMessage(); + } + + + + + private void llclose() { + cgi.close(); + } + + private void llsend(ubyte[] data) { + cgi.write(data); + cgi.flush(); + } + + void unregisterActiveSocket(WebSocket) {} + + /* copy/paste section { */ + + private int readyState_; + private ubyte[] receiveBuffer; + private size_t receiveBufferUsedLength; + + private Config config; + + enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. + enum OPEN = 1; /// The connection is open and ready to communicate. + enum CLOSING = 2; /// The connection is in the process of closing. + enum CLOSED = 3; /// The connection is closed or couldn't be opened. + + /++ + + +/ + /// Group: foundational + static struct Config { + /++ + These control the size of the receive buffer. + + It starts at the initial size, will temporarily + balloon up to the maximum size, and will reuse + a buffer up to the likely size. + + Anything larger than the maximum size will cause + the connection to be aborted and an exception thrown. + This is to protect you against a peer trying to + exhaust your memory, while keeping the user-level + processing simple. + +/ + size_t initialReceiveBufferSize = 4096; + size_t likelyReceiveBufferSize = 4096; /// ditto + size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto + + /++ + Maximum combined size of a message. + +/ + size_t maximumMessageSize = 10 * 1024 * 1024; + + string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; + string origin; /// Origin URL to send with the handshake, if desired. + string protocol; /// the protocol header, if desired. + + int pingFrequency = 5000; /// Amount of time (in msecs) of idleness after which to send an automatic ping + } + + /++ + Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. + +/ + int readyState() { + return readyState_; + } + + /++ + Closes the connection, sending a graceful teardown message to the other side. + +/ + /// Group: foundational + void close(int code = 0, string reason = null) + //in (reason.length < 123) + in { assert(reason.length < 123); } do + { + if(readyState_ != OPEN) + return; // it cool, we done + WebSocketFrame wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.close; + wss.data = cast(ubyte[]) reason.dup; + wss.send(&llsend); + + readyState_ = CLOSING; + + llclose(); + } + + /++ + Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. + +/ + /// Group: foundational + void ping() { + WebSocketFrame wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.ping; + wss.send(&llsend); + } + + // automatically handled.... + void pong() { + WebSocketFrame wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.pong; + wss.send(&llsend); + } + + /++ + Sends a text message through the websocket. + +/ + /// Group: foundational + void send(in char[] textData) { + WebSocketFrame wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.text; + wss.data = cast(ubyte[]) textData.dup; + wss.send(&llsend); + } + + /++ + Sends a binary message through the websocket. + +/ + /// Group: foundational + void send(in ubyte[] binaryData) { + WebSocketFrame wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.binary; + wss.data = cast(ubyte[]) binaryData.dup; + wss.send(&llsend); + } + + /++ + Waits for and returns the next complete message on the socket. + + Note that the onmessage function is still called, right before + this returns. + +/ + /// Group: blocking_api + public WebSocketFrame waitForNextMessage() { + do { + auto m = processOnce(); + if(m.populated) + return m; + } while(lowLevelReceive()); + + throw new ConnectionClosedException("Websocket receive timed out"); + //return WebSocketFrame.init; // FIXME? maybe. + } + + /++ + Tells if [waitForNextMessage] would block. + +/ + /// Group: blocking_api + public bool waitForNextMessageWouldBlock() { + checkAgain: + if(isMessageBuffered()) + return false; + if(!isDataPending()) + return true; + while(isDataPending()) + lowLevelReceive(); + goto checkAgain; + } + + /++ + Is there a message in the buffer already? + If `true`, [waitForNextMessage] is guaranteed to return immediately. + If `false`, check [isDataPending] as the next step. + +/ + /// Group: blocking_api + public bool isMessageBuffered() { + ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; + auto s = d; + if(d.length) { + auto orig = d; + auto m = WebSocketFrame.read(d); + // that's how it indicates that it needs more data + if(d !is orig) + return true; + } + + return false; + } + + private ubyte continuingType; + private ubyte[] continuingData; + //private size_t continuingDataLength; + + private WebSocketFrame processOnce() { + ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; + auto s = d; + // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. + WebSocketFrame m; + if(d.length) { + auto orig = d; + m = WebSocketFrame.read(d); + // that's how it indicates that it needs more data + if(d is orig) + return WebSocketFrame.init; + m.unmaskInPlace(); + switch(m.opcode) { + case WebSocketOpcode.continuation: + if(continuingData.length + m.data.length > config.maximumMessageSize) + throw new Exception("message size exceeded"); + + continuingData ~= m.data; + if(m.fin) { + if(ontextmessage) + ontextmessage(cast(char[]) continuingData); + if(onbinarymessage) + onbinarymessage(continuingData); + + continuingData = null; + } + break; + case WebSocketOpcode.text: + if(m.fin) { + if(ontextmessage) + ontextmessage(m.textData); + } else { + continuingType = m.opcode; + //continuingDataLength = 0; + continuingData = null; + continuingData ~= m.data; + } + break; + case WebSocketOpcode.binary: + if(m.fin) { + if(onbinarymessage) + onbinarymessage(m.data); + } else { + continuingType = m.opcode; + //continuingDataLength = 0; + continuingData = null; + continuingData ~= m.data; + } + break; + case WebSocketOpcode.close: + readyState_ = CLOSED; + if(onclose) + onclose(); + + unregisterActiveSocket(this); + break; + case WebSocketOpcode.ping: + pong(); + break; + case WebSocketOpcode.pong: + // just really references it is still alive, nbd. + break; + default: // ignore though i could and perhaps should throw too + } + } + + // the recv thing can be invalidated so gotta copy it over ugh + if(d.length) { + m.data = m.data.dup(); + } + + import core.stdc.string; + memmove(receiveBuffer.ptr, d.ptr, d.length); + receiveBufferUsedLength = d.length; + + return m; + } + + private void autoprocess() { + // FIXME + do { + processOnce(); + } while(lowLevelReceive()); + } + + + void delegate() onclose; /// + void delegate() onerror; /// + void delegate(in char[]) ontextmessage; /// + void delegate(in ubyte[]) onbinarymessage; /// + void delegate() onopen; /// + + /++ + + +/ + /// Group: browser_api + void onmessage(void delegate(in char[]) dg) { + ontextmessage = dg; + } + + /// ditto + void onmessage(void delegate(in ubyte[]) dg) { + onbinarymessage = dg; + } + + /* } end copy/paste */ + + + } + + bool websocketRequested(Cgi cgi) { + return + "sec-websocket-key" in cgi.requestHeaders + && + "connection" in cgi.requestHeaders && + cgi.requestHeaders["connection"].asLowerCase().canFind("upgrade") + && + "upgrade" in cgi.requestHeaders && + cgi.requestHeaders["upgrade"].asLowerCase().equal("websocket") + ; + } + + WebSocket acceptWebsocket(Cgi cgi) { + assert(!cgi.closed); + assert(!cgi.outputtedResponseData); + cgi.setResponseStatus("101 Switching Protocols"); + cgi.header("Upgrade: WebSocket"); + cgi.header("Connection: upgrade"); + + string key = cgi.requestHeaders["sec-websocket-key"]; + key ~= "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; // the defined guid from the websocket spec + + import std.digest.sha; + auto hash = sha1Of(key); + auto accept = Base64.encode(hash); + + cgi.header(("Sec-WebSocket-Accept: " ~ accept).idup); + + cgi.websocketMode = true; + cgi.write(""); + + cgi.flush(); + + return new WebSocket(cgi); + } + + // FIXME get websocket to work on other modes, not just embedded_httpd + + /* copy/paste in http2.d { */ + enum WebSocketOpcode : ubyte { + continuation = 0, + text = 1, + binary = 2, + // 3, 4, 5, 6, 7 RESERVED + close = 8, + ping = 9, + pong = 10, + // 11,12,13,14,15 RESERVED + } + + public struct WebSocketFrame { + private bool populated; + bool fin; + bool rsv1; + bool rsv2; + bool rsv3; + WebSocketOpcode opcode; // 4 bits + bool masked; + ubyte lengthIndicator; // don't set this when building one to send + ulong realLength; // don't use when sending + ubyte[4] maskingKey; // don't set this when sending + ubyte[] data; + + static WebSocketFrame simpleMessage(WebSocketOpcode opcode, void[] data) { + WebSocketFrame msg; + msg.fin = true; + msg.opcode = opcode; + msg.data = cast(ubyte[]) data.dup; + + return msg; + } + + private void send(scope void delegate(ubyte[]) llsend) { + ubyte[64] headerScratch; + int headerScratchPos = 0; + + realLength = data.length; + + { + ubyte b1; + b1 |= cast(ubyte) opcode; + b1 |= rsv3 ? (1 << 4) : 0; + b1 |= rsv2 ? (1 << 5) : 0; + b1 |= rsv1 ? (1 << 6) : 0; + b1 |= fin ? (1 << 7) : 0; + + headerScratch[0] = b1; + headerScratchPos++; + } + + { + headerScratchPos++; // we'll set header[1] at the end of this + auto rlc = realLength; + ubyte b2; + b2 |= masked ? (1 << 7) : 0; + + assert(headerScratchPos == 2); + + if(realLength > 65535) { + // use 64 bit length + b2 |= 0x7f; + + // FIXME: double check endinaness + foreach(i; 0 .. 8) { + headerScratch[2 + 7 - i] = rlc & 0x0ff; + rlc >>>= 8; + } + + headerScratchPos += 8; + } else if(realLength > 125) { + // use 16 bit length + b2 |= 0x7e; + + // FIXME: double check endinaness + foreach(i; 0 .. 2) { + headerScratch[2 + 1 - i] = rlc & 0x0ff; + rlc >>>= 8; + } + + headerScratchPos += 2; + } else { + // use 7 bit length + b2 |= realLength & 0b_0111_1111; + } + + headerScratch[1] = b2; + } + + //assert(!masked, "masking key not properly implemented"); + if(masked) { + // FIXME: randomize this + headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; + headerScratchPos += 4; + + // we'll just mask it in place... + int keyIdx = 0; + foreach(i; 0 .. data.length) { + data[i] = data[i] ^ maskingKey[keyIdx]; + if(keyIdx == 3) + keyIdx = 0; + else + keyIdx++; + } + } + + //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); + llsend(headerScratch[0 .. headerScratchPos]); + llsend(data); + } + + static WebSocketFrame read(ref ubyte[] d) { + WebSocketFrame msg; + + auto orig = d; + + WebSocketFrame needsMoreData() { + d = orig; + return WebSocketFrame.init; + } + + if(d.length < 2) + return needsMoreData(); + + ubyte b = d[0]; + + msg.populated = true; + + msg.opcode = cast(WebSocketOpcode) (b & 0x0f); + b >>= 4; + msg.rsv3 = b & 0x01; + b >>= 1; + msg.rsv2 = b & 0x01; + b >>= 1; + msg.rsv1 = b & 0x01; + b >>= 1; + msg.fin = b & 0x01; + + b = d[1]; + msg.masked = (b & 0b1000_0000) ? true : false; + msg.lengthIndicator = b & 0b0111_1111; + + d = d[2 .. $]; + + if(msg.lengthIndicator == 0x7e) { + // 16 bit length + msg.realLength = 0; + + if(d.length < 2) return needsMoreData(); + + foreach(i; 0 .. 2) { + msg.realLength |= d[0] << ((1-i) * 8); + d = d[1 .. $]; + } + } else if(msg.lengthIndicator == 0x7f) { + // 64 bit length + msg.realLength = 0; + + if(d.length < 8) return needsMoreData(); + + foreach(i; 0 .. 8) { + msg.realLength |= ulong(d[0]) << ((7-i) * 8); + d = d[1 .. $]; + } + } else { + // 7 bit length + msg.realLength = msg.lengthIndicator; + } + + if(msg.masked) { + + if(d.length < 4) return needsMoreData(); + + msg.maskingKey = d[0 .. 4]; + d = d[4 .. $]; + } + + if(msg.realLength > d.length) { + return needsMoreData(); + } + + msg.data = d[0 .. cast(size_t) msg.realLength]; + d = d[cast(size_t) msg.realLength .. $]; + + return msg; + } + + void unmaskInPlace() { + if(this.masked) { + int keyIdx = 0; + foreach(i; 0 .. this.data.length) { + this.data[i] = this.data[i] ^ this.maskingKey[keyIdx]; + if(keyIdx == 3) + keyIdx = 0; + else + keyIdx++; + } + } + } + + char[] textData() { + return cast(char[]) data; + } + } + /* } */ +} + + +version(Windows) +{ + version(CRuntime_DigitalMars) + { + extern(C) int setmode(int, int) nothrow @nogc; + } + else version(CRuntime_Microsoft) + { + extern(C) int _setmode(int, int) nothrow @nogc; + alias setmode = _setmode; + } + else static assert(0); +} + +version(Posix) { + import core.sys.posix.unistd; + version(CRuntime_Musl) {} else { + private extern(C) int posix_spawn(pid_t*, const char*, void*, void*, const char**, const char**); + } +} + + +// FIXME: these aren't quite public yet. +//private: + +// template for laziness +void startAddonServer()(string arg) { + version(OSX) { + assert(0, "Not implemented"); + } else version(linux) { + import core.sys.posix.unistd; + pid_t pid; + const(char)*[16] args; + args[0] = "ARSD_CGI_ADDON_SERVER"; + args[1] = arg.ptr; + posix_spawn(&pid, "/proc/self/exe", + null, + null, + args.ptr, + null // env + ); + } else version(Windows) { + wchar[2048] filename; + auto len = GetModuleFileNameW(null, filename.ptr, cast(DWORD) filename.length); + if(len == 0 || len == filename.length) + throw new Exception("could not get process name to start helper server"); + + STARTUPINFOW startupInfo; + startupInfo.cb = cast(DWORD) startupInfo.sizeof; + PROCESS_INFORMATION processInfo; + + import std.utf; + + // I *MIGHT* need to run it as a new job or a service... + auto ret = CreateProcessW( + filename.ptr, + toUTF16z(arg), + null, // process attributes + null, // thread attributes + false, // inherit handles + 0, // creation flags + null, // environment + null, // working directory + &startupInfo, + &processInfo + ); + + if(!ret) + throw new Exception("create process failed"); + + // when done with those, if we set them + /* + CloseHandle(hStdInput); + CloseHandle(hStdOutput); + CloseHandle(hStdError); + */ + + } else static assert(0, "Websocket server not implemented on this system yet (email me, i can prolly do it if you need it)"); +} + +// template for laziness +/* + The websocket server is a single-process, single-thread, event + I/O thing. It is passed websockets from other CGI processes + and is then responsible for handling their messages and responses. + Note that the CGI process is responsible for websocket setup, + including authentication, etc. + + It also gets data sent to it by other processes and is responsible + for distributing that, as necessary. +*/ +void runWebsocketServer()() { + assert(0, "not implemented"); +} + +void sendToWebsocketServer(WebSocket ws, string group) { + assert(0, "not implemented"); +} + +void sendToWebsocketServer(string content, string group) { + assert(0, "not implemented"); +} + + +void runEventServer()() { + runAddonServer("/tmp/arsd_cgi_event_server", new EventSourceServerImplementation()); +} + +void runTimerServer()() { + runAddonServer("/tmp/arsd_scheduled_job_server", new ScheduledJobServerImplementation()); +} + +version(Posix) { + alias LocalServerConnectionHandle = int; + alias CgiConnectionHandle = int; + alias SocketConnectionHandle = int; + + enum INVALID_CGI_CONNECTION_HANDLE = -1; +} else version(Windows) { + alias LocalServerConnectionHandle = HANDLE; + version(embedded_httpd_threads) { + alias CgiConnectionHandle = SOCKET; + enum INVALID_CGI_CONNECTION_HANDLE = INVALID_SOCKET; + } else version(fastcgi) { + alias CgiConnectionHandle = void*; // Doesn't actually work! But I don't want compile to fail pointlessly at this point. + enum INVALID_CGI_CONNECTION_HANDLE = null; + } else version(scgi) { + alias CgiConnectionHandle = SOCKET; + enum INVALID_CGI_CONNECTION_HANDLE = INVALID_SOCKET; + } else { /* version(plain_cgi) */ + alias CgiConnectionHandle = HANDLE; + enum INVALID_CGI_CONNECTION_HANDLE = null; + } + alias SocketConnectionHandle = SOCKET; +} + +version(with_addon_servers_connections) +LocalServerConnectionHandle openLocalServerConnection()(string name, string arg) { + version(Posix) { + import core.sys.posix.unistd; + import core.sys.posix.sys.un; + + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + if(sock == -1) + throw new Exception("socket " ~ to!string(errno)); + + scope(failure) + close(sock); + + cloexec(sock); + + // add-on server processes are assumed to be local, and thus will + // use unix domain sockets. Besides, I want to pass sockets to them, + // so it basically must be local (except for the session server, but meh). + sockaddr_un addr; + addr.sun_family = AF_UNIX; + version(linux) { + // on linux, we will use the abstract namespace + addr.sun_path[0] = 0; + addr.sun_path[1 .. name.length + 1] = cast(typeof(addr.sun_path[])) name[]; + } else { + // but otherwise, just use a file cuz we must. + addr.sun_path[0 .. name.length] = cast(typeof(addr.sun_path[])) name[]; + } + + bool alreadyTried; + + try_again: + + if(connect(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) { + if(!alreadyTried && errno == ECONNREFUSED) { + // try auto-spawning the server, then attempt connection again + startAddonServer(arg); + import core.thread; + Thread.sleep(50.msecs); + alreadyTried = true; + goto try_again; + } else + throw new Exception("connect " ~ to!string(errno)); + } + + return sock; + } else version(Windows) { + return null; // FIXME + } +} + +version(with_addon_servers_connections) +void closeLocalServerConnection(LocalServerConnectionHandle handle) { + version(Posix) { + import core.sys.posix.unistd; + close(handle); + } else version(Windows) + CloseHandle(handle); +} + +void runSessionServer()() { + runAddonServer("/tmp/arsd_session_server", new BasicDataServerImplementation()); +} + +version(Posix) +private void makeNonBlocking(int fd) { + import core.sys.posix.fcntl; + auto flags = fcntl(fd, F_GETFL, 0); + if(flags == -1) + throw new Exception("fcntl get"); + flags |= O_NONBLOCK; + auto s = fcntl(fd, F_SETFL, flags); + if(s == -1) + throw new Exception("fcntl set"); +} + +import core.stdc.errno; + +struct IoOp { + @disable this(); + @disable this(this); + + /* + So we want to be able to eventually handle generic sockets too. + */ + + enum Read = 1; + enum Write = 2; + enum Accept = 3; + enum ReadSocketHandle = 4; + + // Your handler may be called in a different thread than the one that initiated the IO request! + // It is also possible to have multiple io requests being called simultaneously. Use proper thread safety caution. + private bool delegate(IoOp*, int) handler; // returns true if you are done and want it to be closed + private void delegate(IoOp*) closeHandler; + private void delegate(IoOp*) completeHandler; + private int internalFd; + private int operation; + private int bufferLengthAllocated; + private int bufferLengthUsed; + private ubyte[1] internalBuffer; // it can be overallocated! + + ubyte[] allocatedBuffer() return { + return internalBuffer.ptr[0 .. bufferLengthAllocated]; + } + + ubyte[] usedBuffer() return { + return allocatedBuffer[0 .. bufferLengthUsed]; + } + + void reset() { + bufferLengthUsed = 0; + } + + int fd() { + return internalFd; + } +} + +IoOp* allocateIoOp(int fd, int operation, int bufferSize, bool delegate(IoOp*, int) handler) { + import core.stdc.stdlib; + + auto ptr = calloc(IoOp.sizeof + bufferSize, 1); + if(ptr is null) + assert(0); // out of memory! + + auto op = cast(IoOp*) ptr; + + op.handler = handler; + op.internalFd = fd; + op.operation = operation; + op.bufferLengthAllocated = bufferSize; + op.bufferLengthUsed = 0; + + import core.memory; + + GC.addRoot(ptr); + + return op; +} + +void freeIoOp(ref IoOp* ptr) { + + import core.memory; + GC.removeRoot(ptr); + + import core.stdc.stdlib; + free(ptr); + ptr = null; +} + +version(Posix) +version(with_addon_servers_connections) +void nonBlockingWrite(EventIoServer eis, int connection, const void[] data) { + + //import std.stdio : writeln; writeln(cast(string) data); + + import core.sys.posix.unistd; + + auto ret = write(connection, data.ptr, data.length); + if(ret != data.length) { + if(ret == 0 || (ret == -1 && (errno == EPIPE || errno == ETIMEDOUT))) { + // the file is closed, remove it + eis.fileClosed(connection); + } else + throw new Exception("alas " ~ to!string(ret) ~ " " ~ to!string(errno)); // FIXME + } +} +version(Windows) +version(with_addon_servers_connections) +void nonBlockingWrite(EventIoServer eis, int connection, const void[] data) { + // FIXME +} + +bool isInvalidHandle(CgiConnectionHandle h) { + return h == INVALID_CGI_CONNECTION_HANDLE; +} + +/+ +https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsarecv +https://support.microsoft.com/en-gb/help/181611/socket-overlapped-i-o-versus-blocking-nonblocking-mode +https://stackoverflow.com/questions/18018489/should-i-use-iocps-or-overlapped-wsasend-receive +https://docs.microsoft.com/en-us/windows/desktop/fileio/i-o-completion-ports +https://docs.microsoft.com/en-us/windows/desktop/fileio/createiocompletionport +https://docs.microsoft.com/en-us/windows/desktop/api/mswsock/nf-mswsock-acceptex +https://docs.microsoft.com/en-us/windows/desktop/Sync/waitable-timer-objects +https://docs.microsoft.com/en-us/windows/desktop/api/synchapi/nf-synchapi-setwaitabletimer +https://docs.microsoft.com/en-us/windows/desktop/Sync/using-a-waitable-timer-with-an-asynchronous-procedure-call +https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsagetoverlappedresult + ++/ + +/++ + You can customize your server by subclassing the appropriate server. Then, register your + subclass at compile time with the [registerEventIoServer] template, or implement your own + main function and call it yourself. + + $(TIP If you make your subclass a `final class`, there is a slight performance improvement.) ++/ +version(with_addon_servers_connections) +interface EventIoServer { + bool handleLocalConnectionData(IoOp* op, int receivedFd); + void handleLocalConnectionClose(IoOp* op); + void handleLocalConnectionComplete(IoOp* op); + void wait_timeout(); + void fileClosed(int fd); + + void epoll_fd(int fd); +} + +// the sink should buffer it +private void serialize(T)(scope void delegate(scope ubyte[]) sink, T t) { + static if(is(T == struct)) { + foreach(member; __traits(allMembers, T)) + serialize(sink, __traits(getMember, t, member)); + } else static if(is(T : int)) { + // no need to think of endianness just because this is only used + // for local, same-machine stuff anyway. thanks private lol + sink((cast(ubyte*) &t)[0 .. t.sizeof]); + } else static if(is(T == string) || is(T : const(ubyte)[])) { + // these are common enough to optimize + int len = cast(int) t.length; // want length consistent size tho, in case 32 bit program sends to 64 bit server, etc. + sink((cast(ubyte*) &len)[0 .. int.sizeof]); + sink(cast(ubyte[]) t[]); + } else static if(is(T : A[], A)) { + // generic array is less optimal but still prolly ok + int len = cast(int) t.length; + sink((cast(ubyte*) &len)[0 .. int.sizeof]); + foreach(item; t) + serialize(sink, item); + } else static assert(0, T.stringof); +} + +// all may be stack buffers, so use cautio +private void deserialize(T)(scope ubyte[] delegate(int sz) get, scope void delegate(T) dg) { + static if(is(T == struct)) { + T t; + foreach(member; __traits(allMembers, T)) + deserialize!(typeof(__traits(getMember, T, member)))(get, (mbr) { __traits(getMember, t, member) = mbr; }); + dg(t); + } else static if(is(T : int)) { + // no need to think of endianness just because this is only used + // for local, same-machine stuff anyway. thanks private lol + T t; + auto data = get(t.sizeof); + t = (cast(T[]) data)[0]; + dg(t); + } else static if(is(T == string) || is(T : const(ubyte)[])) { + // these are common enough to optimize + int len; + auto data = get(len.sizeof); + len = (cast(int[]) data)[0]; + + /* + typeof(T[0])[2000] stackBuffer; + T buffer; + + if(len < stackBuffer.length) + buffer = stackBuffer[0 .. len]; + else + buffer = new T(len); + + data = get(len * typeof(T[0]).sizeof); + */ + + T t = cast(T) get(len * cast(int) typeof(T.init[0]).sizeof); + + dg(t); + } else static if(is(T == E[], E)) { + T t; + int len; + auto data = get(len.sizeof); + len = (cast(int[]) data)[0]; + t.length = len; + foreach(ref e; t) { + deserialize!E(get, (ele) { e = ele; }); + } + dg(t); + } else static assert(0, T.stringof); +} + +unittest { + serialize((ubyte[] b) { + deserialize!int( sz => b[0 .. sz], (t) { assert(t == 1); }); + }, 1); + serialize((ubyte[] b) { + deserialize!int( sz => b[0 .. sz], (t) { assert(t == 56674); }); + }, 56674); + ubyte[1000] buffer; + int bufferPoint; + void add(ubyte[] b) { + buffer[bufferPoint .. bufferPoint + b.length] = b[]; + bufferPoint += b.length; + } + ubyte[] get(int sz) { + auto b = buffer[bufferPoint .. bufferPoint + sz]; + bufferPoint += sz; + return b; + } + serialize(&add, "test here"); + bufferPoint = 0; + deserialize!string(&get, (t) { assert(t == "test here"); }); + bufferPoint = 0; + + struct Foo { + int a; + ubyte c; + string d; + } + serialize(&add, Foo(403, 37, "amazing")); + bufferPoint = 0; + deserialize!Foo(&get, (t) { + assert(t.a == 403); + assert(t.c == 37); + assert(t.d == "amazing"); + }); + bufferPoint = 0; +} + +/* + Here's the way the RPC interface works: + + You define the interface that lists the functions you can call on the remote process. + The interface may also have static methods for convenience. These forward to a singleton + instance of an auto-generated class, which actually sends the args over the pipe. + + An impl class actually implements it. A receiving server deserializes down the pipe and + calls methods on the class. + + I went with the interface to get some nice compiler checking and documentation stuff. + + I could have skipped the interface and just implemented it all from the server class definition + itself, but then the usage may call the method instead of rpcing it; I just like having the user + interface and the implementation separate so you aren't tempted to `new impl` to call the methods. + + + I fiddled with newlines in the mixin string to ensure the assert line numbers matched up to the source code line number. Idk why dmd didn't do this automatically, but it was important to me. + + Realistically though the bodies would just be + connection.call(this.mangleof, args...) sooooo. + + FIXME: overloads aren't supported +*/ + +/// Base for storing sessions in an array. Exists primarily for internal purposes and you should generally not use this. +interface SessionObject {} + +private immutable void delegate(string[])[string] scheduledJobHandlers; +private immutable void delegate(string[])[string] websocketServers; + +version(with_breaking_cgi_features) +mixin(q{ + +mixin template ImplementRpcClientInterface(T, string serverPath, string cmdArg) { + static import std.traits; + + // derivedMembers on an interface seems to give exactly what I want: the virtual functions we need to implement. so I am just going to use it directly without more filtering. + static foreach(idx, member; __traits(derivedMembers, T)) { + static if(__traits(isVirtualMethod, __traits(getMember, T, member))) + mixin( q{ + std.traits.ReturnType!(__traits(getMember, T, member)) + } ~ member ~ q{(std.traits.Parameters!(__traits(getMember, T, member)) params) + { + SerializationBuffer buffer; + auto i = cast(ushort) idx; + serialize(&buffer.sink, i); + serialize(&buffer.sink, __traits(getMember, T, member).mangleof); + foreach(param; params) + serialize(&buffer.sink, param); + + auto sendable = buffer.sendable; + + version(Posix) {{ + auto ret = send(connectionHandle, sendable.ptr, sendable.length, 0); + + if(ret == -1) { + throw new Exception("send returned -1, errno: " ~ to!string(errno)); + } else if(ret == 0) { + throw new Exception("Connection to addon server lost"); + } if(ret < sendable.length) + throw new Exception("Send failed to send all"); + assert(ret == sendable.length); + }} // FIXME Windows impl + + static if(!is(typeof(return) == void)) { + // there is a return value; we need to wait for it too + version(Posix) { + ubyte[3000] revBuffer; + auto ret = recv(connectionHandle, revBuffer.ptr, revBuffer.length, 0); + auto got = revBuffer[0 .. ret]; + + int dataLocation; + ubyte[] grab(int sz) { + auto dataLocation1 = dataLocation; + dataLocation += sz; + return got[dataLocation1 .. dataLocation]; + } + + typeof(return) retu; + deserialize!(typeof(return))(&grab, (a) { retu = a; }); + return retu; + } else { + // FIXME Windows impl + return typeof(return).init; + } + + } + }}); + } + + private static typeof(this) singletonInstance; + private LocalServerConnectionHandle connectionHandle; + + static typeof(this) connection() { + if(singletonInstance is null) { + singletonInstance = new typeof(this)(); + singletonInstance.connect(); + } + return singletonInstance; + } + + void connect() { + connectionHandle = openLocalServerConnection(serverPath, cmdArg); + } + + void disconnect() { + closeLocalServerConnection(connectionHandle); + } +} + +void dispatchRpcServer(Interface, Class)(Class this_, ubyte[] data, int fd) if(is(Class : Interface)) { + ushort calledIdx; + string calledFunction; + + int dataLocation; + ubyte[] grab(int sz) { + if(sz == 0) assert(0); + auto d = data[dataLocation .. dataLocation + sz]; + dataLocation += sz; + return d; + } + + again: + + deserialize!ushort(&grab, (a) { calledIdx = a; }); + deserialize!string(&grab, (a) { calledFunction = a; }); + + import std.traits; + + sw: switch(calledIdx) { + foreach(idx, memberName; __traits(derivedMembers, Interface)) + static if(__traits(isVirtualMethod, __traits(getMember, Interface, memberName))) { + case idx: + assert(calledFunction == __traits(getMember, Interface, memberName).mangleof); + + Parameters!(__traits(getMember, Interface, memberName)) params; + foreach(ref param; params) + deserialize!(typeof(param))(&grab, (a) { param = a; }); + + static if(is(ReturnType!(__traits(getMember, Interface, memberName)) == void)) { + __traits(getMember, this_, memberName)(params); + } else { + auto ret = __traits(getMember, this_, memberName)(params); + SerializationBuffer buffer; + serialize(&buffer.sink, ret); + + auto sendable = buffer.sendable; + + version(Posix) { + auto r = send(fd, sendable.ptr, sendable.length, 0); + if(r == -1) { + throw new Exception("send returned -1, errno: " ~ to!string(errno)); + } else if(r == 0) { + throw new Exception("Connection to addon client lost"); + } if(r < sendable.length) + throw new Exception("Send failed to send all"); + + } // FIXME Windows impl + } + break sw; + } + default: assert(0); + } + + if(dataLocation != data.length) + goto again; +} + + +private struct SerializationBuffer { + ubyte[2048] bufferBacking; + int bufferLocation; + void sink(scope ubyte[] data) { + bufferBacking[bufferLocation .. bufferLocation + data.length] = data[]; + bufferLocation += data.length; + } + + ubyte[] sendable() return { + return bufferBacking[0 .. bufferLocation]; + } +} + +/* + FIXME: + add a version command line arg + version data in the library + management gui as external program + + at server with event_fd for each run + use .mangleof in the at function name + + i think the at server will have to: + pipe args to the child + collect child output for logging + get child return value for logging + + on windows timers work differently. idk how to best combine with the io stuff. + + will have to have dump and restore too, so i can restart without losing stuff. +*/ + +/++ + A convenience object for talking to the [BasicDataServer] from a higher level. + See: [Cgi.getSessionObject]. + + You pass it a `Data` struct describing the data you want saved in the session. + Then, this class will generate getter and setter properties that allow access + to that data. + + Note that each load and store will be done as-accessed; it doesn't front-load + mutable data nor does it batch updates out of fear of read-modify-write race + conditions. (In fact, right now it does this for everything, but in the future, + I might batch load `immutable` members of the Data struct.) + + At some point in the future, I might also let it do different backends, like + a client-side cookie store too, but idk. + + Note that the plain-old-data members of your `Data` struct are wrapped by this + interface via a static foreach to make property functions. + + See_Also: [MockSession] ++/ +interface Session(Data) : SessionObject { + @property string sessionId() const; + + /++ + Starts a new session. Note that a session is also + implicitly started as soon as you write data to it, + so if you need to alter these parameters from their + defaults, be sure to explicitly call this BEFORE doing + any writes to session data. + + Params: + idleLifetime = How long, in seconds, the session + should remain in memory when not being read from + or written to. The default is one day. + + NOT IMPLEMENTED + + useExtendedLifetimeCookie = The session ID is always + stored in a HTTP cookie, and by default, that cookie + is discarded when the user closes their browser. + + But if you set this to true, it will use a non-perishable + cookie for the given idleLifetime. + + NOT IMPLEMENTED + +/ + void start(int idleLifetime = 2600 * 24, bool useExtendedLifetimeCookie = false); + + /++ + Regenerates the session ID and updates the associated + cookie. + + This is also your chance to change immutable data + (not yet implemented). + +/ + void regenerateId(); + + /++ + Terminates this session, deleting all saved data. + +/ + void terminate(); + + /++ + Plain-old-data members of your `Data` struct are wrapped here via + the property getters and setters. + + If the member is a non-string array, it returns a magical array proxy + object which allows for atomic appends and replaces via overloaded operators. + You can slice this to get a range representing a $(B const) view of the array. + This is to protect you against read-modify-write race conditions. + +/ + static foreach(memberName; __traits(allMembers, Data)) + static if(is(typeof(__traits(getMember, Data, memberName)))) + mixin(q{ + @property inout(typeof(__traits(getMember, Data, memberName))) } ~ memberName ~ q{ () inout; + @property typeof(__traits(getMember, Data, memberName)) } ~ memberName ~ q{ (typeof(__traits(getMember, Data, memberName)) value); + }); + +} + +/++ + An implementation of [Session] that works on real cgi connections utilizing the + [BasicDataServer]. + + As opposed to a [MockSession] which is made for testing purposes. + + You will not construct one of these directly. See [Cgi.getSessionObject] instead. ++/ +class BasicDataServerSession(Data) : Session!Data { + private Cgi cgi; + private string sessionId_; + + public @property string sessionId() const { + return sessionId_; + } + + protected @property string sessionId(string s) { + return this.sessionId_ = s; + } + + private this(Cgi cgi) { + this.cgi = cgi; + if(auto ptr = "sessionId" in cgi.cookies) + sessionId = (*ptr).length ? *ptr : null; + } + + void start(int idleLifetime = 2600 * 24, bool useExtendedLifetimeCookie = false) { + assert(sessionId is null); + + // FIXME: what if there is a session ID cookie, but no corresponding session on the server? + + import std.random, std.conv; + sessionId = to!string(uniform(1, long.max)); + + BasicDataServer.connection.createSession(sessionId, idleLifetime); + setCookie(); + } + + protected void setCookie() { + cgi.setCookie( + "sessionId", sessionId, + 0 /* expiration */, + "/" /* path */, + null /* domain */, + true /* http only */, + cgi.https /* if the session is started on https, keep it there, otherwise, be flexible */); + } + + void regenerateId() { + if(sessionId is null) { + start(); + return; + } + import std.random, std.conv; + auto oldSessionId = sessionId; + sessionId = to!string(uniform(1, long.max)); + BasicDataServer.connection.renameSession(oldSessionId, sessionId); + setCookie(); + } + + void terminate() { + BasicDataServer.connection.destroySession(sessionId); + sessionId = null; + setCookie(); + } + + static foreach(memberName; __traits(allMembers, Data)) + static if(is(typeof(__traits(getMember, Data, memberName)))) + mixin(q{ + @property inout(typeof(__traits(getMember, Data, memberName))) } ~ memberName ~ q{ () inout { + if(sessionId is null) + return typeof(return).init; + + import std.traits; + auto v = BasicDataServer.connection.getSessionData(sessionId, fullyQualifiedName!Data ~ "." ~ memberName); + if(v.length == 0) + return typeof(return).init; + import std.conv; + // why this cast? to doesn't like being given an inout argument. so need to do it without that, then + // we need to return it and that needed the cast. It should be fine since we basically respect constness.. + // basically. Assuming the session is POD this should be fine. + return cast(typeof(return)) to!(typeof(__traits(getMember, Data, memberName)))(v); + } + @property typeof(__traits(getMember, Data, memberName)) } ~ memberName ~ q{ (typeof(__traits(getMember, Data, memberName)) value) { + if(sessionId is null) + start(); + import std.conv; + import std.traits; + BasicDataServer.connection.setSessionData(sessionId, fullyQualifiedName!Data ~ "." ~ memberName, to!string(value)); + return value; + } + }); +} + +/++ + A mock object that works like the real session, but doesn't actually interact with any actual database or http connection. + Simply stores the data in its instance members. ++/ +class MockSession(Data) : Session!Data { + pure { + @property string sessionId() const { return "mock"; } + void start(int idleLifetime = 2600 * 24, bool useExtendedLifetimeCookie = false) {} + void regenerateId() {} + void terminate() {} + + private Data store_; + + static foreach(memberName; __traits(allMembers, Data)) + static if(is(typeof(__traits(getMember, Data, memberName)))) + mixin(q{ + @property inout(typeof(__traits(getMember, Data, memberName))) } ~ memberName ~ q{ () inout { + return __traits(getMember, store_, memberName); + } + @property typeof(__traits(getMember, Data, memberName)) } ~ memberName ~ q{ (typeof(__traits(getMember, Data, memberName)) value) { + return __traits(getMember, store_, memberName) = value; + } + }); + } +} + +/++ + Direct interface to the basic data add-on server. You can + typically use [Cgi.getSessionObject] as a more convenient interface. ++/ +version(with_addon_servers_connections) +interface BasicDataServer { + /// + void createSession(string sessionId, int lifetime); + /// + void renewSession(string sessionId, int lifetime); + /// + void destroySession(string sessionId); + /// + void renameSession(string oldSessionId, string newSessionId); + + /// + void setSessionData(string sessionId, string dataKey, string dataValue); + /// + string getSessionData(string sessionId, string dataKey); + + /// + static BasicDataServerConnection connection() { + return BasicDataServerConnection.connection(); + } +} + +version(with_addon_servers_connections) +class BasicDataServerConnection : BasicDataServer { + mixin ImplementRpcClientInterface!(BasicDataServer, "/tmp/arsd_session_server", "--session-server"); +} + +version(with_addon_servers) +final class BasicDataServerImplementation : BasicDataServer, EventIoServer { + + void createSession(string sessionId, int lifetime) { + sessions[sessionId.idup] = Session(lifetime); + } + void destroySession(string sessionId) { + sessions.remove(sessionId); + } + void renewSession(string sessionId, int lifetime) { + sessions[sessionId].lifetime = lifetime; + } + void renameSession(string oldSessionId, string newSessionId) { + sessions[newSessionId.idup] = sessions[oldSessionId]; + sessions.remove(oldSessionId); + } + void setSessionData(string sessionId, string dataKey, string dataValue) { + if(sessionId !in sessions) + createSession(sessionId, 3600); // FIXME? + sessions[sessionId].values[dataKey.idup] = dataValue.idup; + } + string getSessionData(string sessionId, string dataKey) { + if(auto session = sessionId in sessions) { + if(auto data = dataKey in (*session).values) + return *data; + else + return null; // no such data + + } else { + return null; // no session + } + } + + + protected: + + struct Session { + int lifetime; + + string[string] values; + } + + Session[string] sessions; + + bool handleLocalConnectionData(IoOp* op, int receivedFd) { + auto data = op.usedBuffer; + dispatchRpcServer!BasicDataServer(this, data, op.fd); + return false; + } + + void handleLocalConnectionClose(IoOp* op) {} // doesn't really matter, this is a fairly stateless go + void handleLocalConnectionComplete(IoOp* op) {} // again, irrelevant + void wait_timeout() {} + void fileClosed(int fd) {} // stateless so irrelevant + void epoll_fd(int fd) {} +} + +/++ + See [schedule] to make one of these. You then call one of the methods here to set it up: + + --- + schedule!fn(args).at(DateTime(2019, 8, 7, 12, 00, 00)); // run the function at August 7, 2019, 12 noon UTC + schedule!fn(args).delay(6.seconds); // run it after waiting 6 seconds + schedule!fn(args).asap(); // run it in the background as soon as the event loop gets around to it + --- ++/ +version(with_addon_servers_connections) +struct ScheduledJobHelper { + private string func; + private string[] args; + private bool consumed; + + private this(string func, string[] args) { + this.func = func; + this.args = args; + } + + ~this() { + assert(consumed); + } + + /++ + Schedules the job to be run at the given time. + +/ + void at(DateTime when, immutable TimeZone timezone = UTC()) { + consumed = true; + + auto conn = ScheduledJobServerConnection.connection; + import std.file; + auto st = SysTime(when, timezone); + auto jobId = conn.scheduleJob(1, cast(int) st.toUnixTime(), thisExePath, func, args); + } + + /++ + Schedules the job to run at least after the specified delay. + +/ + void delay(Duration delay) { + consumed = true; + + auto conn = ScheduledJobServerConnection.connection; + import std.file; + auto jobId = conn.scheduleJob(0, cast(int) delay.total!"seconds", thisExePath, func, args); + } + + /++ + Runs the job in the background ASAP. + + $(NOTE It may run in a background thread. Don't segfault!) + +/ + void asap() { + consumed = true; + + auto conn = ScheduledJobServerConnection.connection; + import std.file; + auto jobId = conn.scheduleJob(0, 1, thisExePath, func, args); + } + + /+ + /++ + Schedules the job to recur on the given pattern. + +/ + void recur(string spec) { + + } + +/ +} + +/++ + First step to schedule a job on the scheduled job server. + + The scheduled job needs to be a top-level function that doesn't read any + variables from outside its arguments because it may be run in a new process, + without any context existing later. + + You MUST set details on the returned object to actually do anything! ++/ +template schedule(alias fn, T...) if(is(typeof(fn) == function)) { + /// + ScheduledJobHelper schedule(T args) { + // this isn't meant to ever be called, but instead just to + // get the compiler to type check the arguments passed for us + auto sample = delegate() { + fn(args); + }; + string[] sargs; + foreach(arg; args) + sargs ~= to!string(arg); + return ScheduledJobHelper(fn.mangleof, sargs); + } + + shared static this() { + scheduledJobHandlers[fn.mangleof] = delegate(string[] sargs) { + import std.traits; + Parameters!fn args; + foreach(idx, ref arg; args) + arg = to!(typeof(arg))(sargs[idx]); + fn(args); + }; + } +} + +/// +interface ScheduledJobServer { + /// Use the [schedule] function for a higher-level interface. + int scheduleJob(int whenIs, int when, string executable, string func, string[] args); + /// + void cancelJob(int jobId); +} + +version(with_addon_servers_connections) +class ScheduledJobServerConnection : ScheduledJobServer { + mixin ImplementRpcClientInterface!(ScheduledJobServer, "/tmp/arsd_scheduled_job_server", "--timer-server"); +} + +version(with_addon_servers) +final class ScheduledJobServerImplementation : ScheduledJobServer, EventIoServer { + // FIXME: we need to handle SIGCHLD in this somehow + // whenIs is 0 for relative, 1 for absolute + protected int scheduleJob(int whenIs, int when, string executable, string func, string[] args) { + auto nj = nextJobId; + nextJobId++; + + version(linux) { + import core.sys.linux.timerfd; + import core.sys.linux.epoll; + import core.sys.posix.unistd; + + + auto fd = timerfd_create(CLOCK_REALTIME, TFD_NONBLOCK | TFD_CLOEXEC); + if(fd == -1) + throw new Exception("fd timer create failed"); + + foreach(ref arg; args) + arg = arg.idup; + auto job = Job(executable.idup, func.idup, .dup(args), fd, nj); + + itimerspec value; + value.it_value.tv_sec = when; + value.it_value.tv_nsec = 0; + + value.it_interval.tv_sec = 0; + value.it_interval.tv_nsec = 0; + + if(timerfd_settime(fd, whenIs == 1 ? TFD_TIMER_ABSTIME : 0, &value, null) == -1) + throw new Exception("couldn't set fd timer"); + + auto op = allocateIoOp(fd, IoOp.Read, 16, (IoOp* op, int fd) { + jobs.remove(nj); + epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, null); + close(fd); + + + spawnProcess([job.executable, "--timed-job", job.func] ~ job.args); + + return true; + }); + scope(failure) + freeIoOp(op); + + epoll_event ev; + ev.events = EPOLLIN | EPOLLET; + ev.data.ptr = op; + if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, fd, &ev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + + jobs[nj] = job; + return nj; + } else assert(0); + } + + protected void cancelJob(int jobId) { + version(linux) { + auto job = jobId in jobs; + if(job is null) + return; + + jobs.remove(jobId); + + version(linux) { + import core.sys.linux.timerfd; + import core.sys.linux.epoll; + import core.sys.posix.unistd; + epoll_ctl(epoll_fd, EPOLL_CTL_DEL, job.timerfd, null); + close(job.timerfd); + } + } + jobs.remove(jobId); + } + + int nextJobId = 1; + static struct Job { + string executable; + string func; + string[] args; + int timerfd; + int id; + } + Job[int] jobs; + + + // event io server methods below + + bool handleLocalConnectionData(IoOp* op, int receivedFd) { + auto data = op.usedBuffer; + dispatchRpcServer!ScheduledJobServer(this, data, op.fd); + return false; + } + + void handleLocalConnectionClose(IoOp* op) {} // doesn't really matter, this is a fairly stateless go + void handleLocalConnectionComplete(IoOp* op) {} // again, irrelevant + void wait_timeout() {} + void fileClosed(int fd) {} // stateless so irrelevant + + int epoll_fd_; + void epoll_fd(int fd) {this.epoll_fd_ = fd; } + int epoll_fd() { return epoll_fd_; } +} + +/// +version(with_addon_servers_connections) +interface EventSourceServer { + /++ + sends this cgi request to the event server so it will be fed events. You should not do anything else with the cgi object after this. + + $(WARNING This API is extremely unstable. I might change it or remove it without notice.) + + See_Also: + [sendEvent] + +/ + public static void adoptConnection(Cgi cgi, in char[] eventUrl) { + /* + If lastEventId is missing or empty, you just get new events as they come. + + If it is set from something else, it sends all since then (that are still alive) + down the pipe immediately. + + The reason it can come from the header is that's what the standard defines for + browser reconnects. The reason it can come from a query string is just convenience + in catching up in a user-defined manner. + + The reason the header overrides the query string is if the browser tries to reconnect, + it will send the header AND the query (it reconnects to the same url), so we just + want to do the restart thing. + + Note that if you ask for "0" as the lastEventId, it will get ALL still living events. + */ + string lastEventId = cgi.lastEventId; + if(lastEventId.length == 0 && "lastEventId" in cgi.get) + lastEventId = cgi.get["lastEventId"]; + + cgi.setResponseContentType("text/event-stream"); + cgi.write(":\n", false); // to initialize the chunking and send headers before keeping the fd for later + cgi.flush(); + + cgi.closed = true; + auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server", "--event-server"); + scope(exit) + closeLocalServerConnection(s); + + version(fastcgi) + throw new Exception("sending fcgi connections not supported"); + else { + auto fd = cgi.getOutputFileHandle(); + if(isInvalidHandle(fd)) + throw new Exception("bad fd from cgi!"); + + EventSourceServerImplementation.SendableEventConnection sec; + sec.populate(cgi.responseChunked, eventUrl, lastEventId); + + version(Posix) { + auto res = write_fd(s, cast(void*) &sec, sec.sizeof, fd); + assert(res == sec.sizeof); + } else version(Windows) { + // FIXME + } + } + } + + /++ + Sends an event to the event server, starting it if necessary. The event server will distribute it to any listening clients, and store it for `lifetime` seconds for any later listening clients to catch up later. + + $(WARNING This API is extremely unstable. I might change it or remove it without notice.) + + Params: + url = A string identifying this event "bucket". Listening clients must also connect to this same string. I called it `url` because I envision it being just passed as the url of the request. + event = the event type string, which is used in the Javascript addEventListener API on EventSource + data = the event data. Available in JS as `event.data`. + lifetime = the amount of time to keep this event for replaying on the event server. + + See_Also: + [sendEventToEventServer] + +/ + public static void sendEvent(string url, string event, string data, int lifetime) { + auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server", "--event-server"); + scope(exit) + closeLocalServerConnection(s); + + EventSourceServerImplementation.SendableEvent sev; + sev.populate(url, event, data, lifetime); + + version(Posix) { + auto ret = send(s, &sev, sev.sizeof, 0); + assert(ret == sev.sizeof); + } else version(Windows) { + // FIXME + } + } + + /++ + Messages sent to `url` will also be sent to anyone listening on `forwardUrl`. + + See_Also: [disconnect] + +/ + void connect(string url, string forwardUrl); + + /++ + Disconnects `forwardUrl` from `url` + + See_Also: [connect] + +/ + void disconnect(string url, string forwardUrl); +} + +/// +version(with_addon_servers) +final class EventSourceServerImplementation : EventSourceServer, EventIoServer { + + protected: + + void connect(string url, string forwardUrl) { + pipes[url] ~= forwardUrl; + } + void disconnect(string url, string forwardUrl) { + auto t = url in pipes; + if(t is null) + return; + foreach(idx, n; (*t)) + if(n == forwardUrl) { + (*t)[idx] = (*t)[$-1]; + (*t) = (*t)[0 .. $-1]; + break; + } + } + + bool handleLocalConnectionData(IoOp* op, int receivedFd) { + if(receivedFd != -1) { + //writeln("GOT FD ", receivedFd, " -- ", op.usedBuffer); + + //core.sys.posix.unistd.write(receivedFd, "hello".ptr, 5); + + SendableEventConnection* got = cast(SendableEventConnection*) op.usedBuffer.ptr; + + auto url = got.url.idup; + eventConnectionsByUrl[url] ~= EventConnection(receivedFd, got.responseChunked > 0 ? true : false); + + // FIXME: catch up on past messages here + } else { + auto data = op.usedBuffer; + auto event = cast(SendableEvent*) data.ptr; + + if(event.magic == 0xdeadbeef) { + handleInputEvent(event); + + if(event.url in pipes) + foreach(pipe; pipes[event.url]) { + event.url = pipe; + handleInputEvent(event); + } + } else { + dispatchRpcServer!EventSourceServer(this, data, op.fd); + } + } + return false; + } + void handleLocalConnectionClose(IoOp* op) { + fileClosed(op.fd); + } + void handleLocalConnectionComplete(IoOp* op) {} + + void wait_timeout() { + // just keeping alive + foreach(url, connections; eventConnectionsByUrl) + foreach(connection; connections) + if(connection.needsChunking) + nonBlockingWrite(this, connection.fd, "1b\r\nevent: keepalive\ndata: ok\n\n\r\n"); + else + nonBlockingWrite(this, connection.fd, "event: keepalive\ndata: ok\n\n\r\n"); + } + + void fileClosed(int fd) { + outer: foreach(url, ref connections; eventConnectionsByUrl) { + foreach(idx, conn; connections) { + if(fd == conn.fd) { + connections[idx] = connections[$-1]; + connections = connections[0 .. $ - 1]; + continue outer; + } + } + } + } + + void epoll_fd(int fd) {} + + + private: + + + struct SendableEventConnection { + ubyte responseChunked; + + int urlLength; + char[256] urlBuffer = 0; + + int lastEventIdLength; + char[32] lastEventIdBuffer = 0; + + char[] url() return { + return urlBuffer[0 .. urlLength]; + } + void url(in char[] u) { + urlBuffer[0 .. u.length] = u[]; + urlLength = cast(int) u.length; + } + char[] lastEventId() return { + return lastEventIdBuffer[0 .. lastEventIdLength]; + } + void populate(bool responseChunked, in char[] url, in char[] lastEventId) + in { + assert(url.length < this.urlBuffer.length); + assert(lastEventId.length < this.lastEventIdBuffer.length); + } + do { + this.responseChunked = responseChunked ? 1 : 0; + this.urlLength = cast(int) url.length; + this.lastEventIdLength = cast(int) lastEventId.length; + + this.urlBuffer[0 .. url.length] = url[]; + this.lastEventIdBuffer[0 .. lastEventId.length] = lastEventId[]; + } + } + + struct SendableEvent { + int magic = 0xdeadbeef; + int urlLength; + char[256] urlBuffer = 0; + int typeLength; + char[32] typeBuffer = 0; + int messageLength; + char[2048 * 4] messageBuffer = 0; // this is an arbitrary limit, it needs to fit comfortably in stack (including in a fiber) and be a single send on the kernel side cuz of the impl... i think this is ok for a unix socket. + int _lifetime; + + char[] message() return { + return messageBuffer[0 .. messageLength]; + } + char[] type() return { + return typeBuffer[0 .. typeLength]; + } + char[] url() return { + return urlBuffer[0 .. urlLength]; + } + void url(in char[] u) { + urlBuffer[0 .. u.length] = u[]; + urlLength = cast(int) u.length; + } + int lifetime() { + return _lifetime; + } + + /// + void populate(string url, string type, string message, int lifetime) + in { + assert(url.length < this.urlBuffer.length); + assert(type.length < this.typeBuffer.length); + assert(message.length < this.messageBuffer.length); + } + do { + this.urlLength = cast(int) url.length; + this.typeLength = cast(int) type.length; + this.messageLength = cast(int) message.length; + this._lifetime = lifetime; + + this.urlBuffer[0 .. url.length] = url[]; + this.typeBuffer[0 .. type.length] = type[]; + this.messageBuffer[0 .. message.length] = message[]; + } + } + + struct EventConnection { + int fd; + bool needsChunking; + } + + private EventConnection[][string] eventConnectionsByUrl; + private string[][string] pipes; + + private void handleInputEvent(scope SendableEvent* event) { + static int eventId; + + static struct StoredEvent { + int id; + string type; + string message; + int lifetimeRemaining; + } + + StoredEvent[][string] byUrl; + + int thisId = ++eventId; + + if(event.lifetime) + byUrl[event.url.idup] ~= StoredEvent(thisId, event.type.idup, event.message.idup, event.lifetime); + + auto connectionsPtr = event.url in eventConnectionsByUrl; + EventConnection[] connections; + if(connectionsPtr is null) + return; + else + connections = *connectionsPtr; + + char[4096] buffer; + char[] formattedMessage; + + void append(const char[] a) { + // the 6's here are to leave room for a HTTP chunk header, if it proves necessary + buffer[6 + formattedMessage.length .. 6 + formattedMessage.length + a.length] = a[]; + formattedMessage = buffer[6 .. 6 + formattedMessage.length + a.length]; + } + + import std.algorithm.iteration; + + if(connections.length) { + append("id: "); + append(to!string(thisId)); + append("\n"); + + append("event: "); + append(event.type); + append("\n"); + + foreach(line; event.message.splitter("\n")) { + append("data: "); + append(line); + append("\n"); + } + + append("\n"); + } + + // chunk it for HTTP! + auto len = toHex(formattedMessage.length); + buffer[4 .. 6] = "\r\n"[]; + buffer[4 - len.length .. 4] = len[]; + buffer[6 + formattedMessage.length] = '\r'; + buffer[6 + formattedMessage.length + 1] = '\n'; + + auto chunkedMessage = buffer[4 - len.length .. 6 + formattedMessage.length +2]; + // done + + // FIXME: send back requests when needed + // FIXME: send a single ":\n" every 15 seconds to keep alive + + foreach(connection; connections) { + if(connection.needsChunking) { + nonBlockingWrite(this, connection.fd, chunkedMessage); + } else { + nonBlockingWrite(this, connection.fd, formattedMessage); + } + } + } +} + +void runAddonServer(EIS)(string localListenerName, EIS eis) if(is(EIS : EventIoServer)) { + version(Posix) { + + import core.sys.posix.unistd; + import core.sys.posix.fcntl; + import core.sys.posix.sys.un; + + import core.sys.posix.signal; + signal(SIGPIPE, SIG_IGN); + + static extern(C) void sigchldhandler(int) { + int status; + import w = core.sys.posix.sys.wait; + w.wait(&status); + } + signal(SIGCHLD, &sigchldhandler); + + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + if(sock == -1) + throw new Exception("socket " ~ to!string(errno)); + + scope(failure) + close(sock); + + cloexec(sock); + + // add-on server processes are assumed to be local, and thus will + // use unix domain sockets. Besides, I want to pass sockets to them, + // so it basically must be local (except for the session server, but meh). + sockaddr_un addr; + addr.sun_family = AF_UNIX; + version(linux) { + // on linux, we will use the abstract namespace + addr.sun_path[0] = 0; + addr.sun_path[1 .. localListenerName.length + 1] = cast(typeof(addr.sun_path[])) localListenerName[]; + } else { + // but otherwise, just use a file cuz we must. + addr.sun_path[0 .. localListenerName.length] = cast(typeof(addr.sun_path[])) localListenerName[]; + } + + if(bind(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) + throw new Exception("bind " ~ to!string(errno)); + + if(listen(sock, 128) == -1) + throw new Exception("listen " ~ to!string(errno)); + + makeNonBlocking(sock); + + version(linux) { + import core.sys.linux.epoll; + auto epoll_fd = epoll_create1(EPOLL_CLOEXEC); + if(epoll_fd == -1) + throw new Exception("epoll_create1 " ~ to!string(errno)); + scope(failure) + close(epoll_fd); + } else { + import core.sys.posix.poll; + } + + version(linux) + eis.epoll_fd = epoll_fd; + + auto acceptOp = allocateIoOp(sock, IoOp.Read, 0, null); + scope(exit) + freeIoOp(acceptOp); + + version(linux) { + epoll_event ev; + ev.events = EPOLLIN | EPOLLET; + ev.data.ptr = acceptOp; + if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sock, &ev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + + epoll_event[64] events; + } else { + pollfd[] pollfds; + IoOp*[int] ioops; + pollfds ~= pollfd(sock, POLLIN); + ioops[sock] = acceptOp; + } + + import core.time : MonoTime, seconds; + + MonoTime timeout = MonoTime.currTime + 15.seconds; + + while(true) { + + // FIXME: it should actually do a timerfd that runs on any thing that hasn't been run recently + + int timeout_milliseconds = 0; // -1; // infinite + + timeout_milliseconds = cast(int) (timeout - MonoTime.currTime).total!"msecs"; + if(timeout_milliseconds < 0) + timeout_milliseconds = 0; + + //writeln("waiting for ", name); + + version(linux) { + auto nfds = epoll_wait(epoll_fd, events.ptr, events.length, timeout_milliseconds); + if(nfds == -1) { + if(errno == EINTR) + continue; + throw new Exception("epoll_wait " ~ to!string(errno)); + } + } else { + int nfds = poll(pollfds.ptr, cast(int) pollfds.length, timeout_milliseconds); + size_t lastIdx = 0; + } + + if(nfds == 0) { + eis.wait_timeout(); + timeout += 15.seconds; + } + + foreach(idx; 0 .. nfds) { + version(linux) { + auto flags = events[idx].events; + auto ioop = cast(IoOp*) events[idx].data.ptr; + } else { + IoOp* ioop; + foreach(tidx, thing; pollfds[lastIdx .. $]) { + if(thing.revents) { + ioop = ioops[thing.fd]; + lastIdx += tidx + 1; + break; + } + } + } + + //writeln(flags, " ", ioop.fd); + + void newConnection() { + // on edge triggering, it is important that we get it all + while(true) { + version(Android) { + auto size = cast(int) addr.sizeof; + } else { + auto size = cast(uint) addr.sizeof; + } + auto ns = accept(sock, cast(sockaddr*) &addr, &size); + if(ns == -1) { + if(errno == EAGAIN || errno == EWOULDBLOCK) { + // all done, got it all + break; + } + throw new Exception("accept " ~ to!string(errno)); + } + cloexec(ns); + + makeNonBlocking(ns); + auto niop = allocateIoOp(ns, IoOp.ReadSocketHandle, 4096 * 4, &eis.handleLocalConnectionData); + niop.closeHandler = &eis.handleLocalConnectionClose; + niop.completeHandler = &eis.handleLocalConnectionComplete; + scope(failure) freeIoOp(niop); + + version(linux) { + epoll_event nev; + nev.events = EPOLLIN | EPOLLET; + nev.data.ptr = niop; + if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, ns, &nev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + } else { + bool found = false; + foreach(ref pfd; pollfds) { + if(pfd.fd < 0) { + pfd.fd = ns; + found = true; + } + } + if(!found) + pollfds ~= pollfd(ns, POLLIN); + ioops[ns] = niop; + } + } + } + + bool newConnectionCondition() { + version(linux) + return ioop.fd == sock && (flags & EPOLLIN); + else + return pollfds[idx].fd == sock && (pollfds[idx].revents & POLLIN); + } + + if(newConnectionCondition()) { + newConnection(); + } else if(ioop.operation == IoOp.ReadSocketHandle) { + while(true) { + int in_fd; + auto got = read_fd(ioop.fd, ioop.allocatedBuffer.ptr, ioop.allocatedBuffer.length, &in_fd); + if(got == -1) { + if(errno == EAGAIN || errno == EWOULDBLOCK) { + // all done, got it all + if(ioop.completeHandler) + ioop.completeHandler(ioop); + break; + } + throw new Exception("recv " ~ to!string(errno)); + } + + if(got == 0) { + if(ioop.closeHandler) { + ioop.closeHandler(ioop); + version(linux) {} // nothing needed + else { + foreach(ref pfd; pollfds) { + if(pfd.fd == ioop.fd) + pfd.fd = -1; + } + } + } + close(ioop.fd); + freeIoOp(ioop); + break; + } + + ioop.bufferLengthUsed = cast(int) got; + ioop.handler(ioop, in_fd); + } + } else if(ioop.operation == IoOp.Read) { + while(true) { + auto got = read(ioop.fd, ioop.allocatedBuffer.ptr, ioop.allocatedBuffer.length); + if(got == -1) { + if(errno == EAGAIN || errno == EWOULDBLOCK) { + // all done, got it all + if(ioop.completeHandler) + ioop.completeHandler(ioop); + break; + } + throw new Exception("recv " ~ to!string(ioop.fd) ~ " errno " ~ to!string(errno)); + } + + if(got == 0) { + if(ioop.closeHandler) + ioop.closeHandler(ioop); + close(ioop.fd); + freeIoOp(ioop); + break; + } + + ioop.bufferLengthUsed = cast(int) got; + if(ioop.handler(ioop, ioop.fd)) { + close(ioop.fd); + freeIoOp(ioop); + break; + } + } + } + + // EPOLLHUP? + } + } + } else version(Windows) { + + // set up a named pipe + // https://msdn.microsoft.com/en-us/library/windows/desktop/ms724251(v=vs.85).aspx + // https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsaduplicatesocketw + // https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-getnamedpipeserverprocessid + + } else static assert(0); +} + + +version(with_sendfd) +// copied from the web and ported from C +// see https://stackoverflow.com/questions/2358684/can-i-share-a-file-descriptor-to-another-process-on-linux-or-are-they-local-to-t +ssize_t write_fd(int fd, void *ptr, size_t nbytes, int sendfd) { + msghdr msg; + iovec[1] iov; + + version(OSX) { + //msg.msg_accrights = cast(cattr_t) &sendfd; + //msg.msg_accrightslen = int.sizeof; + } else version(Android) { + } else { + union ControlUnion { + cmsghdr cm; + char[CMSG_SPACE(int.sizeof)] control; + } + + ControlUnion control_un; + cmsghdr* cmptr; + + msg.msg_control = control_un.control.ptr; + msg.msg_controllen = control_un.control.length; + + cmptr = CMSG_FIRSTHDR(&msg); + cmptr.cmsg_len = CMSG_LEN(int.sizeof); + cmptr.cmsg_level = SOL_SOCKET; + cmptr.cmsg_type = SCM_RIGHTS; + *(cast(int *) CMSG_DATA(cmptr)) = sendfd; + } + + msg.msg_name = null; + msg.msg_namelen = 0; + + iov[0].iov_base = ptr; + iov[0].iov_len = nbytes; + msg.msg_iov = iov.ptr; + msg.msg_iovlen = 1; + + return sendmsg(fd, &msg, 0); +} + +version(with_sendfd) +// copied from the web and ported from C +ssize_t read_fd(int fd, void *ptr, size_t nbytes, int *recvfd) { + msghdr msg; + iovec[1] iov; + ssize_t n; + int newfd; + + version(OSX) { + //msg.msg_accrights = cast(cattr_t) recvfd; + //msg.msg_accrightslen = int.sizeof; + } else version(Android) { + } else { + union ControlUnion { + cmsghdr cm; + char[CMSG_SPACE(int.sizeof)] control; + } + ControlUnion control_un; + cmsghdr* cmptr; + + msg.msg_control = control_un.control.ptr; + msg.msg_controllen = control_un.control.length; + } + + msg.msg_name = null; + msg.msg_namelen = 0; + + iov[0].iov_base = ptr; + iov[0].iov_len = nbytes; + msg.msg_iov = iov.ptr; + msg.msg_iovlen = 1; + + if ( (n = recvmsg(fd, &msg, 0)) <= 0) + return n; + + version(OSX) { + //if(msg.msg_accrightslen != int.sizeof) + //*recvfd = -1; + } else version(Android) { + } else { + if ( (cmptr = CMSG_FIRSTHDR(&msg)) != null && + cmptr.cmsg_len == CMSG_LEN(int.sizeof)) { + if (cmptr.cmsg_level != SOL_SOCKET) + throw new Exception("control level != SOL_SOCKET"); + if (cmptr.cmsg_type != SCM_RIGHTS) + throw new Exception("control type != SCM_RIGHTS"); + *recvfd = *(cast(int *) CMSG_DATA(cmptr)); + } else + *recvfd = -1; /* descriptor was not passed */ + } + + return n; +} +/* end read_fd */ + + +/* + Event source stuff + + The api is: + + sendEvent(string url, string type, string data, int timeout = 60*10); + + attachEventListener(string url, int fd, lastId) + + + It just sends to all attached listeners, and stores it until the timeout + for replaying via lastEventId. +*/ + +/* + Session process stuff + + it stores it all. the cgi object has a session object that can grab it + + session may be done in the same process if possible, there is a version + switch to choose if you want to override. +*/ + +struct DispatcherDefinition(alias dispatchHandler, DispatcherDetails = typeof(null)) {// if(is(typeof(dispatchHandler("str", Cgi.init, void) == bool))) { // bool delegate(string urlPrefix, Cgi cgi) dispatchHandler; + alias handler = dispatchHandler; + string urlPrefix; + bool rejectFurther; + immutable(DispatcherDetails) details; +} + +private string urlify(string name) pure { + return beautify(name, '-', true); +} + +private string beautify(string name, char space = ' ', bool allLowerCase = false) pure { + if(name == "id") + return allLowerCase ? name : "ID"; + + char[160] buffer; + int bufferIndex = 0; + bool shouldCap = true; + bool shouldSpace; + bool lastWasCap; + foreach(idx, char ch; name) { + if(bufferIndex == buffer.length) return name; // out of space, just give up, not that important + + if((ch >= 'A' && ch <= 'Z') || ch == '_') { + if(lastWasCap) { + // two caps in a row, don't change. Prolly acronym. + } else { + if(idx) + shouldSpace = true; // new word, add space + } + + lastWasCap = true; + } else { + lastWasCap = false; + } + + if(shouldSpace) { + buffer[bufferIndex++] = space; + if(bufferIndex == buffer.length) return name; // out of space, just give up, not that important + shouldSpace = false; + } + if(shouldCap) { + if(ch >= 'a' && ch <= 'z') + ch -= 32; + shouldCap = false; + } + if(allLowerCase && ch >= 'A' && ch <= 'Z') + ch += 32; + buffer[bufferIndex++] = ch; + } + return buffer[0 .. bufferIndex].idup; +} + +/* +string urlFor(alias func)() { + return __traits(identifier, func); +} +*/ + +/++ + UDA: The name displayed to the user in auto-generated HTML. + + Default is `beautify(identifier)`. ++/ +struct DisplayName { + string name; +} + +/++ + UDA: The name used in the URL or web parameter. + + Default is `urlify(identifier)` for functions and `identifier` for parameters and data members. ++/ +struct UrlName { + string name; +} + +/++ + UDA: default format to respond for this method ++/ +struct DefaultFormat { string value; } + +class MissingArgumentException : Exception { + string functionName; + string argumentName; + string argumentType; + + this(string functionName, string argumentName, string argumentType, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + this.functionName = functionName; + this.argumentName = argumentName; + this.argumentType = argumentType; + + super("Missing Argument: " ~ this.argumentName, file, line, next); + } +} + +/++ + You can throw this from an api handler to indicate a 404 response. This is done by the presentExceptionAsHtml function in the presenter. + + History: + Added December 15, 2021 (dub v10.5) ++/ +class ResourceNotFoundException : Exception { + string resourceType; + string resourceId; + + this(string resourceType, string resourceId, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + this.resourceType = resourceType; + this.resourceId = resourceId; + + super("Resource not found: " ~ resourceType ~ " " ~ resourceId, file, line, next); + } + +} + +/++ + This can be attached to any constructor or function called from the cgi system. + + If it is present, the function argument can NOT be set from web params, but instead + is set to the return value of the given `func`. + + If `func` can take a parameter of type [Cgi], it will be passed the one representing + the current request. Otherwise, it must take zero arguments. + + Any params in your function of type `Cgi` are automatically assumed to take the cgi object + for the connection. Any of type [Session] (with an argument) is also assumed to come from + the cgi object. + + const arguments are also supported. ++/ +struct ifCalledFromWeb(alias func) {} + +// it only looks at query params for GET requests, the rest must be in the body for a function argument. +auto callFromCgi(alias method, T)(T dg, Cgi cgi) { + + // FIXME: any array of structs should also be settable or gettable from csv as well. + + // FIXME: think more about checkboxes and bools. + + import std.traits; + + Parameters!method params; + alias idents = ParameterIdentifierTuple!method; + alias defaults = ParameterDefaults!method; + + const(string)[] names; + const(string)[] values; + + // first, check for missing arguments and initialize to defaults if necessary + + static if(is(typeof(method) P == __parameters)) + foreach(idx, param; P) {{ + // see: mustNotBeSetFromWebParams + static if(is(param : Cgi)) { + static assert(!is(param == immutable)); + cast() params[idx] = cgi; + } else static if(is(param == Session!D, D)) { + static assert(!is(param == immutable)); + cast() params[idx] = cgi.getSessionObject!D(); + } else { + bool populated; + foreach(uda; __traits(getAttributes, P[idx .. idx + 1])) { + static if(is(uda == ifCalledFromWeb!func, alias func)) { + static if(is(typeof(func(cgi)))) + params[idx] = func(cgi); + else + params[idx] = func(); + + populated = true; + } + } + + if(!populated) { + static if(__traits(compiles, { params[idx] = param.getAutomaticallyForCgi(cgi); } )) { + params[idx] = param.getAutomaticallyForCgi(cgi); + populated = true; + } + } + + if(!populated) { + auto ident = idents[idx]; + if(cgi.requestMethod == Cgi.RequestMethod.GET) { + if(ident !in cgi.get) { + static if(is(defaults[idx] == void)) { + static if(is(param == bool)) + params[idx] = false; + else + throw new MissingArgumentException(__traits(identifier, method), ident, param.stringof); + } else + params[idx] = defaults[idx]; + } + } else { + if(ident !in cgi.post) { + static if(is(defaults[idx] == void)) { + static if(is(param == bool)) + params[idx] = false; + else + throw new MissingArgumentException(__traits(identifier, method), ident, param.stringof); + } else + params[idx] = defaults[idx]; + } + } + } + } + }} + + // second, parse the arguments in order to build up arrays, etc. + + static bool setVariable(T)(string name, string paramName, T* what, string value) { + static if(is(T == struct)) { + if(name == paramName) { + *what = T.init; + return true; + } else { + // could be a child. gonna allow either obj.field OR obj[field] + + string afterName; + + if(name[paramName.length] == '[') { + int count = 1; + auto idx = paramName.length + 1; + while(idx < name.length && count > 0) { + if(name[idx] == '[') + count++; + else if(name[idx] == ']') { + count--; + if(count == 0) break; + } + idx++; + } + + if(idx == name.length) + return false; // malformed + + auto insideBrackets = name[paramName.length + 1 .. idx]; + afterName = name[idx + 1 .. $]; + + name = name[0 .. paramName.length]; + + paramName = insideBrackets; + + } else if(name[paramName.length] == '.') { + paramName = name[paramName.length + 1 .. $]; + name = paramName; + int p = 0; + foreach(ch; paramName) { + if(ch == '.' || ch == '[') + break; + p++; + } + + afterName = paramName[p .. $]; + paramName = paramName[0 .. p]; + } else { + return false; + } + + if(paramName.length) + // set the child member + switch(paramName) { + foreach(idx, memberName; __traits(allMembers, T)) + static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { + // data member! + case memberName: + return setVariable(name ~ afterName, paramName, &(__traits(getMember, *what, memberName)), value); + } + default: + // ok, not a member + } + } + + return false; + } else static if(is(T == enum)) { + *what = to!T(value); + return true; + } else static if(isSomeString!T || isIntegral!T || isFloatingPoint!T) { + *what = to!T(value); + return true; + } else static if(is(T == bool)) { + *what = value == "1" || value == "yes" || value == "t" || value == "true" || value == "on"; + return true; + } else static if(is(T == K[], K)) { + K tmp; + if(name == paramName) { + // direct - set and append + if(setVariable(name, paramName, &tmp, value)) { + (*what) ~= tmp; + return true; + } else { + return false; + } + } else { + // child, append to last element + // FIXME: what about range violations??? + auto ptr = &(*what)[(*what).length - 1]; + return setVariable(name, paramName, ptr, value); + + } + } else static if(is(T == V[K], K, V)) { + // assoc array, name[key] is valid + if(name == paramName) { + // no action necessary + return true; + } else if(name[paramName.length] == '[') { + int count = 1; + auto idx = paramName.length + 1; + while(idx < name.length && count > 0) { + if(name[idx] == '[') + count++; + else if(name[idx] == ']') { + count--; + if(count == 0) break; + } + idx++; + } + if(idx == name.length) + return false; // malformed + + auto insideBrackets = name[paramName.length + 1 .. idx]; + auto afterName = name[idx + 1 .. $]; + + auto k = to!K(insideBrackets); + V v; + if(auto ptr = k in *what) + v = *ptr; + + name = name[0 .. paramName.length]; + //writeln(name, afterName, " ", paramName); + + auto ret = setVariable(name ~ afterName, paramName, &v, value); + if(ret) { + (*what)[k] = v; + return true; + } + } + + return false; + } else { + static assert(0, "unsupported type for cgi call " ~ T.stringof); + } + + //return false; + } + + void setArgument(string name, string value) { + int p; + foreach(ch; name) { + if(ch == '.' || ch == '[') + break; + p++; + } + + auto paramName = name[0 .. p]; + + sw: switch(paramName) { + static if(is(typeof(method) P == __parameters)) + foreach(idx, param; P) { + static if(mustNotBeSetFromWebParams!(P[idx], __traits(getAttributes, P[idx .. idx + 1]))) { + // cannot be set from the outside + } else { + case idents[idx]: + static if(is(param == Cgi.UploadedFile)) { + params[idx] = cgi.files[name]; + } else { + setVariable(name, paramName, ¶ms[idx], value); + } + break sw; + } + } + default: + // ignore; not relevant argument + } + } + + if(cgi.requestMethod == Cgi.RequestMethod.GET) { + names = cgi.allGetNamesInOrder; + values = cgi.allGetValuesInOrder; + } else { + names = cgi.allPostNamesInOrder; + values = cgi.allPostValuesInOrder; + } + + foreach(idx, name; names) { + setArgument(name, values[idx]); + } + + static if(is(ReturnType!method == void)) { + typeof(null) ret; + dg(params); + } else { + auto ret = dg(params); + } + + // FIXME: format return values + // options are: json, html, csv. + // also may need to wrap in envelope format: none, html, or json. + return ret; +} + +private bool mustNotBeSetFromWebParams(T, attrs...)() { + static if(is(T : const(Cgi))) { + return true; + } else static if(is(T : const(Session!D), D)) { + return true; + } else static if(__traits(compiles, T.getAutomaticallyForCgi(Cgi.init))) { + return true; + } else { + foreach(uda; attrs) + static if(is(uda == ifCalledFromWeb!func, alias func)) + return true; + return false; + } +} + +private bool hasIfCalledFromWeb(attrs...)() { + foreach(uda; attrs) + static if(is(uda == ifCalledFromWeb!func, alias func)) + return true; + return false; +} + +/++ + Implies POST path for the thing itself, then GET will get the automatic form. + + The given customizer, if present, will be called as a filter on the Form object. + + History: + Added December 27, 2020 ++/ +template AutomaticForm(alias customizer) { } + +/++ + This is meant to be returned by a function that takes a form POST submission. You + want to set the url of the new resource it created, which is set as the http + Location header for a "201 Created" result, and you can also set a separate + destination for browser users, which it sets via a "Refresh" header. + + The `resourceRepresentation` should generally be the thing you just created, and + it will be the body of the http response when formatted through the presenter. + The exact thing is up to you - it could just return an id, or the whole object, or + perhaps a partial object. + + Examples: + --- + class Test : WebObject { + @(Cgi.RequestMethod.POST) + CreatedResource!int makeThing(string value) { + return CreatedResource!int(value.to!int, "/resources/id"); + } + } + --- + + History: + Added December 18, 2021 ++/ +struct CreatedResource(T) { + static if(!is(T == void)) + T resourceRepresentation; + string resourceUrl; + string refreshUrl; +} + +/+ +/++ + This can be attached as a UDA to a handler to add a http Refresh header on a + successful run. (It will not be attached if the function throws an exception.) + This will refresh the browser the given number of seconds after the page loads, + to the url returned by `urlFunc`, which can be either a static function or a + member method of the current handler object. + + You might use this for a POST handler that is normally used from ajax, but you + want it to degrade gracefully to a temporarily flashed message before reloading + the main page. + + History: + Added December 18, 2021 ++/ +struct Refresh(alias urlFunc) { + int waitInSeconds; + + string url() { + static if(__traits(isStaticFunction, urlFunc)) + return urlFunc(); + else static if(is(urlFunc : string)) + return urlFunc; + } +} ++/ + +/+ +/++ + Sets a filter to be run before + + A before function can do validations of params and log and stop the function from running. ++/ +template Before(alias b) {} +template After(alias b) {} ++/ + +/+ + Argument conversions: for the most part, it is to!Thing(string). + + But arrays and structs are a bit different. Arrays come from the cgi array. Thus + they are passed + + arr=foo&arr=bar <-- notice the same name. + + Structs are first declared with an empty thing, then have their members set individually, + with dot notation. The members are not required, just the initial declaration. + + struct Foo { + int a; + string b; + } + void test(Foo foo){} + + foo&foo.a=5&foo.b=str <-- the first foo declares the arg, the others set the members + + Arrays of structs use this declaration. + + void test(Foo[] foo) {} + + foo&foo.a=5&foo.b=bar&foo&foo.a=9 + + You can use a hidden input field in HTML forms to achieve this. The value of the naked name + declaration is ignored. + + Mind that order matters! The declaration MUST come first in the string. + + Arrays of struct members follow this rule recursively. + + struct Foo { + int[] a; + } + + foo&foo.a=1&foo.a=2&foo&foo.a=1 + + + Associative arrays are formatted with brackets, after a declaration, like structs: + + foo&foo[key]=value&foo[other_key]=value + + + Note: for maximum compatibility with outside code, keep your types simple. Some libraries + do not support the strict ordering requirements to work with these struct protocols. + + FIXME: also perhaps accept application/json to better work with outside trash. + + + Return values are also auto-formatted according to user-requested type: + for json, it loops over and converts. + for html, basic types are strings. Arrays are
    . Structs are
    . Arrays of structs are tables! ++/ + +/++ + A web presenter is responsible for rendering things to HTML to be usable + in a web browser. + + They are passed as template arguments to the base classes of [WebObject] + + Responsible for displaying stuff as HTML. You can put this into your own aggregate + and override it. Use forwarding and specialization to customize it. + + When you inherit from it, pass your own class as the CRTP argument. This lets the base + class templates and your overridden templates work with each other. + + --- + class MyPresenter : WebPresenter!(MyPresenter) { + @Override + void presentSuccessfulReturnAsHtml(T : CustomType)(Cgi cgi, T ret, typeof(null) meta) { + // present the CustomType + } + @Override + void presentSuccessfulReturnAsHtml(T)(Cgi cgi, T ret, typeof(null) meta) { + // handle everything else via the super class, which will call + // back to your class when appropriate + super.presentSuccessfulReturnAsHtml(cgi, ret); + } + } + --- + + The meta argument in there can be overridden by your own facility. + ++/ +class WebPresenter(CRTP) { + + /// A UDA version of the built-in `override`, to be used for static template polymorphism + /// If you override a plain method, use `override`. If a template, use `@Override`. + enum Override; + + string script() { + return ` + `; + } + + string style() { + return ` + :root { + --mild-border: #ccc; + --middle-border: #999; + --accent-color: #f2f2f2; + --sidebar-color: #fefefe; + } + ` ~ genericFormStyling() ~ genericSiteStyling(); + } + + string genericFormStyling() { + return +q"css + table.automatic-data-display { + border-collapse: collapse; + border: solid 1px var(--mild-border); + } + + table.automatic-data-display td { + vertical-align: top; + border: solid 1px var(--mild-border); + padding: 2px 4px; + } + + table.automatic-data-display th { + border: solid 1px var(--mild-border); + border-bottom: solid 1px var(--middle-border); + padding: 2px 4px; + } + + ol.automatic-data-display { + margin: 0px; + list-style-position: inside; + padding: 0px; + } + + dl.automatic-data-display { + + } + + .automatic-form { + max-width: 600px; + } + + .form-field { + margin: 0.5em; + padding-left: 0.5em; + } + + .label-text { + display: block; + font-weight: bold; + margin-left: -0.5em; + } + + .submit-button-holder { + padding-left: 2em; + } + + .add-array-button { + + } +css"; + } + + string genericSiteStyling() { + return +q"css + * { box-sizing: border-box; } + html, body { margin: 0px; } + body { + font-family: sans-serif; + } + header { + background: var(--accent-color); + height: 64px; + } + footer { + background: var(--accent-color); + height: 64px; + } + #site-container { + display: flex; + } + main { + flex: 1 1 auto; + order: 2; + min-height: calc(100vh - 64px - 64px); + padding: 4px; + padding-left: 1em; + } + #sidebar { + flex: 0 0 16em; + order: 1; + background: var(--sidebar-color); + } +css"; + } + + import arsd.dom; + Element htmlContainer() { + auto document = new Document(q"html + + + + + + D Application + + + +
    +
    +
    + +
    +
    + + + +html", true, true); + + return document.requireSelector("main"); + } + + /// Renders a response as an HTTP error + void renderBasicError(Cgi cgi, int httpErrorCode) { + cgi.setResponseStatus(getHttpCodeText(httpErrorCode)); + auto c = htmlContainer(); + c.innerText = getHttpCodeText(httpErrorCode); + cgi.setResponseContentType("text/html; charset=utf-8"); + cgi.write(c.parentDocument.toString(), true); + } + + template methodMeta(alias method) { + enum methodMeta = null; + } + + void presentSuccessfulReturn(T, Meta)(Cgi cgi, T ret, Meta meta, string format) { + switch(format) { + case "html": + (cast(CRTP) this).presentSuccessfulReturnAsHtml(cgi, ret, meta); + break; + case "json": + import arsd.jsvar; + static if(is(typeof(ret) == MultipleResponses!Types, Types...)) { + var json; + foreach(index, type; Types) { + if(ret.contains == index) + json = ret.payload[index]; + } + } else { + var json = ret; + } + var envelope = json; // var.emptyObject; + /* + envelope.success = true; + envelope.result = json; + envelope.error = null; + */ + cgi.setResponseContentType("application/json"); + cgi.write(envelope.toJson(), true); + break; + default: + cgi.setResponseStatus("406 Not Acceptable"); // not exactly but sort of. + } + } + + /// typeof(null) (which is also used to represent functions returning `void`) do nothing + /// in the default presenter - allowing the function to have full low-level control over the + /// response. + void presentSuccessfulReturn(T : typeof(null), Meta)(Cgi cgi, T ret, Meta meta, string format) { + // nothing intentionally! + } + + /// Redirections are forwarded to [Cgi.setResponseLocation] + void presentSuccessfulReturn(T : Redirection, Meta)(Cgi cgi, T ret, Meta meta, string format) { + cgi.setResponseLocation(ret.to, true, getHttpCodeText(ret.code)); + } + + /// [CreatedResource]s send code 201 and will set the given urls, then present the given representation. + void presentSuccessfulReturn(T : CreatedResource!R, Meta, R)(Cgi cgi, T ret, Meta meta, string format) { + cgi.setResponseStatus(getHttpCodeText(201)); + if(ret.resourceUrl.length) + cgi.header("Location: " ~ ret.resourceUrl); + if(ret.refreshUrl.length) + cgi.header("Refresh: 0;" ~ ret.refreshUrl); + static if(!is(R == void)) + presentSuccessfulReturn(cgi, ret.resourceRepresentation, meta, format); + } + + /// Multiple responses deconstruct the algebraic type and forward to the appropriate handler at runtime + void presentSuccessfulReturn(T : MultipleResponses!Types, Meta, Types...)(Cgi cgi, T ret, Meta meta, string format) { + bool outputted = false; + foreach(index, type; Types) { + if(ret.contains == index) { + assert(!outputted); + outputted = true; + (cast(CRTP) this).presentSuccessfulReturn(cgi, ret.payload[index], meta, format); + } + } + if(!outputted) + assert(0); + } + + /++ + An instance of the [arsd.dom.FileResource] interface has its own content type; assume it is a download of some sort if the filename member is non-null of the FileResource interface. + +/ + void presentSuccessfulReturn(T : FileResource, Meta)(Cgi cgi, T ret, Meta meta, string format) { + cgi.setCache(true); // not necessarily true but meh + if(auto fn = ret.filename()) { + cgi.header("Content-Disposition: attachment; filename="~fn~";"); + } + cgi.setResponseContentType(ret.contentType); + cgi.write(ret.getData(), true); + } + + /// And the default handler for HTML will call [formatReturnValueAsHtml] and place it inside the [htmlContainer]. + void presentSuccessfulReturnAsHtml(T)(Cgi cgi, T ret, typeof(null) meta) { + auto container = this.htmlContainer(); + container.appendChild(formatReturnValueAsHtml(ret)); + cgi.write(container.parentDocument.toString(), true); + } + + /++ + + History: + Added January 23, 2023 (dub v11.0) + +/ + void presentExceptionalReturn(Meta)(Cgi cgi, Throwable t, Meta meta, string format) { + switch(format) { + case "html": + presentExceptionAsHtml(cgi, t, meta); + break; + default: + } + } + + + /++ + If you override this, you will need to cast the exception type `t` dynamically, + but can then use the template arguments here to refer back to the function. + + `func` is an alias to the method itself, and `dg` is a callable delegate to the same + method on the live object. You could, in theory, change arguments and retry, but I + provide that information mostly with the expectation that you will use them to make + useful forms or richer error messages for the user. + + History: + BREAKING CHANGE on January 23, 2023 (v11.0 ): it previously took an `alias func` and `T dg` to call the function again. + I removed this in favor of a `Meta` param. + + Before: `void presentExceptionAsHtml(alias func, T)(Cgi cgi, Throwable t, T dg)` + + After: `void presentExceptionAsHtml(Meta)(Cgi cgi, Throwable t, Meta meta)` + + If you used the func for something, move that something into your `methodMeta` template. + + What is the benefit of this change? Somewhat smaller executables and faster builds thanks to more reused functions, together with + enabling an easier implementation of [presentExceptionalReturn]. + +/ + void presentExceptionAsHtml(Meta)(Cgi cgi, Throwable t, Meta meta) { + Form af; + /+ + foreach(attr; __traits(getAttributes, func)) { + static if(__traits(isSame, attr, AutomaticForm)) { + af = createAutomaticFormForFunction!(func)(dg); + } + } + +/ + presentExceptionAsHtmlImpl(cgi, t, af); + } + + void presentExceptionAsHtmlImpl(Cgi cgi, Throwable t, Form automaticForm) { + if(auto e = cast(ResourceNotFoundException) t) { + auto container = this.htmlContainer(); + + container.addChild("p", e.msg); + + if(!cgi.outputtedResponseData) + cgi.setResponseStatus("404 Not Found"); + cgi.write(container.parentDocument.toString(), true); + } else if(auto mae = cast(MissingArgumentException) t) { + if(automaticForm is null) + goto generic; + auto container = this.htmlContainer(); + if(cgi.requestMethod == Cgi.RequestMethod.POST) + container.appendChild(Element.make("p", "Argument `" ~ mae.argumentName ~ "` of type `" ~ mae.argumentType ~ "` is missing")); + container.appendChild(automaticForm); + + cgi.write(container.parentDocument.toString(), true); + } else { + generic: + auto container = this.htmlContainer(); + + // import std.stdio; writeln(t.toString()); + + container.appendChild(exceptionToElement(t)); + + container.addChild("h4", "GET"); + foreach(k, v; cgi.get) { + auto deets = container.addChild("details"); + deets.addChild("summary", k); + deets.addChild("div", v); + } + + container.addChild("h4", "POST"); + foreach(k, v; cgi.post) { + auto deets = container.addChild("details"); + deets.addChild("summary", k); + deets.addChild("div", v); + } + + + if(!cgi.outputtedResponseData) + cgi.setResponseStatus("500 Internal Server Error"); + cgi.write(container.parentDocument.toString(), true); + } + } + + Element exceptionToElement(Throwable t) { + auto div = Element.make("div"); + div.addClass("exception-display"); + + div.addChild("p", t.msg); + div.addChild("p", "Inner code origin: " ~ typeid(t).name ~ "@" ~ t.file ~ ":" ~ to!string(t.line)); + + auto pre = div.addChild("pre"); + string s; + s = t.toString(); + Element currentBox; + bool on = false; + foreach(line; s.splitLines) { + if(!on && line.startsWith("-----")) + on = true; + if(!on) continue; + if(line.indexOf("arsd/") != -1) { + if(currentBox is null) { + currentBox = pre.addChild("details"); + currentBox.addChild("summary", "Framework code"); + } + currentBox.addChild("span", line ~ "\n"); + } else { + pre.addChild("span", line ~ "\n"); + currentBox = null; + } + } + + return div; + } + + /++ + Returns an element for a particular type + +/ + Element elementFor(T)(string displayName, string name, Element function() udaSuggestion) { + import std.traits; + + auto div = Element.make("div"); + div.addClass("form-field"); + + static if(is(T == Cgi.UploadedFile)) { + Element lbl; + if(displayName !is null) { + lbl = div.addChild("label"); + lbl.addChild("span", displayName, "label-text"); + lbl.appendText(" "); + } else { + lbl = div; + } + auto i = lbl.addChild("input", name); + i.attrs.name = name; + i.attrs.type = "file"; + } else static if(is(T == enum)) { + Element lbl; + if(displayName !is null) { + lbl = div.addChild("label"); + lbl.addChild("span", displayName, "label-text"); + lbl.appendText(" "); + } else { + lbl = div; + } + auto i = lbl.addChild("select", name); + i.attrs.name = name; + + foreach(memberName; __traits(allMembers, T)) + i.addChild("option", memberName); + + } else static if(is(T == struct)) { + if(displayName !is null) + div.addChild("span", displayName, "label-text"); + auto fieldset = div.addChild("fieldset"); + fieldset.addChild("legend", beautify(T.stringof)); // FIXME + fieldset.addChild("input", name); + foreach(idx, memberName; __traits(allMembers, T)) + static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { + fieldset.appendChild(elementFor!(typeof(__traits(getMember, T, memberName)))(beautify(memberName), name ~ "." ~ memberName, null /* FIXME: pull off the UDA */)); + } + } else static if(isSomeString!T || isIntegral!T || isFloatingPoint!T) { + Element lbl; + if(displayName !is null) { + lbl = div.addChild("label"); + lbl.addChild("span", displayName, "label-text"); + lbl.appendText(" "); + } else { + lbl = div; + } + Element i; + if(udaSuggestion) { + i = udaSuggestion(); + lbl.appendChild(i); + } else { + i = lbl.addChild("input", name); + } + i.attrs.name = name; + static if(isSomeString!T) + i.attrs.type = "text"; + else + i.attrs.type = "number"; + if(i.tagName == "textarea") + i.textContent = to!string(T.init); + else + i.attrs.value = to!string(T.init); + } else static if(is(T == bool)) { + Element lbl; + if(displayName !is null) { + lbl = div.addChild("label"); + lbl.addChild("span", displayName, "label-text"); + lbl.appendText(" "); + } else { + lbl = div; + } + auto i = lbl.addChild("input", name); + i.attrs.type = "checkbox"; + i.attrs.value = "true"; + i.attrs.name = name; + } else static if(is(T == K[], K)) { + auto templ = div.addChild("template"); + templ.appendChild(elementFor!(K)(null, name, null /* uda??*/)); + if(displayName !is null) + div.addChild("span", displayName, "label-text"); + auto btn = div.addChild("button"); + btn.addClass("add-array-button"); + btn.attrs.type = "button"; + btn.innerText = "Add"; + btn.attrs.onclick = q{ + var a = document.importNode(this.parentNode.firstChild.content, true); + this.parentNode.insertBefore(a, this); + }; + } else static if(is(T == V[K], K, V)) { + div.innerText = "assoc array not implemented for automatic form at this time"; + } else { + static assert(0, "unsupported type for cgi call " ~ T.stringof); + } + + + return div; + } + + /// creates a form for gathering the function's arguments + Form createAutomaticFormForFunction(alias method, T)(T dg) { + + auto form = cast(Form) Element.make("form"); + + form.method = "POST"; // FIXME + + form.addClass("automatic-form"); + + string formDisplayName = beautify(__traits(identifier, method)); + foreach(attr; __traits(getAttributes, method)) + static if(is(typeof(attr) == DisplayName)) + formDisplayName = attr.name; + form.addChild("h3", formDisplayName); + + import std.traits; + + //Parameters!method params; + //alias idents = ParameterIdentifierTuple!method; + //alias defaults = ParameterDefaults!method; + + static if(is(typeof(method) P == __parameters)) + foreach(idx, _; P) {{ + + alias param = P[idx .. idx + 1]; + + static if(!mustNotBeSetFromWebParams!(param[0], __traits(getAttributes, param))) { + string displayName = beautify(__traits(identifier, param)); + Element function() element; + foreach(attr; __traits(getAttributes, param)) { + static if(is(typeof(attr) == DisplayName)) + displayName = attr.name; + else static if(is(typeof(attr) : typeof(element))) { + element = attr; + } + } + auto i = form.appendChild(elementFor!(param)(displayName, __traits(identifier, param), element)); + if(i.querySelector("input[type=file]") !is null) + form.setAttribute("enctype", "multipart/form-data"); + } + }} + + form.addChild("div", Html(``), "submit-button-holder"); + + return form; + } + + /// creates a form for gathering object members (for the REST object thing right now) + Form createAutomaticFormForObject(T)(T obj) { + auto form = cast(Form) Element.make("form"); + + form.addClass("automatic-form"); + + form.addChild("h3", beautify(__traits(identifier, T))); + + import std.traits; + + //Parameters!method params; + //alias idents = ParameterIdentifierTuple!method; + //alias defaults = ParameterDefaults!method; + + foreach(idx, memberName; __traits(derivedMembers, T)) {{ + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + string displayName = beautify(memberName); + Element function() element; + foreach(attr; __traits(getAttributes, __traits(getMember, T, memberName))) + static if(is(typeof(attr) == DisplayName)) + displayName = attr.name; + else static if(is(typeof(attr) : typeof(element))) + element = attr; + form.appendChild(elementFor!(typeof(__traits(getMember, T, memberName)))(displayName, memberName, element)); + + form.setValue(memberName, to!string(__traits(getMember, obj, memberName))); + }}} + + form.addChild("div", Html(``), "submit-button-holder"); + + return form; + } + + /// + Element formatReturnValueAsHtml(T)(T t) { + import std.traits; + + static if(is(T == typeof(null))) { + return Element.make("span"); + } else static if(is(T : Element)) { + return t; + } else static if(is(T == MultipleResponses!Types, Types...)) { + foreach(index, type; Types) { + if(t.contains == index) + return formatReturnValueAsHtml(t.payload[index]); + } + assert(0); + } else static if(is(T == Paginated!E, E)) { + auto e = Element.make("div").addClass("paginated-result"); + e.appendChild(formatReturnValueAsHtml(t.items)); + if(t.nextPageUrl.length) + e.appendChild(Element.make("a", "Next Page", t.nextPageUrl)); + return e; + } else static if(isIntegral!T || isSomeString!T || isFloatingPoint!T) { + return Element.make("span", to!string(t), "automatic-data-display"); + } else static if(is(T == V[K], K, V)) { + auto dl = Element.make("dl"); + dl.addClass("automatic-data-display associative-array"); + foreach(k, v; t) { + dl.addChild("dt", to!string(k)); + dl.addChild("dd", formatReturnValueAsHtml(v)); + } + return dl; + } else static if(is(T == struct)) { + auto dl = Element.make("dl"); + dl.addClass("automatic-data-display struct"); + + foreach(idx, memberName; __traits(allMembers, T)) + static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { + dl.addChild("dt", beautify(memberName)); + dl.addChild("dd", formatReturnValueAsHtml(__traits(getMember, t, memberName))); + } + + return dl; + } else static if(is(T == bool)) { + return Element.make("span", t ? "true" : "false", "automatic-data-display"); + } else static if(is(T == E[], E)) { + static if(is(E : RestObject!Proxy, Proxy)) { + // treat RestObject similar to struct + auto table = cast(Table) Element.make("table"); + table.addClass("automatic-data-display"); + string[] names; + foreach(idx, memberName; __traits(derivedMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + names ~= beautify(memberName); + } + table.appendHeaderRow(names); + + foreach(l; t) { + auto tr = table.appendRow(); + foreach(idx, memberName; __traits(derivedMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + static if(memberName == "id") { + string val = to!string(__traits(getMember, l, memberName)); + tr.addChild("td", Element.make("a", val, E.stringof.toLower ~ "s/" ~ val)); // FIXME + } else { + tr.addChild("td", formatReturnValueAsHtml(__traits(getMember, l, memberName))); + } + } + } + + return table; + } else static if(is(E == struct)) { + // an array of structs is kinda special in that I like + // having those formatted as tables. + auto table = cast(Table) Element.make("table"); + table.addClass("automatic-data-display"); + string[] names; + foreach(idx, memberName; __traits(allMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + names ~= beautify(memberName); + } + table.appendHeaderRow(names); + + foreach(l; t) { + auto tr = table.appendRow(); + foreach(idx, memberName; __traits(allMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + tr.addChild("td", formatReturnValueAsHtml(__traits(getMember, l, memberName))); + } + } + + return table; + } else { + // otherwise, I will just make a list. + auto ol = Element.make("ol"); + ol.addClass("automatic-data-display"); + foreach(e; t) + ol.addChild("li", formatReturnValueAsHtml(e)); + return ol; + } + } else static if(is(T : Object)) { + static if(is(typeof(t.toHtml()))) // FIXME: maybe i will make this an interface + return Element.make("div", t.toHtml()); + else + return Element.make("div", t.toString()); + } else static assert(0, "bad return value for cgi call " ~ T.stringof); + + assert(0); + } + +} + +/++ + The base class for the [dispatcher] function and object support. ++/ +class WebObject { + //protected Cgi cgi; + + protected void initialize(Cgi cgi) { + //this.cgi = cgi; + } +} + +/++ + Can return one of the given types, decided at runtime. The syntax + is to declare all the possible types in the return value, then you + can `return typeof(return)(...value...)` to construct it. + + It has an auto-generated constructor for each value it can hold. + + --- + MultipleResponses!(Redirection, string) getData(int how) { + if(how & 1) + return typeof(return)(Redirection("http://dpldocs.info/")); + else + return typeof(return)("hi there!"); + } + --- + + If you have lots of returns, you could, inside the function, `alias r = typeof(return);` to shorten it a little. ++/ +struct MultipleResponses(T...) { + private size_t contains; + private union { + private T payload; + } + + static foreach(index, type; T) + public this(type t) { + contains = index; + payload[index] = t; + } + + /++ + This is primarily for testing. It is your way of getting to the response. + + Let's say you wanted to test that one holding a Redirection and a string actually + holds a string, by name of "test": + + --- + auto valueToTest = your_test_function(); + + valueToTest.visit( + (Redirection r) { assert(0); }, // got a redirection instead of a string, fail the test + (string s) { assert(s == "test"); } // right value, go ahead and test it. + ); + --- + + History: + Was horribly broken until June 16, 2022. Ironically, I wrote it for tests but never actually tested it. + It tried to use alias lambdas before, but runtime delegates work much better so I changed it. + +/ + void visit(Handlers...)(Handlers handlers) { + template findHandler(type, int count, HandlersToCheck...) { + static if(HandlersToCheck.length == 0) + enum findHandler = -1; + else { + static if(is(typeof(HandlersToCheck[0].init(type.init)))) + enum findHandler = count; + else + enum findHandler = findHandler!(type, count + 1, HandlersToCheck[1 .. $]); + } + } + foreach(index, type; T) { + enum handlerIndex = findHandler!(type, 0, Handlers); + static if(handlerIndex == -1) + static assert(0, "Type " ~ type.stringof ~ " was not handled by visitor"); + else { + if(index == this.contains) + handlers[handlerIndex](this.payload[index]); + } + } + } + + /+ + auto toArsdJsvar()() { + import arsd.jsvar; + return var(null); + } + +/ +} + +// FIXME: implement this somewhere maybe +struct RawResponse { + int code; + string[] headers; + const(ubyte)[] responseBody; +} + +/++ + You can return this from [WebObject] subclasses for redirections. + + (though note the static types means that class must ALWAYS redirect if + you return this directly. You might want to return [MultipleResponses] if it + can be conditional) ++/ +struct Redirection { + string to; /// The URL to redirect to. + int code = 303; /// The HTTP code to return. +} + +/++ + Serves a class' methods, as a kind of low-state RPC over the web. To be used with [dispatcher]. + + Usage of this function will add a dependency on [arsd.dom] and [arsd.jsvar] unless you have overriden + the presenter in the dispatcher. + + FIXME: explain this better + + You can overload functions to a limited extent: you can provide a zero-arg and non-zero-arg function, + and non-zero-arg functions can filter via UDAs for various http methods. Do not attempt other overloads, + the runtime result of that is undefined. + + A method is assumed to allow any http method unless it lists some in UDAs, in which case it is limited to only those. + (this might change, like maybe i will use pure as an indicator GET is ok. idk.) + + $(WARNING + --- + // legal in D, undefined runtime behavior with cgi.d, it may call either method + // even if you put different URL udas on it, the current code ignores them. + void foo(int a) {} + void foo(string a) {} + --- + ) + + See_Also: [serveRestObject], [serveStaticFile] ++/ +auto serveApi(T)(string urlPrefix) { + assert(urlPrefix[$ - 1] == '/'); + return serveApiInternal!T(urlPrefix); +} + +private string nextPieceFromSlash(ref string remainingUrl) { + if(remainingUrl.length == 0) + return remainingUrl; + int slash = 0; + while(slash < remainingUrl.length && remainingUrl[slash] != '/') // && remainingUrl[slash] != '.') + slash++; + + // I am specifically passing `null` to differentiate it vs empty string + // so in your ctor, `items` means new T(null) and `items/` means new T("") + auto ident = remainingUrl.length == 0 ? null : remainingUrl[0 .. slash]; + // so if it is the last item, the dot can be used to load an alternative view + // otherwise tho the dot is considered part of the identifier + // FIXME + + // again notice "" vs null here! + if(slash == remainingUrl.length) + remainingUrl = null; + else + remainingUrl = remainingUrl[slash + 1 .. $]; + + return ident; +} + +/++ + UDA used to indicate to the [dispatcher] that a trailing slash should always be added to or removed from the url. It will do it as a redirect header as-needed. ++/ +enum AddTrailingSlash; +/// ditto +enum RemoveTrailingSlash; + +private auto serveApiInternal(T)(string urlPrefix) { + + import arsd.dom; + import arsd.jsvar; + + static bool internalHandler(Presenter)(string urlPrefix, Cgi cgi, Presenter presenter, immutable void* details) { + string remainingUrl = cgi.pathInfo[urlPrefix.length .. $]; + + try { + // see duplicated code below by searching subresource_ctor + // also see mustNotBeSetFromWebParams + + static if(is(typeof(T.__ctor) P == __parameters)) { + P params; + + foreach(pidx, param; P) { + static if(is(param : Cgi)) { + static assert(!is(param == immutable)); + cast() params[pidx] = cgi; + } else static if(is(param == Session!D, D)) { + static assert(!is(param == immutable)); + cast() params[pidx] = cgi.getSessionObject!D(); + + } else { + static if(hasIfCalledFromWeb!(__traits(getAttributes, P[pidx .. pidx + 1]))) { + foreach(uda; __traits(getAttributes, P[pidx .. pidx + 1])) { + static if(is(uda == ifCalledFromWeb!func, alias func)) { + static if(is(typeof(func(cgi)))) + params[pidx] = func(cgi); + else + params[pidx] = func(); + } + } + } else { + + static if(__traits(compiles, { params[pidx] = param.getAutomaticallyForCgi(cgi); } )) { + params[pidx] = param.getAutomaticallyForCgi(cgi); + } else static if(is(param == string)) { + auto ident = nextPieceFromSlash(remainingUrl); + params[pidx] = ident; + } else static assert(0, "illegal type for subresource " ~ param.stringof); + } + } + } + + auto obj = new T(params); + } else { + auto obj = new T(); + } + + return internalHandlerWithObject(obj, remainingUrl, cgi, presenter); + } catch(Throwable t) { + switch(cgi.request("format", "html")) { + case "html": + static void dummy() {} + presenter.presentExceptionAsHtml(cgi, t, null); + return true; + case "json": + var envelope = var.emptyObject; + envelope.success = false; + envelope.result = null; + envelope.error = t.toString(); + cgi.setResponseContentType("application/json"); + cgi.write(envelope.toJson(), true); + return true; + default: + throw t; + // return true; + } + // return true; + } + + assert(0); + } + + static bool internalHandlerWithObject(T, Presenter)(T obj, string remainingUrl, Cgi cgi, Presenter presenter) { + + obj.initialize(cgi); + + /+ + Overload rules: + Any unique combination of HTTP verb and url path can be dispatched to function overloads + statically. + + Moreover, some args vs no args can be overloaded dynamically. + +/ + + auto methodNameFromUrl = nextPieceFromSlash(remainingUrl); + /+ + auto orig = remainingUrl; + assert(0, + (orig is null ? "__null" : orig) + ~ " .. " ~ + (methodNameFromUrl is null ? "__null" : methodNameFromUrl)); + +/ + + if(methodNameFromUrl is null) + methodNameFromUrl = "__null"; + + string hack = to!string(cgi.requestMethod) ~ " " ~ methodNameFromUrl; + + if(remainingUrl.length) + hack ~= "/"; + + switch(hack) { + foreach(methodName; __traits(derivedMembers, T)) + static if(methodName != "__ctor") + foreach(idx, overload; __traits(getOverloads, T, methodName)) { + static if(is(typeof(overload) P == __parameters)) + static if(is(typeof(overload) R == return)) + static if(__traits(getProtection, overload) == "public" || __traits(getProtection, overload) == "export") + { + static foreach(urlNameForMethod; urlNamesForMethod!(overload, urlify(methodName))) + case urlNameForMethod: + + static if(is(R : WebObject)) { + // if it returns a WebObject, it is considered a subresource. That means the url is dispatched like the ctor above. + + // the only argument it is allowed to take, outside of cgi, session, and set up thingies, is a single string + + // subresource_ctor + // also see mustNotBeSetFromWebParams + + P params; + + string ident; + + foreach(pidx, param; P) { + static if(is(param : Cgi)) { + static assert(!is(param == immutable)); + cast() params[pidx] = cgi; + } else static if(is(param == typeof(presenter))) { + cast() param[pidx] = presenter; + } else static if(is(param == Session!D, D)) { + static assert(!is(param == immutable)); + cast() params[pidx] = cgi.getSessionObject!D(); + } else { + static if(hasIfCalledFromWeb!(__traits(getAttributes, P[pidx .. pidx + 1]))) { + foreach(uda; __traits(getAttributes, P[pidx .. pidx + 1])) { + static if(is(uda == ifCalledFromWeb!func, alias func)) { + static if(is(typeof(func(cgi)))) + params[pidx] = func(cgi); + else + params[pidx] = func(); + } + } + } else { + + static if(__traits(compiles, { params[pidx] = param.getAutomaticallyForCgi(cgi); } )) { + params[pidx] = param.getAutomaticallyForCgi(cgi); + } else static if(is(param == string)) { + ident = nextPieceFromSlash(remainingUrl); + if(ident is null) { + // trailing slash mandated on subresources + cgi.setResponseLocation(cgi.pathInfo ~ "/"); + return true; + } else { + params[pidx] = ident; + } + } else static assert(0, "illegal type for subresource " ~ param.stringof); + } + } + } + + auto nobj = (__traits(getOverloads, obj, methodName)[idx])(ident); + return internalHandlerWithObject!(typeof(nobj), Presenter)(nobj, remainingUrl, cgi, presenter); + } else { + // 404 it if any url left - not a subresource means we don't get to play with that! + if(remainingUrl.length) + return false; + + bool automaticForm; + + foreach(attr; __traits(getAttributes, overload)) + static if(is(attr == AddTrailingSlash)) { + if(remainingUrl is null) { + cgi.setResponseLocation(cgi.pathInfo ~ "/"); + return true; + } + } else static if(is(attr == RemoveTrailingSlash)) { + if(remainingUrl !is null) { + cgi.setResponseLocation(cgi.pathInfo[0 .. lastIndexOf(cgi.pathInfo, "/")]); + return true; + } + + } else static if(__traits(isSame, AutomaticForm, attr)) { + automaticForm = true; + } + + /+ + int zeroArgOverload = -1; + int overloadCount = cast(int) __traits(getOverloads, T, methodName).length; + bool calledWithZeroArgs = true; + foreach(k, v; cgi.get) + if(k != "format") { + calledWithZeroArgs = false; + break; + } + foreach(k, v; cgi.post) + if(k != "format") { + calledWithZeroArgs = false; + break; + } + + // first, we need to go through and see if there is an empty one, since that + // changes inside. But otherwise, all the stuff I care about can be done via + // simple looping (other improper overloads might be flagged for runtime semantic check) + // + // an argument of type Cgi is ignored for these purposes + static foreach(idx, overload; __traits(getOverloads, T, methodName)) {{ + static if(is(typeof(overload) P == __parameters)) + static if(P.length == 0) + zeroArgOverload = cast(int) idx; + else static if(P.length == 1 && is(P[0] : Cgi)) + zeroArgOverload = cast(int) idx; + }} + // FIXME: static assert if there are multiple non-zero-arg overloads usable with a single http method. + bool overloadHasBeenCalled = false; + static foreach(idx, overload; __traits(getOverloads, T, methodName)) {{ + bool callFunction = true; + // there is a zero arg overload and this is NOT it, and we have zero args - don't call this + if(overloadCount > 1 && zeroArgOverload != -1 && idx != zeroArgOverload && calledWithZeroArgs) + callFunction = false; + // if this is the zero-arg overload, obviously it cannot be called if we got any args. + if(overloadCount > 1 && idx == zeroArgOverload && !calledWithZeroArgs) + callFunction = false; + + // FIXME: so if you just add ?foo it will give the error below even when. this might not be a great idea. + + bool hadAnyMethodRestrictions = false; + bool foundAcceptableMethod = false; + foreach(attr; __traits(getAttributes, overload)) { + static if(is(typeof(attr) == Cgi.RequestMethod)) { + hadAnyMethodRestrictions = true; + if(attr == cgi.requestMethod) + foundAcceptableMethod = true; + } + } + + if(hadAnyMethodRestrictions && !foundAcceptableMethod) + callFunction = false; + + /+ + The overloads we really want to allow are the sane ones + from the web perspective. Which is likely on HTTP verbs, + for the most part, but might also be potentially based on + some args vs zero args, or on argument names. Can't really + do argument types very reliable through the web though; those + should probably be different URLs. + + Even names I feel is better done inside the function, so I'm not + going to support that here. But the HTTP verbs and zero vs some + args makes sense - it lets you define custom forms pretty easily. + + Moreover, I'm of the opinion that empty overload really only makes + sense on GET for this case. On a POST, it is just a missing argument + exception and that should be handled by the presenter. But meh, I'll + let the user define that, D only allows one empty arg thing anyway + so the method UDAs are irrelevant. + +/ + if(callFunction) + +/ + + auto format = cgi.request("format", defaultFormat!overload()); + auto wantsFormFormat = format.startsWith("form-"); + + if(wantsFormFormat || (automaticForm && cgi.requestMethod == Cgi.RequestMethod.GET)) { + // Should I still show the form on a json thing? idk... + auto ret = presenter.createAutomaticFormForFunction!((__traits(getOverloads, obj, methodName)[idx]))(&(__traits(getOverloads, obj, methodName)[idx])); + presenter.presentSuccessfulReturn(cgi, ret, presenter.methodMeta!(__traits(getOverloads, obj, methodName)[idx]), wantsFormFormat ? format["form_".length .. $] : "html"); + return true; + } + + try { + // a void return (or typeof(null) lol) means you, the user, is doing it yourself. Gives full control. + auto ret = callFromCgi!(__traits(getOverloads, obj, methodName)[idx])(&(__traits(getOverloads, obj, methodName)[idx]), cgi); + presenter.presentSuccessfulReturn(cgi, ret, presenter.methodMeta!(__traits(getOverloads, obj, methodName)[idx]), format); + } catch(Throwable t) { + // presenter.presentExceptionAsHtml!(__traits(getOverloads, obj, methodName)[idx])(cgi, t, &(__traits(getOverloads, obj, methodName)[idx])); + presenter.presentExceptionalReturn(cgi, t, presenter.methodMeta!(__traits(getOverloads, obj, methodName)[idx]), format); + } + return true; + //}} + + //cgi.header("Accept: POST"); // FIXME list the real thing + //cgi.setResponseStatus("405 Method Not Allowed"); // again, not exactly, but sort of. no overload matched our args, almost certainly due to http verb filtering. + //return true; + } + } + } + case "GET script.js": + cgi.setResponseContentType("text/javascript"); + cgi.gzipResponse = true; + cgi.write(presenter.script(), true); + return true; + case "GET style.css": + cgi.setResponseContentType("text/css"); + cgi.gzipResponse = true; + cgi.write(presenter.style(), true); + return true; + default: + return false; + } + + assert(0); + } + return DispatcherDefinition!internalHandler(urlPrefix, false); +} + +string defaultFormat(alias method)() { + bool nonConstConditionForWorkingAroundASpuriousDmdWarning = true; + foreach(attr; __traits(getAttributes, method)) { + static if(is(typeof(attr) == DefaultFormat)) { + if(nonConstConditionForWorkingAroundASpuriousDmdWarning) + return attr.value; + } + } + return "html"; +} + +struct Paginated(T) { + T[] items; + string nextPageUrl; +} + +template urlNamesForMethod(alias method, string default_) { + string[] helper() { + auto verb = Cgi.RequestMethod.GET; + bool foundVerb = false; + bool foundNoun = false; + + string def = default_; + + bool hasAutomaticForm = false; + + foreach(attr; __traits(getAttributes, method)) { + static if(is(typeof(attr) == Cgi.RequestMethod)) { + verb = attr; + if(foundVerb) + assert(0, "Multiple http verbs on one function is not currently supported"); + foundVerb = true; + } + static if(is(typeof(attr) == UrlName)) { + if(foundNoun) + assert(0, "Multiple url names on one function is not currently supported"); + foundNoun = true; + def = attr.name; + } + static if(__traits(isSame, attr, AutomaticForm)) { + hasAutomaticForm = true; + } + } + + if(def is null) + def = "__null"; + + string[] ret; + + static if(is(typeof(method) R == return)) { + static if(is(R : WebObject)) { + def ~= "/"; + foreach(v; __traits(allMembers, Cgi.RequestMethod)) + ret ~= v ~ " " ~ def; + } else { + if(hasAutomaticForm) { + ret ~= "GET " ~ def; + ret ~= "POST " ~ def; + } else { + ret ~= to!string(verb) ~ " " ~ def; + } + } + } else static assert(0); + + return ret; + } + enum urlNamesForMethod = helper(); +} + + + enum AccessCheck { + allowed, + denied, + nonExistant, + } + + enum Operation { + show, + create, + replace, + remove, + update + } + + enum UpdateResult { + accessDenied, + noSuchResource, + success, + failure, + unnecessary + } + + enum ValidationResult { + valid, + invalid + } + + +/++ + The base of all REST objects, to be used with [serveRestObject] and [serveRestCollectionOf]. + + WARNING: this is not stable. ++/ +class RestObject(CRTP) : WebObject { + + import arsd.dom; + import arsd.jsvar; + + /// Prepare the object to be shown. + void show() {} + /// ditto + void show(string urlId) { + load(urlId); + show(); + } + + /// Override this to provide access control to this object. + AccessCheck accessCheck(string urlId, Operation operation) { + return AccessCheck.allowed; + } + + ValidationResult validate() { + // FIXME + return ValidationResult.valid; + } + + string getUrlSlug() { + import std.conv; + static if(is(typeof(CRTP.id))) + return to!string((cast(CRTP) this).id); + else + return null; + } + + // The functions with more arguments are the low-level ones, + // they forward to the ones with fewer arguments by default. + + // POST on a parent collection - this is called from a collection class after the members are updated + /++ + Given a populated object, this creates a new entry. Returns the url identifier + of the new object. + +/ + string create(scope void delegate() applyChanges) { + applyChanges(); + save(); + return getUrlSlug(); + } + + void replace() { + save(); + } + void replace(string urlId, scope void delegate() applyChanges) { + load(urlId); + applyChanges(); + replace(); + } + + void update(string[] fieldList) { + save(); + } + void update(string urlId, scope void delegate() applyChanges, string[] fieldList) { + load(urlId); + applyChanges(); + update(fieldList); + } + + void remove() {} + + void remove(string urlId) { + load(urlId); + remove(); + } + + abstract void load(string urlId); + abstract void save(); + + Element toHtml(Presenter)(Presenter presenter) { + import arsd.dom; + import std.conv; + auto obj = cast(CRTP) this; + auto div = Element.make("div"); + div.addClass("Dclass_" ~ CRTP.stringof); + div.dataset.url = getUrlSlug(); + bool first = true; + foreach(idx, memberName; __traits(derivedMembers, CRTP)) + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + if(!first) div.addChild("br"); else first = false; + div.appendChild(presenter.formatReturnValueAsHtml(__traits(getMember, obj, memberName))); + } + return div; + } + + var toJson() { + import arsd.jsvar; + var v = var.emptyObject(); + auto obj = cast(CRTP) this; + foreach(idx, memberName; __traits(derivedMembers, CRTP)) + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + v[memberName] = __traits(getMember, obj, memberName); + } + return v; + } + + /+ + auto structOf(this This) { + + } + +/ +} + +// FIXME XSRF token, prolly can just put in a cookie and then it needs to be copied to header or form hidden value +// https://use-the-index-luke.com/sql/partial-results/fetch-next-page + +/++ + Base class for REST collections. ++/ +class CollectionOf(Obj) : RestObject!(CollectionOf) { + /// You might subclass this and use the cgi object's query params + /// to implement a search filter, for example. + /// + /// FIXME: design a way to auto-generate that form + /// (other than using the WebObject thing above lol + // it'll prolly just be some searchParams UDA or maybe an enum. + // + // pagination too perhaps. + // + // and sorting too + IndexResult index() { return IndexResult.init; } + + string[] sortableFields() { return null; } + string[] searchableFields() { return null; } + + struct IndexResult { + Obj[] results; + + string[] sortableFields; + + string previousPageIdentifier; + string nextPageIdentifier; + string firstPageIdentifier; + string lastPageIdentifier; + + int numberOfPages; + } + + override string create(scope void delegate() applyChanges) { assert(0); } + override void load(string urlId) { assert(0); } + override void save() { assert(0); } + override void show() { + index(); + } + override void show(string urlId) { + show(); + } + + /// Proxy POST requests (create calls) to the child collection + alias PostProxy = Obj; +} + +/++ + Serves a REST object, similar to a Ruby on Rails resource. + + You put data members in your class. cgi.d will automatically make something out of those. + + It will call your constructor with the ID from the URL. This may be null. + It will then populate the data members from the request. + It will then call a method, if present, telling what happened. You don't need to write these! + It finally returns a reply. + + Your methods are passed a list of fields it actually set. + + The URL mapping - despite my general skepticism of the wisdom - matches up with what most REST + APIs I have used seem to follow. (I REALLY want to put trailing slashes on it though. Works better + with relative linking. But meh.) + + GET /items -> index. all values not set. + GET /items/id -> get. only ID will be set, other params ignored. + POST /items -> create. values set as given + PUT /items/id -> replace. values set as given + or POST /items/id with cgi.post["_method"] (thus urlencoded or multipart content-type) set to "PUT" to work around browser/html limitation + a GET with cgi.get["_method"] (in the url) set to "PUT" will render a form. + PATCH /items/id -> update. values set as given, list of changed fields passed + or POST /items/id with cgi.post["_method"] == "PATCH" + DELETE /items/id -> destroy. only ID guaranteed to be set + or POST /items/id with cgi.post["_method"] == "DELETE" + + Following the stupid convention, there will never be a trailing slash here, and if it is there, it will + redirect you away from it. + + API clients should set the `Accept` HTTP header to application/json or the cgi.get["_format"] = "json" var. + + I will also let you change the default, if you must. + + // One add-on is validation. You can issue a HTTP GET to a resource with _method = VALIDATE to check potential changes. + + You can define sub-resources on your object inside the object. These sub-resources are also REST objects + that follow the same thing. They may be individual resources or collections themselves. + + Your class is expected to have at least the following methods: + + FIXME: i kinda wanna add a routes object to the initialize call + + create + Create returns the new address on success, some code on failure. + show + index + update + remove + + You will want to be able to customize the HTTP, HTML, and JSON returns but generally shouldn't have to - the defaults + should usually work. The returned JSON will include a field "href" on all returned objects along with "id". Or omething like that. + + Usage of this function will add a dependency on [arsd.dom] and [arsd.jsvar]. + + NOT IMPLEMENTED + + + Really, a collection is a resource with a bunch of subresources. + + GET /items + index because it is GET on the top resource + + GET /items/foo + item but different than items? + + class Items { + + } + + ... but meh, a collection can be automated. not worth making it + a separate thing, let's look at a real example. Users has many + items and a virtual one, /users/current. + + the individual users have properties and two sub-resources: + session, which is just one, and comments, a collection. + + class User : RestObject!() { // no parent + int id; + string name; + + // the default implementations of the urlId ones is to call load(that_id) then call the arg-less one. + // but you can override them to do it differently. + + // any member which is of type RestObject can be linked automatically via href btw. + + void show() {} + void show(string urlId) {} // automated! GET of this specific thing + void create() {} // POST on a parent collection - this is called from a collection class after the members are updated + void replace(string urlId) {} // this is the PUT; really, it just updates all fields. + void update(string urlId, string[] fieldList) {} // PATCH, it updates some fields. + void remove(string urlId) {} // DELETE + + void load(string urlId) {} // the default implementation of show() populates the id, then + + this() {} + + mixin Subresource!Session; + mixin Subresource!Comment; + } + + class Session : RestObject!() { + // the parent object may not be fully constructed/loaded + this(User parent) {} + + } + + class Comment : CollectionOf!Comment { + this(User parent) {} + } + + class Users : CollectionOf!User { + // but you don't strictly need ANYTHING on a collection; it will just... collect. Implement the subobjects. + void index() {} // GET on this specific thing; just like show really, just different name for the different semantics. + User create() {} // You MAY implement this, but the default is to create a new object, populate it from args, and then call create() on the child + } + ++/ +auto serveRestObject(T)(string urlPrefix) { + assert(urlPrefix[0] == '/'); + assert(urlPrefix[$ - 1] != '/', "Do NOT use a trailing slash on REST objects."); + static bool internalHandler(Presenter)(string urlPrefix, Cgi cgi, Presenter presenter, immutable void* details) { + string url = cgi.pathInfo[urlPrefix.length .. $]; + + if(url.length && url[$ - 1] == '/') { + // remove the final slash... + cgi.setResponseLocation(cgi.scriptName ~ cgi.pathInfo[0 .. $ - 1]); + return true; + } + + return restObjectServeHandler!T(cgi, presenter, url); + } + return DispatcherDefinition!internalHandler(urlPrefix, false); +} + +/+ +/// Convenience method for serving a collection. It will be named the same +/// as type T, just with an s at the end. If you need any further, just +/// write the class yourself. +auto serveRestCollectionOf(T)(string urlPrefix) { + assert(urlPrefix[0] == '/'); + mixin(`static class `~T.stringof~`s : CollectionOf!(T) {}`); + return serveRestObject!(mixin(T.stringof ~ "s"))(urlPrefix); +} ++/ + +bool restObjectServeHandler(T, Presenter)(Cgi cgi, Presenter presenter, string url) { + string urlId = null; + if(url.length && url[0] == '/') { + // asking for a subobject + urlId = url[1 .. $]; + foreach(idx, ch; urlId) { + if(ch == '/') { + urlId = urlId[0 .. idx]; + break; + } + } + } + + // FIXME handle other subresources + + static if(is(T : CollectionOf!(C), C)) { + if(urlId !is null) { + return restObjectServeHandler!(C, Presenter)(cgi, presenter, url); // FIXME? urlId); + } + } + + // FIXME: support precondition failed, if-modified-since, expectation failed, etc. + + auto obj = new T(); + obj.initialize(cgi); + // FIXME: populate reflection info delegates + + + // FIXME: I am not happy with this. + switch(urlId) { + case "script.js": + cgi.setResponseContentType("text/javascript"); + cgi.gzipResponse = true; + cgi.write(presenter.script(), true); + return true; + case "style.css": + cgi.setResponseContentType("text/css"); + cgi.gzipResponse = true; + cgi.write(presenter.style(), true); + return true; + default: + // intentionally blank + } + + + + + static void applyChangesTemplate(Obj)(Cgi cgi, Obj obj) { + foreach(idx, memberName; __traits(derivedMembers, Obj)) + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + __traits(getMember, obj, memberName) = cgi.request(memberName, __traits(getMember, obj, memberName)); + } + } + void applyChanges() { + applyChangesTemplate(cgi, obj); + } + + string[] modifiedList; + + void writeObject(bool addFormLinks) { + if(cgi.request("format") == "json") { + cgi.setResponseContentType("application/json"); + cgi.write(obj.toJson().toString, true); + } else { + auto container = presenter.htmlContainer(); + if(addFormLinks) { + static if(is(T : CollectionOf!(C), C)) + container.appendHtml(` + + + + `); + else + container.appendHtml(` + Back +
    + + +
    + `); + } + container.appendChild(obj.toHtml(presenter)); + cgi.write(container.parentDocument.toString, true); + } + } + + // FIXME: I think I need a set type in here.... + // it will be nice to pass sets of members. + + try + switch(cgi.requestMethod) { + case Cgi.RequestMethod.GET: + // I could prolly use template this parameters in the implementation above for some reflection stuff. + // sure, it doesn't automatically work in subclasses... but I instantiate here anyway... + + // automatic forms here for usable basic auto site from browser. + // even if the format is json, it could actually send out the links and formats, but really there i'ma be meh. + switch(cgi.request("_method", "GET")) { + case "GET": + static if(is(T : CollectionOf!(C), C)) { + auto results = obj.index(); + if(cgi.request("format", "html") == "html") { + auto container = presenter.htmlContainer(); + auto html = presenter.formatReturnValueAsHtml(results.results); + container.appendHtml(` +
    + +
    + `); + + container.appendChild(html); + cgi.write(container.parentDocument.toString, true); + } else { + cgi.setResponseContentType("application/json"); + import arsd.jsvar; + var json = var.emptyArray; + foreach(r; results.results) { + var o = var.emptyObject; + foreach(idx, memberName; __traits(derivedMembers, typeof(r))) + static if(__traits(compiles, __traits(getMember, r, memberName).offsetof)) { + o[memberName] = __traits(getMember, r, memberName); + } + + json ~= o; + } + cgi.write(json.toJson(), true); + } + } else { + obj.show(urlId); + writeObject(true); + } + break; + case "PATCH": + obj.load(urlId); + goto case; + case "PUT": + case "POST": + // an editing form for the object + auto container = presenter.htmlContainer(); + static if(__traits(compiles, () { auto o = new obj.PostProxy(); })) { + auto form = (cgi.request("_method") == "POST") ? presenter.createAutomaticFormForObject(new obj.PostProxy()) : presenter.createAutomaticFormForObject(obj); + } else { + auto form = presenter.createAutomaticFormForObject(obj); + } + form.attrs.method = "POST"; + form.setValue("_method", cgi.request("_method", "GET")); + container.appendChild(form); + cgi.write(container.parentDocument.toString(), true); + break; + case "DELETE": + // FIXME: a delete form for the object (can be phrased "are you sure?") + auto container = presenter.htmlContainer(); + container.appendHtml(` +
    + Are you sure you want to delete this item? + + +
    + + `); + cgi.write(container.parentDocument.toString(), true); + break; + default: + cgi.write("bad method\n", true); + } + break; + case Cgi.RequestMethod.POST: + // this is to allow compatibility with HTML forms + switch(cgi.request("_method", "POST")) { + case "PUT": + goto PUT; + case "PATCH": + goto PATCH; + case "DELETE": + goto DELETE; + case "POST": + static if(__traits(compiles, () { auto o = new obj.PostProxy(); })) { + auto p = new obj.PostProxy(); + void specialApplyChanges() { + applyChangesTemplate(cgi, p); + } + string n = p.create(&specialApplyChanges); + } else { + string n = obj.create(&applyChanges); + } + + auto newUrl = cgi.scriptName ~ cgi.pathInfo ~ "/" ~ n; + cgi.setResponseLocation(newUrl); + cgi.setResponseStatus("201 Created"); + cgi.write(`The object has been created.`); + break; + default: + cgi.write("bad method\n", true); + } + // FIXME this should be valid on the collection, but not the child.... + // 303 See Other + break; + case Cgi.RequestMethod.PUT: + PUT: + obj.replace(urlId, &applyChanges); + writeObject(false); + break; + case Cgi.RequestMethod.PATCH: + PATCH: + obj.update(urlId, &applyChanges, modifiedList); + writeObject(false); + break; + case Cgi.RequestMethod.DELETE: + DELETE: + obj.remove(urlId); + cgi.setResponseStatus("204 No Content"); + break; + default: + // FIXME: OPTIONS, HEAD + } + catch(Throwable t) { + presenter.presentExceptionAsHtml(cgi, t); + } + + return true; +} + +/+ +struct SetOfFields(T) { + private void[0][string] storage; + void set(string what) { + //storage[what] = + } + void unset(string what) {} + void setAll() {} + void unsetAll() {} + bool isPresent(string what) { return false; } +} ++/ + +/+ +enum readonly; +enum hideonindex; ++/ + +/++ + Returns true if I recommend gzipping content of this type. You might + want to call it from your Presenter classes before calling cgi.write. + + --- + cgi.setResponseContentType(yourContentType); + cgi.gzipResponse = gzipRecommendedForContentType(yourContentType); + cgi.write(yourData, true); + --- + + This is used internally by [serveStaticFile], [serveStaticFileDirectory], [serveStaticData], and maybe others I forgot to update this doc about. + + + The implementation considers text content to be recommended to gzip. This may change, but it seems reasonable enough for now. + + History: + Added January 28, 2023 (dub v11.0) ++/ +bool gzipRecommendedForContentType(string contentType) { + if(contentType.startsWith("text/")) + return true; + if(contentType.startsWith("application/javascript")) + return true; + + return false; +} + +/++ + Serves a static file. To be used with [dispatcher]. + + See_Also: [serveApi], [serveRestObject], [dispatcher], [serveRedirect] ++/ +auto serveStaticFile(string urlPrefix, string filename = null, string contentType = null) { +// https://baus.net/on-tcp_cork/ +// man 2 sendfile + assert(urlPrefix[0] == '/'); + if(filename is null) + filename = decodeComponent(urlPrefix[1 .. $]); // FIXME is this actually correct? + if(contentType is null) { + contentType = contentTypeFromFileExtension(filename); + } + + static struct DispatcherDetails { + string filename; + string contentType; + } + + static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) { + if(details.contentType.indexOf("image/") == 0 || details.contentType.indexOf("audio/") == 0) + cgi.setCache(true); + cgi.setResponseContentType(details.contentType); + cgi.gzipResponse = gzipRecommendedForContentType(details.contentType); + cgi.write(std.file.read(details.filename), true); + return true; + } + return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, true, DispatcherDetails(filename, contentType)); +} + +/++ + Serves static data. To be used with [dispatcher]. + + History: + Added October 31, 2021 ++/ +auto serveStaticData(string urlPrefix, immutable(void)[] data, string contentType = null) { + assert(urlPrefix[0] == '/'); + if(contentType is null) { + contentType = contentTypeFromFileExtension(urlPrefix); + } + + static struct DispatcherDetails { + immutable(void)[] data; + string contentType; + } + + static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) { + cgi.setCache(true); + cgi.setResponseContentType(details.contentType); + cgi.write(details.data, true); + return true; + } + return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, true, DispatcherDetails(data, contentType)); +} + +string contentTypeFromFileExtension(string filename) { + if(filename.endsWith(".png")) + return "image/png"; + if(filename.endsWith(".apng")) + return "image/apng"; + if(filename.endsWith(".svg")) + return "image/svg+xml"; + if(filename.endsWith(".jpg")) + return "image/jpeg"; + if(filename.endsWith(".html")) + return "text/html"; + if(filename.endsWith(".css")) + return "text/css"; + if(filename.endsWith(".js")) + return "application/javascript"; + if(filename.endsWith(".wasm")) + return "application/wasm"; + if(filename.endsWith(".mp3")) + return "audio/mpeg"; + if(filename.endsWith(".pdf")) + return "application/pdf"; + return null; +} + +/// This serves a directory full of static files, figuring out the content-types from file extensions. +/// It does not let you to descend into subdirectories (or ascend out of it, of course) +auto serveStaticFileDirectory(string urlPrefix, string directory = null, bool recursive = false) { + assert(urlPrefix[0] == '/'); + assert(urlPrefix[$-1] == '/'); + + static struct DispatcherDetails { + string directory; + bool recursive; + } + + if(directory is null) + directory = urlPrefix[1 .. $]; + + if(directory.length == 0) + directory = "./"; + + assert(directory[$-1] == '/'); + + static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) { + auto file = decodeComponent(cgi.pathInfo[urlPrefix.length .. $]); // FIXME: is this actually correct + + if(details.recursive) { + // never allow a backslash since it isn't in a typical url anyway and makes the following checks easier + if(file.indexOf("\\") != -1) + return false; + + import std.path; + + file = std.path.buildNormalizedPath(file); + enum upOneDir = ".." ~ std.path.dirSeparator; + + // also no point doing any kind of up directory things since that makes it more likely to break out of the parent + if(file == ".." || file.startsWith(upOneDir)) + return false; + if(std.path.isAbsolute(file)) + return false; + + // FIXME: if it has slashes and stuff, should we redirect to the canonical resource? or what? + + // once it passes these filters it is probably ok. + } else { + if(file.indexOf("/") != -1 || file.indexOf("\\") != -1) + return false; + } + + auto contentType = contentTypeFromFileExtension(file); + + auto fn = details.directory ~ file; + if(std.file.exists(fn)) { + //if(contentType.indexOf("image/") == 0) + //cgi.setCache(true); + //else if(contentType.indexOf("audio/") == 0) + cgi.setCache(true); + cgi.setResponseContentType(contentType); + cgi.gzipResponse = gzipRecommendedForContentType(contentType); + cgi.write(std.file.read(fn), true); + return true; + } else { + return false; + } + } + + return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, false, DispatcherDetails(directory, recursive)); +} + +/++ + Redirects one url to another + + See_Also: [dispatcher], [serveStaticFile] ++/ +auto serveRedirect(string urlPrefix, string redirectTo, int code = 303) { + assert(urlPrefix[0] == '/'); + static struct DispatcherDetails { + string redirectTo; + string code; + } + + static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) { + cgi.setResponseLocation(details.redirectTo, true, details.code); + return true; + } + + + return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, true, DispatcherDetails(redirectTo, getHttpCodeText(code))); +} + +/// Used exclusively with `dispatchTo` +struct DispatcherData(Presenter) { + Cgi cgi; /// You can use this cgi object. + Presenter presenter; /// This is the presenter from top level, and will be forwarded to the sub-dispatcher. + size_t pathInfoStart; /// This is forwarded to the sub-dispatcher. It may be marked private later, or at least read-only. +} + +/++ + Dispatches the URL to a specific function. ++/ +auto handleWith(alias handler)(string urlPrefix) { + // cuz I'm too lazy to do it better right now + static class Hack : WebObject { + static import std.traits; + @UrlName("") + auto handle(std.traits.Parameters!handler args) { + return handler(args); + } + } + + return urlPrefix.serveApiInternal!Hack; +} + +/++ + Dispatches the URL (and anything under it) to another dispatcher function. The function should look something like this: + + --- + bool other(DD)(DD dd) { + return dd.dispatcher!( + "/whatever".serveRedirect("/success"), + "/api/".serveApi!MyClass + ); + } + --- + + The `DD` in there will be an instance of [DispatcherData] which you can inspect, or forward to another dispatcher + here. It is a template to account for any Presenter type, so you can do compile-time analysis in your presenters. + Or, of course, you could just use the exact type in your own code. + + You return true if you handle the given url, or false if not. Just returning the result of [dispatcher] will do a + good job. + + ++/ +auto dispatchTo(alias handler)(string urlPrefix) { + assert(urlPrefix[0] == '/'); + assert(urlPrefix[$-1] != '/'); + static bool internalHandler(Presenter)(string urlPrefix, Cgi cgi, Presenter presenter, const void* details) { + return handler(DispatcherData!Presenter(cgi, presenter, urlPrefix.length)); + } + + return DispatcherDefinition!(internalHandler)(urlPrefix, false); +} + +/++ + See [serveStaticFile] if you want to serve a file off disk. + + History: + Added January 28, 2023 (dub v11.0) ++/ +auto serveStaticData(string urlPrefix, immutable(ubyte)[] data, string contentType, string filenameToSuggestAsDownload = null) { + assert(urlPrefix[0] == '/'); + + static struct DispatcherDetails { + immutable(ubyte)[] data; + string contentType; + string filenameToSuggestAsDownload; + } + + static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) { + cgi.setCache(true); + cgi.setResponseContentType(details.contentType); + if(details.filenameToSuggestAsDownload.length) + cgi.header("Content-Disposition: attachment; filename=\""~details.filenameToSuggestAsDownload~"\""); + cgi.gzipResponse = gzipRecommendedForContentType(details.contentType); + cgi.write(details.data, true); + return true; + } + return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, true, DispatcherDetails(data, contentType, filenameToSuggestAsDownload)); +} + +/++ + Placeholder for use with [dispatchSubsection]'s `NewPresenter` argument to indicate you want to keep the parent's presenter. + + History: + Added January 28, 2023 (dub v11.0) ++/ +alias KeepExistingPresenter = typeof(null); + +/++ + For use with [dispatchSubsection]. Calls your filter with the request and if your filter returns false, + this issues the given errorCode and stops processing. + + --- + bool hasAdminPermissions(Cgi cgi) { + return true; + } + + mixin DispatcherMain!( + "/admin".dispatchSubsection!( + passFilterOrIssueError!(hasAdminPermissions, 403), + KeepExistingPresenter, + "/".serveApi!AdminFunctions + ) + ); + --- + + History: + Added January 28, 2023 (dub v11.0) ++/ +template passFilterOrIssueError(alias filter, int errorCode) { + bool passFilterOrIssueError(DispatcherDetails)(DispatcherDetails dd) { + if(filter(dd.cgi)) + return true; + dd.presenter.renderBasicError(dd.cgi, errorCode); + return false; + } +} + +/++ + Allows for a subsection of your dispatched urls to be passed through other a pre-request filter, optionally pick up an new presenter class, + and then be dispatched to their own handlers. + + --- + /+ + // a long-form filter function + bool permissionCheck(DispatcherData)(DispatcherData dd) { + // you are permitted to call mutable methods on the Cgi object + // Note if you use a Cgi subclass, you can try dynamic casting it back to your custom type to attach per-request data + // though much of the request is immutable so there's only so much you're allowed to do to modify it. + + if(checkPermissionOnRequest(dd.cgi)) { + return true; // OK, allow processing to continue + } else { + dd.presenter.renderBasicError(dd.cgi, 403); // reply forbidden to the requester + return false; // and stop further processing into this subsection + } + } + +/ + + // but you can also do short-form filters: + + bool permissionCheck(Cgi cgi) { + return ("ok" in cgi.get) !is null; + } + + // handler for the subsection + class AdminClass : WebObject { + int foo() { return 5; } + } + + // handler for the main site + class TheMainSite : WebObject {} + + mixin DispatcherMain!( + "/admin".dispatchSubsection!( + // converts our short-form filter into a long-form filter + passFilterOrIssueError!(permissionCheck, 403), + // can use a new presenter if wanted for the subsection + KeepExistingPresenter, + // and then provide child route dispatchers + "/".serveApi!AdminClass + ), + // and back to the top level + "/".serveApi!TheMainSite + ); + --- + + Note you can encapsulate sections in files like this: + + --- + auto adminDispatcher(string urlPrefix) { + return urlPrefix.dispatchSubsection!( + .... + ); + } + + mixin DispatcherMain!( + "/admin".adminDispatcher, + // and so on + ) + --- + + If you want no filter, you can pass `(cgi) => true` as the filter to approve all requests. + + If you want to keep the same presenter as the parent, use [KeepExistingPresenter] as the presenter argument. + + + History: + Added January 28, 2023 (dub v11.0) ++/ +auto dispatchSubsection(alias PreRequestFilter, NewPresenter, definitions...)(string urlPrefix) { + assert(urlPrefix[0] == '/'); + assert(urlPrefix[$-1] != '/'); + static bool internalHandler(Presenter)(string urlPrefix, Cgi cgi, Presenter presenter, const void* details) { + static if(!is(PreRequestFilter == typeof(null))) { + if(!PreRequestFilter(DispatcherData!Presenter(cgi, presenter, urlPrefix.length))) + return true; // we handled it by rejecting it + } + + static if(is(NewPresenter == Presenter) || is(NewPresenter == typeof(null))) { + return dispatcher!definitions(DispatcherData!Presenter(cgi, presenter, urlPrefix.length)); + } else { + auto newPresenter = new NewPresenter(); + return dispatcher!(definitions(DispatcherData!NewPresenter(cgi, presenter, urlPrefix.length))); + } + } + + return DispatcherDefinition!(internalHandler)(urlPrefix, false); +} + +/++ + A URL dispatcher. + + --- + if(cgi.dispatcher!( + "/api/".serveApi!MyApiClass, + "/objects/lol".serveRestObject!MyRestObject, + "/file.js".serveStaticFile, + "/admin/".dispatchTo!adminHandler + )) return; + --- + + + You define a series of url prefixes followed by handlers. + + You may want to do different pre- and post- processing there, for example, + an authorization check and different page layout. You can use different + presenters and different function chains. See [dispatchSubsection] for details. + + [dispatchTo] will send the request to another function for handling. ++/ +template dispatcher(definitions...) { + bool dispatcher(Presenter)(Cgi cgi, Presenter presenterArg = null) { + static if(is(Presenter == typeof(null))) { + static class GenericWebPresenter : WebPresenter!(GenericWebPresenter) {} + auto presenter = new GenericWebPresenter(); + } else + alias presenter = presenterArg; + + return dispatcher(DispatcherData!(typeof(presenter))(cgi, presenter, 0)); + } + + bool dispatcher(DispatcherData)(DispatcherData dispatcherData) if(!is(DispatcherData : Cgi)) { + // I can prolly make this more efficient later but meh. + foreach(definition; definitions) { + if(definition.rejectFurther) { + if(dispatcherData.cgi.pathInfo[dispatcherData.pathInfoStart .. $] == definition.urlPrefix) { + auto ret = definition.handler( + dispatcherData.cgi.pathInfo[0 .. dispatcherData.pathInfoStart + definition.urlPrefix.length], + dispatcherData.cgi, dispatcherData.presenter, definition.details); + if(ret) + return true; + } + } else if( + dispatcherData.cgi.pathInfo[dispatcherData.pathInfoStart .. $].startsWith(definition.urlPrefix) && + // cgi.d dispatcher urls must be complete or have a /; + // "foo" -> thing should NOT match "foobar", just "foo" or "foo/thing" + (definition.urlPrefix[$-1] == '/' || (dispatcherData.pathInfoStart + definition.urlPrefix.length) == dispatcherData.cgi.pathInfo.length + || dispatcherData.cgi.pathInfo[dispatcherData.pathInfoStart + definition.urlPrefix.length] == '/') + ) { + auto ret = definition.handler( + dispatcherData.cgi.pathInfo[0 .. dispatcherData.pathInfoStart + definition.urlPrefix.length], + dispatcherData.cgi, dispatcherData.presenter, definition.details); + if(ret) + return true; + } + } + return false; + } +} + +}); + +private struct StackBuffer { + char[1024] initial = void; + char[] buffer; + size_t position; + + this(int a) { + buffer = initial[]; + position = 0; + } + + void add(in char[] what) { + if(position + what.length > buffer.length) + buffer.length = position + what.length + 1024; // reallocate with GC to handle special cases + buffer[position .. position + what.length] = what[]; + position += what.length; + } + + void add(in char[] w1, in char[] w2, in char[] w3 = null) { + add(w1); + add(w2); + add(w3); + } + + void add(long v) { + char[16] buffer = void; + auto pos = buffer.length; + bool negative; + if(v < 0) { + negative = true; + v = -v; + } + do { + buffer[--pos] = cast(char) (v % 10 + '0'); + v /= 10; + } while(v); + + if(negative) + buffer[--pos] = '-'; + + auto res = buffer[pos .. $]; + + add(res[]); + } + + char[] get() @nogc { + return buffer[0 .. position]; + } +} + +// duplicated in http2.d +private static string getHttpCodeText(int code) pure nothrow @nogc { + switch(code) { + case 200: return "200 OK"; + case 201: return "201 Created"; + case 202: return "202 Accepted"; + case 203: return "203 Non-Authoritative Information"; + case 204: return "204 No Content"; + case 205: return "205 Reset Content"; + case 206: return "206 Partial Content"; + // + case 300: return "300 Multiple Choices"; + case 301: return "301 Moved Permanently"; + case 302: return "302 Found"; + case 303: return "303 See Other"; + case 304: return "304 Not Modified"; + case 305: return "305 Use Proxy"; + case 307: return "307 Temporary Redirect"; + case 308: return "308 Permanent Redirect"; + + // + case 400: return "400 Bad Request"; + case 401: return "401 Unauthorized"; + case 402: return "402 Payment Required"; + case 403: return "403 Forbidden"; + case 404: return "404 Not Found"; + case 405: return "405 Method Not Allowed"; + case 406: return "406 Not Acceptable"; + case 407: return "407 Proxy Authentication Required"; + case 408: return "408 Request Timeout"; + case 409: return "409 Conflict"; + case 410: return "410 Gone"; + case 411: return "411 Length Required"; + case 412: return "412 Precondition Failed"; + case 413: return "413 Payload Too Large"; + case 414: return "414 URI Too Long"; + case 415: return "415 Unsupported Media Type"; + case 416: return "416 Range Not Satisfiable"; + case 417: return "417 Expectation Failed"; + case 418: return "418 I'm a teapot"; + case 421: return "421 Misdirected Request"; + case 422: return "422 Unprocessable Entity (WebDAV)"; + case 423: return "423 Locked (WebDAV)"; + case 424: return "424 Failed Dependency (WebDAV)"; + case 425: return "425 Too Early"; + case 426: return "426 Upgrade Required"; + case 428: return "428 Precondition Required"; + case 431: return "431 Request Header Fields Too Large"; + case 451: return "451 Unavailable For Legal Reasons"; + + case 500: return "500 Internal Server Error"; + case 501: return "501 Not Implemented"; + case 502: return "502 Bad Gateway"; + case 503: return "503 Service Unavailable"; + case 504: return "504 Gateway Timeout"; + case 505: return "505 HTTP Version Not Supported"; + case 506: return "506 Variant Also Negotiates"; + case 507: return "507 Insufficient Storage (WebDAV)"; + case 508: return "508 Loop Detected (WebDAV)"; + case 510: return "510 Not Extended"; + case 511: return "511 Network Authentication Required"; + // + default: assert(0, "Unsupported http code"); + } +} + + +/+ +/++ + This is the beginnings of my web.d 2.0 - it dispatches web requests to a class object. + + It relies on jsvar.d and dom.d. + + + You can get javascript out of it to call. The generated functions need to look + like + + function name(a,b,c,d,e) { + return _call("name", {"realName":a,"sds":b}); + } + + And _call returns an object you can call or set up or whatever. ++/ +bool apiDispatcher()(Cgi cgi) { + import arsd.jsvar; + import arsd.dom; +} ++/ +version(linux) +private extern(C) int eventfd (uint initval, int flags) nothrow @trusted @nogc; +/* +Copyright: Adam D. Ruppe, 2008 - 2023 +License: [http://www.boost.org/LICENSE_1_0.txt|Boost License 1.0]. +Authors: Adam D. Ruppe + + Copyright Adam D. Ruppe 2008 - 2023. +Distributed under the Boost Software License, Version 1.0. + (See accompanying file LICENSE_1_0.txt or copy at + http://www.boost.org/LICENSE_1_0.txt) +*/ \ No newline at end of file diff --git a/src/clientSideFiltering.d b/src/clientSideFiltering.d new file mode 100644 index 00000000..ed67c629 --- /dev/null +++ b/src/clientSideFiltering.d @@ -0,0 +1,384 @@ +// What is this module called? +module clientSideFiltering; + +// What does this module require to function? +import std.algorithm; +import std.array; +import std.file; +import std.path; +import std.regex; +import std.stdio; +import std.string; + +// What other modules that we have created do we need to import? +import config; +import util; +import log; + +class ClientSideFiltering { + // Class variables + ApplicationConfig appConfig; + string[] paths; + string[] businessSharedItemsList; + Regex!char fileMask; + Regex!char directoryMask; + bool skipDirStrictMatch = false; + bool skipDotfiles = false; + + this(ApplicationConfig appConfig) { + // Configure the class varaible to consume the application configuration + this.appConfig = appConfig; + } + + // Initialise the required items + bool initialise() { + // + log.vdebug("Configuring Client Side Filtering (Selective Sync)"); + // Load the sync_list file if it exists + if (exists(appConfig.syncListFilePath)){ + loadSyncList(appConfig.syncListFilePath); + } + + // Load the Business Shared Items file if it exists + if (exists(appConfig.businessSharedItemsFilePath)){ + loadBusinessSharedItems(appConfig.businessSharedItemsFilePath); + } + + // Configure skip_dir, skip_file, skip-dir-strict-match & skip_dotfiles from config entries + // Handle skip_dir configuration in config file + log.vdebug("Configuring skip_dir ..."); + log.vdebug("skip_dir: ", appConfig.getValueString("skip_dir")); + setDirMask(appConfig.getValueString("skip_dir")); + + // Was --skip-dir-strict-match configured? + log.vdebug("Configuring skip_dir_strict_match ..."); + log.vdebug("skip_dir_strict_match: ", appConfig.getValueBool("skip_dir_strict_match")); + if (appConfig.getValueBool("skip_dir_strict_match")) { + setSkipDirStrictMatch(); + } + + // Was --skip-dot-files configured? + log.vdebug("Configuring skip_dotfiles ..."); + log.vdebug("skip_dotfiles: ", appConfig.getValueBool("skip_dotfiles")); + if (appConfig.getValueBool("skip_dotfiles")) { + setSkipDotfiles(); + } + + // Handle skip_file configuration in config file + log.vdebug("Configuring skip_file ..."); + // Validate skip_file to ensure that this does not contain an invalid configuration + // Do not use a skip_file entry of .* as this will prevent correct searching of local changes to process. + foreach(entry; appConfig.getValueString("skip_file").split("|")){ + if (entry == ".*") { + // invalid entry element detected + log.error("ERROR: Invalid skip_file entry '.*' detected"); + return false; + } + } + + // All skip_file entries are valid + log.vdebug("skip_file: ", appConfig.getValueString("skip_file")); + setFileMask(appConfig.getValueString("skip_file")); + + // All configured OK + return true; + } + + // Load sync_list file if it exists + void loadSyncList(string filepath) { + // open file as read only + auto file = File(filepath, "r"); + auto range = file.byLine(); + foreach (line; range) { + // Skip comments in file + if (line.length == 0 || line[0] == ';' || line[0] == '#') continue; + paths ~= buildNormalizedPath(line); + } + file.close(); + } + + // load business_shared_folders file + void loadBusinessSharedItems(string filepath) { + // open file as read only + auto file = File(filepath, "r"); + auto range = file.byLine(); + foreach (line; range) { + // Skip comments in file + if (line.length == 0 || line[0] == ';' || line[0] == '#') continue; + businessSharedItemsList ~= buildNormalizedPath(line); + } + file.close(); + } + + // Configure the regex that will be used for 'skip_file' + void setFileMask(const(char)[] mask) { + fileMask = wild2regex(mask); + log.vdebug("Selective Sync File Mask: ", fileMask); + } + + // Configure the regex that will be used for 'skip_dir' + void setDirMask(const(char)[] dirmask) { + directoryMask = wild2regex(dirmask); + log.vdebug("Selective Sync Directory Mask: ", directoryMask); + } + + // Configure skipDirStrictMatch if function is called + // By default, skipDirStrictMatch = false; + void setSkipDirStrictMatch() { + skipDirStrictMatch = true; + } + + // Configure skipDotfiles if function is called + // By default, skipDotfiles = false; + void setSkipDotfiles() { + skipDotfiles = true; + } + + // return value of skipDotfiles + bool getSkipDotfiles() { + return skipDotfiles; + } + + // Match against sync_list only + bool isPathExcludedViaSyncList(string path) { + // Debug output that we are performing a 'sync_list' inclusion / exclusion test + return isPathExcluded(path, paths); + } + + // config file skip_dir parameter + bool isDirNameExcluded(string name) { + // Does the directory name match skip_dir config entry? + // Returns true if the name matches a skip_dir config entry + // Returns false if no match + log.vdebug("skip_dir evaluation for: ", name); + + // Try full path match first + if (!name.matchFirst(directoryMask).empty) { + log.vdebug("'!name.matchFirst(directoryMask).empty' returned true = matched"); + return true; + } else { + // Do we check the base name as well? + if (!skipDirStrictMatch) { + log.vdebug("No Strict Matching Enforced"); + + // Test the entire path working backwards from child + string path = buildNormalizedPath(name); + string checkPath; + auto paths = pathSplitter(path); + + foreach_reverse(directory; paths) { + if (directory != "/") { + // This will add a leading '/' but that needs to be stripped to check + checkPath = "/" ~ directory ~ checkPath; + if(!checkPath.strip('/').matchFirst(directoryMask).empty) { + log.vdebug("'!checkPath.matchFirst(directoryMask).empty' returned true = matched"); + return true; + } + } + } + } else { + log.vdebug("Strict Matching Enforced - No Match"); + } + } + // no match + return false; + } + + // config file skip_file parameter + bool isFileNameExcluded(string name) { + // Does the file name match skip_file config entry? + // Returns true if the name matches a skip_file config entry + // Returns false if no match + log.vdebug("skip_file evaluation for: ", name); + + // Try full path match first + if (!name.matchFirst(fileMask).empty) { + return true; + } else { + // check just the file name + string filename = baseName(name); + if(!filename.matchFirst(fileMask).empty) { + return true; + } + } + // no match + return false; + } + + // test if the given path is not included in the allowed paths + // if there are no allowed paths always return false + private bool isPathExcluded(string path, string[] allowedPaths) { + // function variables + bool exclude = false; + bool exludeDirectMatch = false; // will get updated to true, if there is a pattern match to sync_list entry + bool excludeMatched = false; // will get updated to true, if there is a pattern match to sync_list entry + bool finalResult = true; // will get updated to false, if pattern match to sync_list entry + int offset; + string wildcard = "*"; + + // always allow the root + if (path == ".") return false; + // if there are no allowed paths always return false + if (allowedPaths.empty) return false; + path = buildNormalizedPath(path); + log.vdebug("Evaluation against 'sync_list' for this path: ", path); + log.vdebug("[S]exclude = ", exclude); + log.vdebug("[S]exludeDirectMatch = ", exludeDirectMatch); + log.vdebug("[S]excludeMatched = ", excludeMatched); + + // unless path is an exact match, entire sync_list entries need to be processed to ensure + // negative matches are also correctly detected + foreach (allowedPath; allowedPaths) { + // is this an inclusion path or finer grained exclusion? + switch (allowedPath[0]) { + case '-': + // sync_list path starts with '-', this user wants to exclude this path + exclude = true; + // If the sync_list entry starts with '-/' offset needs to be 2, else 1 + if (startsWith(allowedPath, "-/")){ + // Offset needs to be 2 + offset = 2; + } else { + // Offset needs to be 1 + offset = 1; + } + break; + case '!': + // sync_list path starts with '!', this user wants to exclude this path + exclude = true; + // If the sync_list entry starts with '!/' offset needs to be 2, else 1 + if (startsWith(allowedPath, "!/")){ + // Offset needs to be 2 + offset = 2; + } else { + // Offset needs to be 1 + offset = 1; + } + break; + case '/': + // sync_list path starts with '/', this user wants to include this path + // but a '/' at the start causes matching issues, so use the offset for comparison + exclude = false; + offset = 1; + break; + + default: + // no negative pattern, default is to not exclude + exclude = false; + offset = 0; + } + + // What are we comparing against? + log.vdebug("Evaluation against 'sync_list' entry: ", allowedPath); + + // Generate the common prefix from the path vs the allowed path + auto comm = commonPrefix(path, allowedPath[offset..$]); + + // Is path is an exact match of the allowed path? + if (comm.length == path.length) { + // we have a potential exact match + // strip any potential '/*' from the allowed path, to avoid a potential lesser common match + string strippedAllowedPath = strip(allowedPath[offset..$], "/*"); + + if (path == strippedAllowedPath) { + // we have an exact path match + log.vdebug("exact path match"); + if (!exclude) { + log.vdebug("Evaluation against 'sync_list' result: direct match"); + finalResult = false; + // direct match, break and go sync + break; + } else { + log.vdebug("Evaluation against 'sync_list' result: direct match - path to be excluded"); + // do not set excludeMatched = true here, otherwise parental path also gets excluded + // flag exludeDirectMatch so that a 'wildcard match' will not override this exclude + exludeDirectMatch = true; + // final result + finalResult = true; + } + } else { + // no exact path match, but something common does match + log.vdebug("something 'common' matches the input path"); + auto splitAllowedPaths = pathSplitter(strippedAllowedPath); + string pathToEvaluate = ""; + foreach(base; splitAllowedPaths) { + pathToEvaluate ~= base; + if (path == pathToEvaluate) { + // The input path matches what we want to evaluate against as a direct match + if (!exclude) { + log.vdebug("Evaluation against 'sync_list' result: direct match for parental path item"); + finalResult = false; + // direct match, break and go sync + break; + } else { + log.vdebug("Evaluation against 'sync_list' result: direct match for parental path item but to be excluded"); + finalResult = true; + // do not set excludeMatched = true here, otherwise parental path also gets excluded + } + } + pathToEvaluate ~= dirSeparator; + } + } + } + + // Is path is a subitem/sub-folder of the allowed path? + if (comm.length == allowedPath[offset..$].length) { + // The given path is potentially a subitem of an allowed path + // We want to capture sub-folders / files of allowed paths here, but not explicitly match other items + // if there is no wildcard + auto subItemPathCheck = allowedPath[offset..$] ~ "/"; + if (canFind(path, subItemPathCheck)) { + // The 'path' includes the allowed path, and is 'most likely' a sub-path item + if (!exclude) { + log.vdebug("Evaluation against 'sync_list' result: parental path match"); + finalResult = false; + // parental path matches, break and go sync + break; + } else { + log.vdebug("Evaluation against 'sync_list' result: parental path match but must be excluded"); + finalResult = true; + excludeMatched = true; + } + } + } + + // Does the allowed path contain a wildcard? (*) + if (canFind(allowedPath[offset..$], wildcard)) { + // allowed path contains a wildcard + // manually replace '*' for '.*' to be compatible with regex + string regexCompatiblePath = replace(allowedPath[offset..$], "*", ".*"); + auto allowedMask = regex(regexCompatiblePath); + if (matchAll(path, allowedMask)) { + // regex wildcard evaluation matches + // if we have a prior pattern match for an exclude, excludeMatched = true + if (!exclude && !excludeMatched && !exludeDirectMatch) { + // nothing triggered an exclusion before evaluation against wildcard match attempt + log.vdebug("Evaluation against 'sync_list' result: wildcard pattern match"); + finalResult = false; + } else { + log.vdebug("Evaluation against 'sync_list' result: wildcard pattern matched but must be excluded"); + finalResult = true; + excludeMatched = true; + } + } + } + } + // Interim results + log.vdebug("[F]exclude = ", exclude); + log.vdebug("[F]exludeDirectMatch = ", exludeDirectMatch); + log.vdebug("[F]excludeMatched = ", excludeMatched); + + // If exclude or excludeMatched is true, then finalResult has to be true + if ((exclude) || (excludeMatched) || (exludeDirectMatch)) { + finalResult = true; + } + + // results + if (finalResult) { + log.vdebug("Evaluation against 'sync_list' final result: EXCLUDED"); + } else { + log.vdebug("Evaluation against 'sync_list' final result: included for sync"); + } + return finalResult; + } +} \ No newline at end of file diff --git a/src/config.d b/src/config.d new file mode 100644 index 00000000..e39ab216 --- /dev/null +++ b/src/config.d @@ -0,0 +1,2107 @@ +// What is this module called? +module config; + +// What does this module require to function? +import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit; +import std.stdio; +import std.process; +import std.regex; +import std.string; +import std.algorithm.searching; +import std.algorithm.sorting: sort; +import std.file; +import std.conv; +import std.path; +import std.getopt; +import std.format; +import std.ascii; + +// What other modules that we have created do we need to import? +import log; +import util; + +class ApplicationConfig { + // Application default values - these do not change + // - Compile time regex + immutable auto configRegex = ctRegex!(`^(\w+)\s*=\s*"(.*)"\s*$`); + // - Default directory to store data + immutable string defaultSyncDir = "~/OneDrive"; + // - Default Directory Permissions + immutable long defaultDirectoryPermissionMode = 700; + // - Default File Permissions + immutable long defaultFilePermissionMode = 600; + // - Default types of files to skip + // v2.0.x - 2.4.x: ~*|.~*|*.tmp + // v2.5.x : ~*|.~*|*.tmp|*.swp|*.partial + immutable string defaultSkipFile = "~*|.~*|*.tmp|*.swp|*.partial"; + // - Default directories to skip (default is skip none) + immutable string defaultSkipDir = ""; + // - Default log directory + immutable string defaultLogFileDir = "/var/log/onedrive"; + // - Default configuration directory + immutable string defaultConfigDirName = "~/.config/onedrive"; + + // Microsoft Requirements + // - Default Application ID (abraunegg) + immutable string defaultApplicationId = "d50ca740-c83f-4d1b-b616-12c519384f0c"; + // - Microsoft User Agent ISV Tag + immutable string isvTag = "ISV"; + // - Microsoft User Agent Company name + immutable string companyName = "abraunegg"; + // - Microsoft Application name as per Microsoft Azure application registration + immutable string appTitle = "OneDrive Client for Linux"; + // Comply with OneDrive traffic decoration requirements + // https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online + // - Identify as ISV and include Company Name, App Name separated by a pipe character and then adding Version number separated with a slash character + + //immutable string defaultUserAgent = isvTag ~ "|" ~ companyName ~ "|" ~ appTitle ~ "/" ~ strip(import("version")); + immutable string defaultUserAgent = isvTag ~ "|" ~ companyName ~ "|" ~ appTitle ~ "/" ~ "v2.5.0-alpha-2"; + + // HTTP Struct items, used for configuring HTTP() + // Curl Timeout Handling + // libcurl dns_cache_timeout timeout + immutable int defaultDnsTimeout = 60; + // Connect timeout for HTTP|HTTPS connections + immutable int defaultConnectTimeout = 30; + // Default data timeout for HTTP + // curl.d has a default of: _defaultDataTimeout = dur!"minutes"(2); + immutable int defaultDataTimeout = 240; + // Maximum time any operation is allowed to take + // This includes dns resolution, connecting, data transfer, etc. + immutable int defaultOperationTimeout = 3600; + // Specify what IP protocol version should be used when communicating with OneDrive + immutable int defaultIpProtocol = 0; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only + // Specify how many redirects should be allowed + immutable int defaultMaxRedirects = 5; + + // Azure Active Directory & Graph Explorer Endpoints + // - Global & Default + immutable string globalAuthEndpoint = "https://login.microsoftonline.com"; + immutable string globalGraphEndpoint = "https://graph.microsoft.com"; + // - US Government L4 + immutable string usl4AuthEndpoint = "https://login.microsoftonline.us"; + immutable string usl4GraphEndpoint = "https://graph.microsoft.us"; + // - US Government L5 + immutable string usl5AuthEndpoint = "https://login.microsoftonline.us"; + immutable string usl5GraphEndpoint = "https://dod-graph.microsoft.us"; + // - Germany + immutable string deAuthEndpoint = "https://login.microsoftonline.de"; + immutable string deGraphEndpoint = "https://graph.microsoft.de"; + // - China + immutable string cnAuthEndpoint = "https://login.chinacloudapi.cn"; + immutable string cnGraphEndpoint = "https://microsoftgraph.chinacloudapi.cn"; + + // Application items that depend on application run-time environment, thus cannot be immutable + // Public variables + // Was the application just authorised - paste of response uri + bool applicationAuthorizeResponseUri = false; + // Store the 'refresh_token' file path + string refreshTokenFilePath = ""; + // Store the refreshToken for use within the application + string refreshToken; + // Store the 'session_upload.CRC32-HASH' file path + string uploadSessionFilePath = ""; + + bool apiWasInitialised = false; + bool syncEngineWasInitialised = false; + string accountType; + string defaultDriveId; + string defaultRootId; + ulong remainingFreeSpace = 0; + bool quotaAvailable = true; + bool quotaRestricted = false; + + bool fullScanTrueUpRequired = false; + bool surpressLoggingOutput = false; + + // This is the value that needs testing when we are actually downloading and uploading data + ulong concurrentThreads = 16; + + // All application run-time paths are formulated from this as a set of defaults + // - What is the home path of the actual 'user' that is running the application + string defaultHomePath = ""; + // - What is the config path for the application. By default, this is ~/.config/onedrive but can be overridden by using --confdir + private string configDirName = defaultConfigDirName; + // - In case we have to use a system config directory such as '/etc/onedrive' or similar, store that path in this variable + private string systemConfigDirName = ""; + // - Store the configured converted octal value for directory permissions + private int configuredDirectoryPermissionMode; + // - Store the configured converted octal value for file permissions + private int configuredFilePermissionMode; + // - Store the 'delta_link' file path + private string deltaLinkFilePath = ""; + // - Store the 'items.sqlite3' file path + string databaseFilePath = ""; + // - Store the 'items-dryrun.sqlite3' file path + string databaseFilePathDryRun = ""; + // - Store the user 'config' file path + private string userConfigFilePath = ""; + // - Store the system 'config' file path + private string systemConfigFilePath = ""; + // - What is the 'config' file path that will be used? + private string applicableConfigFilePath = ""; + // - Store the 'sync_list' file path + string syncListFilePath = ""; + // - Store the 'business_shared_items' file path + string businessSharedItemsFilePath = ""; + + // Hash files so that we can detect when the configuration has changed, in items that will require a --resync + private string configHashFile = ""; + private string configBackupFile = ""; + private string syncListHashFile = ""; + private string businessSharedItemsHashFile = ""; + + // Store the actual 'runtime' hash + private string currentConfigHash = ""; + private string currentSyncListHash = ""; + private string currentBusinessSharedItemsHash = ""; + + // Store the previous config files hash values (file contents) + private string previousConfigHash = ""; + private string previousSyncListHash = ""; + private string previousBusinessSharedItemsHash = ""; + + // Store items that come in from the 'config' file, otherwise these need to be set the the defaults + private string configFileSyncDir = defaultSyncDir; + private string configFileSkipFile = defaultSkipFile; + private string configFileSkipDir = ""; // Default here is no directories are skipped + private string configFileDriveId = ""; // Default here is that no drive id is specified + private bool configFileSkipDotfiles = false; + private bool configFileSkipSymbolicLinks = false; + private bool configFileSyncBusinessSharedItems = false; + + // File permission values (set via initialise function) + private int convertedPermissionValue; + + // Array of values that are the actual application runtime configuration + // The values stored in these array's are the actual application configuration which can then be accessed by getValue & setValue + string[string] stringValues; + long[string] longValues; + bool[string] boolValues; + + bool shellEnvironmentSet = false; + + // Initialise the application configuration + bool initialise(string confdirOption) { + + // Default runtime configuration - entries in config file ~/.config/onedrive/config or derived from variables above + // An entry here means it can be set via the config file if there is a coresponding entry, read from config and set via update_from_args() + // The below becomes the 'default' application configuration before config file and/or cli options are overlayed on top + + // - Set the required default values + stringValues["application_id"] = defaultApplicationId; + stringValues["log_dir"] = defaultLogFileDir; + stringValues["skip_dir"] = defaultSkipDir; + stringValues["skip_file"] = defaultSkipFile; + stringValues["sync_dir"] = defaultSyncDir; + stringValues["user_agent"] = defaultUserAgent; + // - The 'drive_id' is used when we specify a specific OneDrive ID when attempting to sync Shared Folders and SharePoint items + stringValues["drive_id"] = ""; + // Support National Azure AD endpoints as per https://docs.microsoft.com/en-us/graph/deployments + // By default, if empty, use standard Azure AD URL's + // Will support the following options: + // - USL4 + // AD Endpoint: https://login.microsoftonline.us + // Graph Endpoint: https://graph.microsoft.us + // - USL5 + // AD Endpoint: https://login.microsoftonline.us + // Graph Endpoint: https://dod-graph.microsoft.us + // - DE + // AD Endpoint: https://portal.microsoftazure.de + // Graph Endpoint: https://graph.microsoft.de + // - CN + // AD Endpoint: https://login.chinacloudapi.cn + // Graph Endpoint: https://microsoftgraph.chinacloudapi.cn + stringValues["azure_ad_endpoint"] = ""; + + // Support single-tenant applications that are not able to use the "common" multiplexer + stringValues["azure_tenant_id"] = ""; + // - Store how many times was --verbose added + longValues["verbose"] = log.verbose; // might also be initialised by the first getopt call! + // - The amount of time (seconds) between monitor sync loops + longValues["monitor_interval"] = 300; + // - What size of file should be skipped? + longValues["skip_size"] = 0; + // - How many 'loops' when using --monitor, before we print out high frequency recurring items? + longValues["monitor_log_frequency"] = 12; + // - Number of N sync runs before performing a full local scan of sync_dir + // By default 12 which means every ~60 minutes a full disk scan of sync_dir will occur + // 'monitor_interval' * 'monitor_fullscan_frequency' = 3600 = 1 hour + longValues["monitor_fullscan_frequency"] = 12; + // - Number of children in a path that is locally removed which will be classified as a 'big data delete' + longValues["classify_as_big_delete"] = 1000; + // - Configure the default folder permission attributes for newly created folders + longValues["sync_dir_permissions"] = defaultDirectoryPermissionMode; + // - Configure the default file permission attributes for newly created file + longValues["sync_file_permissions"] = defaultFilePermissionMode; + // - Configure download / upload rate limits + longValues["rate_limit"] = 0; + // - To ensure we do not fill up the load disk, how much disk space should be reserved by default + longValues["space_reservation"] = 50 * 2^^20; // 50 MB as Bytes + + // HTTPS & CURL Operation Settings + // - Maximum time an operation is allowed to take + // This includes dns resolution, connecting, data transfer, etc. + longValues["operation_timeout"] = defaultOperationTimeout; + // libcurl dns_cache_timeout timeout + longValues["dns_timeout"] = defaultDnsTimeout; + // Timeout for HTTPS connections + longValues["connect_timeout"] = defaultConnectTimeout; + // Timeout for activity on a HTTPS connection + longValues["data_timeout"] = defaultDataTimeout; + // What IP protocol version should be used when communicating with OneDrive + longValues["ip_protocol_version"] = defaultIpProtocol; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only + + // - Do we wish to upload only? + boolValues["upload_only"] = false; + // - Do we need to check for the .nomount file on the mount point? + boolValues["check_nomount"] = false; + // - Do we need to check for the .nosync file anywhere? + boolValues["check_nosync"] = false; + // - Do we wish to download only? + boolValues["download_only"] = false; + // - Do we disable notifications? + boolValues["disable_notifications"] = false; + // - Do we bypass all the download validation? + // This is critically important not to disable, but because of SharePoint 'feature' can be highly desirable to enable + boolValues["disable_download_validation"] = false; + // - Do we bypass all the upload validation? + // This is critically important not to disable, but because of SharePoint 'feature' can be highly desirable to enable + boolValues["disable_upload_validation"] = false; + // - Do we enable logging? + boolValues["enable_logging"] = false; + // - Do we force HTTP 1.1 for connections to the OneDrive API + // By default we use the curl library default, which should be HTTP2 for most operations governed by the OneDrive API + boolValues["force_http_11"] = false; + // - Do we treat the local file system as the source of truth for our data? + boolValues["local_first"] = false; + // - Do we ignore local file deletes, so that all files are retained online? + boolValues["no_remote_delete"] = false; + // - Do we skip symbolic links? + boolValues["skip_symlinks"] = false; + // - Do we enable debugging for all HTTPS flows. Critically important for debugging API issues. + boolValues["debug_https"] = false; + // - Do we skip .files and .folders? + boolValues["skip_dotfiles"] = false; + // - Do we perform a 'dry-run' with no local or remote changes actually being performed? + boolValues["dry_run"] = false; + // - Do we sync all the files in the 'sync_dir' root? + boolValues["sync_root_files"] = false; + // - Do we delete source after successful transfer? + boolValues["remove_source_files"] = false; + // - Do we perform strict matching for skip_dir? + boolValues["skip_dir_strict_match"] = false; + // - Do we perform a --resync? + boolValues["resync"] = false; + // - resync now needs to be acknowledged based on the 'risk' of using it + boolValues["resync_auth"] = false; + // - Ignore data safety checks and overwrite local data rather than preserve & rename + // This is a config file option ONLY + boolValues["bypass_data_preservation"] = false; + // - Allow enable / disable of the syncing of OneDrive Business Shared items (files & folders) via configuration file + boolValues["sync_business_shared_items"] = false; + // - Log to application output running configuration values + boolValues["display_running_config"] = false; + // - Configure read-only authentication scope + boolValues["read_only_auth_scope"] = false; + // - Flag to cleanup local files when using --download-only + boolValues["cleanup_local_files"] = false; + + // Webhook Feature Options + stringValues["webhook_public_url"] = ""; + stringValues["webhook_listening_host"] = ""; + longValues["webhook_listening_port"] = 8888; + longValues["webhook_expiration_interval"] = 3600 * 24; + longValues["webhook_renewal_interval"] = 3600 * 12; + boolValues["webhook_enabled"] = false; + + // Print in debug the application version as soon as possible + //log.vdebug("Application Version: ", strip(import("version"))); + string tempVersion = "v2.5.0-alpha-2" ~ " GitHub version: " ~ strip(import("version")); + log.vdebug("Application Version: ", tempVersion); + + // EXPAND USERS HOME DIRECTORY + // Determine the users home directory. + // Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts + // Check for HOME environment variable + if (environment.get("HOME") != ""){ + // Use HOME environment variable + log.vdebug("runtime_environment: HOME environment variable detected, expansion of '~' should be possible"); + defaultHomePath = environment.get("HOME"); + shellEnvironmentSet = true; + } else { + if ((environment.get("SHELL") == "") && (environment.get("USER") == "")){ + // No shell is set or username - observed case when running as systemd service under CentOS 7.x + log.vdebug("runtime_environment: No HOME, SHELL or USER environment variable configuration detected. Expansion of '~' not possible"); + defaultHomePath = "/root"; + shellEnvironmentSet = false; + + } else { + // A shell & valid user is set, but no HOME is set, use ~ which can be expanded + log.vdebug("runtime_environment: SHELL and USER environment variable detected, expansion of '~' should be possible"); + defaultHomePath = "~"; + shellEnvironmentSet = true; + } + } + // outcome of setting defaultHomePath + log.vdebug("runtime_environment: Calculated defaultHomePath: ", defaultHomePath); + + // DEVELOPER OPTIONS + // display_memory = true | false + // - It may be desirable to display the memory usage of the application to assist with diagnosing memory issues with the application + // - This is especially beneficial when debugging or performing memory tests with Valgrind + boolValues["display_memory"] = false; + // monitor_max_loop = long value + // - It may be desirable to, when running in monitor mode, force monitor mode to 'quit' after X number of loops + // - This is especially beneficial when debugging or performing memory tests with Valgrind + longValues["monitor_max_loop"] = 0; + // display_sync_options = true | false + // - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging + boolValues["display_sync_options"] = false; + // force_children_scan = true | false + // - Force client to use /children rather than /delta to query changes on OneDrive + // - This option flags nationalCloudDeployment as true, forcing the client to act like it is using a National Cloud Deployment model + boolValues["force_children_scan"] = false; + // display_processing_time = true | false + // - Enabling this option will add function processing times to the console output + // - This then enables tracking of where the application is spending most amount of time when processing data when users have questions re performance + boolValues["display_processing_time"] = false; + + // Function variables + string configDirBase; + string systemConfigDirBase; + bool configurationInitialised = false; + + // Initialise the application configuration, using the provided --confdir option was passed in + if (!confdirOption.empty) { + // A CLI 'confdir' was passed in + // Clean up any stray " .. these should not be there for correct process handling of the configuration option + confdirOption = strip(confdirOption,"\""); + log.vdebug("configDirName: CLI override to set configDirName to: ", confdirOption); + if (canFind(confdirOption,"~")) { + // A ~ was found + log.vdebug("configDirName: A '~' was found in configDirName, using the calculated 'defaultHomePath' to replace '~'"); + configDirName = defaultHomePath ~ strip(confdirOption,"~","~"); + } else { + configDirName = confdirOption; + } + } else { + // Determine the base directory relative to which user specific configuration files should be stored + if (environment.get("XDG_CONFIG_HOME") != ""){ + log.vdebug("configDirBase: XDG_CONFIG_HOME environment variable set"); + configDirBase = environment.get("XDG_CONFIG_HOME"); + } else { + // XDG_CONFIG_HOME does not exist on systems where X11 is not present - ie - headless systems / servers + log.vdebug("configDirBase: WARNING - no XDG_CONFIG_HOME environment variable set"); + configDirBase = buildNormalizedPath(buildPath(defaultHomePath, ".config")); + // Also set up a path to pre-shipped shared configs (which can be overridden by supplying a config file in userspace) + systemConfigDirBase = "/etc"; + } + // Output configDirBase calculation + log.vdebug("configDirBase: ", configDirBase); + // Set the calculated application configuration directory + log.vdebug("configDirName: Configuring application to use calculated config path"); + // configDirBase contains the correct path so we do not need to check for presence of '~' + configDirName = buildNormalizedPath(buildPath(configDirBase, "onedrive")); + // systemConfigDirBase contains the correct path so we do not need to check for presence of '~' + systemConfigDirName = buildNormalizedPath(buildPath(systemConfigDirBase, "onedrive")); + } + + // Configuration directory should now have been correctly identified + if (!exists(configDirName)) { + // create the directory + mkdirRecurse(configDirName); + // Configure the applicable permissions for the folder + configDirName.setAttributes(returnRequiredDirectoryPermisions()); + } else { + // The config path exists + // The path that exists must be a directory, not a file + if (!isDir(configDirName)) { + if (!confdirOption.empty) { + // the configuration path was passed in by the user .. user error + log.error("ERROR: --confdir entered value is an existing file instead of an existing directory"); + } else { + // other error + log.error("ERROR: " ~ confdirOption ~ " is a file rather than a directory"); + } + // Must exit + exit(EXIT_FAILURE); + } + } + + // Update application set variables based on configDirName + // - What is the full path for the 'refresh_token' + refreshTokenFilePath = buildNormalizedPath(buildPath(configDirName, "refresh_token")); + // - What is the full path for the 'delta_link' + deltaLinkFilePath = buildNormalizedPath(buildPath(configDirName, "delta_link")); + // - What is the full path for the 'items.sqlite3' - the database cache file + databaseFilePath = buildNormalizedPath(buildPath(configDirName, "items.sqlite3")); + // - What is the full path for the 'items-dryrun.sqlite3' - the dry-run database cache file + databaseFilePathDryRun = buildNormalizedPath(buildPath(configDirName, "items-dryrun.sqlite3")); + // - What is the full path for the 'resume_upload' + uploadSessionFilePath = buildNormalizedPath(buildPath(configDirName, "session_upload")); + // - What is the full path for the 'sync_list' file + syncListFilePath = buildNormalizedPath(buildPath(configDirName, "sync_list")); + // - What is the full path for the 'config' - the user file to configure the application + userConfigFilePath = buildNormalizedPath(buildPath(configDirName, "config")); + // - What is the full path for the system 'config' file if it is required + systemConfigFilePath = buildNormalizedPath(buildPath(systemConfigDirName, "config")); + + // - What is the full path for the 'business_shared_items' + businessSharedItemsFilePath = buildNormalizedPath(buildPath(configDirName, "business_shared_items")); + + // To determine if any configuration items has changed, where a --resync would be required, we need to have a hash file for the following items + // - 'config.backup' file + // - applicable 'config' file + // - 'sync_list' file + // - 'business_shared_items' file + configBackupFile = buildNormalizedPath(buildPath(configDirName, ".config.backup")); + configHashFile = buildNormalizedPath(buildPath(configDirName, ".config.hash")); + syncListHashFile = buildNormalizedPath(buildPath(configDirName, ".sync_list.hash")); + businessSharedItemsHashFile = buildNormalizedPath(buildPath(configDirName, ".business_shared_items.hash")); + + // Debug Output for application set variables based on configDirName + log.vdebug("refreshTokenFilePath = ", refreshTokenFilePath); + log.vdebug("deltaLinkFilePath = ", deltaLinkFilePath); + log.vdebug("databaseFilePath = ", databaseFilePath); + log.vdebug("databaseFilePathDryRun = ", databaseFilePathDryRun); + log.vdebug("uploadSessionFilePath = ", uploadSessionFilePath); + log.vdebug("userConfigFilePath = ", userConfigFilePath); + log.vdebug("syncListFilePath = ", syncListFilePath); + log.vdebug("systemConfigFilePath = ", systemConfigFilePath); + log.vdebug("configBackupFile = ", configBackupFile); + log.vdebug("configHashFile = ", configHashFile); + log.vdebug("syncListHashFile = ", syncListHashFile); + log.vdebug("businessSharedItemsFilePath = ", businessSharedItemsFilePath); + log.vdebug("businessSharedItemsHashFile = ", businessSharedItemsHashFile); + + // Configure the Hash and Backup File Permission Value + string valueToConvert = to!string(defaultFilePermissionMode); + auto convertedValue = parse!long(valueToConvert, 8); + convertedPermissionValue = to!int(convertedValue); + + // Initialise the application using the configuration file if it exists + if (!exists(userConfigFilePath)) { + // 'user' configuration file does not exist + // Is there a system configuration file? + if (!exists(systemConfigFilePath)) { + // 'system' configuration file does not exist + log.vlog("No user or system config file found, using application defaults"); + applicableConfigFilePath = userConfigFilePath; + configurationInitialised = true; + } else { + // 'system' configuration file exists + // can we load the configuration file without error? + if (loadConfigFile(systemConfigFilePath)) { + // configuration file loaded without error + log.log("System configuration file successfully loaded"); + // Set 'applicableConfigFilePath' to equal the 'config' we loaded + applicableConfigFilePath = systemConfigFilePath; + // Update the configHashFile path value to ensure we are using the system 'config' file for the hash + configHashFile = buildNormalizedPath(buildPath(systemConfigDirName, ".config.hash")); + configurationInitialised = true; + } else { + // there was a problem loading the configuration file + log.log("\nSystem configuration file has errors - please check your configuration"); + } + } + } else { + // 'user' configuration file exists + // can we load the configuration file without error? + if (loadConfigFile(userConfigFilePath)) { + // configuration file loaded without error + log.log("Configuration file successfully loaded"); + // Set 'applicableConfigFilePath' to equal the 'config' we loaded + applicableConfigFilePath = userConfigFilePath; + configurationInitialised = true; + } else { + // there was a problem loading the configuration file + log.log("\nConfiguration file has errors - please check your configuration"); + } + } + + // Advise the user path that we will use for the application state data + if (canFind(applicableConfigFilePath, configDirName)) { + log.vlog("Using 'user' configuration path for application state data: ", configDirName); + } else { + if (canFind(applicableConfigFilePath, systemConfigDirName)) { + log.vlog("Using 'system' configuration path for application state data: ", systemConfigDirName); + } + } + + // return if the configuration was initialised + return configurationInitialised; + } + + // Create a backup of the 'config' file if it does not exist + void createBackupConfigFile() { + if (!getValueBool("dry_run")) { + // Is there a backup of the config file if the config file exists? + if (exists(applicableConfigFilePath)) { + log.vdebug("Creating a backup of the applicable config file"); + // create backup copy of current config file + std.file.copy(applicableConfigFilePath, configBackupFile); + // File Copy should only be readable by the user who created it - 0600 permissions needed + configBackupFile.setAttributes(convertedPermissionValue); + } + } else { + // --dry-run scenario ... technically we should not be making any local file changes ....... + log.log("DRY RUN: Not creating backup config file as --dry-run has been used"); + } + } + + // Return a given string value based on the provided key + string getValueString(string key) { + auto p = key in stringValues; + if (p) { + return *p; + } else { + throw new Exception("Missing config value: " ~ key); + } + } + + // Return a given long value based on the provided key + long getValueLong(string key) { + auto p = key in longValues; + if (p) { + return *p; + } else { + throw new Exception("Missing config value: " ~ key); + } + } + + // Return a given bool value based on the provided key + bool getValueBool(string key) { + auto p = key in boolValues; + if (p) { + return *p; + } else { + throw new Exception("Missing config value: " ~ key); + } + } + + // Set a given string value based on the provided key + void setValueString(string key, string value) { + stringValues[key] = value; + } + + // Set a given long value based on the provided key + void setValueLong(string key, long value) { + longValues[key] = value; + } + + // Set a given long value based on the provided key + void setValueBool(string key, bool value) { + boolValues[key] = value; + } + + // Configure the directory octal permission value + void configureRequiredDirectoryPermisions() { + // return the directory permission mode required + // - return octal!defaultDirectoryPermissionMode; ... cant be used .. which is odd + // Error: variable defaultDirectoryPermissionMode cannot be read at compile time + if (getValueLong("sync_dir_permissions") != defaultDirectoryPermissionMode) { + // return user configured permissions as octal integer + string valueToConvert = to!string(getValueLong("sync_dir_permissions")); + auto convertedValue = parse!long(valueToConvert, 8); + configuredDirectoryPermissionMode = to!int(convertedValue); + } else { + // return default as octal integer + string valueToConvert = to!string(defaultDirectoryPermissionMode); + auto convertedValue = parse!long(valueToConvert, 8); + configuredDirectoryPermissionMode = to!int(convertedValue); + } + } + + // Configure the file octal permission value + void configureRequiredFilePermisions() { + // return the file permission mode required + // - return octal!defaultFilePermissionMode; ... cant be used .. which is odd + // Error: variable defaultFilePermissionMode cannot be read at compile time + if (getValueLong("sync_file_permissions") != defaultFilePermissionMode) { + // return user configured permissions as octal integer + string valueToConvert = to!string(getValueLong("sync_file_permissions")); + auto convertedValue = parse!long(valueToConvert, 8); + configuredFilePermissionMode = to!int(convertedValue); + } else { + // return default as octal integer + string valueToConvert = to!string(defaultFilePermissionMode); + auto convertedValue = parse!long(valueToConvert, 8); + configuredFilePermissionMode = to!int(convertedValue); + } + } + + // Read the configuredDirectoryPermissionMode and return + int returnRequiredDirectoryPermisions() { + if (configuredDirectoryPermissionMode == 0) { + // the configured value is zero, this means that directories would get + // values of d--------- + configureRequiredDirectoryPermisions(); + } + return configuredDirectoryPermissionMode; + } + + // Read the configuredFilePermissionMode and return + int returnRequiredFilePermisions() { + if (configuredFilePermissionMode == 0) { + // the configured value is zero + configureRequiredFilePermisions(); + } + return configuredFilePermissionMode; + } + + // Load a configuration file from the provided filename + private bool loadConfigFile(string filename) { + // configure function variables + try { + log.log("Reading configuration file: ", filename); + readText(filename); + } catch (std.file.FileException e) { + // Unable to access required file + log.error("ERROR: Unable to access ", e.msg); + // Use exit scopes to shutdown API + return false; + } + + // We were able to readText the config file - so, we should be able to open and read it + auto file = File(filename, "r"); + string lineBuffer; + + // configure scopes + // - failure + scope(failure) { + // close file if open + if (file.isOpen()){ + // close open file + file.close(); + } + } + // - exit + scope(exit) { + // close file if open + if (file.isOpen()){ + // close open file + file.close(); + } + } + + // read file line by line + auto range = file.byLine(); + foreach (line; range) { + lineBuffer = stripLeft(line).to!string; + if (lineBuffer.length == 0 || lineBuffer[0] == ';' || lineBuffer[0] == '#') continue; + auto c = lineBuffer.matchFirst(configRegex); + if (!c.empty) { + c.popFront(); // skip the whole match + string key = c.front.dup; + auto p = key in boolValues; + if (p) { + c.popFront(); + // only accept "true" as true value. TODO Should we support other formats? + setValueBool(key, c.front.dup == "true" ? true : false); + + // skip_dotfiles tracking for change + if (key == "skip_dotfiles") { + configFileSkipDotfiles = true; + } + + // skip_symlinks tracking for change + if (key == "skip_symlinks") { + configFileSkipSymbolicLinks = true; + } + + // sync_business_shared_items tracking for change + if (key == "sync_business_shared_items") { + configFileSyncBusinessSharedItems = true; + } + + } else { + auto pp = key in stringValues; + if (pp) { + c.popFront(); + setValueString(key, c.front.dup); + // detect need for --resync for these: + // --syncdir ARG + // --skip-file ARG + // --skip-dir ARG + + // sync_dir + if (key == "sync_dir") { + // configure a temp variable + string tempSyncDirValue = c.front.dup; + // is this empty ? + if (!strip(tempSyncDirValue).empty) { + configFileSyncDir = tempSyncDirValue; + } else { + // sync_dir cannot be empty + log.error("Invalid value for key in config file: ", key); + log.error("sync_dir in config file cannot be empty - this is a fatal error and must be corrected"); + exit(EXIT_FAILURE); + } + } + + // skip_file + if (key == "skip_file") { + // Handle multiple entries of skip_file + if (configFileSkipFile.empty) { + // currently no entry exists + configFileSkipFile = c.front.dup; + } else { + // add to existing entry + configFileSkipFile = configFileSkipFile ~ "|" ~ to!string(c.front.dup); + setValueString("skip_file", configFileSkipFile); + } + } + + // skip_dir + if (key == "skip_dir") { + // Handle multiple entries of skip_dir + if (configFileSkipDir.empty) { + // currently no entry exists + configFileSkipDir = c.front.dup; + } else { + // add to existing entry + configFileSkipDir = configFileSkipDir ~ "|" ~ to!string(c.front.dup); + setValueString("skip_dir", configFileSkipDir); + } + } + + // --single-directory Strip quotation marks from path + // This is an issue when using ONEDRIVE_SINGLE_DIRECTORY with Docker + if (key == "single_directory") { + // Strip quotation marks from provided path + string configSingleDirectory = strip(to!string(c.front.dup), "\""); + setValueString("single_directory", configSingleDirectory); + } + + // Azure AD Configuration + if (key == "azure_ad_endpoint") { + string azureConfigValue = strip(c.front.dup); + switch(azureConfigValue) { + case "": + log.log("Using config option for Global Azure AD Endpoints"); + break; + case "USL4": + log.log("Using config option for Azure AD for US Government Endpoints"); + break; + case "USL5": + log.log("Using config option for Azure AD for US Government Endpoints (DOD)"); + break; + case "DE": + log.log("Using config option for Azure AD Germany"); + break; + case "CN": + log.log("Using config option for Azure AD China operated by 21Vianet"); + break; + // Default - all other entries + default: + log.log("Unknown Azure AD Endpoint - using Global Azure AD Endpoints"); + } + } + + // Application ID + if (key == "application_id") { + // This key cannot be empty + string tempApplicationId = strip(c.front.dup); + if (tempApplicationId.empty) { + log.log("Invalid value for key in config file - using default value: ", key); + log.vdebug("application_id in config file cannot be empty - using default application_id"); + setValueString("application_id", defaultApplicationId); + } else { + setValueString("application_id", tempApplicationId); + } + } + + // Drive ID + if (key == "drive_id") { + // This key cannot be empty + string tempApplicationId = strip(c.front.dup); + if (tempApplicationId.empty) { + log.error("Invalid value for key in config file: ", key); + log.error("drive_id in config file cannot be empty - this is a fatal error and must be corrected"); + exit(EXIT_FAILURE); + } else { + setValueString("drive_id", tempApplicationId); + configFileDriveId = tempApplicationId; + } + } + + // Log Directory + if (key == "log_dir") { + // This key cannot be empty + string tempLogDir = strip(c.front.dup); + if (tempLogDir.empty) { + log.log("Invalid value for key in config file - using default value: ", key); + log.vdebug("log_dir in config file cannot be empty - using default log_dir"); + setValueString("log_dir", defaultLogFileDir); + } else { + setValueString("log_dir", tempLogDir); + } + } + + } else { + auto ppp = key in longValues; + if (ppp) { + c.popFront(); + ulong thisConfigValue; + + // Can this value actually be converted to an integer? + try { + thisConfigValue = to!long(c.front.dup); + } catch (std.conv.ConvException) { + log.log("Invalid value for key in config file: ", key); + return false; + } + + setValueLong(key, thisConfigValue); + + // if key is 'monitor_interval' the value must be 300 or greater + if (key == "monitor_interval") { + // temp value + ulong tempValue = thisConfigValue; + // the temp value needs to be greater than 300 + if (tempValue < 300) { + log.log("Invalid value for key in config file - using default value: ", key); + tempValue = 300; + } + setValueLong("monitor_interval", to!long(tempValue)); + } + + // if key is 'monitor_fullscan_frequency' the value must be 12 or greater + if (key == "monitor_fullscan_frequency") { + // temp value + ulong tempValue = thisConfigValue; + // the temp value needs to be greater than 12 + if (tempValue < 12) { + log.log("Invalid value for key in config file - using default value: ", key); + tempValue = 12; + } + setValueLong("monitor_fullscan_frequency", to!long(tempValue)); + } + + // if key is 'space_reservation' we have to calculate MB -> bytes + if (key == "space_reservation") { + // temp value + ulong tempValue = thisConfigValue; + // a value of 0 needs to be made at least 1MB .. + if (tempValue == 0) { + log.log("Invalid value for key in config file - using 1MB: ", key); + tempValue = 1; + } + setValueLong("space_reservation", to!long(tempValue * 2^^20)); + } + + // if key is 'ip_protocol_version' this has to be a value of 0 or 1 or 2 .. nothing else + if (key == "ip_protocol_version") { + // temp value + ulong tempValue = thisConfigValue; + // If greater than 2, set to default + if (tempValue > 2) { + log.log("Invalid value for key in config file - using default value: ", key); + // Set to default of 0 + tempValue = 0; + } + setValueLong("ip_protocol_version", to!long(tempValue)); + } + + } else { + log.log("Unknown key in config file: ", key); + + // Application configuration update required for Business Shared Folders + if (key == "sync_business_shared_folders") { + log.log("\nThe method to sync Business Shared Folders has changed. Please read the updated documentation and update your client configuration."); + } + // Return false + return false; + } + } + } + } else { + log.log("Malformed config line: ", lineBuffer); + return false; + } + } + + // Close the file access + file.close(); + // Free object and memory + object.destroy(file); + object.destroy(range); + return true; + } + + // Update the application configuration based on CLI passed in parameters + void updateFromArgs(string[] cliArgs) { + // Add additional options that are NOT configurable via config file + stringValues["create_directory"] = ""; + stringValues["create_share_link"] = ""; + stringValues["destination_directory"] = ""; + stringValues["get_file_link"] = ""; + stringValues["modified_by"] = ""; + stringValues["sharepoint_library_name"] = ""; + stringValues["remove_directory"] = ""; + stringValues["single_directory"] = ""; + stringValues["source_directory"] = ""; + stringValues["auth_files"] = ""; + stringValues["auth_response"] = ""; + boolValues["display_config"] = false; + boolValues["display_sync_status"] = false; + boolValues["print_token"] = false; + boolValues["logout"] = false; + boolValues["reauth"] = false; + boolValues["monitor"] = false; + boolValues["synchronize"] = false; + boolValues["force"] = false; + boolValues["list_business_shared_items"] = false; + boolValues["force_sync"] = false; + boolValues["with_editing_perms"] = false; + + // Application Startup option validation + try { + string tmpStr; + bool tmpBol; + long tmpVerb; + // duplicated from main.d to get full help output! + auto opt = getopt( + + cliArgs, + std.getopt.config.bundling, + std.getopt.config.caseSensitive, + "auth-files", + "Perform authentication not via interactive dialog but via files read/writes to these files.", + &stringValues["auth_files"], + "auth-response", + "Perform authentication not via interactive dialog but via providing the response url directly.", + &stringValues["auth_response"], + "check-for-nomount", + "Check for the presence of .nosync in the syncdir root. If found, do not perform sync.", + &boolValues["check_nomount"], + "check-for-nosync", + "Check for the presence of .nosync in each directory. If found, skip directory from sync.", + &boolValues["check_nosync"], + "classify-as-big-delete", + "Number of children in a path that is locally removed which will be classified as a 'big data delete'", + &longValues["classify_as_big_delete"], + "cleanup-local-files", + "Cleanup additional local files when using --download-only. This will remove local data.", + &boolValues["cleanup_local_files"], + "create-directory", + "Create a directory on OneDrive - no sync will be performed.", + &stringValues["create_directory"], + "create-share-link", + "Create a shareable link for an existing file on OneDrive", + &stringValues["create_share_link"], + "debug-https", + "Debug OneDrive HTTPS communication.", + &boolValues["debug_https"], + "destination-directory", + "Destination directory for renamed or move on OneDrive - no sync will be performed.", + &stringValues["destination_directory"], + "disable-notifications", + "Do not use desktop notifications in monitor mode.", + &boolValues["disable_notifications"], + "disable-download-validation", + "Disable download validation when downloading from OneDrive", + &boolValues["disable_download_validation"], + "disable-upload-validation", + "Disable upload validation when uploading to OneDrive", + &boolValues["disable_upload_validation"], + "display-config", + "Display what options the client will use as currently configured - no sync will be performed.", + &boolValues["display_config"], + "display-running-config", + "Display what options the client has been configured to use on application startup.", + &boolValues["display_running_config"], + "display-sync-status", + "Display the sync status of the client - no sync will be performed.", + &boolValues["display_sync_status"], + "download-only", + "Replicate the OneDrive online state locally, by only downloading changes from OneDrive. Do not upload local changes to OneDrive.", + &boolValues["download_only"], + "dry-run", + "Perform a trial sync with no changes made", + &boolValues["dry_run"], + "enable-logging", + "Enable client activity to a separate log file", + &boolValues["enable_logging"], + "force-http-11", + "Force the use of HTTP 1.1 for all operations", + &boolValues["force_http_11"], + "force", + "Force the deletion of data when a 'big delete' is detected", + &boolValues["force"], + "force-sync", + "Force a synchronization of a specific folder, only when using --sync --single-directory and ignore all non-default skip_dir and skip_file rules", + &boolValues["force_sync"], + "get-file-link", + "Display the file link of a synced file", + &stringValues["get_file_link"], + "get-sharepoint-drive-id", + "Query and return the Office 365 Drive ID for a given Office 365 SharePoint Shared Library", + &stringValues["sharepoint_library_name"], + "get-O365-drive-id", + "Query and return the Office 365 Drive ID for a given Office 365 SharePoint Shared Library (DEPRECIATED)", + &stringValues["sharepoint_library_name"], + "local-first", + "Synchronize from the local directory source first, before downloading changes from OneDrive.", + &boolValues["local_first"], + "log-dir", + "Directory where logging output is saved to, needs to end with a slash.", + &stringValues["log_dir"], + "logout", + "Logout the current user", + &boolValues["logout"], + "modified-by", + "Display the last modified by details of a given path", + &stringValues["modified_by"], + "monitor|m", + "Keep monitoring for local and remote changes", + &boolValues["monitor"], + "monitor-interval", + "Number of seconds by which each sync operation is undertaken when idle under monitor mode.", + &longValues["monitor_interval"], + "monitor-fullscan-frequency", + "Number of sync runs before performing a full local scan of the synced directory", + &longValues["monitor_fullscan_frequency"], + "monitor-log-frequency", + "Frequency of logging in monitor mode", + &longValues["monitor_log_frequency"], + "no-remote-delete", + "Do not delete local file 'deletes' from OneDrive when using --upload-only", + &boolValues["no_remote_delete"], + "print-access-token", + "Print the access token, useful for debugging", + &boolValues["print_token"], + "reauth", + "Reauthenticate the client with OneDrive", + &boolValues["reauth"], + "resync", + "Forget the last saved state, perform a full sync", + &boolValues["resync"], + "resync-auth", + "Approve the use of performing a --resync action", + &boolValues["resync_auth"], + "remove-directory", + "Remove a directory on OneDrive - no sync will be performed.", + &stringValues["remove_directory"], + "remove-source-files", + "Remove source file after successful transfer to OneDrive when using --upload-only", + &boolValues["remove_source_files"], + "single-directory", + "Specify a single local directory within the OneDrive root to sync.", + &stringValues["single_directory"], + "skip-dot-files", + "Skip dot files and folders from syncing", + &boolValues["skip_dotfiles"], + "skip-file", + "Skip any files that match this pattern from syncing", + &stringValues["skip_file"], + "skip-dir", + "Skip any directories that match this pattern from syncing", + &stringValues["skip_dir"], + "skip-size", + "Skip new files larger than this size (in MB)", + &longValues["skip_size"], + "skip-dir-strict-match", + "When matching skip_dir directories, only match explicit matches", + &boolValues["skip_dir_strict_match"], + "skip-symlinks", + "Skip syncing of symlinks", + &boolValues["skip_symlinks"], + "source-directory", + "Source directory to rename or move on OneDrive - no sync will be performed.", + &stringValues["source_directory"], + "space-reservation", + "The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation", + &longValues["space_reservation"], + "syncdir", + "Specify the local directory used for synchronization to OneDrive", + &stringValues["sync_dir"], + "sync|s", + "Perform a synchronisation with Microsoft OneDrive", + &boolValues["synchronize"], + "synchronize", + "Perform a synchronisation with Microsoft OneDrive (DEPRECIATED)", + &boolValues["synchronize"], + "sync-root-files", + "Sync all files in sync_dir root when using sync_list.", + &boolValues["sync_root_files"], + "upload-only", + "Replicate the locally configured sync_dir state to OneDrive, by only uploading local changes to OneDrive. Do not download changes from OneDrive.", + &boolValues["upload_only"], + "confdir", + "Set the directory used to store the configuration files", + &tmpStr, + "verbose|v+", + "Print more details, useful for debugging (repeat for extra debugging)", + &tmpVerb, + "version", + "Print the version and exit", + &tmpBol, + "with-editing-perms", + "Create a read-write shareable link for an existing file on OneDrive when used with --create-share-link ", + &boolValues["with_editing_perms"] + ); + if (opt.helpWanted) { + outputLongHelp(opt.options); + exit(EXIT_SUCCESS); + } + } catch (GetOptException e) { + log.error(e.msg); + log.error("Try 'onedrive -h' for more information"); + exit(EXIT_FAILURE); + } catch (Exception e) { + // error + log.error(e.msg); + log.error("Try 'onedrive -h' for more information"); + exit(EXIT_FAILURE); + } + } + + // Check the arguments passed in for any that will be depreciated + void checkDepreciatedOptions(string[] cliArgs) { + + bool depreciatedCommandsFound = false; + + foreach (cliArg; cliArgs) { + // Check each CLI arg for items that have been depreciated + + // --synchronize depreciated in v2.5.0, will be removed in future version + if (cliArg == "--synchronize") { + writeln(); + log.error("DEPRECIATION WARNING: --synchronize has been depreciated in favour of --sync or -s"); + depreciatedCommandsFound = true; + } + + // --get-O365-drive-id depreciated in v2.5.0, will be removed in future version + if (cliArg == "--get-O365-drive-id") { + writeln(); + log.error("DEPRECIATION WARNING: --get-O365-drive-id has been depreciated in favour of --get-sharepoint-drive-id"); + depreciatedCommandsFound = true; + } + + } + + if (depreciatedCommandsFound) { + log.error("DEPRECIATION WARNING: Depreciated commands will be removed in a future release."); + writeln(); + } + } + + // Display the applicable application configuration + void displayApplicationConfiguration() { + if (getValueBool("display_running_config")) { + writeln("--------------- Application Runtime Configuration ---------------"); + } + + // Display application version + //writeln("onedrive version = ", strip(import("version"))); + + string tempVersion = "v2.5.0-alpha-2" ~ " GitHub version: " ~ strip(import("version")); + writeln("onedrive version = ", tempVersion); + + // Display all of the pertinent configuration options + writeln("Config path = ", configDirName); + // Does a config file exist or are we using application defaults + writeln("Config file found in config path = ", exists(applicableConfigFilePath)); + + // Is config option drive_id configured? + writeln("Config option 'drive_id' = ", getValueString("drive_id")); + + // Config Options as per 'config' file + writeln("Config option 'sync_dir' = ", getValueString("sync_dir")); + + // logging and notifications + writeln("Config option 'enable_logging' = ", getValueBool("enable_logging")); + writeln("Config option 'log_dir' = ", getValueString("log_dir")); + writeln("Config option 'disable_notifications' = ", getValueBool("disable_notifications")); + + // skip files and directory and 'matching' policy + writeln("Config option 'skip_dir' = ", getValueString("skip_dir")); + writeln("Config option 'skip_dir_strict_match' = ", getValueBool("skip_dir_strict_match")); + writeln("Config option 'skip_file' = ", getValueString("skip_file")); + writeln("Config option 'skip_dotfiles' = ", getValueBool("skip_dotfiles")); + writeln("Config option 'skip_symlinks' = ", getValueBool("skip_symlinks")); + + // --monitor sync process options + writeln("Config option 'monitor_interval' = ", getValueLong("monitor_interval")); + writeln("Config option 'monitor_log_frequency' = ", getValueLong("monitor_log_frequency")); + writeln("Config option 'monitor_fullscan_frequency' = ", getValueLong("monitor_fullscan_frequency")); + + // sync process and method + writeln("Config option 'read_only_auth_scope' = ", getValueBool("read_only_auth_scope")); + writeln("Config option 'dry_run' = ", getValueBool("dry_run")); + writeln("Config option 'upload_only' = ", getValueBool("upload_only")); + writeln("Config option 'download_only' = ", getValueBool("download_only")); + writeln("Config option 'local_first' = ", getValueBool("local_first")); + writeln("Config option 'check_nosync' = ", getValueBool("check_nosync")); + writeln("Config option 'check_nomount' = ", getValueBool("check_nomount")); + writeln("Config option 'resync' = ", getValueBool("resync")); + writeln("Config option 'resync_auth' = ", getValueBool("resync_auth")); + writeln("Config option 'cleanup_local_files' = ", getValueBool("cleanup_local_files")); + + // data integrity + writeln("Config option 'classify_as_big_delete' = ", getValueLong("classify_as_big_delete")); + writeln("Config option 'disable_upload_validation' = ", getValueBool("disable_upload_validation")); + writeln("Config option 'disable_download_validation' = ", getValueBool("disable_download_validation")); + writeln("Config option 'bypass_data_preservation' = ", getValueBool("bypass_data_preservation")); + writeln("Config option 'no_remote_delete' = ", getValueBool("no_remote_delete")); + writeln("Config option 'remove_source_files' = ", getValueBool("remove_source_files")); + writeln("Config option 'sync_dir_permissions' = ", getValueLong("sync_dir_permissions")); + writeln("Config option 'sync_file_permissions' = ", getValueLong("sync_file_permissions")); + writeln("Config option 'space_reservation' = ", getValueLong("space_reservation")); + + // curl operations + writeln("Config option 'application_id' = ", getValueString("application_id")); + writeln("Config option 'azure_ad_endpoint' = ", getValueString("azure_ad_endpoint")); + writeln("Config option 'azure_tenant_id' = ", getValueString("azure_tenant_id")); + writeln("Config option 'user_agent' = ", getValueString("user_agent")); + writeln("Config option 'force_http_11' = ", getValueBool("force_http_11")); + writeln("Config option 'debug_https' = ", getValueBool("debug_https")); + writeln("Config option 'rate_limit' = ", getValueLong("rate_limit")); + writeln("Config option 'operation_timeout' = ", getValueLong("operation_timeout")); + writeln("Config option 'dns_timeout' = ", getValueLong("dns_timeout")); + writeln("Config option 'connect_timeout' = ", getValueLong("connect_timeout")); + writeln("Config option 'data_timeout' = ", getValueLong("data_timeout")); + writeln("Config option 'ip_protocol_version' = ", getValueLong("ip_protocol_version")); + + // Is sync_list configured ? + writeln("\nConfig option 'sync_root_files' = ", getValueBool("sync_root_files")); + if (exists(syncListFilePath)){ + + writeln("Selective sync 'sync_list' configured = true"); + writeln("sync_list contents:"); + // Output the sync_list contents + auto syncListFile = File(syncListFilePath, "r"); + auto range = syncListFile.byLine(); + foreach (line; range) + { + writeln(line); + } + } else { + writeln("Selective sync 'sync_list' configured = false"); + + } + + // Is sync_business_shared_items enabled and configured ? + writeln("\nConfig option 'sync_business_shared_items' = ", getValueBool("sync_business_shared_items")); + + if (exists(businessSharedItemsFilePath)){ + writeln("Selective Business Shared Items configured = true"); + writeln("sync_business_shared_items contents:"); + // Output the sync_business_shared_items contents + auto businessSharedItemsFileList = File(businessSharedItemsFilePath, "r"); + auto range = businessSharedItemsFileList.byLine(); + foreach (line; range) + { + writeln(line); + } + } else { + writeln("Selective Business Shared Items configured = false"); + } + + // Are webhooks enabled? + writeln("\nConfig option 'webhook_enabled' = ", getValueBool("webhook_enabled")); + if (getValueBool("webhook_enabled")) { + writeln("Config option 'webhook_public_url' = ", getValueString("webhook_public_url")); + writeln("Config option 'webhook_listening_host' = ", getValueString("webhook_listening_host")); + writeln("Config option 'webhook_listening_port' = ", getValueLong("webhook_listening_port")); + writeln("Config option 'webhook_expiration_interval' = ", getValueLong("webhook_expiration_interval")); + writeln("Config option 'webhook_renewal_interval' = ", getValueLong("webhook_renewal_interval")); + } + + if (getValueBool("display_running_config")) { + writeln("-----------------------------------------------------------------"); + } + } + + // Prompt the user to accept the risk of using --resync + bool displayResyncRiskForAcceptance() { + // what is the user risk acceptance? + bool userRiskAcceptance = false; + + // Did the user use --resync-auth or 'resync_auth' in the config file to negate presenting this message? + if (!getValueBool("resync_auth")) { + // need to prompt user + char response; + + // --resync warning message + writeln("\nThe usage of --resync will delete your local 'onedrive' client state, thus no record of your current 'sync status' will exist."); + writeln("This has the potential to overwrite local versions of files with perhaps older versions of documents downloaded from OneDrive, resulting in local data loss."); + writeln("If in doubt, backup your local data before using --resync"); + write("\nAre you sure you wish to proceed with --resync? [Y/N] "); + + try { + // Attempt to read user response + string input = readln().strip; + if (input.length > 0) { + response = std.ascii.toUpper(input[0]); + } + } catch (std.format.FormatException e) { + userRiskAcceptance = false; + // Caught an error + return EXIT_FAILURE; + } + + // What did the user enter? + log.vdebug("--resync warning User Response Entered: ", (to!string(response))); + + // Evaluate user repsonse + if ((to!string(response) == "y") || (to!string(response) == "Y")) { + // User has accepted --resync risk to proceed + userRiskAcceptance = true; + // Are you sure you wish .. does not use writeln(); + write("\n"); + } + } else { + // resync_auth is true + userRiskAcceptance = true; + } + + // Return the --resync acceptance or not + return userRiskAcceptance; + } + + // Prompt the user to accept the risk of using --force-sync + bool displayForceSyncRiskForAcceptance() { + // what is the user risk acceptance? + bool userRiskAcceptance = false; + + // need to prompt user + char response; + + // --force-sync warning message + writeln("\nThe use of --force-sync will reconfigure the application to use defaults. This may have untold and unknown future impacts."); + writeln("By proceeding in using this option you accept any impacts including any data loss that may occur as a result of using --force-sync."); + write("\nAre you sure you wish to proceed with --force-sync [Y/N] "); + + try { + // Attempt to read user response + string input = readln().strip; + if (input.length > 0) { + response = std.ascii.toUpper(input[0]); + } + } catch (std.format.FormatException e) { + userRiskAcceptance = false; + // Caught an error + return EXIT_FAILURE; + } + + // What did the user enter? + log.vdebug("--force-sync warning User Response Entered: ", (to!string(response))); + + // Evaluate user repsonse + if ((to!string(response) == "y") || (to!string(response) == "Y")) { + // User has accepted --force-sync risk to proceed + userRiskAcceptance = true; + // Are you sure you wish .. does not use writeln(); + write("\n"); + } + + // Return the --resync acceptance or not + return userRiskAcceptance; + } + + // Check the application configuration for any changes that need to trigger a --resync + // This function is only called if --resync is not present + bool applicationChangeWhereResyncRequired() { + // Default is that no resync is required + bool resyncRequired = false; + + // Configuration File Flags + bool configFileOptionsDifferent = false; + bool syncListFileDifferent = false; + bool syncDirDifferent = false; + bool skipFileDifferent = false; + bool skipDirDifferent = false; + bool skipDotFilesDifferent = false; + bool skipSymbolicLinksDifferent = false; + bool driveIdDifferent = false; + bool syncBusinessSharedItemsDifferent = false; + bool businessSharedItemsFileDifferent = false; + + // Create the required initial hash files + createRequiredInitialConfigurationHashFiles(); + + // Read in the existing hash file values + readExistingConfigurationHashFiles(); + + // Was the 'sync_list' file updated? + if (currentSyncListHash != previousSyncListHash) { + // Debugging output to assist what changed + log.vdebug("sync_list file has been updated, --resync needed"); + syncListFileDifferent = true; + } + + // Was the 'business_shared_items' file updated? + if (currentBusinessSharedItemsHash != previousBusinessSharedItemsHash) { + // Debugging output to assist what changed + log.vdebug("business_shared_folders file has been updated, --resync needed"); + businessSharedItemsFileDifferent = true; + } + + // Was the 'config' file updated between last execution and this execution? + if (currentConfigHash != previousConfigHash) { + // config file was updated, however we only want to trigger a --resync requirement if sync_dir, skip_dir, skip_file or drive_id was modified + log.log("Application configuration file has been updated, checking if --resync needed"); + log.vdebug("Using this configBackupFile: ", configBackupFile); + + if (exists(configBackupFile)) { + // check backup config what has changed for these configuration options if anything + // # drive_id = "" + // # sync_dir = "~/OneDrive" + // # skip_file = "~*|.~*|*.tmp|*.swp|*.partial" + // # skip_dir = "" + // # skip_dotfiles = "" + // # skip_symlinks = "" + // # sync_business_shared_items = "" + string[string] backupConfigStringValues; + backupConfigStringValues["drive_id"] = ""; + backupConfigStringValues["sync_dir"] = ""; + backupConfigStringValues["skip_file"] = ""; + backupConfigStringValues["skip_dir"] = ""; + backupConfigStringValues["skip_dotfiles"] = ""; + backupConfigStringValues["skip_symlinks"] = ""; + backupConfigStringValues["sync_business_shared_items"] = ""; + + // bool flags to trigger if the entries that trigger a --resync were found in the backup config file + // if these were not in the backup file, they may have been added ... thus new, thus we need to double check the existing + // config file to see if this was a newly added config option + bool drive_id_present = false; + bool sync_dir_present = false; + bool skip_file_present = false; + bool skip_dir_present = false; + bool skip_dotfiles_present = false; + bool skip_symlinks_present = false; + bool sync_business_shared_items_present = false; + + // Common debug message if an element is different + string configOptionModifiedMessage = " was modified since the last time the application was successfully run, --resync required"; + + auto configBackupFileHandle = File(configBackupFile, "r"); + string lineBuffer; + + // read configBackupFile line by line + auto range = configBackupFileHandle.byLine(); + // for each line + foreach (line; range) { + log.vdebug("Backup Config Line: ", lineBuffer); + lineBuffer = stripLeft(line).to!string; + if (lineBuffer.length == 0 || lineBuffer[0] == ';' || lineBuffer[0] == '#') continue; + auto c = lineBuffer.matchFirst(configRegex); + if (!c.empty) { + c.popFront(); // skip the whole match + string key = c.front.dup; + log.vdebug("Backup Config Key: ", key); + auto p = key in backupConfigStringValues; + if (p) { + c.popFront(); + // compare this key + if (key == "drive_id") { + drive_id_present = true; + if (c.front.dup != getValueString("drive_id")) { + log.vdebug(key, configOptionModifiedMessage); + configFileOptionsDifferent = true; + } + } + + if (key == "sync_dir") { + sync_dir_present = true; + if (c.front.dup != getValueString("sync_dir")) { + log.vdebug(key, configOptionModifiedMessage); + configFileOptionsDifferent = true; + } + } + + if (key == "skip_file") { + skip_file_present = true; + if (c.front.dup != getValueString("skip_file")) { + log.vdebug(key, configOptionModifiedMessage); + configFileOptionsDifferent = true; + } + } + + if (key == "skip_dir") { + skip_dir_present = true; + if (c.front.dup != getValueString("skip_dir")) { + log.vdebug(key, configOptionModifiedMessage); + configFileOptionsDifferent = true; + } + } + + if (key == "skip_dotfiles") { + skip_dotfiles_present = true; + if (c.front.dup != to!string(getValueBool("skip_dotfiles"))) { + log.vdebug(key, configOptionModifiedMessage); + configFileOptionsDifferent = true; + } + } + + if (key == "skip_symlinks") { + skip_symlinks_present = true; + if (c.front.dup != to!string(getValueBool("skip_symlinks"))) { + log.vdebug(key, configOptionModifiedMessage); + configFileOptionsDifferent = true; + } + } + + if (key == "sync_business_shared_items") { + sync_business_shared_items_present = true; + if (c.front.dup != to!string(getValueBool("sync_business_shared_items"))) { + log.vdebug(key, configOptionModifiedMessage); + configFileOptionsDifferent = true; + } + } + } + } + } + + // close file if open + if (configBackupFileHandle.isOpen()) { + // close open file + configBackupFileHandle.close(); + } + + // Were any of the items that trigger a --resync not in the existing backup 'config' file .. thus newly added? + if ((!drive_id_present) || (!sync_dir_present) || (! skip_file_present) || (!skip_dir_present) || (!skip_dotfiles_present) || (!skip_symlinks_present)) { + log.vdebug("drive_id present in config backup: ", drive_id_present); + log.vdebug("sync_dir present in config backup: ", sync_dir_present); + log.vdebug("skip_file present in config backup: ", skip_file_present); + log.vdebug("skip_dir present in config backup: ", skip_dir_present); + log.vdebug("skip_dotfiles present in config backup: ", skip_dotfiles_present); + log.vdebug("skip_symlinks present in config backup: ", skip_symlinks_present); + log.vdebug("sync_business_shared_items present in config backup: ", sync_business_shared_items_present); + + if ((!drive_id_present) && (configFileDriveId != "")) { + writeln("drive_id newly added ... --resync needed"); + configFileOptionsDifferent = true; + driveIdDifferent = true; + } + + if ((!sync_dir_present) && (configFileSyncDir != defaultSyncDir)) { + writeln("sync_dir newly added ... --resync needed"); + configFileOptionsDifferent = true; + syncDirDifferent = true; + } + + if ((!skip_file_present) && (configFileSkipFile != defaultSkipFile)) { + writeln("skip_file newly added ... --resync needed"); + configFileOptionsDifferent = true; + skipFileDifferent = true; + } + + if ((!skip_dir_present) && (configFileSkipDir != "")) { + writeln("skip_dir newly added ... --resync needed"); + configFileOptionsDifferent = true; + skipFileDifferent = true; + } + + if ((!skip_dotfiles_present) && (configFileSkipDotfiles)) { + writeln("skip_dotfiles newly added ... --resync needed"); + configFileOptionsDifferent = true; + skipDotFilesDifferent = true; + } + + if ((!skip_symlinks_present) && (configFileSkipSymbolicLinks)) { + writeln("skip_symlinks newly added ... --resync needed"); + configFileOptionsDifferent = true; + skipSymbolicLinksDifferent = true; + } + + if ((!sync_business_shared_items_present) && (configFileSyncBusinessSharedItems)) { + writeln("sync_business_shared_items newly added ... --resync needed"); + configFileOptionsDifferent = true; + syncBusinessSharedItemsDifferent = true; + } + } + } else { + // no backup to check + log.log("WARNING: no backup config file was found, unable to validate if any changes made"); + } + } + + // config file set options can be changed via CLI input, specifically these will impact sync and a --resync will be needed: + // --syncdir ARG + // --skip-file ARG + // --skip-dir ARG + // --skip-dot-files + // --skip-symlinks + + if (exists(applicableConfigFilePath)) { + // config file exists + // was the sync_dir updated by CLI? + if (configFileSyncDir != "") { + // sync_dir was set in config file + if (configFileSyncDir != getValueString("sync_dir")) { + // config file was set and CLI input changed this + log.vdebug("sync_dir: CLI override of config file option, --resync needed"); + syncDirDifferent = true; + } + } + + // was the skip_file updated by CLI? + if (configFileSkipFile != "") { + // skip_file was set in config file + if (configFileSkipFile != getValueString("skip_file")) { + // config file was set and CLI input changed this + log.vdebug("skip_file: CLI override of config file option, --resync needed"); + skipFileDifferent = true; + } + } + + // was the skip_dir updated by CLI? + if (configFileSkipDir != "") { + // skip_dir was set in config file + if (configFileSkipDir != getValueString("skip_dir")) { + // config file was set and CLI input changed this + log.vdebug("skip_dir: CLI override of config file option, --resync needed"); + skipDirDifferent = true; + } + } + + // was skip_dotfiles updated by --skip-dot-files ? + if (!configFileSkipDotfiles) { + // was not set in config file + if (getValueBool("skip_dotfiles")) { + // --skip-dot-files passed in + log.vdebug("skip_dotfiles: CLI override of config file option, --resync needed"); + skipDotFilesDifferent = true; + } + } + + // was skip_symlinks updated by --skip-symlinks ? + if (!configFileSkipSymbolicLinks) { + // was not set in config file + if (getValueBool("skip_symlinks")) { + // --skip-symlinks passed in + log.vdebug("skip_symlinks: CLI override of config file option, --resync needed"); + skipSymbolicLinksDifferent = true; + } + } + } + + // Did any of the config files or CLI options trigger a --resync requirement? + log.vdebug("configFileOptionsDifferent: ", configFileOptionsDifferent); + // Options + log.vdebug("driveIdDifferent: ", driveIdDifferent); + log.vdebug("syncDirDifferent: ", syncDirDifferent); + log.vdebug("skipFileDifferent: ", skipFileDifferent); + log.vdebug("skipDirDifferent: ", skipDirDifferent); + log.vdebug("skipDotFilesDifferent: ", skipDotFilesDifferent); + log.vdebug("skipSymbolicLinksDifferent: ", skipSymbolicLinksDifferent); + log.vdebug("syncBusinessSharedItemsDifferent: ", syncBusinessSharedItemsDifferent); + // Files + log.vdebug("syncListFileDifferent: ", syncListFileDifferent); + log.vdebug("businessSharedItemsFileDifferent: ", businessSharedItemsFileDifferent); + + if ((configFileOptionsDifferent) || (syncListFileDifferent) || (businessSharedItemsFileDifferent) || (syncDirDifferent) || (skipFileDifferent) || (skipDirDifferent) || (driveIdDifferent) || (skipDotFilesDifferent) || (skipSymbolicLinksDifferent) || (syncBusinessSharedItemsDifferent) ) { + // set the flag + resyncRequired = true; + } + return resyncRequired; + } + + // Cleanup hash files that require to be cleaned up when a --resync is issued + void cleanupHashFilesDueToResync() { + if (!getValueBool("dry_run")) { + // cleanup hash files + log.vdebug("Cleaning up configuration hash files"); + safeRemove(configHashFile); + safeRemove(syncListHashFile); + safeRemove(businessSharedItemsHashFile); + } else { + // --dry-run scenario ... technically we should not be making any local file changes ....... + log.log("DRY RUN: Not removing hash files as --dry-run has been used"); + } + } + + // For each of the config files, update the hash data in the hash files + void updateHashContentsForConfigFiles() { + // Are we in a --dry-run scenario? + if (!getValueBool("dry_run")) { + // Not a dry-run scenario, update the applicable files + // Update applicable 'config' files + if (exists(applicableConfigFilePath)) { + // Update the hash of the applicable config file + log.vdebug("Updating applicable config file hash"); + std.file.write(configHashFile, computeQuickXorHash(applicableConfigFilePath)); + // Hash file should only be readable by the user who created it - 0600 permissions needed + configHashFile.setAttributes(convertedPermissionValue); + } + // Update 'sync_list' files + if (exists(syncListFilePath)) { + // update sync_list hash + log.vdebug("Updating sync_list hash"); + std.file.write(syncListHashFile, computeQuickXorHash(syncListFilePath)); + // Hash file should only be readable by the user who created it - 0600 permissions needed + syncListHashFile.setAttributes(convertedPermissionValue); + } + + + // Update 'update business_shared_items' files + if (exists(businessSharedItemsFilePath)) { + // update business_shared_folders hash + log.vdebug("Updating business_shared_items hash"); + std.file.write(businessSharedItemsHashFile, computeQuickXorHash(businessSharedItemsFilePath)); + // Hash file should only be readable by the user who created it - 0600 permissions needed + businessSharedItemsHashFile.setAttributes(convertedPermissionValue); + } + + } else { + // --dry-run scenario ... technically we should not be making any local file changes ....... + log.log("DRY RUN: Not updating hash files as --dry-run has been used"); + } + } + + // Create any required hash files for files that help us determine if the configuration has changed since last run + void createRequiredInitialConfigurationHashFiles() { + // Does a 'config' file exist with a valid hash file + if (exists(applicableConfigFilePath)) { + if (!exists(configHashFile)) { + // no existing hash file exists + std.file.write(configHashFile, "initial-hash"); + // Hash file should only be readable by the user who created it - 0600 permissions needed + configHashFile.setAttributes(convertedPermissionValue); + } + // Generate the runtime hash for the 'config' file + currentConfigHash = computeQuickXorHash(applicableConfigFilePath); + } + + // Does a 'sync_list' file exist with a valid hash file + if (exists(syncListFilePath)) { + if (!exists(syncListHashFile)) { + // no existing hash file exists + std.file.write(syncListHashFile, "initial-hash"); + // Hash file should only be readable by the user who created it - 0600 permissions needed + syncListHashFile.setAttributes(convertedPermissionValue); + } + // Generate the runtime hash for the 'sync_list' file + currentSyncListHash = computeQuickXorHash(syncListFilePath); + } + + // Does a 'business_shared_items' file exist with a valid hash file + if (exists(businessSharedItemsFilePath)) { + if (!exists(businessSharedItemsHashFile)) { + // no existing hash file exists + std.file.write(businessSharedItemsHashFile, "initial-hash"); + // Hash file should only be readable by the user who created it - 0600 permissions needed + businessSharedItemsHashFile.setAttributes(convertedPermissionValue); + } + // Generate the runtime hash for the 'sync_list' file + currentBusinessSharedItemsHash = computeQuickXorHash(businessSharedItemsFilePath); + } + } + + // Read in the text values of the previous configurations + int readExistingConfigurationHashFiles() { + if (exists(configHashFile)) { + try { + previousConfigHash = readText(configHashFile); + } catch (std.file.FileException e) { + // Unable to access required file + log.error("ERROR: Unable to access ", e.msg); + // Use exit scopes to shutdown API + return EXIT_FAILURE; + } + } + + if (exists(syncListHashFile)) { + try { + previousSyncListHash = readText(syncListHashFile); + } catch (std.file.FileException e) { + // Unable to access required file + log.error("ERROR: Unable to access ", e.msg); + // Use exit scopes to shutdown API + return EXIT_FAILURE; + } + } + if (exists(businessSharedItemsHashFile)) { + try { + previousBusinessSharedItemsHash = readText(businessSharedItemsHashFile); + } catch (std.file.FileException e) { + // Unable to access required file + log.error("ERROR: Unable to access ", e.msg); + // Use exit scopes to shutdown API + return EXIT_FAILURE; + } + } + return 0; + } + + // Check for basic option conflicts - flags that should not be used together and/or flag combinations that conflict with each other + bool checkForBasicOptionConflicts() { + + bool operationalConflictDetected = false; + + // What are the permission that have been set for the application? + // These are relevant for: + // - The ~/OneDrive parent folder or 'sync_dir' configured item + // - Any new folder created under ~/OneDrive or 'sync_dir' + // - Any new file created under ~/OneDrive or 'sync_dir' + // valid permissions are 000 -> 777 - anything else is invalid + if ((getValueLong("sync_dir_permissions") < 0) || (getValueLong("sync_file_permissions") < 0) || (getValueLong("sync_dir_permissions") > 777) || (getValueLong("sync_file_permissions") > 777)) { + log.error("ERROR: Invalid 'User|Group|Other' permissions set within config file. Please check your configuration."); + operationalConflictDetected = true; + } else { + // Debug log output what permissions are being set to + log.vdebug("Configuring default new folder permissions as: ", getValueLong("sync_dir_permissions")); + configureRequiredDirectoryPermisions(); + log.vdebug("Configuring default new file permissions as: ", getValueLong("sync_file_permissions")); + configureRequiredFilePermisions(); + } + + // --upload-only and --download-only cannot be used together + if ((getValueBool("upload_only")) && (getValueBool("download_only"))) { + log.error("ERROR: --upload-only and --download-only cannot be used together. Use one, not both at the same time."); + operationalConflictDetected = true; + } + + // --sync and --monitor cannot be used together + if ((getValueBool("synchronize")) && (getValueBool("monitor"))) { + log.error("ERROR: --sync and --monitor cannot be used together. Use one, not both at the same time."); + operationalConflictDetected = true; + } + + // --no-remote-delete can ONLY be enabled when --upload-only is used + if ((getValueBool("no_remote_delete")) && (!getValueBool("upload_only"))) { + log.error("ERROR: --no-remote-delete can only be used with --upload-only."); + operationalConflictDetected = true; + } + + // --remove-source-files can ONLY be enabled when --upload-only is used + if ((getValueBool("remove_source_files")) && (!getValueBool("upload_only"))) { + log.error("ERROR: --remove-source-files can only be used with --upload-only."); + operationalConflictDetected = true; + } + + // --cleanup-local-files can ONLY be enabled when --download-only is used + if ((getValueBool("cleanup_local_files")) && (!getValueBool("download_only"))) { + log.error("ERROR: --cleanup-local-files can only be used with --download-only."); + operationalConflictDetected = true; + } + + // --list-shared-folders cannot be used with --resync and/or --resync-auth + if ((getValueBool("list_business_shared_items")) && ((getValueBool("resync")) || (getValueBool("resync_auth")))) { + log.error("ERROR: --list-shared-folders cannot be used with --resync or --resync-auth."); + operationalConflictDetected = true; + } + + // --display-sync-status cannot be used with --resync and/or --resync-auth + if ((getValueBool("display_sync_status")) && ((getValueBool("resync")) || (getValueBool("resync_auth")))) { + log.error("ERROR: --display-sync-status cannot be used with --resync or --resync-auth."); + operationalConflictDetected = true; + } + + // --modified-by cannot be used with --resync and/or --resync-auth + if ((!getValueString("modified_by").empty) && ((getValueBool("resync")) || (getValueBool("resync_auth")))) { + log.error("ERROR: --modified-by cannot be used with --resync or --resync-auth."); + operationalConflictDetected = true; + } + + // --get-file-link cannot be used with --resync and/or --resync-auth + if ((!getValueString("get_file_link").empty) && ((getValueBool("resync")) || (getValueBool("resync_auth")))) { + log.error("ERROR: --get-file-link cannot be used with --resync or --resync-auth."); + operationalConflictDetected = true; + } + + // --create-share-link cannot be used with --resync and/or --resync-auth + if ((!getValueString("create_share_link").empty) && ((getValueBool("resync")) || (getValueBool("resync_auth")))) { + log.error("ERROR: --create-share-link cannot be used with --resync or --resync-auth."); + operationalConflictDetected = true; + } + + // --get-O365-drive-id cannot be used with --resync and/or --resync-auth + if ((!getValueString("sharepoint_library_name").empty) && ((getValueBool("resync")) || (getValueBool("resync_auth")))) { + log.error("ERROR: --get-O365-drive-id cannot be used with --resync or --resync-auth."); + operationalConflictDetected = true; + } + + // --monitor and --download-only cannot be used together + if ((getValueBool("monitor")) && (getValueBool("download_only"))) { + log.error("ERROR: --monitor and --download-only cannot be used together."); + operationalConflictDetected = true; + } + + // --force-sync can only be used when using --sync --single-directory + if (getValueBool("force_sync")) { + + bool conflict = false; + // Should not be used with --monitor + if (getValueBool("monitor")) conflict = true; + // single_directory must not be empty + if (getValueString("single_directory").empty) conflict = true; + if (conflict) { + log.error("ERROR: --force-sync can only be used with --sync --single-directory."); + operationalConflictDetected = true; + } + } + + // When using 'azure_ad_endpoint', 'azure_tenant_id' cannot be empty + if ((!getValueString("azure_ad_endpoint").empty) && (getValueString("azure_tenant_id").empty)) { + log.error("ERROR: config option 'azure_tenant_id' cannot be empty when 'azure_ad_endpoint' is configured."); + operationalConflictDetected = true; + } + + // When using --enable-logging the 'log_dir' cannot be empty + if ((getValueBool("enable_logging")) && (getValueString("log_dir").empty)) { + log.error("ERROR: config option 'log_dir' cannot be empty when 'enable_logging' is configured."); + operationalConflictDetected = true; + } + + // When using --syncdir, the value cannot be empty. + if (strip(getValueString("sync_dir")).empty) { + log.error("ERROR: --syncdir value cannot be empty."); + operationalConflictDetected = true; + } + + // Return bool value indicating if we have an operational conflict + return operationalConflictDetected; + } + + // Reset skip_file and skip_dir to application defaults when --force-sync is used + void resetSkipToDefaults() { + // skip_file + log.vdebug("original skip_file: ", getValueString("skip_file")); + log.vdebug("resetting skip_file to application defaults"); + setValueString("skip_file", defaultSkipFile); + log.vdebug("reset skip_file: ", getValueString("skip_file")); + // skip_dir + log.vdebug("original skip_dir: ", getValueString("skip_dir")); + log.vdebug("resetting skip_dir to application defaults"); + setValueString("skip_dir", defaultSkipDir); + log.vdebug("reset skip_dir: ", getValueString("skip_dir")); + } + + // Initialise the correct 'sync_dir' expanding any '~' if present + string initialiseRuntimeSyncDirectory() { + + string runtimeSyncDirectory; + + log.vdebug("sync_dir: Setting runtimeSyncDirectory from config value 'sync_dir'"); + + if (!shellEnvironmentSet){ + log.vdebug("sync_dir: No SHELL or USER environment variable configuration detected"); + // No shell or user set, so expandTilde() will fail - usually headless system running under init.d / systemd or potentially Docker + // Does the 'currently configured' sync_dir include a ~ + if (canFind(getValueString("sync_dir"), "~")) { + // A ~ was found in sync_dir + log.vdebug("sync_dir: A '~' was found in 'sync_dir', using the calculated 'homePath' to replace '~' as no SHELL or USER environment variable set"); + runtimeSyncDirectory = buildNormalizedPath(buildPath(defaultHomePath, strip(getValueString("sync_dir"), "~"))); + } else { + // No ~ found in sync_dir, use as is + log.vdebug("sync_dir: Using configured 'sync_dir' path as-is as no SHELL or USER environment variable configuration detected"); + runtimeSyncDirectory = getValueString("sync_dir"); + } + } else { + // A shell and user environment variable is set, expand any ~ as this will be expanded correctly if present + if (canFind(getValueString("sync_dir"), "~")) { + log.vdebug("sync_dir: A '~' was found in the configured 'sync_dir', automatically expanding as SHELL and USER environment variable is set"); + runtimeSyncDirectory = expandTilde(getValueString("sync_dir")); + } else { + // No ~ found in sync_dir, does the path begin with a '/' ? + log.vdebug("sync_dir: Using configured 'sync_dir' path as-is as however SHELL or USER environment variable configuration detected - should be placed in USER home directory"); + if (!startsWith(getValueString("sync_dir"), "/")) { + log.vdebug("Configured 'sync_dir' does not start with a '/' or '~/' - adjusting configured 'sync_dir' to use User Home Directory as base for 'sync_dir' path"); + string updatedPathWithHome = "~/" ~ getValueString("sync_dir"); + runtimeSyncDirectory = expandTilde(updatedPathWithHome); + } else { + log.vdebug("use 'sync_dir' as is - no touch"); + runtimeSyncDirectory = getValueString("sync_dir"); + } + } + } + + // What will runtimeSyncDirectory be actually set to? + log.vdebug("runtimeSyncDirectory set to: ", runtimeSyncDirectory); + return runtimeSyncDirectory; + } + + // Initialise the correct 'log_dir' when application logging to a separate file is enabled with 'enable_logging' and expanding any '~' if present + string initialiseLogDirectory() { + + string initialisedLogDirPath; + + log.vdebug("log_dir: Setting runtime application log from config value 'log_dir'"); + + if (getValueString("log_dir") != defaultLogFileDir) { + // User modified 'log_dir' to be used with 'enable_logging' + // if 'log_dir' contains a '~' this needs to be expanded correctly + if (canFind(getValueString("log_dir"), "~")) { + // ~ needs to be expanded correctly + if (!shellEnvironmentSet) { + // No shell or user environment variable set, so expandTilde() will fail - usually headless system running under init.d / systemd or potentially Docker + log.vdebug("log_dir: A '~' was found in log_dir, using the calculated 'homePath' to replace '~' as no SHELL or USER environment variable set"); + initialisedLogDirPath = buildNormalizedPath(buildPath(defaultHomePath, strip(getValueString("log_dir"), "~"))); + } else { + // A shell and user environment variable is set, expand any ~ as this will be expanded correctly if present + log.vdebug("log_dir: A '~' was found in the configured 'log_dir', automatically expanding as SHELL and USER environment variable is set"); + initialisedLogDirPath = expandTilde(getValueString("log_dir")); + } + } else { + // '~' not found in log_dir entry, use as is + initialisedLogDirPath = getValueString("log_dir"); + } + } else { + // Default 'log_dir' to be used with 'enable_logging' + initialisedLogDirPath = defaultLogFileDir; + } + + // Return the initialised application log path + return initialisedLogDirPath; + } +} + +// Output the full application help when --help is passed in +void outputLongHelp(Option[] opt) { + auto argsNeedingOptions = [ + "--auth-files", + "--auth-response", + "--confdir", + "--create-directory", + "--classify-as-big-delete", + "--create-share-link", + "--destination-directory", + "--get-file-link", + "--get-O365-drive-id", + "--log-dir", + "--min-notify-changes", + "--modified-by", + "--monitor-interval", + "--monitor-log-frequency", + "--monitor-fullscan-frequency", + "--operation-timeout", + "--remove-directory", + "--single-directory", + "--skip-dir", + "--skip-file", + "--skip-size", + "--source-directory", + "--space-reservation", + "--syncdir", + "--user-agent" ]; + writeln(`OneDrive - a client for OneDrive Cloud Services + + Usage: + onedrive [options] --sync + Do a one time synchronization + onedrive [options] --monitor + Monitor filesystem and sync regularly + onedrive [options] --display-config + Display the currently used configuration + onedrive [options] --display-sync-status + Query OneDrive service and report on pending changes + onedrive -h | --help + Show this help screen + onedrive --version + Show version + + Options: + `); + foreach (it; opt.sort!("a.optLong < b.optLong")) { + writefln(" %s%s%s%s\n %s", + it.optLong, + it.optShort == "" ? "" : " " ~ it.optShort, + argsNeedingOptions.canFind(it.optLong) ? " ARG" : "", + it.required ? " (required)" : "", it.help); + } +} \ No newline at end of file diff --git a/src/curlEngine.d b/src/curlEngine.d new file mode 100644 index 00000000..c56d1efc --- /dev/null +++ b/src/curlEngine.d @@ -0,0 +1,111 @@ +// What is this module called? +module curlEngine; + +// What does this module require to function? +import std.net.curl; +import etc.c.curl: CurlOption; +import std.datetime; + +// What other modules that we have created do we need to import? +import log; + +import std.stdio; + +class CurlEngine { + HTTP http; + + this() { + http = HTTP(); + } + + void initialise(long dnsTimeout, long connectTimeout, long dataTimeout, long operationTimeout, int maxRedirects, bool httpsDebug, string userAgent, bool httpProtocol, long userRateLimit, long protocolVersion) { + // Curl Timeout Handling + + // libcurl dns_cache_timeout timeout + // https://curl.se/libcurl/c/CURLOPT_DNS_CACHE_TIMEOUT.html + // https://dlang.org/library/std/net/curl/http.dns_timeout.html + http.dnsTimeout = (dur!"seconds"(dnsTimeout)); + + // Timeout for HTTPS connections + // https://curl.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html + // https://dlang.org/library/std/net/curl/http.connect_timeout.html + http.connectTimeout = (dur!"seconds"(connectTimeout)); + + // Timeout for activity on connection + // This is a DMD | DLANG specific item, not a libcurl item + // https://dlang.org/library/std/net/curl/http.data_timeout.html + // https://raw.githubusercontent.com/dlang/phobos/master/std/net/curl.d - private enum _defaultDataTimeout = dur!"minutes"(2); + http.dataTimeout = (dur!"seconds"(dataTimeout)); + + // Maximum time any operation is allowed to take + // This includes dns resolution, connecting, data transfer, etc. + // https://curl.se/libcurl/c/CURLOPT_TIMEOUT_MS.html + // https://dlang.org/library/std/net/curl/http.operation_timeout.html + http.operationTimeout = (dur!"seconds"(operationTimeout)); + + // Specify how many redirects should be allowed + http.maxRedirects(maxRedirects); + // Debug HTTPS + http.verbose = httpsDebug; + // Use the configured 'user_agent' value + http.setUserAgent = userAgent; + // What IP protocol version should be used when using Curl - IPv4 & IPv6, IPv4 or IPv6 + http.handle.set(CurlOption.ipresolve,protocolVersion); // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only + + // What version of HTTP protocol do we use? + // Curl >= 7.62.0 defaults to http2 for a significant number of operations + if (httpProtocol) { + // Downgrade to HTTP 1.1 - yes version = 2 is HTTP 1.1 + http.handle.set(CurlOption.http_version,2); + } + + // Configure upload / download rate limits if configured + // 131072 = 128 KB/s - minimum for basic application operations to prevent timeouts + // A 0 value means rate is unlimited, and is the curl default + if (userRateLimit > 0) { + // set rate limit + http.handle.set(CurlOption.max_send_speed_large,userRateLimit); + http.handle.set(CurlOption.max_recv_speed_large,userRateLimit); + } + + // Explicitly set these libcurl options + // https://curl.se/libcurl/c/CURLOPT_NOSIGNAL.html + // Ensure that nosignal is set to 0 - Setting CURLOPT_NOSIGNAL to 0 makes libcurl ask the system to ignore SIGPIPE signals + http.handle.set(CurlOption.nosignal,0); + + // https://curl.se/libcurl/c/CURLOPT_TCP_NODELAY.html + // Ensure that TCP_NODELAY is set to 0 to ensure that TCP NAGLE is enabled + http.handle.set(CurlOption.tcp_nodelay,0); + + // https://curl.se/libcurl/c/CURLOPT_FORBID_REUSE.html + // CURLOPT_FORBID_REUSE - make connection get closed at once after use + // Ensure that we ARE NOT reusing TCP sockets connections - setting to 0 ensures that we ARE reusing connections (we did this in v2.4.xx) to ensure connections remained open and usable + // Setting this to 1 ensures that when we close the curl instance, any open sockets are closed - which we need to do when running + // multiple threads and API instances at the same time otherwise we run out of local files | sockets pretty quickly + // The libcurl default is 1 - ensure we are configuring not to reuse connections and leave unused sockets open + http.handle.set(CurlOption.forbid_reuse,1); + + if (httpsDebug) { + // Output what options we are using so that in the debug log this can be tracked + log.vdebug("http.dnsTimeout = ", dnsTimeout); + log.vdebug("http.connectTimeout = ", connectTimeout); + log.vdebug("http.dataTimeout = ", dataTimeout); + log.vdebug("http.operationTimeout = ", operationTimeout); + log.vdebug("http.maxRedirects = ", maxRedirects); + log.vdebug("http.CurlOption.ipresolve = ", protocolVersion); + } + } + + void setMethodPost() { + http.method = HTTP.Method.post; + } + + void setMethodPatch() { + http.method = HTTP.Method.patch; + } + + void setDisableSSLVerifyPeer() { + log.vdebug("Switching off CurlOption.ssl_verifypeer"); + http.handle.set(CurlOption.ssl_verifypeer, 0); + } +} \ No newline at end of file diff --git a/src/itemdb.d b/src/itemdb.d new file mode 100644 index 00000000..2ee44fec --- /dev/null +++ b/src/itemdb.d @@ -0,0 +1,696 @@ +// What is this module called? +module itemdb; + +// What does this module require to function? +import std.datetime; +import std.exception; +import std.path; +import std.string; +import std.stdio; +import std.algorithm.searching; +import core.stdc.stdlib; +import std.json; +import std.conv; + +// What other modules that we have created do we need to import? +import sqlite; +import util; +import log; + +enum ItemType { + file, + dir, + remote, + unknown +} + +struct Item { + string driveId; + string id; + string name; + string remoteName; + ItemType type; + string eTag; + string cTag; + SysTime mtime; + string parentId; + string quickXorHash; + string sha256Hash; + string remoteDriveId; + string remoteId; + string syncStatus; + string size; +} + +// Construct an Item struct from a JSON driveItem +Item makeDatabaseItem(JSONValue driveItem) { + + Item item = { + id: driveItem["id"].str, + name: "name" in driveItem ? driveItem["name"].str : null, // name may be missing for deleted files in OneDrive Business + eTag: "eTag" in driveItem ? driveItem["eTag"].str : null, // eTag is not returned for the root in OneDrive Business + cTag: "cTag" in driveItem ? driveItem["cTag"].str : null, // cTag is missing in old files (and all folders in OneDrive Business) + remoteName: "actualOnlineName" in driveItem ? driveItem["actualOnlineName"].str : null, // actualOnlineName is only used with OneDrive Business Shared Folders + }; + + // OneDrive API Change: https://github.com/OneDrive/onedrive-api-docs/issues/834 + // OneDrive no longer returns lastModifiedDateTime if the item is deleted by OneDrive + if(isItemDeleted(driveItem)) { + // Set mtime to SysTime(0) + item.mtime = SysTime(0); + } else { + // Item is not in a deleted state + // Resolve 'Key not found: fileSystemInfo' when then item is a remote item + // https://github.com/abraunegg/onedrive/issues/11 + if (isItemRemote(driveItem)) { + // remoteItem is a OneDrive object that exists on a 'different' OneDrive drive id, when compared to account default + // Normally, the 'remoteItem' field will contain 'fileSystemInfo' however, if the user uses the 'Add Shortcut ..' option in OneDrive WebUI + // to create a 'link', this object, whilst remote, does not have 'fileSystemInfo' in the expected place, thus leading to a application crash + // See: https://github.com/abraunegg/onedrive/issues/1533 + if ("fileSystemInfo" in driveItem["remoteItem"]) { + // 'fileSystemInfo' is in 'remoteItem' which will be the majority of cases + item.mtime = SysTime.fromISOExtString(driveItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str); + } else { + // is a remote item, but 'fileSystemInfo' is missing from 'remoteItem' + if ("fileSystemInfo" in driveItem) { + item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str); + } + } + } else { + // Does fileSystemInfo exist at all ? + if ("fileSystemInfo" in driveItem) { + item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str); + } + } + } + + // Set this item object type + bool typeSet = false; + if (isItemFile(driveItem)) { + // 'file' object exists in the JSON + log.vdebug("Flagging object as a file"); + typeSet = true; + item.type = ItemType.file; + } + + if (isItemFolder(driveItem)) { + // 'folder' object exists in the JSON + log.vdebug("Flagging object as a directory"); + typeSet = true; + item.type = ItemType.dir; + } + + if (isItemRemote(driveItem)) { + // 'remote' object exists in the JSON + log.vdebug("Flagging object as a remote"); + typeSet = true; + item.type = ItemType.remote; + } + + // root and remote items do not have parentReference + if (!isItemRoot(driveItem) && ("parentReference" in driveItem) != null) { + item.driveId = driveItem["parentReference"]["driveId"].str; + if (hasParentReferenceId(driveItem)) { + item.parentId = driveItem["parentReference"]["id"].str; + } + } + + // extract the file hash and file size + if (isItemFile(driveItem) && ("hashes" in driveItem["file"])) { + // Get file size + if (hasFileSize(driveItem)) { + item.size = to!string(driveItem["size"].integer); + // Get quickXorHash as default + if ("quickXorHash" in driveItem["file"]["hashes"]) { + item.quickXorHash = driveItem["file"]["hashes"]["quickXorHash"].str; + } else { + log.vdebug("quickXorHash is missing from ", driveItem["id"].str); + } + + // If quickXorHash is empty .. + if (item.quickXorHash.empty) { + // Is there a sha256Hash? + if ("sha256Hash" in driveItem["file"]["hashes"]) { + item.sha256Hash = driveItem["file"]["hashes"]["sha256Hash"].str; + } else { + log.vdebug("sha256Hash is missing from ", driveItem["id"].str); + } + } + } else { + // So that we have at least a zero value here as the API provided no 'size' data for this file item + item.size = "0"; + } + } + + // Is the object a remote drive item - living on another driveId ? + if (isItemRemote(driveItem)) { + item.remoteDriveId = driveItem["remoteItem"]["parentReference"]["driveId"].str; + item.remoteId = driveItem["remoteItem"]["id"].str; + } + + // We have 3 different operational modes where 'item.syncStatus' is used to flag if an item is synced or not: + // - National Cloud Deployments do not support /delta as a query + // - When using --single-directory + // - When using --download-only --cleanup-local-files + // + // Thus we need to track in the database that this item is in sync + // As we are making an item, set the syncStatus to Y + // ONLY when either of the three modes above are being used, all the existing DB entries will get set to N + // so when processing /children, it can be identified what the 'deleted' difference is + item.syncStatus = "Y"; + + // Return the created item + return item; +} + +final class ItemDatabase { + // increment this for every change in the db schema + immutable int itemDatabaseVersion = 12; + + Database db; + string insertItemStmt; + string updateItemStmt; + string selectItemByIdStmt; + string selectItemByRemoteIdStmt; + string selectItemByParentIdStmt; + string deleteItemByIdStmt; + bool databaseInitialised = false; + + this(const(char)[] filename) { + db = Database(filename); + int dbVersion; + try { + dbVersion = db.getVersion(); + } catch (SqliteException e) { + // An error was generated - what was the error? + if (e.msg == "database is locked") { + writeln(); + log.error("ERROR: onedrive application is already running - check system process list for active application instances"); + log.vlog(" - Use 'sudo ps aufxw | grep onedrive' to potentially determine acive running process"); + writeln(); + } else { + writeln(); + log.error("ERROR: An internal database error occurred: " ~ e.msg); + writeln(); + } + return; + } + + if (dbVersion == 0) { + createTable(); + } else if (db.getVersion() != itemDatabaseVersion) { + log.log("The item database is incompatible, re-creating database table structures"); + db.exec("DROP TABLE item"); + createTable(); + } + // Set the enforcement of foreign key constraints. + // https://www.sqlite.org/pragma.html#pragma_foreign_keys + // PRAGMA foreign_keys = boolean; + db.exec("PRAGMA foreign_keys = TRUE"); + // Set the recursive trigger capability + // https://www.sqlite.org/pragma.html#pragma_recursive_triggers + // PRAGMA recursive_triggers = boolean; + db.exec("PRAGMA recursive_triggers = TRUE"); + // Set the journal mode for databases associated with the current connection + // https://www.sqlite.org/pragma.html#pragma_journal_mode + db.exec("PRAGMA journal_mode = WAL"); + // Automatic indexing is enabled by default as of version 3.7.17 + // https://www.sqlite.org/pragma.html#pragma_automatic_index + // PRAGMA automatic_index = boolean; + db.exec("PRAGMA automatic_index = FALSE"); + // Tell SQLite to store temporary tables in memory. This will speed up many read operations that rely on temporary tables, indices, and views. + // https://www.sqlite.org/pragma.html#pragma_temp_store + db.exec("PRAGMA temp_store = MEMORY"); + // Tell SQlite to cleanup database table size + // https://www.sqlite.org/pragma.html#pragma_auto_vacuum + // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL; + db.exec("PRAGMA auto_vacuum = FULL"); + // This pragma sets or queries the database connection locking-mode. The locking-mode is either NORMAL or EXCLUSIVE. + // https://www.sqlite.org/pragma.html#pragma_locking_mode + // PRAGMA schema.locking_mode = NORMAL | EXCLUSIVE + db.exec("PRAGMA locking_mode = EXCLUSIVE"); + + insertItemStmt = " + INSERT OR REPLACE INTO item (driveId, id, name, remoteName, type, eTag, cTag, mtime, parentId, quickXorHash, sha256Hash, remoteDriveId, remoteId, syncStatus, size) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15) + "; + updateItemStmt = " + UPDATE item + SET name = ?3, remoteName = ?4, type = ?5, eTag = ?6, cTag = ?7, mtime = ?8, parentId = ?9, quickXorHash = ?10, sha256Hash = ?11, remoteDriveId = ?12, remoteId = ?13, syncStatus = ?14, size = ?15 + WHERE driveId = ?1 AND id = ?2 + "; + selectItemByIdStmt = " + SELECT * + FROM item + WHERE driveId = ?1 AND id = ?2 + "; + selectItemByRemoteIdStmt = " + SELECT * + FROM item + WHERE remoteDriveId = ?1 AND remoteId = ?2 + "; + selectItemByParentIdStmt = "SELECT * FROM item WHERE driveId = ? AND parentId = ?"; + deleteItemByIdStmt = "DELETE FROM item WHERE driveId = ? AND id = ?"; + + // flag that the database is accessible and we have control + databaseInitialised = true; + } + + bool isDatabaseInitialised() { + return databaseInitialised; + } + + void createTable() { + db.exec("CREATE TABLE item ( + driveId TEXT NOT NULL, + id TEXT NOT NULL, + name TEXT NOT NULL, + remoteName TEXT, + type TEXT NOT NULL, + eTag TEXT, + cTag TEXT, + mtime TEXT NOT NULL, + parentId TEXT, + quickXorHash TEXT, + sha256Hash TEXT, + remoteDriveId TEXT, + remoteId TEXT, + deltaLink TEXT, + syncStatus TEXT, + size TEXT, + PRIMARY KEY (driveId, id), + FOREIGN KEY (driveId, parentId) + REFERENCES item (driveId, id) + ON DELETE CASCADE + ON UPDATE RESTRICT + )"); + db.exec("CREATE INDEX name_idx ON item (name)"); + db.exec("CREATE INDEX remote_idx ON item (remoteDriveId, remoteId)"); + db.exec("CREATE INDEX item_children_idx ON item (driveId, parentId)"); + db.exec("CREATE INDEX selectByPath_idx ON item (name, driveId, parentId)"); + db.setVersion(itemDatabaseVersion); + } + + void insert(const ref Item item) { + auto p = db.prepare(insertItemStmt); + bindItem(item, p); + p.exec(); + } + + void update(const ref Item item) { + auto p = db.prepare(updateItemStmt); + bindItem(item, p); + p.exec(); + } + + void dump_open_statements() { + db.dump_open_statements(); + } + + int db_checkpoint() { + return db.db_checkpoint(); + } + + void upsert(const ref Item item) { + auto s = db.prepare("SELECT COUNT(*) FROM item WHERE driveId = ? AND id = ?"); + s.bind(1, item.driveId); + s.bind(2, item.id); + auto r = s.exec(); + Statement stmt; + if (r.front[0] == "0") stmt = db.prepare(insertItemStmt); + else stmt = db.prepare(updateItemStmt); + bindItem(item, stmt); + stmt.exec(); + } + + Item[] selectChildren(const(char)[] driveId, const(char)[] id) { + auto p = db.prepare(selectItemByParentIdStmt); + p.bind(1, driveId); + p.bind(2, id); + auto res = p.exec(); + Item[] items; + while (!res.empty) { + items ~= buildItem(res); + res.step(); + } + return items; + } + + bool selectById(const(char)[] driveId, const(char)[] id, out Item item) { + auto p = db.prepare(selectItemByIdStmt); + p.bind(1, driveId); + p.bind(2, id); + auto r = p.exec(); + if (!r.empty) { + item = buildItem(r); + return true; + } + return false; + } + + bool selectByRemoteId(const(char)[] remoteDriveId, const(char)[] remoteId, out Item item) { + auto p = db.prepare(selectItemByRemoteIdStmt); + p.bind(1, remoteDriveId); + p.bind(2, remoteId); + auto r = p.exec(); + if (!r.empty) { + item = buildItem(r); + return true; + } + return false; + } + + // returns true if an item id is in the database + bool idInLocalDatabase(const(string) driveId, const(string)id) { + auto p = db.prepare(selectItemByIdStmt); + p.bind(1, driveId); + p.bind(2, id); + auto r = p.exec(); + if (!r.empty) { + return true; + } + return false; + } + + // returns the item with the given path + // the path is relative to the sync directory ex: "./Music/Turbo Killer.mp3" + bool selectByPath(const(char)[] path, string rootDriveId, out Item item) { + Item currItem = { driveId: rootDriveId }; + + // Issue https://github.com/abraunegg/onedrive/issues/578 + if (startsWith(path, "./") || path == ".") { + // Need to remove the . from the path prefix + path = "root/" ~ path.chompPrefix("."); + } else { + // Leave path as it is + path = "root/" ~ path; + } + + auto s = db.prepare("SELECT * FROM item WHERE name = ?1 AND driveId IS ?2 AND parentId IS ?3"); + foreach (name; pathSplitter(path)) { + s.bind(1, name); + s.bind(2, currItem.driveId); + s.bind(3, currItem.id); + auto r = s.exec(); + if (r.empty) return false; + currItem = buildItem(r); + + // if the item is of type remote substitute it with the child + if (currItem.type == ItemType.remote) { + Item child; + if (selectById(currItem.remoteDriveId, currItem.remoteId, child)) { + assert(child.type != ItemType.remote, "The type of the child cannot be remote"); + currItem = child; + } + } + } + item = currItem; + return true; + } + + // same as selectByPath() but it does not traverse remote folders + bool selectByPathWithoutRemote(const(char)[] path, string rootDriveId, out Item item) { + Item currItem = { driveId: rootDriveId }; + + // Issue https://github.com/abraunegg/onedrive/issues/578 + if (startsWith(path, "./") || path == ".") { + // Need to remove the . from the path prefix + path = "root/" ~ path.chompPrefix("."); + } else { + // Leave path as it is + path = "root/" ~ path; + } + + auto s = db.prepare("SELECT * FROM item WHERE name IS ?1 AND driveId IS ?2 AND parentId IS ?3"); + foreach (name; pathSplitter(path)) { + s.bind(1, name); + s.bind(2, currItem.driveId); + s.bind(3, currItem.id); + auto r = s.exec(); + if (r.empty) return false; + currItem = buildItem(r); + } + item = currItem; + return true; + } + + void deleteById(const(char)[] driveId, const(char)[] id) { + auto p = db.prepare(deleteItemByIdStmt); + p.bind(1, driveId); + p.bind(2, id); + p.exec(); + } + + private void bindItem(const ref Item item, ref Statement stmt) { + with (stmt) with (item) { + bind(1, driveId); + bind(2, id); + bind(3, name); + bind(4, remoteName); + string typeStr = null; + final switch (type) with (ItemType) { + case file: typeStr = "file"; break; + case dir: typeStr = "dir"; break; + case remote: typeStr = "remote"; break; + case unknown: typeStr = "unknown"; break; + } + bind(5, typeStr); + bind(6, eTag); + bind(7, cTag); + bind(8, mtime.toISOExtString()); + bind(9, parentId); + bind(10, quickXorHash); + bind(11, sha256Hash); + bind(12, remoteDriveId); + bind(13, remoteId); + bind(14, syncStatus); + bind(15, size); + } + } + + private Item buildItem(Statement.Result result) { + assert(!result.empty, "The result must not be empty"); + assert(result.front.length == 16, "The result must have 16 columns"); + Item item = { + + // column 0: driveId + // column 1: id + // column 2: name + // column 3: remoteName - only used when there is a difference in the local name & remote shared folder name + // column 4: type + // column 5: eTag + // column 6: cTag + // column 7: mtime + // column 8: parentId + // column 9: quickXorHash + // column 10: sha256Hash + // column 11: remoteDriveId + // column 12: remoteId + // column 13: deltaLink + // column 14: syncStatus + // column 15: size + + driveId: result.front[0].dup, + id: result.front[1].dup, + name: result.front[2].dup, + remoteName: result.front[3].dup, + // Column 4 is type - not set here + eTag: result.front[5].dup, + cTag: result.front[6].dup, + mtime: SysTime.fromISOExtString(result.front[7]), + parentId: result.front[8].dup, + quickXorHash: result.front[9].dup, + sha256Hash: result.front[10].dup, + remoteDriveId: result.front[11].dup, + remoteId: result.front[12].dup, + // Column 13 is deltaLink - not set here + syncStatus: result.front[14].dup, + size: result.front[15].dup + }; + switch (result.front[4]) { + case "file": item.type = ItemType.file; break; + case "dir": item.type = ItemType.dir; break; + case "remote": item.type = ItemType.remote; break; + default: assert(0, "Invalid item type"); + } + return item; + } + + // computes the path of the given item id + // the path is relative to the sync directory ex: "Music/Turbo Killer.mp3" + // the trailing slash is not added even if the item is a directory + string computePath(const(char)[] driveId, const(char)[] id) { + assert(driveId && id); + string path; + Item item; + auto s = db.prepare("SELECT * FROM item WHERE driveId = ?1 AND id = ?2"); + auto s2 = db.prepare("SELECT driveId, id FROM item WHERE remoteDriveId = ?1 AND remoteId = ?2"); + while (true) { + s.bind(1, driveId); + s.bind(2, id); + auto r = s.exec(); + if (!r.empty) { + item = buildItem(r); + if (item.type == ItemType.remote) { + // substitute the last name with the current + ptrdiff_t idx = indexOf(path, '/'); + path = idx >= 0 ? item.name ~ path[idx .. $] : item.name; + } else { + if (path) path = item.name ~ "/" ~ path; + else path = item.name; + } + id = item.parentId; + } else { + if (id == null) { + // check for remoteItem + s2.bind(1, item.driveId); + s2.bind(2, item.id); + auto r2 = s2.exec(); + if (r2.empty) { + // root reached + assert(path.length >= 4); + // remove "root/" from path string if it exists + if (path.length >= 5) { + if (canFind(path, "root/")){ + path = path[5 .. $]; + } + } else { + path = path[4 .. $]; + } + // special case of computing the path of the root itself + if (path.length == 0) path = "."; + break; + } else { + // remote folder + driveId = r2.front[0].dup; + id = r2.front[1].dup; + } + } else { + // broken tree + log.vdebug("The following generated a broken tree query:"); + log.vdebug("Drive ID: ", driveId); + log.vdebug("Item ID: ", id); + assert(0); + } + } + } + return path; + } + + Item[] selectRemoteItems() { + Item[] items; + auto stmt = db.prepare("SELECT * FROM item WHERE remoteDriveId IS NOT NULL"); + auto res = stmt.exec(); + while (!res.empty) { + items ~= buildItem(res); + res.step(); + } + return items; + } + + string getDeltaLink(const(char)[] driveId, const(char)[] id) { + // Log what we received + log.vdebug("DeltaLink Query (driveId): ", driveId); + log.vdebug("DeltaLink Query (id): ", id); + + assert(driveId && id); + auto stmt = db.prepare("SELECT deltaLink FROM item WHERE driveId = ?1 AND id = ?2"); + stmt.bind(1, driveId); + stmt.bind(2, id); + auto res = stmt.exec(); + if (res.empty) return null; + return res.front[0].dup; + } + + void setDeltaLink(const(char)[] driveId, const(char)[] id, const(char)[] deltaLink) { + assert(driveId && id); + assert(deltaLink); + auto stmt = db.prepare("UPDATE item SET deltaLink = ?3 WHERE driveId = ?1 AND id = ?2"); + stmt.bind(1, driveId); + stmt.bind(2, id); + stmt.bind(3, deltaLink); + stmt.exec(); + } + + // National Cloud Deployments (US and DE) do not support /delta as a query + // We need to track in the database that this item is in sync + // As we query /children to get all children from OneDrive, update anything in the database + // to be flagged as not-in-sync, thus, we can use that flag to determing what was previously + // in-sync, but now deleted on OneDrive + void downgradeSyncStatusFlag(const(char)[] driveId, const(char)[] id) { + assert(driveId); + auto stmt = db.prepare("UPDATE item SET syncStatus = 'N' WHERE driveId = ?1 AND id = ?2"); + stmt.bind(1, driveId); + stmt.bind(2, id); + stmt.exec(); + } + + // National Cloud Deployments (US and DE) do not support /delta as a query + // Select items that have a out-of-sync flag set + Item[] selectOutOfSyncItems(const(char)[] driveId) { + assert(driveId); + Item[] items; + auto stmt = db.prepare("SELECT * FROM item WHERE syncStatus = 'N' AND driveId = ?1"); + stmt.bind(1, driveId); + auto res = stmt.exec(); + while (!res.empty) { + items ~= buildItem(res); + res.step(); + } + return items; + } + + // OneDrive Business Folders are stored in the database potentially without a root | parentRoot link + // Select items associated with the provided driveId + Item[] selectByDriveId(const(char)[] driveId) { + assert(driveId); + Item[] items; + auto stmt = db.prepare("SELECT * FROM item WHERE driveId = ?1 AND parentId IS NULL"); + stmt.bind(1, driveId); + auto res = stmt.exec(); + while (!res.empty) { + items ~= buildItem(res); + res.step(); + } + return items; + } + + // Select all items associated with the provided driveId + Item[] selectAllItemsByDriveId(const(char)[] driveId) { + assert(driveId); + Item[] items; + auto stmt = db.prepare("SELECT * FROM item WHERE driveId = ?1"); + stmt.bind(1, driveId); + auto res = stmt.exec(); + while (!res.empty) { + items ~= buildItem(res); + res.step(); + } + return items; + } + + // Perform a vacuum on the database, commit WAL / SHM to file + void performVacuum() { + try { + auto stmt = db.prepare("VACUUM;"); + stmt.exec(); + } catch (SqliteException e) { + writeln(); + log.error("ERROR: Unable to perform a database vacuum: " ~ e.msg); + writeln(); + } + } + + // Select distinct driveId items from database + string[] selectDistinctDriveIds() { + string[] driveIdArray; + auto stmt = db.prepare("SELECT DISTINCT driveId FROM item;"); + auto res = stmt.exec(); + if (res.empty) return driveIdArray; + while (!res.empty) { + driveIdArray ~= res.front[0].dup; + res.step(); + } + return driveIdArray; + } +} \ No newline at end of file diff --git a/src/log.d b/src/log.d new file mode 100644 index 00000000..12afa140 --- /dev/null +++ b/src/log.d @@ -0,0 +1,252 @@ +// What is this module called? +module log; + +// What does this module require to function? +import std.stdio; +import std.file; +import std.datetime; +import std.process; +import std.conv; +import std.path; +import std.string; +import core.memory; +import core.sys.posix.pwd; +import core.sys.posix.unistd; +import core.stdc.string : strlen; +import std.algorithm : splitter; + +version(Notifications) { + import dnotify; +} + +// module variables +// verbose logging count +long verbose; +// do we write a log file? ... this should be a config falue +bool writeLogFile = false; +// did the log file write fail? +bool logFileWriteFailFlag = false; +private bool triggerNotification; + +// shared string variable for username +string username; +string logFilePath; +string logFileName; +string logFileFullPath; + +void initialise(string logDir) { + writeLogFile = true; + + // Configure various variables + username = getUserName(); + logFilePath = logDir; + logFileName = username ~ ".onedrive.log"; + logFileFullPath = buildPath(logFilePath, logFileName); + + if (!exists(logFilePath)){ + // logfile path does not exist + try { + mkdirRecurse(logFilePath); + } + catch (std.file.FileException e) { + // we got an error .. + writeln(); + writeln("ERROR: Unable to access ", logFilePath); + writeln("ERROR: Please manually create '",logFilePath, "' and set appropriate permissions to allow write access"); + writeln("ERROR: The requested client activity log will instead be located in your users home directory"); + writeln(); + + // set the flag so we dont keep printing this sort of message + logFileWriteFailFlag = true; + } + } +} + +void enableNotifications(bool value) { + version(Notifications) { + // if we try to enable notifications, check for server availability + // and disable in case dbus server is not reachable + if (value) { + auto serverAvailable = dnotify.check_availability(); + if (!serverAvailable) { + log("Notification (dbus) server not available, disabling"); + value = false; + } + } + } + triggerNotification = value; +} + +void log(T...)(T args) { + writeln(args); + if(writeLogFile){ + // Write to log file + logfileWriteLine(args); + } +} + +void logAndNotify(T...)(T args) { + notify(args); + log(args); +} + +void fileOnly(T...)(T args) { + if(writeLogFile){ + // Write to log file + logfileWriteLine(args); + } +} + +void vlog(T...)(T args) { + if (verbose >= 1) { + writeln(args); + if(writeLogFile){ + // Write to log file + logfileWriteLine(args); + } + } +} + +void vdebug(T...)(T args) { + if (verbose >= 2) { + writeln("[DEBUG] ", args); + if(writeLogFile){ + // Write to log file + logfileWriteLine("[DEBUG] ", args); + } + } +} + +void vdebugNewLine(T...)(T args) { + if (verbose >= 2) { + writeln("\n[DEBUG] ", args); + if(writeLogFile){ + // Write to log file + logfileWriteLine("\n[DEBUG] ", args); + } + } +} + +void error(T...)(T args) { + stderr.writeln(args); + if(writeLogFile){ + // Write to log file + logfileWriteLine(args); + } +} + +void errorAndNotify(T...)(T args) { + notify(args); + error(args); +} + +void notify(T...)(T args) { + version(Notifications) { + if (triggerNotification) { + string result; + foreach (index, arg; args) { + result ~= to!string(arg); + if (index != args.length - 1) + result ~= " "; + } + auto n = new Notification("OneDrive", result, "IGNORED"); + try { + n.show(); + // Sent message to notification daemon + if (verbose >= 2) { + writeln("[DEBUG] Sent notification to notification service. If notification is not displayed, check dbus or notification-daemon for errors"); + } + + } catch (Throwable e) { + vlog("Got exception from showing notification: ", e); + } + } + } +} + +private void logfileWriteLine(T...)(T args) { + static import std.exception; + // Write to log file + auto currentTime = Clock.currTime(); + auto timeString = leftJustify(currentTime.toString(), 28, '0'); + File logFile; + + // Resolve: std.exception.ErrnoException@std/stdio.d(423): Cannot open file `/var/log/onedrive/xxxxx.onedrive.log' in mode `a' (Permission denied) + try { + logFile = File(logFileFullPath, "a"); + } + catch (std.exception.ErrnoException e) { + // We cannot open the log file logFileFullPath for writing + // The user is not part of the standard 'users' group (GID 100) + // Change logfile to ~/onedrive.log putting the log file in the users home directory + + if (!logFileWriteFailFlag) { + // write out error message that we cant log to the requested file + writeln(); + writeln("ERROR: Unable to write activity log to ", logFileFullPath); + writeln("ERROR: Please set appropriate permissions to allow write access to the logging directory for your user account"); + writeln("ERROR: The requested client activity log will instead be located in your users home directory"); + writeln(); + + // set the flag so we dont keep printing this error message + logFileWriteFailFlag = true; + } + + string homePath = environment.get("HOME"); + string logFileFullPathAlternate = homePath ~ "/onedrive.log"; + logFile = File(logFileFullPathAlternate, "a"); + } + // Write to the log file + logFile.writeln(timeString, "\t", args); + logFile.close(); +} + +private string getUserName() { + auto pw = getpwuid(getuid); + + // get required details + auto runtime_pw_name = pw.pw_name[0 .. strlen(pw.pw_name)].splitter(','); + auto runtime_pw_uid = pw.pw_uid; + auto runtime_pw_gid = pw.pw_gid; + + // user identifiers from process + vdebug("Process ID: ", pw); + vdebug("User UID: ", runtime_pw_uid); + vdebug("User GID: ", runtime_pw_gid); + + // What should be returned as username? + if (!runtime_pw_name.empty && runtime_pw_name.front.length){ + // user resolved + vdebug("User Name: ", runtime_pw_name.front.idup); + return runtime_pw_name.front.idup; + } else { + // Unknown user? + vdebug("User Name: unknown"); + return "unknown"; + } +} + +void displayMemoryUsagePreGC() { + // Display memory usage + writeln(); + writeln("Memory Usage pre GC (KB)"); + writeln("------------------------"); + writeMemoryStats(); + writeln(); +} + +void displayMemoryUsagePostGC() { + // Display memory usage + writeln(); + writeln("Memory Usage post GC (KB)"); + writeln("-------------------------"); + writeMemoryStats(); + writeln(); +} + +void writeMemoryStats() { + // write memory stats + writeln("memory usedSize = ", (GC.stats.usedSize/1024)); + writeln("memory freeSize = ", (GC.stats.freeSize/1024)); + writeln("memory allocatedInCurrentThread = ", (GC.stats.allocatedInCurrentThread/1024)); +} \ No newline at end of file diff --git a/src/main.d b/src/main.d new file mode 100644 index 00000000..f1d574f1 --- /dev/null +++ b/src/main.d @@ -0,0 +1,1001 @@ +// What is this module called? +module main; + +// What does this module require to function? +import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit; +import core.stdc.signal; +import core.memory; +import core.time; +import core.thread; +import std.stdio; +import std.getopt; +import std.string; +import std.file; +import std.process; +import std.algorithm; +import std.path; +import std.concurrency; +import std.parallelism; +import std.conv; +import std.traits; +import std.net.curl: CurlException; +import std.datetime; + +// What other modules that we have created do we need to import? +import config; +import log; +import curlEngine; +import util; +import onedrive; +import syncEngine; +import itemdb; +import clientSideFiltering; +import monitor; + +// What other constant variables do we require? +const int EXIT_RESYNC_REQUIRED = 126; + +// Class objects +ApplicationConfig appConfig; +OneDriveApi oneDriveApiInstance; +SyncEngine syncEngineInstance; +ItemDatabase itemDB; +ClientSideFiltering selectiveSync; + +int main(string[] cliArgs) { + // Disable buffering on stdout - this is needed so that when we are using plain write() it will go to the terminal + stdout.setvbuf(0, _IONBF); + // Required main function variables + string genericHelpMessage = "Try 'onedrive --help' for more information"; + // If the user passes in --confdir we need to store this as a variable + string confdirOption = ""; + // Are we online? + bool online = false; + // Does the operating environment have shell environment variables set + bool shellEnvSet = false; + // What is the runtime syncronisation directory that will be used + // Typically this will be '~/OneDrive' .. however tilde expansion is unreliable + string runtimeSyncDirectory = ""; + // Configure the runtime database file path. Typically this will be the default, but in a --dry-run scenario, we use a separate database file + string runtimeDatabaseFile = ""; + + // Application Start Time - used during monitor loop to detail how long it has been running for + auto applicationStartTime = Clock.currTime(); + + // DEVELOPER OPTIONS OUTPUT VARIABLES + bool displayMemoryUsage = false; + bool displaySyncOptions = false; + + // Define 'exit' and 'failure' scopes + scope(exit) { + // detail what scope was called + log.vdebug("Exit scope was called"); + + // Was itemDB initialised? + if (itemDB !is null) { + // Make sure the .wal file is incorporated into the main db before we exit + itemDB.performVacuum(); + } + + // Cleanup any existing dry-run elements ... these should never be left hanging around + cleanupDryRunDatabaseFiles(appConfig.databaseFilePathDryRun); + + // Free object and memory + object.destroy(appConfig); + object.destroy(oneDriveApiInstance); + object.destroy(selectiveSync); + object.destroy(syncEngineInstance); + object.destroy(itemDB); + } + + scope(failure) { + // detail what scope was called + log.vdebug("Failure scope was called"); + + // Was itemDB initialised? + if (itemDB !is null) { + // Make sure the .wal file is incorporated into the main db before we exit + itemDB.performVacuum(); + } + + // Cleanup any existing dry-run elements ... these should never be left hanging around + cleanupDryRunDatabaseFiles(appConfig.databaseFilePathDryRun); + + // Free object and memory + object.destroy(appConfig); + object.destroy(oneDriveApiInstance); + object.destroy(selectiveSync); + object.destroy(syncEngineInstance); + object.destroy(itemDB); + } + + // Read in application options as passed in + try { + bool printVersion = false; + auto cliOptions = getopt( + cliArgs, + std.getopt.config.passThrough, + std.getopt.config.bundling, + std.getopt.config.caseSensitive, + "confdir", "Set the directory used to store the configuration files", &confdirOption, + "verbose|v+", "Print more details, useful for debugging (repeat for extra debugging)", &log.verbose, + "version", "Print the version and exit", &printVersion + ); + + // Print help and exit + if (cliOptions.helpWanted) { + cliArgs ~= "--help"; + } + // Print the version and exit + if (printVersion) { + //writeln("onedrive ", strip(import("version"))); + string tempVersion = "v2.5.0-alpha-2" ~ " GitHub version: " ~ strip(import("version")); + writeln(tempVersion); + return EXIT_SUCCESS; + } + } catch (GetOptException e) { + // Option errors + log.error(e.msg); + log.error(genericHelpMessage); + return EXIT_FAILURE; + } catch (Exception e) { + // Generic error + log.error(e.msg); + log.error(genericHelpMessage); + return EXIT_FAILURE; + } + + // How was this application started - what options were passed in + log.vdebug("passed in options: ", cliArgs); + log.vdebug("note --confdir and --verbose not listed in 'cliArgs'"); + + // Create a new AppConfig object with default values, + appConfig = new ApplicationConfig(); + // Initialise the application configuration, utilising --confdir if it was passed in + // Otherwise application defaults will be used to configure the application + if (!appConfig.initialise(confdirOption)) { + // There was an error loading the user specified application configuration + // Error message already printed + return EXIT_FAILURE; + } + + // Update the existing application configuration (default or 'config' file) from any passed in command line arguments + appConfig.updateFromArgs(cliArgs); + + // Depreciated options check now that the config file (if present) and CLI options have all been parsed + appConfig.checkDepreciatedOptions(cliArgs); + + // Configure GUI Notifications + // - This option is reverse action. If 'disable_notifications' is 'true', we need to send 'false' + if (appConfig.getValueBool("disable_notifications")){ + // disable_notifications is true, initialise with false + log.enableNotifications(false); + } else { + log.enableNotifications(true); + } + + // Configure application logging to a log file only if enabled + // This is the earliest point to do so as the client configuration has been read in, CLI arguments have been processed. + // Either of those could be enabling logging + if (appConfig.getValueBool("enable_logging")){ + // configure the application logging directory + string initialisedLogDirPath = appConfig.initialiseLogDirectory(); + // Initialise using the configured logging directory + log.vlog("Using the following path to store the runtime application log: ", initialisedLogDirPath); + log.initialise(initialisedLogDirPath); + } + + // Configure Client Side Filtering (selective sync) by parsing and getting a usable regex for skip_file, skip_dir and sync_list config components + selectiveSync = new ClientSideFiltering(appConfig); + if (!selectiveSync.initialise()) { + // exit here as something triggered a selective sync configuration failure + return EXIT_FAILURE; + } + + // Set runtimeDatabaseFile, this will get updated if we are using --dry-run + runtimeDatabaseFile = appConfig.databaseFilePath; + + // Read in 'sync_dir' from appConfig with '~' if present expanded + runtimeSyncDirectory = appConfig.initialiseRuntimeSyncDirectory(); + + // DEVELOPER OPTIONS OUTPUT + // Set to display memory details as early as possible + displayMemoryUsage = appConfig.getValueBool("display_memory"); + // set to display sync options + displaySyncOptions = appConfig.getValueBool("display_sync_options"); + + // Display the current application configuration (based on all defaults, 'config' file parsing and/or options passed in via the CLI) and exit if --display-config has been used + if ((appConfig.getValueBool("display_config")) || (appConfig.getValueBool("display_running_config"))) { + // Display the application configuration + appConfig.displayApplicationConfiguration(); + // Do we exit? We exit only if '--display-config' has been used + if (appConfig.getValueBool("display_config")) { + return EXIT_SUCCESS; + } + } + + // Check for basic application option conflicts - flags that should not be used together and/or flag combinations that conflict with each other, values that should be present and are not + if (appConfig.checkForBasicOptionConflicts) { + // Any error will have been printed by the function itself + return EXIT_FAILURE; + } + + // Check for --dry-run operation + // If this has been requested, we need to ensure that all actions are performed against the dry-run database copy, and, + // no actual action takes place - such as deleting files if deleted online, moving files if moved online or local, downloading new & changed files, uploading new & changed files + if (appConfig.getValueBool("dry_run")) { + // this is a --dry-run operation + log.log("DRY-RUN Configured. Output below shows what 'would' have occurred."); + + // Cleanup any existing dry-run elements ... these should never be left hanging around + cleanupDryRunDatabaseFiles(appConfig.databaseFilePathDryRun); + + // Make a copy of the original items.sqlite3 for use as the dry run copy if it exists + if (exists(appConfig.databaseFilePath)) { + // In a --dry-run --resync scenario, we should not copy the existing database file + if (!appConfig.getValueBool("resync")) { + // Copy the existing DB file to the dry-run copy + log.log("DRY-RUN: Copying items.sqlite3 to items-dryrun.sqlite3 to use for dry run operations"); + copy(appConfig.databaseFilePath,appConfig.databaseFilePathDryRun); + } else { + // No database copy due to --resync + log.log("DRY-RUN: No database copy created for --dry-run due to --resync also being used"); + } + } + // update runtimeDatabaseFile now that we are using the dry run path + runtimeDatabaseFile = appConfig.databaseFilePathDryRun; + } + + // Handle --logout as separate item, do not 'resync' on a --logout + if (appConfig.getValueBool("logout")) { + log.vdebug("--logout requested"); + log.log("Deleting the saved authentication status ..."); + if (!appConfig.getValueBool("dry_run")) { + safeRemove(appConfig.refreshTokenFilePath); + } else { + // --dry-run scenario ... technically we should not be making any local file changes ....... + log.log("DRY RUN: Not removing the saved authentication status"); + } + // Exit + return EXIT_SUCCESS; + } + + // Handle --reauth to re-authenticate the client + if (appConfig.getValueBool("reauth")) { + log.vdebug("--reauth requested"); + log.log("Deleting the saved authentication status ... re-authentication requested"); + if (!appConfig.getValueBool("dry_run")) { + safeRemove(appConfig.refreshTokenFilePath); + } else { + // --dry-run scenario ... technically we should not be making any local file changes ....... + log.log("DRY RUN: Not removing the saved authentication status"); + } + } + + // --resync should be considered a 'last resort item' or if the application configuration has changed, where a resync is needed .. the user needs to 'accept' this warning to proceed + // If --resync has not been used (bool value is false), check the application configuration for 'changes' that require a --resync to ensure that the data locally reflects the users requested configuration + if (appConfig.getValueBool("resync")) { + // what is the risk acceptance for --resync? + bool resyncRiskAcceptance = appConfig.displayResyncRiskForAcceptance(); + log.vdebug("Returned --resync risk acceptance: ", resyncRiskAcceptance); + // Action based on user response + if (!resyncRiskAcceptance){ + // --resync risk not accepted + return EXIT_FAILURE; + } else { + log.vdebug("--resync issued and risk accepted"); + // --resync risk accepted, perform a cleanup of items that require a cleanup + appConfig.cleanupHashFilesDueToResync(); + // Make a backup of the applicable configuration file + appConfig.createBackupConfigFile(); + // Update hash files and generate a new config backup + appConfig.updateHashContentsForConfigFiles(); + // Remove the items database + processResyncDatabaseRemoval(runtimeDatabaseFile); + } + } else { + // Has any of our application configuration that would require a --resync been changed? + if (appConfig.applicationChangeWhereResyncRequired()) { + // Application configuration has changed however --resync not issued, fail fast + log.error("\nAn application configuration change has been detected where a --resync is required\n"); + return EXIT_RESYNC_REQUIRED; + } else { + // No configuration change that requires a --resync to be issued + // Make a backup of the applicable configuration file + appConfig.createBackupConfigFile(); + // Update hash files and generate a new config backup + appConfig.updateHashContentsForConfigFiles(); + } + } + + // Implement https://github.com/abraunegg/onedrive/issues/1129 + // Force a synchronization of a specific folder, only when using --synchronize --single-directory and ignoring all non-default skip_dir and skip_file rules + if (appConfig.getValueBool("force_sync")) { + // appConfig.checkForBasicOptionConflicts() has already checked for the basic requirements for --force-sync + log.log("\nWARNING: Overriding application configuration to use application defaults for skip_dir and skip_file due to --synch --single-directory --force-sync being used"); + bool forceSyncRiskAcceptance = appConfig.displayForceSyncRiskForAcceptance(); + log.vdebug("Returned --force-sync risk acceptance: ", forceSyncRiskAcceptance); + // Action based on user response + if (!forceSyncRiskAcceptance){ + // --force-sync risk not accepted + return EXIT_FAILURE; + } else { + // --force-sync risk accepted + // reset set config using function to use application defaults + appConfig.resetSkipToDefaults(); + // update sync engine regex with reset defaults + selectiveSync.setDirMask(appConfig.getValueString("skip_dir")); + selectiveSync.setFileMask(appConfig.getValueString("skip_file")); + } + } + + // Test if OneDrive service can be reached, exit if it cant be reached + log.vdebug("Testing network to ensure network connectivity to Microsoft OneDrive Service"); + online = testInternetReachability(appConfig); + + // If we are not 'online' - how do we handle this situation? + if (!online) { + // We are unable to initialise the OneDrive API as we are not online + if (!appConfig.getValueBool("monitor")) { + // Running as --synchronize + log.error("Unable to reach Microsoft OneDrive API service, unable to initialise application\n"); + return EXIT_FAILURE; + } else { + // Running as --monitor + log.error("Unable to reach the Microsoft OneDrive API service at this point in time, re-trying network tests based on applicable intervals\n"); + if (!retryInternetConnectivtyTest(appConfig)) { + return EXIT_FAILURE; + } + } + } + + // This needs to be a separate 'if' statement, as, if this was an 'if-else' from above, if we were originally offline and using --monitor, we would never get to this point + if (online) { + // Check Application Version + log.vlog("Checking Application Version ..."); + checkApplicationVersion(); + // Initialise the OneDrive API + log.vlog("Attempting to initialise the OneDrive API ..."); + oneDriveApiInstance = new OneDriveApi(appConfig); + appConfig.apiWasInitialised = oneDriveApiInstance.initialise(); + if (appConfig.apiWasInitialised) { + log.vlog("The OneDrive API was initialised successfully"); + // Flag that we were able to initalise the API in the application config + oneDriveApiInstance.debugOutputConfiguredAPIItems(); + + // Need to configure the itemDB and syncEngineInstance for 'sync' and 'non-sync' operations + log.vlog("Opening the item database ..."); + // Configure the Item Database + itemDB = new ItemDatabase(runtimeDatabaseFile); + // Was the database successfully initialised? + if (!itemDB.isDatabaseInitialised()) { + // no .. destroy class + itemDB = null; + // exit application + return EXIT_FAILURE; + } + + // Initialise the syncEngine + syncEngineInstance = new SyncEngine(appConfig, itemDB, selectiveSync); + appConfig.syncEngineWasInitialised = syncEngineInstance.initialise(); + + // Are we not doing a --sync or a --monitor operation? Both of these will be false if they are not set + if ((!appConfig.getValueBool("synchronize")) && (!appConfig.getValueBool("monitor"))) { + + // Are we performing some sort of 'no-sync' task? + // - Are we obtaining the Office 365 Drive ID for a given Office 365 SharePoint Shared Library? + // - Are we createing a shareable link for an existing file on OneDrive? + // ... others ?? + + if (appConfig.getValueString("sharepoint_library_name") != "") { + syncEngineInstance.querySiteCollectionForDriveID(appConfig.getValueString("sharepoint_library_name")); + // Exit application + // Use exit scopes to shutdown API and cleanup data + return EXIT_SUCCESS; + } + + + // If we get to this point, we have not performed a 'no-sync' task .. + log.error("\n --sync or --monitor switches missing from your command line input. Please add one (not both) of these switches to your command line or use 'onedrive --help' for further assistance.\n"); + log.error("No OneDrive sync will be performed without one of these two arguments being present.\n"); + // Use exit scopes to shutdown API + // invalidSyncExit = true; + return EXIT_FAILURE; + } + // We do not need this instance, as the API was initialised, and individual instances are used during sync process + oneDriveApiInstance.shutdown(); + } else { + // API could not be initialised + log.error("The OneDrive API could not be initialised"); + return EXIT_FAILURE; + } + } + + // Configure the sync direcory based on the runtimeSyncDirectory configured directory + try { + if (!exists(runtimeSyncDirectory)) { + log.vdebug("runtimeSyncDirectory: Configured 'sync_dir' is missing locally. Creating: ", runtimeSyncDirectory); + try { + // Attempt to create the sync dir we have been configured with + mkdirRecurse(runtimeSyncDirectory); + // Configure the applicable permissions for the folder + log.vdebug("Setting directory permissions for: ", runtimeSyncDirectory); + runtimeSyncDirectory.setAttributes(appConfig.returnRequiredDirectoryPermisions()); + } catch (std.file.FileException e) { + // Creating the sync directory failed + log.error("ERROR: Unable to create local OneDrive 'sync_dir' - ", e.msg); + // Use exit scopes to shutdown API + return EXIT_FAILURE; + } + } + } catch (std.file.FileException e) { + // Creating the sync directory failed + log.error("ERROR: Unable to test the existence of the configured OneDrive 'sync_dir' - ", e.msg); + // Use exit scopes to shutdown API + return EXIT_FAILURE; + } + + // Change the working directory to the 'sync_dir' as configured + log.log("All application operations will be performed in: ", runtimeSyncDirectory); + chdir(runtimeSyncDirectory); + + // Do we need to validate the runtimeSyncDirectory to check for the presence of a '.nosync' file + checkForNoMountScenario(); + + // Set the default thread pool value - hard coded to 16 + defaultPoolThreads(to!int(appConfig.concurrentThreads)); + + // Is the sync engine initiallised correctly? + if (appConfig.syncEngineWasInitialised) { + // Configure some initial variables + string singleDirectoryPath; + string localPath = "."; + string remotePath = "/"; + + // Are we doing a single directory operation (--single-directory) ? + if (!appConfig.getValueString("single_directory").empty) { + // Set singleDirectoryPath + singleDirectoryPath = appConfig.getValueString("single_directory"); + + // Ensure that this is a normalised relative path to runtimeSyncDirectory + string normalisedRelativePath = replace(buildNormalizedPath(absolutePath(singleDirectoryPath)), buildNormalizedPath(absolutePath(runtimeSyncDirectory)), "." ); + + // The user provided a directory to sync within the configured 'sync_dir' path + // This also validates if the path being used exists online and/or does not have a 'case-insensitive match' + syncEngineInstance.setSingleDirectoryScope(normalisedRelativePath); + + // Does the directory we want to sync actually exist locally? + if (!exists(singleDirectoryPath)) { + // The requested path to use with --single-directory does not exist locally within the configured 'sync_dir' + log.logAndNotify("WARNING: The requested path for --single-directory does not exist locally. Creating requested path within ", runtimeSyncDirectory); + // Make the required --single-directory path locally + mkdirRecurse(singleDirectoryPath); + // Configure the applicable permissions for the folder + log.vdebug("Setting directory permissions for: ", singleDirectoryPath); + singleDirectoryPath.setAttributes(appConfig.returnRequiredDirectoryPermisions()); + } + + // Update the paths that we use to perform the sync actions + localPath = singleDirectoryPath; + remotePath = singleDirectoryPath; + + // Display that we are syncing from a specific path due to --single-directory + log.vlog("Syncing changes from this selected path: ", singleDirectoryPath); + } + + // Are we doing a --sync operation? This includes doing any --single-directory operations + if (appConfig.getValueBool("synchronize")) { + // Did the user specify --upload-only? + if (appConfig.getValueBool("upload_only")) { + // Perform the --upload-only sync process + performUploadOnlySyncProcess(localPath); + } + + // Did the user specify --download-only? + if (appConfig.getValueBool("download_only")) { + // Only download data from OneDrive + syncEngineInstance.syncOneDriveAccountToLocalDisk(); + // Perform the DB consistency check + // This will also delete any out-of-sync flagged items if configured to do so + syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); + // Do we cleanup local files? + // - Deletes of data from online will already have been performed, but what we are now doing is searching the local filesystem + // for any new data locally, that usually would be uploaded to OneDrive, but instead, because of the options being + // used, will need to be deleted from the local filesystem + if (appConfig.getValueBool("cleanup_local_files")) { + // Perform the filesystem walk + syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); + } + } + + // If no use of --upload-only or --download-only + if ((!appConfig.getValueBool("upload_only")) && (!appConfig.getValueBool("download_only"))) { + // Perform the standard sync process + performStandardSyncProcess(localPath); + } + + // Detail the outcome of the sync process + displaySyncOutcome(); + } + + // Are we doing a --monitor operation? + if (appConfig.getValueBool("monitor")) { + // What are the current values for the platform we are running on + // Max number of open files /proc/sys/fs/file-max + string maxOpenFiles = strip(readText("/proc/sys/fs/file-max")); + // What is the currently configured maximum inotify watches that can be used + // /proc/sys/user/max_inotify_watches + string maxInotifyWatches = strip(readText("/proc/sys/user/max_inotify_watches")); + + // Start the monitor process + log.log("OneDrive synchronisation interval (seconds): ", appConfig.getValueLong("monitor_interval")); + log.vlog("Maximum allowed open files: ", maxOpenFiles); + log.vlog("Maximum allowed inotify watches: ", maxInotifyWatches); + + // Configure the monitor class + Monitor filesystemMonitor = new Monitor(appConfig, selectiveSync); + + // Delegated function for when inotify detects a new local directory has been created + filesystemMonitor.onDirCreated = delegate(string path) { + // Handle .folder creation if skip_dotfiles is enabled + if ((appConfig.getValueBool("skip_dotfiles")) && (isDotFile(path))) { + log.vlog("[M] Skipping watching local path - .folder found & --skip-dot-files enabled: ", path); + } else { + log.vlog("[M] Local directory created: ", path); + try { + syncEngineInstance.scanLocalFilesystemPathForNewData(path); + } catch (CurlException e) { + log.vlog("Offline, cannot create remote dir!"); + } catch(Exception e) { + log.logAndNotify("Cannot create remote directory: ", e.msg); + } + } + }; + + // Delegated function for when inotify detects a local file has been changed + filesystemMonitor.onFileChanged = delegate(string path) { + log.vlog("[M] Local file changed: ", path); + try { + syncEngineInstance.scanLocalFilesystemPathForNewData(dirName(path)); + } catch (CurlException e) { + log.vlog("Offline, cannot upload changed item!"); + } catch(Exception e) { + log.logAndNotify("Cannot upload file changes/creation: ", e.msg); + } + }; + + // Delegated function for when inotify detects a delete event + filesystemMonitor.onDelete = delegate(string path) { + log.log("Received inotify delete event from operating system .. attempting item deletion as requested"); + log.vlog("[M] Local item deleted: ", path); + try { + syncEngineInstance.deleteByPath(path); + } catch (CurlException e) { + log.vlog("Offline, cannot delete item!"); + } catch(SyncException e) { + if (e.msg == "The item to delete is not in the local database") { + log.vlog("Item cannot be deleted from OneDrive because it was not found in the local database"); + } else { + log.logAndNotify("Cannot delete remote item: ", e.msg); + } + } catch(Exception e) { + log.logAndNotify("Cannot delete remote item: ", e.msg); + } + }; + + // Delegated function for when inotify detects a move event + filesystemMonitor.onMove = delegate(string from, string to) { + log.vlog("[M] Local item moved: ", from, " -> ", to); + try { + // Handle .folder -> folder if skip_dotfiles is enabled + if ((appConfig.getValueBool("skip_dotfiles")) && (isDotFile(from))) { + // .folder -> folder handling - has to be handled as a new folder + syncEngineInstance.scanLocalFilesystemPathForNewData(to); + } else { + syncEngineInstance.uploadMoveItem(from, to); + } + } catch (CurlException e) { + log.vlog("Offline, cannot move item!"); + } catch(Exception e) { + log.logAndNotify("Cannot move item: ", e.msg); + } + }; + + // Handle SIGINT and SIGTERM + signal(SIGINT, &exitHandler); + signal(SIGTERM, &exitHandler); + + // Initialise the filesystem monitor class + try { + log.log("Initialising filesystem inotify monitoring ..."); + filesystemMonitor.initialise(); + log.log("Performing initial syncronisation to ensure consistent local state ..."); + } catch (MonitorException e) { + // monitor class initialisation failed + log.error("ERROR: ", e.msg); + return EXIT_FAILURE; + } + + // Filesystem monitor loop + bool performMonitor = true; + ulong monitorLoopFullCount = 0; + ulong fullScanFrequencyLoopCount = 0; + ulong monitorLogOutputLoopCount = 0; + immutable auto checkOnlineInterval = dur!"seconds"(appConfig.getValueLong("monitor_interval")); + immutable auto githubCheckInterval = dur!"seconds"(86400); + immutable ulong fullScanFrequency = appConfig.getValueLong("monitor_fullscan_frequency"); + immutable ulong logOutputSupressionInterval = appConfig.getValueLong("monitor_log_frequency"); + MonoTime lastCheckTime = MonoTime.currTime(); + MonoTime lastGitHubCheckTime = MonoTime.currTime(); + string loopStartOutputMessage = "################################################## NEW LOOP ##################################################"; + string loopStopOutputMessage = "################################################ LOOP COMPLETE ###############################################"; + + while (performMonitor) { + + // Do we need to validate the runtimeSyncDirectory to check for the presence of a '.nosync' file - the disk may have been ejected .. + checkForNoMountScenario(); + + try { + // Process any inotify events + filesystemMonitor.update(true); + } catch (MonitorException e) { + // Catch any exceptions thrown by inotify / monitor engine + log.error("ERROR: The following inotify error was generated: ", e.msg); + } + + // Check for notifications pushed from Microsoft to the webhook + bool notificationReceived = false; + + // Check here for a webhook notification + + // Get the current time this loop is starting + auto currentTime = MonoTime.currTime(); + + // Do we perform a sync with OneDrive? + if (notificationReceived || (currentTime - lastCheckTime > checkOnlineInterval) || (monitorLoopFullCount == 0)) { + // Increment relevant counters + monitorLoopFullCount++; + fullScanFrequencyLoopCount++; + monitorLogOutputLoopCount++; + + // Do we flag to perform a full scan of the online data? + if (fullScanFrequencyLoopCount > fullScanFrequency) { + // set full scan trigger for true up + log.vdebug("Enabling Full Scan True Up (fullScanFrequencyLoopCount > fullScanFrequency), resetting fullScanFrequencyLoopCount = 1"); + fullScanFrequencyLoopCount = 1; + appConfig.fullScanTrueUpRequired = true; + } else { + // unset full scan trigger for true up + log.vdebug("Disabling Full Scan True Up"); + appConfig.fullScanTrueUpRequired = false; + } + + log.vdebug(loopStartOutputMessage); + log.vdebug("Total Run-Time Loop Number: ", monitorLoopFullCount); + log.vdebug("Full Scan Freqency Loop Number: ", fullScanFrequencyLoopCount); + SysTime startFunctionProcessingTime = Clock.currTime(); + log.vdebug("Start Monitor Loop Time: ", startFunctionProcessingTime); + + // Do we perform any monitor console logging output surpression? + // 'monitor_log_frequency' controls how often, in a non-verbose application output mode, how often + // the full output of what is occuring is done. This is done to lessen the 'verbosity' of non-verbose + // logging, but only when running in --monitor + if (monitorLogOutputLoopCount > logOutputSupressionInterval) { + // unsurpress the logging output + monitorLogOutputLoopCount = 1; + log.vdebug("Unsuppressing log output"); + appConfig.surpressLoggingOutput = false; + } else { + // do we surpress the logging output to absolute minimal + if (monitorLoopFullCount == 1) { + // application startup with --monitor + log.vdebug("Unsuppressing initial sync log output"); + appConfig.surpressLoggingOutput = false; + } else { + // only surpress if we are not doing --verbose or higher + if (log.verbose == 0) { + log.vdebug("Suppressing --monitor log output"); + appConfig.surpressLoggingOutput = true; + } else { + log.vdebug("Unsuppressing log output"); + appConfig.surpressLoggingOutput = false; + } + } + } + + // How long has the application been running for? + auto elapsedTime = Clock.currTime() - applicationStartTime; + log.vdebug("Application run-time thus far: ", elapsedTime); + + // Need to re-validate that the client is still online for this loop + if (testInternetReachability(appConfig)) { + // Starting a sync + log.log("Starting a sync with Microsoft OneDrive"); + + // Did the user specify --upload-only? + if (appConfig.getValueBool("upload_only")) { + // Perform the --upload-only sync process + performUploadOnlySyncProcess(localPath, filesystemMonitor); + } else { + // Perform the standard sync process + performStandardSyncProcess(localPath, filesystemMonitor); + } + + // Discard any inotify events generated as part of any sync operation + filesystemMonitor.update(false); + + // Detail the outcome of the sync process + displaySyncOutcome(); + + if (appConfig.fullScanTrueUpRequired) { + // Write WAL and SHM data to file for this loop + log.vdebug("Merge contents of WAL and SHM files into main database file"); + itemDB.performVacuum(); + } + } else { + // Not online + log.log("Microsoft OneDrive service is not reachable at this time. Will re-try on next loop attempt."); + } + + // Output end of loop processing times + SysTime endFunctionProcessingTime = Clock.currTime(); + log.vdebug("End Monitor Loop Time: ", endFunctionProcessingTime); + log.vdebug("Elapsed Monitor Loop Processing Time: ", (endFunctionProcessingTime - startFunctionProcessingTime)); + + // Display memory details before cleanup + if (displayMemoryUsage) log.displayMemoryUsagePreGC(); + // Perform Garbage Cleanup + GC.collect(); + // Return free memory to the OS + GC.minimize(); + // Display memory details after cleanup + if (displayMemoryUsage) log.displayMemoryUsagePostGC(); + + // Log that this loop is complete + log.vdebug(loopStopOutputMessage); + // performSync complete, set lastCheckTime to current time + lastCheckTime = MonoTime.currTime(); + + // Developer break via config option + if (appConfig.getValueLong("monitor_max_loop") > 0) { + // developer set option to limit --monitor loops + if (monitorLoopFullCount == (appConfig.getValueLong("monitor_max_loop"))) { + performMonitor = false; + log.log("Exiting after ", monitorLoopFullCount, " loops due to developer set option"); + } + } + } + // Sleep the monitor thread for 1 second, loop around and pick up any inotify changes + Thread.sleep(dur!"seconds"(1)); + } + } + } else { + // Exit application as the sync engine could not be initialised + log.error("Application Sync Engine could not be initialised correctly"); + // Use exit scope + return EXIT_FAILURE; + } + + /** + // Before we exit, if we are using --dry-run, clean up the local syste, + if (appConfig.getValueBool("dry_run")) { + // Cleanup any existing dry-run elements ... these should never be left hanging around + cleanupDryRunDatabaseFiles(appConfig.databaseFilePathDryRun); + } + **/ + + // Exit application using exit scope + if (!syncEngineInstance.syncFailures) { + return EXIT_SUCCESS; + } else { + return EXIT_FAILURE; + } +} + +void performUploadOnlySyncProcess(string localPath, Monitor filesystemMonitor = null) { + // Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive + syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); + if (appConfig.getValueBool("monitor")) { + // Handle any inotify events whilst the DB was being scanned + filesystemMonitor.update(true); + } + + // Scan the configured 'sync_dir' for new data to upload + syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); + if (appConfig.getValueBool("monitor")) { + // Handle any new inotify events whilst the local filesystem was being scanned + filesystemMonitor.update(true); + } +} + +void performStandardSyncProcess(string localPath, Monitor filesystemMonitor = null) { + + // If we are performing log supression, output this message so the user knows what is happening + if (appConfig.surpressLoggingOutput) { + log.log("Syncing changes from Microsoft OneDrive ..."); + } + + // Zero out these arrays + syncEngineInstance.fileDownloadFailures = []; + syncEngineInstance.fileUploadFailures = []; + + // Which way do we sync first? + // OneDrive first then local changes (normal operational process that uses OneDrive as the source of truth) + // Local First then OneDrive changes (alternate operation process to use local files as source of truth) + if (appConfig.getValueBool("local_first")) { + // Local data first + // Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive + syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); + if (appConfig.getValueBool("monitor")) { + // Handle any inotify events whilst the DB was being scanned + filesystemMonitor.update(true); + } + + // Scan the configured 'sync_dir' for new data to upload to OneDrive + syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); + if (appConfig.getValueBool("monitor")) { + // Handle any new inotify events whilst the local filesystem was being scanned + filesystemMonitor.update(true); + } + + // Download data from OneDrive last + syncEngineInstance.syncOneDriveAccountToLocalDisk(); + if (appConfig.getValueBool("monitor")) { + // Cancel out any inotify events from downloading data + filesystemMonitor.update(false); + } + } else { + // Normal sync + // Download data from OneDrive first + syncEngineInstance.syncOneDriveAccountToLocalDisk(); + if (appConfig.getValueBool("monitor")) { + // Cancel out any inotify events from downloading data + filesystemMonitor.update(false); + } + + // Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive + syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); + if (appConfig.getValueBool("monitor")) { + // Handle any inotify events whilst the DB was being scanned + filesystemMonitor.update(true); + } + + // Scan the configured 'sync_dir' for new data to upload to OneDrive + syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); + if (appConfig.getValueBool("monitor")) { + // Handle any new inotify events whilst the local filesystem was being scanned + filesystemMonitor.update(true); + } + + // Make sure we sync any DB data to this point, but only if not in --monitor mode + // In --monitor mode, this is handled within the 'loop', based on when the full scan true up is being performed + if (!appConfig.getValueBool("monitor")) { + itemDB.performVacuum(); + } + + // Perform the final true up scan to ensure we have correctly replicated the current online state locally + if (!appConfig.surpressLoggingOutput) { + log.log("Performing a final true-up scan of online data from Microsoft OneDrive"); + } + // We pass in the 'appConfig.fullScanTrueUpRequired' value which then flags do we use the configured 'deltaLink' + // If 'appConfig.fullScanTrueUpRequired' is true, we do not use the 'deltaLink' if we are in --monitor mode, thus forcing a full scan true up + syncEngineInstance.syncOneDriveAccountToLocalDisk(); + if (appConfig.getValueBool("monitor")) { + // Cancel out any inotify events from downloading data + filesystemMonitor.update(false); + } + } +} + +void displaySyncOutcome() { + + // Detail any download or upload transfer failures + syncEngineInstance.displaySyncFailures(); + + // Sync is either complete or partially complete + if (!syncEngineInstance.syncFailures) { + // No download or upload issues + if (!appConfig.getValueBool("monitor")) writeln(); // Add an additional line break so that this is clear when using --sync + log.log("Sync with Microsoft OneDrive is complete"); + } else { + log.log("\nSync with Microsoft OneDrive has completed, however there are items that failed to sync."); + // Due to how the OneDrive API works 'changes' such as add new files online, rename files online, delete files online are only sent once when using the /delta API call. + // That we failed to download it, we need to track that, and then issue a --resync to download any of these failed files .. unfortunate, but there is no easy way here + if (!syncEngineInstance.fileDownloadFailures.empty) { + log.log("To fix any download failures you may need to perform a --resync to ensure this system is correctly synced with your Microsoft OneDrive Account"); + } + if (!syncEngineInstance.fileUploadFailures.empty) { + log.log("To fix any upload failures you may need to perform a --resync to ensure this system is correctly synced with your Microsoft OneDrive Account"); + } + // So that from a logging perspective these messages are clear, add a line break in + writeln(); + } +} + +void processResyncDatabaseRemoval(string databaseFilePathToRemove) { + log.vdebug("Testing if we have exclusive access to local database file"); + // Are we the only running instance? Test that we can open the database file path + itemDB = new ItemDatabase(databaseFilePathToRemove); + + // did we successfully initialise the database class? + if (!itemDB.isDatabaseInitialised()) { + // no .. destroy class + itemDB = null; + // exit application - void function, force exit this way + exit(-1); + } + + // If we have exclusive access we will not have exited + // destroy access test + destroy(itemDB); + // delete application sync state + log.log("Deleting the saved application sync status ..."); + if (!appConfig.getValueBool("dry_run")) { + safeRemove(databaseFilePathToRemove); + } else { + // --dry-run scenario ... technically we should not be making any local file changes ....... + log.log("DRY RUN: Not removing the saved application sync status"); + } +} + +void cleanupDryRunDatabaseFiles(string dryRunDatabaseFile) { + // Temp variables + string dryRunShmFile = dryRunDatabaseFile ~ "-shm"; + string dryRunWalFile = dryRunDatabaseFile ~ "-wal"; + + // If the dry run database exists, clean this up + if (exists(dryRunDatabaseFile)) { + // remove the existing file + log.vdebug("DRY-RUN: Removing items-dryrun.sqlite3 as it still exists for some reason"); + safeRemove(dryRunDatabaseFile); + } + + // silent cleanup of shm files if it exists + if (exists(dryRunShmFile)) { + // remove items-dryrun.sqlite3-shm + log.vdebug("DRY-RUN: Removing items-dryrun.sqlite3-shm as it still exists for some reason"); + safeRemove(dryRunShmFile); + } + + // silent cleanup of wal files if it exists + if (exists(dryRunWalFile)) { + // remove items-dryrun.sqlite3-wal + log.vdebug("DRY-RUN: Removing items-dryrun.sqlite3-wal as it still exists for some reason"); + safeRemove(dryRunWalFile); + } +} + +void checkForNoMountScenario() { + // If this is a 'mounted' folder, the 'mount point' should have this file to help the application stop any action to preserve data because the drive to mount is not currently mounted + if (appConfig.getValueBool("check_nomount")) { + // we were asked to check the mount point for the presence of a '.nosync' file + if (exists(".nosync")) { + log.logAndNotify("ERROR: .nosync file found in directory mount point. Aborting application startup process to safeguard data."); + exit(EXIT_FAILURE); + } + } +} + +// Getting around the @nogc problem +// https://p0nce.github.io/d-idioms/#Bypassing-@nogc +auto assumeNoGC(T) (T t) if (isFunctionPointer!T || isDelegate!T) { + enum attrs = functionAttributes!T | FunctionAttribute.nogc; + return cast(SetFunctionAttributes!(T, functionLinkage!T, attrs)) t; +} + +// Catch CTRL-C +extern(C) nothrow @nogc @system void exitHandler(int value) { + try { + assumeNoGC ( () { + log.log("Got termination signal, performing clean up"); + // was itemDb initialised? + if (itemDB.isDatabaseInitialised()) { + // Make sure the .wal file is incorporated into the main db before we exit + log.log("Shutting down DB connection and merging temporary data"); + itemDB.performVacuum(); + destroy(itemDB); + } + })(); + } catch(Exception e) {} + exit(0); +} \ No newline at end of file diff --git a/src/monitor.d b/src/monitor.d new file mode 100644 index 00000000..beb6baff --- /dev/null +++ b/src/monitor.d @@ -0,0 +1,413 @@ +// What is this module called? +module monitor; + +// What does this module require to function? +import core.stdc.errno; +import core.stdc.stdlib; +import core.sys.linux.sys.inotify; +import core.sys.posix.poll; +import core.sys.posix.unistd; +import std.algorithm; +import std.exception; +import std.file; +import std.path; +import std.regex; +import std.stdio; +import std.string; +import std.conv; + +// What other modules that we have created do we need to import? +import config; +import util; +import log; +import clientSideFiltering; + +// Relevant inotify events +private immutable uint32_t mask = IN_CLOSE_WRITE | IN_CREATE | IN_DELETE | IN_MOVE | IN_IGNORED | IN_Q_OVERFLOW; + +class MonitorException: ErrnoException { + @safe this(string msg, string file = __FILE__, size_t line = __LINE__) { + super(msg, file, line); + } +} + +final class Monitor { + // Class variables + ApplicationConfig appConfig; + ClientSideFiltering selectiveSync; + + // Are we verbose in logging output + bool verbose = false; + // skip symbolic links + bool skip_symlinks = false; + // check for .nosync if enabled + bool check_nosync = false; + + // Configure Private Class Variables + // inotify file descriptor + private int fd; + // map every inotify watch descriptor to its directory + private string[int] wdToDirName; + // map the inotify cookies of move_from events to their path + private string[int] cookieToPath; + // buffer to receive the inotify events + private void[] buffer; + + // Configure function delegates + void delegate(string path) onDirCreated; + void delegate(string path) onFileChanged; + void delegate(string path) onDelete; + void delegate(string from, string to) onMove; + + // Configure the class varaible to consume the application configuration including selective sync + this(ApplicationConfig appConfig, ClientSideFiltering selectiveSync) { + this.appConfig = appConfig; + this.selectiveSync = selectiveSync; + } + + // Initialise the monitor class + void initialise() { + // Configure the variables + skip_symlinks = appConfig.getValueBool("skip_symlinks"); + check_nosync = appConfig.getValueBool("check_nosync"); + if (appConfig.getValueLong("verbose") > 0) { + verbose = true; + } + + assert(onDirCreated && onFileChanged && onDelete && onMove); + fd = inotify_init(); + if (fd < 0) throw new MonitorException("inotify_init failed"); + if (!buffer) buffer = new void[4096]; + + // from which point do we start watching for changes? + string monitorPath; + if (appConfig.getValueString("single_directory") != ""){ + // single directory in use, monitor only this path + monitorPath = "./" ~ appConfig.getValueString("single_directory"); + } else { + // default + monitorPath = "."; + } + addRecursive(monitorPath); + } + + // Shutdown the monitor class + void shutdown() { + if (fd > 0) close(fd); + wdToDirName = null; + } + + // Recursivly add this path to be monitored + private void addRecursive(string dirname) { + // skip non existing/disappeared items + if (!exists(dirname)) { + log.vlog("Not adding non-existing/disappeared directory: ", dirname); + return; + } + + // Skip the monitoring of any user filtered items + if (dirname != ".") { + // Is the directory name a match to a skip_dir entry? + // The path that needs to be checked needs to include the '/' + // This due to if the user has specified in skip_dir an exclusive path: '/path' - that is what must be matched + if (isDir(dirname)) { + if (selectiveSync.isDirNameExcluded(dirname.strip('.'))) { + // dont add a watch for this item + log.vdebug("Skipping monitoring due to skip_dir match: ", dirname); + return; + } + } + if (isFile(dirname)) { + // Is the filename a match to a skip_file entry? + // The path that needs to be checked needs to include the '/' + // This due to if the user has specified in skip_file an exclusive path: '/path/file' - that is what must be matched + if (selectiveSync.isFileNameExcluded(dirname.strip('.'))) { + // dont add a watch for this item + log.vdebug("Skipping monitoring due to skip_file match: ", dirname); + return; + } + } + // is the path exluded by sync_list? + if (selectiveSync.isPathExcludedViaSyncList(buildNormalizedPath(dirname))) { + // dont add a watch for this item + log.vdebug("Skipping monitoring due to sync_list match: ", dirname); + return; + } + } + + // skip symlinks if configured + if (isSymlink(dirname)) { + // if config says so we skip all symlinked items + if (skip_symlinks) { + // dont add a watch for this directory + return; + } + } + + // Do we need to check for .nosync? Only if check_nosync is true + if (check_nosync) { + if (exists(buildNormalizedPath(dirname) ~ "/.nosync")) { + log.vlog("Skipping watching path - .nosync found & --check-for-nosync enabled: ", buildNormalizedPath(dirname)); + return; + } + } + + // passed all potential exclusions + // add inotify watch for this path / directory / file + log.vdebug("Calling add() for this dirname: ", dirname); + add(dirname); + + // if this is a directory, recursivly add this path + if (isDir(dirname)) { + // try and get all the directory entities for this path + try { + auto pathList = dirEntries(dirname, SpanMode.shallow, false); + foreach(DirEntry entry; pathList) { + if (entry.isDir) { + log.vdebug("Calling addRecursive() for this directory: ", entry.name); + addRecursive(entry.name); + } + } + // catch any error which is generated + } catch (std.file.FileException e) { + // Standard filesystem error + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + return; + } catch (Exception e) { + // Issue #1154 handling + // Need to check for: Failed to stat file in error message + if (canFind(e.msg, "Failed to stat file")) { + // File system access issue + log.error("ERROR: The local file system returned an error with the following message:"); + log.error(" Error Message: ", e.msg); + log.error("ACCESS ERROR: Please check your UID and GID access to this file, as the permissions on this file is preventing this application to read it"); + log.error("\nFATAL: Exiting application to avoid deleting data due to local file system access issues\n"); + // Must exit here + exit(-1); + } else { + // some other error + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + return; + } + } + } + } + + // Add this path to be monitored + private void add(string pathname) { + int wd = inotify_add_watch(fd, toStringz(pathname), mask); + if (wd < 0) { + if (errno() == ENOSPC) { + // Get the current value + ulong maxInotifyWatches = to!int(strip(readText("/proc/sys/user/max_inotify_watches"))); + log.log("The user limit on the total number of inotify watches has been reached."); + log.log("Your current limit of inotify watches is: ", maxInotifyWatches); + log.log("It is recommended that you change the max number of inotify watches to at least double your existing value."); + log.log("To change the current max number of watches to " , (maxInotifyWatches * 2) , " run:"); + log.log("EXAMPLE: sudo sysctl fs.inotify.max_user_watches=", (maxInotifyWatches * 2)); + } + if (errno() == 13) { + if ((selectiveSync.getSkipDotfiles()) && (isDotFile(pathname))) { + // no misleading output that we could not add a watch due to permission denied + return; + } else { + log.vlog("WARNING: inotify_add_watch failed - permission denied: ", pathname); + return; + } + } + // Flag any other errors + log.error("ERROR: inotify_add_watch failed: ", pathname); + return; + } + + // Add path to inotify watch - required regardless if a '.folder' or 'folder' + wdToDirName[wd] = buildNormalizedPath(pathname) ~ "/"; + log.vdebug("inotify_add_watch successfully added for: ", pathname); + + // Do we log that we are monitoring this directory? + if (isDir(pathname)) { + // This is a directory + // is the path exluded if skip_dotfiles configured and path is a .folder? + if ((selectiveSync.getSkipDotfiles()) && (isDotFile(pathname))) { + // no misleading output that we are monitoring this directory + return; + } + // Log that this is directory is being monitored + log.vlog("Monitoring directory: ", pathname); + } + } + + // Remove a watch descriptor + private void remove(int wd) { + assert(wd in wdToDirName); + int ret = inotify_rm_watch(fd, wd); + if (ret < 0) throw new MonitorException("inotify_rm_watch failed"); + log.vlog("Monitored directory removed: ", wdToDirName[wd]); + wdToDirName.remove(wd); + } + + // Remove the watch descriptors associated to the given path + private void remove(const(char)[] path) { + path ~= "/"; + foreach (wd, dirname; wdToDirName) { + if (dirname.startsWith(path)) { + int ret = inotify_rm_watch(fd, wd); + if (ret < 0) throw new MonitorException("inotify_rm_watch failed"); + wdToDirName.remove(wd); + log.vlog("Monitored directory removed: ", dirname); + } + } + } + + // Return the file path from an inotify event + private string getPath(const(inotify_event)* event) { + string path = wdToDirName[event.wd]; + if (event.len > 0) path ~= fromStringz(event.name.ptr); + log.vdebug("inotify path event for: ", path); + return path; + } + + // Update + void update(bool useCallbacks = true) { + + pollfd fds = { + fd: fd, + events: POLLIN + }; + + while (true) { + int ret = poll(&fds, 1, 0); + if (ret == -1) throw new MonitorException("poll failed"); + else if (ret == 0) break; // no events available + + size_t length = read(fd, buffer.ptr, buffer.length); + if (length == -1) throw new MonitorException("read failed"); + + int i = 0; + while (i < length) { + inotify_event *event = cast(inotify_event*) &buffer[i]; + string path; + string evalPath; + // inotify event debug + log.vdebug("inotify event wd: ", event.wd); + log.vdebug("inotify event mask: ", event.mask); + log.vdebug("inotify event cookie: ", event.cookie); + log.vdebug("inotify event len: ", event.len); + log.vdebug("inotify event name: ", event.name); + if (event.mask & IN_ACCESS) log.vdebug("inotify event flag: IN_ACCESS"); + if (event.mask & IN_MODIFY) log.vdebug("inotify event flag: IN_MODIFY"); + if (event.mask & IN_ATTRIB) log.vdebug("inotify event flag: IN_ATTRIB"); + if (event.mask & IN_CLOSE_WRITE) log.vdebug("inotify event flag: IN_CLOSE_WRITE"); + if (event.mask & IN_CLOSE_NOWRITE) log.vdebug("inotify event flag: IN_CLOSE_NOWRITE"); + if (event.mask & IN_MOVED_FROM) log.vdebug("inotify event flag: IN_MOVED_FROM"); + if (event.mask & IN_MOVED_TO) log.vdebug("inotify event flag: IN_MOVED_TO"); + if (event.mask & IN_CREATE) log.vdebug("inotify event flag: IN_CREATE"); + if (event.mask & IN_DELETE) log.vdebug("inotify event flag: IN_DELETE"); + if (event.mask & IN_DELETE_SELF) log.vdebug("inotify event flag: IN_DELETE_SELF"); + if (event.mask & IN_MOVE_SELF) log.vdebug("inotify event flag: IN_MOVE_SELF"); + if (event.mask & IN_UNMOUNT) log.vdebug("inotify event flag: IN_UNMOUNT"); + if (event.mask & IN_Q_OVERFLOW) log.vdebug("inotify event flag: IN_Q_OVERFLOW"); + if (event.mask & IN_IGNORED) log.vdebug("inotify event flag: IN_IGNORED"); + if (event.mask & IN_CLOSE) log.vdebug("inotify event flag: IN_CLOSE"); + if (event.mask & IN_MOVE) log.vdebug("inotify event flag: IN_MOVE"); + if (event.mask & IN_ONLYDIR) log.vdebug("inotify event flag: IN_ONLYDIR"); + if (event.mask & IN_DONT_FOLLOW) log.vdebug("inotify event flag: IN_DONT_FOLLOW"); + if (event.mask & IN_EXCL_UNLINK) log.vdebug("inotify event flag: IN_EXCL_UNLINK"); + if (event.mask & IN_MASK_ADD) log.vdebug("inotify event flag: IN_MASK_ADD"); + if (event.mask & IN_ISDIR) log.vdebug("inotify event flag: IN_ISDIR"); + if (event.mask & IN_ONESHOT) log.vdebug("inotify event flag: IN_ONESHOT"); + if (event.mask & IN_ALL_EVENTS) log.vdebug("inotify event flag: IN_ALL_EVENTS"); + + // skip events that need to be ignored + if (event.mask & IN_IGNORED) { + // forget the directory associated to the watch descriptor + wdToDirName.remove(event.wd); + goto skip; + } else if (event.mask & IN_Q_OVERFLOW) { + throw new MonitorException("Inotify overflow, events missing"); + } + + // if the event is not to be ignored, obtain path + path = getPath(event); + // configure the skip_dir & skip skip_file comparison item + evalPath = path.strip('.'); + + // Skip events that should be excluded based on application configuration + // We cant use isDir or isFile as this information is missing from the inotify event itself + // Thus this causes a segfault when attempting to query this - https://github.com/abraunegg/onedrive/issues/995 + + // Based on the 'type' of event & object type (directory or file) check that path against the 'right' user exclusions + // Directory events should only be compared against skip_dir and file events should only be compared against skip_file + if (event.mask & IN_ISDIR) { + // The event in question contains IN_ISDIR event mask, thus highly likely this is an event on a directory + // This due to if the user has specified in skip_dir an exclusive path: '/path' - that is what must be matched + if (selectiveSync.isDirNameExcluded(evalPath)) { + // The path to evaluate matches a path that the user has configured to skip + goto skip; + } + } else { + // The event in question missing the IN_ISDIR event mask, thus highly likely this is an event on a file + // This due to if the user has specified in skip_file an exclusive path: '/path/file' - that is what must be matched + if (selectiveSync.isFileNameExcluded(evalPath)) { + // The path to evaluate matches a file that the user has configured to skip + goto skip; + } + } + + // is the path, excluded via sync_list + if (selectiveSync.isPathExcludedViaSyncList(path)) { + // The path to evaluate matches a directory or file that the user has configured not to include in the sync + goto skip; + } + + // handle the inotify events + if (event.mask & IN_MOVED_FROM) { + log.vdebug("event IN_MOVED_FROM: ", path); + cookieToPath[event.cookie] = path; + } else if (event.mask & IN_MOVED_TO) { + log.vdebug("event IN_MOVED_TO: ", path); + if (event.mask & IN_ISDIR) addRecursive(path); + auto from = event.cookie in cookieToPath; + if (from) { + cookieToPath.remove(event.cookie); + if (useCallbacks) onMove(*from, path); + } else { + // item moved from the outside + if (event.mask & IN_ISDIR) { + if (useCallbacks) onDirCreated(path); + } else { + if (useCallbacks) onFileChanged(path); + } + } + } else if (event.mask & IN_CREATE) { + log.vdebug("event IN_CREATE: ", path); + if (event.mask & IN_ISDIR) { + addRecursive(path); + if (useCallbacks) onDirCreated(path); + } + } else if (event.mask & IN_DELETE) { + log.vdebug("event IN_DELETE: ", path); + if (useCallbacks) onDelete(path); + } else if ((event.mask & IN_CLOSE_WRITE) && !(event.mask & IN_ISDIR)) { + log.vdebug("event IN_CLOSE_WRITE and ...: ", path); + if (useCallbacks) onFileChanged(path); + } else { + log.vdebug("event unhandled: ", path); + assert(0); + } + + skip: + i += inotify_event.sizeof + event.len; + } + // assume that the items moved outside the watched directory have been deleted + foreach (cookie, path; cookieToPath) { + log.vdebug("deleting (post loop): ", path); + if (useCallbacks) onDelete(path); + remove(path); + cookieToPath.remove(cookie); + } + // Debug Log that all inotify events are flushed + log.vdebug("inotify events flushed"); + } + } +} diff --git a/src/notifications/README b/src/notifications/README new file mode 100644 index 00000000..7385cb31 --- /dev/null +++ b/src/notifications/README @@ -0,0 +1,10 @@ +The files in this directory have been obtained form the following places: + +dnotify.d + https://github.com/Dav1dde/dnotify/blob/master/dnotify.d + License: Creative Commons Zro 1.0 Universal + see https://github.com/Dav1dde/dnotify/blob/master/LICENSE + +notify.d + https://github.com/D-Programming-Deimos/libnotify/blob/master/deimos/notify/notify.d + License: GNU Lesser General Public License (LGPL) 2.1 or upwards, see file diff --git a/src/notifications/dnotify.d b/src/notifications/dnotify.d new file mode 100644 index 00000000..1cc09356 --- /dev/null +++ b/src/notifications/dnotify.d @@ -0,0 +1,323 @@ +module dnotify; + +private { + import std.string : toStringz; + import std.conv : to; + import std.traits : isPointer, isArray; + import std.variant : Variant; + import std.array : appender; + + import deimos.notify.notify; +} + +public import deimos.notify.notify : NOTIFY_EXPIRES_DEFAULT, NOTIFY_EXPIRES_NEVER, + NotifyUrgency; + + +version(NoPragma) { +} else { + pragma(lib, "notify"); + pragma(lib, "gmodule"); + pragma(lib, "glib-2.0"); +} + +extern (C) { + private void g_free(void* mem); + private void g_list_free(GList* glist); +} + +version(NoGdk) { +} else { + version(NoPragma) { + } else { + pragma(lib, "gdk_pixbuf"); + } + + private: + extern (C) { + GdkPixbuf* gdk_pixbuf_new_from_file(const(char)* filename, GError **error); + } +} + +class NotificationError : Exception { + string message; + GError* gerror; + + this(GError* gerror) { + this.message = to!(string)(gerror.message); + this.gerror = gerror; + + super(this.message); + } + + this(string message) { + this.message = message; + + super(message); + } +} + +bool check_availability() { + // notify_init might return without dbus server actually started + // try to check for running dbus server + char **ret_name; + char **ret_vendor; + char **ret_version; + char **ret_spec_version; + bool ret; + try { + return notify_get_server_info(ret_name, ret_vendor, ret_version, ret_spec_version); + } catch (NotificationError e) { + throw new NotificationError("Cannot find dbus server!"); + } +} + +void init(in char[] name) { + notify_init(name.toStringz()); +} + +alias notify_is_initted is_initted; +alias notify_uninit uninit; + +static this() { + init(__FILE__); +} + +static ~this() { + uninit(); +} + +string get_app_name() { + return to!(string)(notify_get_app_name()); +} + +void set_app_name(in char[] app_name) { + notify_set_app_name(app_name.toStringz()); +} + +string[] get_server_caps() { + auto result = appender!(string[])(); + + GList* list = notify_get_server_caps(); + if(list !is null) { + for(GList* c = list; c !is null; c = c.next) { + result.put(to!(string)(cast(char*)c.data)); + g_free(c.data); + } + + g_list_free(list); + } + + return result.data; +} + +struct ServerInfo { + string name; + string vendor; + string version_; + string spec_version; +} + +ServerInfo get_server_info() { + char* name; + char* vendor; + char* version_; + char* spec_version; + notify_get_server_info(&name, &vendor, &version_, &spec_version); + + scope(exit) { + g_free(name); + g_free(vendor); + g_free(version_); + g_free(spec_version); + } + + return ServerInfo(to!string(name), to!string(vendor), to!string(version_), to!string(spec_version)); +} + + +struct Action { + const(char[]) id; + const(char[]) label; + NotifyActionCallback callback; + void* user_ptr; +} + + +class Notification { + NotifyNotification* notify_notification; + + const(char)[] summary; + const(char)[] body_; + const(char)[] icon; + + bool closed = true; + + private int _timeout = NOTIFY_EXPIRES_DEFAULT; + const(char)[] _category; + NotifyUrgency _urgency; + GdkPixbuf* _image; + Variant[const(char)[]] _hints; + const(char)[] _app_name; + Action[] _actions; + + this(in char[] summary, in char[] body_, in char[] icon="") + in { assert(is_initted(), "call dnotify.init() before using Notification"); } + do { + this.summary = summary; + this.body_ = body_; + this.icon = icon; + notify_notification = notify_notification_new(summary.toStringz(), body_.toStringz(), icon.toStringz()); + } + + bool update(in char[] summary, in char[] body_, in char[] icon="") { + this.summary = summary; + this.body_ = body_; + this.icon = icon; + return notify_notification_update(notify_notification, summary.toStringz(), body_.toStringz(), icon.toStringz()); + } + + void show() { + GError* ge; + + if(!notify_notification_show(notify_notification, &ge)) { + throw new NotificationError(ge); + } + } + + @property int timeout() { return _timeout; } + @property void timeout(int timeout) { + this._timeout = timeout; + notify_notification_set_timeout(notify_notification, timeout); + } + + @property const(char[]) category() { return _category; } + @property void category(in char[] category) { + this._category = category; + notify_notification_set_category(notify_notification, category.toStringz()); + } + + @property NotifyUrgency urgency() { return _urgency; } + @property void urgency(NotifyUrgency urgency) { + this._urgency = urgency; + notify_notification_set_urgency(notify_notification, urgency); + } + + + void set_image(GdkPixbuf* pixbuf) { + notify_notification_set_image_from_pixbuf(notify_notification, pixbuf); + //_image = pixbuf; + } + + version(NoGdk) { + } else { + void set_image(in char[] filename) { + GError* ge; + // TODO: free pixbuf + GdkPixbuf* pixbuf = gdk_pixbuf_new_from_file(filename.toStringz(), &ge); + + if(pixbuf is null) { + if(ge is null) { + throw new NotificationError("Unable to load file: " ~ filename.idup); + } else { + throw new NotificationError(ge); + } + } + assert(notify_notification !is null); + notify_notification_set_image_from_pixbuf(notify_notification, pixbuf); // TODO: fix segfault + //_image = pixbuf; + } + } + + @property GdkPixbuf* image() { return _image; } + + // using deprecated set_hint_* functions (GVariant is an opaque structure, which needs the glib) + void set_hint(T)(in char[] key, T value) { + static if(is(T == int)) { + notify_notification_set_hint_int32(notify_notification, key, value); + } else static if(is(T == uint)) { + notify_notification_set_hint_uint32(notify_notification, key, value); + } else static if(is(T == double)) { + notify_notification_set_hint_double(notify_notification, key, value); + } else static if(is(T : const(char)[])) { + notify_notification_set_hint_string(notify_notification, key, value.toStringz()); + } else static if(is(T == ubyte)) { + notify_notification_set_hint_byte(notify_notification, key, value); + } else static if(is(T == ubyte[])) { + notify_notification_set_hint_byte_array(notify_notification, key, value.ptr, value.length); + } else { + static assert(false, "unsupported value for Notification.set_hint"); + } + + _hints[key] = Variant(value); + } + + // unset hint? + + Variant get_hint(in char[] key) { + return _hints[key]; + } + + @property const(char)[] app_name() { return _app_name; } + @property void app_name(in char[] name) { + this._app_name = app_name; + notify_notification_set_app_name(notify_notification, app_name.toStringz()); + } + + void add_action(T)(in char[] action, in char[] label, NotifyActionCallback callback, T user_data) { + static if(isPointer!T) { + void* user_ptr = cast(void*)user_data; + } else static if(isArray!T) { + void* user_ptr = cast(void*)user_data.ptr; + } else { + void* user_ptr = cast(void*)&user_data; + } + + notify_notification_add_action(notify_notification, action.toStringz(), label.toStringz(), + callback, user_ptr, null); + + _actions ~= Action(action, label, callback, user_ptr); + } + + void add_action()(Action action) { + notify_notification_add_action(notify_notification, action.id.toStringz(), action.label.toStringz(), + action.callback, action.user_ptr, null); + + _actions ~= action; + } + + @property Action[] actions() { return _actions; } + + void clear_actions() { + notify_notification_clear_actions(notify_notification); + } + + void close() { + GError* ge; + + if(!notify_notification_close(notify_notification, &ge)) { + throw new NotificationError(ge); + } + } + + @property int closed_reason() { + return notify_notification_get_closed_reason(notify_notification); + } +} + + +version(TestMain) { + import std.stdio; + + void main() { + writeln(get_app_name()); + set_app_name("bla"); + writeln(get_app_name()); + writeln(get_server_caps()); + writeln(get_server_info()); + + auto n = new Notification("foo", "bar", "notification-message-im"); + n.timeout = 3; + n.show(); + } +} diff --git a/src/notifications/notify.d b/src/notifications/notify.d new file mode 100644 index 00000000..c549e397 --- /dev/null +++ b/src/notifications/notify.d @@ -0,0 +1,195 @@ +/** + * Copyright (C) 2004-2006 Christian Hammond + * Copyright (C) 2010 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +module deimos.notify.notify; + + +enum NOTIFY_VERSION_MAJOR = 0; +enum NOTIFY_VERSION_MINOR = 7; +enum NOTIFY_VERSION_MICRO = 5; + +template NOTIFY_CHECK_VERSION(int major, int minor, int micro) { + enum NOTIFY_CHECK_VERSION = ((NOTIFY_VERSION_MAJOR > major) || + (NOTIFY_VERSION_MAJOR == major && NOTIFY_VERSION_MINOR > minor) || + (NOTIFY_VERSION_MAJOR == major && NOTIFY_VERSION_MINOR == minor && + NOTIFY_VERSION_MICRO >= micro)); +} + + +alias ulong GType; +alias void function(void*) GFreeFunc; + +struct GError { + uint domain; + int code; + char* message; +} + +struct GList { + void* data; + GList* next; + GList* prev; +} + +// dummies +struct GdkPixbuf {} +struct GObject {} +struct GObjectClass {} +struct GVariant {} + +GType notify_urgency_get_type(); + +/** + * NOTIFY_EXPIRES_DEFAULT: + * + * The default expiration time on a notification. + */ +enum NOTIFY_EXPIRES_DEFAULT = -1; + +/** + * NOTIFY_EXPIRES_NEVER: + * + * The notification never expires. It stays open until closed by the calling API + * or the user. + */ +enum NOTIFY_EXPIRES_NEVER = 0; + +// #define NOTIFY_TYPE_NOTIFICATION (notify_notification_get_type ()) +// #define NOTIFY_NOTIFICATION(o) (G_TYPE_CHECK_INSTANCE_CAST ((o), NOTIFY_TYPE_NOTIFICATION, NotifyNotification)) +// #define NOTIFY_NOTIFICATION_CLASS(k) (G_TYPE_CHECK_CLASS_CAST((k), NOTIFY_TYPE_NOTIFICATION, NotifyNotificationClass)) +// #define NOTIFY_IS_NOTIFICATION(o) (G_TYPE_CHECK_INSTANCE_TYPE ((o), NOTIFY_TYPE_NOTIFICATION)) +// #define NOTIFY_IS_NOTIFICATION_CLASS(k) (G_TYPE_CHECK_CLASS_TYPE ((k), NOTIFY_TYPE_NOTIFICATION)) +// #define NOTIFY_NOTIFICATION_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS ((o), NOTIFY_TYPE_NOTIFICATION, NotifyNotificationClass)) + +extern (C) { + struct NotifyNotificationPrivate; + + struct NotifyNotification { + /*< private >*/ + GObject parent_object; + + NotifyNotificationPrivate *priv; + } + + struct NotifyNotificationClass { + GObjectClass parent_class; + + /* Signals */ + void function(NotifyNotification *notification) closed; + } + + + /** + * NotifyUrgency: + * @NOTIFY_URGENCY_LOW: Low urgency. Used for unimportant notifications. + * @NOTIFY_URGENCY_NORMAL: Normal urgency. Used for most standard notifications. + * @NOTIFY_URGENCY_CRITICAL: Critical urgency. Used for very important notifications. + * + * The urgency level of the notification. + */ + enum NotifyUrgency { + NOTIFY_URGENCY_LOW, + NOTIFY_URGENCY_NORMAL, + NOTIFY_URGENCY_CRITICAL, + + } + + /** + * NotifyActionCallback: + * @notification: + * @action: + * @user_data: + * + * An action callback function. + */ + alias void function(NotifyNotification* notification, char* action, void* user_data) NotifyActionCallback; + + + GType notify_notification_get_type(); + + NotifyNotification* notify_notification_new(const(char)* summary, const(char)* body_, const(char)* icon); + + bool notify_notification_update(NotifyNotification* notification, const(char)* summary, const(char)* body_, const(char)* icon); + + bool notify_notification_show(NotifyNotification* notification, GError** error); + + void notify_notification_set_timeout(NotifyNotification* notification, int timeout); + + void notify_notification_set_category(NotifyNotification* notification, const(char)* category); + + void notify_notification_set_urgency(NotifyNotification* notification, NotifyUrgency urgency); + + void notify_notification_set_image_from_pixbuf(NotifyNotification* notification, GdkPixbuf* pixbuf); + + void notify_notification_set_icon_from_pixbuf(NotifyNotification* notification, GdkPixbuf* icon); + + void notify_notification_set_hint_int32(NotifyNotification* notification, const(char)* key, int value); + void notify_notification_set_hint_uint32(NotifyNotification* notification, const(char)* key, uint value); + + void notify_notification_set_hint_double(NotifyNotification* notification, const(char)* key, double value); + + void notify_notification_set_hint_string(NotifyNotification* notification, const(char)* key, const(char)* value); + + void notify_notification_set_hint_byte(NotifyNotification* notification, const(char)* key, ubyte value); + + void notify_notification_set_hint_byte_array(NotifyNotification* notification, const(char)* key, const(ubyte)* value, ulong len); + + void notify_notification_set_hint(NotifyNotification* notification, const(char)* key, GVariant* value); + + void notify_notification_set_app_name(NotifyNotification* notification, const(char)* app_name); + + void notify_notification_clear_hints(NotifyNotification* notification); + + void notify_notification_add_action(NotifyNotification* notification, const(char)* action, const(char)* label, + NotifyActionCallback callback, void* user_data, GFreeFunc free_func); + + void notify_notification_clear_actions(NotifyNotification* notification); + bool notify_notification_close(NotifyNotification* notification, GError** error); + + int notify_notification_get_closed_reason(const NotifyNotification* notification); + + + + bool notify_init(const(char)* app_name); + void notify_uninit(); + bool notify_is_initted(); + + const(char)* notify_get_app_name(); + void notify_set_app_name(const(char)* app_name); + + GList *notify_get_server_caps(); + + bool notify_get_server_info(char** ret_name, char** ret_vendor, char** ret_version, char** ret_spec_version); +} + +version(MainTest) { + import std.string; + + void main() { + + notify_init("test".toStringz()); + + auto n = notify_notification_new("summary".toStringz(), "body".toStringz(), "none".toStringz()); + GError* ge; + notify_notification_show(n, &ge); + + scope(success) notify_uninit(); + } +} diff --git a/src/onedrive.d b/src/onedrive.d new file mode 100644 index 00000000..f978e85f --- /dev/null +++ b/src/onedrive.d @@ -0,0 +1,1504 @@ +// What is this module called? +module onedrive; + +// What does this module require to function? +import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit; +import core.memory; +import core.thread; +import std.stdio; +import std.string; +import std.utf; +import std.file; +import std.exception; +import std.regex; +import std.json; +import std.algorithm.searching; +import std.net.curl; +import std.datetime; +import std.path; +import std.conv; +import std.math; +import std.uri; + +// What other modules that we have created do we need to import? +import config; +import log; +import util; +import curlEngine; +import progress; + +class OneDriveException: Exception { + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/errors + int httpStatusCode; + JSONValue error; + + @safe pure this(int httpStatusCode, string reason, string file = __FILE__, size_t line = __LINE__) { + this.httpStatusCode = httpStatusCode; + this.error = error; + string msg = format("HTTP request returned status code %d (%s)", httpStatusCode, reason); + super(msg, file, line); + } + + this(int httpStatusCode, string reason, ref const JSONValue error, string file = __FILE__, size_t line = __LINE__) { + this.httpStatusCode = httpStatusCode; + this.error = error; + string msg = format("HTTP request returned status code %d (%s)\n%s", httpStatusCode, reason, toJSON(error, true)); + super(msg, file, line); + } +} + +class OneDriveApi { + // Class variables + ApplicationConfig appConfig; + CurlEngine curlEngine; + string clientId = ""; + string companyName = ""; + string authUrl = ""; + string redirectUrl = ""; + string tokenUrl = ""; + string driveUrl = ""; + string driveByIdUrl = ""; + string sharedWithMeUrl = ""; + string itemByIdUrl = ""; + string itemByPathUrl = ""; + string siteSearchUrl = ""; + string siteDriveUrl = ""; + string subscriptionUrl = ""; + string tenantId = ""; + string authScope = ""; + string refreshToken = ""; + string accessToken = ""; + SysTime accessTokenExpiration; + bool dryRun = false; + bool debugResponse = false; + ulong retryAfterValue = 0; + + this(ApplicationConfig appConfig) { + // Configure the class varaible to consume the application configuration + this.appConfig = appConfig; + // Configure the major API Query URL's, based on using application configuration + // These however can be updated by config option 'azure_ad_endpoint', thus handled differently + + // Drive Queries + driveUrl = appConfig.globalGraphEndpoint ~ "/v1.0/me/drive"; + driveByIdUrl = appConfig.globalGraphEndpoint ~ "/v1.0/drives/"; + + // What is 'shared with me' Query + sharedWithMeUrl = appConfig.globalGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe"; + + // Item Queries + itemByIdUrl = appConfig.globalGraphEndpoint ~ "/v1.0/me/drive/items/"; + itemByPathUrl = appConfig.globalGraphEndpoint ~ "/v1.0/me/drive/root:/"; + + // Office 365 / SharePoint Queries + siteSearchUrl = appConfig.globalGraphEndpoint ~ "/v1.0/sites?search"; + siteDriveUrl = appConfig.globalGraphEndpoint ~ "/v1.0/sites/"; + + // Subscriptions + subscriptionUrl = appConfig.globalGraphEndpoint ~ "/v1.0/subscriptions"; + } + + // Initialise the OneDrive API class + bool initialise() { + // Initialise the curl engine + curlEngine = new CurlEngine(); + curlEngine.initialise(appConfig.getValueLong("dns_timeout"), appConfig.getValueLong("connect_timeout"), appConfig.getValueLong("data_timeout"), appConfig.getValueLong("operation_timeout"), appConfig.defaultMaxRedirects, appConfig.getValueBool("debug_https"), appConfig.getValueString("user_agent"), appConfig.getValueBool("force_http_11"), appConfig.getValueLong("rate_limit"), appConfig.getValueLong("ip_protocol_version")); + + // Authorised value to return + bool authorised = false; + + // Did the user specify --dry-run + dryRun = appConfig.getValueBool("dry_run"); + + // Did the user specify --debug-https + debugResponse = appConfig.getValueBool("debug_https"); + + // Set clientId to use the configured 'application_id' + clientId = appConfig.getValueString("application_id"); + if (clientId != appConfig.defaultApplicationId) { + // a custom 'application_id' was set + companyName = "custom_application"; + } + + // Do we have a custom Azure Tenant ID? + if (!appConfig.getValueString("azure_tenant_id").empty) { + // Use the value entered by the user + tenantId = appConfig.getValueString("azure_tenant_id"); + } else { + // set to common + tenantId = "common"; + } + + // Did the user specify a 'drive_id' ? + if (!appConfig.getValueString("drive_id").empty) { + // Update base URL's + driveUrl = driveByIdUrl ~ appConfig.getValueString("drive_id"); + itemByIdUrl = driveUrl ~ "/items"; + itemByPathUrl = driveUrl ~ "/root:/"; + } + + // Configure the authentication scope + if (appConfig.getValueBool("read_only_auth_scope")) { + // read-only authentication scopes has been requested + authScope = "&scope=Files.Read%20Files.Read.All%20Sites.Read.All%20offline_access&response_type=code&prompt=login&redirect_uri="; + } else { + // read-write authentication scopes will be used (default) + authScope = "&scope=Files.ReadWrite%20Files.ReadWrite.All%20Sites.ReadWrite.All%20offline_access&response_type=code&prompt=login&redirect_uri="; + } + + // Configure Azure AD endpoints if 'azure_ad_endpoint' is configured + string azureConfigValue = appConfig.getValueString("azure_ad_endpoint"); + switch(azureConfigValue) { + case "": + if (tenantId == "common") { + if (!appConfig.apiWasInitialised) log.log("Configuring Global Azure AD Endpoints"); + } else { + if (!appConfig.apiWasInitialised) log.log("Configuring Global Azure AD Endpoints - Single Tenant Application"); + } + // Authentication + authUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize"; + redirectUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient"; + tokenUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token"; + break; + case "USL4": + if (!appConfig.apiWasInitialised) log.log("Configuring Azure AD for US Government Endpoints"); + // Authentication + authUrl = appConfig.usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize"; + tokenUrl = appConfig.usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token"; + if (clientId == appConfig.defaultApplicationId) { + // application_id == default + log.vdebug("USL4 AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint"); + redirectUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient"; + } else { + // custom application_id + redirectUrl = appConfig.usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient"; + } + + // Drive Queries + driveUrl = appConfig.usl4GraphEndpoint ~ "/v1.0/me/drive"; + driveByIdUrl = appConfig.usl4GraphEndpoint ~ "/v1.0/drives/"; + // Item Queries + itemByIdUrl = appConfig.usl4GraphEndpoint ~ "/v1.0/me/drive/items/"; + itemByPathUrl = appConfig.usl4GraphEndpoint ~ "/v1.0/me/drive/root:/"; + // Office 365 / SharePoint Queries + siteSearchUrl = appConfig.usl4GraphEndpoint ~ "/v1.0/sites?search"; + siteDriveUrl = appConfig.usl4GraphEndpoint ~ "/v1.0/sites/"; + // Shared With Me + sharedWithMeUrl = appConfig.usl4GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe"; + // Subscriptions + subscriptionUrl = appConfig.usl4GraphEndpoint ~ "/v1.0/subscriptions"; + break; + case "USL5": + if (!appConfig.apiWasInitialised) log.log("Configuring Azure AD for US Government Endpoints (DOD)"); + // Authentication + authUrl = appConfig.usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize"; + tokenUrl = appConfig.usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token"; + if (clientId == appConfig.defaultApplicationId) { + // application_id == default + log.vdebug("USL5 AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint"); + redirectUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient"; + } else { + // custom application_id + redirectUrl = appConfig.usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient"; + } + + // Drive Queries + driveUrl = appConfig.usl5GraphEndpoint ~ "/v1.0/me/drive"; + driveByIdUrl = appConfig.usl5GraphEndpoint ~ "/v1.0/drives/"; + // Item Queries + itemByIdUrl = appConfig.usl5GraphEndpoint ~ "/v1.0/me/drive/items/"; + itemByPathUrl = appConfig.usl5GraphEndpoint ~ "/v1.0/me/drive/root:/"; + // Office 365 / SharePoint Queries + siteSearchUrl = appConfig.usl5GraphEndpoint ~ "/v1.0/sites?search"; + siteDriveUrl = appConfig.usl5GraphEndpoint ~ "/v1.0/sites/"; + // Shared With Me + sharedWithMeUrl = appConfig.usl5GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe"; + // Subscriptions + subscriptionUrl = appConfig.usl5GraphEndpoint ~ "/v1.0/subscriptions"; + break; + case "DE": + if (!appConfig.apiWasInitialised) log.log("Configuring Azure AD Germany"); + // Authentication + authUrl = appConfig.deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize"; + tokenUrl = appConfig.deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token"; + if (clientId == appConfig.defaultApplicationId) { + // application_id == default + log.vdebug("DE AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint"); + redirectUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient"; + } else { + // custom application_id + redirectUrl = appConfig.deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient"; + } + + // Drive Queries + driveUrl = appConfig.deGraphEndpoint ~ "/v1.0/me/drive"; + driveByIdUrl = appConfig.deGraphEndpoint ~ "/v1.0/drives/"; + // Item Queries + itemByIdUrl = appConfig.deGraphEndpoint ~ "/v1.0/me/drive/items/"; + itemByPathUrl = appConfig.deGraphEndpoint ~ "/v1.0/me/drive/root:/"; + // Office 365 / SharePoint Queries + siteSearchUrl = appConfig.deGraphEndpoint ~ "/v1.0/sites?search"; + siteDriveUrl = appConfig.deGraphEndpoint ~ "/v1.0/sites/"; + // Shared With Me + sharedWithMeUrl = appConfig.deGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe"; + // Subscriptions + subscriptionUrl = appConfig.deGraphEndpoint ~ "/v1.0/subscriptions"; + break; + case "CN": + if (!appConfig.apiWasInitialised) log.log("Configuring AD China operated by 21Vianet"); + // Authentication + authUrl = appConfig.cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize"; + tokenUrl = appConfig.cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token"; + if (clientId == appConfig.defaultApplicationId) { + // application_id == default + log.vdebug("CN AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint"); + redirectUrl = appConfig.globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient"; + } else { + // custom application_id + redirectUrl = appConfig.cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient"; + } + + // Drive Queries + driveUrl = appConfig.cnGraphEndpoint ~ "/v1.0/me/drive"; + driveByIdUrl = appConfig.cnGraphEndpoint ~ "/v1.0/drives/"; + // Item Queries + itemByIdUrl = appConfig.cnGraphEndpoint ~ "/v1.0/me/drive/items/"; + itemByPathUrl = appConfig.cnGraphEndpoint ~ "/v1.0/me/drive/root:/"; + // Office 365 / SharePoint Queries + siteSearchUrl = appConfig.cnGraphEndpoint ~ "/v1.0/sites?search"; + siteDriveUrl = appConfig.cnGraphEndpoint ~ "/v1.0/sites/"; + // Shared With Me + sharedWithMeUrl = appConfig.cnGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe"; + // Subscriptions + subscriptionUrl = appConfig.cnGraphEndpoint ~ "/v1.0/subscriptions"; + break; + // Default - all other entries + default: + if (!appConfig.apiWasInitialised) log.log("Unknown Azure AD Endpoint request - using Global Azure AD Endpoints"); + } + + // Has the application been authenticated? + if (!exists(appConfig.refreshTokenFilePath)) { + log.vdebug("Application has no 'refresh_token' thus needs to be authenticated"); + authorised = authorise(); + } else { + // Try and read the value from the appConfig if it is set, rather than trying to read the value from disk + if (!appConfig.refreshToken.empty) { + log.vdebug("Read token from appConfig"); + refreshToken = strip(appConfig.refreshToken); + authorised = true; + } else { + // Try and read the file from disk + try { + refreshToken = strip(readText(appConfig.refreshTokenFilePath)); + // is the refresh_token empty? + if (refreshToken.empty) { + log.error("refreshToken exists but is empty: ", appConfig.refreshTokenFilePath); + authorised = authorise(); + } else { + // existing token not empty + authorised = true; + // update appConfig.refreshToken + appConfig.refreshToken = refreshToken; + } + } catch (FileException e) { + authorised = authorise(); + } catch (std.utf.UTFException e) { + // path contains characters which generate a UTF exception + log.error("Cannot read refreshToken from: ", appConfig.refreshTokenFilePath); + log.error(" Error Reason:", e.msg); + authorised = false; + } + } + + if (refreshToken.empty) { + // PROBLEM + writeln("refreshToken is empty !!!!!!!!!! will cause 4xx errors"); + } + } + // Return if we are authorised + log.vdebug("Authorised State: ", authorised); + return authorised; + } + + // If the API has been configured correctly, print the items that been configured + void debugOutputConfiguredAPIItems() { + // Debug output of configured URL's + // Application Identification + log.vdebug("Configured clientId ", clientId); + log.vdebug("Configured userAgent ", appConfig.getValueString("user_agent")); + // Authentication + log.vdebug("Configured authScope: ", authScope); + log.vdebug("Configured authUrl: ", authUrl); + log.vdebug("Configured redirectUrl: ", redirectUrl); + log.vdebug("Configured tokenUrl: ", tokenUrl); + // Drive Queries + log.vdebug("Configured driveUrl: ", driveUrl); + log.vdebug("Configured driveByIdUrl: ", driveByIdUrl); + // Shared With Me + log.vdebug("Configured sharedWithMeUrl: ", sharedWithMeUrl); + // Item Queries + log.vdebug("Configured itemByIdUrl: ", itemByIdUrl); + log.vdebug("Configured itemByPathUrl: ", itemByPathUrl); + // SharePoint Queries + log.vdebug("Configured siteSearchUrl: ", siteSearchUrl); + log.vdebug("Configured siteDriveUrl: ", siteDriveUrl); + } + + // Shutdown OneDrive API Curl Engine + void shutdown() { + // Delete subscription if there exists any + //deleteSubscription(); + + // Reset any values to defaults, freeing any set objects + curlEngine.http.clearRequestHeaders(); + curlEngine.http.onSend = null; + curlEngine.http.onReceive = null; + curlEngine.http.onReceiveHeader = null; + curlEngine.http.onReceiveStatusLine = null; + curlEngine.http.contentLength = 0; + // Shut down the curl instance & close any open sockets + curlEngine.http.shutdown(); + // Free object and memory + object.destroy(curlEngine); + } + + // Authenticate this client against Microsoft OneDrive API + bool authorise() { + + char[] response; + // What URL should be presented to the user to access + string url = authUrl ~ "?client_id=" ~ clientId ~ authScope ~ redirectUrl; + // Configure automated authentication if --auth-files authUrl:responseUrl is being used + string authFilesString = appConfig.getValueString("auth_files"); + string authResponseString = appConfig.getValueString("auth_response"); + + if (!authResponseString.empty) { + // read the response from authResponseString + response = cast(char[]) authResponseString; + } else if (authFilesString != "") { + string[] authFiles = authFilesString.split(":"); + string authUrl = authFiles[0]; + string responseUrl = authFiles[1]; + auto authUrlFile = File(authUrl, "w"); + authUrlFile.write(url); + authUrlFile.close(); + + log.log("Client requires authentication before proceeding. Waiting for --auth-files elements to be available."); + + while (!exists(responseUrl)) { + Thread.sleep(dur!("msecs")(100)); + } + + // read response from provided from OneDrive + try { + response = cast(char[]) read(responseUrl); + } catch (OneDriveException e) { + // exception generated + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + return false; + } + + // try to remove old files + try { + std.file.remove(authUrl); + std.file.remove(responseUrl); + } catch (FileException e) { + log.error("Cannot remove files ", authUrl, " ", responseUrl); + return false; + } + } else { + log.log("Authorise this application by visiting:\n"); + write(url, "\n\n", "Enter the response uri from your browser: "); + readln(response); + appConfig.applicationAuthorizeResponseUri = true; + } + // match the authorization code + auto c = matchFirst(response, r"(?:[\?&]code=)([\w\d-.]+)"); + if (c.empty) { + log.log("An empty or invalid response uri was entered"); + return false; + } + c.popFront(); // skip the whole match + redeemToken(c.front); + + + return true; + + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get + JSONValue getDefaultDriveDetails() { + checkAccessTokenExpired(); + string url; + url = driveUrl; + return get(driveUrl); + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get + JSONValue getDefaultRootDetails() { + checkAccessTokenExpired(); + string url; + url = driveUrl ~ "/root"; + return get(url); + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get + JSONValue getDriveIdRoot(string driveId) { + checkAccessTokenExpired(); + string url; + url = driveByIdUrl ~ driveId ~ "/root"; + return get(url); + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get + JSONValue getDriveQuota(string driveId) { + checkAccessTokenExpired(); + string url; + url = driveByIdUrl ~ driveId ~ "/"; + url ~= "?select=quota"; + return get(url); + } + + // Return the details of the specified path, by giving the path we wish to query + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get + JSONValue getPathDetails(string path) { + checkAccessTokenExpired(); + string url; + if ((path == ".")||(path == "/")) { + url = driveUrl ~ "/root/"; + } else { + url = itemByPathUrl ~ encodeComponent(path) ~ ":/"; + } + return get(url); + } + + // Return the details of the specified item based on its driveID and itemID + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get + JSONValue getPathDetailsById(string driveId, string id) { + checkAccessTokenExpired(); + string url; + url = driveByIdUrl ~ driveId ~ "/items/" ~ id; + //url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; + return get(url); + } + + // Return the requested details of the specified path on the specified drive id and path + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get + JSONValue getPathDetailsByDriveId(string driveId, string path) { + checkAccessTokenExpired(); + string url; + // Required format: /drives/{drive-id}/root:/{item-path} + url = driveByIdUrl ~ driveId ~ "/root:/" ~ encodeComponent(path); + return get(url); + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta + JSONValue viewChangesByItemId(string driveId, string id, string deltaLink) { + checkAccessTokenExpired(); + + // If Business Account add addIncludeFeatureRequestHeader() which should add Prefer: Include-Feature=AddToOneDrive + if ((appConfig.accountType != "personal") && ( appConfig.getValueBool("sync_business_shared_items"))) { + addIncludeFeatureRequestHeader(); + } + + string url; + // configure deltaLink to query + if (deltaLink.empty) { + url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/delta"; + } else { + url = deltaLink; + } + return get(url); + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_list_children + JSONValue listChildren(string driveId, string id, string nextLink) { + checkAccessTokenExpired(); + + // If Business Account add addIncludeFeatureRequestHeader() which should add Prefer: Include-Feature=AddToOneDrive + if ((appConfig.accountType != "personal") && ( appConfig.getValueBool("sync_business_shared_items"))) { + addIncludeFeatureRequestHeader(); + } + + string url; + // configure URL to query + if (nextLink.empty) { + url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/children"; + //url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; + } else { + url = nextLink; + } + return get(url); + } + + // https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_search + JSONValue searchDriveForPath(string driveId, string path) { + checkAccessTokenExpired(); + string url; + url = "https://graph.microsoft.com/v1.0/drives/" ~ driveId ~ "/root/search(q='" ~ encodeComponent(path) ~ "')"; + return get(url); + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_update + JSONValue updateById(const(char)[] driveId, const(char)[] id, JSONValue data, const(char)[] eTag = null) { + checkAccessTokenExpired(); + const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id; + if (eTag) curlEngine.http.addRequestHeader("If-Match", eTag); + curlEngine.http.addRequestHeader("Content-Type", "application/json"); + return patch(url, data.toString()); + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delete + void deleteById(const(char)[] driveId, const(char)[] id, const(char)[] eTag = null) { + checkAccessTokenExpired(); + const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id; + //TODO: investigate why this always fail with 412 (Precondition Failed) + //if (eTag) http.addRequestHeader("If-Match", eTag); + performDelete(url); + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_post_children + JSONValue createById(string parentDriveId, string parentId, JSONValue item) { + checkAccessTokenExpired(); + string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ "/children"; + curlEngine.http.addRequestHeader("Content-Type", "application/json"); + return post(url, item.toString()); + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content + JSONValue simpleUpload(string localPath, string parentDriveId, string parentId, string filename) { + checkAccessTokenExpired(); + string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/content"; + return upload(localPath, url); + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content + JSONValue simpleUploadReplace(string localPath, string driveId, string id) { + checkAccessTokenExpired(); + string url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content"; + return upload(localPath, url); + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession + //JSONValue createUploadSession(string parentDriveId, string parentId, string filename, string eTag = null, JSONValue item = null) { + JSONValue createUploadSession(string parentDriveId, string parentId, string filename, const(char)[] eTag = null, JSONValue item = null) { + checkAccessTokenExpired(); + string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/createUploadSession"; + // eTag If-Match header addition commented out for the moment + // At some point, post the creation of this upload session the eTag is being 'updated' by OneDrive, thus when uploadFragment() is used + // this generates a 412 Precondition Failed and then a 416 Requested Range Not Satisfiable + // This needs to be investigated further as to why this occurs + //if (eTag) curlEngine.http.addRequestHeader("If-Match", eTag); + curlEngine.http.addRequestHeader("Content-Type", "application/json"); + return post(url, item.toString()); + } + + // https://dev.onedrive.com/items/upload_large_files.htm + JSONValue uploadFragment(string uploadUrl, string filepath, long offset, long offsetSize, long fileSize) { + checkAccessTokenExpired(); + // open file as read-only in binary mode + + // If we upload a modified file, with the current known online eTag, this gets changed when the session is started - thus, the tail end of uploading + // a fragment fails with a 412 Precondition Failed and then a 416 Requested Range Not Satisfiable + // For the moment, comment out adding the If-Match header in createUploadSession, which then avoids this issue + + auto file = File(filepath, "rb"); + file.seek(offset); + string contentRange = "bytes " ~ to!string(offset) ~ "-" ~ to!string(offset + offsetSize - 1) ~ "/" ~ to!string(fileSize); + log.vdebugNewLine("contentRange: ", contentRange); + + // function scopes + scope(exit) { + curlEngine.http.clearRequestHeaders(); + curlEngine.http.onSend = null; + curlEngine.http.onReceive = null; + curlEngine.http.onReceiveHeader = null; + curlEngine.http.onReceiveStatusLine = null; + curlEngine.http.contentLength = 0; + // close file if open + if (file.isOpen()){ + // close open file + file.close(); + } + } + + curlEngine.http.method = HTTP.Method.put; + curlEngine.http.url = uploadUrl; + curlEngine.http.addRequestHeader("Content-Range", contentRange); + curlEngine.http.onSend = data => file.rawRead(data).length; + // convert offsetSize to ulong + curlEngine.http.contentLength = to!ulong(offsetSize); + auto response = performHTTPOperation(); + checkHttpResponseCode(response); + return response; + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/site_search?view=odsp-graph-online + JSONValue o365SiteSearch(string nextLink) { + checkAccessTokenExpired(); + string url; + // configure URL to query + if (nextLink.empty) { + url = siteSearchUrl ~ "=*"; + } else { + url = nextLink; + } + return get(url); + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_list?view=odsp-graph-online + JSONValue o365SiteDrives(string site_id){ + checkAccessTokenExpired(); + string url; + url = siteDriveUrl ~ site_id ~ "/drives"; + return get(url); + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get_content + void downloadById(const(char)[] driveId, const(char)[] id, string saveToPath, long fileSize) { + checkAccessTokenExpired(); + scope(failure) { + if (exists(saveToPath)) { + // try and remove the file, catch error + try { + remove(saveToPath); + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } + } + + // Create the required local directory + string newPath = dirName(saveToPath); + + // Does the path exist locally? + if (!exists(newPath)) { + try { + log.vdebug("Requested path does not exist, creating directory structure: ", newPath); + mkdirRecurse(newPath); + // Configure the applicable permissions for the folder + log.vdebug("Setting directory permissions for: ", newPath); + newPath.setAttributes(appConfig.returnRequiredDirectoryPermisions()); + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } + + const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content?AVOverride=1"; + // Download file + downloadFile(url, saveToPath, fileSize); + // Does path exist? + if (exists(saveToPath)) { + // File was downloaded successfully - configure the applicable permissions for the file + log.vdebug("Setting file permissions for: ", saveToPath); + saveToPath.setAttributes(appConfig.returnRequiredFilePermisions()); + } + } + + // Return the actual siteSearchUrl being used and/or requested when performing 'siteQuery = onedrive.o365SiteSearch(nextLink);' call + string getSiteSearchUrl() { + return siteSearchUrl; + } + + // Return the current value of retryAfterValue + ulong getRetryAfterValue() { + return retryAfterValue; + } + + // Reset the current value of retryAfterValue to 0 after it has been used + void resetRetryAfterValue() { + retryAfterValue = 0; + } + + private void addAccessTokenHeader() { + curlEngine.http.addRequestHeader("Authorization", accessToken); + } + + private void addIncludeFeatureRequestHeader() { + log.vdebug("Adding 'Include-Feature=AddToOneDrive' API request header as 'sync_business_shared_items' config option is enabled"); + curlEngine.http.addRequestHeader("Prefer", "Include-Feature=AddToOneDrive"); + } + + private void acquireToken(char[] postData) { + JSONValue response; + + try { + response = post(tokenUrl, postData); + } catch (OneDriveException e) { + // an error was generated + if ((e.httpStatusCode == 400) || (e.httpStatusCode == 401)) { + // Handle an unauthorised client + handleClientUnauthorised(e.httpStatusCode, e.msg); + } else { + if (e.httpStatusCode >= 500) { + // There was a HTTP 5xx Server Side Error - retry + acquireToken(postData); + } else { + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + } + } + } + + if (response.type() == JSONType.object) { + // Has the client been configured to use read_only_auth_scope + if (appConfig.getValueBool("read_only_auth_scope")) { + // read_only_auth_scope has been configured + if ("scope" in response){ + string effectiveScopes = response["scope"].str(); + // Display the effective authentication scopes + writeln(); + writeln("Effective API Authentication Scopes: ", effectiveScopes); + // if we have any write scopes, we need to tell the user to update an remove online prior authentication and exit application + if (canFind(effectiveScopes, "Write")) { + // effective scopes contain write scopes .. so not a read-only configuration + writeln(); + writeln("ERROR: You have authentication scopes that allow write operations. You need to remove your existing application access consent"); + writeln(); + writeln("Please login to https://account.live.com/consent/Manage and remove your existing application access consent"); + writeln(); + // force exit + shutdown(); + exit(-1); + } + } + } + + if ("access_token" in response){ + accessToken = "bearer " ~ strip(response["access_token"].str); + + // Do we print the current access token + if (log.verbose > 1) { + if (appConfig.getValueBool("debug_https")) { + if (appConfig.getValueBool("print_token")) { + // This needs to be highly restricted in output .... + log.vdebug("CAUTION - KEEP THIS SAFE: Current access token: ", accessToken); + } + } + } + + refreshToken = strip(response["refresh_token"].str); + accessTokenExpiration = Clock.currTime() + dur!"seconds"(response["expires_in"].integer()); + if (!dryRun) { + // Update the refreshToken in appConfig so that we can reuse it + if (appConfig.refreshToken.empty) { + // The access token is empty + log.vdebug("Updating appConfig.refreshToken with new refreshToken as appConfig.refreshToken is empty"); + appConfig.refreshToken = refreshToken; + } else { + // Is the access token different? + if (appConfig.refreshToken != refreshToken) { + // Update the memory version + log.vdebug("Updating appConfig.refreshToken with updated refreshToken"); + appConfig.refreshToken = refreshToken; + } + } + + // try and update the refresh_token file on disk + try { + log.vdebug("Updating refreshToken on disk"); + std.file.write(appConfig.refreshTokenFilePath, refreshToken); + log.vdebug("Setting file permissions for: ", appConfig.refreshTokenFilePath); + appConfig.refreshTokenFilePath.setAttributes(appConfig.returnRequiredFilePermisions()); + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } + } else { + log.error("\nInvalid authentication response from OneDrive. Please check the response uri\n"); + // re-authorize + authorise(); + } + } else { + log.log("Invalid response from the OneDrive API. Unable to initialise OneDrive API instance."); + exit(-1); + } + } + + private void checkAccessTokenExpired() { + try { + if (Clock.currTime() >= accessTokenExpiration) { + newToken(); + } + } catch (OneDriveException e) { + if (e.httpStatusCode == 400 || e.httpStatusCode == 401) { + // flag error and notify + writeln(); + log.errorAndNotify("ERROR: Refresh token invalid, use --reauth to authorize the client again."); + writeln(); + // set error message + e.msg ~= "\nRefresh token invalid, use --reauth to authorize the client again"; + } + } + } + + private void performDelete(const(char)[] url) { + scope(exit) curlEngine.http.clearRequestHeaders(); + curlEngine.http.method = HTTP.Method.del; + curlEngine.http.url = url; + addAccessTokenHeader(); + auto response = performHTTPOperation(); + checkHttpResponseCode(response); + } + + private void downloadFile(const(char)[] url, string filename, long fileSize) { + // Threshold for displaying download bar + long thresholdFileSize = 4 * 2^^20; // 4 MiB + + // To support marking of partially-downloaded files, + string originalFilename = filename; + string downloadFilename = filename ~ ".partial"; + + // open downloadFilename as write in binary mode + auto file = File(downloadFilename, "wb"); + + // function scopes + scope(exit) { + curlEngine.http.clearRequestHeaders(); + curlEngine.http.onSend = null; + curlEngine.http.onReceive = null; + curlEngine.http.onReceiveHeader = null; + curlEngine.http.onReceiveStatusLine = null; + curlEngine.http.contentLength = 0; + // Reset onProgress to not display anything for next download + curlEngine.http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) + { + return 0; + }; + // close file if open + if (file.isOpen()){ + // close open file + file.close(); + } + } + + curlEngine.http.method = HTTP.Method.get; + curlEngine.http.url = url; + addAccessTokenHeader(); + + curlEngine.http.onReceive = (ubyte[] data) { + file.rawWrite(data); + return data.length; + }; + + if (fileSize >= thresholdFileSize){ + // Download Progress Bar + size_t iteration = 20; + Progress p = new Progress(iteration); + p.title = "Downloading"; + writeln(); + bool barInit = false; + real previousProgressPercent = -1.0; + real percentCheck = 5.0; + long segmentCount = 1; + // Setup progress bar to display + curlEngine.http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) + { + // For each onProgress, what is the % of dlnow to dltotal + // floor - rounds down to nearest whole number + real currentDLPercent = floor(double(dlnow)/dltotal*100); + // Have we started downloading? + if (currentDLPercent > 0){ + // We have started downloading + log.vdebugNewLine("Data Received = ", dlnow); + log.vdebug("Expected Total = ", dltotal); + log.vdebug("Percent Complete = ", currentDLPercent); + // Every 5% download we need to increment the download bar + + // Has the user set a data rate limit? + // when using rate_limit, we will get odd download rates, for example: + // Percent Complete = 24 + // Data Received = 13080163 + // Expected Total = 52428800 + // Percent Complete = 24 + // Data Received = 13685777 + // Expected Total = 52428800 + // Percent Complete = 26 <---- jumps to 26% missing 25%, thus fmod misses incrementing progress bar + // Data Received = 13685777 + // Expected Total = 52428800 + // Percent Complete = 26 + + if (appConfig.getValueLong("rate_limit") > 0) { + // User configured rate limit + // How much data should be in each segment to qualify for 5% + ulong dataPerSegment = to!ulong(floor(double(dltotal)/iteration)); + // How much data received do we need to validate against + ulong thisSegmentData = dataPerSegment * segmentCount; + ulong nextSegmentData = dataPerSegment * (segmentCount + 1); + // Has the data that has been received in a 5% window that we need to increment the progress bar at + if ((dlnow > thisSegmentData) && (dlnow < nextSegmentData) && (previousProgressPercent != currentDLPercent) || (dlnow == dltotal)) { + // Downloaded data equals approx 5% + log.vdebug("Incrementing Progress Bar using calculated 5% of data received"); + // Downloading 50% |oooooooooooooooooooo | ETA 00:01:40 + // increment progress bar + p.next(); + // update values + log.vdebug("Setting previousProgressPercent to ", currentDLPercent); + previousProgressPercent = currentDLPercent; + log.vdebug("Incrementing segmentCount"); + segmentCount++; + } + } else { + // Is currentDLPercent divisible by 5 leaving remainder 0 and does previousProgressPercent not equal currentDLPercent + if ((isIdentical(fmod(currentDLPercent, percentCheck), 0.0)) && (previousProgressPercent != currentDLPercent)) { + // currentDLPercent matches a new increment + log.vdebug("Incrementing Progress Bar using fmod match"); + // Downloading 50% |oooooooooooooooooooo | ETA 00:01:40 + // increment progress bar + p.next(); + // update values + previousProgressPercent = currentDLPercent; + } + } + } else { + if ((currentDLPercent == 0) && (!barInit)) { + // Initialise the download bar at 0% + // Downloading 0% | | ETA --:--:--: + p.next(); + barInit = true; + } + } + return 0; + }; + + // Perform download & display progress bar + try { + // try and catch any curl error + curlEngine.http.perform(); + // Check the HTTP Response headers - needed for correct 429 handling + // check will be performed in checkHttpCode() + writeln(); + // Reset onProgress to not display anything for next download done using exit scope + } catch (CurlException e) { + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + } + // free progress bar memory + p = null; + } else { + // No progress bar + try { + // try and catch any curl error + curlEngine.http.perform(); + // Check the HTTP Response headers - needed for correct 429 handling + // check will be performed in checkHttpCode() + } catch (CurlException e) { + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + } + } + + // Rename downloaded file + rename(downloadFilename, originalFilename); + + // Check the HTTP response code, which, if a 429, will also check response headers + checkHttpCode(); + } + + private JSONValue get(string url, bool skipToken = false) { + scope(exit) curlEngine.http.clearRequestHeaders(); + log.vdebug("Request URL = ", url); + curlEngine.http.method = HTTP.Method.get; + curlEngine.http.url = url; + if (!skipToken) addAccessTokenHeader(); // HACK: requestUploadStatus + JSONValue response; + response = performHTTPOperation(); + checkHttpResponseCode(response); + // OneDrive API Response Debugging if --https-debug is being used + if (debugResponse){ + log.vdebug("OneDrive API Response: ", response); + } + return response; + } + + private void newToken() { + string postData = + "client_id=" ~ clientId ~ + "&redirect_uri=" ~ redirectUrl ~ + "&refresh_token=" ~ refreshToken ~ + "&grant_type=refresh_token"; + char[] strArr = postData.dup; + acquireToken(strArr); + } + + private auto patch(T)(const(char)[] url, const(T)[] patchData) { + curlEngine.setMethodPatch(); + curlEngine.http.url = url; + addAccessTokenHeader(); + auto response = perform(patchData); + checkHttpResponseCode(response); + return response; + } + + private auto post(T)(string url, const(T)[] postData) { + curlEngine.setMethodPost(); + curlEngine.http.url = url; + addAccessTokenHeader(); + auto response = perform(postData); + checkHttpResponseCode(response); + return response; + } + + private JSONValue perform(const(void)[] sendData) { + scope(exit) { + curlEngine.http.onSend = null; + curlEngine.http.contentLength = 0; + } + if (sendData) { + curlEngine.http.contentLength = sendData.length; + curlEngine.http.onSend = (void[] buf) { + import std.algorithm: min; + size_t minLen = min(buf.length, sendData.length); + if (minLen == 0) return 0; + buf[0 .. minLen] = sendData[0 .. minLen]; + sendData = sendData[minLen .. $]; + return minLen; + }; + } else { + curlEngine.http.onSend = buf => 0; + } + auto response = performHTTPOperation(); + return response; + } + + private JSONValue performHTTPOperation() { + scope(exit) curlEngine.http.onReceive = null; + char[] content; + JSONValue json; + + curlEngine.http.onReceive = (ubyte[] data) { + content ~= data; + // HTTP Server Response Code Debugging if --https-debug is being used + if (debugResponse){ + log.vdebug("onedrive.performHTTPOperation() => OneDrive HTTP Server Response: ", curlEngine.http.statusLine.code); + } + return data.length; + }; + + try { + curlEngine.http.perform(); + // Check the HTTP Response headers - needed for correct 429 handling + checkHTTPResponseHeaders(); + } catch (CurlException e) { + // Parse and display error message received from OneDrive + log.vdebug("onedrive.performHTTPOperation() Generated a OneDrive CurlException"); + auto errorArray = splitLines(e.msg); + string errorMessage = errorArray[0]; + + // what is contained in the curl error message? + if (canFind(errorMessage, "Couldn't connect to server on handle") || canFind(errorMessage, "Couldn't resolve host name on handle") || canFind(errorMessage, "Timeout was reached on handle")) { + // This is a curl timeout + // or is this a 408 request timeout + // https://github.com/abraunegg/onedrive/issues/694 + // Back off & retry with incremental delay + int retryCount = 10000; + int retryAttempts = 0; + int backoffInterval = 0; + int maxBackoffInterval = 3600; + int timestampAlign = 0; + bool retrySuccess = false; + SysTime currentTime; + + // what caused the initial curl exception? + if (canFind(errorMessage, "Couldn't connect to server on handle")) log.vdebug("Unable to connect to server - HTTPS access blocked?"); + if (canFind(errorMessage, "Couldn't resolve host name on handle")) log.vdebug("Unable to resolve server - DNS access blocked?"); + if (canFind(errorMessage, "Timeout was reached on handle")) log.vdebug("A timeout was triggered - data too slow, no response ... use --debug-https to diagnose further"); + + while (!retrySuccess){ + try { + // configure libcurl to perform a fresh connection + log.vdebug("Configuring libcurl to use a fresh connection for re-try"); + curlEngine.http.handle.set(CurlOption.fresh_connect,1); + // try the access + curlEngine.http.perform(); + // Check the HTTP Response headers - needed for correct 429 handling + checkHTTPResponseHeaders(); + // no error from http.perform() on re-try + log.log("Internet connectivity to Microsoft OneDrive service has been restored"); + // unset the fresh connect option as this then creates performance issues if left enabled + log.vdebug("Unsetting libcurl to use a fresh connection as this causes a performance impact if left enabled"); + curlEngine.http.handle.set(CurlOption.fresh_connect,0); + // connectivity restored + retrySuccess = true; + } catch (CurlException e) { + // when was the exception generated + currentTime = Clock.currTime(); + // Increment retry attempts + retryAttempts++; + if (canFind(e.msg, "Couldn't connect to server on handle") || canFind(e.msg, "Couldn't resolve host name on handle") || canFind(errorMessage, "Timeout was reached on handle")) { + // no access to Internet + writeln(); + log.error("ERROR: There was a timeout in accessing the Microsoft OneDrive service - Internet connectivity issue?"); + // what is the error reason to assis the user as what to check + if (canFind(e.msg, "Couldn't connect to server on handle")) { + log.log(" - Check HTTPS access or Firewall Rules"); + timestampAlign = 9; + } + if (canFind(e.msg, "Couldn't resolve host name on handle")) { + log.log(" - Check DNS resolution or Firewall Rules"); + timestampAlign = 0; + } + + // increment backoff interval + backoffInterval++; + int thisBackOffInterval = retryAttempts*backoffInterval; + + // display retry information + currentTime.fracSecs = Duration.zero; + auto timeString = currentTime.toString(); + log.vlog(" Retry attempt: ", retryAttempts); + log.vlog(" This attempt timestamp: ", timeString); + if (thisBackOffInterval > maxBackoffInterval) { + thisBackOffInterval = maxBackoffInterval; + } + + // detail when the next attempt will be tried + // factor in the delay for curl to generate the exception - otherwise the next timestamp appears to be 'out' even though technically correct + auto nextRetry = currentTime + dur!"seconds"(thisBackOffInterval) + dur!"seconds"(timestampAlign); + log.vlog(" Next retry in approx: ", (thisBackOffInterval + timestampAlign), " seconds"); + log.vlog(" Next retry approx: ", nextRetry); + + // thread sleep + Thread.sleep(dur!"seconds"(thisBackOffInterval)); + } + if (retryAttempts == retryCount) { + // we have attempted to re-connect X number of times + // false set this to true to break out of while loop + retrySuccess = true; + } + } + } + if (retryAttempts >= retryCount) { + log.error(" ERROR: Unable to reconnect to the Microsoft OneDrive service after ", retryCount, " attempts lasting over 1.2 years!"); + throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?"); + } + } else { + + // what error was returned? + if (canFind(errorMessage, "Problem with the SSL CA cert (path? access rights?) on handle")) { + // error setting certificate verify locations: + // CAfile: /etc/pki/tls/certs/ca-bundle.crt + // CApath: none + // + // Tell the Curl Engine to bypass SSL check - essentially SSL is passing back a bad value due to 'stdio' compile time option + // Further reading: + // https://github.com/curl/curl/issues/6090 + // https://github.com/openssl/openssl/issues/7536 + // https://stackoverflow.com/questions/45829588/brew-install-fails-curl77-error-setting-certificate-verify + // https://forum.dlang.org/post/vwvkbubufexgeuaxhqfl@forum.dlang.org + + log.vdebug("Problem with reading the SSL CA cert via libcurl - attempting work around"); + curlEngine.setDisableSSLVerifyPeer(); + // retry origional call + performHTTPOperation(); + } else { + // Log that an error was returned + log.error("ERROR: OneDrive returned an error with the following message:"); + // Some other error was returned + log.error(" Error Message: ", errorMessage); + log.error(" Calling Function: ", getFunctionName!({})); + + // Was this a curl initialization error? + if (canFind(errorMessage, "Failed initialization on handle")) { + // initialization error ... prevent a run-away process if we have zero disk space + ulong localActualFreeSpace = getAvailableDiskSpace("."); + if (localActualFreeSpace == 0) { + // force exit + shutdown(); + exit(-1); + } + } + } + } + // return an empty JSON for handling + return json; + } + + try { + json = content.parseJSON(); + } catch (JSONException e) { + // Log that a JSON Exception was caught, dont output the HTML response from OneDrive + log.vdebug("JSON Exception caught when performing HTTP operations - use --debug-https to diagnose further"); + } + return json; + } + + private void redeemToken(char[] authCode){ + char[] postData = + "client_id=" ~ clientId ~ + "&redirect_uri=" ~ redirectUrl ~ + "&code=" ~ authCode ~ + "&grant_type=authorization_code"; + acquireToken(postData); + } + + private JSONValue upload(string filepath, string url) { + checkAccessTokenExpired(); + // open file as read-only in binary mode + auto file = File(filepath, "rb"); + + // function scopes + scope(exit) { + curlEngine.http.clearRequestHeaders(); + curlEngine.http.onSend = null; + curlEngine.http.onReceive = null; + curlEngine.http.onReceiveHeader = null; + curlEngine.http.onReceiveStatusLine = null; + curlEngine.http.contentLength = 0; + // close file if open + if (file.isOpen()){ + // close open file + file.close(); + } + } + + curlEngine.http.method = HTTP.Method.put; + curlEngine.http.url = url; + addAccessTokenHeader(); + curlEngine.http.addRequestHeader("Content-Type", "application/octet-stream"); + curlEngine.http.onSend = data => file.rawRead(data).length; + curlEngine.http.contentLength = file.size; + auto response = performHTTPOperation(); + checkHttpResponseCode(response); + return response; + } + + private void checkHTTPResponseHeaders() { + // Get the HTTP Response headers - needed for correct 429 handling + auto responseHeaders = curlEngine.http.responseHeaders(); + if (debugResponse){ + log.vdebug("curlEngine.http.perform() => HTTP Response Headers: ", responseHeaders); + } + + // is retry-after in the response headers + if ("retry-after" in curlEngine.http.responseHeaders) { + // Set the retry-after value + log.vdebug("curlEngine.http.perform() => Received a 'Retry-After' Header Response with the following value: ", curlEngine.http.responseHeaders["retry-after"]); + log.vdebug("curlEngine.http.perform() => Setting retryAfterValue to: ", curlEngine.http.responseHeaders["retry-after"]); + retryAfterValue = to!ulong(curlEngine.http.responseHeaders["retry-after"]); + } + } + + private void checkHttpResponseCode(JSONValue response) { + switch(curlEngine.http.statusLine.code) { + // 0 - OK ... HTTP2 version of 200 OK + case 0: + break; + // 100 - Continue + case 100: + break; + // 200 - OK + case 200: + // No Log .. + break; + // 201 - Created OK + // 202 - Accepted + // 204 - Deleted OK + case 201,202,204: + // No actions, but log if verbose logging + //log.vlog("OneDrive Response: '", curlEngine.http.statusLine.code, " - ", curlEngine.http.statusLine.reason, "'"); + break; + + // 302 - resource found and available at another location, redirect + case 302: + break; + + // 400 - Bad Request + case 400: + // Bad Request .. how should we act? + // make sure this is thrown so that it is caught + throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); + + // 403 - Forbidden + case 403: + // OneDrive responded that the user is forbidden + log.vlog("OneDrive returned a 'HTTP 403 - Forbidden' - gracefully handling error"); + // Throw this as a specific exception so this is caught when performing 'siteQuery = onedrive.o365SiteSearch(nextLink);' call + throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); + + // 412 - Precondition Failed + case 412: + // Throw this as a specific exception so this is caught when performing sync.uploadLastModifiedTime + throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); + + // Server side (OneDrive) Errors + // 500 - Internal Server Error + // 502 - Bad Gateway + // 503 - Service Unavailable + // 504 - Gateway Timeout (Issue #320) + case 500: + // Throw this as a specific exception so this is caught + throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); + + case 502: + // Throw this as a specific exception so this is caught + throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); + + case 503: + // Throw this as a specific exception so this is caught + throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); + + case 504: + // Throw this as a specific exception so this is caught + throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); + + // Default - all other errors that are not a 2xx or a 302 + default: + if (curlEngine.http.statusLine.code / 100 != 2 && curlEngine.http.statusLine.code != 302) { + throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); + } + } + } + + private void checkHttpCode() { + // https://dev.onedrive.com/misc/errors.htm + // https://developer.overdrive.com/docs/reference-guide + + /* + HTTP/1.1 Response handling + + Errors in the OneDrive API are returned using standard HTTP status codes, as well as a JSON error response object. The following HTTP status codes should be expected. + + Status code Status message Description + 100 Continue Continue + 200 OK Request was handled OK + 201 Created This means you've made a successful POST to checkout, lock in a format, or place a hold + 204 No Content This means you've made a successful DELETE to remove a hold or return a title + + 400 Bad Request Cannot process the request because it is malformed or incorrect. + 401 Unauthorized Required authentication information is either missing or not valid for the resource. + 403 Forbidden Access is denied to the requested resource. The user might not have enough permission. + 404 Not Found The requested resource doesn’t exist. + 405 Method Not Allowed The HTTP method in the request is not allowed on the resource. + 406 Not Acceptable This service doesn’t support the format requested in the Accept header. + 408 Request Time out Not expected from OneDrive, but can be used to handle Internet connection failures the same (fallback and try again) + 409 Conflict The current state conflicts with what the request expects. For example, the specified parent folder might not exist. + 410 Gone The requested resource is no longer available at the server. + 411 Length Required A Content-Length header is required on the request. + 412 Precondition Failed A precondition provided in the request (such as an if-match header) does not match the resource's current state. + 413 Request Entity Too Large The request size exceeds the maximum limit. + 415 Unsupported Media Type The content type of the request is a format that is not supported by the service. + 416 Requested Range Not Satisfiable The specified byte range is invalid or unavailable. + 422 Unprocessable Entity Cannot process the request because it is semantically incorrect. + 429 Too Many Requests Client application has been throttled and should not attempt to repeat the request until an amount of time has elapsed. + + 500 Internal Server Error There was an internal server error while processing the request. + 501 Not Implemented The requested feature isn’t implemented. + 502 Bad Gateway The service was unreachable + 503 Service Unavailable The service is temporarily unavailable. You may repeat the request after a delay. There may be a Retry-After header. + 507 Insufficient Storage The maximum storage quota has been reached. + 509 Bandwidth Limit Exceeded Your app has been throttled for exceeding the maximum bandwidth cap. Your app can retry the request again after more time has elapsed. + + HTTP/2 Response handling + + 0 OK + + */ + + switch(curlEngine.http.statusLine.code) + { + // 0 - OK ... HTTP2 version of 200 OK + case 0: + break; + // 100 - Continue + case 100: + break; + // 200 - OK + case 200: + // No Log .. + break; + // 201 - Created OK + // 202 - Accepted + // 204 - Deleted OK + case 201,202,204: + // No actions, but log if verbose logging + //log.vlog("OneDrive Response: '", http.statusLine.code, " - ", http.statusLine.reason, "'"); + break; + + // 302 - resource found and available at another location, redirect + case 302: + break; + + // 400 - Bad Request + case 400: + // Bad Request .. how should we act? + log.vlog("OneDrive returned a 'HTTP 400 - Bad Request' - gracefully handling error"); + break; + + // 403 - Forbidden + case 403: + // OneDrive responded that the user is forbidden + log.vlog("OneDrive returned a 'HTTP 403 - Forbidden' - gracefully handling error"); + break; + + // 404 - Item not found + case 404: + // Item was not found - do not throw an exception + log.vlog("OneDrive returned a 'HTTP 404 - Item not found' - gracefully handling error"); + break; + + // 408 - Request Timeout + case 408: + // Request to connect to OneDrive service timed out + log.vlog("Request Timeout - gracefully handling error"); + throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?"); + + // 409 - Conflict + case 409: + // Conflict handling .. how should we act? This only really gets triggered if we are using --local-first & we remove items.db as the DB thinks the file is not uploaded but it is + log.vlog("OneDrive returned a 'HTTP 409 - Conflict' - gracefully handling error"); + break; + + // 412 - Precondition Failed + case 412: + // A precondition provided in the request (such as an if-match header) does not match the resource's current state. + log.vlog("OneDrive returned a 'HTTP 412 - Precondition Failed' - gracefully handling error"); + break; + + // 415 - Unsupported Media Type + case 415: + // Unsupported Media Type ... sometimes triggered on image files, especially PNG + log.vlog("OneDrive returned a 'HTTP 415 - Unsupported Media Type' - gracefully handling error"); + break; + + // 429 - Too Many Requests + case 429: + // Too many requests in a certain time window + // Check the HTTP Response headers - needed for correct 429 handling + checkHTTPResponseHeaders(); + // https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online + log.vlog("OneDrive returned a 'HTTP 429 - Too Many Requests' - gracefully handling error"); + throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason); + + // Server side (OneDrive) Errors + // 500 - Internal Server Error + // 502 - Bad Gateway + // 503 - Service Unavailable + // 504 - Gateway Timeout (Issue #320) + case 500: + // No actions + log.vlog("OneDrive returned a 'HTTP 500 Internal Server Error' - gracefully handling error"); + break; + + case 502: + // No actions + log.vlog("OneDrive returned a 'HTTP 502 Bad Gateway Error' - gracefully handling error"); + break; + + case 503: + // No actions + log.vlog("OneDrive returned a 'HTTP 503 Service Unavailable Error' - gracefully handling error"); + break; + + case 504: + // No actions + log.vlog("OneDrive returned a 'HTTP 504 Gateway Timeout Error' - gracefully handling error"); + break; + + // "else" + default: + throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason); + } + } +} \ No newline at end of file diff --git a/src/progress.d b/src/progress.d new file mode 100644 index 00000000..03d5481a --- /dev/null +++ b/src/progress.d @@ -0,0 +1,160 @@ +// What is this module called? +module progress; + +// What does this module require to function? +import std.stdio; +import std.range; +import std.format; +import std.datetime; +import core.sys.posix.unistd; +import core.sys.posix.sys.ioctl; + +// What other modules that we have created do we need to import? + +class Progress +{ + private: + + immutable static size_t default_width = 80; + size_t max_width = 40; + size_t width = default_width; + + ulong start_time; + string caption = "Progress"; + size_t iterations; + size_t counter; + + + size_t getTerminalWidth() { + size_t column = default_width; + version (CRuntime_Musl) { + } else version(Android) { + } else { + winsize ws; + if(ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) != -1 && ws.ws_col > 0) { + column = ws.ws_col; + } + } + + return column; + } + + + void clear() { + write("\r"); + for(auto i = 0; i < width; i++) write(" "); + write("\r"); + } + + + int calc_eta() { + immutable auto ratio = cast(double)counter / iterations; + auto current_time = Clock.currTime.toUnixTime(); + auto duration = cast(int)(current_time - start_time); + int hours, minutes, seconds; + double elapsed = (current_time - start_time); + int eta_sec = cast(int)((elapsed / ratio) - elapsed); + + // Return an ETA or Duration? + if (eta_sec != 0){ + return eta_sec; + } else { + return duration; + } + } + + + string progressbarText(string header_text, string footer_text) { + immutable auto ratio = cast(double)counter / iterations; + string result = ""; + + double bar_length = width - header_text.length - footer_text.length; + if(bar_length > max_width && max_width > 0) { + bar_length = max_width; + } + size_t i = 0; + for(; i < ratio * bar_length; i++) result ~= "o"; + for(; i < bar_length; i++) result ~= " "; + + return header_text ~ result ~ footer_text; + } + + + void print() { + immutable auto ratio = cast(double)counter / iterations; + auto header = appender!string(); + auto footer = appender!string(); + + header.formattedWrite("%s %3d%% |", caption, cast(int)(ratio * 100)); + + if(counter <= 0 || ratio == 0.0) { + footer.formattedWrite("| ETA --:--:--:"); + } else { + int h, m, s; + dur!"seconds"(calc_eta()) + .split!("hours", "minutes", "seconds")(h, m, s); + if (counter != iterations){ + footer.formattedWrite("| ETA %02d:%02d:%02d ", h, m, s); + } else { + footer.formattedWrite("| DONE IN %02d:%02d:%02d ", h, m, s); + } + } + + write(progressbarText(header.data, footer.data)); + } + + + void update() { + width = getTerminalWidth(); + + clear(); + + print(); + stdout.flush(); + } + + + public: + + this(size_t iterations) { + if(iterations <= 0) iterations = 1; + + counter = -1; + this.iterations = iterations; + start_time = Clock.currTime.toUnixTime; + } + + @property { + string title() { return caption; } + string title(string text) { return caption = text; } + } + + @property { + size_t count() { return counter; } + size_t count(size_t val) { + if(val > iterations) val = iterations; + return counter = val; + } + } + + @property { + size_t maxWidth() { return max_width; } + size_t maxWidth(size_t w) { + return max_width = w; + } + } + + void reset() { + counter = -1; + start_time = Clock.currTime.toUnixTime; + } + + void next() { + counter++; + if(counter > iterations) counter = iterations; + + update(); + } + + +} diff --git a/src/qxor.d b/src/qxor.d new file mode 100644 index 00000000..64de204f --- /dev/null +++ b/src/qxor.d @@ -0,0 +1,78 @@ +// What is this module called? +module qxor; + +// What does this module require to function? +import std.algorithm; +import std.digest; + +// Implementation of the QuickXorHash algorithm in D +// https://github.com/OneDrive/onedrive-api-docs/blob/live/docs/code-snippets/quickxorhash.md +struct QuickXor +{ + private enum int widthInBits = 160; + private enum size_t lengthInBytes = (widthInBits - 1) / 8 + 1; + private enum size_t lengthInQWords = (widthInBits - 1) / 64 + 1; + private enum int bitsInLastCell = widthInBits % 64; // 32 + private enum int shift = 11; + + private ulong[lengthInQWords] _data; + private ulong _lengthSoFar; + private int _shiftSoFar; + + nothrow @safe void put(scope const(ubyte)[] array...) + { + int vectorArrayIndex = _shiftSoFar / 64; + int vectorOffset = _shiftSoFar % 64; + immutable size_t iterations = min(array.length, widthInBits); + + for (size_t i = 0; i < iterations; i++) { + immutable bool isLastCell = vectorArrayIndex == _data.length - 1; + immutable int bitsInVectorCell = isLastCell ? bitsInLastCell : 64; + + if (vectorOffset <= bitsInVectorCell - 8) { + for (size_t j = i; j < array.length; j += widthInBits) { + _data[vectorArrayIndex] ^= cast(ulong) array[j] << vectorOffset; + } + } else { + int index1 = vectorArrayIndex; + int index2 = isLastCell ? 0 : (vectorArrayIndex + 1); + ubyte low = cast(ubyte) (bitsInVectorCell - vectorOffset); + + ubyte xoredByte = 0; + for (size_t j = i; j < array.length; j += widthInBits) { + xoredByte ^= array[j]; + } + + _data[index1] ^= cast(ulong) xoredByte << vectorOffset; + _data[index2] ^= cast(ulong) xoredByte >> low; + } + + vectorOffset += shift; + if (vectorOffset >= bitsInVectorCell) { + vectorArrayIndex = isLastCell ? 0 : vectorArrayIndex + 1; + vectorOffset -= bitsInVectorCell; + } + } + + _shiftSoFar = cast(int) (_shiftSoFar + shift * (array.length % widthInBits)) % widthInBits; + _lengthSoFar += array.length; + + } + + nothrow @safe void start() + { + _data = _data.init; + _shiftSoFar = 0; + _lengthSoFar = 0; + } + + nothrow @trusted ubyte[lengthInBytes] finish() + { + ubyte[lengthInBytes] tmp; + tmp[0 .. lengthInBytes] = (cast(ubyte*) _data)[0 .. lengthInBytes]; + for (size_t i = 0; i < 8; i++) { + tmp[lengthInBytes - 8 + i] ^= (cast(ubyte*) &_lengthSoFar)[i]; + } + return tmp; + } +} \ No newline at end of file diff --git a/src/sqlite.d b/src/sqlite.d new file mode 100644 index 00000000..97e39289 --- /dev/null +++ b/src/sqlite.d @@ -0,0 +1,207 @@ +// What is this module called? +module sqlite; + +// What does this module require to function? +import std.stdio; +import etc.c.sqlite3; +import std.string: fromStringz, toStringz; +import core.stdc.stdlib; +import std.conv; + +// What other modules that we have created do we need to import? +import log; + +extern (C) immutable(char)* sqlite3_errstr(int); // missing from the std library + +static this() { + if (sqlite3_libversion_number() < 3006019) { + throw new SqliteException("sqlite 3.6.19 or newer is required"); + } +} + +private string ifromStringz(const(char)* cstr) { + return fromStringz(cstr).dup; +} + +class SqliteException: Exception { + @safe pure nothrow this(string msg, string file = __FILE__, size_t line = __LINE__, Throwable next = null) + { + super(msg, file, line, next); + } + + @safe pure nothrow this(string msg, Throwable next, string file = __FILE__, size_t line = __LINE__) + { + super(msg, file, line, next); + } +} + +struct Database { + private sqlite3* pDb; + + this(const(char)[] filename) { + open(filename); + } + + ~this() { + close(); + } + + int db_checkpoint() { + return sqlite3_wal_checkpoint(pDb, null); + } + + void dump_open_statements() { + log.log("Dumping open statements: \n"); + auto p = sqlite3_next_stmt(pDb, null); + while (p != null) { + log.log (" - " ~ ifromStringz(sqlite3_sql(p)) ~ "\n"); + p = sqlite3_next_stmt(pDb, p); + } + } + + + void open(const(char)[] filename) { + // https://www.sqlite.org/c3ref/open.html + int rc = sqlite3_open(toStringz(filename), &pDb); + if (rc == SQLITE_CANTOPEN) { + // Database cannot be opened + log.error("\nThe database cannot be opened. Please check the permissions of " ~ filename ~ "\n"); + close(); + exit(-1); + } + if (rc != SQLITE_OK) { + log.error("\nA database access error occurred: " ~ getErrorMessage() ~ "\n"); + close(); + exit(-1); + } + sqlite3_extended_result_codes(pDb, 1); // always use extended result codes + } + + void exec(const(char)[] sql) { + // https://www.sqlite.org/c3ref/exec.html + int rc = sqlite3_exec(pDb, toStringz(sql), null, null, null); + if (rc != SQLITE_OK) { + log.error("\nA database execution error occurred: "~ getErrorMessage() ~ "\n"); + log.error("Please retry your command with --resync to fix any local database corruption issues.\n"); + close(); + exit(-1); + } + } + + int getVersion() { + int userVersion; + extern (C) int callback(void* user_version, int count, char** column_text, char** column_name) { + import core.stdc.stdlib: atoi; + *(cast(int*) user_version) = atoi(*column_text); + return 0; + } + int rc = sqlite3_exec(pDb, "PRAGMA user_version", &callback, &userVersion, null); + if (rc != SQLITE_OK) { + throw new SqliteException(ifromStringz(sqlite3_errmsg(pDb))); + } + return userVersion; + } + + string getErrorMessage() { + return ifromStringz(sqlite3_errmsg(pDb)); + } + + void setVersion(int userVersion) { + import std.conv: to; + exec("PRAGMA user_version=" ~ to!string(userVersion)); + } + + Statement prepare(const(char)[] zSql) { + Statement s; + // https://www.sqlite.org/c3ref/prepare.html + int rc = sqlite3_prepare_v2(pDb, zSql.ptr, cast(int) zSql.length, &s.pStmt, null); + if (rc != SQLITE_OK) { + throw new SqliteException(ifromStringz(sqlite3_errmsg(pDb))); + } + return s; + } + + void close() { + // https://www.sqlite.org/c3ref/close.html + sqlite3_close_v2(pDb); + pDb = null; + } +} + +struct Statement { + struct Result { + private sqlite3_stmt* pStmt; + private const(char)[][] row; + + private this(sqlite3_stmt* pStmt) { + this.pStmt = pStmt; + step(); // initialize the range + } + + @property bool empty() { + return row.length == 0; + } + + @property auto front() { + return row; + } + + alias step popFront; + + void step() { + // https://www.sqlite.org/c3ref/step.html + int rc = sqlite3_step(pStmt); + if (rc == SQLITE_BUSY) { + // Database is locked by another onedrive process + log.error("The database is currently locked by another process - cannot sync"); + return; + } + if (rc == SQLITE_DONE) { + row.length = 0; + } else if (rc == SQLITE_ROW) { + // https://www.sqlite.org/c3ref/data_count.html + int count = 0; + count = sqlite3_data_count(pStmt); + row = new const(char)[][count]; + foreach (size_t i, ref column; row) { + // https://www.sqlite.org/c3ref/column_blob.html + column = fromStringz(sqlite3_column_text(pStmt, to!int(i))); + } + } else { + string errorMessage = ifromStringz(sqlite3_errmsg(sqlite3_db_handle(pStmt))); + log.error("\nA database statement execution error occurred: "~ errorMessage ~ "\n"); + log.error("Please retry your command with --resync to fix any local database corruption issues.\n"); + exit(-1); + } + } + } + + private sqlite3_stmt* pStmt; + + ~this() { + // https://www.sqlite.org/c3ref/finalize.html + sqlite3_finalize(pStmt); + } + + void bind(int index, const(char)[] value) { + reset(); + // https://www.sqlite.org/c3ref/bind_blob.html + int rc = sqlite3_bind_text(pStmt, index, value.ptr, cast(int) value.length, SQLITE_STATIC); + if (rc != SQLITE_OK) { + throw new SqliteException(ifromStringz(sqlite3_errmsg(sqlite3_db_handle(pStmt)))); + } + } + + Result exec() { + reset(); + return Result(pStmt); + } + + private void reset() { + // https://www.sqlite.org/c3ref/reset.html + int rc = sqlite3_reset(pStmt); + if (rc != SQLITE_OK) { + throw new SqliteException(ifromStringz(sqlite3_errmsg(sqlite3_db_handle(pStmt)))); + } + } +} \ No newline at end of file diff --git a/src/sync.d b/src/sync.d new file mode 100644 index 00000000..dd664bea --- /dev/null +++ b/src/sync.d @@ -0,0 +1,6535 @@ +// What is this module called? +module syncEngine; + +// What does this module require to function? +import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit; +import core.thread; +import core.time; +import std.algorithm; +import std.array; +import std.concurrency; +import std.conv; +import std.datetime; +import std.encoding; +import std.exception; +import std.file; +import std.json; +import std.parallelism; +import std.path; +import std.range; +import std.regex; +import std.stdio; +import std.string; +import std.uni; +import std.uri; +import std.utf; + +// What other modules that we have created do we need to import? +import config; +import log; +import util; +import onedrive; +import itemdb; +import clientSideFiltering; +import progress; + +class posixException: Exception { + @safe pure this(string localTargetName, string remoteTargetName) { + string msg = format("POSIX 'case-insensitive match' between '%s' (local) and '%s' (online) which violates the Microsoft OneDrive API namespace convention", localTargetName, remoteTargetName); + super(msg); + } +} + +class SyncException: Exception { + @nogc @safe pure nothrow this(string msg, string file = __FILE__, size_t line = __LINE__) { + super(msg, file, line); + } +} + +class SyncEngine { + // Class Variables + ApplicationConfig appConfig; + OneDriveApi oneDriveApiInstance; + ItemDatabase itemDB; + ClientSideFiltering selectiveSync; + + // Array of directory databaseItem.id to skip while applying the changes. + // These are the 'parent path' id's that are being excluded, so if the parent id is in here, the child needs to be skipped as well + string[] skippedItems; + // Array of databaseItem.id to delete after the changes have been downloaded + string[2][] idsToDelete; + // Array of JSON items which are files or directories that are not 'root', skipped or to be deleted, that need to be processed + JSONValue[] jsonItemsToProcess; + // Array of JSON items which are files that are not 'root', skipped or to be deleted, that need to be downloaded + JSONValue[] fileJSONItemsToDownload; + // Array of paths that failed to download + string[] fileDownloadFailures; + // Array of all OneDrive driveId's that have been seen + string[] driveIDsArray; + // List of items we fake created when using --dry-run + string[2][] idsFaked; + // List of paths we fake deleted when using --dry-run + string[] pathFakeDeletedArray; + // Array of database Parent Item ID, Item ID & Local Path where the content has changed and needs to be uploaded + string[3][] databaseItemsWhereContentHasChanged; + // Array of local file paths that need to be uploaded as new itemts to OneDrive + string[] newLocalFilesToUploadToOneDrive; + // Array of local file paths that failed to be uploaded to OneDrive + string[] fileUploadFailures; + // List of path names changed online, but not changed locally when using --dry-run + string[] pathsRenamed; + // List of paths that were a POSIX case-insensitive match, thus could not be created online + string[] posixViolationPaths; + // List of local paths, that, when using the OneDrive Business Shared Folders feature, then diabling it, folder still exists locally and online + // This list of local paths need to be skipped + string[] businessSharedFoldersOnlineToSkip; + + // Flag that there were upload or download failures listed + bool syncFailures = false; + // Is sync_list configured + bool syncListConfigured = false; + // Was --dry-run used? + bool dryRun = false; + // Was --upload-only used? + bool uploadOnly = false; + // Was --remove-source-files used? + // Flag to set whether the local file should be deleted once it is successfully uploaded to OneDrive + bool localDeleteAfterUpload = false; + + // Do we configure to disable the download validation routine due to --disable-download-validation + // We will always validate our downloads + // However, when downloading files from SharePoint, the OneDrive API will not advise the correct file size + // which means that the application thinks the file download has failed as the size is different / hash is different + // See: https://github.com/abraunegg/onedrive/discussions/1667 + bool disableDownloadValidation = false; + + // Do we configure to disable the upload validation routine due to --disable-upload-validation + // We will always validate our uploads + // However, when uploading a file that can contain metadata SharePoint will associate some + // metadata from the library the file is uploaded to directly in the file which breaks this validation. + // See: https://github.com/abraunegg/onedrive/issues/205 + // See: https://github.com/OneDrive/onedrive-api-docs/issues/935 + bool disableUploadValidation = false; + + // Do we perform a local cleanup of files that are 'extra' on the local file system, when using --download-only + bool cleanupLocalFiles = false; + // Are we performing a --single-directory sync ? + bool singleDirectoryScope = false; + string singleDirectoryScopeDriveId; + string singleDirectoryScopeItemId; + // Is National Cloud Deployments configured ? + bool nationalCloudDeployment = false; + // Do we configure not to perform a remote file delete if --upload-only & --no-remote-delete configured + bool noRemoteDelete = false; + // Is bypass_data_preservation set via config file + // Local data loss MAY occur in this scenario + bool bypassDataPreservation = false; + // Maximum file size upload + // https://support.microsoft.com/en-us/office/invalid-file-names-and-file-types-in-onedrive-and-sharepoint-64883a5d-228e-48f5-b3d2-eb39e07630fa?ui=en-us&rs=en-us&ad=us + // July 2020, maximum file size for all accounts is 100GB + // January 2021, maximum file size for all accounts is 250GB + ulong maxUploadFileSize = 268435456000; // 250GB + // Threshold after which files will be uploaded using an upload session + ulong sessionThresholdFileSize = 4 * 2^^20; // 4 MiB + // File size limit for file operations that the user has configured + ulong fileSizeLimit; + // Total data to upload + ulong totalDataToUpload; + // How many items have been processed for the active operation + ulong processedCount; + + // Are we creating a simulated /delta response? This is critically important in terms of how we 'update' the database + bool generateSimulatedDeltaResponse = false; + + // Configure this class instance + this(ApplicationConfig appConfig, ItemDatabase itemDB, ClientSideFiltering selectiveSync) { + // Configure the class varaible to consume the application configuration + this.appConfig = appConfig; + // Configure the class varaible to consume the database configuration + this.itemDB = itemDB; + // Configure the class variable to consume the selective sync (skip_dir, skip_file and sync_list) configuration + this.selectiveSync = selectiveSync; + + // Configure the dryRun flag to capture if --dry-run was used + // Application startup already flagged we are also in a --dry-run state, so no need to output anything else here + this.dryRun = appConfig.getValueBool("dry_run"); + + // Configure file size limit + if (appConfig.getValueLong("skip_size") != 0) { + fileSizeLimit = appConfig.getValueLong("skip_size") * 2^^20; + fileSizeLimit = (fileSizeLimit == 0) ? ulong.max : fileSizeLimit; + } + + // Is there a sync_list file present? + if (exists(appConfig.syncListFilePath)) this.syncListConfigured = true; + + // Configure the uploadOnly flag to capture if --upload-only was used + if (appConfig.getValueBool("upload_only")) { + log.vdebug("Configuring uploadOnly flag to TRUE as --upload-only passed in or configured"); + this.uploadOnly = true; + } + + // Configure the localDeleteAfterUpload flag + if (appConfig.getValueBool("remove_source_files")) { + log.vdebug("Configuring localDeleteAfterUpload flag to TRUE as --remove-source-files passed in or configured"); + this.localDeleteAfterUpload = true; + } + + // Configure the disableDownloadValidation flag + if (appConfig.getValueBool("disable_download_validation")) { + log.vdebug("Configuring disableDownloadValidation flag to TRUE as --disable-download-validation passed in or configured"); + this.disableDownloadValidation = true; + } + + // Configure the disableUploadValidation flag + if (appConfig.getValueBool("disable_upload_validation")) { + log.vdebug("Configuring disableUploadValidation flag to TRUE as --disable-upload-validation passed in or configured"); + this.disableUploadValidation = true; + } + + // Do we configure to clean up local files if using --download-only ? + if ((appConfig.getValueBool("download_only")) && (appConfig.getValueBool("cleanup_local_files"))) { + // --download-only and --cleanup-local-files were passed in + log.log("WARNING: Application has been configured to cleanup local files that are not present online."); + log.log("WARNING: Local data loss MAY occur in this scenario if you are expecting data to remain archived locally."); + // Set the flag + this.cleanupLocalFiles = true; + } + + // Do we configure to NOT perform a remote delete if --upload-only & --no-remote-delete configured ? + if ((appConfig.getValueBool("upload_only")) && (appConfig.getValueBool("no_remote_delete"))) { + // --upload-only and --no-remote-delete were passed in + log.log("WARNING: Application has been configured NOT to cleanup remote files that are deleted locally."); + // Set the flag + this.noRemoteDelete = true; + } + + // Are we forcing to use /children scan instead of /delta to simulate National Cloud Deployment use of /children? + if (appConfig.getValueBool("force_children_scan")) { + log.log("Forcing client to use /children API call rather than /delta API to retrieve objects from the OneDrive API"); + this.nationalCloudDeployment = true; + } + + // Are we forcing the client to bypass any data preservation techniques to NOT rename any local files if there is a conflict? + // The enabling of this function could lead to data loss + if (appConfig.getValueBool("bypass_data_preservation")) { + log.log("WARNING: Application has been configured to bypass local data preservation in the event of file conflict."); + log.log("WARNING: Local data loss MAY occur in this scenario."); + this.bypassDataPreservation = true; + } + + // Did the user configure a specific rate limit for the application? + if (appConfig.getValueLong("rate_limit") > 0) { + // User configured rate limit + log.log("User Configured Rate Limit: ", appConfig.getValueLong("rate_limit")); + + // If user provided rate limit is < 131072, flag that this is too low, setting to the recommended minimum of 131072 + if (appConfig.getValueLong("rate_limit") < 131072) { + // user provided limit too low + log.log("WARNING: User configured rate limit too low for normal application processing and preventing application timeouts. Overriding to recommended minimum of 131072 (128KB/s)"); + appConfig.setValueLong("rate_limit", 131072); + } + } + + // Did the user downgrade all HTTP operations to force HTTP 1.1 + if (appConfig.getValueBool("force_http_11")) { + // User is forcing downgrade to curl to use HTTP 1.1 for all operations + log.vlog("Downgrading all HTTP operations to HTTP/1.1 due to user configuration"); + } else { + // Use curl defaults + log.vdebug("Using Curl defaults for HTTP operational protocol version (potentially HTTP/2)"); + } + } + + // Initialise the Sync Engine class + bool initialise() { + + // create a new instance of the OneDrive API + oneDriveApiInstance = new OneDriveApi(appConfig); + if (oneDriveApiInstance.initialise()) { + log.log("Sync Engine Initialised with new Onedrive API instance"); + // Get the default account drive details + getDefaultDriveDetails(); + getDefaultRootDetails(); + displaySyncEngineDetails(); + } else { + // API could not be initialised + log.error("OneDrive API could not be initialised"); + // Shutdown API instance + oneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(oneDriveApiInstance); + exit(-1); + } + // Shutdown API instance + oneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(oneDriveApiInstance); + return true; + } + + // Get Default Drive Details for this Account + void getDefaultDriveDetails() { + + // Function variables + JSONValue defaultOneDriveDriveDetails; + + // Get Default Drive Details for this Account + try { + log.vdebug("Getting Account Default Drive Details"); + defaultOneDriveDriveDetails = oneDriveApiInstance.getDefaultDriveDetails(); + } catch (OneDriveException exception) { + log.vdebug("defaultOneDriveDriveDetails = oneDriveApiInstance.getDefaultDriveDetails() generated a OneDriveException"); + + string thisFunctionName = getFunctionName!({}); + + if ((exception.httpStatusCode == 400) || (exception.httpStatusCode == 401)) { + // Handle the 400 | 401 error + handleClientUnauthorised(exception.httpStatusCode, exception.msg); + } + + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(oneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) ||(exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to query Account Default Drive Details - retrying applicable request in 30 seconds"); + log.vdebug("defaultOneDriveDriveDetails = oneDriveApiInstance.getDefaultDriveDetails() previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429 and 504 - but loop back calling this function + log.vdebug("Retrying Function: getDefaultDriveDetails()"); + getDefaultDriveDetails(); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + } + } + + // If the JSON response is a correct JSON object, and has an 'id' we can set these details + if ((defaultOneDriveDriveDetails.type() == JSONType.object) && (hasId(defaultOneDriveDriveDetails))) { + log.vdebug("OneDrive Account Default Drive Details: ", defaultOneDriveDriveDetails); + appConfig.accountType = defaultOneDriveDriveDetails["driveType"].str; + appConfig.defaultDriveId = defaultOneDriveDriveDetails["id"].str; + + // Get the initial remaining size from OneDrive API response JSON + // This will be updated as we upload data to OneDrive + if (hasQuota(defaultOneDriveDriveDetails)) { + if ("remaining" in defaultOneDriveDriveDetails["quota"]){ + // use the value provided + appConfig.remainingFreeSpace = defaultOneDriveDriveDetails["quota"]["remaining"].integer; + } + } + + // In some cases OneDrive Business configurations 'restrict' quota details thus is empty / blank / negative value / zero + if (appConfig.remainingFreeSpace <= 0) { + // free space is <= 0 .. why ? + if ("remaining" in defaultOneDriveDriveDetails["quota"]) { + if (appConfig.accountType == "personal") { + // zero space available + log.error("ERROR: OneDrive account currently has zero space available. Please free up some space online."); + appConfig.quotaAvailable = false; + } else { + // zero space available is being reported, maybe being restricted? + log.error("WARNING: OneDrive quota information is being restricted or providing a zero value. Please fix by speaking to your OneDrive / Office 365 Administrator."); + appConfig.quotaRestricted = true; + } + } else { + // json response was missing a 'remaining' value + if (appConfig.accountType == "personal") { + log.error("ERROR: OneDrive quota information is missing. Potentially your OneDrive account currently has zero space available. Please free up some space online."); + appConfig.quotaAvailable = false; + } else { + // quota details not available + log.error("ERROR: OneDrive quota information is being restricted. Please fix by speaking to your OneDrive / Office 365 Administrator."); + appConfig.quotaRestricted = true; + } + } + } + // What did we set based on the data from the JSON + log.vdebug("appConfig.accountType = ", appConfig.accountType); + log.vdebug("appConfig.defaultDriveId = ", appConfig.defaultDriveId); + log.vdebug("appConfig.remainingFreeSpace = ", appConfig.remainingFreeSpace); + log.vdebug("appConfig.quotaAvailable = ", appConfig.quotaAvailable); + log.vdebug("appConfig.quotaRestricted = ", appConfig.quotaRestricted); + + // Make sure that appConfig.defaultDriveId is in our driveIDs array to use when checking if item is in database + // Keep the driveIDsArray with unique entries only + if (!canFind(driveIDsArray, appConfig.defaultDriveId)) { + // Add this drive id to the array to search with + driveIDsArray ~= appConfig.defaultDriveId; + } + } else { + // Handle the invalid JSON response + invalidJSONResponseFromOneDriveAPI(); + } + } + + // Get Default Root Details for this Account + void getDefaultRootDetails() { + + // Function variables + JSONValue defaultOneDriveRootDetails; + + // Get Default Root Details for this Account + try { + log.vdebug("Getting Account Default Root Details"); + defaultOneDriveRootDetails = oneDriveApiInstance.getDefaultRootDetails(); + } catch (OneDriveException exception) { + log.vdebug("defaultOneDriveRootDetails = oneDriveApiInstance.getDefaultRootDetails() generated a OneDriveException"); + + string thisFunctionName = getFunctionName!({}); + + if ((exception.httpStatusCode == 400) || (exception.httpStatusCode == 401)) { + // Handle the 400 | 401 error + handleClientUnauthorised(exception.httpStatusCode, exception.msg); + } + + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(oneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to query Account Default Root Details - retrying applicable request in 30 seconds"); + log.vdebug("defaultOneDriveRootDetails = oneDriveApiInstance.getDefaultRootDetails() previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: getDefaultRootDetails()"); + getDefaultRootDetails(); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + } + } + + // If the JSON response is a correct JSON object, and has an 'id' we can set these details + if ((defaultOneDriveRootDetails.type() == JSONType.object) && (hasId(defaultOneDriveRootDetails))) { + log.vdebug("OneDrive Account Default Root Details: ", defaultOneDriveRootDetails); + appConfig.defaultRootId = defaultOneDriveRootDetails["id"].str; + log.vdebug("appConfig.defaultRootId = ", appConfig.defaultRootId); + // Save the item to the database, so the account root drive is is always going to be present in the DB + saveItem(defaultOneDriveRootDetails); + } else { + // Handle the invalid JSON response + invalidJSONResponseFromOneDriveAPI(); + } + } + + // Perform a sync of the OneDrive Account + // - Query /delta + // - If singleDirectoryScope or nationalCloudDeployment is used we need to generate a /delta like response + // - Process changes (add, changes, moves, deletes) + // - Process any items to add (download data to local) + // - Detail any files that we failed to download + // - Process any deletes (remove local data) + // - Walk local file system for any differences (new files / data to upload to OneDrive) + void syncOneDriveAccountToLocalDisk() { + + // performFullScanTrueUp value + log.vdebug("Perform a Full Scan True-Up: ", appConfig.fullScanTrueUpRequired); + // Fetch the API response of /delta to track changes on OneDrive + fetchOneDriveDeltaAPIResponse(null, null, null); + // Process any download activities or cleanup actions + processDownloadActivities(); + + // If singleDirectoryScope is false, we are not targeting a single directory + // but if true, the target 'could' be a shared folder - so dont try and scan it again + if (!singleDirectoryScope) { + // OneDrive Shared Folder Handling + if (appConfig.accountType == "personal") { + // Personal Account Type + // https://github.com/OneDrive/onedrive-api-docs/issues/764 + + // Get the Remote Items from the Database + Item[] remoteItems = itemDB.selectRemoteItems(); + foreach (remoteItem; remoteItems) { + // Check if this path is specifically excluded by 'skip_dir', but only if 'skip_dir' is not empty + if (appConfig.getValueString("skip_dir") != "") { + // The path that needs to be checked needs to include the '/' + // This due to if the user has specified in skip_dir an exclusive path: '/path' - that is what must be matched + if (selectiveSync.isDirNameExcluded(remoteItem.name)) { + // This directory name is excluded + log.vlog("Skipping item - excluded by skip_dir config: ", remoteItem.name); + continue; + } + } + + // Directory name is not excluded or skip_dir is not populated + if (!appConfig.surpressLoggingOutput) { + log.log("Syncing this OneDrive Personal Shared Folder: ", remoteItem.name); + } + // Check this OneDrive Personal Shared Folder for changes + fetchOneDriveDeltaAPIResponse(remoteItem.remoteDriveId, remoteItem.remoteId, remoteItem.name); + // Process any download activities or cleanup actions for this OneDrive Personal Shared Folder + processDownloadActivities(); + } + } else { + // Is this a Business Account with Sync Business Shared Items enabled? + if ((appConfig.accountType == "business") && ( appConfig.getValueBool("sync_business_shared_items"))) { + + // Business Account Shared Items Handling + // - OneDrive Business Shared Folder + // - OneDrive Business Shared Files ?? + // - SharePoint Links + + // Get the Remote Items from the Database + Item[] remoteItems = itemDB.selectRemoteItems(); + + foreach (remoteItem; remoteItems) { + // Check if this path is specifically excluded by 'skip_dir', but only if 'skip_dir' is not empty + if (appConfig.getValueString("skip_dir") != "") { + // The path that needs to be checked needs to include the '/' + // This due to if the user has specified in skip_dir an exclusive path: '/path' - that is what must be matched + if (selectiveSync.isDirNameExcluded(remoteItem.name)) { + // This directory name is excluded + log.vlog("Skipping item - excluded by skip_dir config: ", remoteItem.name); + continue; + } + } + + // Directory name is not excluded or skip_dir is not populated + if (!appConfig.surpressLoggingOutput) { + log.log("Syncing this OneDrive Business Shared Folder: ", remoteItem.name); + } + + log.vdebug("Fetching /delta API response for:"); + log.vdebug(" remoteItem.remoteDriveId: ", remoteItem.remoteDriveId); + log.vdebug(" remoteItem.remoteId: ", remoteItem.remoteId); + + // Check this OneDrive Personal Shared Folder for changes + fetchOneDriveDeltaAPIResponse(remoteItem.remoteDriveId, remoteItem.remoteId, remoteItem.name); + + // Process any download activities or cleanup actions for this OneDrive Personal Shared Folder + processDownloadActivities(); + } + } + } + } + } + + // Configure singleDirectoryScope = true if this function is called + // By default, singleDirectoryScope = false + void setSingleDirectoryScope(string normalisedSingleDirectoryPath) { + + // Function variables + Item searchItem; + JSONValue onlinePathData; + + // Set the main flag + singleDirectoryScope = true; + + // What are we doing? + log.log("The OneDrive Client was asked to search for this directory online and create it if it's not located: ", normalisedSingleDirectoryPath); + + // Query the OneDrive API for the specified path online + // In a --single-directory scenario, we need to travervse the entire path that we are wanting to sync + // and then check the path element does it exist online, if it does, is it a POSIX match, or if it does not, create the path + // Once we have searched online, we have the right drive id and item id so that we can downgrade the sync status, then build up + // any object items from that location + // This is because, in a --single-directory scenario, any folder in the entire path tree could be a 'case-insensitive match' + + try { + onlinePathData = queryOneDriveForSpecificPathAndCreateIfMissing(normalisedSingleDirectoryPath, true); + } catch (posixException e) { + displayPosixErrorMessage(e.msg); + log.error("ERROR: Requested directory to search for and potentially create has a 'case-insensitive match' to an existing directory on OneDrive online."); + } + + // Was a valid JSON response provided? + if (onlinePathData.type() == JSONType.object) { + // Valid JSON item was returned + searchItem = makeItem(onlinePathData); + log.vdebug("searchItem: ", searchItem); + + // Is this item a potential Shared Folder? + // Is this JSON a remote object + if (isItemRemote(onlinePathData)) { + // The path we are seeking is remote to our account drive id + searchItem.driveId = onlinePathData["remoteItem"]["parentReference"]["driveId"].str; + searchItem.id = onlinePathData["remoteItem"]["id"].str; + } + + // Set these items so that these can be used as required + singleDirectoryScopeDriveId = searchItem.driveId; + singleDirectoryScopeItemId = searchItem.id; + } else { + log.error("\nThe requested --single-directory path to sync has generated an error. Please correct this error and try again.\n"); + exit(EXIT_FAILURE); + } + } + + // Query OneDrive API for /delta changes and iterate through items online + void fetchOneDriveDeltaAPIResponse(string driveIdToQuery = null, string itemIdToQuery = null, string sharedFolderName = null) { + + string deltaLink = null; + string deltaLinkAvailable; + JSONValue deltaChanges; + ulong responseBundleCount; + ulong jsonItemsReceived = 0; + + // Reset jsonItemsToProcess & processedCount + jsonItemsToProcess = []; + processedCount = 0; + + // Was a driveId provided as an input + //if (driveIdToQuery == "") { + if (strip(driveIdToQuery).empty) { + // No provided driveId to query, use the account default + log.vdebug("driveIdToQuery was empty, setting to appConfig.defaultDriveId"); + driveIdToQuery = appConfig.defaultDriveId; + log.vdebug("driveIdToQuery: ", driveIdToQuery); + } + + // Was an itemId provided as an input + //if (itemIdToQuery == "") { + if (strip(itemIdToQuery).empty) { + // No provided itemId to query, use the account default + log.vdebug("itemIdToQuery was empty, setting to appConfig.defaultRootId"); + itemIdToQuery = appConfig.defaultRootId; + log.vdebug("itemIdToQuery: ", itemIdToQuery); + } + + // What OneDrive API query do we use? + // - Are we running against a National Cloud Deployments that does not support /delta ? + // National Cloud Deployments do not support /delta as a query + // https://docs.microsoft.com/en-us/graph/deployments#supported-features + // + // - Are we performing a --single-directory sync, which will exclude many items online, focusing in on a specific online directory + // + // - Are we performing a --download-only --cleanup-local-files action? + // - If we are, and we use a normal /delta query, we get all the local 'deleted' objects as well. + // - If the user deletes a folder online, then replaces it online, we download the deletion events and process the new 'upload' via the web iterface .. + // the net effect of this, is that the valid local files we want to keep, are actually deleted ...... not desirable + if ((singleDirectoryScope) || (nationalCloudDeployment) || (cleanupLocalFiles)) { + // Generate a simulated /delta response so that we correctly capture the current online state, less any 'online' delete and replace activity + generateSimulatedDeltaResponse = true; + } + + // What /delta query do we use? + if (!generateSimulatedDeltaResponse) { + // This should be the majority default pathway application use + // Get the current delta link from the database for this DriveID and RootID + deltaLinkAvailable = itemDB.getDeltaLink(driveIdToQuery, itemIdToQuery); + if (!deltaLinkAvailable.empty) { + log.vdebug("Using stored deltaLink"); + deltaLink = deltaLinkAvailable; + } + + // Do we need to perform a Full Scan True Up? Is 'appConfig.fullScanTrueUpRequired' set to 'true'? + if (appConfig.fullScanTrueUpRequired) { + log.log("Performing a full scan of online data to ensure consistent local state"); + log.vdebug("Setting deltaLink = null"); + deltaLink = null; + } + + // Dynamic output for a non-verbose run so that the user knows something is happening + if (log.verbose <= 1) { + if (!appConfig.surpressLoggingOutput) { + log.fileOnly("Fetching items from the OneDrive API for Drive ID: ", driveIdToQuery); + write("Fetching items from the OneDrive API for Drive ID: ", driveIdToQuery, " ."); + } + } else { + log.vdebug("Fetching /delta response from the OneDrive API for driveId: ", driveIdToQuery); + } + + for (;;) { + responseBundleCount++; + // Get the /delta changes via the OneDrive API + log.vdebug(""); + log.vdebug("------------------------------------------------------------------"); + log.vdebug("driveIdToQuery: ", driveIdToQuery); + log.vdebug("itemIdToQuery: ", itemIdToQuery); + log.vdebug("deltaLink: ", deltaLink); + log.vdebug("------------------------------------------------------------------"); + + // getDeltaChangesByItemId has the re-try logic for transient errors + deltaChanges = getDeltaChangesByItemId(driveIdToQuery, itemIdToQuery, deltaLink); + + // If deltaChanges is an invalid JSON object, must exit + if (deltaChanges.type() != JSONType.object){ + // Handle the invalid JSON response + invalidJSONResponseFromOneDriveAPI(); + } + + ulong nrChanges = count(deltaChanges["value"].array); + int changeCount = 0; + + if (log.verbose <= 1) { + // Dynamic output for a non-verbose run so that the user knows something is happening + if (!appConfig.surpressLoggingOutput) { + write("."); + } + } else { + log.vdebug("API Response Bundle: ", responseBundleCount, " - Quantity of 'changes|items' in this bundle to process: ", nrChanges); + } + + jsonItemsReceived = jsonItemsReceived + nrChanges; + + // This means we are most likely processing 200+ items at the same time as the OneDrive API bundles the JSON items + // into 200+ bundle lots and there is zero way to configure or change this + // The API response however cannot be run in parallel as the OneDrive API sends the JSON items in the order in which they must be processed + foreach (onedriveJSONItem; deltaChanges["value"].array) { + // increment change count for this item + changeCount++; + // Process the OneDrive object item JSON + processDeltaJSONItem(onedriveJSONItem, nrChanges, changeCount, responseBundleCount, singleDirectoryScope); + } + + // The response may contain either @odata.deltaLink or @odata.nextLink + if ("@odata.deltaLink" in deltaChanges) { + deltaLink = deltaChanges["@odata.deltaLink"].str; + log.vdebug("Setting next deltaLink to (@odata.deltaLink): ", deltaLink); + } + // Update the deltaLink in the database so that we can reuse this + if (!deltaLink.empty) { + log.vdebug("Updating completed deltaLink in DB to: ", deltaLink); + itemDB.setDeltaLink(driveIdToQuery, itemIdToQuery, deltaLink); + } + // Update deltaLink to next changeSet bundle + if ("@odata.nextLink" in deltaChanges) { + deltaLink = deltaChanges["@odata.nextLink"].str; + // Update deltaLinkAvailable to next changeSet bundle to quantify how many changes we have to process + deltaLinkAvailable = deltaChanges["@odata.nextLink"].str; + log.vdebug("Setting next deltaLink & deltaLinkAvailable to (@odata.nextLink): ", deltaLink); + } + else break; + } + + // To finish off the JSON processing items, this is needed to reflect this in the log + log.vdebug("------------------------------------------------------------------"); + + // Log that we have finished querying the /delta API + if (log.verbose <= 1) { + if (!appConfig.surpressLoggingOutput) { + write("\n"); + } + } else { + log.vdebug("Finished processing /delta JSON response from the OneDrive API"); + } + + // If this was set, now unset it, as this will have been completed, so that for a true up, we dont do a double full scan + if (appConfig.fullScanTrueUpRequired) { + log.vdebug("Unsetting fullScanTrueUpRequired as this has been performed"); + appConfig.fullScanTrueUpRequired = false; + } + } else { + // We have to generate our own /delta response + // Log what we are doing so that the user knows something is happening + if (!appConfig.surpressLoggingOutput) { + log.log("Generating a /delta compatible JSON response from the OneDrive API ..."); + } + + // Why are are generating a /delta response + log.vdebug("Why are we generating a /delta response:"); + log.vdebug(" singleDirectoryScope: ", singleDirectoryScope); + log.vdebug(" nationalCloudDeployment: ", nationalCloudDeployment); + log.vdebug(" cleanupLocalFiles: ", cleanupLocalFiles); + + // What 'path' are we going to start generating the response for + string pathToQuery; + + // If --single-directory has been called, use the value that has been set + if (singleDirectoryScope) { + pathToQuery = appConfig.getValueString("single_directory"); + } + + // We could also be syncing a Shared Folder of some description + if (!sharedFolderName.empty) { + pathToQuery = sharedFolderName; + } + + // Generate the simulated /delta response + // + // The generated /delta response however contains zero deleted JSON items, so the only way that we can track this, is if the object was in sync + // we have the object in the database, thus, what we need to do is for every DB object in the tree of items, flag 'syncStatus' as 'N', then when we process + // the returned JSON items from the API, we flag the item as back in sync, then we can cleanup any out-of-sync items + // + // The flagging of the local database items to 'N' is handled within the generateDeltaResponse() function + // + // When these JSON items are then processed, if the item exists online, and is in the DB, and that the values match, the DB item is flipped back to 'Y' + // This then allows the application to look for any remaining 'N' values, and delete these as no longer needed locally + deltaChanges = generateDeltaResponse(pathToQuery); + + ulong nrChanges = count(deltaChanges["value"].array); + int changeCount = 0; + log.vdebug("API Response Bundle: ", responseBundleCount, " - Quantity of 'changes|items' in this bundle to process: ", nrChanges); + jsonItemsReceived = jsonItemsReceived + nrChanges; + + // The API response however cannot be run in parallel as the OneDrive API sends the JSON items in the order in which they must be processed + foreach (onedriveJSONItem; deltaChanges["value"].array) { + // increment change count for this item + changeCount++; + // Process the OneDrive object item JSON + processDeltaJSONItem(onedriveJSONItem, nrChanges, changeCount, responseBundleCount, singleDirectoryScope); + } + + // To finish off the JSON processing items, this is needed to reflect this in the log + log.vdebug("------------------------------------------------------------------"); + + // Log that we have finished generating our self generated /delta response + if (!appConfig.surpressLoggingOutput) { + log.log("Finished processing self generated /delta JSON response from the OneDrive API"); + } + } + + // We have JSON items received from the OneDrive API + log.vdebug("Number of JSON Objects received from OneDrive API: ", jsonItemsReceived); + log.vdebug("Number of JSON Objects already processed (root and deleted items): ", (jsonItemsReceived - jsonItemsToProcess.length)); + + // We should have now at least processed all the JSON items as returned by the /delta call + // Additionally, we should have a new array, that now contains all the JSON items we need to process that are non 'root' or deleted items + log.vdebug("Number of JSON items to process is: ", jsonItemsToProcess.length); + + // Lets deal with the JSON items in a batch process + ulong batchSize = 500; + ulong batchCount = (jsonItemsToProcess.length + batchSize - 1) / batchSize; + ulong batchesProcessed = 0; + + if (log.verbose == 0) { + // Dynamic output for a non-verbose run so that the user knows something is happening + if (!appConfig.surpressLoggingOutput) { + log.log("Processing changes and items received from Microsoft OneDrive ..."); + } + } + + foreach (batchOfJSONItems; jsonItemsToProcess.chunks(batchSize)) { + // Chunk the total items to process into 500 lot items + batchesProcessed++; + log.vlog("Processing OneDrive JSON item batch [", batchesProcessed,"/", batchCount, "] to ensure consistent local state"); + processJSONItemsInBatch(batchOfJSONItems, batchesProcessed, batchCount); + // To finish off the JSON processing items, this is needed to reflect this in the log + log.vdebug("------------------------------------------------------------------"); + } + + log.vdebug("Number of JSON items to process is: ", jsonItemsToProcess.length); + log.vdebug("Number of JSON items processed was: ", processedCount); + + // Free up memory and items processed as it is pointless now having this data around + jsonItemsToProcess = []; + + // Keep the driveIDsArray with unique entries only + if (!canFind(driveIDsArray, driveIdToQuery)) { + // Add this driveId to the array of driveId's we know about + driveIDsArray ~= driveIdToQuery; + } + } + + // Process the /delta API JSON response items + void processDeltaJSONItem(JSONValue onedriveJSONItem, ulong nrChanges, int changeCount, ulong responseBundleCount, bool singleDirectoryScope) { + + // Variables for this foreach loop + string thisItemId; + bool itemIsRoot = false; + bool handleItemAsRootObject = false; + bool itemIsDeletedOnline = false; + bool itemHasParentReferenceId = false; + bool itemHasParentReferencePath = false; + bool itemIdMatchesDefaultRootId = false; + bool itemNameExplicitMatchRoot = false; + string objectParentDriveId; + + log.vdebug("------------------------------------------------------------------"); + log.vdebug("Processing OneDrive Item ", changeCount, " of ", nrChanges, " from API Response Bundle ", responseBundleCount); + log.vdebug("Raw JSON OneDrive Item: ", onedriveJSONItem); + // What is this item's id + thisItemId = onedriveJSONItem["id"].str; + // Is this a deleted item - only calculate this once + itemIsDeletedOnline = isItemDeleted(onedriveJSONItem); + + if(!itemIsDeletedOnline){ + // This is not a deleted item + log.vdebug("This item is not a OneDrive deletion change"); + // Only calculate this once + itemIsRoot = isItemRoot(onedriveJSONItem); + itemHasParentReferenceId = hasParentReferenceId(onedriveJSONItem); + itemIdMatchesDefaultRootId = (thisItemId == appConfig.defaultRootId); + itemNameExplicitMatchRoot = (onedriveJSONItem["name"].str == "root"); + objectParentDriveId = onedriveJSONItem["parentReference"]["driveId"].str; + + // Shared Folder Items + // !hasParentReferenceId(id) + // !hasParentReferenceId(path) + + // Test is this is the OneDrive Users Root? + // Debug output of change evaluation items + log.vdebug("defaultRootId = ", appConfig.defaultRootId); + log.vdebug("'search id' = ", thisItemId); + log.vdebug("id == defaultRootId = ", itemIdMatchesDefaultRootId); + log.vdebug("isItemRoot(onedriveJSONItem) = ", itemIsRoot); + log.vdebug("onedriveJSONItem['name'].str == 'root' = ", itemNameExplicitMatchRoot); + log.vdebug("itemHasParentReferenceId = ", itemHasParentReferenceId); + + if ( (itemIdMatchesDefaultRootId || singleDirectoryScope) && itemIsRoot && itemNameExplicitMatchRoot) { + // This IS a OneDrive Root item or should be classified as such in the case of 'singleDirectoryScope' + log.vdebug("JSON item will flagged as a 'root' item"); + handleItemAsRootObject = true; + } + } + + // How do we handle this JSON item from the OneDrive API? + // Is this a confirmed 'root' item, has no Parent ID, or is a Deleted Item + if (handleItemAsRootObject || !itemHasParentReferenceId || itemIsDeletedOnline){ + // Is a root item, has no id in parentReference or is a OneDrive deleted item + log.vdebug("objectParentDriveId = ", objectParentDriveId); + log.vdebug("handleItemAsRootObject = ", handleItemAsRootObject); + log.vdebug("itemHasParentReferenceId = ", itemHasParentReferenceId); + log.vdebug("itemIsDeletedOnline = ", itemIsDeletedOnline); + log.vdebug("Handling change immediately as 'root item', or has no parent reference id or is a deleted item"); + // OK ... do something with this JSON post here .... + processRootAndDeletedJSONItems(onedriveJSONItem, objectParentDriveId, handleItemAsRootObject, itemIsDeletedOnline, itemHasParentReferenceId); + } else { + // Do we need to update this RAW JSON from OneDrive? + if ( (objectParentDriveId != appConfig.defaultDriveId) && (appConfig.accountType == "business") && (appConfig.getValueBool("sync_business_shared_items")) ) { + // Potentially need to update this JSON data + log.vdebug("Potentially need to update this source JSON .... need to check the database"); + + // Check the DB for 'remote' objects, searching 'remoteDriveId' and 'remoteId' items for this remoteItem.driveId and remoteItem.id + Item remoteDBItem; + itemDB.selectByRemoteId(objectParentDriveId, thisItemId, remoteDBItem); + + // Is the data that was returned from the database what we are looking for? + if ((remoteDBItem.remoteDriveId == objectParentDriveId) && (remoteDBItem.remoteId == thisItemId)) { + // Yes, this is the record we are looking for + log.vdebug("DB Item response for remoteDBItem: ", remoteDBItem); + + // Must compare remoteDBItem.name with remoteItem.name + if (remoteDBItem.name != onedriveJSONItem["name"].str) { + // Update JSON Item + string actualOnlineName = onedriveJSONItem["name"].str; + log.vdebug("Updating source JSON 'name' to that which is the actual local directory"); + log.vdebug("onedriveJSONItem['name'] was: ", onedriveJSONItem["name"].str); + log.vdebug("Updating onedriveJSONItem['name'] to: ", remoteDBItem.name); + onedriveJSONItem["name"] = remoteDBItem.name; + log.vdebug("onedriveJSONItem['name'] now: ", onedriveJSONItem["name"].str); + // Add the original name to the JSON + onedriveJSONItem["actualOnlineName"] = actualOnlineName; + } + } + } + + // Add this JSON item for further processing + log.vdebug("Adding this Raw JSON OneDrive Item to jsonItemsToProcess array for further processing"); + jsonItemsToProcess ~= onedriveJSONItem; + } + } + + // Process 'root' and 'deleted' OneDrive JSON items + void processRootAndDeletedJSONItems(JSONValue onedriveJSONItem, string driveId, bool handleItemAsRootObject, bool itemIsDeletedOnline, bool itemHasParentReferenceId) { + + // Use the JSON elements rather can computing a DB struct via makeItem() + string thisItemId = onedriveJSONItem["id"].str; + string thisItemDriveId = onedriveJSONItem["parentReference"]["driveId"].str; + + // Check if the item has been seen before + Item existingDatabaseItem; + bool existingDBEntry = itemDB.selectById(thisItemDriveId, thisItemId, existingDatabaseItem); + + // Is the item deleted online? + if(!itemIsDeletedOnline) { + + // Is the item a confirmed root object? + + // The JSON item should be considered a 'root' item if: + // 1. Contains a ["root"] element + // 2. Has no ["parentReference"]["id"] ... #323 & #324 highlighted that this is false as some 'root' shared objects now can have an 'id' element .. OneDrive API change + // 2. Has no ["parentReference"]["path"] + // 3. Was detected by an input flag as to be handled as a root item regardless of actual status + + if ((handleItemAsRootObject) || (!itemHasParentReferenceId)) { + log.vdebug("Handing JSON object as OneDrive 'root' object"); + if (!existingDBEntry) { + // we have not seen this item before + saveItem(onedriveJSONItem); + } + } + } else { + // Change is to delete an item + log.vdebug("Handing a OneDrive Deleted Item"); + if (existingDBEntry) { + // Flag to delete + log.vdebug("Flagging to delete item locally: ", onedriveJSONItem); + idsToDelete ~= [thisItemDriveId, thisItemId]; + } else { + // Flag to ignore + log.vdebug("Flagging item to skip: ", onedriveJSONItem); + skippedItems ~= thisItemId; + } + } + } + + // Process each of the elements contained in jsonItemsToProcess[] + void processJSONItemsInBatch(JSONValue[] array, ulong batchGroup, ulong batchCount) { + + ulong batchElementCount = array.length; + + foreach (i, onedriveJSONItem; array.enumerate) { + // Use the JSON elements rather can computing a DB struct via makeItem() + ulong elementCount = i +1; + + // To show this is the processing for this particular item, start off with this breaker line + log.vdebug("------------------------------------------------------------------"); + log.vdebug("Processing OneDrive JSON item ", elementCount, " of ", batchElementCount, " as part of JSON Item Batch ", batchGroup, " of ", batchCount); + log.vdebug("Raw JSON OneDrive Item: ", onedriveJSONItem); + + string thisItemId = onedriveJSONItem["id"].str; + string thisItemDriveId = onedriveJSONItem["parentReference"]["driveId"].str; + string thisItemParentId = onedriveJSONItem["parentReference"]["id"].str; + string thisItemName = onedriveJSONItem["name"].str; + + // Create an empty item struct for an existing DB item + Item existingDatabaseItem; + + // Do we NOT want this item? + bool unwanted = false; // meaning by default we will WANT this item + // Is this parent is in the database + bool parentInDatabase = false; + // What is the path of the new item + string newItemPath; + + // Configure the remoteItem - so if it is used, it can be utilised later + Item remoteItem; + + // Check the database for an existing entry for this JSON item + bool existingDBEntry = itemDB.selectById(thisItemDriveId, thisItemId, existingDatabaseItem); + + // Calculate if the Parent Item is in the database so that it can be re-used + parentInDatabase = itemDB.idInLocalDatabase(thisItemDriveId, thisItemParentId); + + // Calculate the path of this JSON item, but we can only do this if the parent is in the database + if (parentInDatabase) { + // Calculate this items path + newItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ thisItemName; + log.vdebug("New Item calculated full path is: ", newItemPath); + } else { + // Parent not in the database + // Is the parent a 'folder' from another user? ie - is this a 'shared folder' that has been shared with us? + log.vdebug("Parent ID is not in DB .. "); + // Why? + if (thisItemDriveId == appConfig.defaultDriveId) { + // Flagging as unwanted + log.vdebug("Flagging as unwanted: thisItemDriveId (", thisItemDriveId,"), thisItemParentId (", thisItemParentId,") not in local database"); + if (skippedItems.find(thisItemParentId).length != 0) { + log.vdebug("Reason: thisItemParentId listed within skippedItems"); + } + unwanted = true; + } else { + // Edge case as the parent (from another users OneDrive account) will never be in the database - potentially a shared object? + log.vdebug("Potential Shared Object Item: ", onedriveJSONItem); + // Format the OneDrive change into a consumable object for the database + remoteItem = makeItem(onedriveJSONItem); + log.vdebug("The reported parentId is not in the database. This potentially is a shared folder as 'remoteItem.driveId' != 'appConfig.defaultDriveId'. Relevant Details: remoteItem.driveId (", remoteItem.driveId,"), remoteItem.parentId (", remoteItem.parentId,")"); + + if (appConfig.accountType == "personal") { + // Personal Account Handling + // Ensure that this item has no parent + log.vdebug("Setting remoteItem.parentId to be null"); + remoteItem.parentId = null; + // Add this record to the local database + log.vdebug("Update/Insert local database with remoteItem details with remoteItem.parentId as null: ", remoteItem); + itemDB.upsert(remoteItem); + } else { + // Business or SharePoint Account Handling + log.vdebug("Handling a Business or SharePoint Shared Item JSON object"); + + if (appConfig.accountType == "business") { + // Create a DB Tie Record for this parent object + Item parentItem; + parentItem.driveId = onedriveJSONItem["parentReference"]["driveId"].str; + parentItem.id = onedriveJSONItem["parentReference"]["id"].str; + parentItem.name = "root"; + parentItem.type = ItemType.dir; + parentItem.mtime = remoteItem.mtime; + parentItem.parentId = null; + + // Add this parent record to the local database + log.vdebug("Insert local database with remoteItem parent details: ", parentItem); + itemDB.upsert(parentItem); + + // Ensure that this item has no parent + log.vdebug("Setting remoteItem.parentId to be null"); + remoteItem.parentId = null; + + // Check the DB for 'remote' objects, searching 'remoteDriveId' and 'remoteId' items for this remoteItem.driveId and remoteItem.id + Item remoteDBItem; + itemDB.selectByRemoteId(remoteItem.driveId, remoteItem.id, remoteDBItem); + + // Must compare remoteDBItem.name with remoteItem.name + if ((!remoteDBItem.name.empty) && (remoteDBItem.name != remoteItem.name)) { + // Update DB Item + log.vdebug("The shared item stored in OneDrive, has a different name to the actual name on the remote drive"); + log.vdebug("Updating remoteItem.name JSON data with the actual name being used on account drive and local folder"); + log.vdebug("remoteItem.name was: ", remoteItem.name); + log.vdebug("Updating remoteItem.name to: ", remoteDBItem.name); + remoteItem.name = remoteDBItem.name; + log.vdebug("Setting remoteItem.remoteName to: ", onedriveJSONItem["name"].str); + + // Update JSON Item + remoteItem.remoteName = onedriveJSONItem["name"].str; + log.vdebug("Updating source JSON 'name' to that which is the actual local directory"); + log.vdebug("onedriveJSONItem['name'] was: ", onedriveJSONItem["name"].str); + log.vdebug("Updating onedriveJSONItem['name'] to: ", remoteDBItem.name); + onedriveJSONItem["name"] = remoteDBItem.name; + log.vdebug("onedriveJSONItem['name'] now: ", onedriveJSONItem["name"].str); + + // Update newItemPath value + newItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ remoteDBItem.name; + log.vdebug("New Item updated calculated full path is: ", newItemPath); + } + + // Add this record to the local database + log.vdebug("Update/Insert local database with remoteItem details: ", remoteItem); + itemDB.upsert(remoteItem); + } + } + } + } + + // Check the skippedItems array for the parent id of this JSONItem if this is something we need to skip + if (!unwanted) { + if (skippedItems.find(thisItemParentId).length != 0) { + // Flag this JSON item as unwanted + log.vdebug("Flagging as unwanted: find(thisItemParentId).length != 0"); + unwanted = true; + + // Is this item id in the database? + if (existingDBEntry) { + // item exists in database, most likely moved out of scope for current client configuration + log.vdebug("This item was previously synced / seen by the client"); + if (("name" in onedriveJSONItem["parentReference"]) != null) { + + // How is this out of scope? + // is sync_list configured + if (syncListConfigured) { + // sync_list configured and in use + if (selectiveSync.isPathExcludedViaSyncList(onedriveJSONItem["parentReference"]["name"].str)) { + // Previously synced item is now out of scope as it has been moved out of what is included in sync_list + log.vdebug("This previously synced item is now excluded from being synced due to sync_list exclusion"); + } + } + // flag to delete local file as it now is no longer in sync with OneDrive + log.vdebug("Flagging to delete item locally: ", onedriveJSONItem); + idsToDelete ~= [thisItemDriveId, thisItemId]; + } + } + } + } + + // Check the item type - if it not an item type that we support, we cant process the JSON item + if (!unwanted) { + if (isItemFile(onedriveJSONItem)) { + log.vdebug("The item we are syncing is a file"); + } else if (isItemFolder(onedriveJSONItem)) { + log.vdebug("The item we are syncing is a folder"); + } else if (isItemRemote(onedriveJSONItem)) { + log.vdebug("The item we are syncing is a remote item"); + } else { + // Why was this unwanted? + if (newItemPath.empty) { + // Compute this item path & need the full path for this file + newItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ thisItemName; + log.vdebug("New Item calculated full path is: ", newItemPath); + } + // Microsoft OneNote container objects present as neither folder or file but has file size + if ((!isItemFile(onedriveJSONItem)) && (!isItemFolder(onedriveJSONItem)) && (hasFileSize(onedriveJSONItem))) { + // Log that this was skipped as this was a Microsoft OneNote item and unsupported + log.vlog("The Microsoft OneNote Notebook '", newItemPath, "' is not supported by this client"); + } else { + // Log that this item was skipped as unsupported + log.vlog("The OneDrive item '", newItemPath, "' is not supported by this client"); + } + unwanted = true; + log.vdebug("Flagging as unwanted: item type is not supported"); + } + } + + // Check if this is excluded by config option: skip_dir + if (!unwanted) { + // Only check path if config is != "" + if (!appConfig.getValueString("skip_dir").empty) { + // Is the item a folder? + if (isItemFolder(onedriveJSONItem)) { + // work out the 'snippet' path where this folder would be created + string simplePathToCheck = ""; + string complexPathToCheck = ""; + string matchDisplay = ""; + + if (hasParentReference(onedriveJSONItem)) { + // we need to workout the FULL path for this item + // simple path + if (("name" in onedriveJSONItem["parentReference"]) != null) { + simplePathToCheck = onedriveJSONItem["parentReference"]["name"].str ~ "/" ~ onedriveJSONItem["name"].str; + } else { + simplePathToCheck = onedriveJSONItem["name"].str; + } + log.vdebug("skip_dir path to check (simple): ", simplePathToCheck); + + // complex path + if (parentInDatabase) { + // build up complexPathToCheck + complexPathToCheck = buildNormalizedPath(newItemPath); + } else { + log.vdebug("Parent details not in database - unable to compute complex path to check"); + } + if (!complexPathToCheck.empty) { + log.vdebug("skip_dir path to check (complex): ", complexPathToCheck); + } + } else { + simplePathToCheck = onedriveJSONItem["name"].str; + } + + // If 'simplePathToCheck' or 'complexPathToCheck' is of the following format: root:/folder + // then isDirNameExcluded matching will not work + // Clean up 'root:' if present + if (startsWith(simplePathToCheck, "root:")){ + log.vdebug("Updating simplePathToCheck to remove 'root:'"); + simplePathToCheck = strip(simplePathToCheck, "root:"); + } + if (startsWith(complexPathToCheck, "root:")){ + log.vdebug("Updating complexPathToCheck to remove 'root:'"); + complexPathToCheck = strip(complexPathToCheck, "root:"); + } + + // OK .. what checks are we doing? + if ((!simplePathToCheck.empty) && (complexPathToCheck.empty)) { + // just a simple check + log.vdebug("Performing a simple check only"); + unwanted = selectiveSync.isDirNameExcluded(simplePathToCheck); + } else { + // simple and complex + log.vdebug("Performing a simple then complex path match if required"); + // simple first + log.vdebug("Performing a simple check first"); + unwanted = selectiveSync.isDirNameExcluded(simplePathToCheck); + matchDisplay = simplePathToCheck; + if (!unwanted) { + log.vdebug("Simple match was false, attempting complex match"); + // simple didnt match, perform a complex check + unwanted = selectiveSync.isDirNameExcluded(complexPathToCheck); + matchDisplay = complexPathToCheck; + } + } + // result + log.vdebug("skip_dir exclude result (directory based): ", unwanted); + if (unwanted) { + // This path should be skipped + log.vlog("Skipping item - excluded by skip_dir config: ", matchDisplay); + } + } + // Is the item a file? + // We need to check to see if this files path is excluded as well + if (isItemFile(onedriveJSONItem)) { + + string pathToCheck; + // does the newItemPath start with '/'? + if (!startsWith(newItemPath, "/")){ + // path does not start with '/', but we need to check skip_dir entries with and without '/' + // so always make sure we are checking a path with '/' + pathToCheck = '/' ~ dirName(newItemPath); + } else { + pathToCheck = dirName(newItemPath); + } + + // perform the check + unwanted = selectiveSync.isDirNameExcluded(pathToCheck); + // result + log.vdebug("skip_dir exclude result (file based): ", unwanted); + if (unwanted) { + // this files path should be skipped + log.vlog("Skipping item - file path is excluded by skip_dir config: ", newItemPath); + } + } + } + } + + // Check if this is excluded by config option: skip_file + if (!unwanted) { + // Is the JSON item a file? + if (isItemFile(onedriveJSONItem)) { + // skip_file can contain 4 types of entries: + // - wildcard - *.txt + // - text + wildcard - name*.txt + // - full path + combination of any above two - /path/name*.txt + // - full path to file - /path/to/file.txt + + // is the parent id in the database? + if (parentInDatabase) { + // Compute this item path & need the full path for this file + if (newItemPath.empty) { + newItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ thisItemName; + log.vdebug("New Item calculated full path is: ", newItemPath); + } + + // The path that needs to be checked needs to include the '/' + // This due to if the user has specified in skip_file an exclusive path: '/path/file' - that is what must be matched + // However, as 'path' used throughout, use a temp variable with this modification so that we use the temp variable for exclusion checks + string exclusionTestPath = ""; + if (!startsWith(newItemPath, "/")){ + // Add '/' to the path + exclusionTestPath = '/' ~ newItemPath; + } + + log.vdebug("skip_file item to check: ", exclusionTestPath); + unwanted = selectiveSync.isFileNameExcluded(exclusionTestPath); + log.vdebug("Result: ", unwanted); + if (unwanted) log.vlog("Skipping item - excluded by skip_file config: ", thisItemName); + } else { + // parent id is not in the database + unwanted = true; + log.vlog("Skipping file - parent path not present in local database"); + } + } + } + + // Check if this is included or excluded by use of sync_list + if (!unwanted) { + // No need to try and process something against a sync_list if it has been configured + if (syncListConfigured) { + // Compute the item path if empty - as to check sync_list we need an actual path to check + if (newItemPath.empty) { + // Calculate this items path + newItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ thisItemName; + log.vdebug("New Item calculated full path is: ", newItemPath); + } + + // What path are we checking? + log.vdebug("sync_list item to check: ", newItemPath); + + // Unfortunatly there is no avoiding this call to check if the path is excluded|included via sync_list + if (selectiveSync.isPathExcludedViaSyncList(newItemPath)) { + // selective sync advised to skip, however is this a file and are we configured to upload / download files in the root? + if ((isItemFile(onedriveJSONItem)) && (appConfig.getValueBool("sync_root_files")) && (rootName(newItemPath) == "") ) { + // This is a file + // We are configured to sync all files in the root + // This is a file in the logical root + unwanted = false; + } else { + // path is unwanted + unwanted = true; + log.vlog("Skipping item - excluded by sync_list config: ", newItemPath); + // flagging to skip this item now, but does this exist in the DB thus needs to be removed / deleted? + if (existingDBEntry) { + // flag to delete + log.vlog("Flagging item for local delete as item exists in database: ", newItemPath); + idsToDelete ~= [thisItemDriveId, thisItemId]; + } + } + } + } + } + + // Check if the user has configured to skip downloading .files or .folders: skip_dotfiles + if (!unwanted) { + if (appConfig.getValueBool("skip_dotfiles")) { + if (isDotFile(newItemPath)) { + log.vlog("Skipping item - .file or .folder: ", newItemPath); + unwanted = true; + } + } + } + + // Check if this should be skipped due to a --check-for-nosync directive (.nosync)? + if (!unwanted) { + if (appConfig.getValueBool("check_nosync")) { + // need the parent path for this object + string parentPath = dirName(newItemPath); + // Check for the presence of a .nosync in the parent path + if (exists(parentPath ~ "/.nosync")) { + log.vlog("Skipping downloading item - .nosync found in parent folder & --check-for-nosync is enabled: ", newItemPath); + unwanted = true; + } + } + } + + // Check if this is excluded by a user set maximum filesize to download + if (!unwanted) { + if (isItemFile(onedriveJSONItem)) { + if (fileSizeLimit != 0) { + if (onedriveJSONItem["size"].integer >= fileSizeLimit) { + log.vlog("Skipping item - excluded by skip_size config: ", thisItemName, " (", onedriveJSONItem["size"].integer/2^^20, " MB)"); + } + } + } + } + + // At this point all the applicable checks on this JSON object from OneDrive are complete: + // - skip_file + // - skip_dir + // - sync_list + // - skip_dotfiles + // - check_nosync + // - skip_size + // - We know if this item exists in the DB or not in the DB + + // We know if this JSON item is unwanted or not + if (unwanted) { + // This JSON item is NOT wanted - it is excluded + log.vdebug("Skipping OneDrive change as this is determined to be unwanted"); + // Add to the skippedItems array, but only if it is a directory ... pointless adding 'files' here, as it is the 'id' we check as the parent path which can only be a directory + if (!isItemFile(onedriveJSONItem)) { + skippedItems ~= thisItemId; + } + } else { + // This JSON item is wanted - we need to process this JSON item further + // Take the JSON item and create a consumable object for eventual database insertion + Item newDatabaseItem = makeItem(onedriveJSONItem); + + if (existingDBEntry) { + // The details of this JSON item are already in the DB + // Is the item in the DB the same as the JSON data provided - or is the JSON data advising this is an updated file? + log.vdebug("OneDrive change is an update to an existing local item"); + // Compute the existing item path + // NOTE: + // string existingItemPath = computeItemPath(existingDatabaseItem.driveId, existingDatabaseItem.id); + // + // This will calculate the path as follows: + // + // existingItemPath: Document.txt + // + // Whereas above we use the following + // + // newItemPath = computeItemPath(newDatabaseItem.driveId, newDatabaseItem.parentId) ~ "/" ~ newDatabaseItem.name; + // + // Which generates the following path: + // + // changedItemPath: ./Document.txt + // + // Need to be consistent here with how 'newItemPath' was calculated + string existingItemPath = computeItemPath(existingDatabaseItem.driveId, existingDatabaseItem.parentId) ~ "/" ~ existingDatabaseItem.name; + // Attempt to apply this changed item + applyPotentiallyChangedItem(existingDatabaseItem, existingItemPath, newDatabaseItem, newItemPath, onedriveJSONItem); + } else { + // Action this JSON item as a new item as we have no DB record of it + // The actual item may actually exist locally already, meaning that just the database is out-of-date or missing the data due to --resync + // But we also cannot compute the newItemPath as the parental objects may not exist as well + log.vdebug("OneDrive change is potentially a new local item"); + + // Attempt to apply this potentially new item + applyPotentiallyNewLocalItem(newDatabaseItem, onedriveJSONItem, newItemPath); + } + } + + // Tracking as to if this item was processed + processedCount++; + } + } + + // Perform the download of any required objects in parallel + void processDownloadActivities() { + + // Are there any items to delete locally? Cleanup space locally first + if (!idsToDelete.empty) { + // There are elements that potentially need to be deleted locally + log.vlog("Items to potentially delete locally: ", idsToDelete.length); + if (appConfig.getValueBool("download_only")) { + // Download only has been configured + if (cleanupLocalFiles) { + // Process online deleted items + log.vlog("Processing local deletion activity as --download-only & --cleanup-local-files configured"); + processDeleteItems(); + } else { + // Not cleaning up local files + log.vlog("Skipping local deletion activity as --download-only has been used"); + } + } else { + // Not using --download-only process normally + processDeleteItems(); + } + // Cleanup array memory + idsToDelete = []; + } + + // Are there any items to download post fetching and processing the /delta data? + if (!fileJSONItemsToDownload.empty) { + // There are elements to download + log.vlog("Number of items to download from OneDrive: ", fileJSONItemsToDownload.length); + downloadOneDriveItems(); + // Cleanup array memory + fileJSONItemsToDownload = []; + } + + // Are there any skipped items still? + if (!skippedItems.empty) { + // Cleanup array memory + skippedItems = []; + } + } + + // If the JSON item is not in the database, it is potentially a new item that we need to action + void applyPotentiallyNewLocalItem(Item newDatabaseItem, JSONValue onedriveJSONItem, string newItemPath) { + + // The JSON and Database items being passed in here have passed the following checks: + // - skip_file + // - skip_dir + // - sync_list + // - skip_dotfiles + // - check_nosync + // - skip_size + // - Is not currently cached in the local database + // As such, we should not be doing any other checks here to determine if the JSON item is wanted .. it is + + if (exists(newItemPath)) { + // Issue #2209 fix - test if path is a bad symbolic link + if (isSymlink(newItemPath)) { + log.vdebug("Path on local disk is a symbolic link ........"); + if (!exists(readLink(newItemPath))) { + // reading the symbolic link failed + log.vdebug("Reading the symbolic link target failed ........ "); + log.logAndNotify("Skipping item - invalid symbolic link: ", newItemPath); + return; + } + } + + // Path exists locally, is not a bad symbolic link + // Test if this item is actually in-sync + // What is the source of this item data? + string itemSource = "remote"; + if (isItemSynced(newDatabaseItem, newItemPath, itemSource)) { + // Item details from OneDrive and local item details in database are in-sync + log.vdebug("The item to sync is already present on the local filesystem and is in-sync with what is reported online"); + log.vdebug("Update/Insert local database with item details"); + log.vdebug("item details to update/insert: ", newDatabaseItem); + itemDB.upsert(newDatabaseItem); + return; + } else { + // Item details from OneDrive and local item details in database are NOT in-sync + log.vdebug("The item to sync exists locally but is NOT in the local database - otherwise this would be handled as changed item"); + + // Which object is newer? The local file or the remote file? + SysTime localModifiedTime = timeLastModified(newItemPath).toUTC(); + SysTime itemModifiedTime = newDatabaseItem.mtime; + // Reduce time resolution to seconds before comparing + localModifiedTime.fracSecs = Duration.zero; + itemModifiedTime.fracSecs = Duration.zero; + + // If we need to rename the file, what do we rename it to? + auto ext = extension(newItemPath); + auto renamedNewItemPath = newItemPath.chomp(ext) ~ "-" ~ deviceName ~ ext; + + // Is the local modified time greater than that from OneDrive? + if (localModifiedTime > itemModifiedTime) { + // Local file is newer than item on OneDrive based on file modified time + // Is this item id in the database? + if (itemDB.idInLocalDatabase(newDatabaseItem.driveId, newDatabaseItem.id)) { + // item id is in the database + // no local rename + // no download needed + log.vlog("Local item modified time is newer based on UTC time conversion - keeping local item as this exists in the local database"); + log.vdebug("Skipping OneDrive change as this is determined to be unwanted due to local item modified time being newer than OneDrive item and present in the sqlite database"); + } else { + // item id is not in the database .. maybe a --resync ? + // file exists locally but is not in the sqlite database - maybe a failed download? + log.vlog("Local item does not exist in local database - replacing with file from OneDrive - failed download?"); + + // In a --resync scenario or if items.sqlite3 was deleted before startup we have zero way of knowing IF the local file is meant to be the right file + // To this pint we have passed the following checks: + // 1. Any client side filtering checks - this determined this is a file that is wanted + // 2. A file with the exact name exists locally + // 3. The local modified time > remote modified time + // 4. The id of the item from OneDrive is not in the database + + // Has the user configured to IGNORE local data protection rules? + if (bypassDataPreservation) { + // The user has configured to ignore data safety checks and overwrite local data rather than preserve & rename + log.vlog("WARNING: Local Data Protection has been disabled. You may experience data loss on this file: ", newItemPath); + } else { + // local data protection is configured, renaming local file + log.log("The local item is out-of-sync with OneDrive, renaming to preserve existing file and prevent local data loss: ", newItemPath, " -> ", renamedNewItemPath); + // perform the rename action of the local file + if (!dryRun) { + // Perform the local rename of the existing local file + safeRename(newItemPath, renamedNewItemPath, dryRun); + } else { + // Expectation here is that there is a new file locally (renamedNewItemPath) however as we don't create this, the "new file" will not be uploaded as it does not exist + log.vdebug("DRY-RUN: Skipping local file rename"); + } + } + } + } else { + // Remote file is newer than the existing local item + log.vlog("Remote item modified time is newer based on UTC time conversion"); // correct message, remote item is newer + log.vdebug("localModifiedTime (local file): ", localModifiedTime); + log.vdebug("itemModifiedTime (OneDrive item): ", itemModifiedTime); + + // Has the user configured to IGNORE local data protection rules? + if (bypassDataPreservation) { + // The user has configured to ignore data safety checks and overwrite local data rather than preserve & rename + log.vlog("WARNING: Local Data Protection has been disabled. You may experience data loss on this file: ", newItemPath); + } else { + // local data protection is configured, renaming local file + log.vlog("The local item is out-of-sync with OneDrive, renaming to preserve existing file and prevent data loss: ", newItemPath, " -> ", renamedNewItemPath); + // perform the rename action of the local file + if (!dryRun) { + // Perform the local rename of the existing local file + safeRename(newItemPath, renamedNewItemPath, dryRun); + } else { + // Expectation here is that there is a new file locally (renamedNewItemPath) however as we don't create this, the "new file" will not be uploaded as it does not exist + log.vdebug("DRY-RUN: Skipping local file rename"); + } + } + } + } + } + + // Path does not exist locally (should not exist locally if renamed file) - this will be a new file download or new folder creation + // How to handle this Potentially New Local Item JSON ? + final switch (newDatabaseItem.type) { + case ItemType.file: + // Add to the items to download array for processing + fileJSONItemsToDownload ~= onedriveJSONItem; + break; + case ItemType.dir: + case ItemType.remote: + log.log("Creating local directory: ", newItemPath); + if (!dryRun) { + try { + // Create the new directory + log.vdebug("Requested path does not exist, creating directory structure: ", newItemPath); + mkdirRecurse(newItemPath); + // Configure the applicable permissions for the folder + log.vdebug("Setting directory permissions for: ", newItemPath); + newItemPath.setAttributes(appConfig.returnRequiredDirectoryPermisions()); + // Update the time of the folder to match the last modified time as is provided by OneDrive + // If there are any files then downloaded into this folder, the last modified time will get + // updated by the local Operating System with the latest timestamp - as this is normal operation + // as the directory has been modified + log.vdebug("Setting directory lastModifiedDateTime for: ", newItemPath , " to ", newDatabaseItem.mtime); + log.vdebug("Calling setTimes() for this file: ", newItemPath); + setTimes(newItemPath, newDatabaseItem.mtime, newDatabaseItem.mtime); + // Save the item to the database + writeln("Line: 1612"); + saveItem(onedriveJSONItem); + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } else { + // we dont create the directory, but we need to track that we 'faked it' + idsFaked ~= [newDatabaseItem.driveId, newDatabaseItem.id]; + // Save the item to the dry-run database + writeln("Line: 1622"); + saveItem(onedriveJSONItem); + } + break; + case ItemType.unknown: + // Unknown type - we dont action or sync these items + break; + } + } + + // If the JSON item IS in the database, this will be an update to an existing in-sync item + void applyPotentiallyChangedItem(Item existingDatabaseItem, string existingItemPath, Item changedOneDriveItem, string changedItemPath, JSONValue onedriveJSONItem) { + + // If we are moving the item, we do not need to download it again + bool itemWasMoved = false; + + // Do we need to actually update the database with the details that were provided by the OneDrive API? + // Calculate these time items from the provided items + SysTime existingItemModifiedTime = existingDatabaseItem.mtime; + existingItemModifiedTime.fracSecs = Duration.zero; + SysTime changedOneDriveItemModifiedTime = changedOneDriveItem.mtime; + changedOneDriveItemModifiedTime.fracSecs = Duration.zero; + + if (existingDatabaseItem.eTag != changedOneDriveItem.eTag) { + // The eTag has changed to what we previously cached + if (existingItemPath != changedItemPath) { + // Log that we are changing / moving an item to a new name + log.log("Moving ", existingItemPath, " to ", changedItemPath); + // Is the destination path empty .. or does something exist at that location? + if (exists(changedItemPath)) { + // Destination we are moving to exists ... + Item changedLocalItem; + // Query DB for this changed item in specified path that exists and see if it is in-sync + if (itemDB.selectByPath(changedItemPath, changedOneDriveItem.driveId, changedLocalItem)) { + // The 'changedItemPath' is in the database + string itemSource = "database"; + if (isItemSynced(changedLocalItem, changedItemPath, itemSource)) { + // The destination item is in-sync + log.vlog("Destination is in sync and will be overwritten"); + } else { + // The destination item is different + log.vlog("The destination is occupied with a different item, renaming the conflicting file..."); + // Backup this item, passing in if we are performing a --dry-run or not + safeBackup(changedItemPath, dryRun); + } + } else { + // The to be overwritten item is not already in the itemdb, so it should saved to avoid data loss + log.vlog("The destination is occupied by an existing un-synced file, renaming the conflicting file..."); + // Backup this item, passing in if we are performing a --dry-run or not + safeBackup(changedItemPath, dryRun); + } + } + + // Try and rename path, catch any exception generated + try { + // Rename this item, passing in if we are performing a --dry-run or not + safeRename(existingItemPath, changedItemPath, dryRun); + + // If the item is a file, make sure that the local timestamp now is the same as the timestamp online + // Otherwise when we do the DB check, the move on the file system, the file technically has a newer timestamp + // which is 'correct' .. but we need to report locally the online timestamp here as the move was made online + if (changedOneDriveItem.type == ItemType.file) { + setTimes(changedItemPath, changedOneDriveItem.mtime, changedOneDriveItem.mtime); + } + + // Flag that the item was moved | renamed + itemWasMoved = true; + + // If we are in a --dry-run situation, the actual rename did not occur - but we need to track like it did + if (dryRun) { + // Track this as a faked id item + idsFaked ~= [changedOneDriveItem.driveId, changedOneDriveItem.id]; + // We also need to track that we did not rename this path + pathsRenamed ~= [existingItemPath]; + } + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } + + // What sort of changed item is this? + // Is it a file, and we did not move it .. + if ((changedOneDriveItem.type == ItemType.file) && (!itemWasMoved)) { + // The eTag is notorious for being 'changed' online by some backend Microsoft process + if (existingDatabaseItem.quickXorHash != changedOneDriveItem.quickXorHash) { + // Add to the items to download array for processing - the file hash we previously recorded is not the same as online + fileJSONItemsToDownload ~= onedriveJSONItem; + } else { + // If the timestamp is different, or we are running a client operational mode that does not support /delta queries - we have to update the DB with the details from OneDrive + // Unfortunatly because of the consequence of Nataional Cloud Deployments not supporting /delta queries, the application uses the local database to flag what is out-of-date / track changes + // This means that the constant disk writing to the database fix implemented with https://github.com/abraunegg/onedrive/pull/2004 cannot be utilised when using these operational modes + // as all records are touched / updated when performing the OneDrive sync operations. The impacted operational modes are: + // - National Cloud Deployments do not support /delta as a query + // - When using --single-directory + // - When using --download-only --cleanup-local-files + + // Is the last modified timestamp in the DB the same as the API data or are we running an operational mode where we simulated the /delta response? + if ((existingItemModifiedTime != changedOneDriveItemModifiedTime) || (generateSimulatedDeltaResponse)) { + // Save this item in the database + writeln("Line: 1722"); + // Add to the local database + log.vdebug("Adding changed OneDrive Item to database: ", changedOneDriveItem); + itemDB.upsert(changedOneDriveItem); + } + } + } else { + // Save this item in the database + writeln("Line: 1730"); + saveItem(onedriveJSONItem); + + // If the 'Add shortcut to My files' link was the item that was actually renamed .. we have to update our DB records + if (changedOneDriveItem.type == ItemType.remote) { + // Select remote item data from the database + Item existingRemoteDbItem; + itemDB.selectById(changedOneDriveItem.remoteDriveId, changedOneDriveItem.remoteId, existingRemoteDbItem); + // Update the 'name' in existingRemoteDbItem and save it back to the database + // This is the local name stored on disk that was just 'moved' + existingRemoteDbItem.name = changedOneDriveItem.name; + itemDB.upsert(existingRemoteDbItem); + } + } + } else { + // The existingDatabaseItem.eTag == changedOneDriveItem.eTag .. nothing has changed eTag wise + + // If the timestamp is different, or we are running a client operational mode that does not support /delta queries - we have to update the DB with the details from OneDrive + // Unfortunatly because of the consequence of Nataional Cloud Deployments not supporting /delta queries, the application uses the local database to flag what is out-of-date / track changes + // This means that the constant disk writing to the database fix implemented with https://github.com/abraunegg/onedrive/pull/2004 cannot be utilised when using these operational modes + // as all records are touched / updated when performing the OneDrive sync operations. The impacted operational modes are: + // - National Cloud Deployments do not support /delta as a query + // - When using --single-directory + // - When using --download-only --cleanup-local-files + + // Is the last modified timestamp in the DB the same as the API data or are we running an operational mode where we simulated the /delta response? + if ((existingItemModifiedTime != changedOneDriveItemModifiedTime) || (generateSimulatedDeltaResponse)) { + // Database update needed for this item because our local record is out-of-date + writeln("Line: 1758"); + // Add to the local database + log.vdebug("Adding changed OneDrive Item to database: ", changedOneDriveItem); + itemDB.upsert(changedOneDriveItem); + } + } + } + + // Download new file items as identified + void downloadOneDriveItems() { + // Lets deal with all the JSON items that need to be downloaded in a batch process + ulong batchSize = appConfig.concurrentThreads; + ulong batchCount = (fileJSONItemsToDownload.length + batchSize - 1) / batchSize; + ulong batchesProcessed = 0; + + foreach (chunk; fileJSONItemsToDownload.chunks(batchSize)) { + // send an array containing 'appConfig.concurrentThreads' (16) JSON items to download + downloadOneDriveItemsInParallel(chunk); + } + } + + // Download items in parallel + void downloadOneDriveItemsInParallel(JSONValue[] array) { + // This function recieved an array of 16 JSON items to download + foreach (i, onedriveJSONItem; taskPool.parallel(array)) { + // Take each JSON item and + downloadFileItem(onedriveJSONItem); + } + } + + // Perform the actual download of an object from OneDrive + void downloadFileItem(JSONValue onedriveJSONItem) { + + bool downloadFailed = false; + string OneDriveFileXORHash; + string OneDriveFileSHA256Hash; + ulong jsonFileSize = 0; + + // Download item specifics + string downloadDriveId = onedriveJSONItem["parentReference"]["driveId"].str; + string downloadParentId = onedriveJSONItem["parentReference"]["id"].str; + string downloadItemName = onedriveJSONItem["name"].str; + string downloadItemId = onedriveJSONItem["id"].str; + + // Calculate this items path + string newItemPath = computeItemPath(downloadDriveId, downloadParentId) ~ "/" ~ downloadItemName; + log.vdebug("New Item calculated full path is: ", newItemPath); + + // Is the item reported as Malware ? + if (isMalware(onedriveJSONItem)){ + // OneDrive reports that this file is malware + log.error("ERROR: MALWARE DETECTED IN FILE - DOWNLOAD SKIPPED: ", newItemPath); + downloadFailed = true; + } else { + // Grab this file's filesize + if (hasFileSize(onedriveJSONItem)) { + // Use the configured filesize as reported by OneDrive + jsonFileSize = onedriveJSONItem["size"].integer; + } else { + // filesize missing + log.vdebug("WARNING: onedriveJSONItem['size'] is missing"); + } + + // Configure the hashes for comparison post download + if (hasHashes(onedriveJSONItem)) { + // File details returned hash details + // QuickXorHash + if (hasQuickXorHash(onedriveJSONItem)) { + // Use the provided quickXorHash as reported by OneDrive + if (onedriveJSONItem["file"]["hashes"]["quickXorHash"].str != "") { + OneDriveFileXORHash = onedriveJSONItem["file"]["hashes"]["quickXorHash"].str; + } + } else { + // Fallback: Check for SHA256Hash + if (hasSHA256Hash(onedriveJSONItem)) { + // Use the provided sha256Hash as reported by OneDrive + if (onedriveJSONItem["file"]["hashes"]["sha256Hash"].str != "") { + OneDriveFileSHA256Hash = onedriveJSONItem["file"]["hashes"]["sha256Hash"].str; + } + } + } + } else { + // file hash data missing + log.vdebug("WARNING: onedriveJSONItem['file']['hashes'] is missing - unable to compare file hash after download"); + } + + // Is there enough free space locally to download the file + // - We can use '.' here as we change the current working directory to the configured 'sync_dir' + ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace(".")); + // So that we are not responsible in making the disk 100% full if we can download the file, compare the current available space against the reservation set and file size + // The reservation value is user configurable in the config file, 50MB by default + ulong freeSpaceReservation = appConfig.getValueLong("space_reservation"); + // debug output + log.vdebug("Local Disk Space Actual: ", localActualFreeSpace); + log.vdebug("Free Space Reservation: ", freeSpaceReservation); + log.vdebug("File Size to Download: ", jsonFileSize); + + // Calculate if we can actually download file - is there enough free space? + if ((localActualFreeSpace < freeSpaceReservation) || (jsonFileSize > localActualFreeSpace)) { + // localActualFreeSpace is less than freeSpaceReservation .. insufficient free space + // jsonFileSize is greater than localActualFreeSpace .. insufficient free space + log.log("Downloading file ", newItemPath, " ... failed!"); + log.log("Insufficient local disk space to download file"); + downloadFailed = true; + } else { + // If we are in a --dry-run situation - if not, actually perform the download + if (!dryRun) { + // Attempt to download the file as there is enough free space locally + OneDriveApi downloadFileOneDriveApiInstance; + downloadFileOneDriveApiInstance = new OneDriveApi(appConfig); + try { + downloadFileOneDriveApiInstance.initialise(); + downloadFileOneDriveApiInstance.downloadById(downloadDriveId, downloadItemId, newItemPath, jsonFileSize); + downloadFileOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(downloadFileOneDriveApiInstance); + } catch (OneDriveException exception) { + log.vdebug("downloadFileOneDriveApiInstance.downloadById(downloadDriveId, downloadItemId, newItemPath, jsonFileSize); generated a OneDriveException"); + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(downloadFileOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to download an item from OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + downloadFileItem(onedriveJSONItem); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + } + + } catch (FileException e) { + // There was a file system error + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + downloadFailed = true; + } catch (ErrnoException e) { + // There was a file system error + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + downloadFailed = true; + } + + // If we get to this point, something was downloaded .. does it match what we expected? + if (exists(newItemPath)) { + // When downloading some files from SharePoint, the OneDrive API reports one file size, + // but the SharePoint HTTP Server sends a totally different byte count for the same file + // we have implemented --disable-download-validation to disable these checks + + if (!disableDownloadValidation) { + // A 'file' was downloaded - does what we downloaded = reported jsonFileSize or if there is some sort of funky local disk compression going on + // Does the file hash OneDrive reports match what we have locally? + string onlineFileHash; + string downloadedFileHash; + ulong downloadFileSize = getSize(newItemPath); + + if (!OneDriveFileXORHash.empty) { + onlineFileHash = OneDriveFileXORHash; + // Calculate the QuickXOHash for this file + downloadedFileHash = computeQuickXorHash(newItemPath); + } else { + onlineFileHash = OneDriveFileSHA256Hash; + // Fallback: Calculate the SHA256 Hash for this file + downloadedFileHash = computeSHA256Hash(newItemPath); + } + + if ((downloadFileSize == jsonFileSize) && (downloadedFileHash == onlineFileHash)) { + // Downloaded file matches size and hash + log.vdebug("Downloaded file matches reported size and reported file hash"); + try { + // get the mtime from the JSON data + SysTime itemModifiedTime; + if (isItemRemote(onedriveJSONItem)) { + // remote file item + itemModifiedTime = SysTime.fromISOExtString(onedriveJSONItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str); + } else { + // not a remote item + itemModifiedTime = SysTime.fromISOExtString(onedriveJSONItem["fileSystemInfo"]["lastModifiedDateTime"].str); + } + + // set the correct time on the downloaded file + log.vdebug("Calling setTimes() for this file: ", newItemPath); + setTimes(newItemPath, itemModifiedTime, itemModifiedTime); + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } else { + // Downloaded file does not match size or hash .. which is it? + bool downloadValueMismatch = false; + // Size error? + if (downloadFileSize != jsonFileSize) { + // downloaded file size does not match + downloadValueMismatch = true; + log.vdebug("Actual file size on disk: ", downloadFileSize); + log.vdebug("OneDrive API reported size: ", jsonFileSize); + log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why."); + } + // Hash Error + if (downloadedFileHash != onlineFileHash) { + // downloaded file hash does not match + downloadValueMismatch = true; + log.vdebug("Actual local file hash: ", downloadedFileHash); + log.vdebug("OneDrive API reported hash: ", onlineFileHash); + log.error("ERROR: File download hash mis-match. Increase logging verbosity to determine why."); + } + // .heic data loss check + // - https://github.com/abraunegg/onedrive/issues/2471 + // - https://github.com/OneDrive/onedrive-api-docs/issues/1532 + // - https://github.com/OneDrive/onedrive-api-docs/issues/1723 + if (downloadValueMismatch && (toLower(extension(newItemPath)) == ".heic")) { + // Need to display a message to the user that they have experienced data loss + log.error("DATA-LOSS: File downloaded has experienced data loss due to a Microsoft OneDrive API bug. DO NOT DELETE THIS FILE ONLINE."); + log.vlog(" Please read https://github.com/OneDrive/onedrive-api-docs/issues/1723 for more details."); + } + + // Add some workaround messaging for SharePoint + if (appConfig.accountType == "documentLibrary"){ + // It has been seen where SharePoint / OneDrive API reports one size via the JSON + // but the content length and file size written to disk is totally different - example: + // From JSON: "size": 17133 + // From HTTPS Server: < Content-Length: 19340 + // with no logical reason for the difference, except for a 302 redirect before file download + log.error("INFO: It is most likely that a SharePoint OneDrive API issue is the root cause. Add --disable-download-validation to work around this issue but downloaded data integrity cannot be guaranteed."); + } else { + // other account types + log.error("INFO: Potentially add --disable-download-validation to work around this issue but downloaded data integrity cannot be guaranteed."); + } + // We do not want this local file to remain on the local file system as it failed the integrity checks + log.log("Removing file ", newItemPath, " due to failed integrity checks"); + if (!dryRun) { + safeRemove(newItemPath); + } + downloadFailed = true; + } + } else { + // Download validation checks were disabled + log.vdebug("Downloaded file validation disabled due to --disable-download-validation"); + log.vlog("WARNING: Skipping download integrity check for: ", newItemPath); + } // end of (!disableDownloadValidation) + } else { + log.error("ERROR: File failed to download. Increase logging verbosity to determine why."); + downloadFailed = true; + } + } + } + + // File should have been downloaded + if (!downloadFailed) { + // Download did not fail + log.log("Downloading file ", newItemPath, " ... done"); + // Save this item into the database + writeln("Line: 1992"); + saveItem(onedriveJSONItem); + + /** + log.vdebug("Inserting new item details to local database"); + // What was the item that was saved + log.vdebug("item details: ", newDatabaseItem); + itemDB.upsert(newDatabaseItem); + **/ + + + // If we are in a --dry-run situation - if we are, we need to track that we faked the download + if (dryRun) { + // track that we 'faked it' + idsFaked ~= [downloadDriveId, downloadItemId]; + } + } else { + // Output download failed + log.log("Downloading file ", newItemPath, " ... failed!"); + // Add the path to a list of items that failed to download + fileDownloadFailures ~= newItemPath; + } + } + } + + // Test if the given item is in-sync. Returns true if the given item corresponds to the local one + bool isItemSynced(Item item, string path, string itemSource) { + + if (!exists(path)) return false; + final switch (item.type) { + case ItemType.file: + if (isFile(path)) { + // can we actually read the local file? + if (readLocalFile(path)){ + // local file is readable + SysTime localModifiedTime = timeLastModified(path).toUTC(); + SysTime itemModifiedTime = item.mtime; + // Reduce time resolution to seconds before comparing + localModifiedTime.fracSecs = Duration.zero; + itemModifiedTime.fracSecs = Duration.zero; + if (localModifiedTime == itemModifiedTime) { + return true; + } else { + log.vlog("Local item time discrepancy detected: ", path); + log.vlog("This local item has a different modified time ", localModifiedTime, " when compared to ", itemSource, " modified time ", itemModifiedTime); + // The file has been modified ... is the hash the same? + // Test the file hash as the date / time stamp is different + // Generating a hash is computationally expensive - we only generate the hash if timestamp was different + if (testFileHash(path, item)) { + // The hash is the same .. so we need to fix-up the timestamp depending on where it is wrong + log.vlog("Local item has the same hash value as the item online - correcting timestamp"); + // Test if the local timestamp is newer + if (localModifiedTime > itemModifiedTime) { + // The source of the out-of-date timestamp was OneDrive and this needs to be corrected to avoid always generating a hash test if timestamp is different + log.vlog("The source of the incorrect timestamp was OneDrive online - correcting timestamp online"); + if (!dryRun) { + // Attempt to update the online date time stamp + uploadLastModifiedTime(item.driveId, item.id, localModifiedTime.toUTC(), item.eTag); + } + } else { + // The source of the out-of-date timestamp was the local file and this needs to be corrected to avoid always generating a hash test if timestamp is different + log.vlog("The source of the incorrect timestamp was the local file - correcting timestamp locally"); + if (!dryRun) { + log.vdebug("Calling setTimes() for this file: ", path); + setTimes(path, item.mtime, item.mtime); + } + } + return true; + } else { + // The hash is different so the content of the file has to be different as to what is stored online + log.vlog("The local item has a different hash when compared to ", itemSource, " item hash"); + return false; + } + } + } else { + // Unable to read local file + log.log("Unable to determine the sync state of this file as it cannot be read (file permissions or file corruption): ", path); + return false; + } + } else { + log.vlog("The local item is a directory but should be a file"); + } + break; + case ItemType.dir: + case ItemType.remote: + if (isDir(path)) { + return true; + } else { + log.vlog("The local item is a file but should be a directory"); + } + break; + case ItemType.unknown: + // Unknown type - return true but we dont action or sync these items + return true; + } + return false; + } + + // Get the /delta data using the provided details + JSONValue getDeltaChangesByItemId(string selectedDriveId, string selectedItemId, string providedDeltaLink) { + + // Function variables + JSONValue deltaChangesBundle; + // Get the /delta data for this account | driveId | deltaLink combination + // Create a new API Instance for this thread and initialise it + OneDriveApi getDeltaQueryOneDriveApiInstance; + getDeltaQueryOneDriveApiInstance = new OneDriveApi(appConfig); + getDeltaQueryOneDriveApiInstance.initialise(); + try { + deltaChangesBundle = getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink); + } catch (OneDriveException exception) { + // caught an exception + log.log("------------------------------------------------------------------"); + log.log("getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink) generated a OneDriveException"); + log.log(""); + log.log("selectedDriveId: ", selectedDriveId); + log.log("selectedItemId: ", selectedItemId); + log.log("providedDeltaLink: ", providedDeltaLink); + log.log("------------------------------------------------------------------"); + + auto errorArray = splitLines(exception.msg); + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(getDeltaQueryOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + log.log(errorArray[0], " when attempting to query OneDrive API for Delta Changes - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Query: "); + deltaChangesBundle = getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink); + } else { + // Default operation if not 408,429,503,504 errors + if (exception.httpStatusCode == 410) { + log.log("\nWARNING: The OneDrive API responded with an error that indicates the locally stored deltaLink value is invalid"); + // Essentially the 'providedDeltaLink' that we have stored is no longer available ... re-try without the stored deltaLink + log.log("WARNING: Retrying OneDrive API call without using the locally stored deltaLink value"); + // Configure an empty deltaLink + log.vdebug("Delta link expired for 'getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink)', setting 'deltaLink = null'"); + string emptyDeltaLink; + deltaChangesBundle = getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, emptyDeltaLink); + } else { + // display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } + } + } + + // Check the response JSON + if (deltaChangesBundle.type() != JSONType.object){ + // Handle the invalid JSON response + invalidJSONResponseFromOneDriveAPI(); + } + + // Shutdown the API + getDeltaQueryOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(getDeltaQueryOneDriveApiInstance); + return deltaChangesBundle; + } + + // Common code to handle a 408 or 429 response from the OneDrive API + void handleOneDriveThrottleRequest(OneDriveApi activeOneDriveApiInstance) { + + // If OneDrive sends a status code 429 then this function will be used to process the Retry-After response header which contains the value by which we need to wait + log.vdebug("Handling a OneDrive HTTP 429 Response Code (Too Many Requests)"); + // Read in the Retry-After HTTP header as set and delay as per this value before retrying the request + auto retryAfterValue = activeOneDriveApiInstance.getRetryAfterValue(); + log.vdebug("Using Retry-After Value = ", retryAfterValue); + + // HTTP request returned status code 429 (Too Many Requests) + // https://github.com/abraunegg/onedrive/issues/133 + // https://github.com/abraunegg/onedrive/issues/815 + + ulong delayBeforeRetry = 0; + if (retryAfterValue != 0) { + // Use the HTTP Response Header Value + delayBeforeRetry = retryAfterValue; + } else { + // Use a 120 second delay as a default given header value was zero + // This value is based on log files and data when determining correct process for 429 response handling + delayBeforeRetry = 120; + // Update that we are over-riding the provided value with a default + log.vdebug("HTTP Response Header retry-after value was 0 - Using a preconfigured default of: ", delayBeforeRetry); + } + + // Sleep thread as per request + log.log("Thread sleeping due to 'HTTP request returned status code 429' - The request has been throttled"); + log.log("Sleeping for ", delayBeforeRetry, " seconds"); + Thread.sleep(dur!"seconds"(delayBeforeRetry)); + + // Reset retry-after value to zero as we have used this value now and it may be changed in the future to a different value + activeOneDriveApiInstance.resetRetryAfterValue(); + } + + // If the JSON response is not correct JSON object, exit + void invalidJSONResponseFromOneDriveAPI() { + + log.error("ERROR: Query of the OneDrive API returned an invalid JSON response"); + // Must exit + exit(-1); + } + + // Handle an unhandled API error + void defaultUnhandledHTTPErrorCode(OneDriveException exception) { + + // display error + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + // Must exit here + exit(-1); + } + + // Display the pertinant details of the sync engine + void displaySyncEngineDetails() { + + // Display accountType, defaultDriveId, defaultRootId & remainingFreeSpace for verbose logging purposes + //log.vlog("Application version: ", strip(import("version"))); + + string tempVersion = "v2.5.0-alpha-2" ~ " GitHub version: " ~ strip(import("version")); + log.vlog("Application version: ", tempVersion); + + log.vlog("Account Type: ", appConfig.accountType); + log.vlog("Default Drive ID: ", appConfig.defaultDriveId); + log.vlog("Default Root ID: ", appConfig.defaultRootId); + + // What do we display here for space remaining + if (appConfig.remainingFreeSpace > 0) { + // Display the actual value + log.vlog("Remaining Free Space: ", (appConfig.remainingFreeSpace/1024) , " KB"); + } else { + // zero or non-zero value or restricted + if (!appConfig.quotaRestricted){ + log.vlog("Remaining Free Space: 0 KB"); + } else { + log.vlog("Remaining Free Space: Not Available"); + } + } + } + + // Query itemdb.computePath() and catch potential assert when DB consistency issue occurs + string computeItemPath(string thisDriveId, string thisItemId) { + + // static declare this for this function + static import core.exception; + string calculatedPath; + log.vdebug("Attempting to calculate local filesystem path for ", thisDriveId, " and ", thisItemId); + try { + calculatedPath = itemDB.computePath(thisDriveId, thisItemId); + } catch (core.exception.AssertError) { + // broken tree in the database, we cant compute the path for this item id, exit + log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); + // Must exit here to preserve data + exit(-1); + } + + // return calculated path as string + return calculatedPath; + } + + // Try and compute the file hash for the given item + bool testFileHash(string path, Item item) { + + // Generate QuickXORHash first before attempting to generate any other type of hash + if (item.quickXorHash) { + if (item.quickXorHash == computeQuickXorHash(path)) return true; + } else if (item.sha256Hash) { + if (item.sha256Hash == computeSHA256Hash(path)) return true; + } + return false; + } + + // Process items that need to be removed + void processDeleteItems() { + + foreach_reverse (i; idsToDelete) { + Item item; + string path; + if (!itemDB.selectById(i[0], i[1], item)) continue; // check if the item is in the db + // Compute this item path + path = computeItemPath(i[0], i[1]); + + // Log the action if the path exists .. it may of already been removed and this is a legacy array item + if (exists(path)) { + if (item.type == ItemType.file) { + log.log("Trying to delete file ", path); + } else { + log.log("Trying to delete directory ", path); + } + } + + // Process the database entry removal. In a --dry-run scenario, this is being done against a DB copy + itemDB.deleteById(item.driveId, item.id); + if (item.remoteDriveId != null) { + // delete the linked remote folder + itemDB.deleteById(item.remoteDriveId, item.remoteId); + } + + // Add to pathFakeDeletedArray + // We dont want to try and upload this item again, so we need to track this object + if (dryRun) { + // We need to add './' here so that it can be correctly searched to ensure it is not uploaded + string pathToAdd = "./" ~ path; + pathFakeDeletedArray ~= pathToAdd; + } + + bool needsRemoval = false; + if (exists(path)) { + // path exists on the local system + // make sure that the path refers to the correct item + Item pathItem; + if (itemDB.selectByPath(path, item.driveId, pathItem)) { + if (pathItem.id == item.id) { + needsRemoval = true; + } else { + log.log("Skipped due to id difference!"); + } + } else { + // item has disappeared completely + needsRemoval = true; + } + } + if (needsRemoval) { + // Log the action + if (item.type == ItemType.file) { + log.log("Deleting file ", path); + } else { + log.log("Deleting directory ", path); + } + + // Perform the action + if (!dryRun) { + if (isFile(path)) { + remove(path); + } else { + try { + // Remove any children of this path if they still exist + // Resolve 'Directory not empty' error when deleting local files + foreach (DirEntry child; dirEntries(path, SpanMode.depth, false)) { + attrIsDir(child.linkAttributes) ? rmdir(child.name) : remove(child.name); + } + // Remove the path now that it is empty of children + rmdirRecurse(path); + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } + } + } + } + + if (!dryRun) { + // Cleanup array memory + idsToDelete = []; + } + } + + // Update the timestamp of an object online + void uploadLastModifiedTime(string driveId, string id, SysTime mtime, string eTag) { + + string itemModifiedTime; + itemModifiedTime = mtime.toISOExtString(); + JSONValue data = [ + "fileSystemInfo": JSONValue([ + "lastModifiedDateTime": itemModifiedTime + ]) + ]; + + // What eTag value do we use? + string eTagValue; + if (appConfig.accountType == "personal") { + eTagValue = null; + } else { + eTagValue = eTag; + } + + JSONValue response; + // Create a new OneDrive API instance + OneDriveApi uploadLastModifiedTimeApiInstance; + uploadLastModifiedTimeApiInstance = new OneDriveApi(appConfig); + uploadLastModifiedTimeApiInstance.initialise(); + + // Try and update the online last modified time + try { + // Use this instance + response = uploadLastModifiedTimeApiInstance.updateById(driveId, id, data, eTagValue); + // Shut the instance down + uploadLastModifiedTimeApiInstance.shutdown(); + // Free object and memory + object.destroy(uploadLastModifiedTimeApiInstance); + } catch (OneDriveException exception) { + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(uploadLastModifiedTimeApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to update the timestamp on an item on OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + uploadLastModifiedTime(driveId, id, mtime, eTag); + return; + } else { + // Default operation if not 408,429,503,504 errors + if (exception.httpStatusCode == 409) { + // ETag does not match current item's value - use a null eTag + log.vdebug("Retrying Function: ", thisFunctionName); + uploadLastModifiedTime(driveId, id, mtime, null); + } else { + // display what the error is + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + } + } + } + + // Is the response a valid JSON object - validation checking done in saveItem + writeln("Line: 2428"); + saveItem(response); + } + + // Perform a database integrity check - checking all the items that are in-sync at the moment, validating what we know should be on disk, to what is actually on disk + void performDatabaseConsistencyAndIntegrityCheck() { + + // Log what we are doing + if (!appConfig.surpressLoggingOutput) { + log.log("Performing a database consistency and integrity check on locally stored data ... "); + } + + // What driveIDsArray do we use? If we are doing a --single-directory we need to use just the drive id associated with that operation + string[] consistencyCheckDriveIdsArray; + if (singleDirectoryScope) { + consistencyCheckDriveIdsArray ~= singleDirectoryScopeDriveId; + } else { + consistencyCheckDriveIdsArray = driveIDsArray; + } + + // Create a new DB blank item + Item item; + // Use the array we populate, rather than selecting all distinct driveId's from the database + foreach (driveId; consistencyCheckDriveIdsArray) { + // Make the logging more accurate - we cant update driveId as this then breaks the below queries + log.vlog("Processing DB entries for this Drive ID: ", driveId); + + // What OneDrive API query do we use? + // - Are we running against a National Cloud Deployments that does not support /delta ? + // National Cloud Deployments do not support /delta as a query + // https://docs.microsoft.com/en-us/graph/deployments#supported-features + // + // - Are we performing a --single-directory sync, which will exclude many items online, focusing in on a specific online directory + // + // - Are we performing a --download-only --cleanup-local-files action? + // + // If we did, we self generated a /delta response, thus need to now process elements that are still flagged as out-of-sync + if ((singleDirectoryScope) || (nationalCloudDeployment) || (cleanupLocalFiles)) { + // Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB + // Normally, this is done at the end of processing all /delta queries, however when using --single-directory or a National Cloud Deployments is configured + // We cant use /delta to query the OneDrive API as National Cloud Deployments dont support /delta + // https://docs.microsoft.com/en-us/graph/deployments#supported-features + // We dont use /delta for --single-directory as, in order to sync a single path with /delta, we need to query the entire OneDrive API JSON data to then filter out + // objects that we dont want, thus, it is easier to use the same method as National Cloud Deployments, but query just the objects we are after + + // For each unique OneDrive driveID we know about + Item[] outOfSyncItems = itemDB.selectOutOfSyncItems(driveId); + foreach (outOfSyncItem; outOfSyncItems) { + if (!dryRun) { + // clean up idsToDelete + idsToDelete.length = 0; + assumeSafeAppend(idsToDelete); + // flag to delete local file as it now is no longer in sync with OneDrive + log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive"); + log.vdebug("outOfSyncItem: ", outOfSyncItem); + idsToDelete ~= [outOfSyncItem.driveId, outOfSyncItem.id]; + // delete items in idsToDelete + if (idsToDelete.length > 0) processDeleteItems(); + } + } + + // Fetch database items associated with this path + Item[] driveItems; + if (singleDirectoryScope) { + // Use the --single-directory items we previously configured + // - query database for children objects using those items + driveItems = getChildren(singleDirectoryScopeDriveId, singleDirectoryScopeItemId); + } else { + // Check everything associated with each driveId we know about + log.vdebug("Selecting DB items via itemDB.selectByDriveId(driveId)"); + // Query database + driveItems = itemDB.selectByDriveId(driveId); + } + + log.vdebug("Database items to process for this driveId: ", driveItems.count); + // Process each database database item associated with the driveId + foreach(dbItem; driveItems) { + // Does it still exist on disk in the location the DB thinks it is + checkDatabaseItemForConsistency(dbItem); + } + } else { + // Check everything associated with each driveId we know about + log.vdebug("Selecting DB items via itemDB.selectByDriveId(driveId)"); + // Query database + auto driveItems = itemDB.selectByDriveId(driveId); + log.vdebug("Database items to process for this driveId: ", driveItems.count); + // Process each database database item associated with the driveId + foreach(dbItem; driveItems) { + // Does it still exist on disk in the location the DB thinks it is + checkDatabaseItemForConsistency(dbItem); + } + } + } + + // Are we doing a --download-only sync? + if (!appConfig.getValueBool("download_only")) { + // Do we have any known items, where the content has changed locally, that needs to be uploaded? + if (!databaseItemsWhereContentHasChanged.empty) { + // There are changed local files that were in the DB to upload + log.log("Changed local items to upload to OneDrive: ", databaseItemsWhereContentHasChanged.length); + processChangedLocalItemsToUpload(); + // Cleanup array memory + databaseItemsWhereContentHasChanged = []; + } + } + } + + // Check this Database Item for its consistency on disk + void checkDatabaseItemForConsistency(Item dbItem) { + + // What is the local path item + string localFilePath; + // Do we want to onward process this item? + bool unwanted = false; + + // Compute this dbItem path early as we we use this path often + localFilePath = buildNormalizedPath(computeItemPath(dbItem.driveId, dbItem.id)); + + // To improve logging output for this function, what is the 'logical path'? + string logOutputPath; + if (localFilePath == ".") { + // get the configured sync_dir + logOutputPath = buildNormalizedPath(appConfig.getValueString("sync_dir")); + } else { + // use what was computed + logOutputPath = localFilePath; + } + + // Log what we are doing + log.vlog("Processing ", logOutputPath); + + // Determine which action to take + final switch (dbItem.type) { + case ItemType.file: + // Logging output + checkFileDatabaseItemForConsistency(dbItem, localFilePath); + break; + case ItemType.dir: + // Logging output + checkDirectoryDatabaseItemForConsistency(dbItem, localFilePath); + break; + case ItemType.remote: + // checkRemoteDirectoryDatabaseItemForConsistency(dbItem, localFilePath); + break; + case ItemType.unknown: + // Unknown type - we dont action these items + break; + } + } + + // Perform the database consistency check on this file item + void checkFileDatabaseItemForConsistency(Item dbItem, string localFilePath) { + + // What is the source of this item data? + string itemSource = "database"; + + // Does this item|file still exist on disk? + if (exists(localFilePath)) { + // Path exists locally, is this path a file? + if (isFile(localFilePath)) { + // Can we actually read the local file? + if (readLocalFile(localFilePath)){ + // File is readable + SysTime localModifiedTime = timeLastModified(localFilePath).toUTC(); + SysTime itemModifiedTime = dbItem.mtime; + // Reduce time resolution to seconds before comparing + itemModifiedTime.fracSecs = Duration.zero; + localModifiedTime.fracSecs = Duration.zero; + + if (localModifiedTime != itemModifiedTime) { + // The modified dates are different + log.vdebug("The local item has a different modified time ", localModifiedTime, " when compared to ", itemSource, " modified time ", itemModifiedTime); + // Test the file hash + if (!testFileHash(localFilePath, dbItem)) { + // Is the local file 'newer' or 'older' (ie was an old file 'restored locally' by a different backup / replacement process?) + if (localModifiedTime >= itemModifiedTime) { + // Local file is newer + if (!appConfig.getValueBool("download_only")) { + log.vlog("The file content has changed locally and has a newer timestamp, thus needs to be uploaded to OneDrive"); + // Add to an array of files we need to upload as this file has changed locally in-between doing the /delta check and performing this check + databaseItemsWhereContentHasChanged ~= [dbItem.driveId, dbItem.id, localFilePath]; + } else { + log.vlog("The file content has changed locally and has a newer timestamp. The file will remain different to online file due to --download-only being used"); + } + } else { + // Local file is older - data recovery process? something else? + if (!appConfig.getValueBool("download_only")) { + log.vlog("The file content has changed locally and file now has a older timestamp. Uploading this file to OneDrive may potentially cause data-loss online"); + // Add to an array of files we need to upload as this file has changed locally in-between doing the /delta check and performing this check + databaseItemsWhereContentHasChanged ~= [dbItem.driveId, dbItem.id, localFilePath]; + } else { + log.vlog("The file content has changed locally and file now has a older timestamp. The file will remain different to online file due to --download-only being used"); + } + } + } else { + // The file contents have not changed, but the modified timestamp has + log.vlog("The last modified timestamp has changed however the file content has not changed"); + log.vlog("The local item has the same hash value as the item online - correcting timestamp online"); + if (!dryRun) { + // Attempt to update the online date time stamp + uploadLastModifiedTime(dbItem.driveId, dbItem.id, localModifiedTime.toUTC(), dbItem.eTag); + } + } + } else { + // The file has not changed + log.vlog("The file has not changed"); + } + } else { + //The file is not readable - skipped + log.log("Skipping processing this file as it cannot be read (file permissions or file corruption): ", localFilePath); + } + } else { + // The item was a file but now is a directory + log.vlog("The item was a file but now is a directory"); + } + } else { + // File does not exist locally, but is in our database as a dbItem containing all the data was passed into this function + // If we are in a --dry-run situation - this file may never have existed as we never downloaded it + if (!dryRun) { + // Not --dry-run situation + log.vlog("The file has been deleted locally"); + // Upload to OneDrive the instruction to delete this item. This will handle the 'noRemoteDelete' flag if set + uploadDeletedItem(dbItem, localFilePath); + } else { + // We are in a --dry-run situation, file appears to have been deleted locally - this file may never have existed locally as we never downloaded it due to --dry-run + // Did we 'fake create it' as part of --dry-run ? + bool idsFakedMatch = false; + foreach (i; idsFaked) { + if (i[1] == dbItem.id) { + log.vdebug("Matched faked file which is 'supposed' to exist but not created due to --dry-run use"); + log.vlog("The file has not changed"); + idsFakedMatch = true; + } + } + if (!idsFakedMatch) { + // dbItem.id did not match a 'faked' download new file creation - so this in-sync object was actually deleted locally, but we are in a --dry-run situation + log.vlog("The file has been deleted locally"); + // Upload to OneDrive the instruction to delete this item. This will handle the 'noRemoteDelete' flag if set + uploadDeletedItem(dbItem, localFilePath); + } + } + } + } + + // Perform the database consistency check on this directory item + void checkDirectoryDatabaseItemForConsistency(Item dbItem, string localFilePath) { + + // What is the source of this item data? + string itemSource = "database"; + + // Does this item|directory still exist on disk? + if (exists(localFilePath)) { + // Fix https://github.com/abraunegg/onedrive/issues/1915 + try { + if (!isDir(localFilePath)) { + log.vlog("The item was a directory but now it is a file"); + uploadDeletedItem(dbItem, localFilePath); + uploadNewFile(localFilePath); + } else { + // Directory still exists locally + log.vlog("The directory has not changed"); + // When we are using --single-directory, we use a the getChildren() call to get all children of a path, meaning all children are already traversed + // Thus, if we traverse the path of this directory .. we end up with double processing & log output .. which is not ideal + if (!singleDirectoryScope) { + // loop through the children + foreach (Item child; itemDB.selectChildren(dbItem.driveId, dbItem.id)) { + checkDatabaseItemForConsistency(child); + } + } + } + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } else { + // Directory does not exist locally, but it is in our database as a dbItem containing all the data was passed into this function + // If we are in a --dry-run situation - this directory may never have existed as we never created it + if (!dryRun) { + // Not --dry-run situation + if (!appConfig.getValueBool("monitor")) { + // Not in --monitor mode + log.vlog("The directory has been deleted locally"); + } else { + // Appropriate message as we are in --monitor mode + log.vlog("The directory appears to have been deleted locally .. but we are running in --monitor mode. This may have been 'moved' on the local filesystem rather than being 'deleted'"); + log.vdebug("Most likely cause - 'inotify' event was missing for whatever action was taken locally or action taken when application was stopped"); + } + // A moved directory will be uploaded as 'new', delete the old directory and database reference + // Upload to OneDrive the instruction to delete this item. This will handle the 'noRemoteDelete' flag if set + uploadDeletedItem(dbItem, localFilePath); + } else { + // We are in a --dry-run situation, directory appears to have been deleted locally - this directory may never have existed locally as we never created it due to --dry-run + // Did we 'fake create it' as part of --dry-run ? + bool idsFakedMatch = false; + foreach (i; idsFaked) { + if (i[1] == dbItem.id) { + log.vdebug("Matched faked dir which is 'supposed' to exist but not created due to --dry-run use"); + log.vlog("The directory has not changed"); + idsFakedMatch = true; + } + } + if (!idsFakedMatch) { + // dbItem.id did not match a 'faked' download new directory creation - so this in-sync object was actually deleted locally, but we are in a --dry-run situation + log.vlog("The directory has been deleted locally"); + // Upload to OneDrive the instruction to delete this item. This will handle the 'noRemoteDelete' flag if set + uploadDeletedItem(dbItem, localFilePath); + } else { + // When we are using --single-directory, we use a the getChildren() call to get all children of a path, meaning all children are already traversed + // Thus, if we traverse the path of this directory .. we end up with double processing & log output .. which is not ideal + if (!singleDirectoryScope) { + // loop through the children + foreach (Item child; itemDB.selectChildren(dbItem.driveId, dbItem.id)) { + checkDatabaseItemForConsistency(child); + } + } + } + } + } + } + + /** + // Perform the database consistency check on this remote directory item + void checkRemoteDirectoryDatabaseItemForConsistency(Item dbItem, string localFilePath) { + + writeln("CODING TO DO: checkRemoteDirectoryDatabaseItemForConsistency"); + } + **/ + + // Does this Database Item (directory or file) get excluded from any operation based on any client side filtering rules? + bool checkDBItemAndPathAgainstClientSideFiltering(Item dbItem, string localFilePath) { + + // Check the item and path against client side filtering rules + // Return a true|false response + bool clientSideRuleExcludesItem = false; + + // Is this item a directory or 'remote' type? A 'remote' type is a folder DB tie so should be compared as directory for exclusion + if ((dbItem.type == ItemType.dir) || (dbItem.type == ItemType.remote)) { + + // Directory Path Tests + if (!clientSideRuleExcludesItem) { + // Do we need to check for .nosync? Only if --check-for-nosync was passed in + if (appConfig.getValueBool("check_nosync")) { + if (exists(localFilePath ~ "/.nosync")) { + log.vlog("Skipping item - .nosync found & --check-for-nosync enabled: ", localFilePath); + clientSideRuleExcludesItem = true; + } + } + + // Is this item excluded by user configuration of skip_dir? + if (!clientSideRuleExcludesItem) { + clientSideRuleExcludesItem = selectiveSync.isDirNameExcluded(dbItem.name); + } + } + } + + // Is this item a file? + if (dbItem.type == ItemType.file) { + // Is this item excluded by user configuration of skip_file? + if (!clientSideRuleExcludesItem) { + clientSideRuleExcludesItem = selectiveSync.isFileNameExcluded(dbItem.name); + } + + if (!clientSideRuleExcludesItem) { + // Check if file should be skipped based on user configured size limit 'skip_size' + if (fileSizeLimit != 0) { + // Get the file size + ulong thisFileSize = getSize(localFilePath); + if (thisFileSize >= fileSizeLimit) { + clientSideRuleExcludesItem = true; + log.vlog("Skipping item - excluded by skip_size config: ", localFilePath, " (", thisFileSize/2^^20," MB)"); + } + } + } + } + + if (!clientSideRuleExcludesItem) { + // Is sync_list configured? + if (syncListConfigured) { + // Is this item excluded by user configuration of sync_list? + clientSideRuleExcludesItem = selectiveSync.isPathExcludedViaSyncList(localFilePath); + } + } + + // Return bool value + return clientSideRuleExcludesItem; + } + + // Does this local path (directory or file) conform with the Microsoft Naming Restrictions? + bool checkPathAgainstMicrosoftNamingRestrictions(string localFilePath) { + + // Check if the given path violates certain Microsoft restrictions and limitations + // Return a true|false response + bool invalidPath = false; + + // Check against Microsoft OneDrive restriction and limitations about Windows naming files + if (!invalidPath) { + if (!isValidName(localFilePath)) { + log.logAndNotify("Skipping item - invalid name (Microsoft Naming Convention): ", localFilePath); + invalidPath = true; + } + } + + // Check for bad whitespace items + if (!invalidPath) { + if (!containsBadWhiteSpace(localFilePath)) { + log.logAndNotify("Skipping item - invalid name (Contains an invalid whitespace item): ", localFilePath); + invalidPath = true; + } + } + + // Check for HTML ASCII Codes as part of file name + if (!invalidPath) { + if (!containsASCIIHTMLCodes(localFilePath)) { + log.logAndNotify("Skipping item - invalid name (Contains HTML ASCII Code): ", localFilePath); + invalidPath = true; + } + } + // Return if this is a valid path + return invalidPath; + } + + // Does this local path (directory or file) get excluded from any operation based on any client side filtering rules? + bool checkPathAgainstClientSideFiltering(string localFilePath) { + + // Unlike checkDBItemAndPathAgainstClientSideFiltering - we need to check the path only + + // Check the path against client side filtering rules + // - check_nosync + // - skip_dotfiles + // - skip_symlinks + // - skip_file + // - skip_dir + // - sync_list + // - skip_size + // Return a true|false response + + bool clientSideRuleExcludesPath = false; + + // - check_nosync + if (!clientSideRuleExcludesPath) { + // Do we need to check for .nosync? Only if --check-for-nosync was passed in + if (appConfig.getValueBool("check_nosync")) { + if (exists(localFilePath ~ "/.nosync")) { + log.vlog("Skipping item - .nosync found & --check-for-nosync enabled: ", localFilePath); + clientSideRuleExcludesPath = true; + } + } + } + + // - skip_dotfiles + if (!clientSideRuleExcludesPath) { + // Do we need to check skip dot files if configured + if (appConfig.getValueBool("skip_dotfiles")) { + if (isDotFile(localFilePath)) { + log.vlog("Skipping item - .file or .folder: ", localFilePath); + clientSideRuleExcludesPath = true; + } + } + } + + // - skip_symlinks + if (!clientSideRuleExcludesPath) { + // Is the path a symbolic link + if (isSymlink(localFilePath)) { + // if config says so we skip all symlinked items + if (appConfig.getValueBool("skip_symlinks")) { + log.vlog("Skipping item - skip symbolic links configured: ", localFilePath); + clientSideRuleExcludesPath = true; + + } + // skip unexisting symbolic links + else if (!exists(readLink(localFilePath))) { + // reading the symbolic link failed - is the link a relative symbolic link + // drwxrwxr-x. 2 alex alex 46 May 30 09:16 . + // drwxrwxr-x. 3 alex alex 35 May 30 09:14 .. + // lrwxrwxrwx. 1 alex alex 61 May 30 09:16 absolute.txt -> /home/alex/OneDrivePersonal/link_tests/intercambio/prueba.txt + // lrwxrwxrwx. 1 alex alex 13 May 30 09:16 relative.txt -> ../prueba.txt + // + // absolute links will be able to be read, but 'relative' links will fail, because they cannot be read based on the current working directory 'sync_dir' + string currentSyncDir = getcwd(); + string fullLinkPath = buildNormalizedPath(absolutePath(localFilePath)); + string fileName = baseName(fullLinkPath); + string parentLinkPath = dirName(fullLinkPath); + // test if this is a 'relative' symbolic link + chdir(parentLinkPath); + auto relativeLink = readLink(fileName); + auto relativeLinkTest = exists(readLink(fileName)); + // reset back to our 'sync_dir' + chdir(currentSyncDir); + // results + if (relativeLinkTest) { + log.vdebug("Not skipping item - symbolic link is a 'relative link' to target ('", relativeLink, "') which can be supported: ", localFilePath); + } else { + log.logAndNotify("Skipping item - invalid symbolic link: ", localFilePath); + clientSideRuleExcludesPath = true; + } + } + } + } + + // Is this item excluded by user configuration of skip_dir or skip_file? + if (!clientSideRuleExcludesPath) { + if (localFilePath != ".") { + // skip_dir handling + if (isDir(localFilePath)) { + log.vdebug("Checking local path: ", localFilePath); + // Only check path if config is != "" + if (appConfig.getValueString("skip_dir") != "") { + // The path that needs to be checked needs to include the '/' + // This due to if the user has specified in skip_dir an exclusive path: '/path' - that is what must be matched + if (selectiveSync.isDirNameExcluded(localFilePath.strip('.'))) { + log.vlog("Skipping item - excluded by skip_dir config: ", localFilePath); + clientSideRuleExcludesPath = true; + } + } + } + + // skip_file handling + if (isFile(localFilePath)) { + log.vdebug("Checking file: ", localFilePath); + // The path that needs to be checked needs to include the '/' + // This due to if the user has specified in skip_file an exclusive path: '/path/file' - that is what must be matched + if (selectiveSync.isFileNameExcluded(localFilePath.strip('.'))) { + log.vlog("Skipping item - excluded by skip_file config: ", localFilePath); + clientSideRuleExcludesPath = true; + } + } + } + } + + // Is this item excluded by user configuration of sync_list? + if (!clientSideRuleExcludesPath) { + if (localFilePath != ".") { + if (syncListConfigured) { + // sync_list configured and in use + if (selectiveSync.isPathExcludedViaSyncList(localFilePath)) { + if ((isFile(localFilePath)) && (appConfig.getValueBool("sync_root_files")) && (rootName(localFilePath.strip('.').strip('/')) == "")) { + log.vdebug("Not skipping path due to sync_root_files inclusion: ", localFilePath); + } else { + if (exists(appConfig.syncListFilePath)){ + // skipped most likely due to inclusion in sync_list + log.vlog("Skipping item - excluded by sync_list config: ", localFilePath); + clientSideRuleExcludesPath = true; + } else { + // skipped for some other reason + log.vlog("Skipping item - path excluded by user config: ", localFilePath); + clientSideRuleExcludesPath = true; + } + } + } + } + } + } + + // Check if this is excluded by a user set maximum filesize to upload + if (!clientSideRuleExcludesPath) { + if (isFile(localFilePath)) { + if (fileSizeLimit != 0) { + // Get the file size + ulong thisFileSize = getSize(localFilePath); + if (thisFileSize >= fileSizeLimit) { + log.vlog("Skipping item - excluded by skip_size config: ", localFilePath, " (", thisFileSize/2^^20," MB)"); + } + } + } + } + + return clientSideRuleExcludesPath; + } + + // Does this JSON item (as received from OneDrive API) get excluded from any operation based on any client side filtering rules? + // This function is only used when we are fetching objects from the OneDrive API using a /children query to help speed up what object we query + bool checkJSONAgainstClientSideFiltering(JSONValue onedriveJSONItem) { + + bool clientSideRuleExcludesPath = false; + + // Check the path against client side filtering rules + // - check_nosync (MISSING) + // - skip_dotfiles (MISSING) + // - skip_symlinks (MISSING) + // - skip_file (MISSING) + // - skip_dir + // - sync_list + // - skip_size (MISSING) + // Return a true|false response + + // Use the JSON elements rather can computing a DB struct via makeItem() + string thisItemId = onedriveJSONItem["id"].str; + string thisItemDriveId = onedriveJSONItem["parentReference"]["driveId"].str; + string thisItemParentId = onedriveJSONItem["parentReference"]["id"].str; + string thisItemName = onedriveJSONItem["name"].str; + + // Is this parent is in the database + bool parentInDatabase = false; + + // Calculate if the Parent Item is in the database so that it can be re-used + parentInDatabase = itemDB.idInLocalDatabase(thisItemDriveId, thisItemParentId); + + // Check if this is excluded by config option: skip_dir + if (!clientSideRuleExcludesPath) { + // Only check path if config is != "" + if (!appConfig.getValueString("skip_dir").empty) { + // Is the item a folder? + if (isItemFolder(onedriveJSONItem)) { + // work out the 'snippet' path where this folder would be created + string simplePathToCheck = ""; + string complexPathToCheck = ""; + string matchDisplay = ""; + + if (hasParentReference(onedriveJSONItem)) { + // we need to workout the FULL path for this item + // simple path + if (("name" in onedriveJSONItem["parentReference"]) != null) { + simplePathToCheck = onedriveJSONItem["parentReference"]["name"].str ~ "/" ~ onedriveJSONItem["name"].str; + } else { + simplePathToCheck = onedriveJSONItem["name"].str; + } + log.vdebug("skip_dir path to check (simple): ", simplePathToCheck); + + // complex path + if (parentInDatabase) { + // build up complexPathToCheck + //complexPathToCheck = buildNormalizedPath(newItemPath); + complexPathToCheck = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ thisItemName; + } else { + log.vdebug("Parent details not in database - unable to compute complex path to check"); + } + if (!complexPathToCheck.empty) { + log.vdebug("skip_dir path to check (complex): ", complexPathToCheck); + } + } else { + simplePathToCheck = onedriveJSONItem["name"].str; + } + + // If 'simplePathToCheck' or 'complexPathToCheck' is of the following format: root:/folder + // then isDirNameExcluded matching will not work + // Clean up 'root:' if present + if (startsWith(simplePathToCheck, "root:")){ + log.vdebug("Updating simplePathToCheck to remove 'root:'"); + simplePathToCheck = strip(simplePathToCheck, "root:"); + } + if (startsWith(complexPathToCheck, "root:")){ + log.vdebug("Updating complexPathToCheck to remove 'root:'"); + complexPathToCheck = strip(complexPathToCheck, "root:"); + } + + // OK .. what checks are we doing? + if ((!simplePathToCheck.empty) && (complexPathToCheck.empty)) { + // just a simple check + log.vdebug("Performing a simple check only"); + clientSideRuleExcludesPath = selectiveSync.isDirNameExcluded(simplePathToCheck); + } else { + // simple and complex + log.vdebug("Performing a simple then complex path match if required"); + // simple first + log.vdebug("Performing a simple check first"); + clientSideRuleExcludesPath = selectiveSync.isDirNameExcluded(simplePathToCheck); + matchDisplay = simplePathToCheck; + if (!clientSideRuleExcludesPath) { + log.vdebug("Simple match was false, attempting complex match"); + // simple didnt match, perform a complex check + clientSideRuleExcludesPath = selectiveSync.isDirNameExcluded(complexPathToCheck); + matchDisplay = complexPathToCheck; + } + } + // result + log.vdebug("skip_dir exclude result (directory based): ", clientSideRuleExcludesPath); + if (clientSideRuleExcludesPath) { + // This path should be skipped + log.vlog("Skipping item - excluded by skip_dir config: ", matchDisplay); + } + } + } + } + + // Check if this is included or excluded by use of sync_list + if (!clientSideRuleExcludesPath) { + // No need to try and process something against a sync_list if it has been configured + if (syncListConfigured) { + // Compute the item path if empty - as to check sync_list we need an actual path to check + + // What is the path of the new item + string newItemPath; + + // Is the parent in the database? If not, we cannot compute the the full path based on the database entries + // In a --resync scenario - the database is empty + if (parentInDatabase) { + // Calculate this items path based on database entries + newItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ thisItemName; + } else { + newItemPath = thisItemName; + } + + // What path are we checking? + log.vdebug("sync_list item to check: ", newItemPath); + + // Unfortunatly there is no avoiding this call to check if the path is excluded|included via sync_list + if (selectiveSync.isPathExcludedViaSyncList(newItemPath)) { + // selective sync advised to skip, however is this a file and are we configured to upload / download files in the root? + if ((isItemFile(onedriveJSONItem)) && (appConfig.getValueBool("sync_root_files")) && (rootName(newItemPath) == "") ) { + // This is a file + // We are configured to sync all files in the root + // This is a file in the logical root + clientSideRuleExcludesPath = false; + } else { + // path is unwanted + clientSideRuleExcludesPath = true; + log.vlog("Skipping item - excluded by sync_list config: ", newItemPath); + } + } + } + } + + // return if path is excluded + return clientSideRuleExcludesPath; + } + + // Process the list of local changes to upload to OneDrive + void processChangedLocalItemsToUpload() { + + // Each element in this array 'databaseItemsWhereContentHasChanged' is an Database Item ID that has been modified locally + ulong batchSize = appConfig.concurrentThreads; + ulong batchCount = (databaseItemsWhereContentHasChanged.length + batchSize - 1) / batchSize; + ulong batchesProcessed = 0; + + // For each batch of files to upload, upload the changed data to OneDrive + foreach (chunk; databaseItemsWhereContentHasChanged.chunks(batchSize)) { + uploadChangedLocalFileToOneDrive(chunk); + } + } + + // Upload changed local files to OneDrive in parallel + void uploadChangedLocalFileToOneDrive(string[3][] array) { + + foreach (i, localItemDetails; taskPool.parallel(array)) { + + log.vdebug("Thread ", i, " Starting: ", Clock.currTime()); + + // These are the details of the item we need to upload + string changedItemParentId = localItemDetails[0]; + string changedItemId = localItemDetails[1]; + string localFilePath = localItemDetails[2]; + + // How much space is remaining on OneDrive + ulong remainingFreeSpace; + // Did the upload fail? + bool uploadFailed = false; + // Did we skip due to exceeding maximum allowed size? + bool skippedMaxSize = false; + // Did we skip to an exception error? + bool skippedExceptionError = false; + + // Unfortunatly, we cant store an array of Item's ... so we have to re-query the DB again - unavoidable extra processing here + // This is because the Item[] has no other functions to allow is to parallel process those elements, so we have to use a string array as input to this function + Item dbItem; + itemDB.selectById(changedItemParentId, changedItemId, dbItem); + + // Query the available space online + // This will update appConfig.quotaAvailable & appConfig.quotaRestricted values + remainingFreeSpace = getRemainingFreeSpace(dbItem.driveId); + + // Get the file size + ulong thisFileSizeLocal = getSize(localFilePath); + ulong thisFileSizeFromDB = to!ulong(dbItem.size); + + // remainingFreeSpace online includes the current file online + // we need to remove the online file (add back the existing file size) then take away the new local file size to get a new approximate value + ulong calculatedSpaceOnlinePostUpload = (remainingFreeSpace + thisFileSizeFromDB) - thisFileSizeLocal; + + // Based on what we know, for this thread - can we safely upload this modified local file? + log.vdebug("This Thread Current Free Space Online: ", remainingFreeSpace); + log.vdebug("This Thread Calculated Free Space Online Post Upload: ", calculatedSpaceOnlinePostUpload); + + JSONValue uploadResponse; + + bool spaceAvailableOnline = false; + // If 'personal' accounts, if driveId == defaultDriveId, then we will have data - appConfig.quotaAvailable will be updated + // If 'personal' accounts, if driveId != defaultDriveId, then we will not have quota data - appConfig.quotaRestricted will be set as true + // If 'business' accounts, if driveId == defaultDriveId, then we will have data + // If 'business' accounts, if driveId != defaultDriveId, then we will have data, but it will be a 0 value - appConfig.quotaRestricted will be set as true + + // What was the latest getRemainingFreeSpace() value? + if (appConfig.quotaAvailable) { + // Our query told us we have free space online .. if we upload this file, will we exceed space online - thus upload will fail during upload? + if (calculatedSpaceOnlinePostUpload > 0) { + // Based on this thread action, we beleive that there is space available online to upload - proceed + spaceAvailableOnline = true; + } + } + // Is quota being restricted? + if (appConfig.quotaRestricted) { + // Space available online is being restricted - so we have no way to really know if there is space available online + spaceAvailableOnline = true; + } + + // Do we have space available or is space available being restricted (so we make the blind assumption that there is space available) + if (spaceAvailableOnline) { + // Does this file exceed the maximum file size to upload to OneDrive? + if (thisFileSizeLocal <= maxUploadFileSize) { + // Attempt to upload the modified file + // Error handling is in performModifiedFileUpload(), and the JSON that is responded with - will either be null or a valid JSON object containing the upload result + uploadResponse = performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal); + + // Evaluate the returned JSON uploadResponse + // If there was an error uploading the file, uploadResponse should be empty and invalid + if (uploadResponse.type() != JSONType.object){ + uploadFailed = true; + skippedExceptionError = true; + } + + } else { + // Skip file - too large + uploadFailed = true; + skippedMaxSize = true; + } + } else { + // Cant upload this file - no space available + uploadFailed = true; + } + + // Did the upload fail? + if (uploadFailed) { + // Upload failed .. why? + // No space available online + if (!spaceAvailableOnline) { + log.logAndNotify("Skipping uploading modified file ", localFilePath, " due to insufficient free space available on OneDrive"); + } + // File exceeds max allowed size + if (skippedMaxSize) { + log.logAndNotify("Skipping uploading this modified file as it exceeds the maximum size allowed by OneDrive: ", localFilePath); + } + // Generic message + if (skippedExceptionError) { + // normal failure message if API or exception error generated + log.logAndNotify("Uploading modified file ", localFilePath, " ... failed!"); + } + } else { + // Upload was successful + log.logAndNotify("Uploading modified file ", localFilePath, " ... done."); + + // Save JSON item in database + writeln("Line: 3269"); + saveItem(uploadResponse); + + if (!dryRun) { + // Check the integrity of the uploaded modified file + performUploadIntegrityValidationChecks(uploadResponse, localFilePath, thisFileSizeLocal); + + // Update the date / time of the file online to match the local item + // Get the local file last modified time + SysTime localModifiedTime = timeLastModified(localFilePath).toUTC(); + localModifiedTime.fracSecs = Duration.zero; + // Get the latest eTag, and use that + string etagFromUploadResponse = uploadResponse["eTag"].str; + // Attempt to update the online date time stamp based on our local data + uploadLastModifiedTime(dbItem.driveId, dbItem.id, localModifiedTime, etagFromUploadResponse); + } + } + + log.vdebug("Thread ", i, " Finished: ", Clock.currTime()); + + } // end of 'foreach (i, localItemDetails; array.enumerate)' + } + + // Perform the upload of a locally modified file to OneDrive + JSONValue performModifiedFileUpload(Item dbItem, string localFilePath, ulong thisFileSizeLocal) { + + JSONValue uploadResponse; + OneDriveApi uploadFileOneDriveApiInstance; + uploadFileOneDriveApiInstance = new OneDriveApi(appConfig); + uploadFileOneDriveApiInstance.initialise(); + + // Is this a dry-run scenario? + if (!dryRun) { + // Do we use simpleUpload or create an upload session? + bool useSimpleUpload = false; + + //if ((appConfig.accountType == "personal") && (thisFileSizeLocal <= sessionThresholdFileSize)) { + + if (thisFileSizeLocal <= sessionThresholdFileSize) { + useSimpleUpload = true; + } + + // We can only upload zero size files via simpleFileUpload regardless of account type + // Reference: https://github.com/OneDrive/onedrive-api-docs/issues/53 + // Additionally, all files where file size is < 4MB should be uploaded by simpleUploadReplace - everything else should use a session to upload the modified file + + if ((thisFileSizeLocal == 0) || (useSimpleUpload)) { + // Must use Simple Upload to replace the file online + try { + uploadResponse = uploadFileOneDriveApiInstance.simpleUploadReplace(localFilePath, dbItem.driveId, dbItem.id); + } catch (OneDriveException exception) { + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to upload a modified file to OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + } + + } catch (FileException e) { + // filesystem error + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } else { + // Configure JSONValue variables we use for a session upload + JSONValue currentOnlineData; + JSONValue uploadSessionData; + string currentETag; + + // As this is a unique thread, the sessionFilePath for where we save the data needs to be unique + // The best way to do this is calculate the CRC32 of the file, and use this as the suffix of the session file we save + string threadUploadSessionFilePath = appConfig.uploadSessionFilePath ~ "." ~ computeCRC32(localFilePath); + + // Get the absolute latest object details from online + try { + currentOnlineData = uploadFileOneDriveApiInstance.getPathDetailsByDriveId(dbItem.driveId, localFilePath); + } catch (OneDriveException exception) { + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to obtain latest file details from OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + } + + } + + // Was a valid JSON response provided? + if (currentOnlineData.type() == JSONType.object) { + // Does the response contain an eTag? + if (hasETag(currentOnlineData)) { + // Use the value returned from online + currentETag = currentOnlineData["eTag"].str; + } else { + // Use the database value + currentETag = dbItem.eTag; + } + } else { + // no valid JSON response + currentETag = dbItem.eTag; + } + + // Create the Upload Session + try { + uploadSessionData = createSessionFileUpload(uploadFileOneDriveApiInstance, localFilePath, dbItem.driveId, dbItem.parentId, baseName(localFilePath), currentETag, threadUploadSessionFilePath); + } catch (OneDriveException exception) { + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to create an upload session on OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + } + + } catch (FileException e) { + writeln("DEBUG TO REMOVE: Modified file upload FileException Handling (Create the Upload Session)"); + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + + // Perform the Upload using the session + try { + uploadResponse = performSessionFileUpload(uploadFileOneDriveApiInstance, thisFileSizeLocal, uploadSessionData, threadUploadSessionFilePath); + } catch (OneDriveException exception) { + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to upload a file via a session to OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + } + + } catch (FileException e) { + writeln("DEBUG TO REMOVE: Modified file upload FileException Handling (Perform the Upload using the session)"); + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } + + } else { + // We are in a --dry-run scenario + uploadResponse = createFakeResponse(localFilePath); + } + + // Debug Log the modified upload response + log.vdebug("Modified File Upload Response: ", uploadResponse); + + // Shutdown the API instance + uploadFileOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(uploadFileOneDriveApiInstance); + // Return JSON + return uploadResponse; + } + + // Query the OneDrive API using the provided driveId to get the latest quota details + ulong getRemainingFreeSpace(string driveId) { + + // Get the quota details for this driveId, as this could have changed since we started the application - the user could have added / deleted data online, or purchased additional storage + // Quota details are ONLY available for the main default driveId, as the OneDrive API does not provide quota details for shared folders + + JSONValue currentDriveQuota; + ulong remainingQuota; + + try { + // Create a new OneDrive API instance + OneDriveApi getCurrentDriveQuotaApiInstance; + getCurrentDriveQuotaApiInstance = new OneDriveApi(appConfig); + getCurrentDriveQuotaApiInstance.initialise(); + log.vdebug("Seeking available quota for this drive id: ", driveId); + currentDriveQuota = getCurrentDriveQuotaApiInstance.getDriveQuota(driveId); + // Shut this API instance down + getCurrentDriveQuotaApiInstance.shutdown(); + // Free object and memory + object.destroy(getCurrentDriveQuotaApiInstance); + } catch (OneDriveException e) { + log.vdebug("currentDriveQuota = onedrive.getDriveQuota(driveId) generated a OneDriveException"); + } + + // validate that currentDriveQuota is a JSON value + if (currentDriveQuota.type() == JSONType.object) { + // Response from API contains valid data + // If 'personal' accounts, if driveId == defaultDriveId, then we will have data + // If 'personal' accounts, if driveId != defaultDriveId, then we will not have quota data + // If 'business' accounts, if driveId == defaultDriveId, then we will have data + // If 'business' accounts, if driveId != defaultDriveId, then we will have data, but it will be a 0 value + + if ("quota" in currentDriveQuota){ + if (driveId == appConfig.defaultDriveId) { + // We potentially have updated quota remaining details available + // However in some cases OneDrive Business configurations 'restrict' quota details thus is empty / blank / negative value / zero + if ("remaining" in currentDriveQuota["quota"]){ + // We have valid quota remaining details returned for the provided drive id + remainingQuota = currentDriveQuota["quota"]["remaining"].integer; + + if (remainingQuota <= 0) { + if (appConfig.accountType == "personal"){ + // zero space available + log.error("ERROR: OneDrive account currently has zero space available. Please free up some space online or purchase additional space."); + remainingQuota = 0; + appConfig.quotaAvailable = false; + } else { + // zero space available is being reported, maybe being restricted? + log.error("WARNING: OneDrive quota information is being restricted or providing a zero value. Please fix by speaking to your OneDrive / Office 365 Administrator."); + remainingQuota = 0; + appConfig.quotaRestricted = true; + } + } + } + } else { + // quota details returned, but for a drive id that is not ours + if ("remaining" in currentDriveQuota["quota"]){ + // remaining is in the quota JSON response + if (currentDriveQuota["quota"]["remaining"].integer <= 0) { + // value returned is 0 or less than 0 + log.vlog("OneDrive quota information is set at zero, as this is not our drive id, ignoring"); + remainingQuota = 0; + appConfig.quotaRestricted = true; + } + } + } + } else { + // No quota details returned + if (driveId == appConfig.defaultDriveId) { + // no quota details returned for current drive id + log.error("ERROR: OneDrive quota information is missing. Potentially your OneDrive account currently has zero space available. Please free up some space online or purchase additional space."); + remainingQuota = 0; + appConfig.quotaRestricted = true; + } else { + // quota details not available + log.vdebug("WARNING: OneDrive quota information is being restricted as this is not our drive id."); + remainingQuota = 0; + appConfig.quotaRestricted = true; + } + } + } + + // what was the determined available quota? + log.vdebug("Available quota: ", remainingQuota); + return remainingQuota; + } + + // Perform a filesystem walk to uncover new data to upload to OneDrive + void scanLocalFilesystemPathForNewData(string path) { + + // To improve logging output for this function, what is the 'logical path' we are scanning for file & folder differences? + string logPath; + if (path == ".") { + // get the configured sync_dir + logPath = buildNormalizedPath(appConfig.getValueString("sync_dir")); + } else { + // use what was passed in + if (!appConfig.getValueBool("monitor")) { + logPath = buildNormalizedPath(appConfig.getValueString("sync_dir")) ~ "/" ~ path; + } else { + logPath = path; + } + } + + // Log the action that we are performing + if (!appConfig.surpressLoggingOutput) { + if (!cleanupLocalFiles) { + log.log("Scanning the local file system '", logPath, "' for new data to upload ..."); + } else { + log.log("Scanning the local file system '", logPath, "' for data to cleanup ..."); + } + } + + auto startTime = Clock.currTime(); + log.vdebug("Starting Filesystem Walk: ", startTime); + + // Perform the filesystem walk of this path, building an array of new items to upload + scanPathForNewData(path); + + // To finish off the processing items, this is needed to reflect this in the log + log.vdebug("------------------------------------------------------------------"); + + auto finishTime = Clock.currTime(); + log.vdebug("Finished Filesystem Walk: ", finishTime); + + auto elapsedTime = finishTime - startTime; + log.vdebug("Elapsed Time Filesystem Walk: ", elapsedTime); + + // Upload new data that has been identified + // Are there any items to download post fetching the /delta data? + if (!newLocalFilesToUploadToOneDrive.empty) { + // There are elements to upload + log.vlog("New items to upload to OneDrive: ", newLocalFilesToUploadToOneDrive.length); + + // How much data do we need to upload? This is important, as, we need to know how much data to determine if all the files can be uploaded + foreach (uploadFilePath; newLocalFilesToUploadToOneDrive) { + // validate that the path actually exists so that it can be counted + if (exists(uploadFilePath)) { + totalDataToUpload = totalDataToUpload + getSize(uploadFilePath); + } + } + + // How many bytes to upload + if (totalDataToUpload < 1024) { + // Display as Bytes to upload + log.vlog("Total New Data to Upload: ", totalDataToUpload, " Bytes"); + } else { + if ((totalDataToUpload > 1024) && (totalDataToUpload < 1048576)) { + // Display as KB to upload + log.vlog("Total New Data to Upload: ", (totalDataToUpload / 1024), " KB"); + } else { + // Display as MB to upload + log.vlog("Total New Data to Upload: ", (totalDataToUpload / 1024 / 1024), " MB"); + } + } + + // How much space is available (Account Drive ID) + // The file, could be uploaded to a shared folder, which, we are not tracking how much free space is available there ... + log.vdebug("Current Available Space Online (Account Drive ID): ", (appConfig.remainingFreeSpace / 1024 / 1024), " MB"); + + // Perform the upload + uploadNewLocalFileItems(); + + // Cleanup array memory + newLocalFilesToUploadToOneDrive = []; + } + } + + // Scan this path for new data + void scanPathForNewData(string path) { + + ulong maxPathLength; + ulong pathWalkLength; + + // Add this logging break to assist with what was checked for each path + if (path != ".") { + log.vdebug("------------------------------------------------------------------"); + } + + // https://support.microsoft.com/en-us/help/3125202/restrictions-and-limitations-when-you-sync-files-and-folders + // If the path is greater than allowed characters, then one drive will return a '400 - Bad Request' + // Need to ensure that the URI is encoded before the check is made: + // - 400 Character Limit for OneDrive Business / Office 365 + // - 430 Character Limit for OneDrive Personal + + // Configure maxPathLength based on account type + if (appConfig.accountType == "personal") { + // Personal Account + maxPathLength = 430; + } else { + // Business Account / Office365 / SharePoint + maxPathLength = 400; + } + + // A short lived item that has already disappeared will cause an error - is the path still valid? + if (!exists(path)) { + log.log("Skipping item - path has disappeared: ", path); + return; + } + + // Calculate the path length by walking the path and catch any UTF-8 sequence errors at the same time + // https://github.com/skilion/onedrive/issues/57 + // https://github.com/abraunegg/onedrive/issues/487 + // https://github.com/abraunegg/onedrive/issues/1192 + try { + pathWalkLength = path.byGrapheme.walkLength; + } catch (std.utf.UTFException e) { + // Path contains characters which generate a UTF exception + log.logAndNotify("Skipping item - invalid UTF sequence: ", path); + log.vdebug(" Error Reason:", e.msg); + return; + } + + // Is the path length is less than maxPathLength + if (pathWalkLength < maxPathLength) { + // Is this path unwanted + bool unwanted = false; + + // First check of this item - if we are in a --dry-run scenario, we may have 'fake deleted' this path + // thus, the entries are not in the dry-run DB copy, thus, at this point the client thinks that this is an item to upload + // Check this 'path' for an entry in pathFakeDeletedArray - if it is there, this is unwanted + if (dryRun) { + // Is this path in the array of fake deleted items? If yes, return early, nothing else to do, save processing + if (canFind(pathFakeDeletedArray, path)) return; + } + + // This not a Client Side Filtering check, nor a Microsoft Check, but is a sanity check that the path provided is UTF encoded correctly + // Check the std.encoding of the path against: Unicode 5.0, ASCII, ISO-8859-1, ISO-8859-2, WINDOWS-1250, WINDOWS-1251, WINDOWS-1252 + if (!unwanted) { + if(!isValid(path)) { + // Path is not valid according to https://dlang.org/phobos/std_encoding.html + log.logAndNotify("Skipping item - invalid character encoding sequence: ", path); + unwanted = true; + } + } + + // Check this path against the Client Side Filtering Rules + // - check_nosync + // - skip_dotfiles + // - skip_symlinks + // - skip_file + // - skip_dir + // - sync_list + // - skip_size + if (!unwanted) { + unwanted = checkPathAgainstClientSideFiltering(path); + } + + // Check this path against the Microsoft Naming Conventions & Restristions + // - Microsoft OneDrive restriction and limitations about Windows naming files + // - Bad whitespace items + // - HTML ASCII Codes as part of file name + if (!unwanted) { + unwanted = checkPathAgainstMicrosoftNamingRestrictions(path); + } + + if (!unwanted) { + // At this point, this path, we want to scan for new data as it is not excluded + if (isDir(path)) { + // Check if this path in the database + bool directoryFoundInDB = pathFoundInDatabase(path); + + // Was the path found in the database? + if (!directoryFoundInDB) { + // Path not found in database when searching all drive id's + if (!cleanupLocalFiles) { + // --download-only --cleanup-local-files not used + // Create this directory on OneDrive so that we can upload files to it + createDirectoryOnline(path); + } else { + // we need to clean up this directory + log.log("Removing local directory as --download-only & --cleanup-local-files configured"); + // Remove any children of this path if they still exist + // Resolve 'Directory not empty' error when deleting local files + try { + foreach (DirEntry child; dirEntries(path, SpanMode.depth, false)) { + // what sort of child is this? + if (isDir(child.name)) { + log.log("Removing local directory: ", child.name); + } else { + log.log("Removing local file: ", child.name); + } + + // are we in a --dry-run scenario? + if (!dryRun) { + // No --dry-run ... process local delete + if (exists(child)) { + try { + attrIsDir(child.linkAttributes) ? rmdir(child.name) : remove(child.name); + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } + } + } + // Remove the path now that it is empty of children + log.log("Removing local directory: ", path); + // are we in a --dry-run scenario? + if (!dryRun) { + // No --dry-run ... process local delete + try { + rmdirRecurse(path); + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + return; + } + } + } + + // flag for if we are going traverse this path + bool skipFolderTraverse = false; + + // Before we traverse this 'path', we need to make a last check to see if this was just excluded + if (appConfig.accountType == "business") { + // search businessSharedFoldersOnlineToSkip for this path + if (canFind(businessSharedFoldersOnlineToSkip, path)) { + // This path was skipped - why? + log.logAndNotify("Skipping item '", path, "' due to this path matching an existing online Business Shared Folder name"); + skipFolderTraverse = true; + } + } + + // Do we traverse this path? + if (!skipFolderTraverse) { + // Try and access this directory and any path below + try { + auto entries = dirEntries(path, SpanMode.shallow, false); + foreach (DirEntry entry; entries) { + string thisPath = entry.name; + scanPathForNewData(thisPath); + } + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + return; + } + } + } else { + // https://github.com/abraunegg/onedrive/issues/984 + // path is not a directory, is it a valid file? + // pipes - whilst technically valid files, are not valid for this client + // prw-rw-r--. 1 user user 0 Jul 7 05:55 my_pipe + if (isFile(path)) { + // Path is a valid file, not a pipe + bool fileFoundInDB = pathFoundInDatabase(path); + // Was the file found in the database? + if (!fileFoundInDB) { + // File not found in database when searching all drive id's + // Do we upload the file or clean up the file? + if (!cleanupLocalFiles) { + // --download-only --cleanup-local-files not used + // Add this path as a file we need to upload + log.vdebug("OneDrive Client flagging to upload this file to OneDrive: ", path); + newLocalFilesToUploadToOneDrive ~= path; + } else { + // we need to clean up this file + log.log("Removing local file as --download-only & --cleanup-local-files configured"); + // are we in a --dry-run scenario? + log.log("Removing local file: ", path); + if (!dryRun) { + // No --dry-run ... process local file delete + safeRemove(path); + } + } + } + } else { + // path is not a valid file + log.logAndNotify("Skipping item - item is not a valid file: ", path); + } + } + } + } else { + // This path was skipped - why? + log.logAndNotify("Skipping item '", path, "' due to the full path exceeding ", maxPathLength, " characters (Microsoft OneDrive limitation)"); + } + } + + // Query the database to determine if this path is within the existing database + bool pathFoundInDatabase(string searchPath) { + + // Check if this path in the database + Item databaseItem; + bool pathFoundInDB = false; + foreach (driveId; driveIDsArray) { + if (itemDB.selectByPath(searchPath, driveId, databaseItem)) { + pathFoundInDB = true; + } + } + return pathFoundInDB; + } + + // Create a new directory online on OneDrive + // - Test if we can get the parent path details from the database, otherwise we need to search online + // for the path flow and create the folder that way + void createDirectoryOnline(string thisNewPathToCreate) { + + log.log("OneDrive Client requested to create this directory online: ", thisNewPathToCreate); + + Item parentItem; + JSONValue onlinePathData; + + // Create a new API Instance for this thread and initialise it + OneDriveApi createDirectoryOnlineOneDriveApiInstance; + createDirectoryOnlineOneDriveApiInstance = new OneDriveApi(appConfig); + createDirectoryOnlineOneDriveApiInstance.initialise(); + + // What parent path to use? + string parentPath = dirName(thisNewPathToCreate); // will be either . or something else + + // Configure the parentItem by if this is the account 'root' use the root details, or search the database for the parent details + if (parentPath == ".") { + // Parent path is '.' which is the account root + // Use client defaults + parentItem.driveId = appConfig.defaultDriveId; // Should give something like 12345abcde1234a1 + parentItem.id = appConfig.defaultRootId; // Should give something like 12345ABCDE1234A1!101 + } else { + // Query the parent path online + log.vdebug("Attempting to query Local Database for this parent path: ", parentPath); + + // Attempt a 2 step process to work out where to create the directory + // Step 1: Query the DB first + // Step 2: Query online as last resort + + // Step 1: Check if this path in the database + Item databaseItem; + bool pathFoundInDB = false; + foreach (driveId; driveIDsArray) { + if (itemDB.selectByPath(parentPath, driveId, databaseItem)) { + pathFoundInDB = true; + log.vdebug("databaseItem: ", databaseItem); + log.vdebug("pathFoundInDB: ", pathFoundInDB); + } + } + + // Step 2: Query for the path online + if (!pathFoundInDB) { + + try { + log.vdebug("Attempting to query OneDrive Online for this parent path as path not found in local database: ", parentPath); + onlinePathData = createDirectoryOnlineOneDriveApiInstance.getPathDetails(parentPath); + writeln("Line: 3961"); + saveItem(onlinePathData); + parentItem = makeItem(onlinePathData); + } catch (OneDriveException exception) { + + if (exception.httpStatusCode == 404) { + // Parent does not exist ... need to create parent + log.vdebug("Parent path does not exist online: ", parentPath); + createDirectoryOnline(parentPath); + } else { + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(createDirectoryOnlineOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to create a remote directory on OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + createDirectoryOnline(thisNewPathToCreate); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } + } + } + + } else { + // parent path found in database ... use those details ... + parentItem = databaseItem; + } + + } + + // Make sure the full path does not exist online, this should generate a 404 response, to which then the folder will be created online + try { + // Try and query the OneDrive API for the path we need to create + log.vdebug("Attempting to query OneDrive for this path: ", thisNewPathToCreate); + + if (parentItem.driveId == appConfig.defaultDriveId) { + // Use getPathDetailsByDriveId + onlinePathData = createDirectoryOnlineOneDriveApiInstance.getPathDetailsByDriveId(parentItem.driveId, thisNewPathToCreate); + } else { + // If the parentItem.driveId is not our driveId - the path we are looking for will not be at the logical location that getPathDetailsByDriveId + // can use - as it will always return a 404 .. even if the path actually exists (which is the whole point of this test) + // Search the parentItem.driveId for any folder name match that we are going to create, then compare response JSON items with parentItem.id + // If no match, the folder we want to create does not exist at the location we are seeking to create it at, thus generate a 404 + onlinePathData = createDirectoryOnlineOneDriveApiInstance.searchDriveForPath(parentItem.driveId, baseName(thisNewPathToCreate)); + + // Process the response from searching the drive + ulong responseCount = count(onlinePathData["value"].array); + if (responseCount > 0) { + // Search 'name' matches were found .. need to match these against parentItem.id + bool foundDirectoryOnline = false; + JSONValue foundDirectoryJSONItem; + // Items were returned .. but is one of these what we are looking for? + foreach (childJSON; onlinePathData["value"].array) { + // Is this item not a file? + if (!isFileItem(childJSON)) { + Item thisChildItem = makeItem(childJSON); + // Direct Match Check + if ((parentItem.id == thisChildItem.parentId) && (baseName(thisNewPathToCreate) == thisChildItem.name)) { + // High confidence that this child folder is a direct match we are trying to create and it already exists online + log.vdebug("Path we are searching for exists online: ", baseName(thisNewPathToCreate)); + log.vdebug("childJSON: ", childJSON); + foundDirectoryOnline = true; + foundDirectoryJSONItem = childJSON; + break; + } + // Full Lower Case POSIX Match Check + string childAsLower = toLower(childJSON["name"].str); + string thisFolderNameAsLower = toLower(baseName(thisNewPathToCreate)); + + if (childAsLower == thisFolderNameAsLower) { + // This is a POSIX 'case in-sensitive match' ..... + // Local item name has a 'case-insensitive match' to an existing item on OneDrive + foundDirectoryOnline = true; + foundDirectoryJSONItem = childJSON; + break; + } + } + } + + if (foundDirectoryOnline) { + // Directory we are seeking was found online ... + onlinePathData = foundDirectoryJSONItem; + } else { + // No 'search item matches found' - raise a 404 so that the exception handling will take over to create the folder + throw new OneDriveException(404, "Name not found via search"); + } + } else { + // No 'search item matches found' - raise a 404 so that the exception handling will take over to create the folder + throw new OneDriveException(404, "Name not found via search"); + } + } + } catch (OneDriveException exception) { + if (exception.httpStatusCode == 404) { + // This is a good error - it means that the directory to create 100% does not exist online + // The directory was not found on the drive id we queried + log.vlog("The requested directory to create was not found on OneDrive - creating remote directory: ", thisNewPathToCreate); + + // Build up the create directory request + JSONValue createDirectoryOnlineAPIResponse; + JSONValue newDriveItem = [ + "name": JSONValue(baseName(thisNewPathToCreate)), + "folder": parseJSON("{}") + ]; + + // Submit the creation request + // Fix for https://github.com/skilion/onedrive/issues/356 + if (!dryRun) { + try { + // Attempt to create a new folder on the configured parent driveId & parent id + createDirectoryOnlineAPIResponse = createDirectoryOnlineOneDriveApiInstance.createById(parentItem.driveId, parentItem.id, newDriveItem); + // Is the response a valid JSON object - validation checking done in saveItem + writeln("Line: 4093"); + saveItem(createDirectoryOnlineAPIResponse); + // Log that the directory was created + log.log("Successfully created the remote directory ", thisNewPathToCreate, " on OneDrive"); + } catch (OneDriveException exception) { + if (exception.httpStatusCode == 409) { + // OneDrive API returned a 404 (above) to say the directory did not exist + // but when we attempted to create it, OneDrive responded that it now already exists + log.vlog("OneDrive reported that ", thisNewPathToCreate, " already exists .. OneDrive API race condition"); + return; + } else { + // some other error from OneDrive was returned - display what it is + log.error("OneDrive generated an error when creating this path: ", thisNewPathToCreate); + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + return; + } + } + } else { + // Simulate a successful 'directory create' & save it to the dryRun database copy + // The simulated response has to pass 'makeItem' as part of saveItem + auto fakeResponse = createFakeResponse(thisNewPathToCreate); + writeln("Line: 4114"); + saveItem(fakeResponse); + } + + // Shutdown API instance + createDirectoryOnlineOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(createDirectoryOnlineOneDriveApiInstance); + return; + + } else { + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(createDirectoryOnlineOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to create a remote directory on OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + createDirectoryOnline(thisNewPathToCreate); + } else { + // Re-Try + createDirectoryOnline(thisNewPathToCreate); + } + } + } + + // If we get to this point - onlinePathData = createDirectoryOnlineOneDriveApiInstance.getPathDetailsByDriveId(parentItem.driveId, thisNewPathToCreate) generated a 'valid' response .... + // This means that the folder potentially exists online .. which is odd .. as it should not have existed + if (onlinePathData.type() == JSONType.object){ + // A valid object was responded with + if (onlinePathData["name"].str == baseName(thisNewPathToCreate)) { + // OneDrive 'name' matches local path name + if (appConfig.accountType == "business") { + // We are a business account, this existing online folder, could be a Shared Online Folder and is the 'Add shortcut to My files' item + log.vdebug("onlinePathData: ", onlinePathData); + if (isItemRemote(onlinePathData)) { + // The folder is a remote item ... we do not want to create this ... + log.vdebug("Remote Existing Online Folder is most likely a OneDrive Shared Business Folder Link added by 'Add shortcut to My files'"); + log.vdebug("We need to skip this path: ", thisNewPathToCreate); + // Add this path to businessSharedFoldersOnlineToSkip + businessSharedFoldersOnlineToSkip ~= [thisNewPathToCreate]; + // no save to database, no online create + return; + } + } + + log.vlog("The requested directory to create was found on OneDrive - skipping creating the directory: ", thisNewPathToCreate); + // Is the response a valid JSON object - validation checking done in saveItem + writeln("Line: 4178"); + saveItem(onlinePathData); + return; + } else { + // Normally this would throw an error, however we cant use throw new posixException() + string msg = format("POSIX 'case-insensitive match' between '%s' (local) and '%s' (online) which violates the Microsoft OneDrive API namespace convention", baseName(thisNewPathToCreate), onlinePathData["name"].str); + displayPosixErrorMessage(msg); + log.error("ERROR: Requested directory to create has a 'case-insensitive match' to an existing directory on OneDrive online."); + log.error("ERROR: To resolve, rename this local directory: ", buildNormalizedPath(absolutePath(thisNewPathToCreate))); + log.log("Skipping creating this directory online due to 'case-insensitive match': ", thisNewPathToCreate); + // Add this path to posixViolationPaths + posixViolationPaths ~= [thisNewPathToCreate]; + return; + } + } else { + // response is not valid JSON, an error was returned from OneDrive + log.error("ERROR: There was an error performing this operation on OneDrive"); + log.error("ERROR: Increase logging verbosity to assist determining why."); + log.log("Skipping: ", buildNormalizedPath(absolutePath(thisNewPathToCreate))); + return; + } + } + + // Test that the online name actually matches the requested local name + void performPosixTest(string localNameToCheck, string onlineName) { + + // https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file + // Do not assume case sensitivity. For example, consider the names OSCAR, Oscar, and oscar to be the same, + // even though some file systems (such as a POSIX-compliant file system) may consider them as different. + // Note that NTFS supports POSIX semantics for case sensitivity but this is not the default behavior. + if (localNameToCheck != onlineName) { + // POSIX Error + // Local item name has a 'case-insensitive match' to an existing item on OneDrive + throw new posixException(localNameToCheck, onlineName); + } + } + + // Upload new file items as identified + void uploadNewLocalFileItems() { + + // Lets deal with the new local items in a batch process + ulong batchSize = appConfig.concurrentThreads; + ulong batchCount = (newLocalFilesToUploadToOneDrive.length + batchSize - 1) / batchSize; + ulong batchesProcessed = 0; + + foreach (chunk; newLocalFilesToUploadToOneDrive.chunks(batchSize)) { + uploadNewLocalFileItemsInParallel(chunk); + } + } + + // Upload the file batches in parallel + void uploadNewLocalFileItemsInParallel(string[] array) { + + foreach (i, fileToUpload; taskPool.parallel(array)) { + log.vdebug("Upload Thread ", i, " Starting: ", Clock.currTime()); + uploadNewFile(fileToUpload); + log.vdebug("Upload Thread ", i, " Finished: ", Clock.currTime()); + } + } + + // Upload a new file to OneDrive + void uploadNewFile(string fileToUpload) { + + // Debug for the moment + log.vdebug("fileToUpload: ", fileToUpload); + + // These are the details of the item we need to upload + // How much space is remaining on OneDrive + ulong remainingFreeSpaceOnline; + // Did the upload fail? + bool uploadFailed = false; + // Did we skip due to exceeding maximum allowed size? + bool skippedMaxSize = false; + // Did we skip to an exception error? + bool skippedExceptionError = false; + // Is the parent path in the item database? + bool parentPathFoundInDB = false; + // Get this file size + ulong thisFileSize; + // Is there space available online + bool spaceAvailableOnline = false; + + // Check the database for the parent path of fileToUpload + Item parentItem; + // What parent path to use? + string parentPath = dirName(fileToUpload); // will be either . or something else + if (parentPath == "."){ + // Assume this is a new file in the users configured sync_dir root + // Use client defaults + parentItem.id = appConfig.defaultRootId; // Should give something like 12345ABCDE1234A1!101 + parentItem.driveId = appConfig.defaultDriveId; // Should give something like 12345abcde1234a1 + parentPathFoundInDB = true; + } else { + // Query the database using each of the driveId's we are using + foreach (driveId; driveIDsArray) { + // Query the database for this parent path using each driveId + Item dbResponse; + if(itemDB.selectByPath(parentPath, driveId, dbResponse)){ + // parent path was found in the database + parentItem = dbResponse; + parentPathFoundInDB = true; + } + } + } + + // If the parent path was found in the DB, to ensure we are uploading the the right location 'parentItem.driveId' must not be empty + if ((parentPathFoundInDB) && (parentItem.driveId.empty)) { + // switch to using defaultDriveId + log.log("parentItem.driveId is empty - using defaultDriveId for upload API calls"); + parentItem.driveId = appConfig.defaultDriveId; + } + + // Can we read the file - as a permissions issue or actual file corruption will cause a failure + // Resolves: https://github.com/abraunegg/onedrive/issues/113 + if (readLocalFile(fileToUpload)) { + if (parentPathFoundInDB) { + // The local file can be read - so we can read it to attemtp to upload it in this thread + // Get the file size + thisFileSize = getSize(fileToUpload); + // Does this file exceed the maximum filesize for OneDrive + // Resolves: https://github.com/skilion/onedrive/issues/121 , https://github.com/skilion/onedrive/issues/294 , https://github.com/skilion/onedrive/issues/329 + if (thisFileSize <= maxUploadFileSize) { + // Is there enough free space on OneDrive when we started this thread, to upload the file to OneDrive? + remainingFreeSpaceOnline = getRemainingFreeSpace(parentItem.driveId); + log.vdebug("Current Available Space Online (Upload Target Drive ID): ", (remainingFreeSpaceOnline / 1024 / 1024), " MB"); + + // When we compare the space online to the total we are trying to upload - is there space online? + ulong calculatedSpaceOnlinePostUpload = remainingFreeSpaceOnline - thisFileSize; + + // If 'personal' accounts, if driveId == defaultDriveId, then we will have data - appConfig.quotaAvailable will be updated + // If 'personal' accounts, if driveId != defaultDriveId, then we will not have quota data - appConfig.quotaRestricted will be set as true + // If 'business' accounts, if driveId == defaultDriveId, then we will have data + // If 'business' accounts, if driveId != defaultDriveId, then we will have data, but it will be a 0 value - appConfig.quotaRestricted will be set as true + + if (remainingFreeSpaceOnline > totalDataToUpload) { + // Space available + spaceAvailableOnline = true; + } else { + // we need to look more granular + // What was the latest getRemainingFreeSpace() value? + if (appConfig.quotaAvailable) { + // Our query told us we have free space online .. if we upload this file, will we exceed space online - thus upload will fail during upload? + if (calculatedSpaceOnlinePostUpload > 0) { + // Based on this thread action, we beleive that there is space available online to upload - proceed + spaceAvailableOnline = true; + } + } + } + + // Is quota being restricted? + if (appConfig.quotaRestricted) { + // If the upload target drive is not our drive id, then it is a shared folder .. we need to print a space warning message + if (parentItem.driveId != appConfig.defaultDriveId) { + // Different message depending on account type + if (appConfig.accountType == "personal") { + log.vlog("WARNING: Shared Folder OneDrive quota information is being restricted or providing a zero value. Space available online cannot be guaranteed."); + } else { + log.vlog("WARNING: Shared Folder OneDrive quota information is being restricted or providing a zero value. Please fix by speaking to your OneDrive / Office 365 Administrator."); + } + } else { + if (appConfig.accountType == "personal") { + log.vlog("WARNING: OneDrive quota information is being restricted or providing a zero value. Space available online cannot be guaranteed."); + } else { + log.vlog("WARNING: OneDrive quota information is being restricted or providing a zero value. Please fix by speaking to your OneDrive / Office 365 Administrator."); + } + } + // Space available online is being restricted - so we have no way to really know if there is space available online + spaceAvailableOnline = true; + } + + // Do we have space available or is space available being restricted (so we make the blind assumption that there is space available) + if (spaceAvailableOnline) { + // We need to check that this new local file does not exist on OneDrive + + // Create a new API Instance for this thread and initialise it + OneDriveApi checkFileOneDriveApiInstance; + checkFileOneDriveApiInstance = new OneDriveApi(appConfig); + checkFileOneDriveApiInstance.initialise(); + + JSONValue fileDetailsFromOneDrive; + + // https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file + // Do not assume case sensitivity. For example, consider the names OSCAR, Oscar, and oscar to be the same, + // even though some file systems (such as a POSIX-compliant file systems that Linux use) may consider them as different. + // Note that NTFS supports POSIX semantics for case sensitivity but this is not the default behavior, OneDrive does not use this. + + // In order to upload this file - this query HAS to respond as a 404 - Not Found + + // Does this 'file' already exist on OneDrive? + try { + fileDetailsFromOneDrive = checkFileOneDriveApiInstance.getPathDetailsByDriveId(parentItem.driveId, fileToUpload); + // Portable Operating System Interface (POSIX) testing of JSON response from OneDrive API + performPosixTest(baseName(fileToUpload), fileDetailsFromOneDrive["name"].str); + } catch (OneDriveException exception) { + // If we get a 404 .. the file is not online .. this is what we want .. file does not exist online + if (exception.httpStatusCode == 404) { + // The file has been checked, client side filtering checked, does not exist online - we need to upload it + log.vdebug("fileDetailsFromOneDrive = checkFileOneDriveApiInstance.getPathDetailsByDriveId(parentItem.driveId, fileToUpload); generated a 404 - file does not exist online - must upload it"); + uploadFailed = performNewFileUpload(parentItem, fileToUpload, thisFileSize); + } else { + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(checkFileOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to validate file details on OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + uploadNewFile(fileToUpload); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } + + } + } catch (posixException e) { + displayPosixErrorMessage(e.msg); + uploadFailed = true; + } + + // No 404 or otherwise, meaning that the file already exists online and passes the POSIX test ... + log.vdebug("fileDetailsFromOneDrive after exist online check: ", fileDetailsFromOneDrive); + // Does the data from online match our local file? + if (performUploadIntegrityValidationChecks(fileDetailsFromOneDrive, fileToUpload, thisFileSize)) { + // yes, save the data + writeln("Line: 4421"); + saveItem(fileDetailsFromOneDrive); + } + + // Operations in this thread are done / complete - either upload was done or it failed + checkFileOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(checkFileOneDriveApiInstance); + } else { + // skip file upload - insufficent space to upload + log.log("Skipping uploading this new file as it exceeds the available free space on OneDrive: ", fileToUpload); + uploadFailed = true; + } + } else { + // Skip file upload - too large + log.log("Skipping uploading this new file as it exceeds the maximum size allowed by OneDrive: ", fileToUpload); + uploadFailed = true; + } + } else { + // why was the parent path not in the database? + if (canFind(posixViolationPaths, parentPath)) { + log.error("ERROR: POSIX 'case-insensitive match' for the parent path which violates the Microsoft OneDrive API namespace convention."); + } else { + log.error("ERROR: Parent path is not in the database or online."); + } + log.error("ERROR: Unable to upload this file: ", fileToUpload); + uploadFailed = true; + } + } else { + // Unable to read local file + log.log("Skipping uploading this file as it cannot be read (file permissions or file corruption): ", fileToUpload); + uploadFailed = true; + } + + // Upload success or failure? + if (uploadFailed) { + // Need to add this to fileUploadFailures to capture at the end + fileUploadFailures ~= fileToUpload; + } + } + + // Perform the actual upload to OneDrive + bool performNewFileUpload(Item parentItem, string fileToUpload, ulong thisFileSize) { + + // Assume that by default the upload fails + bool uploadFailed = true; + + // OneDrive API Upload Response + JSONValue uploadResponse; + + // Create the OneDriveAPI Upload Instance + OneDriveApi uploadFileOneDriveApiInstance; + uploadFileOneDriveApiInstance = new OneDriveApi(appConfig); + uploadFileOneDriveApiInstance.initialise(); + + // Calculate upload speed + auto uploadStartTime = Clock.currTime(); + + // Is this a dry-run scenario? + if (!dryRun) { + // Not a dry-run situation + // Do we use simpleUpload or create an upload session? + bool useSimpleUpload = false; + if (thisFileSize <= sessionThresholdFileSize) { + useSimpleUpload = true; + } + + // We can only upload zero size files via simpleFileUpload regardless of account type + // Reference: https://github.com/OneDrive/onedrive-api-docs/issues/53 + // Additionally, only where file size is < 4MB should be uploaded by simpleUpload - everything else should use a session to upload + + if ((thisFileSize == 0) || (useSimpleUpload)) { + try { + // Attempt to upload the zero byte file using simpleUpload for all account types + uploadResponse = uploadFileOneDriveApiInstance.simpleUpload(fileToUpload, parentItem.driveId, parentItem.id, baseName(fileToUpload)); + uploadFailed = false; + log.log("Uploading new file ", fileToUpload, " ... done."); + // Shutdown the API + uploadFileOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(uploadFileOneDriveApiInstance); + } catch (OneDriveException exception) { + // An error was responded with - what was it + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to upload a new file to OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + performNewFileUpload(parentItem, fileToUpload, thisFileSize); + // Return upload status + return uploadFailed; + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + log.log("Uploading new file ", fileToUpload, " ... failed."); + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } + + } catch (FileException e) { + // display the error message + log.log("Uploading new file ", fileToUpload, " ... failed."); + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } else { + // Session Upload for this criteria: + // - Personal Account and file size > 4MB + // - All Business | Office365 | SharePoint files > 0 bytes + JSONValue uploadSessionData; + // As this is a unique thread, the sessionFilePath for where we save the data needs to be unique + // The best way to do this is calculate the CRC32 of the file, and use this as the suffix of the session file we save + string threadUploadSessionFilePath = appConfig.uploadSessionFilePath ~ "." ~ computeCRC32(fileToUpload); + + // Attempt to upload the > 4MB file using an upload session for all account types + try { + // Create the Upload Session + uploadSessionData = createSessionFileUpload(uploadFileOneDriveApiInstance, fileToUpload, parentItem.driveId, parentItem.id, baseName(fileToUpload), null, threadUploadSessionFilePath); + } catch (OneDriveException exception) { + // An error was responded with - what was it + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to create an upload session on OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + performNewFileUpload(parentItem, fileToUpload, thisFileSize); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + log.log("Uploading new file ", fileToUpload, " ... failed."); + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } + + } catch (FileException e) { + // display the error message + log.log("Uploading new file ", fileToUpload, " ... failed."); + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + + // Do we have a valid session URL that we can use ? + if (uploadSessionData.type() == JSONType.object) { + // This is a valid JSON object + bool sessionDataValid = true; + + // Validate that we have the following items which we need + if (!hasUploadURL(uploadSessionData)) { + sessionDataValid = false; + log.vdebug("Session data missing 'uploadUrl'"); + } + + if (!hasNextExpectedRanges(uploadSessionData)) { + sessionDataValid = false; + log.vdebug("Session data missing 'nextExpectedRanges'"); + } + + if (!hasLocalPath(uploadSessionData)) { + sessionDataValid = false; + log.vdebug("Session data missing 'localPath'"); + } + + if (sessionDataValid) { + // We have a valid Upload Session Data we can use + + try { + // Try and perform the upload session + uploadResponse = performSessionFileUpload(uploadFileOneDriveApiInstance, thisFileSize, uploadSessionData, threadUploadSessionFilePath); + + if (uploadResponse.type() == JSONType.object) { + uploadFailed = false; + log.log("Uploading new file ", fileToUpload, " ... done."); + } else { + log.log("Uploading new file ", fileToUpload, " ... failed."); + uploadFailed = true; + } + } catch (OneDriveException exception) { + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to upload a new file via a session to OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + performNewFileUpload(parentItem, fileToUpload, thisFileSize); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + log.log("Uploading new file ", fileToUpload, " ... failed."); + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } + + } + } else { + // No Upload URL or nextExpectedRanges or localPath .. not a valid JSON we can use + log.vlog("Session data is missing required elements to perform a session upload."); + log.log("Uploading new file ", fileToUpload, " ... failed."); + } + } else { + // Create session Upload URL failed + log.log("Uploading new file ", fileToUpload, " ... failed."); + } + } + } else { + // We are in a --dry-run scenario + uploadResponse = createFakeResponse(fileToUpload); + uploadFailed = false; + log.logAndNotify("Uploading new file ", fileToUpload, " ... done."); + } + + // Upload has finished + auto uploadFinishTime = Clock.currTime(); + // If no upload failure, calculate metrics, perform integrity validation + if (!uploadFailed) { + // Upload did not fail ... + auto uploadDuration = uploadFinishTime - uploadStartTime; + log.vdebug("File Size: ", thisFileSize, " Bytes"); + log.vdebug("Upload Duration: ", (uploadDuration.total!"msecs"/1e3), " Seconds"); + auto uploadSpeed = (thisFileSize / (uploadDuration.total!"msecs"/1e3)/ 1024 / 1024); + log.vdebug("Upload Speed: ", uploadSpeed, " Mbps (approx)"); + + // OK as the upload did not fail, we need to save the response from OneDrive, but it has to be a valid JSON response + if (uploadResponse.type() == JSONType.object) { + // check if the path still exists locally before we try to set the file times online - as short lived files, whilst we uploaded it - it may not exist locally aready + if (exists(fileToUpload)) { + if (!dryRun) { + // Check the integrity of the uploaded file, if the local file still exists + performUploadIntegrityValidationChecks(uploadResponse, fileToUpload, thisFileSize); + + // Update the file modified time on OneDrive and save item details to database + // Update the item's metadata on OneDrive + SysTime mtime = timeLastModified(fileToUpload).toUTC(); + mtime.fracSecs = Duration.zero; + string newFileId = uploadResponse["id"].str; + string newFileETag = uploadResponse["eTag"].str; + // Attempt to update the online date time stamp based on our local data + uploadLastModifiedTime(parentItem.driveId, newFileId, mtime, newFileETag); + } + } else { + // will be removed in different event! + log.log("File disappeared locally after upload: ", fileToUpload); + } + } else { + // Log that an invalid JSON object was returned + log.vdebug("uploadFileOneDriveApiInstance.simpleUpload or session.upload call returned an invalid JSON Object from the OneDrive API"); + } + } + + // Return upload status + return uploadFailed; + } + + // Create the OneDrive Upload Session + JSONValue createSessionFileUpload(OneDriveApi activeOneDriveApiInstance, string fileToUpload, string parentDriveId, string parentId, string filename, string eTag, string threadUploadSessionFilePath) { + + // Upload file via a OneDrive API session + JSONValue uploadSession; + + // Calculate modification time + SysTime localFileLastModifiedTime = timeLastModified(fileToUpload).toUTC(); + localFileLastModifiedTime.fracSecs = Duration.zero; + + // Construct the fileSystemInfo JSON component needed to create the Upload Session + JSONValue fileSystemInfo = [ + "item": JSONValue([ + "@microsoft.graph.conflictBehavior": JSONValue("replace"), + "fileSystemInfo": JSONValue([ + "lastModifiedDateTime": localFileLastModifiedTime.toISOExtString() + ]) + ]) + ]; + + // Try to create the upload session for this file + uploadSession = activeOneDriveApiInstance.createUploadSession(parentDriveId, parentId, filename, eTag, fileSystemInfo); + + if (uploadSession.type() == JSONType.object) { + // a valid session object was created + if ("uploadUrl" in uploadSession) { + // Add the file path we are uploading to this JSON Session Data + uploadSession["localPath"] = fileToUpload; + // Save this session + saveSessionFile(threadUploadSessionFilePath, uploadSession); + } + } else { + // no valid session was created + log.vlog("Creation of OneDrive API Upload Session failed."); + // return upload() will return a JSONValue response, create an empty JSONValue response to return + uploadSession = null; + } + // Return the JSON + return uploadSession; + } + + // Save the session upload data + void saveSessionFile(string threadUploadSessionFilePath, JSONValue uploadSessionData) { + + try { + std.file.write(threadUploadSessionFilePath, uploadSessionData.toString()); + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } + + // Perform the upload of file via the Upload Session that was created + JSONValue performSessionFileUpload(OneDriveApi activeOneDriveApiInstance, ulong thisFileSize, JSONValue uploadSessionData, string threadUploadSessionFilePath) { + + // Response for upload + JSONValue uploadResponse; + + // Session JSON needs to contain valid elements + // Get the offset details + ulong fragmentSize = 10 * 2^^20; // 10 MiB + ulong fragmentCount = 0; + ulong fragSize = 0; + ulong offset = uploadSessionData["nextExpectedRanges"][0].str.splitter('-').front.to!ulong; + size_t iteration = (roundTo!int(double(thisFileSize)/double(fragmentSize)))+1; + Progress p = new Progress(iteration); + p.title = "Uploading"; + + // Initialise the download bar at 0% + p.next(); + + // Start the session upload using the active API instance for this thread + while (true) { + fragmentCount++; + log.vdebugNewLine("Fragment: ", fragmentCount, " of ", iteration); + p.next(); + log.vdebugNewLine("fragmentSize: ", fragmentSize, "offset: ", offset, " thisFileSize: ", thisFileSize ); + fragSize = fragmentSize < thisFileSize - offset ? fragmentSize : thisFileSize - offset; + log.vdebugNewLine("Using fragSize: ", fragSize); + + // fragSize must not be a negative value + if (fragSize < 0) { + // Session upload will fail + // not a JSON object - fragment upload failed + log.vlog("File upload session failed - invalid calculation of fragment size"); + if (exists(threadUploadSessionFilePath)) { + remove(threadUploadSessionFilePath); + } + // set uploadResponse to null as error + uploadResponse = null; + return uploadResponse; + } + + // If the resume upload fails, we need to check for a return code here + try { + uploadResponse = activeOneDriveApiInstance.uploadFragment( + uploadSessionData["uploadUrl"].str, + uploadSessionData["localPath"].str, + offset, + fragSize, + thisFileSize + ); + } catch (OneDriveException exception) { + // if a 100 uploadResponse is generated, continue + if (exception.httpStatusCode == 100) { + continue; + } + // There was an error uploadResponse from OneDrive when uploading the file fragment + + // Handle transient errors: + // 408 - Request Time Out + // 429 - Too Many Requests + // 503 - Service Unavailable + // 504 - Gateway Timeout + + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle 'HTTP request returned status code 429 (Too Many Requests)' first + log.vdebug("Fragment upload failed - received throttle request uploadResponse from OneDrive"); + if (exception.httpStatusCode == 429) { + auto retryAfterValue = activeOneDriveApiInstance.getRetryAfterValue(); + log.vdebug("Using Retry-After Value = ", retryAfterValue); + // Sleep thread as per request + log.log("\nThread sleeping due to 'HTTP request returned status code 429' - The request has been throttled"); + log.log("Sleeping for ", retryAfterValue, " seconds"); + Thread.sleep(dur!"seconds"(retryAfterValue)); + log.log("Retrying fragment upload"); + } else { + // Handle 408, 503 and 504 + auto errorArray = splitLines(exception.msg); + auto retryAfterValue = 30; + log.log("\nThread sleeping due to '", errorArray[0], "' - retrying applicable request in 30 seconds"); + log.log("Sleeping for ", retryAfterValue, " seconds"); + Thread.sleep(dur!"seconds"(retryAfterValue)); + log.log("Retrying fragment upload"); + } + } else { + // insert a new line as well, so that the below error is inserted on the console in the right location + log.vlog("\nFragment upload failed - received an exception response from OneDrive API"); + // display what the error is + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + // retry fragment upload in case error is transient + log.vlog("Retrying fragment upload"); + } + + try { + uploadResponse = activeOneDriveApiInstance.uploadFragment( + uploadSessionData["uploadUrl"].str, + uploadSessionData["localPath"].str, + offset, + fragSize, + thisFileSize + ); + } catch (OneDriveException e) { + // OneDrive threw another error on retry + log.vlog("Retry to upload fragment failed"); + // display what the error is + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + // set uploadResponse to null as the fragment upload was in error twice + uploadResponse = null; + } catch (std.exception.ErrnoException e) { + // There was a file system error - display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + return uploadResponse; + } + } + + // was the fragment uploaded without issue? + if (uploadResponse.type() == JSONType.object){ + offset += fragmentSize; + if (offset >= thisFileSize) break; + // update the uploadSessionData details + uploadSessionData["expirationDateTime"] = uploadResponse["expirationDateTime"]; + uploadSessionData["nextExpectedRanges"] = uploadResponse["nextExpectedRanges"]; + saveSessionFile(threadUploadSessionFilePath, uploadSessionData); + } else { + // not a JSON object - fragment upload failed + log.vlog("File upload session failed - invalid response from OneDrive API"); + if (exists(threadUploadSessionFilePath)) { + remove(threadUploadSessionFilePath); + } + // set uploadResponse to null as error + uploadResponse = null; + return uploadResponse; + } + } + + // upload complete + p.next(); + writeln(); + if (exists(threadUploadSessionFilePath)) { + remove(threadUploadSessionFilePath); + } + + // Return the session upload response + return uploadResponse; + } + + // Delete an item on OneDrive + void uploadDeletedItem(Item itemToDelete, string path) { + + // Are we in a situation where we HAVE to keep the data online - do not delete the remote object + if (noRemoteDelete) { + if ((itemToDelete.type == ItemType.dir)) { + // Do not process remote directory delete + log.vlog("Skipping remote directory delete as --upload-only & --no-remote-delete configured"); + } else { + // Do not process remote file delete + log.vlog("Skipping remote file delete as --upload-only & --no-remote-delete configured"); + } + } else { + // Process the delete - delete the object online + log.log("Deleting item from OneDrive: ", path); + bool flagAsBigDelete = false; + + Item[] children; + ulong itemsToDelete; + + if ((itemToDelete.type == ItemType.dir)) { + // Query the database - how many objects will this remove? + children = getChildren(itemToDelete.driveId, itemToDelete.id); + // Count the returned items + the original item (1) + itemsToDelete = count(children) + 1; + log.vdebug("Number of items online to delete: ", itemsToDelete); + } else { + itemsToDelete = 1; + } + + // A local delete of a file|folder when using --monitor will issue a inotify event, which will trigger the local & remote data immediately be deleted + // The user may also be --sync process, so we are checking if something was deleted between application use + if (itemsToDelete >= appConfig.getValueLong("classify_as_big_delete")) { + // A big delete has been detected + flagAsBigDelete = true; + if (!appConfig.getValueBool("force")) { + log.error("ERROR: An attempt to remove a large volume of data from OneDrive has been detected. Exiting client to preserve data on OneDrive"); + log.error("ERROR: To delete a large volume of data use --force or increase the config value 'classify_as_big_delete' to a larger value"); + // Must exit here to preserve data on online + exit(-1); + } + } + + // Are we in a --dry-run scenario? + if (!dryRun) { + // We are not in a dry run scenario + log.vdebug("itemToDelete: ", itemToDelete); + + // Create new OneDrive API Instance + OneDriveApi uploadDeletedItemOneDriveApiInstance; + uploadDeletedItemOneDriveApiInstance = new OneDriveApi(appConfig); + uploadDeletedItemOneDriveApiInstance.initialise(); + + // what item are we trying to delete? + log.vdebug("Attempting to delete this single item id: ", itemToDelete.id, " from drive: ", itemToDelete.driveId); + try { + // perform the delete via the default OneDrive API instance + uploadDeletedItemOneDriveApiInstance.deleteById(itemToDelete.driveId, itemToDelete.id); + // Shutdown API + uploadDeletedItemOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(uploadDeletedItemOneDriveApiInstance); + } catch (OneDriveException e) { + if (e.httpStatusCode == 404) { + // item.id, item.eTag could not be found on the specified driveId + log.vlog("OneDrive reported: The resource could not be found to be deleted."); + } + } + + // Delete the reference in the local database + itemDB.deleteById(itemToDelete.driveId, itemToDelete.id); + if (itemToDelete.remoteId != null) { + // If the item is a remote item, delete the reference in the local database + itemDB.deleteById(itemToDelete.remoteDriveId, itemToDelete.remoteId); + } + + } else { + // log that this is a dry-run activity + log.log("dry run - no delete activity"); + } + } + } + + // Get the children of an item id from the database + Item[] getChildren(string driveId, string id) { + + Item[] children; + children ~= itemDB.selectChildren(driveId, id); + foreach (Item child; children) { + if (child.type != ItemType.file) { + // recursively get the children of this child + children ~= getChildren(child.driveId, child.id); + } + } + return children; + } + + // Perform a 'reverse' delete of all child objects on OneDrive + void performReverseDeletionOfOneDriveItems(Item[] children, Item itemToDelete) { + + // Log what is happening + log.vdebug("Attempting a reverse delete of all child objects from OneDrive"); + + // Create a new API Instance for this thread and initialise it + OneDriveApi performReverseDeletionOneDriveApiInstance; + performReverseDeletionOneDriveApiInstance = new OneDriveApi(appConfig); + performReverseDeletionOneDriveApiInstance.initialise(); + + foreach_reverse (Item child; children) { + // Log the action + log.vdebug("Attempting to delete this child item id: ", child.id, " from drive: ", child.driveId); + // perform the delete via the default OneDrive API instance + performReverseDeletionOneDriveApiInstance.deleteById(child.driveId, child.id, child.eTag); + // delete the child reference in the local database + itemDB.deleteById(child.driveId, child.id); + } + // Log the action + log.vdebug("Attempting to delete this parent item id: ", itemToDelete.id, " from drive: ", itemToDelete.driveId); + // Perform the delete via the default OneDrive API instance + performReverseDeletionOneDriveApiInstance.deleteById(itemToDelete.driveId, itemToDelete.id, itemToDelete.eTag); + // Shutdown API instance + performReverseDeletionOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(performReverseDeletionOneDriveApiInstance); + } + + // Create a fake OneDrive response suitable for use with saveItem + JSONValue createFakeResponse(const(string) path) { + + import std.digest.sha; + // Generate a simulated JSON response which can be used + // At a minimum we need: + // 1. eTag + // 2. cTag + // 3. fileSystemInfo + // 4. file or folder. if file, hash of file + // 5. id + // 6. name + // 7. parent reference + + string fakeDriveId = appConfig.defaultDriveId; + string fakeRootId = appConfig.defaultRootId; + SysTime mtime = timeLastModified(path).toUTC(); + + // Need to update the 'fakeDriveId' & 'fakeRootId' with elements from the --dry-run database + // Otherwise some calls to validate objects will fail as the actual driveId being used is invalid + string parentPath = dirName(path); + Item databaseItem; + + if (parentPath != ".") { + // Not a 'root' parent + // For each driveid in the existing driveIDsArray + foreach (searchDriveId; driveIDsArray) { + log.vdebug("FakeResponse: searching database for: ", searchDriveId, " ", parentPath); + if (itemDB.selectByPath(parentPath, searchDriveId, databaseItem)) { + log.vdebug("FakeResponse: Found Database Item: ", databaseItem); + fakeDriveId = databaseItem.driveId; + fakeRootId = databaseItem.id; + } + } + } + + // real id / eTag / cTag are different format for personal / business account + auto sha1 = new SHA1Digest(); + ubyte[] fakedOneDriveItemValues = sha1.digest(path); + + JSONValue fakeResponse; + + if (isDir(path)) { + // path is a directory + fakeResponse = [ + "id": JSONValue(toHexString(fakedOneDriveItemValues)), + "cTag": JSONValue(toHexString(fakedOneDriveItemValues)), + "eTag": JSONValue(toHexString(fakedOneDriveItemValues)), + "fileSystemInfo": JSONValue([ + "createdDateTime": mtime.toISOExtString(), + "lastModifiedDateTime": mtime.toISOExtString() + ]), + "name": JSONValue(baseName(path)), + "parentReference": JSONValue([ + "driveId": JSONValue(fakeDriveId), + "driveType": JSONValue(appConfig.accountType), + "id": JSONValue(fakeRootId) + ]), + "folder": JSONValue("") + ]; + } else { + // path is a file + // compute file hash - both business and personal responses use quickXorHash + string quickXorHash = computeQuickXorHash(path); + + fakeResponse = [ + "id": JSONValue(toHexString(fakedOneDriveItemValues)), + "cTag": JSONValue(toHexString(fakedOneDriveItemValues)), + "eTag": JSONValue(toHexString(fakedOneDriveItemValues)), + "fileSystemInfo": JSONValue([ + "createdDateTime": mtime.toISOExtString(), + "lastModifiedDateTime": mtime.toISOExtString() + ]), + "name": JSONValue(baseName(path)), + "parentReference": JSONValue([ + "driveId": JSONValue(fakeDriveId), + "driveType": JSONValue(appConfig.accountType), + "id": JSONValue(fakeRootId) + ]), + "file": JSONValue([ + "hashes":JSONValue([ + "quickXorHash": JSONValue(quickXorHash) + ]) + + ]) + ]; + } + + log.vdebug("Generated Fake OneDrive Response: ", fakeResponse); + return fakeResponse; + } + + // Save JSON item details into the item database + void saveItem(JSONValue jsonItem) { + + // jsonItem has to be a valid object + if (jsonItem.type() == JSONType.object) { + // Check if the response JSON has an 'id', otherwise makeItem() fails with 'Key not found: id' + if (hasId(jsonItem)) { + // Are we in a --upload-only & --remove-source-files scenario? + // We do not want to add the item to the database in this situation as there is no local reference to the file post file deletion + // If the item is a directory, we need to add this to the DB, if this is a file, we dont add this, the parent path is not in DB, thus any new files in this directory are not added + if ((uploadOnly) && (localDeleteAfterUpload) && (isItemFile(jsonItem))) { + // Log that we skipping adding item to the local DB and the reason why + log.vdebug("Skipping adding to database as --upload-only & --remove-source-files configured"); + } else { + // What is the JSON item we are trying to create a DB record with? + log.vdebug("saveItem - creating DB item from this JSON: ", jsonItem); + // Takes a JSON input and formats to an item which can be used by the database + Item item = makeItem(jsonItem); + + // Is this JSON item a 'root' item? + if ((isItemRoot(jsonItem)) && (item.name == "root")) { + log.vdebug("Updating DB Item object with correct values as this is a 'root' object"); + item.parentId = null; // ensures that this database entry has no parent + // Check for parentReference + if (hasParentReference(jsonItem)) { + // Set the correct item.driveId + log.vdebug("ROOT JSON Item HAS parentReference .... setting item.driveId = jsonItem['parentReference']['driveId'].str"); + item.driveId = jsonItem["parentReference"]["driveId"].str; + } + + // We only should be adding our account 'root' to the database, not shared folder 'root' items + if (item.driveId != appConfig.defaultDriveId) { + // Shared Folder drive 'root' object .. we dont want this item + log.vdebug("NOT adding 'remote root' object to database: ", item); + return; + } + } + + // Add to the local database + log.vdebug("Adding to database: ", item); + itemDB.upsert(item); + + // If we have a remote drive ID, add this to our list of known drive id's + if (!item.remoteDriveId.empty) { + // Keep the driveIDsArray with unique entries only + if (!canFind(driveIDsArray, item.remoteDriveId)) { + // Add this drive id to the array to search with + driveIDsArray ~= item.remoteDriveId; + } + } + } + } else { + // log error + log.error("ERROR: OneDrive response missing required 'id' element"); + log.error("ERROR: ", jsonItem); + } + } else { + // log error + log.error("ERROR: An error was returned from OneDrive and the resulting response is not a valid JSON object"); + log.error("ERROR: Increase logging verbosity to assist determining why."); + } + } + + // Wrapper function for makeDatabaseItem so we can check to ensure that the item has the required hashes + Item makeItem(JSONValue onedriveJSONItem) { + + // Make the DB Item from the JSON data provided + Item newDatabaseItem = makeDatabaseItem(onedriveJSONItem); + + // Is this a 'file' item that has not been deleted? Deleted items have no hash + if ((newDatabaseItem.type == ItemType.file) && (!isItemDeleted(onedriveJSONItem))) { + // Does this item have a file size attribute? + if (hasFileSize(onedriveJSONItem)) { + // Is the file size greater than 0? + if (onedriveJSONItem["size"].integer > 0) { + // Does the DB item have any hashes as per the API provided JSON data? + if ((newDatabaseItem.quickXorHash.empty) && (newDatabaseItem.sha256Hash.empty)) { + // Odd .. there is no hash for this item .. why is that? + // Is there a 'file' JSON element? + if ("file" in onedriveJSONItem) { + // Microsoft OneDrive OneNote objects will report as files but have 'application/msonenote' and 'application/octet-stream' as mime types + if ((isMicrosoftOneNoteMimeType1(onedriveJSONItem)) || (isMicrosoftOneNoteMimeType2(onedriveJSONItem))) { + // Debug log output that this is a potential OneNote object + log.vdebug("This item is potentially an associated Microsoft OneNote Object Item"); + } else { + // Not a Microsoft OneNote Mime Type Object .. + string apiWarningMessage = "WARNING: OneDrive API inconsistency - this file does not have any hash: "; + // This is computationally expensive .. but we are only doing this if there are no hashses provided + bool parentInDatabase = itemDB.idInLocalDatabase(newDatabaseItem.driveId, newDatabaseItem.parentId); + // Is the parent id in the database? + if (parentInDatabase) { + // This is again computationally expensive .. calculate this item path to advise the user the actual path of this item that has no hash + string newItemPath = computeItemPath(newDatabaseItem.driveId, newDatabaseItem.parentId) ~ "/" ~ newDatabaseItem.name; + log.log(apiWarningMessage, newItemPath); + } else { + // Parent is not in the database .. why? + // Check if the parent item had been skipped .. + if (skippedItems.find(newDatabaseItem.parentId).length != 0) { + log.vdebug(apiWarningMessage, "newDatabaseItem.parentId listed within skippedItems"); + } else { + // Use the item ID .. there is no other reference available, parent is not being skipped, so we should have been able to calculate this - but we could not + log.log(apiWarningMessage, newDatabaseItem.id); + } + } + } + } + } + } else { + // zero file size + log.vdebug("This item file is zero size - potentially no hash provided by the OneDrive API"); + } + } + } + + // Return the new database item + return newDatabaseItem; + } + + // Print the fileDownloadFailures and fileUploadFailures arrays if they are not empty + void displaySyncFailures() { + + // Were there any file download failures? + if (!fileDownloadFailures.empty) { + // There are download failures ... + log.log("\nFailed items to download from OneDrive: ", fileDownloadFailures.length); + foreach(failedFileToDownload; fileDownloadFailures) { + // List the detail of the item that failed to download + log.logAndNotify("Failed to download: ", failedFileToDownload); + + // Is this failed item in the DB? It should not be .. + Item downloadDBItem; + // Need to check all driveid's we know about, not just the defaultDriveId + foreach (searchDriveId; driveIDsArray) { + if (itemDB.selectByPath(failedFileToDownload, searchDriveId, downloadDBItem)) { + // item was found in the DB + log.error("ERROR: Failed Download Path found in database, must delete this item from the database .. it should not be in there if it failed to download"); + // Process the database entry removal. In a --dry-run scenario, this is being done against a DB copy + itemDB.deleteById(downloadDBItem.driveId, downloadDBItem.id); + if (downloadDBItem.remoteDriveId != null) { + // delete the linked remote folder + itemDB.deleteById(downloadDBItem.remoteDriveId, downloadDBItem.remoteId); + } + } + } + } + // Set the flag + syncFailures = true; + } + + // Were there any file upload failures? + if (!fileUploadFailures.empty) { + // There are download failures ... + log.log("\nFailed items to upload to OneDrive: ", fileUploadFailures.length); + foreach(failedFileToUpload; fileUploadFailures) { + // List the path of the item that failed to upload + log.logAndNotify("Failed to upload: ", failedFileToUpload); + + // Is this failed item in the DB? It should not be .. + Item uploadDBItem; + // Need to check all driveid's we know about, not just the defaultDriveId + foreach (searchDriveId; driveIDsArray) { + if (itemDB.selectByPath(failedFileToUpload, searchDriveId, uploadDBItem)) { + // item was found in the DB + log.error("ERROR: Failed Upload Path found in database, must delete this item from the database .. it should not be in there if it failed to upload"); + // Process the database entry removal. In a --dry-run scenario, this is being done against a DB copy + itemDB.deleteById(uploadDBItem.driveId, uploadDBItem.id); + if (uploadDBItem.remoteDriveId != null) { + // delete the linked remote folder + itemDB.deleteById(uploadDBItem.remoteDriveId, uploadDBItem.remoteId); + } + } + } + } + // Set the flag + syncFailures = true; + } + } + + // Generate a /delta compatible response - for use when we cant actually use /delta + // This is required when the application is configured to use National Azure AD deployments as these do not support /delta queries + // The same technique can also be used when we are using --single-directory. The parent objects up to the single directory target can be added, + // then once the target of the --single-directory request is hit, all of the children of that path can be queried, giving a much more focused + // JSON response which can then be processed, negating the need to continuously traverse the tree and 'exclude' items + JSONValue generateDeltaResponse(string pathToQuery = null) { + + // JSON value which will be responded with + JSONValue selfGeneratedDeltaResponse; + + // Function variables + Item searchItem; + JSONValue rootData; + JSONValue driveData; + JSONValue pathData; + JSONValue topLevelChildren; + JSONValue[] childrenData; + string nextLink; + + // Was a path to query passed in? + if (pathToQuery.empty) { + // Will query for the 'root' + pathToQuery = "."; + } + + // Create new OneDrive API Instance + OneDriveApi generateDeltaResponseOneDriveApiInstance; + generateDeltaResponseOneDriveApiInstance = new OneDriveApi(appConfig); + generateDeltaResponseOneDriveApiInstance.initialise(); + + if (!singleDirectoryScope) { + // In a --resync scenario, there is no DB data to query, so we have to query the OneDrive API here to get relevant details + try { + // Query the OneDrive API + pathData = generateDeltaResponseOneDriveApiInstance.getPathDetails(pathToQuery); + // Is the path on OneDrive local or remote to our account drive id? + if (isItemRemote(pathData)) { + // The path we are seeking is remote to our account drive id + searchItem.driveId = pathData["remoteItem"]["parentReference"]["driveId"].str; + searchItem.id = pathData["remoteItem"]["id"].str; + } else { + // The path we are seeking is local to our account drive id + searchItem.driveId = pathData["parentReference"]["driveId"].str; + searchItem.id = pathData["id"].str; + } + } catch (OneDriveException e) { + // Display error message + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + // Must exit here + generateDeltaResponseOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(generateDeltaResponseOneDriveApiInstance); + exit(-1); + } + } else { + // When setSingleDirectoryScope() was called, the following were set to the correct items, even if the path was remote: + // - singleDirectoryScopeDriveId + // - singleDirectoryScopeItemId + // Reuse these prior set values + searchItem.driveId = singleDirectoryScopeDriveId; + searchItem.id = singleDirectoryScopeItemId; + } + + // Before we get any data from the OneDrive API, flag any child object in the database as out-of-sync for this driveId & and object id + // Downgrade ONLY files associated with this driveId and idToQuery + log.vdebug("Downgrading all children for this searchItem.driveId (" ~ searchItem.driveId ~ ") and searchItem.id (" ~ searchItem.id ~ ") to an out-of-sync state"); + auto drivePathChildren = getChildren(searchItem.driveId, searchItem.id); + if (count(drivePathChildren) > 0) { + // Children to process and flag as out-of-sync + foreach (drivePathChild; drivePathChildren) { + // Flag any object in the database as out-of-sync for this driveId & and object id + log.vdebug("Downgrading item as out-of-sync: ", drivePathChild.id); + itemDB.downgradeSyncStatusFlag(drivePathChild.driveId, drivePathChild.id); + } + } + + // Get drive details for the provided driveId + try { + driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id); + } catch (OneDriveException exception) { + log.vdebug("driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id) generated a OneDriveException"); + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to query path details on OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + generateDeltaResponse(pathToQuery); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } + } + + // Was a valid JSON response for 'driveData' provided? + if (driveData.type() == JSONType.object) { + + // Process this initial JSON response + if (!isItemRoot(driveData)) { + // Get root details for the provided driveId + try { + rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId); + } catch (OneDriveException exception) { + log.vdebug("rootData = onedrive.getDriveIdRoot(searchItem.driveId) generated a OneDriveException"); + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to query drive root details on OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.log("Retrying Query: rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId)"); + rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId); + + + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } + + } + // Add driveData JSON data to array + log.vlog("Adding OneDrive root details for processing"); + childrenData ~= rootData; + } + + // Add driveData JSON data to array + log.vlog("Adding OneDrive folder details for processing"); + childrenData ~= driveData; + + } else { + // driveData is an invalid JSON object + writeln("CODING TO DO: The query of OneDrive API to getPathDetailsById generated an invalid JSON response - thus we cant build our own /delta simulated response ... how to handle?"); + // Must exit here + generateDeltaResponseOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(generateDeltaResponseOneDriveApiInstance); + exit(-1); + } + + // For each child object, query the OneDrive API + for (;;) { + // query top level children + try { + topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink); + } catch (OneDriveException exception) { + // OneDrive threw an error + log.vdebug("------------------------------------------------------------------"); + log.vdebug("Query Error: topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)"); + log.vdebug("driveId: ", searchItem.driveId); + log.vdebug("idToQuery: ", searchItem.id); + log.vdebug("nextLink: ", nextLink); + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)"); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to query OneDrive top level drive children on OneDrive - retrying applicable request in 30 seconds"); + log.vdebug("generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink) previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + //log.vdebug("Retrying Query: generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)"); + //topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink); + + log.vdebug("Retrying Function: ", thisFunctionName); + generateDeltaResponse(pathToQuery); + + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } + } + + // process top level children + log.vlog("Adding ", count(topLevelChildren["value"].array), " OneDrive items for processing from the OneDrive 'root' folder"); + foreach (child; topLevelChildren["value"].array) { + // Check for any Client Side Filtering here ... we should skip querying the OneDrive API for 'folders' that we are going to just process and skip anyway. + // This avoids needless calls to the OneDrive API, and potentially speeds up this process. + if (!checkJSONAgainstClientSideFiltering(child)) { + // add this child to the array of objects + childrenData ~= child; + // is this child a folder? + if (isItemFolder(child)) { + // We have to query this folders children if childCount > 0 + if (child["folder"]["childCount"].integer > 0){ + // This child folder has children + string childIdToQuery = child["id"].str; + string childDriveToQuery = child["parentReference"]["driveId"].str; + auto childParentPath = child["parentReference"]["path"].str.split(":"); + string folderPathToScan = childParentPath[1] ~ "/" ~ child["name"].str; + + string pathForLogging; + // Are we in a --single-directory situation? If we are, the path we are using for logging needs to use the input path as a base + if (singleDirectoryScope) { + pathForLogging = appConfig.getValueString("single_directory") ~ "/" ~ child["name"].str; + } else { + pathForLogging = child["name"].str; + } + + // Query the children of this item + JSONValue[] grandChildrenData = queryForChildren(childDriveToQuery, childIdToQuery, folderPathToScan, pathForLogging); + foreach (grandChild; grandChildrenData.array) { + // add the grandchild to the array + childrenData ~= grandChild; + } + } + } + } + } + // If a collection exceeds the default page size (200 items), the @odata.nextLink property is returned in the response + // to indicate more items are available and provide the request URL for the next page of items. + if ("@odata.nextLink" in topLevelChildren) { + // Update nextLink to next changeSet bundle + log.vdebug("Setting nextLink to (@odata.nextLink): ", nextLink); + nextLink = topLevelChildren["@odata.nextLink"].str; + } else break; + } + + // Craft response from all returned JSON elements + selfGeneratedDeltaResponse = [ + "@odata.context": JSONValue("https://graph.microsoft.com/v1.0/$metadata#Collection(driveItem)"), + "value": JSONValue(childrenData.array) + ]; + + // Shutdown API + generateDeltaResponseOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(generateDeltaResponseOneDriveApiInstance); + + // Return the generated JSON response + return selfGeneratedDeltaResponse; + } + + // Query the OneDrive API for the specified child id for any children objects + JSONValue[] queryForChildren(string driveId, string idToQuery, string childParentPath, string pathForLogging) { + + // function variables + JSONValue thisLevelChildren; + JSONValue[] thisLevelChildrenData; + string nextLink; + + for (;;) { + // query this level children + try { + thisLevelChildren = queryThisLevelChildren(driveId, idToQuery, nextLink); + } catch (OneDriveException exception) { + + writeln("CODING TO DO: EXCEPTION HANDLING NEEDED: thisLevelChildren = queryThisLevelChildren(driveId, idToQuery, nextLink)"); + + } + + // Was a valid JSON response for 'thisLevelChildren' provided? + if (thisLevelChildren.type() == JSONType.object) { + // process this level children + if (!childParentPath.empty) { + // We dont use childParentPath to log, as this poses an information leak risk. + // The full parent path of the child, as per the JSON might be: + // /Level 1/Level 2/Level 3/Child Shared Folder/some folder/another folder + // But 'Child Shared Folder' is what is shared, thus '/Level 1/Level 2/Level 3/' is a potential information leak if logged. + // Plus, the application output now shows accuratly what is being shared - so that is a good thing. + log.vlog("Adding ", count(thisLevelChildren["value"].array), " OneDrive items for processing from ", pathForLogging); + } + foreach (child; thisLevelChildren["value"].array) { + // Check for any Client Side Filtering here ... we should skip querying the OneDrive API for 'folders' that we are going to just process and skip anyway. + // This avoids needless calls to the OneDrive API, and potentially speeds up this process. + if (!checkJSONAgainstClientSideFiltering(child)) { + // add this child to the array of objects + thisLevelChildrenData ~= child; + // is this child a folder? + if (isItemFolder(child)){ + // We have to query this folders children if childCount > 0 + if (child["folder"]["childCount"].integer > 0){ + // This child folder has children + string childIdToQuery = child["id"].str; + string childDriveToQuery = child["parentReference"]["driveId"].str; + auto grandchildParentPath = child["parentReference"]["path"].str.split(":"); + string folderPathToScan = grandchildParentPath[1] ~ "/" ~ child["name"].str; + string newLoggingPath = pathForLogging ~ "/" ~ child["name"].str; + JSONValue[] grandChildrenData = queryForChildren(childDriveToQuery, childIdToQuery, folderPathToScan, newLoggingPath); + foreach (grandChild; grandChildrenData.array) { + // add the grandchild to the array + thisLevelChildrenData ~= grandChild; + } + } + } + } + } + // If a collection exceeds the default page size (200 items), the @odata.nextLink property is returned in the response + // to indicate more items are available and provide the request URL for the next page of items. + if ("@odata.nextLink" in thisLevelChildren) { + // Update nextLink to next changeSet bundle + nextLink = thisLevelChildren["@odata.nextLink"].str; + log.vdebug("Setting nextLink to (@odata.nextLink): ", nextLink); + } else break; + + } else { + // Invalid JSON response when querying this level children + log.vdebug("INVALID JSON response when attempting a retry of parent function - queryForChildren(driveId, idToQuery, childParentPath, pathForLogging)"); + // retry thisLevelChildren = queryThisLevelChildren + log.vdebug("Thread sleeping for an additional 30 seconds"); + Thread.sleep(dur!"seconds"(30)); + log.vdebug("Retry this call thisLevelChildren = queryThisLevelChildren(driveId, idToQuery, nextLink)"); + thisLevelChildren = queryThisLevelChildren(driveId, idToQuery, nextLink); + } + } + + // return response + return thisLevelChildrenData; + } + + // Query the OneDrive API for the child objects for this element + JSONValue queryThisLevelChildren(string driveId, string idToQuery, string nextLink) { + + // function variables + JSONValue thisLevelChildren; + + // Create new OneDrive API Instance + OneDriveApi queryChildrenOneDriveApiInstance; + queryChildrenOneDriveApiInstance = new OneDriveApi(appConfig); + queryChildrenOneDriveApiInstance.initialise(); + + // query children + try { + // attempt API call + log.vdebug("Attempting Query: thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)"); + thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink); + log.vdebug("Query 'thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)' performed successfully"); + queryChildrenOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(queryChildrenOneDriveApiInstance); + } catch (OneDriveException exception) { + // OneDrive threw an error + log.vdebug("------------------------------------------------------------------"); + log.vdebug("Query Error: thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)"); + log.vdebug("driveId: ", driveId); + log.vdebug("idToQuery: ", idToQuery); + log.vdebug("nextLink: ", nextLink); + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(queryChildrenOneDriveApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to query OneDrive drive item children - retrying applicable request in 30 seconds"); + log.vdebug("thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink) previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + //log.vdebug("Retrying Query: thisLevelChildren = queryThisLevelChildren(driveId, idToQuery, nextLink)"); + //thisLevelChildren = queryThisLevelChildren(driveId, idToQuery, nextLink); + log.vdebug("Retrying Function: ", thisFunctionName); + queryThisLevelChildren(driveId, idToQuery, nextLink); + + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } + } + + // return response + return thisLevelChildren; + } + + // Traverses the provided path online, via the OneDrive API, following correct parent driveId and itemId elements across the account + // to find if this full path exists. If this path exists online, the last item in the object path will be returned as a full JSON item. + // + // If the createPathIfMissing = false + no path exists online, a null invalid JSON item will be returned. + // If the createPathIfMissing = true + no path exists online, the requested path will be created in the correct location online. The resulting + // response to the directory creation will then be returned. + // + // This function also ensures that each path in the requested path actually matches the requested element to ensure that the OneDrive API response + // is not falsely matching a 'case insensitive' match to the actual request which is a POSIX compliance issue. + JSONValue queryOneDriveForSpecificPathAndCreateIfMissing(string thisNewPathToSearch, bool createPathIfMissing) { + + // function variables + JSONValue getPathDetailsAPIResponse; + string currentPathTree; + Item parentDetails; + JSONValue topLevelChildren; + string nextLink; + bool directoryFoundOnline = false; + bool posixIssue = false; + + // Create a new API Instance for this thread and initialise it + OneDriveApi queryOneDriveForSpecificPath; + queryOneDriveForSpecificPath = new OneDriveApi(appConfig); + queryOneDriveForSpecificPath.initialise(); + + foreach (thisFolderName; pathSplitter(thisNewPathToSearch)) { + log.vdebug("Testing for the existance online of this folder path: ", thisFolderName); + directoryFoundOnline = false; + + // If this is '.' this is the account root + if (thisFolderName == ".") { + currentPathTree = thisFolderName; + } else { + currentPathTree = currentPathTree ~ "/" ~ thisFolderName; + } + + log.vdebug("Attempting to query OneDrive for this path: ", currentPathTree); + + // What query do we use? + if (thisFolderName == ".") { + // Query the root, set the right details + try { + getPathDetailsAPIResponse = queryOneDriveForSpecificPath.getPathDetails(currentPathTree); + parentDetails = makeItem(getPathDetailsAPIResponse); + writeln("Line: 5779"); + saveItem(getPathDetailsAPIResponse); + directoryFoundOnline = true; + } catch (OneDriveException exception) { + + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(queryOneDriveForSpecificPath); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to query path on OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + queryOneDriveForSpecificPathAndCreateIfMissing(thisNewPathToSearch, createPathIfMissing); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } + } + } else { + // Ensure we have a valid driveId to search here + if (parentDetails.driveId.empty) { + parentDetails.driveId = appConfig.defaultDriveId; + } + + // If the prior JSON 'getPathDetailsAPIResponse' is on this account driveId .. then continue to use getPathDetails + if (parentDetails.driveId == appConfig.defaultDriveId) { + + try { + // Query OneDrive API for this path + getPathDetailsAPIResponse = queryOneDriveForSpecificPath.getPathDetails(currentPathTree); + // Portable Operating System Interface (POSIX) testing of JSON response from OneDrive API + performPosixTest(thisFolderName, getPathDetailsAPIResponse["name"].str); + // No POSIX issue with requested path element + parentDetails = makeItem(getPathDetailsAPIResponse); + writeln("Line: 5832"); + saveItem(getPathDetailsAPIResponse); + directoryFoundOnline = true; + + // Is this JSON a remote object + if (isItemRemote(getPathDetailsAPIResponse)) { + // Remote Directory .. need a DB Tie Item + log.vdebug("Creating a DB TIE for this Shared Folder"); + // New DB Tie Item to bind the 'remote' path to our parent path + Item tieDBItem; + // Set the name + tieDBItem.name = parentDetails.name; + // Set the correct item type + tieDBItem.type = ItemType.dir; + // Set the right elements using the 'remote' of the parent as the 'actual' for this DB Tie + tieDBItem.driveId = parentDetails.remoteDriveId; + tieDBItem.id = parentDetails.remoteId; + // Set the correct mtime + tieDBItem.mtime = parentDetails.mtime; + // Add tie DB record to the local database + log.vdebug("Adding tie DB record to database: ", tieDBItem); + itemDB.upsert(tieDBItem); + // Update parentDetails to use the DB Tie record + parentDetails = tieDBItem; + } + + } catch (OneDriveException exception) { + if (exception.httpStatusCode == 404) { + directoryFoundOnline = false; + } else { + + string thisFunctionName = getFunctionName!({}); + // HTTP request returned status code 408,429,503,504 + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // Handle the 429 + if (exception.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(queryOneDriveForSpecificPath); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry ", thisFunctionName); + } + // re-try the specific changes queries + if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { + // 408 - Request Time Out + // 503 - Service Unavailable + // 504 - Gateway Timeout + // Transient error - try again in 30 seconds + auto errorArray = splitLines(exception.msg); + log.log(errorArray[0], " when attempting to query path on OneDrive - retrying applicable request in 30 seconds"); + log.vdebug(thisFunctionName, " previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429, 503, 504 - but loop back calling this function + log.vdebug("Retrying Function: ", thisFunctionName); + queryOneDriveForSpecificPathAndCreateIfMissing(thisNewPathToSearch, createPathIfMissing); + } else { + // Default operation if not 408,429,503,504 errors + // display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } + } + } + } else { + // parentDetails.driveId is not the account drive id - thus will be a remote shared item + log.vdebug("This parent directory is a remote object this next path will be on a remote drive"); + // For this parentDetails.driveId, parentDetails.id object, query the OneDrive API for it's children + for (;;) { + // Query this remote object for its children + topLevelChildren = queryOneDriveForSpecificPath.listChildren(parentDetails.driveId, parentDetails.id, nextLink); + // Process each child + foreach (child; topLevelChildren["value"].array) { + // Is this child a folder? + if (isItemFolder(child)) { + // Is this the child folder we are looking for, and is a POSIX match? + if (child["name"].str == thisFolderName) { + // EXACT MATCH including case sensitivity: Flag that we found the folder online + directoryFoundOnline = true; + // Use these details for the next entry path + getPathDetailsAPIResponse = child; + parentDetails = makeItem(getPathDetailsAPIResponse); + writeln("Line: 5913"); + saveItem(getPathDetailsAPIResponse); + // No need to continue searching + break; + } else { + string childAsLower = toLower(child["name"].str); + string thisFolderNameAsLower = toLower(thisFolderName); + if (childAsLower == thisFolderNameAsLower) { + // This is a POSIX 'case in-sensitive match' ..... + // Local item name has a 'case-insensitive match' to an existing item on OneDrive + posixIssue = true; + throw new posixException(thisFolderName, child["name"].str); + } + } + } + } + + if (directoryFoundOnline) { + // We found the folder, no need to continue searching nextLink data + break; + } + + // If a collection exceeds the default page size (200 items), the @odata.nextLink property is returned in the response + // to indicate more items are available and provide the request URL for the next page of items. + if ("@odata.nextLink" in topLevelChildren) { + // Update nextLink to next changeSet bundle + log.vdebug("Setting nextLink to (@odata.nextLink): ", nextLink); + nextLink = topLevelChildren["@odata.nextLink"].str; + } else break; + } + } + } + + // If we did not find the folder, we need to create this folder + if (!directoryFoundOnline) { + // Folder not found online + // Set any response to be an invalid JSON item + getPathDetailsAPIResponse = null; + // Was there a POSIX issue? + if (!posixIssue) { + // No POSIX issue + if (createPathIfMissing) { + // Create this path as it is missing on OneDrive online and there is no POSIX issue with a 'case-insensitive match' + log.vdebug("FOLDER NOT FOUND ONLINE AND WE ARE REQUESTED TO CREATE IT"); + log.vdebug("Create folder on this drive: ", parentDetails.driveId); + log.vdebug("Create folder as a child on this object: ", parentDetails.id); + log.vdebug("Create this folder name: ", thisFolderName); + + JSONValue newDriveItem = [ + "name": JSONValue(thisFolderName), + "folder": parseJSON("{}") + ]; + + JSONValue createByIdAPIResponse; + // Submit the creation request + // Fix for https://github.com/skilion/onedrive/issues/356 + if (!dryRun) { + try { + // Attempt to create a new folder on the configured parent driveId & parent id + createByIdAPIResponse = queryOneDriveForSpecificPath.createById(parentDetails.driveId, parentDetails.id, newDriveItem); + // Is the response a valid JSON object - validation checking done in saveItem + writeln("Line: 5974"); + saveItem(createByIdAPIResponse); + // Set getPathDetailsAPIResponse to createByIdAPIResponse + getPathDetailsAPIResponse = createByIdAPIResponse; + } catch (OneDriveException e) { + // 409 - API Race Condition + if (e.httpStatusCode == 409) { + // When we attempted to create it, OneDrive responded that it now already exists + log.vlog("OneDrive reported that ", thisFolderName, " already exists .. OneDrive API race condition"); + } else { + // some other error from OneDrive was returned - display what it is + log.error("OneDrive generated an error when creating this path: ", thisFolderName); + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + } + } + } else { + // Simulate a successful 'directory create' & save it to the dryRun database copy + // The simulated response has to pass 'makeItem' as part of saveItem + auto fakeResponse = createFakeResponse(thisNewPathToSearch); + writeln("Line: 5993"); + saveItem(fakeResponse); + } + } + } + } + } + + // Shutdown API instance + queryOneDriveForSpecificPath.shutdown(); + // Free object and memory + object.destroy(queryOneDriveForSpecificPath); + + // Output our search results + log.vdebug("queryOneDriveForSpecificPathAndCreateIfMissing.getPathDetailsAPIResponse = ", getPathDetailsAPIResponse); + return getPathDetailsAPIResponse; + } + + // Delete an item by it's path + // This function is only used in --monitor mode + void deleteByPath(const(string) path) { + + // function variables + Item dbItem; + + // Need to check all driveid's we know about, not just the defaultDriveId + bool itemInDB = false; + foreach (searchDriveId; driveIDsArray) { + if (itemDB.selectByPath(path, searchDriveId, dbItem)) { + // item was found in the DB + itemInDB = true; + break; + } + } + if (!itemInDB) { + throw new SyncException("The item to delete is not in the local database"); + } + + if (dbItem.parentId == null) { + // the item is a remote folder, need to do the operation on the parent + enforce(itemDB.selectByPathWithoutRemote(path, appConfig.defaultDriveId, dbItem)); + } + + try { + if (noRemoteDelete) { + // do not process remote delete + log.vlog("Skipping remote delete as --upload-only & --no-remote-delete configured"); + } else { + uploadDeletedItem(dbItem, path); + } + } catch (OneDriveException e) { + if (e.httpStatusCode == 404) { + log.log(e.msg); + } else { + // display what the error is + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + } + } + } + + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_move + // This function is only called in monitor mode when an move event is coming from + // inotify and we try to move the item. + void uploadMoveItem(string oldPath, string newPath) { + // Log that we are doing a move + log.log("Moving ", oldPath, " to ", newPath); + // Is this move unwanted? + bool unwanted = false; + // Item variables + Item oldItem, newItem, parentItem; + + // This not a Client Side Filtering check, nor a Microsoft Check, but is a sanity check that the path provided is UTF encoded correctly + // Check the std.encoding of the path against: Unicode 5.0, ASCII, ISO-8859-1, ISO-8859-2, WINDOWS-1250, WINDOWS-1251, WINDOWS-1252 + if (!unwanted) { + if(!isValid(newPath)) { + // Path is not valid according to https://dlang.org/phobos/std_encoding.html + log.logAndNotify("Skipping item - invalid character encoding sequence: ", newPath); + unwanted = true; + } + } + + // Check this path against the Client Side Filtering Rules + // - check_nosync + // - skip_dotfiles + // - skip_symlinks + // - skip_file + // - skip_dir + // - sync_list + // - skip_size + if (!unwanted) { + unwanted = checkPathAgainstClientSideFiltering(newPath); + } + + // Check against Microsoft OneDrive restriction and limitations about Windows naming files + if (!unwanted) { + unwanted = checkPathAgainstMicrosoftNamingRestrictions(newPath); + } + + // 'newPath' has passed client side filtering validation + if (!unwanted) { + + if (!itemDB.selectByPath(oldPath, appConfig.defaultDriveId, oldItem)) { + // The old path|item is not synced with the database, upload as a new file + log.log("Moved local item was not in-sync with local databse - uploading as new item"); + uploadNewFile(newPath); + return; + } + + if (oldItem.parentId == null) { + // the item is a remote folder, need to do the operation on the parent + enforce(itemDB.selectByPathWithoutRemote(oldPath, appConfig.defaultDriveId, oldItem)); + } + + if (itemDB.selectByPath(newPath, appConfig.defaultDriveId, newItem)) { + // the destination has been overwritten + log.log("Moved local item overwrote an existing item - deleting old online item"); + uploadDeletedItem(newItem, newPath); + } + + if (!itemDB.selectByPath(dirName(newPath), appConfig.defaultDriveId, parentItem)) { + // the parent item is not in the database + throw new SyncException("Can't move an item to an unsynced directory"); + } + + if (oldItem.driveId != parentItem.driveId) { + // items cannot be moved between drives + uploadDeletedItem(oldItem, oldPath); + + // what sort of move is this? + if (isFile(newPath)) { + // newPath is a file + uploadNewFile(newPath); + } else { + // newPath is a directory + scanLocalFilesystemPathForNewData(newPath); + } + } else { + if (!exists(newPath)) { + log.vlog("uploadMoveItem target has disappeared: ", newPath); + return; + } + + // Configure the modification JSON item + SysTime mtime = timeLastModified(newPath).toUTC(); + JSONValue data = [ + "name": JSONValue(baseName(newPath)), + "parentReference": JSONValue([ + "id": parentItem.id + ]), + "fileSystemInfo": JSONValue([ + "lastModifiedDateTime": mtime.toISOExtString() + ]) + ]; + + // Perform the move operation on OneDrive + JSONValue response; + + // Create a new API Instance for this thread and initialise it + OneDriveApi movePathOnlineApiInstance; + movePathOnlineApiInstance = new OneDriveApi(appConfig); + movePathOnlineApiInstance.initialise(); + + try { + response = movePathOnlineApiInstance.updateById(oldItem.driveId, oldItem.id, data, oldItem.eTag); + } catch (OneDriveException e) { + if (e.httpStatusCode == 412) { + // OneDrive threw a 412 error, most likely: ETag does not match current item's value + // Retry without eTag + log.vdebug("File Move Failed - OneDrive eTag / cTag match issue"); + log.vlog("OneDrive returned a 'HTTP 412 - Precondition Failed' when attempting to move the file - gracefully handling error"); + string nullTag = null; + // move the file but without the eTag + response = movePathOnlineApiInstance.updateById(oldItem.driveId, oldItem.id, data, nullTag); + } + } + // Shutdown API instance + movePathOnlineApiInstance.shutdown(); + // Free object and memory + object.destroy(movePathOnlineApiInstance); + + // save the move response from OneDrive in the database + // Is the response a valid JSON object - validation checking done in saveItem + writeln("Line: 6171"); + saveItem(response); + } + } else { + // Moved item is unwanted + log.log("Item has been moved to a location that is excluded from sync operations. Removing item from OneDrive"); + uploadDeletedItem(oldItem, oldPath); + } + } + + // Perform integrity validation of the file that was uploaded + bool performUploadIntegrityValidationChecks(JSONValue uploadResponse, string localFilePath, ulong localFileSize) { + + bool integrityValid = false; + + if (!disableUploadValidation) { + // Integrity validation has not been disabled (this is the default so we are always integrity checking our uploads) + ulong uploadFileSize = uploadResponse["size"].integer; + string uploadFileHash = uploadResponse["file"]["hashes"]["quickXorHash"].str; + string localFileHash = computeQuickXorHash(localFilePath); + + if ((localFileSize == uploadFileSize) && (localFileHash == uploadFileHash)) { + // Uploaded file integrity intact + log.vdebug("Uploaded local file matches reported online size and hash values"); + integrityValid = true; + } else { + // Upload integrity failure .. what failed? + // There are 2 scenarios where this happens: + // 1. Failed Transfer + // 2. Upload file is going to a SharePoint Site, where Microsoft enriches the file with additional metadata with no way to disable + log.logAndNotify("WARNING: Uploaded file integrity failure for: ", localFilePath); + + // What integrity failed - size? + if (localFileSize != uploadFileSize) { + log.vlog("WARNING: Uploaded file integrity failure - Size Mismatch"); + } + // What integrity failed - hash? + if (localFileHash != uploadFileHash) { + log.vlog("WARNING: Uploaded file integrity failure - Hash Mismatch"); + } + + // What account type is this? + if (appConfig.accountType != "personal") { + // Not a personal account, thus the integrity failure is most likely due to SharePoint + log.vlog("CAUTION: Microsoft SharePoint enhances files after you upload them, which means this file may now have technical differences from your local copy, resulting in an integrity issue."); + log.vlog("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details"); + } + // How can this be disabled? + log.log("To disable the integrity checking of uploaded files use --disable-upload-validation"); + } + } else { + // We are bypassing integrity checks due to --disable-upload-validation + log.vdebug("Upload file validation disabled due to --disable-upload-validation"); + log.vlog("WARNING: Skipping upload integrity check for: ", localFilePath); + } + + // Is the file integrity online valid? + return integrityValid; + } + + // Query Office 365 SharePoint Shared Library site name to obtain it's Drive ID + void querySiteCollectionForDriveID(string sharepointLibraryNameToQuery) + { + // Steps to get the ID: + // 1. Query https://graph.microsoft.com/v1.0/sites?search= with the name entered + // 2. Evaluate the response. A valid response will contain the description and the id. If the response comes back with nothing, the site name cannot be found or no access + // 3. If valid, use the returned ID and query the site drives + // https://graph.microsoft.com/v1.0/sites//drives + // 4. Display Shared Library Name & Drive ID + + string site_id; + string drive_id; + bool found = false; + JSONValue siteQuery; + string nextLink; + string[] siteSearchResults; + + // Create a new API Instance for this thread and initialise it + OneDriveApi querySharePointLibraryNameApiInstance; + querySharePointLibraryNameApiInstance = new OneDriveApi(appConfig); + querySharePointLibraryNameApiInstance.initialise(); + + // The account type must not be a personal account type + if (appConfig.accountType == "personal") { + log.error("ERROR: A OneDrive Personal Account cannot be used with --get-sharepoint-drive-id. Please re-authenticate your client using a OneDrive Business Account."); + return; + } + + // What query are we performing? + writeln(); + log.log("Office 365 Library Name Query: ", sharepointLibraryNameToQuery); + + for (;;) { + try { + siteQuery = querySharePointLibraryNameApiInstance.o365SiteSearch(nextLink); + } catch (OneDriveException e) { + log.error("ERROR: Query of OneDrive for Office 365 Library Name failed"); + // Forbidden - most likely authentication scope needs to be updated + if (e.httpStatusCode == 403) { + log.error("ERROR: Authentication scope needs to be updated. Use --reauth and re-authenticate client."); + return; + } + // Requested resource cannot be found + if (e.httpStatusCode == 404) { + string siteSearchUrl; + if (nextLink.empty) { + siteSearchUrl = querySharePointLibraryNameApiInstance.getSiteSearchUrl(); + } else { + siteSearchUrl = nextLink; + } + // log the error + log.error("ERROR: Your OneDrive Account and Authentication Scope cannot access this OneDrive API: ", siteSearchUrl); + log.error("ERROR: To resolve, please discuss this issue with whomever supports your OneDrive and SharePoint environment."); + return; + } + // HTTP request returned status code 429 (Too Many Requests) + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(querySharePointLibraryNameApiInstance); + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive children"); + } + // HTTP request returned status code 504 (Gateway Timeout) or 429 retry + if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) { + // re-try the specific changes queries + if (e.httpStatusCode == 504) { + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query Sharepoint Sites - retrying applicable request"); + log.vdebug("siteQuery = onedrive.o365SiteSearch(nextLink) previously threw an error - retrying"); + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. + log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); + Thread.sleep(dur!"seconds"(30)); + } + // re-try original request - retried for 429 and 504 + try { + log.vdebug("Retrying Query: siteQuery = onedrive.o365SiteSearch(nextLink)"); + siteQuery = querySharePointLibraryNameApiInstance.o365SiteSearch(nextLink); + log.vdebug("Query 'siteQuery = onedrive.o365SiteSearch(nextLink)' performed successfully on re-try"); + } catch (OneDriveException e) { + // display what the error is + log.vdebug("Query Error: siteQuery = onedrive.o365SiteSearch(nextLink) on re-try after delay"); + // error was not a 504 this time + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + return; + } + } else { + // display what the error is + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + return; + } + } + + // is siteQuery a valid JSON object & contain data we can use? + if ((siteQuery.type() == JSONType.object) && ("value" in siteQuery)) { + // valid JSON object + log.vdebug("O365 Query Response: ", siteQuery); + + foreach (searchResult; siteQuery["value"].array) { + // Need an 'exclusive' match here with sharepointLibraryNameToQuery as entered + log.vdebug("Found O365 Site: ", searchResult); + + // 'displayName' and 'id' have to be present in the search result record in order to query the site + if (("displayName" in searchResult) && ("id" in searchResult)) { + if (sharepointLibraryNameToQuery == searchResult["displayName"].str){ + // 'displayName' matches search request + site_id = searchResult["id"].str; + JSONValue siteDriveQuery; + + try { + siteDriveQuery = querySharePointLibraryNameApiInstance.o365SiteDrives(site_id); + } catch (OneDriveException e) { + log.error("ERROR: Query of OneDrive for Office Site ID failed"); + // display what the error is + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + return; + } + + // is siteDriveQuery a valid JSON object & contain data we can use? + if ((siteDriveQuery.type() == JSONType.object) && ("value" in siteDriveQuery)) { + // valid JSON object + foreach (driveResult; siteDriveQuery["value"].array) { + // Display results + writeln("-----------------------------------------------"); + log.vdebug("Site Details: ", driveResult); + found = true; + writeln("Site Name: ", searchResult["displayName"].str); + writeln("Library Name: ", driveResult["name"].str); + writeln("drive_id: ", driveResult["id"].str); + writeln("Library URL: ", driveResult["webUrl"].str); + } + // closeout + writeln("-----------------------------------------------"); + } else { + // not a valid JSON object + log.error("ERROR: There was an error performing this operation on OneDrive"); + log.error("ERROR: Increase logging verbosity to assist determining why."); + return; + } + } + } else { + // 'displayName', 'id' or ''webUrl' not present in JSON results for a specific site + string siteNameAvailable = "Site 'name' was restricted by OneDrive API permissions"; + bool displayNameAvailable = false; + bool idAvailable = false; + if ("name" in searchResult) siteNameAvailable = searchResult["name"].str; + if ("displayName" in searchResult) displayNameAvailable = true; + if ("id" in searchResult) idAvailable = true; + + // Display error details for this site data + writeln(); + log.error("ERROR: SharePoint Site details not provided for: ", siteNameAvailable); + log.error("ERROR: The SharePoint Site results returned from OneDrive API do not contain the required items to match. Please check your permissions with your site administrator."); + log.error("ERROR: Your site security settings is preventing the following details from being accessed: 'displayName' or 'id'"); + log.vlog(" - Is 'displayName' available = ", displayNameAvailable); + log.vlog(" - Is 'id' available = ", idAvailable); + log.error("ERROR: To debug this further, please increase verbosity (--verbose or --verbose --verbose) to provide further insight as to what details are actually being returned."); + } + } + + if(!found) { + // The SharePoint site we are searching for was not found in this bundle set + // Add to siteSearchResults so we can display what we did find + string siteSearchResultsEntry; + foreach (searchResult; siteQuery["value"].array) { + // We can only add the displayName if it is available + if ("displayName" in searchResult) { + // Use the displayName + siteSearchResultsEntry = " * " ~ searchResult["displayName"].str; + siteSearchResults ~= siteSearchResultsEntry; + } else { + // Add, but indicate displayName unavailable, use id + if ("id" in searchResult) { + siteSearchResultsEntry = " * " ~ "Unknown displayName (Data not provided by API), Site ID: " ~ searchResult["id"].str; + siteSearchResults ~= siteSearchResultsEntry; + } else { + // displayName and id unavailable, display in debug log the entry + log.vdebug("Bad SharePoint Data for site: ", searchResult); + } + } + } + } + } else { + // not a valid JSON object + log.error("ERROR: There was an error performing this operation on OneDrive"); + log.error("ERROR: Increase logging verbosity to assist determining why."); + return; + } + + // If a collection exceeds the default page size (200 items), the @odata.nextLink property is returned in the response + // to indicate more items are available and provide the request URL for the next page of items. + if ("@odata.nextLink" in siteQuery) { + // Update nextLink to next set of SharePoint library names + nextLink = siteQuery["@odata.nextLink"].str; + log.vdebug("Setting nextLink to (@odata.nextLink): ", nextLink); + } else break; + } + + // Was the intended target found? + if(!found) { + + // Was the search a wildcard? + if (sharepointLibraryNameToQuery != "*") { + // Only print this out if the search was not a wildcard + writeln(); + log.error("ERROR: The requested SharePoint site could not be found. Please check it's name and your permissions to access the site."); + } + // List all sites returned to assist user + writeln(); + log.log("The following SharePoint site names were returned:"); + foreach (searchResultEntry; siteSearchResults) { + // list the display name that we use to match against the user query + log.log(searchResultEntry); + } + } + + // Shutdown API instance + querySharePointLibraryNameApiInstance.shutdown(); + // Free object and memory + object.destroy(querySharePointLibraryNameApiInstance); + } +} \ No newline at end of file diff --git a/src/util.d b/src/util.d new file mode 100644 index 00000000..9bb45b1f --- /dev/null +++ b/src/util.d @@ -0,0 +1,784 @@ +// What is this module called? +module util; + +// What does this module require to function? +import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit; +import std.base64; +import std.conv; +import std.digest.crc; +import std.digest.sha; +import std.net.curl; +import std.datetime; +import std.file; +import std.path; +import std.regex; +import std.socket; +import std.stdio; +import std.string; +import std.algorithm; +import std.uri; +import std.json; +import std.traits; +import core.stdc.stdlib; +import core.thread; + +// What other modules that we have created do we need to import? +import log; +import config; +import qxor; +import curlEngine; + +// module variables +shared string deviceName; + +static this() { + deviceName = Socket.hostName; +} + +// Creates a safe backup of the given item, and only performs the function if not in a --dry-run scenario +void safeBackup(const(char)[] path, bool dryRun) { + auto ext = extension(path); + auto newPath = path.chomp(ext) ~ "-" ~ deviceName; + if (exists(newPath ~ ext)) { + int n = 2; + char[] newPath2; + do { + newPath2 = newPath ~ "-" ~ n.to!string; + n++; + } while (exists(newPath2 ~ ext)); + newPath = newPath2; + } + newPath ~= ext; + + // Perform the backup + log.vlog("The local item is out-of-sync with OneDrive, renaming to preserve existing file and prevent data loss: ", path, " -> ", newPath); + if (!dryRun) { + rename(path, newPath); + } else { + log.vdebug("DRY-RUN: Skipping local file backup"); + } +} + +// Rename the given item, and only performs the function if not in a --dry-run scenario +void safeRename(const(char)[] oldPath, const(char)[] newPath, bool dryRun) { + // Perform the rename + if (!dryRun) { + log.vdebug("Calling rename(oldPath, newPath)"); + // rename physical path on disk + rename(oldPath, newPath); + } else { + log.vdebug("DRY-RUN: Skipping local file rename"); + } +} + +// deletes the specified file without throwing an exception if it does not exists +void safeRemove(const(char)[] path) { + if (exists(path)) remove(path); +} + +// returns the CRC32 hex string of a file +string computeCRC32(string path) { + CRC32 crc; + auto file = File(path, "rb"); + foreach (ubyte[] data; chunks(file, 4096)) { + crc.put(data); + } + return crc.finish().toHexString().dup; +} + +// returns the SHA1 hash hex string of a file +string computeSha1Hash(string path) { + SHA1 sha; + auto file = File(path, "rb"); + foreach (ubyte[] data; chunks(file, 4096)) { + sha.put(data); + } + return sha.finish().toHexString().dup; +} + +// returns the quickXorHash base64 string of a file +string computeQuickXorHash(string path) { + QuickXor qxor; + auto file = File(path, "rb"); + foreach (ubyte[] data; chunks(file, 4096)) { + qxor.put(data); + } + return Base64.encode(qxor.finish()); +} + +// returns the SHA256 hex string of a file +string computeSHA256Hash(string path) { + SHA256 sha256; + auto file = File(path, "rb"); + foreach (ubyte[] data; chunks(file, 4096)) { + sha256.put(data); + } + return sha256.finish().toHexString().dup; +} + +// converts wildcards (*, ?) to regex +Regex!char wild2regex(const(char)[] pattern) { + string str; + str.reserve(pattern.length + 2); + str ~= "^"; + foreach (c; pattern) { + switch (c) { + case '*': + str ~= "[^/]*"; + break; + case '.': + str ~= "\\."; + break; + case '?': + str ~= "[^/]"; + break; + case '|': + str ~= "$|^"; + break; + case '+': + str ~= "\\+"; + break; + case ' ': + str ~= "\\s+"; + break; + case '/': + str ~= "\\/"; + break; + case '(': + str ~= "\\("; + break; + case ')': + str ~= "\\)"; + break; + default: + str ~= c; + break; + } + } + str ~= "$"; + return regex(str, "i"); +} + +// Test Internet access to Microsoft OneDrive +bool testInternetReachability(ApplicationConfig appConfig) { + // Use preconfigured object with all the correct http values assigned + auto curlEngine = new CurlEngine(); + curlEngine.initialise(appConfig.getValueLong("dns_timeout"), appConfig.getValueLong("connect_timeout"), appConfig.getValueLong("data_timeout"), appConfig.getValueLong("operation_timeout"), appConfig.defaultMaxRedirects, appConfig.getValueBool("debug_https"), appConfig.getValueString("user_agent"), appConfig.getValueBool("force_http_11"), appConfig.getValueLong("rate_limit"), appConfig.getValueLong("ip_protocol_version")); + + // Configure the remaining items required + // URL to use + curlEngine.http.url = "https://login.microsoftonline.com"; + // HTTP connection test method + curlEngine.http.method = HTTP.Method.head; + // Attempt to contact the Microsoft Online Service + try { + log.vdebug("Attempting to contact Microsoft OneDrive Login Service"); + curlEngine.http.perform(); + log.vdebug("Shutting down HTTP engine as successfully reached OneDrive Login Service"); + curlEngine.http.shutdown(); + // Free object and memory + object.destroy(curlEngine); + return true; + } catch (SocketException e) { + // Socket issue + log.vdebug("HTTP Socket Issue"); + log.error("Cannot connect to Microsoft OneDrive Login Service - Socket Issue"); + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + return false; + } catch (CurlException e) { + // No network connection to OneDrive Service + log.vdebug("No Network Connection"); + log.error("Cannot connect to Microsoft OneDrive Login Service - Network Connection Issue"); + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + return false; + } +} + +// Retry Internet access test to Microsoft OneDrive +bool retryInternetConnectivtyTest(ApplicationConfig appConfig) { + // re-try network connection to OneDrive + // https://github.com/abraunegg/onedrive/issues/1184 + // Back off & retry with incremental delay + int retryCount = 10000; + int retryAttempts = 1; + int backoffInterval = 1; + int maxBackoffInterval = 3600; + bool onlineRetry = false; + bool retrySuccess = false; + while (!retrySuccess){ + // retry to access OneDrive API + backoffInterval++; + int thisBackOffInterval = retryAttempts*backoffInterval; + log.vdebug(" Retry Attempt: ", retryAttempts); + if (thisBackOffInterval <= maxBackoffInterval) { + log.vdebug(" Retry In (seconds): ", thisBackOffInterval); + Thread.sleep(dur!"seconds"(thisBackOffInterval)); + } else { + log.vdebug(" Retry In (seconds): ", maxBackoffInterval); + Thread.sleep(dur!"seconds"(maxBackoffInterval)); + } + // perform the re-rty + onlineRetry = testInternetReachability(appConfig); + if (onlineRetry) { + // We are now online + log.log("Internet connectivity to Microsoft OneDrive service has been restored"); + retrySuccess = true; + } else { + // We are still offline + if (retryAttempts == retryCount) { + // we have attempted to re-connect X number of times + // false set this to true to break out of while loop + retrySuccess = true; + } + } + // Increment & loop around + retryAttempts++; + } + if (!onlineRetry) { + // Not online after 1.2 years of trying + log.error("ERROR: Was unable to reconnect to the Microsoft OneDrive service after 10000 attempts lasting over 1.2 years!"); + } + // return the state + return onlineRetry; +} + +// Can we read the file - as a permissions issue or file corruption will cause a failure +// https://github.com/abraunegg/onedrive/issues/113 +// returns true if file can be accessed +bool readLocalFile(string path) { + try { + // attempt to read up to the first 1 byte of the file + // validates we can 'read' the file based on file permissions + read(path,1); + } catch (std.file.FileException e) { + // unable to read the new local file + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + return false; + } + return true; +} + +// calls globMatch for each string in pattern separated by '|' +bool multiGlobMatch(const(char)[] path, const(char)[] pattern) { + foreach (glob; pattern.split('|')) { + if (globMatch!(std.path.CaseSensitive.yes)(path, glob)) { + return true; + } + } + return false; +} + +bool isValidName(string path) { + // Restriction and limitations about windows naming files + // https://msdn.microsoft.com/en-us/library/aa365247 + // https://support.microsoft.com/en-us/help/3125202/restrictions-and-limitations-when-you-sync-files-and-folders + + // allow root item + if (path == ".") { + return true; + } + + bool matched = true; + string itemName = baseName(path); + + auto invalidNameReg = + ctRegex!( + // Leading whitespace and trailing whitespace/dot + `^\s.*|^.*[\s\.]$|` ~ + // Invalid characters + `.*[<>:"\|\?*/\\].*|` ~ + // Reserved device name and trailing .~ + `(?:^CON|^PRN|^AUX|^NUL|^COM[0-9]|^LPT[0-9])(?:[.].+)?$` + ); + auto m = match(itemName, invalidNameReg); + matched = m.empty; + + // Additional explicit validation checks + if (itemName == ".lock") {matched = false;} + if (itemName == "desktop.ini") {matched = false;} + // _vti_ cannot appear anywhere in a file or folder name + if(canFind(itemName, "_vti_")){matched = false;} + // Item name cannot equal '~' + if (itemName == "~") {matched = false;} + + // return response + return matched; +} + +bool containsBadWhiteSpace(string path) { + // allow root item + if (path == ".") { + return true; + } + + // https://github.com/abraunegg/onedrive/issues/35 + // Issue #35 presented an interesting issue where the filename contained a newline item + // 'State-of-the-art, challenges, and open issues in the integration of Internet of'$'\n''Things and Cloud Computing.pdf' + // When the check to see if this file was present the GET request queries as follows: + // /v1.0/me/drive/root:/.%2FState-of-the-art%2C%20challenges%2C%20and%20open%20issues%20in%20the%20integration%20of%20Internet%20of%0AThings%20and%20Cloud%20Computing.pdf + // The '$'\n'' is translated to %0A which causes the OneDrive query to fail + // Check for the presence of '%0A' via regex + + string itemName = encodeComponent(baseName(path)); + auto invalidWhitespaceReg = + ctRegex!( + // Check for \n which is %0A when encoded + `%0A` + ); + auto m = match(itemName, invalidWhitespaceReg); + return m.empty; +} + +bool containsASCIIHTMLCodes(string path) { + // https://github.com/abraunegg/onedrive/issues/151 + // If a filename contains ASCII HTML codes, regardless of if it gets encoded, it generates an error + // Check if the filename contains an ASCII HTML code sequence + + auto invalidASCIICode = + ctRegex!( + // Check to see if &#XXXX is in the filename + `(?:&#|&#[0-9][0-9]|&#[0-9][0-9][0-9]|&#[0-9][0-9][0-9][0-9])` + ); + + auto m = match(path, invalidASCIICode); + return m.empty; +} + +// Parse and display error message received from OneDrive +void displayOneDriveErrorMessage(string message, string callingFunction) { + writeln(); + log.error("ERROR: Microsoft OneDrive API returned an error with the following message:"); + auto errorArray = splitLines(message); + log.error(" Error Message: ", errorArray[0]); + // Extract 'message' as the reason + JSONValue errorMessage = parseJSON(replace(message, errorArray[0], "")); + + // What is the reason for the error + if (errorMessage.type() == JSONType.object) { + // configure the error reason + string errorReason; + string requestDate; + string requestId; + + // set the reason for the error + try { + // Use error_description as reason + errorReason = errorMessage["error_description"].str; + } catch (JSONException e) { + // we dont want to do anything here + } + + // set the reason for the error + try { + // Use ["error"]["message"] as reason + errorReason = errorMessage["error"]["message"].str; + } catch (JSONException e) { + // we dont want to do anything here + } + + // Display the error reason + if (errorReason.startsWith(" currentTime + // display an information warning that there is a new release available + if (releaseGracePeriod.toUnixTime() > currentTime.toUnixTime()) { + // inside release grace period ... set flag to false + displayObsolete = false; + } else { + // outside grace period + displayObsolete = true; + } + } + + // display version response + writeln(); + if (!displayObsolete) { + // display the new version is available message + log.logAndNotify("INFO: A new onedrive client version is available. Please upgrade your client version when possible."); + } else { + // display the obsolete message + log.logAndNotify("WARNING: Your onedrive client version is now obsolete and unsupported. Please upgrade your client version."); + } + log.log("Current Application Version: ", applicationVersion); + log.log("Version Available: ", latestVersion); + writeln(); + } + } +} + +bool hasId(JSONValue item) { + return ("id" in item) != null; +} + +bool hasQuota(JSONValue item) { + return ("quota" in item) != null; +} + +bool isItemDeleted(JSONValue item) { + return ("deleted" in item) != null; +} + +bool isItemRoot(JSONValue item) { + return ("root" in item) != null; +} + +bool hasParentReference(const ref JSONValue item) { + return ("parentReference" in item) != null; +} + +bool hasParentReferenceId(JSONValue item) { + return ("id" in item["parentReference"]) != null; +} + +bool hasParentReferencePath(JSONValue item) { + return ("path" in item["parentReference"]) != null; +} + +bool isFolderItem(const ref JSONValue item) { + return ("folder" in item) != null; +} + +bool isFileItem(const ref JSONValue item) { + return ("file" in item) != null; +} + +bool isItemRemote(const ref JSONValue item) { + return ("remoteItem" in item) != null; +} + +bool isItemFile(const ref JSONValue item) { + return ("file" in item) != null; +} + +bool isItemFolder(const ref JSONValue item) { + return ("folder" in item) != null; +} + +bool hasFileSize(const ref JSONValue item) { + return ("size" in item) != null; +} + +bool isDotFile(const(string) path) { + // always allow the root + if (path == ".") return false; + auto paths = pathSplitter(buildNormalizedPath(path)); + foreach(base; paths) { + if (startsWith(base, ".")){ + return true; + } + } + return false; +} + +bool isMalware(const ref JSONValue item) { + return ("malware" in item) != null; +} + +bool hasHashes(const ref JSONValue item) { + return ("hashes" in item["file"]) != null; +} + +bool hasQuickXorHash(const ref JSONValue item) { + return ("quickXorHash" in item["file"]["hashes"]) != null; +} + +bool hasSHA256Hash(const ref JSONValue item) { + return ("sha256Hash" in item["file"]["hashes"]) != null; +} + +bool isMicrosoftOneNoteMimeType1(const ref JSONValue item) { + return (item["file"]["mimeType"].str) == "application/msonenote"; +} + +bool isMicrosoftOneNoteMimeType2(const ref JSONValue item) { + return (item["file"]["mimeType"].str) == "application/octet-stream"; +} + +bool hasUploadURL(const ref JSONValue item) { + return ("uploadUrl" in item) != null; +} + +bool hasNextExpectedRanges(const ref JSONValue item) { + return ("nextExpectedRanges" in item) != null; +} + +bool hasLocalPath(const ref JSONValue item) { + return ("localPath" in item) != null; +} + +bool hasETag(const ref JSONValue item) { + return ("eTag" in item) != null; +} + +bool hasSharedElement(const ref JSONValue item) { + return ("eTag" in item) != null; +} \ No newline at end of file