From 00d53f648ec13f3738fb1fd7d597d07470fee578 Mon Sep 17 00:00:00 2001 From: skilion Date: Tue, 1 Sep 2015 20:45:34 +0200 Subject: [PATCH] first commit --- Makefile | 14 +++ Makefile.win | 14 +++ lib/sqlite3.lib | Bin 0 -> 37376 bytes onedrive.conf | 3 + src/cache.d | 163 ++++++++++++++++++++++++++++ src/config.d | 64 +++++++++++ src/main.d | 30 ++++++ src/onedrive.d | 219 ++++++++++++++++++++++++++++++++++++++ src/sqlite.d | 181 +++++++++++++++++++++++++++++++ src/sync.d | 277 ++++++++++++++++++++++++++++++++++++++++++++++++ src/util.d | 44 ++++++++ 11 files changed, 1009 insertions(+) create mode 100644 Makefile create mode 100644 Makefile.win create mode 100644 lib/sqlite3.lib create mode 100644 onedrive.conf create mode 100644 src/cache.d create mode 100644 src/config.d create mode 100644 src/main.d create mode 100644 src/onedrive.d create mode 100644 src/sqlite.d create mode 100644 src/sync.d create mode 100644 src/util.d diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..9a040386 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +DC = dmd +DFLAGS = -unittest -debug -g -od./bin -of./bin/$@ -L-lcurl -L-lsqlite3 + +SOURCES = \ + src/cache.d \ + src/config.d \ + src/main.d \ + src/onedrive.d \ + src/sqlite.d \ + src/sync.d \ + src/util.d + +onedrive: $(SOURCES) + $(DC) $(DFLAGS) $(SOURCES) diff --git a/Makefile.win b/Makefile.win new file mode 100644 index 00000000..0a9f98f1 --- /dev/null +++ b/Makefile.win @@ -0,0 +1,14 @@ +DC = dmd +DFLAGS = -debug -g -od./bin -of./bin/$@ lib/sqlite3.lib + +SOURCES = \ + src/cache.d \ + src/config.d \ + src/main.d \ + src/onedrive.d \ + src/sqlite.d \ + src/sync.d \ + src/util.d + +onedrive: $(SOURCES) + $(DC) $(DFLAGS) $(SOURCES) diff --git a/lib/sqlite3.lib b/lib/sqlite3.lib new file mode 100644 index 0000000000000000000000000000000000000000..3cec9c808d70b9df52a9253f6571137cdd89c713 GIT binary patch literal 37376 zcmdsA4UptTb?^CD)IyHagHz8tj=mz69tK!A2SJd_2^0r{96>$7FwE}$_iki%W|^72 z55mWS5)K7a1k`|__yr;ug^K~AK&3D!Aw(2EN{kh>3?!vu)GE}JqItvhbch&QInR-bXvAHgBTW7evw3HhmAg_@`0w>FG{-tVZ z;5RqyHECEychK&2)sD4>-AjaB1V?V8NF>aN<@`(3o}S-CYx0b_+2(Y++a7FIJ6>cuI27SC%9hb0+@#v0^N z)->jl&2M+E(-|?p3_|Mh{F;U_aqjBnP$QSTp?PqlV8&$bCU#DRY*8x_+%6 zEM310LSFIwnuZyNY<)4>cw@x+vOwe!uV2>~xRlZ#X&<uXD^Ge^@> zy1onydmpba+Z2ei{Lz21{W+`X_pA-g{3n8ui}+8(u%n3V1DWRL#P(q^DCvKIA!X@* z0HbhH`@=|+o!+2KJ^?nGzHOSw2lbO_^kxeRWitKrAfzt(>7hyIQ~hVOe$kp5L7zdR z>6?ae=b1kGFH7H!Ad6oFh&)6;1&*W0=67qKV%rh(YhXw{o?qKE9>V-mwMVc|bA#>8 z&8_ZKr|7F4hZ)1Prx+%HB9HRso=Zl5*q$kRbL1mS={x9X`o4Lfzf!I6{hw@gCyP$A z6@|TO#|dU2dxe3Nq9MnpA?R7t?9=?eNbMf@lk7I};yciATr3_M+dWi316jUtJd+Pu0YPZmTyR`Q+Y!*rH`D>(}G>GK^@sL^X z%`FG!@sNig)esMPU`cPOzYxz)_Bx%`kW!y!wM$a5i5R*Yy*sAF?8T^Dhu_5^^)A=ndufs0dK`ln<(F0LFg9s@;Q>3D2&uhbYnx&cqMhOLR# zplI3vE{z|{u#jJfU&|1)5zgE{Xl-bYD9AZoKY}G}=OMq%`0;Sl{I&!M zd4~M91W&kv{fGG5H=!gj&p!c({8Iho3=;X2=%-6kN9m^pBac)+9RaJKCH*(Hb;SB& z!;7@D{&QnoUl89r5x5L@gnh_z+|)kAS#E3}W@g&M=H_1Sj57HL;E4VQc+~$$pJ!&< zouW5uV!llU&2p~K2@3K8eNM21V{9)FZEnC3W^ON19OMZ0B1Lm7v3@w&o0;uUX{;n( zb`Yd3@v?6Q(;gN`?0+nMUY5E4@i628{>OtwRM!72R*NJ)=}xz)%6))|M|OvJguh3+ zX%p#rY`@H|O>=6(Wu;~FouD8WnC}Ej1`*{Gl;WGsB9~T_eqKgI~i zSK`MQC*>LGPmIdq&Fn>rf;>fkQY_(GGM|2YM!X~D6L64YI-h{%SVsQHc59|LHOD&| zS-gZ`t2&`Dl!i@M4BoZzsD&8_&ihruEDKLN1}68K5}>^tXgI zP5#gZvT3{kK;#kP1%M-$r2n=WnAc0GwFMzQ*Gx1>O(|iW{!fP+~0W!QV;&l z1F_qNwvj)?Q-nzRIo0cSXtGP>TSV2OR$=2j%vYM&?D?viu!T*)u#hk55 zVtu&i_a}Q(#51iAI{@+t^eG};7PQAq%I+sV^`9vQatQu2MFMG%zrpq8`Spvz zTxUqmXNq?tvh`JlhMZ-6m0?Egqxvg5BldX1evgh2r1>R6kz24Ih!B{de$v6CR;PVd zNq*!ZNH^%G2L_sU=H2{*K27%vqMhm!e~Wa3J{j~(pCF!Lgr+H)+6&Oo6RoD8!7M3^9%9cCfZ_cbtop7>}}y4&b7vJ$bMQ3M@}NIm_$1JaXias)!I&D(|A^3 z$S2IdfRiDiyr@!3mSt|1raqNd03r1tuK-Pm^L#_SrPCS=o9*tP=ntFy-qto*#x%ak zP>^4UZ!#?5i}8P_Jwfwg&oZX+5*Ts_@)EF+HVP1pC(v(p=Vm5Y1+wv^D9BC5lVS-| z+)wSa&9Y4QQ_a^%H|(d{29Z+XdQStVRDWYQatQhxqe!=ntoXk@jocLD`k5IRXP>Sg^EJ{6 z>&G?_Wnq2B@pH0C(`l428@~XNwv1ok(fBj^1zY9ZKblOpX@CpRw~b*0Xdf*J6Lw_v zTY(|^mHoiCLSCA3>83@Vq80V0P)KDIITAI^8#P7Jy9lKQIvLfXM!1!&TF?my7) zQl#7*3};IGVE~oV56}|&5FgruU1@yC-y+oz9~yKJ1uw@aA7TW~wo-d&zek$E9=aw% z71tjFO_?suUmO4_hV|zgIg)6vdjrx}a{5{S%=bu5`e&Pr=-+^%2@X`V{#gK0l>WI! zzJCy(a1>`8nlF>){yalL9>JeySfU3g-`Rda=aNhEcLzY)q5r;-??33P1F;IE>mh)U zW>^maD*ZId#rg&QE1(&rNb@m-BA4L55Fs!^d*l!d^)y}+7}5^*NWg)z9YHMb9_NI) zzRdOw04amKZ6hyl$j7M}T?iTsIfRJX;#{f+_%lZiUSHFmhrE8P? zD~2Pd5WmJKIlJgQBh5AM5vTEk`5LK*_`x=Cl5zS7_ZJK}J6@We=P1KMZs9yf8Dcgf zj9;h$mv!vv@zg&95V-~a3~*$E`$>SpI*{%sMPTF+_LCwE%cX?AzkxG?etrihrH{98 z(B3)rloR(T088y1Lb3P3-XTH|QLMj4{)Rpq?OcBmjyynr5hZZ3y2d{-U&RiYVqWgw zG8E(*{9A@4jB)=mLhJS@;?MV^a~$Lt_AhfZ#}dyAMM%S5Z-NgGr1kg`B;*+C@g;aN zN{}zq=UfxYSxW4)U26nGZepJUIAKzvk3^&FJf3imZ>*&HXq$VZRALY8jja*(AicAd z+Jp4gRtbGRu9W8kuKNr`38nN+W8Zn!Z#eG1B+C zoFzUfK}^LP#dj2KadfnqPN>p!+BhTO;W0Z^=*$=3T*xsfW&L#! zq$>UO%_72n{lNMEL2sH)cNVSLrmJkH>oq|@Zo$4KIH`~@{z*{T#-;W83 zu95tDbN$JJw>dKTIS5i0`T1t0@`HLh*@?WKPG2Hb|2x|X>19x#sD9)RgQWGD{3J-I z&t%6yqVb!hY8a9S^6SDsGp@z5f~w0y^l+M%?Y}e znCGhjggp-VYJlbvMt@<@D7SSdh*EzMfss$}7ZHZ#71=NDzn{c3eluQiHWg{bm@!k%x>YLJ3^L z^hM`EExkc&nnLNcz9=x{6Y?_wCp>U{4|_v84lrpi=jQoG1V%1leMcCU5&HLGzcoo+ zPyM_38tDc9ZW}~ujsB3p-N7`yoFP|&Q-7GDAfMn5Gb|Aj+NZf$S}?_k`y2p79>G2V zT&TzK+g!ihoapsfo$~w=faUa|F-Q#k1CwyccUsR3Afz4qLx5t>t&XDpFj{!1B-ft^ zjC_LrL>NwgpdL#bwS9BjGxu)+gmk6}d^&jdVCUtY=#B_ZL3^|1LCE!Fvs6P(L zUN27l8vv0*&>w&!3zR>gl+_O+F!BiUk1(A4!Jb;MFIT4ZM1&&sU{4Vt3<34iB9MmW z^BX`&Im~Z>2C~6`=JIzdpj`e2h&&Sc+s2?pSTD5QprbSex&NlU5}e+5$Z?Qcs2ApF zZfLllVv$TmDf0&cL;8unnx-%qk{Z@Kga)fq`Fz`26Y@yr+aiRCLH~*Lxg51N@t-jq zc?AC%qXaHt`lEatk+~eT5Wlz@Rmv|fMzQ=ddt&vV+@7Sj7t{GkE-x~D*zXF5!_xTE z0Z?Kgp7o8~v}3-rWqQDm*_Gsb9)c7@{ON%u(ogh>=+gs)ZAxTH_9K{#haw-*CqM|i zB7Gu+xvr;CGQSZVdC2@mD1i&;6U6IV{8^nWKlBjfA@cIg;+1E4MD}ptD-qfL;}#D> z4q^{IG#@}@4=H}<_}JXY$?Ty3k%!nrfg=|@&&Bs9n)Dh8v*@%Z$*iaQ%P9tO3j50` z60(c#4{oLCl^?sIdkm9j%=U~UN$qcrgFJ)%&CwiJ)Hg;#UTYKk6~mEF(6<;R8ky4v zINuQ?n_~uPyqlpQ*AVY!Si%_X0i)T=%V|F9AxJ&ww+9MQSw2y{a%;PL(1FeNpvfLT z)jt3tm!N+DM@Hz6Ey~QqfN$LU3i>)AE2Rn&5MX0_Bro>@gTmnX}S<;lVrB~Qe`(IxUkG&eZR@9cQY zD_za0Vq%VVLoG~8i51-GTR3;W^~2 zP`;vWfFlnZtz|F@PL2gxj+0K)!ulFqVmf>%Y1U_q-kSxy)Blpc{9nq zZ8YA{D=Ihx$pZrVBNdqey1r=(wR%xi-Z3qi0llg!uQbuNti2uP_DRicEq!8Do{h5y z)%6`!&{M0AtgZT|RV9xv<6%8LUQ?ov(XO<kK?26|DTl&^ItXca#twDW1+J`6@ z*Sn7aCHJ^C?}^ZMkIb&CZkty>#~Z&^qF)Ze1~0r;Hr`lN5u0c4jhT>VTSbYTnzS?T zPu1Rmb`rKtpQ=Is+(^0BbNh2?Grn~K&F>RRy-krCDE0t<&;K5Hj=EXhuRd^H|KHU< z#ZxlZX8nhv9nB@ZL99*v2#Jgh{D(FGUxVtHh}tH>p183p*_CC z%}M%Fd?epVg!IV0Befxd#@aI8xTgYsCfLSBRVA}fDzRl%C2z*qgYLW5q~{=&Ze&~T zRMYCC>YM7agVbl$5lWqTocWUgtOiTeSJZiG)xU|&%l;;Su0X%sNTVmV^!f_K16~#^ zz|$)tZ~r~XPIoG>KCN`@;}`nDimWmdiG;veKUx9%KvJ_0DCl273wwo<@;!6kQ}?Q1 zp|UG>`pT+)x&tTrF_5nt0c+h)6vPDXd#@SejTP8axu|$?Ric^ni1a$JmS8Ey8+ZNR zZN=-mN_|{Cu|$3KxT|Q&|7QAa>WAuEO6{XA+D3VtQgOO4m)lB>(zCM1YCHre;sB# z+r-hA)#VRcq(ub%$}#X7@e{ebLmc5pPx?LHn8uYHF4>-41-p?%j7#glCEmPj*TG}r z&f7cdz(3j&`E(stp4A&$jlVaj&4$}~ij}&ML%yTcJ5HnDZ>G5a%DrDlzip+%2K2d4 zIJ@x~f2n_=fWD2a%!zvaXjLSc>at!}1N&xYhK%f*I(+L(0{X}?7+drQGL|B5*DEu` zeg%4q*5knn=v9awN#{6nTn~|F**UN$K<}d5<)}O^VF&Oe`Vxvue3kJ9V`58fzKTh0 zF=pib9R4WCcdO&)mAaCd&Qdb)2>tt+bzlC9y7skut^PWx#owyOB~}mdzwSfyB34Xa zjM|;J^joS*)}*~1rT2`q!4)v|V$^BWTurK*^u~bS1=_1{!qyWjyz4lTzE(-te_=tG}TCxmBqHmaM$sOX{!b ze|{($m#C3dC>j z3Z7nElL)D74eKXr(8?XU(Qkp33Y+fwN{IT*sIBRwGkfBvU8GN{V~6$$$Z93i{3+z5 zr1iX5mX&0f#`}#o{t7HjIUM{H9Qfazl~Wo zT0K&hpJNv9HmddSAc~h~P27tm*JCXZj))#thb-16>Q2-BxyKg9(pa&Y1-~7R72TXy zv5eFFdL1Gl*W=$sqi~zOOP?d{xk;(h?o~%0tX3|$Y&Cg$+LHWYTd8lH^fRRfv@xLO zNfDlsH3@YzeE`_n5Oe8kVC^{>+6Tq;VR?1b*T~34+ym7Bo2i?d33>~}Q!=0WnksO& zzLqYH=-;5PB~w6zcddb*y7g-#m>WwnO@9$af3GGHpV{WpyVm6!${r^ReKE|v-^155 zbs{h8Da@XwK6Xs(2-Z=kg!X)lDx8eNAFO*-o?S;($z7l(36k4_I}=%bT>jf#Y(KcX$Iv-(LlfMCl*j@YZj_8lbywp!dNQ4YoiZ zSC_c9)E4MgedL%;Mf4Q#b>>l(@8jN)JeN^}Jj3Qgd)CBD+RazYcjQ>^IpFy?=Xd8o zuSUD$q7>fWZ7yI=HhhFLfA#)VOVs&?sI9CCr>f7WTmN)3|H_{)W%D2Y!lPbARl<1w zjlVt_vO}X4#&qj_>F_i3-jm>X@!jToilwYEobP6y^{$NqI0gRg8*VdMAX^G4}Lq0}4BrriXj?xJRXVZSLdQ+?!xhMR>ESEAvnsk0R)JPvX-US#P`7Fl zzd5OMszYk>AVd)^8sL)glem^quMU9Shfo#%71rf~uI-DlT>MbNDS1?C;zKG`{&9R> zb)Ff=-t_ZCb@bq1D*L>B>XHw;!F^h0!>qyJBi{}7%Oqy{n^4=t^+bO7uW&~&se97Q zu7FkI|Af_d!HUrAafZM?@61Z|%FC(#Lpk8zDMj~f)jgd4D|PMBlob9;691d?Gn(3oui{r@ z^$}S6i(iA(7gQmCM$ND}mc9aK$w<}~-t?+5@je(enG*#S1sY;>)7pu1Bd^3ci^*8u z2ay?OEO%f{#}{+lUq$-?Z~$-9DB1If_Py4_o`>J}yacLih?BhhD9#plGfSf5f-p`mzQe;hRkyoTNh`MZsv{2eU41^tGVzunrn z9nKZR6;@3pR*{FPE|{3_k&5VP7R^2nM=58e+E7DwJvN%pjr9}8T_MmT)`85$bJ;rSq7sC3N(R?7+WOy8!R8e-Ehon z_J9)%vF*Gd$!W=TEv`YlTsek-0Hu^tQ8&_?+ z?z8mUL&vG_sn^}1o)TI#&Zvs7e^3(9+fyM5M{4)n%~=_}4Qu;0W(;I$YHzAQe2+UT z`d;)QJgWHgw%&gXq~fw4JYB2bn(e;9I?m9Y=*G`c;4 z@aH=ohFY}E*8Pcu9z@4d>9__e)aKoQ8>=dMPIf2e!;xp@jm=9jsZAQ^{pq?`-mn{{mygM=qT7YI4=a$?(~dXRRHc4H-;q%|v?`LMGXtxm zN*q;l`F2YUw8H9?spcFTuOTb1Kg6F$QKzZkbEs0M?62OWZocFyvj;$yV4gPk?jq=P zdj8W|MSU39*yPL`XMgu$Mo)9^;jImK0P`RMh`aJpzc0h zsozLEhY>g*{d+#t&5a+Gvw^gBifwuVeX5Ux$zoiN+7aZbPlim}&Z4aXKLz_gA@0#1 zt4s8pc{zRfSmNV(W=@WLxj)U^M=xf^;d5Nt5S8kU*Vo@dtr~YTH$tr%bn_wHb+EN+ zb0YpRtX2B6iDQrl`+Hf7(eAZ6bA<^^ZiN~kWK65ZATG2gGhi1eb;dE|MOUj!hw9rB=Yo~g z%R!oE|A)jprLTuvBt0LYKT!eQlo-eD>u`vew*8>Q_L;Pi9vijB$;dHJ6v1g*9|K zS)&idJ;eA_oo+`bY@E;6H9U8QH%;{mF#9v3HX-CKP(eudB<_d(@UT<$8$1i;&Z4{p z^SJDM;$_dO5*>6tPd|OG+MsC@;5}!s6`;*Q^@i10|BX*R>fIGo08MnNZ-gkz?0x9( z(i|;ezQ_vT@(M3tUrApWWw}kape6u{U}uLFs{{f97f`Jh&d}Lai>YI zh}SUxaj?dkHIQS6vj#`t&MnOsr)V3V)fqAJf3|V|dc7DO^4mL0sv}fp{il)~^RW`CO zc%z1c%r8K$O@=o5dr^8ldMT$_ykvD3q6y>eX+(`RCxDMiVj1o$ZHCJEveNTzcVk7= zjQyLq$LYom<69TU!Zt#F98c7TF}d!WTTUmUCxkO=S3VPc{Bx@P@QFY3xq;(-I`FHG zUfHGtAlIscm+ZTJ=fkp|jK#AhviISfI{R-9_S}%yMU2#c0=D1B2H`ZO-hk(JttOad zt^1f8xFq63NnYu92K3J=BX?#^JUUU*H@?@$dj;-IdD7B`_!ctYR(f}jHkcRrgaKB>hi37{s(P0`4j*E literal 0 HcmV?d00001 diff --git a/onedrive.conf b/onedrive.conf new file mode 100644 index 00000000..29e9b881 --- /dev/null +++ b/onedrive.conf @@ -0,0 +1,3 @@ +client_id = "000000004C15842F" +client_secret = "5vWj5xi6rYZM61X81Z9OyXAmjGhVS6Py" +sync_dir = "~/OneDrive/" diff --git a/src/cache.d b/src/cache.d new file mode 100644 index 00000000..6ea70ef6 --- /dev/null +++ b/src/cache.d @@ -0,0 +1,163 @@ +module cache; + +import std.datetime: SysTime, time_t; +import sqlite; + +enum ItemType +{ + file, + dir +} + +struct Item +{ + string id; + string path; + string name; + ItemType type; + string eTag; + string cTag; + SysTime mtime; + string parentId; + string crc32; +} + +struct ItemCache +{ + Database db; + Statement insertItemStmt; + Statement selectItemByIdStmt; + Statement selectItemByPathStmt; + + void init() + { + db = Database("cache.db"); + db.exec("CREATE TABLE IF NOT EXISTS item ( + id TEXT PRIMARY KEY, + path TEXT UNIQUE NOT NULL, + name TEXT NOT NULL, + type TEXT NOT NULL, + eTag TEXT NOT NULL, + cTag TEXT NOT NULL, + mtime TEXT NOT NULL, + parentId TEXT NOT NULL, + crc32 TEXT + )"); + db.exec("CREATE UNIQUE INDEX IF NOT EXISTS path_idx ON item (path)"); + insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, path, name, type, eTag, cTag, mtime, parentId, crc32) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"); + selectItemByIdStmt = db.prepare("SELECT id, path, name, type, eTag, cTag, mtime, parentId, crc32 FROM item WHERE id = ?"); + selectItemByPathStmt = db.prepare("SELECT id, path, name, type, eTag, cTag, mtime, parentId, crc32 FROM item WHERE path = ?"); + } + + void insert(const(char)[] id, const(char)[] name, ItemType type, const(char)[] eTag, const(char)[] cTag, const(char)[] mtime, const(char)[] parentId, const(char)[] crc32) + { + with (insertItemStmt) { + bind(1, id); + bind(2, computePath(name, parentId)); + bind(3, name); + string typeStr = void; + final switch (type) { + case ItemType.file: typeStr = "file"; break; + case ItemType.dir: typeStr = "dir"; break; + } + bind(4, typeStr); + bind(5, eTag); + bind(6, cTag); + bind(7, mtime); + bind(8, parentId); + bind(9, crc32); + exec(); + } + } + + bool selectById(const(char)[] id, out Item item) + { + selectItemByIdStmt.bind(1, id); + auto r = selectItemByIdStmt.exec(); + if (!r.empty) { + item = buildItem(r); + return true; + } + return false; + } + + bool selectByPath(const(char)[] path, out Item item) + { + selectItemByPathStmt.bind(1, path); + auto r = selectItemByPathStmt.exec(); + if (!r.empty) { + item = buildItem(r); + return true; + } + return false; + } + + void updateModifiedTime(const(char)[] id, const(char)[] mtime) + { + auto s = db.prepare("UPDATE mtime FROM item WHERE id = ?"); + s.bind(1, id); + s.exec(); + } + + void deleteById(const(char)[] id) + { + auto s = db.prepare("DELETE FROM item WHERE id = ?"); + s.bind(1, id); + s.exec(); + } + + // returns true if the item has the specified parent + bool hasParent(T)(const(char)[] itemId, T parentId) + if (is(T : const(char)[]) || is(T : const(char[])[])) + { + auto s = db.prepare("SELECT parentId FROM item WHERE id = ?"); + while (true) { + s.bind(1, itemId); + auto r = s.exec(); + if (r.empty) break; + auto currParentId = r.front[0]; + static if (is(T : const(char)[])) { + if (currParentId == parentId) return true; + } else { + foreach (id; parentId) if (currParentId == id) return true; + } + itemId = currParentId.dup; + } + return false; + } + + private Item buildItem(Statement.Result result) + { + assert(!result.empty && result.front.length == 9); + Item item = { + id: result.front[0].dup, + path: result.front[1].dup, + name: result.front[2].dup, + eTag: result.front[4].dup, + cTag: result.front[5].dup, + mtime: SysTime.fromISOExtString(result.front[6]), + parentId: result.front[7].dup, + crc32: result.front[8].dup + }; + switch (result.front[3]) { + case "file": item.type = ItemType.file; break; + case "dir": item.type = ItemType.dir; break; + default: assert(0); + } + return item; + } + + private string computePath(const(char)[] name, const(char)[] parentId) + { + auto s = db.prepare("SELECT name, parentId FROM item WHERE id = ?"); + string path = name.dup; + while (true) { + s.bind(1, parentId); + auto r = s.exec(); + if (r.empty) break; + path = r.front[0].idup ~ "/" ~ path; + parentId = r.front[1].dup; + } + return path; + } +} diff --git a/src/config.d b/src/config.d new file mode 100644 index 00000000..def78456 --- /dev/null +++ b/src/config.d @@ -0,0 +1,64 @@ +import std.regex, std.stdio, std.file; + +final class Config +{ + private string filename; + private string[string] values; + + this(string filename) + { + this.filename = filename; + load(); + } + + string get(string key) + { + return values[key]; + } + + void set(string key, string value) + { + values[key] = value; + } + + void load() + { + values = null; + scope (failure) return; + auto file = File(filename, "r"); + auto r = regex("(?:^\\s*)(\\w+)(?:\\s*=\\s*\")(.*)(?:\"\\s*$)"); + foreach (line; file.byLine()) { + auto c = matchFirst(line, r); + if (!c.empty) { + c.popFront(); // skip whole match + string key = c.front.dup; + c.popFront(); + values[key] = c.front.dup; + } + } + } + + void save() + { + if (exists(filename)) { + string bkpFilename = filename ~ "~"; + rename(filename, bkpFilename); + } + auto file = File(filename, "w"); + foreach (key, value; values) { + file.writeln(key, " = \"", value, "\""); + } + } +} + +unittest +{ + auto cfg = new Config("/tmp/test.conf"); + cfg.set("test1", "1"); + cfg.set("test2", "2"); + cfg.set("test1", "3"); + cfg.save(); + cfg.load(); + assert(cfg.get("test1") == "3"); + assert(cfg.get("test2") == "2"); +} diff --git a/src/main.d b/src/main.d new file mode 100644 index 00000000..657b88de --- /dev/null +++ b/src/main.d @@ -0,0 +1,30 @@ +import std.file; +import config, onedrive, sync; + +private string configFile = "./onedrive.conf"; +private string refreshTokenFile = "refresh_token"; + +void main() +{ + auto cfg = new Config(configFile); + + auto onedrive = new OneDriveApi(cfg.get("client_id"), cfg.get("client_secret")); + try { + string refreshToken = readText(refreshTokenFile); + onedrive.setRefreshToken(refreshToken); + } catch (FileException e) { + onedrive.authorize(); + } + onedrive.onRefreshToken = (string refreshToken) { std.file.write(refreshTokenFile, refreshToken); }; + + auto sync = new SyncEngine(cfg, onedrive); + sync.applyDifferences(); + + /*import std.stdio; + import std.net.curl; + try { + onedrive.simpleUpload("a.txt", "a.txt", "error").toPrettyString.writeln; + } catch (CurlException e) { + writeln("exc ", e.msg); + }*/ +} diff --git a/src/onedrive.d b/src/onedrive.d new file mode 100644 index 00000000..dd87c80a --- /dev/null +++ b/src/onedrive.d @@ -0,0 +1,219 @@ +module onedrive; + +import std.json, std.net.curl, std.path, std.string, std.uni, std.uri; + +extern(C) void signal(int sig, void function(int)); + +private immutable { + string authUrl = "https://login.live.com/oauth20_authorize.srf"; + string redirectUrl = "https://login.live.com/oauth20_desktop.srf"; + string tokenUrl = "https://login.live.com/oauth20_token.srf"; + string itemByIdUrl = "https://api.onedrive.com/v1.0/drive/items/"; + string itemByPathUrl = "https://api.onedrive.com/v1.0/drive/root:/"; +} + +class OneDriveException: Exception +{ + @nogc @safe pure nothrow this(string msg, string file = __FILE__, size_t line = __LINE__, Throwable next = null) + { + super(msg, file, line, next); + } + + @nogc @safe pure nothrow this(string msg, Throwable next, string file = __FILE__, size_t line = __LINE__) + { + super(msg, file, line, next); + } +} + +final class OneDriveApi +{ + private string clientId, clientSecret; + private string refreshToken, accessToken; + private HTTP http; + + void function(string) onRefreshToken; // called when a new refresh_token is received + + this(string clientId, string clientSecret) + { + this.clientId = clientId; + this.clientSecret = clientSecret; + http = HTTP(); + //debug http.verbose = true; + // HACK: prevent SIGPIPE + //import etc.c.curl; + //http.handle.set(CurlOption.nosignal, 0); + //signal(/*SIGPIPE*/ 13, /*SIG_IGN*/ cast(void function(int)) 1); + } + + ~this() + { + http.shutdown(); + } + + void authorize() + { + import std.stdio, std.regex; + string url = authUrl ~ "?client_id=" ~ clientId ~ "&scope=wl.offline_access onedrive.readwrite&response_type=code&redirect_url=" ~ redirectUrl; + writeln("Authorize this app visiting:"); + writeln(url); + + while (true) { + char[] response; + write("Enter the response url: "); + readln(response); + auto c = matchFirst(response, r"(?:code=)(([\w\d]+-){4}[\w\d]+)"); + if (!c.empty) { + c.popFront(); // skip whole match + redeemToken(c.front); + break; + } + } + } + + void setRefreshToken(string refreshToken) + { + this.refreshToken = refreshToken; + newToken(); + } + + string getItemPath(const(char)[] id) + { + JSONValue response = get(itemByIdUrl ~ id ~ "/?select=name,parentReference"); + string path; + try { + path = response["parentReference"].object["path"].str; + } catch (JSONException e) { + // root does not have parentReference + return ""; + } + path = decodeComponent(path[path.indexOf(':') + 1 .. $]); + return buildNormalizedPath("." ~ path ~ "/" ~ response["name"].str); + } + + string getItemId(const(char)[] path) + { + JSONValue response = get(itemByPathUrl ~ encodeComponent(path) ~ ":/?select=id"); + return response["id"].str; + } + + // https://dev.onedrive.com/items/view_changes.htm + JSONValue viewChangesById(const(char)[] id, const(char)[] statusToken) + { + char[] url = itemByIdUrl ~ id ~ "/view.changes"; + if (statusToken) url ~= "?token=" ~ statusToken; + return get(url); + } + + // https://dev.onedrive.com/items/view_changes.htm + JSONValue viewChangesByPath(const(char)[] path, const(char)[] statusToken) + { + char[] url = itemByPathUrl ~ encodeComponent(path).dup ~ ":/view.changes"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,fileSystemInfo,parentReference"; + if (statusToken) url ~= "&token=" ~ statusToken; + return get(url); + } + + // https://dev.onedrive.com/items/download.htm + void downloadById(const(char)[] id, string saveToPath) + { + /*string downloadUrl; + // obtain the download url + http.url = itemByIdUrl ~ id ~ "/content"; + http.method = HTTP.Method.get; + http.maxRedirects = 0; + http.onReceive = (ubyte[] data) { return data.length; }; + http.onReceiveHeader = (in char[] key, in char[] value) { + if (sicmp(key, "location") == 0) { + http.onReceiveHeader = null; + downloadUrl = value.dup; + } + }; + writeln("Obtaining the url ..."); + http.perform(); + check(); + http.maxRedirects = 10; + if (downloadUrl) { + // try to download the file + try { + download(downloadUrl, saveToPath); + } catch (CurlException e) { + import std.file; + if (exists(saveToPath)) remove(saveToPath); + throw new OneDriveException("Download error", e); + } + } else { + throw new OneDriveException("Can't obtain the download url"); + }*/ + char[] url = itemByIdUrl ~ id ~ "/content"; + try { + download(url, saveToPath, http); + } catch (CurlException e) { + import std.file; + if (exists(saveToPath)) remove(saveToPath); + throw new OneDriveException("Download error", e); + } + } + + // https://dev.onedrive.com/items/upload_put.htm + auto simpleUpload(string localPath, const(char)[] remotePath, const(char)[] eTag = null) + { + char[] url = itemByPathUrl ~ remotePath ~ ":/content"; + ubyte[] content; + http.onReceive = (ubyte[] data) { + content ~= data; + return data.length; + }; + if (eTag) http.addRequestHeader("If-Match", eTag); + upload(localPath, url, http); + // remove the if-match header + if (eTag) setAccessToken(accessToken); + check(); + return parseJSON(content); + } + + private void redeemToken(const(char)[] authCode) + { + string postData = "client_id=" ~ clientId ~ "&redirect_url=" ~ redirectUrl ~ "&client_secret=" ~ clientSecret; + postData ~= "&code=" ~ authCode ~ "&grant_type=authorization_code"; + acquireToken(postData); + } + + private void newToken() + { + string postData = "client_id=" ~ clientId ~ "&redirect_url=" ~ redirectUrl ~ "&client_secret=" ~ clientSecret; + postData ~= "&refresh_token=" ~ refreshToken ~ "&grant_type=refresh_token"; + acquireToken(postData); + } + + private void acquireToken(const(char)[] postData) + { + JSONValue response = post(tokenUrl, postData); + setAccessToken(response["access_token"].str()); + refreshToken = response["refresh_token"].str().dup; + if (onRefreshToken) onRefreshToken(refreshToken); + } + + private void setAccessToken(string accessToken) + { + http.clearRequestHeaders(); + this.accessToken = accessToken; + http.addRequestHeader("Authorization", "bearer " ~ accessToken); + } + + private auto get(const(char)[] url) + { + return parseJSON(.get(url, http)); + } + + private auto post(T)(const(char)[] url, const(T)[] postData) + { + return parseJSON(.post(url, postData, http)); + } + + private void check() + { + if (http.statusLine.code / 100 != 2) { + throw new OneDriveException(format("HTTP request returned status code %d (%s)", http.statusLine.code, http.statusLine.reason)); + } + } +} diff --git a/src/sqlite.d b/src/sqlite.d new file mode 100644 index 00000000..b4a8e7d3 --- /dev/null +++ b/src/sqlite.d @@ -0,0 +1,181 @@ +module sqlite; + +import etc.c.sqlite3; +import std.string: fromStringz, toStringz; + +extern (C) immutable(char)* sqlite3_errstr(int); // missing from the std library + +private string ifromStringz(const(char)* cstr) +{ + return fromStringz(cstr).dup; +} + +class SqliteException: Exception +{ + @safe pure nothrow this(string msg, string file = __FILE__, size_t line = __LINE__, Throwable next = null) + { + super(msg, file, line, next); + } + + @safe pure nothrow this(string msg, Throwable next, string file = __FILE__, size_t line = __LINE__) + { + super(msg, file, line, next); + } +} + +struct Database +{ + private sqlite3* pDb; + + this(const(char)[] filename) + { + open(filename); + } + + ~this() + { + close(); + } + + void open(const(char)[] filename) + { + // https://www.sqlite.org/c3ref/open.html + int rc = sqlite3_open(toStringz(filename), &pDb); + if (rc != SQLITE_OK) { + close(); + throw new SqliteException(ifromStringz(sqlite3_errstr(rc))); + } + sqlite3_extended_result_codes(pDb, 1); // always use extended result codes + } + + void exec(const(char)[] sql) + { + // https://www.sqlite.org/c3ref/exec.html + int rc = sqlite3_exec(pDb, toStringz(sql), null, null, null); + if (rc != SQLITE_OK) { + throw new SqliteException(ifromStringz(sqlite3_errmsg(pDb))); + } + } + + Statement prepare(const(char)[] zSql) + { + Statement s; + // https://www.sqlite.org/c3ref/prepare.html + int rc = sqlite3_prepare_v2(pDb, zSql.ptr, cast(int) zSql.length, &s.pStmt, null); + if (rc != SQLITE_OK) { + throw new SqliteException(ifromStringz(sqlite3_errmsg(pDb))); + } + return s; + } + + void close() + { + // https://www.sqlite.org/c3ref/close.html + sqlite3_close_v2(pDb); + pDb = null; + } +} + +struct Statement +{ + struct Result + { + private sqlite3_stmt* pStmt; + private const(char)[][] row; + + private this(sqlite3_stmt* pStmt) + { + this.pStmt = pStmt; + step(); // initialize the range + } + + @property bool empty() + { + return row.length == 0; + } + + @property auto front() + { + return row; + } + + alias step popFront; + + void step() + { + // https://www.sqlite.org/c3ref/step.html + int rc = sqlite3_step(pStmt); + if (rc == SQLITE_DONE) { + row.length = 0; + } else if (rc == SQLITE_ROW) { + // https://www.sqlite.org/c3ref/data_count.html + int count = sqlite3_data_count(pStmt); + row = new const(char)[][count]; + foreach (int i, ref column; row) { + // https://www.sqlite.org/c3ref/column_blob.html + column = fromStringz(sqlite3_column_text(pStmt, i)); + } + } else { + throw new SqliteException(ifromStringz(sqlite3_errmsg(sqlite3_db_handle(pStmt)))); + } + } + } + + private sqlite3_stmt* pStmt; + + ~this() + { + // https://www.sqlite.org/c3ref/finalize.html + sqlite3_finalize(pStmt); + } + + void bind(int index, const(char)[] value) + { + reset(); + // https://www.sqlite.org/c3ref/bind_blob.html + int rc = sqlite3_bind_text(pStmt, index, value.ptr, cast(int) value.length, SQLITE_STATIC); + if (rc != SQLITE_OK) { + throw new SqliteException(ifromStringz(sqlite3_errmsg(sqlite3_db_handle(pStmt)))); + } + } + + Result exec() + { + reset(); + return Result(pStmt); + } + + private void reset() + { + // https://www.sqlite.org/c3ref/reset.html + int rc = sqlite3_reset(pStmt); + if (rc != SQLITE_OK) { + throw new SqliteException(ifromStringz(sqlite3_errmsg(sqlite3_db_handle(pStmt)))); + } + } +} + +unittest +{ + auto db = Database(":memory:"); + db.exec("CREATE TABLE test( + id TEXT PRIMARY KEY, + value TEXT + )"); + + auto s = db.prepare("INSERT INTO test VALUES (?, ?)"); + s.bind(1, "key1"); + s.bind(2, "value1"); + s.exec(); + s.bind(1, "key2"); + s.bind(2, "value2"); + s.exec(); + + s = db.prepare("SELECT * FROM test ORDER BY id ASC"); + auto r = s.exec(); + assert(r.front[0] == "key1"); + r.popFront(); + assert(r.front[1] == "value2"); + r.popFront(); + assert(r.empty); +} diff --git a/src/sync.d b/src/sync.d new file mode 100644 index 00000000..e2ef6a10 --- /dev/null +++ b/src/sync.d @@ -0,0 +1,277 @@ +import std.stdio, std.file, std.json; +import cache, config, onedrive, util; + +private string statusTokenFile = "status_token"; + +private bool isItemFolder(const ref JSONValue item) +{ + scope (failure) return false; + JSONValue folder = item["folder"]; + return true; +} + +private bool isItemFile(const ref JSONValue item) +{ + scope (failure) return false; + JSONValue folder = item["file"]; + return true; +} + +private bool isItemDeleted(const ref JSONValue item) +{ + scope (failure) return false; + return !item["deleted"].isNull(); +} + +class SyncException: Exception +{ + @nogc @safe pure nothrow this(string msg, string file = __FILE__, size_t line = __LINE__, Throwable next = null) + { + super(msg, file, line, next); + } + + @nogc @safe pure nothrow this(string msg, Throwable next, string file = __FILE__, size_t line = __LINE__) + { + super(msg, file, line, next); + } +} + +final class SyncEngine +{ + Config cfg; + OneDriveApi onedrive; + ItemCache itemCache; + string[] itemToDelete; // array of items to be deleted + + this(Config cfg, OneDriveApi onedrive) + { + assert(onedrive); + this.cfg = cfg; + this.onedrive = onedrive; + itemCache.init(); + } + + void applyDifferences() + { + string statusToken; + try { + statusToken = readText(statusTokenFile); + } catch (FileException e) { + writeln("Welcome !"); + } + writeln("Checking for changes..."); + + string currDir = getcwd(); + string syncDir = cfg.get("sync_dir"); + + JSONValue changes; + do { + chdir(syncDir); + changes = onedrive.viewChangesByPath("Politecnico", statusToken); + foreach (item; changes["value"].array) { + applyDifference(item); + } + statusToken = changes["@changes.token"].str; + chdir(currDir); + std.file.write(statusTokenFile, statusToken); + } while (changes["@changes.hasMoreChanges"].type == JSON_TYPE.TRUE); + chdir(syncDir); + deleteFiles(); + chdir(currDir); + } + + private void applyDifference(JSONValue item) + { + string id = item["id"].str; + string name = item["name"].str; + string eTag = item["eTag"].str; + + Item cachedItem; + bool cached = itemCache.selectById(id, cachedItem); + + // skip items already downloaded + //if (cached && cachedItem.eTag == eTag) return; + + writeln("Item ", id, " ", name); + + ItemType type; + if (isItemDeleted(item)) { + writeln("The item is marked for deletion"); + if (cached) { + applyDelete(cachedItem); + } + return; + } else if (isItemFile(item)) { + type = ItemType.file; + writeln("The item is a file"); + } else if (isItemFolder(item)) { + type = ItemType.dir; + writeln("The item is a directory"); + } else { + writeln("The item is neither a file nor a directory, skipping"); + //skippedFolders ~= id; + return; + } + + string cTag = item["cTag"].str; + string mtime = item["fileSystemInfo"].object["lastModifiedDateTime"].str; + string parentId = item["parentReference"].object["id"].str; + + string crc32; + if (type == ItemType.file) { + try { + crc32 = item["file"].object["hashes"].object["crc32Hash"].str; + } catch (JSONException e) { + writeln("The hash is not available"); + } + } + + Item newItem; + itemCache.insert(id, name, type, eTag, cTag, mtime, parentId, crc32); + itemCache.selectById(id, newItem); + + writeln("Path: ", newItem.path); + + try { + if (!cached) { + applyNewItem(newItem); + } else { + applyChangedItem(cachedItem, newItem); + } + } catch (SyncException e) { + itemCache.deleteById(id); + throw e; + } + } + + private void applyDelete(Item item) + { + if (exists(item.path)) { + if (isItemSynced(item, true)) { + addFileToDelete(item.path); + } else { + writeln("The local item is not synced, renaming ..."); + safeRename(item.path); + } + } else { + writeln("The local item is already deleted"); + } + } + private void applyNewItem(Item item) + { + assert(item.id); + if (exists(item.path)) { + if (isItemSynced(item, true)) { + writeln("The item is already present"); + // ensure the modified time is synced + setTimes(item.path, item.mtime, item.mtime); + return; + } else { + writeln("The item is not synced, renaming ..."); + safeRename(item.path); + } + } + final switch (item.type) { + case ItemType.file: + writeln("Downloading ..."); + try { + onedrive.downloadById(item.id, item.path); + } catch (OneDriveException e) { + throw new SyncException("Sync error", e); + } + break; + case ItemType.dir: + writeln("Creating local directory..."); + mkdir(item.path); + break; + } + setTimes(item.path, item.mtime, item.mtime); + } + + private void applyChangedItem(Item oldItem, Item newItem) + { + assert(oldItem.id == newItem.id); + if (exists(oldItem.path)) { + if (isItemSynced(oldItem)) { + if (oldItem.eTag != newItem.eTag) { + assert(oldItem.type == newItem.type); + if (oldItem.path != newItem.path) { + writeln("Moved item ", oldItem.path, " to ", newItem.path); + if (exists(newItem.path)) { + writeln("The destination is occupied, renaming ..."); + safeRename(newItem.path); + } + rename(oldItem.path, newItem.path); + } + if (oldItem.type == ItemType.file && oldItem.cTag != newItem.cTag) { + writeln("Downloading ..."); + onedrive.downloadById(oldItem.id, oldItem.path); + } + setTimes(newItem.path, newItem.mtime, newItem.mtime); + writeln("Updated last modified time"); + } else { + writeln("The item is not changed"); + } + } else { + writeln("The item is not synced, renaming ..."); + safeRename(oldItem.path); + applyNewItem(newItem); + } + } else { + applyNewItem(newItem); + } + } + + // returns true if the given item corresponds to the local one + private bool isItemSynced(Item item, bool checkHash = false) + { + final switch (item.type) { + case ItemType.file: + if (isFile(item.path)) { + SysTime localModifiedTime = timeLastModified(item.path); + import core.time: Duration; + item.mtime.fracSecs = Duration.zero; // HACK + if (localModifiedTime == item.mtime) return true; + else { + writeln("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime); + } + if (checkHash && item.crc32) { + string localCrc32 = computeCrc32(item.path); + if (localCrc32 == item.crc32) return true; + else { + writeln("The local item has a different hash"); + } + } + } else { + writeln("The local item is a directory but should be a file"); + } + break; + case ItemType.dir: + if (isDir(item.path)) return true; + else { + writeln("The local item is a file but should be a directory"); + } + break; + } + return false; + } + + private void addFileToDelete(string path) + { + itemToDelete ~= path; + } + + private void deleteFiles() + { + foreach_reverse (ref path; itemToDelete) { + if (isFile(path)) { + remove(path); + } else { + // TODO: test not empty folder + rmdir(path); + } + } + assumeSafeAppend(itemToDelete); + itemToDelete.length = 0; + } +} diff --git a/src/util.d b/src/util.d new file mode 100644 index 00000000..19bb2f24 --- /dev/null +++ b/src/util.d @@ -0,0 +1,44 @@ +import std.conv: to; +import std.digest.crc; +import std.digest.digest; +import std.stdio; +import std.string: chomp; +import std.file: exists, rename; +import std.path: extension; + +private string deviceName; + +static this() +{ + import std.socket; + deviceName = Socket.hostName; +} + +// give a new name to the specified file or directory +void safeRename(const(char)[] path) +{ + auto ext = extension(path); + auto newPath = path.chomp(ext) ~ "-" ~ deviceName; + if (exists(newPath ~ ext)) { + int n = 2; + char[] newPath2; + do { + newPath2 = newPath ~ "-" ~ n.to!string; + n++; + } while (exists(newPath2 ~ ext)); + newPath = newPath2; + } + newPath ~= ext; + rename(path, newPath); +} + +// return the crc32 hex string of a file +string computeCrc32(string path) +{ + CRC32 crc; + auto file = File(path, "rb"); + foreach (ubyte[] data; chunks(file, 4096)) { + crc.put(data); + } + return crc.finish().toHexString().dup; +}