deblan-mirror/dnote
1
0
Fork 0
mirror of https://github.com/dnote/dnote synced 2026-03-14 14:35:50 +01:00
Code Issues Projects Releases Packages Wiki Activity

Fix an edge case of repeated syncs due to orphaned note (#704)

Browse source 
* Split sync test

* Reproduce a bug

* Fix a bug

* Fix in a more correct way

* Add debug logs
This commit is contained in:
Sung 2025-10-26 11:43:17 -07:00 • committed by GitHub
commit a46afb821f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 1204 additions and 752 deletions
Download patch file Download diff file Expand all files Collapse all files

4
pkg/cli/client/client.go
View file

@ -176,7 +176,7 @@ func doReq(ctx context.DnoteCtx, method, path, body string, options *requestOpti
return nil, errors.Wrap(err, "getting request")
}
log.Debug("HTTP request: %+v\n", req)
log.Debug("HTTP %s %s\n", method, path)
hc := getHTTPClient(ctx, options)
res, err := hc.Do(req)
@ -184,7 +184,7 @@ func doReq(ctx context.DnoteCtx, method, path, body string, options *requestOpti
return res, errors.Wrap(err, "making http request")
}
log.Debug("HTTP response: %+v\n", res)
log.Debug("HTTP %d %s\n", res.StatusCode, res.Status)
if err = checkRespErr(res); err != nil {
return res, errors.Wrap(err, "server responded with an error")

196
pkg/cli/cmd/sync/sync.go
View file

@ -90,6 +90,7 @@ type syncList struct {
ExpungedNotes map[string]bool
ExpungedBooks map[string]bool
MaxUSN int
UserMaxUSN int // Server's actual max USN (for distinguishing empty fragment vs empty server)
MaxCurrentTime int64
}
@ -104,6 +105,7 @@ func processFragments(fragments []client.SyncFragment) (syncList, error) {
expungedNotes := map[string]bool{}
expungedBooks := map[string]bool{}
var maxUSN int
var userMaxUSN int
var maxCurrentTime int64
for _, fragment := range fragments {
@ -123,6 +125,9 @@ func processFragments(fragments []client.SyncFragment) (syncList, error) {
if fragment.FragMaxUSN > maxUSN {
maxUSN = fragment.FragMaxUSN
}
if fragment.UserMaxUSN > userMaxUSN {
userMaxUSN = fragment.UserMaxUSN
}
if fragment.CurrentTime > maxCurrentTime {
maxCurrentTime = fragment.CurrentTime
}
@ -134,6 +139,7 @@ func processFragments(fragments []client.SyncFragment) (syncList, error) {
ExpungedNotes: expungedNotes,
ExpungedBooks: expungedBooks,
MaxUSN: maxUSN,
UserMaxUSN: userMaxUSN,
MaxCurrentTime: maxCurrentTime,
}
@ -180,11 +186,68 @@ func getSyncFragments(ctx context.DnoteCtx, afterUSN int) ([]client.SyncFragment
}
}
log.Debug("received sync fragments: %+v\n", buf)
log.Debug("received sync fragments: %+v\n", redactSyncFragments(buf))
return buf, nil
}
// redactSyncFragments returns a deep copy of sync fragments with sensitive fields (note body, book label) removed for safe logging
func redactSyncFragments(fragments []client.SyncFragment) []client.SyncFragment {
redacted := make([]client.SyncFragment, len(fragments))
for i, frag := range fragments {
// Create new notes with redacted bodies
notes := make([]client.SyncFragNote, len(frag.Notes))
for j, note := range frag.Notes {
notes[j] = client.SyncFragNote{
UUID: note.UUID,
BookUUID: note.BookUUID,
USN: note.USN,
CreatedAt: note.CreatedAt,
UpdatedAt: note.UpdatedAt,
AddedOn: note.AddedOn,
EditedOn: note.EditedOn,
Body: func() string {
if note.Body != "" {
return "<redacted>"
}
return ""
}(),
Deleted: note.Deleted,
}
}
// Create new books with redacted labels
books := make([]client.SyncFragBook, len(frag.Books))
for j, book := range frag.Books {
books[j] = client.SyncFragBook{
UUID: book.UUID,
USN: book.USN,
CreatedAt: book.CreatedAt,
UpdatedAt: book.UpdatedAt,
AddedOn: book.AddedOn,
Label: func() string {
if book.Label != "" {
return "<redacted>"
}
return ""
}(),
Deleted: book.Deleted,
}
}
redacted[i] = client.SyncFragment{
FragMaxUSN: frag.FragMaxUSN,
UserMaxUSN: frag.UserMaxUSN,
CurrentTime: frag.CurrentTime,
Notes: notes,
Books: books,
ExpungedNotes: frag.ExpungedNotes,
ExpungedBooks: frag.ExpungedBooks,
}
}
return redacted
}
// resolveLabel resolves a book label conflict by repeatedly appending an increasing integer
// to the label until it finds a unique label. It returns the first non-conflicting label.
func resolveLabel(tx *database.DB, label string) (string, error) {
@ -540,6 +603,8 @@ func fullSync(ctx context.DnoteCtx, tx *database.DB) error {
log.Debug("performing a full sync\n")
log.Info("resolving delta.")
log.DebugNewline()
list, err := getSyncList(ctx, 0)
if err != nil {
return errors.Wrap(err, "getting sync list")
@ -547,6 +612,8 @@ func fullSync(ctx context.DnoteCtx, tx *database.DB) error {
fmt.Printf(" (total %d).", list.getLength())
log.DebugNewline()
// clean resources that are in erroneous states
if err := cleanLocalNotes(tx, &list); err != nil {
return errors.Wrap(err, "cleaning up local notes")
@ -577,7 +644,7 @@ func fullSync(ctx context.DnoteCtx, tx *database.DB) error {
}
}
err = saveSyncState(tx, list.MaxCurrentTime, list.MaxUSN)
err = saveSyncState(tx, list.MaxCurrentTime, list.MaxUSN, list.UserMaxUSN)
if err != nil {
return errors.Wrap(err, "saving sync state")
}
@ -592,6 +659,8 @@ func stepSync(ctx context.DnoteCtx, tx *database.DB, afterUSN int) error {
log.Info("resolving delta.")
log.DebugNewline()
list, err := getSyncList(ctx, afterUSN)
if err != nil {
return errors.Wrap(err, "getting sync list")
@ -621,7 +690,7 @@ func stepSync(ctx context.DnoteCtx, tx *database.DB, afterUSN int) error {
}
}
err = saveSyncState(tx, list.MaxCurrentTime, list.MaxUSN)
err = saveSyncState(tx, list.MaxCurrentTime, list.MaxUSN, list.UserMaxUSN)
if err != nil {
return errors.Wrap(err, "saving sync state")
}
@ -677,13 +746,9 @@ func sendBooks(ctx context.DnoteCtx, tx *database.DB) (bool, error) {
} else {
resp, err := client.CreateBook(ctx, book.Label)
if err != nil {
// If we get a 409 conflict, it means another client uploaded data.
if isConflictError(err) {
log.Debug("409 conflict creating book %s, will retry after sync\n", book.Label)
isBehind = true
continue
}
return isBehind, errors.Wrap(err, "creating a book")
log.Debug("error creating book (will retry after stepSync): %v\n", err)
isBehind = true
continue
}
_, err = tx.Exec("UPDATE notes SET book_uuid = ? WHERE book_uuid = ?", resp.Book.UUID, book.UUID)
@ -755,9 +820,91 @@ func sendBooks(ctx context.DnoteCtx, tx *database.DB) (bool, error) {
return isBehind, nil
}
// findOrphanedNotes returns a list of all orphaned notes
func findOrphanedNotes(db *database.DB) (int, []struct{ noteUUID, bookUUID string }, error) {
var orphanCount int
err := db.QueryRow(`
SELECT COUNT(*) FROM notes n
WHERE NOT EXISTS (
SELECT 1 FROM books b
WHERE b.uuid = n.book_uuid
AND NOT b.deleted
)
`).Scan(&orphanCount)
if err != nil {
return 0, nil, err
}
if orphanCount == 0 {
return 0, nil, nil
}
rows, err := db.Query(`
SELECT n.uuid, n.book_uuid
FROM notes n
WHERE NOT EXISTS (
SELECT 1 FROM books b
WHERE b.uuid = n.book_uuid
AND NOT b.deleted
)
`)
if err != nil {
return orphanCount, nil, err
}
defer rows.Close()
var orphans []struct{ noteUUID, bookUUID string }
for rows.Next() {
var noteUUID, bookUUID string
if err := rows.Scan(&noteUUID, &bookUUID); err != nil {
continue
}
orphans = append(orphans, struct{ noteUUID, bookUUID string }{noteUUID, bookUUID})
}
return orphanCount, orphans, nil
}
func warnOrphanedNotes(tx *database.DB) {
count, orphans, err := findOrphanedNotes(tx)
if err != nil {
log.Debug("error checking orphaned notes: %v\n", err)
return
}
if count == 0 {
return
}
log.Debug("Found %d orphaned notes (book doesn't exist locally):\n", count)
for _, o := range orphans {
log.Debug("note %s (book %s)\n", o.noteUUID, o.bookUUID)
}
}
// checkPostSyncIntegrity checks for data integrity issues after sync and warns the user
func checkPostSyncIntegrity(db *database.DB) {
count, orphans, err := findOrphanedNotes(db)
if err != nil {
log.Debug("error checking orphaned notes: %v\n", err)
return
}
if count == 0 {
return
}
log.Warnf("Found %d orphaned notes (referencing non-existent or deleted books):\n", count)
for _, o := range orphans {
log.Plainf(" - note %s (missing book: %s)\n", o.noteUUID, o.bookUUID)
}
}
func sendNotes(ctx context.DnoteCtx, tx *database.DB) (bool, error) {
isBehind := false
warnOrphanedNotes(tx)
rows, err := tx.Query("SELECT uuid, book_uuid, body, deleted, usn, added_on FROM notes WHERE dirty")
if err != nil {
return isBehind, errors.Wrap(err, "getting syncable notes")
@ -771,7 +918,7 @@ func sendNotes(ctx context.DnoteCtx, tx *database.DB) (bool, error) {
return isBehind, errors.Wrap(err, "scanning a syncable note")
}
log.Debug("sending note %s\n", note.UUID)
log.Debug("sending note %s (book: %s)\n", note.UUID, note.BookUUID)
var respUSN int
@ -788,8 +935,7 @@ func sendNotes(ctx context.DnoteCtx, tx *database.DB) (bool, error) {
} else {
resp, err := client.CreateNote(ctx, note.BookUUID, note.Body)
if err != nil {
// If we get a 409 conflict, it means another client uploaded data.
log.Debug("error creating note (will retry after sync): %v\n", err)
log.Debug("failed to create note %s (book: %s): %v\n", note.UUID, note.BookUUID, err)
isBehind = true
continue
}
@ -866,6 +1012,8 @@ func sendChanges(ctx context.DnoteCtx, tx *database.DB) (bool, error) {
fmt.Printf(" (total %d).", delta)
log.DebugNewline()
behind1, err := sendBooks(ctx, tx)
if err != nil {
return behind1, errors.Wrap(err, "sending books")
@ -899,10 +1047,24 @@ func updateLastSyncAt(tx *database.DB, val int64) error {
return nil
}
func saveSyncState(tx *database.DB, serverTime int64, serverMaxUSN int) error {
if err := updateLastMaxUSN(tx, serverMaxUSN); err != nil {
return errors.Wrap(err, "updating last max usn")
func saveSyncState(tx *database.DB, serverTime int64, serverMaxUSN int, userMaxUSN int) error {
// Handle last_max_usn update based on server state:
// - If serverMaxUSN > 0: we got data, update to serverMaxUSN
// - If serverMaxUSN == 0 && userMaxUSN > 0: empty fragment (caught up), preserve existing
// - If serverMaxUSN == 0 && userMaxUSN == 0: empty server, reset to 0
if serverMaxUSN > 0 {
if err := updateLastMaxUSN(tx, serverMaxUSN); err != nil {
return errors.Wrap(err, "updating last max usn")
}
} else if userMaxUSN == 0 {
// Server is empty, reset to 0
if err := updateLastMaxUSN(tx, 0); err != nil {
return errors.Wrap(err, "updating last max usn")
}
}
// else: empty fragment but server has data, preserve existing last_max_usn
// Always update last_sync_at (we did communicate with server)
if err := updateLastSyncAt(tx, serverTime); err != nil {
return errors.Wrap(err, "updating last sync at")
}
@ -1065,6 +1227,8 @@ func newRun(ctx context.DnoteCtx) infra.RunEFunc {
log.Success("success\n")
checkPostSyncIntegrity(ctx.DB)
if err := upgrade.Check(ctx); err != nil {
log.Error(errors.Wrap(err, "automatically checking updates").Error())
}

147
pkg/cli/cmd/sync/sync_test.go
View file

@ -36,7 +36,6 @@ import (
"github.com/pkg/errors"
)
func TestProcessFragments(t *testing.T) {
fragments := []client.SyncFragment{
{
@ -98,6 +97,7 @@ func TestProcessFragments(t *testing.T) {
ExpungedNotes: map[string]bool{},
ExpungedBooks: map[string]bool{},
MaxUSN: 10,
UserMaxUSN: 10,
MaxCurrentTime: 1550436136,
}
@ -1796,41 +1796,132 @@ func TestMergeBook(t *testing.T) {
}
func TestSaveServerState(t *testing.T) {
// set up
db := database.InitTestMemoryDB(t)
testutils.LoginDB(t, db)
t.Run("with data received", func(t *testing.T) {
// set up
db := database.InitTestMemoryDB(t)
testutils.LoginDB(t, db)
database.MustExec(t, "inserting last synced at", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemLastSyncAt, int64(1231108742))
database.MustExec(t, "inserting last max usn", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemLastMaxUSN, 8)
database.MustExec(t, "inserting last synced at", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemLastSyncAt, int64(1231108742))
database.MustExec(t, "inserting last max usn", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemLastMaxUSN, 8)
// execute
tx, err := db.Begin()
if err != nil {
t.Fatal(errors.Wrap(err, "beginning a transaction").Error())
}
// execute
tx, err := db.Begin()
if err != nil {
t.Fatal(errors.Wrap(err, "beginning a transaction").Error())
}
serverTime := int64(1541108743)
serverMaxUSN := 100
serverTime := int64(1541108743)
serverMaxUSN := 100
userMaxUSN := 100
err = saveSyncState(tx, serverTime, serverMaxUSN)
if err != nil {
tx.Rollback()
t.Fatal(errors.Wrap(err, "executing").Error())
}
err = saveSyncState(tx, serverTime, serverMaxUSN, userMaxUSN)
if err != nil {
tx.Rollback()
t.Fatal(errors.Wrap(err, "executing").Error())
}
tx.Commit()
tx.Commit()
// test
var lastSyncedAt int64
var lastMaxUSN int
// test
var lastSyncedAt int64
var lastMaxUSN int
database.MustScan(t, "getting system value",
db.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastSyncAt), &lastSyncedAt)
database.MustScan(t, "getting system value",
db.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastMaxUSN), &lastMaxUSN)
database.MustScan(t, "getting system value",
db.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastSyncAt), &lastSyncedAt)
database.MustScan(t, "getting system value",
db.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastMaxUSN), &lastMaxUSN)
assert.Equal(t, lastSyncedAt, serverTime, "last synced at mismatch")
assert.Equal(t, lastMaxUSN, serverMaxUSN, "last max usn mismatch")
assert.Equal(t, lastSyncedAt, serverTime, "last synced at mismatch")
assert.Equal(t, lastMaxUSN, serverMaxUSN, "last max usn mismatch")
})
t.Run("with empty fragment but server has data - preserves last_max_usn", func(t *testing.T) {
// This tests the fix for the infinite sync bug where empty fragments
// would reset last_max_usn to 0, causing the client to re-download all data.
// When serverMaxUSN=0 (no data in fragment) but userMaxUSN>0 (server has data),
// we're caught up and should preserve the existing last_max_usn.
// set up
db := database.InitTestMemoryDB(t)
testutils.LoginDB(t, db)
existingLastMaxUSN := 100
database.MustExec(t, "inserting last synced at", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemLastSyncAt, int64(1231108742))
database.MustExec(t, "inserting last max usn", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemLastMaxUSN, existingLastMaxUSN)
// execute
tx, err := db.Begin()
if err != nil {
t.Fatal(errors.Wrap(err, "beginning a transaction").Error())
}
serverTime := int64(1541108743)
serverMaxUSN := 0 // Empty fragment (no data in this sync)
userMaxUSN := 150 // Server's actual max USN (higher than ours)
err = saveSyncState(tx, serverTime, serverMaxUSN, userMaxUSN)
if err != nil {
tx.Rollback()
t.Fatal(errors.Wrap(err, "executing").Error())
}
tx.Commit()
// test
var lastSyncedAt int64
var lastMaxUSN int
database.MustScan(t, "getting system value",
db.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastSyncAt), &lastSyncedAt)
database.MustScan(t, "getting system value",
db.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastMaxUSN), &lastMaxUSN)
assert.Equal(t, lastSyncedAt, serverTime, "last synced at should be updated")
// last_max_usn should NOT be updated to 0, it should preserve the existing value
assert.Equal(t, lastMaxUSN, existingLastMaxUSN, "last max usn should be preserved when fragment is empty but server has data")
})
t.Run("with empty server - resets last_max_usn to 0", func(t *testing.T) {
// When both serverMaxUSN=0 and userMaxUSN=0, the server is truly empty
// and we should reset last_max_usn to 0.
// set up
db := database.InitTestMemoryDB(t)
testutils.LoginDB(t, db)
database.MustExec(t, "inserting last synced at", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemLastSyncAt, int64(1231108742))
database.MustExec(t, "inserting last max usn", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemLastMaxUSN, 50)
// execute
tx, err := db.Begin()
if err != nil {
t.Fatal(errors.Wrap(err, "beginning a transaction").Error())
}
serverTime := int64(1541108743)
serverMaxUSN := 0 // Empty fragment
userMaxUSN := 0 // Server is actually empty
err = saveSyncState(tx, serverTime, serverMaxUSN, userMaxUSN)
if err != nil {
tx.Rollback()
t.Fatal(errors.Wrap(err, "executing").Error())
}
tx.Commit()
// test
var lastSyncedAt int64
var lastMaxUSN int
database.MustScan(t, "getting system value",
db.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastSyncAt), &lastSyncedAt)
database.MustScan(t, "getting system value",
db.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastMaxUSN), &lastMaxUSN)
assert.Equal(t, lastSyncedAt, serverTime, "last synced at should be updated")
assert.Equal(t, lastMaxUSN, 0, "last max usn should be reset to 0 when server is empty")
})
}
// TestSendBooks tests that books are put to correct 'buckets' by running a test server and recording the

2
pkg/cli/infra/init.go
View file

@ -137,7 +137,7 @@ func Init(versionTag, apiEndpoint, dbPath string) (*context.DnoteCtx, error) {
return nil, errors.Wrap(err, "setting up the context")
}
log.Debug("Running with Dnote context: %+v\n", context.Redact(ctx))
log.Debug("context: %+v\n", context.Redact(ctx))
return &ctx, nil
}

19
pkg/cli/log/log.go
View file

@ -25,6 +25,11 @@ import (
"github.com/fatih/color"
)
const (
debugEnvName = "DNOTE_DEBUG"
debugEnvValue = "1"
)
var (
// ColorRed is a red foreground color
ColorRed = color.New(color.FgRed)
@ -105,9 +110,21 @@ func Askf(msg string, masked bool, v ...interface{}) {
fmt.Fprintf(color.Output, "%s%s %s: ", indent, symbol, fmt.Sprintf(msg, v...))
}
// isDebug returns true if debug mode is enabled
func isDebug() bool {
return os.Getenv(debugEnvName) == debugEnvValue
}
// Debug prints to the console if DNOTE_DEBUG is set
func Debug(msg string, v ...interface{}) {
if os.Getenv("DNOTE_DEBUG") == "1" {
if isDebug() {
fmt.Fprintf(color.Output, "%s %s", ColorGray.Sprint("DEBUG:"), fmt.Sprintf(msg, v...))
}
}
// DebugNewline prints a newline only in debug mode
func DebugNewline() {
if isDebug() {
fmt.Println()
}
}

2
pkg/cli/migrate/migrate.go
View file

@ -144,7 +144,7 @@ func Run(ctx context.DnoteCtx, migrations []migration, mode int) error {
return errors.Wrap(err, "getting the current schema")
}
log.Debug("current schema: %s %d of %d\n", consts.SystemSchema, schema, len(migrations))
log.Debug("%s: %d of %d\n", schemaKey, schema, len(migrations))
toRun := migrations[schema:]

703
pkg/e2e/sync_test.go → pkg/e2e/sync/basic_test.go
View file

@ -16,307 +16,21 @@
* along with Dnote. If not, see <https://www.gnu.org/licenses/>.
*/
package main
package sync
import (
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/dnote/dnote/pkg/assert"
"github.com/dnote/dnote/pkg/cli/consts"
cliDatabase "github.com/dnote/dnote/pkg/cli/database"
"github.com/dnote/dnote/pkg/cli/testutils"
clitest "github.com/dnote/dnote/pkg/cli/testutils"
"github.com/dnote/dnote/pkg/clock"
"github.com/dnote/dnote/pkg/server/app"
"github.com/dnote/dnote/pkg/server/controllers"
"github.com/dnote/dnote/pkg/server/database"
"github.com/dnote/dnote/pkg/server/mailer"
apitest "github.com/dnote/dnote/pkg/server/testutils"
"github.com/pkg/errors"
"gorm.io/gorm"
)
var cliBinaryName string
var serverTime = time.Date(2017, time.March, 14, 21, 15, 0, 0, time.UTC)
var testDir = "./tmp/.dnote"
func init() {
cliBinaryName = fmt.Sprintf("%s/test/cli/test-cli", testDir)
}
// testEnv holds the test environment for a single test
type testEnv struct {
DB *cliDatabase.DB
CmdOpts clitest.RunDnoteCmdOptions
Server *httptest.Server
ServerDB *gorm.DB
TmpDir string
}
// setupTestEnv creates an isolated test environment with its own database and temp directory
func setupTestEnv(t *testing.T) testEnv {
tmpDir := t.TempDir()
// Create .dnote directory
dnoteDir := filepath.Join(tmpDir, consts.DnoteDirName)
if err := os.MkdirAll(dnoteDir, 0755); err != nil {
t.Fatal(errors.Wrap(err, "creating dnote directory"))
}
// Create database at the expected path
dbPath := filepath.Join(dnoteDir, consts.DnoteDBFileName)
db := cliDatabase.InitTestFileDBRaw(t, dbPath)
// Create server
server, serverDB := setupNewServer(t)
// Create config file with this server's endpoint
apiEndpoint := fmt.Sprintf("%s/api", server.URL)
updateConfigAPIEndpoint(t, tmpDir, apiEndpoint)
// Create command options with XDG paths pointing to temp dir
cmdOpts := clitest.RunDnoteCmdOptions{
Env: []string{
fmt.Sprintf("XDG_CONFIG_HOME=%s", tmpDir),
fmt.Sprintf("XDG_DATA_HOME=%s", tmpDir),
fmt.Sprintf("XDG_CACHE_HOME=%s", tmpDir),
},
}
return testEnv{
DB: db,
CmdOpts: cmdOpts,
Server: server,
ServerDB: serverDB,
TmpDir: tmpDir,
}
}
// setupTestServer creates a test server with its own database
func setupTestServer(t *testing.T, serverTime time.Time) (*httptest.Server, *gorm.DB, error) {
db := apitest.InitMemoryDB(t)
mockClock := clock.NewMock()
mockClock.SetNow(serverTime)
a := app.NewTest()
a.Clock = mockClock
a.EmailTemplates = mailer.Templates{}
a.EmailBackend = &apitest.MockEmailbackendImplementation{}
a.DB = db
server, err := controllers.NewServer(&a)
if err != nil {
return nil, nil, errors.Wrap(err, "initializing server")
}
return server, db, nil
}
// setupNewServer creates a new server and returns the server and database.
// This is useful when a test needs to switch to a new empty server.
func setupNewServer(t *testing.T) (*httptest.Server, *gorm.DB) {
server, serverDB, err := setupTestServer(t, serverTime)
if err != nil {
t.Fatal(errors.Wrap(err, "setting up new test server"))
}
t.Cleanup(func() { server.Close() })
return server, serverDB
}
// updateConfigAPIEndpoint updates the config file with the given API endpoint
func updateConfigAPIEndpoint(t *testing.T, tmpDir string, apiEndpoint string) {
dnoteDir := filepath.Join(tmpDir, consts.DnoteDirName)
configPath := filepath.Join(dnoteDir, consts.ConfigFilename)
configContent := fmt.Sprintf("apiEndpoint: %s\n", apiEndpoint)
if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil {
t.Fatal(errors.Wrap(err, "writing config file"))
}
}
// switchToEmptyServer closes the current server and creates a new empty server,
// updating the config file to point to it.
func switchToEmptyServer(t *testing.T, env *testEnv) {
// Close old server
env.Server.Close()
// Create new empty server
env.Server, env.ServerDB = setupNewServer(t)
// Update config file to point to new server
apiEndpoint := fmt.Sprintf("%s/api", env.Server.URL)
updateConfigAPIEndpoint(t, env.TmpDir, apiEndpoint)
}
func TestMain(m *testing.M) {
// Build CLI binary without hardcoded API endpoint
// Each test will create its own server and config file
cmd := exec.Command("go", "build", "--tags", "fts5", "-o", cliBinaryName, "github.com/dnote/dnote/pkg/cli")
var stderr bytes.Buffer
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Print(errors.Wrap(err, "building a CLI binary").Error())
log.Print(stderr.String())
os.Exit(1)
}
os.Exit(m.Run())
}
// helpers
func setupUser(t *testing.T, env testEnv) database.User {
user := apitest.SetupUserData(env.ServerDB, "alice@example.com", "pass1234")
return user
}
func setupUserAndLogin(t *testing.T, env testEnv) database.User {
user := setupUser(t, env)
login(t, env.DB, env.ServerDB, user)
return user
}
// log in the user in CLI
func login(t *testing.T, db *cliDatabase.DB, serverDB *gorm.DB, user database.User) {
session := apitest.SetupSession(serverDB, user)
cliDatabase.MustExec(t, "inserting session_key", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKey, session.Key)
cliDatabase.MustExec(t, "inserting session_key_expiry", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKeyExpiry, session.ExpiresAt.Unix())
}
func apiCreateBook(t *testing.T, env testEnv, user database.User, name, message string) string {
res := doHTTPReq(t, env, "POST", "/v3/books", fmt.Sprintf(`{"name": "%s"}`, name), message, user)
var resp controllers.CreateBookResp
if err := json.NewDecoder(res.Body).Decode(&resp); err != nil {
t.Fatal(errors.Wrap(err, "decoding payload for adding book"))
return ""
}
return resp.Book.UUID
}
func apiPatchBook(t *testing.T, env testEnv, user database.User, uuid, payload, message string) {
doHTTPReq(t, env, "PATCH", fmt.Sprintf("/v3/books/%s", uuid), payload, message, user)
}
func apiDeleteBook(t *testing.T, env testEnv, user database.User, uuid, message string) {
doHTTPReq(t, env, "DELETE", fmt.Sprintf("/v3/books/%s", uuid), "", message, user)
}
func apiCreateNote(t *testing.T, env testEnv, user database.User, bookUUID, body, message string) string {
res := doHTTPReq(t, env, "POST", "/v3/notes", fmt.Sprintf(`{"book_uuid": "%s", "content": "%s"}`, bookUUID, body), message, user)
var resp controllers.CreateNoteResp
if err := json.NewDecoder(res.Body).Decode(&resp); err != nil {
t.Fatal(errors.Wrap(err, "decoding payload for adding note"))
return ""
}
return resp.Result.UUID
}
func apiPatchNote(t *testing.T, env testEnv, user database.User, noteUUID, payload, message string) {
doHTTPReq(t, env, "PATCH", fmt.Sprintf("/v3/notes/%s", noteUUID), payload, message, user)
}
func apiDeleteNote(t *testing.T, env testEnv, user database.User, noteUUID, message string) {
doHTTPReq(t, env, "DELETE", fmt.Sprintf("/v3/notes/%s", noteUUID), "", message, user)
}
func doHTTPReq(t *testing.T, env testEnv, method, path, payload, message string, user database.User) *http.Response {
apiEndpoint := fmt.Sprintf("%s/api", env.Server.URL)
endpoint := fmt.Sprintf("%s%s", apiEndpoint, path)
req, err := http.NewRequest(method, endpoint, strings.NewReader(payload))
if err != nil {
panic(errors.Wrap(err, "constructing http request"))
}
res := apitest.HTTPAuthDo(t, env.ServerDB, req, user)
if res.StatusCode >= 400 {
bs, err := io.ReadAll(res.Body)
if err != nil {
panic(errors.Wrap(err, "parsing response body for error"))
}
t.Errorf("%s. HTTP status %d. Message: %s", message, res.StatusCode, string(bs))
}
return res
}
type setupFunc func(t *testing.T, env testEnv, user database.User) map[string]string
type assertFunc func(t *testing.T, env testEnv, user database.User, ids map[string]string)
func testSyncCmd(t *testing.T, fullSync bool, setup setupFunc, assert assertFunc) {
env := setupTestEnv(t)
user := setupUserAndLogin(t, env)
ids := setup(t, env, user)
if fullSync {
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "-f")
} else {
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync")
}
assert(t, env, user, ids)
}
type systemState struct {
clientNoteCount int
clientBookCount int
clientLastMaxUSN int
clientLastSyncAt int64
serverNoteCount int64
serverBookCount int64
serverUserMaxUSN int
}
// checkState compares the state of the client and the server with the given system state
func checkState(t *testing.T, clientDB *cliDatabase.DB, user database.User, serverDB *gorm.DB, expected systemState) {
var clientBookCount, clientNoteCount int
cliDatabase.MustScan(t, "counting client notes", clientDB.QueryRow("SELECT count(*) FROM notes"), &clientNoteCount)
cliDatabase.MustScan(t, "counting client books", clientDB.QueryRow("SELECT count(*) FROM books"), &clientBookCount)
assert.Equal(t, clientNoteCount, expected.clientNoteCount, "client note count mismatch")
assert.Equal(t, clientBookCount, expected.clientBookCount, "client book count mismatch")
var clientLastMaxUSN int
var clientLastSyncAt int64
cliDatabase.MustScan(t, "finding system last_max_usn", clientDB.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastMaxUSN), &clientLastMaxUSN)
cliDatabase.MustScan(t, "finding system last_sync_at", clientDB.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastSyncAt), &clientLastSyncAt)
assert.Equal(t, clientLastMaxUSN, expected.clientLastMaxUSN, "client last_max_usn mismatch")
assert.Equal(t, clientLastSyncAt, expected.clientLastSyncAt, "client last_sync_at mismatch")
var serverBookCount, serverNoteCount int64
apitest.MustExec(t, serverDB.Model(&database.Note{}).Count(&serverNoteCount), "counting server notes")
apitest.MustExec(t, serverDB.Model(&database.Book{}).Count(&serverBookCount), "counting api notes")
assert.Equal(t, serverNoteCount, expected.serverNoteCount, "server note count mismatch")
assert.Equal(t, serverBookCount, expected.serverBookCount, "server book count mismatch")
var serverUser database.User
apitest.MustExec(t, serverDB.Where("id = ?", user.ID).First(&serverUser), "finding user")
assert.Equal(t, serverUser.MaxUSN, expected.serverUserMaxUSN, "user max_usn mismatch")
}
// tests
func TestSync_Empty(t *testing.T) {
setup := func(t *testing.T, env testEnv, user database.User) map[string]string {
return map[string]string{}
@ -3896,421 +3610,6 @@ func TestFullSync(t *testing.T) {
})
}
func TestSync_EmptyServer(t *testing.T) {
t.Run("sync to empty server after syncing to non-empty server", func(t *testing.T) {
// Test server data loss/wipe scenario (disaster recovery):
// Verify empty server detection works when the server loses all its data
env := setupTestEnv(t)
user := setupUserAndLogin(t, env)
// Step 1: Create local data and sync to server
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync")
// Verify sync succeeded
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 2: Switch to a completely new empty server
switchToEmptyServer(t, &env)
// Recreate user and session on new server
user = setupUserAndLogin(t, env)
// Step 3: Sync again - should detect empty server and prompt user
// User confirms with "y"
clitest.MustWaitDnoteCmd(t, env.CmdOpts, clitest.UserConfirmEmptyServerSync, cliBinaryName, "sync")
// Step 4: Verify data was uploaded to the empty server
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Verify the content is correct on both client and server
var cliNote1JS, cliNote1CSS cliDatabase.Note
var cliBookJS, cliBookCSS cliDatabase.Book
cliDatabase.MustScan(t, "finding cliNote1JS", env.DB.QueryRow("SELECT uuid, body FROM notes WHERE body = ?", "js1"), &cliNote1JS.UUID, &cliNote1JS.Body)
cliDatabase.MustScan(t, "finding cliNote1CSS", env.DB.QueryRow("SELECT uuid, body FROM notes WHERE body = ?", "css1"), &cliNote1CSS.UUID, &cliNote1CSS.Body)
cliDatabase.MustScan(t, "finding cliBookJS", env.DB.QueryRow("SELECT uuid, label FROM books WHERE label = ?", "js"), &cliBookJS.UUID, &cliBookJS.Label)
cliDatabase.MustScan(t, "finding cliBookCSS", env.DB.QueryRow("SELECT uuid, label FROM books WHERE label = ?", "css"), &cliBookCSS.UUID, &cliBookCSS.Label)
assert.Equal(t, cliNote1JS.Body, "js1", "js note body mismatch")
assert.Equal(t, cliNote1CSS.Body, "css1", "css note body mismatch")
assert.Equal(t, cliBookJS.Label, "js", "js book label mismatch")
assert.Equal(t, cliBookCSS.Label, "css", "css book label mismatch")
// Verify on server side
var serverNoteJS, serverNoteCSS database.Note
var serverBookJS, serverBookCSS database.Book
apitest.MustExec(t, env.ServerDB.Where("body = ?", "js1").First(&serverNoteJS), "finding server note js1")
apitest.MustExec(t, env.ServerDB.Where("body = ?", "css1").First(&serverNoteCSS), "finding server note css1")
apitest.MustExec(t, env.ServerDB.Where("label = ?", "js").First(&serverBookJS), "finding server book js")
apitest.MustExec(t, env.ServerDB.Where("label = ?", "css").First(&serverBookCSS), "finding server book css")
assert.Equal(t, serverNoteJS.Body, "js1", "server js note body mismatch")
assert.Equal(t, serverNoteCSS.Body, "css1", "server css note body mismatch")
assert.Equal(t, serverBookJS.Label, "js", "server js book label mismatch")
assert.Equal(t, serverBookCSS.Label, "css", "server css book label mismatch")
})
t.Run("user cancels empty server prompt", func(t *testing.T) {
env := setupTestEnv(t)
user := setupUserAndLogin(t, env)
// Step 1: Create local data and sync to server
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync")
// Verify initial sync succeeded
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 2: Switch to empty server
switchToEmptyServer(t, &env)
user = setupUserAndLogin(t, env)
// Step 3: Sync again but user cancels with "n"
output, err := clitest.WaitDnoteCmd(t, env.CmdOpts, clitest.UserCancelEmptyServerSync, cliBinaryName, "sync")
if err == nil {
t.Fatal("Expected sync to fail when user cancels, but it succeeded")
}
// Verify the prompt appeared
if !strings.Contains(output, clitest.PromptEmptyServer) {
t.Fatalf("Expected empty server warning in output, got: %s", output)
}
// Step 4: Verify local state unchanged (transaction rolled back)
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 0,
serverBookCount: 0,
serverUserMaxUSN: 0,
})
// Verify items still have original USN and dirty=false
var book cliDatabase.Book
var note cliDatabase.Note
cliDatabase.MustScan(t, "checking book state", env.DB.QueryRow("SELECT usn, dirty FROM books WHERE label = ?", "js"), &book.USN, &book.Dirty)
cliDatabase.MustScan(t, "checking note state", env.DB.QueryRow("SELECT usn, dirty FROM notes WHERE body = ?", "js1"), &note.USN, &note.Dirty)
assert.NotEqual(t, book.USN, 0, "book USN should not be reset")
assert.NotEqual(t, note.USN, 0, "note USN should not be reset")
assert.Equal(t, book.Dirty, false, "book should not be marked dirty")
assert.Equal(t, note.Dirty, false, "note should not be marked dirty")
})
t.Run("all local data is marked deleted - should not upload", func(t *testing.T) {
// Test edge case: Server MaxUSN=0, local MaxUSN>0, but all items are deleted=true
// Should NOT prompt because there's nothing to upload
env := setupTestEnv(t)
user := setupUserAndLogin(t, env)
// Step 1: Create local data and sync to server
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync")
// Verify initial sync succeeded
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 2: Delete all local notes and books (mark as deleted)
cliDatabase.MustExec(t, "marking all books deleted", env.DB, "UPDATE books SET deleted = 1")
cliDatabase.MustExec(t, "marking all notes deleted", env.DB, "UPDATE notes SET deleted = 1")
// Step 3: Switch to empty server
switchToEmptyServer(t, &env)
user = setupUserAndLogin(t, env)
// Step 4: Sync - should NOT prompt because bookCount=0 and noteCount=0 (counting only deleted=0)
// This should complete without user interaction
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync")
// Verify no data was uploaded (server still empty, but client still has deleted items)
// Check server is empty
var serverNoteCount, serverBookCount int64
apitest.MustExec(t, env.ServerDB.Model(&database.Note{}).Count(&serverNoteCount), "counting server notes")
apitest.MustExec(t, env.ServerDB.Model(&database.Book{}).Count(&serverBookCount), "counting server books")
assert.Equal(t, serverNoteCount, int64(0), "server should have no notes")
assert.Equal(t, serverBookCount, int64(0), "server should have no books")
// Check client still has the deleted items locally
var clientNoteCount, clientBookCount int
cliDatabase.MustScan(t, "counting client notes", env.DB.QueryRow("SELECT count(*) FROM notes WHERE deleted = 1"), &clientNoteCount)
cliDatabase.MustScan(t, "counting client books", env.DB.QueryRow("SELECT count(*) FROM books WHERE deleted = 1"), &clientBookCount)
assert.Equal(t, clientNoteCount, 2, "client should still have 2 deleted notes")
assert.Equal(t, clientBookCount, 2, "client should still have 2 deleted books")
// Verify lastMaxUSN was reset to 0
var lastMaxUSN int
cliDatabase.MustScan(t, "getting lastMaxUSN", env.DB.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastMaxUSN), &lastMaxUSN)
assert.Equal(t, lastMaxUSN, 0, "lastMaxUSN should be reset to 0")
})
t.Run("race condition - other client uploads first", func(t *testing.T) {
// This test exercises a race condition that can occur during sync:
// While Client A is waiting for user input, Client B uploads data to the server.
//
// The empty server scenario is the natural place to test this because
// an empty server detection triggers a prompt, at which point the test
// can make client B upload data. We trigger the race condition deterministically.
//
// Test flow:
// - Client A detects empty server and prompts user
// - While waiting for confirmation, Client B uploads the same data via API
// - Client A continues and handles the 409 conflict gracefully by:
// 1. Detecting the 409 error when trying to CREATE books that already exist
// 2. Running stepSync to pull the server's books (js, css)
// 3. mergeBook renames local conflicts (js→js_2, css→css_2)
// 4. Retrying sendChanges to upload the renamed books
// - Result: Both clients' data is preserved (4 books total)
env := setupTestEnv(t)
user := setupUserAndLogin(t, env)
// Step 1: Create local data and sync to establish lastMaxUSN > 0
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync")
// Verify initial sync succeeded
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 2: Switch to new empty server to simulate switching to empty server
switchToEmptyServer(t, &env)
// Create user on new server and login
user = setupUserAndLogin(t, env)
// Step 3: Trigger sync which will detect empty server and prompt user
// Inside the callback (before confirming), we simulate Client B uploading via API.
// We wait for the empty server prompt to ensure Client B uploads AFTER
// GetSyncState but BEFORE the sync decision, creating the race condition deterministically
raceCallback := func(stdout io.Reader, stdin io.WriteCloser) error {
// First, wait for the prompt to ensure Client A has obtained the sync state from the server.
clitest.MustWaitForPrompt(t, stdout, clitest.PromptEmptyServer)
// Now Client B uploads the same data via API (after Client A got the sync state from the server
// but before its sync decision)
// This creates the race condition: Client A thinks server is empty, but Client B uploads data
jsBookUUID := apiCreateBook(t, env, user, "js", "client B creating js book")
cssBookUUID := apiCreateBook(t, env, user, "css", "client B creating css book")
apiCreateNote(t, env, user, jsBookUUID, "js1", "client B creating js note")
apiCreateNote(t, env, user, cssBookUUID, "css1", "client B creating css note")
// Now user confirms
if _, err := io.WriteString(stdin, "y\n"); err != nil {
return errors.Wrap(err, "confirming sync")
}
return nil
}
// Step 4: Client A runs sync with race condition
// The 409 conflict is automatically handled:
// - When 409 is detected, isBehind flag is set
// - stepSync pulls Client B's data
// - mergeBook renames Client A's books to js_2, css_2
// - Renamed books are uploaded
// - Both clients' data is preserved.
clitest.MustWaitDnoteCmd(t, env.CmdOpts, raceCallback, cliBinaryName, "sync")
// Verify final state - both clients' data preserved
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 4, // Both clients' notes
clientBookCount: 4, // js, css, js_2, css_2
clientLastMaxUSN: 8, // 4 from Client B + 4 from Client A's renamed books/notes
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 4,
serverBookCount: 4,
serverUserMaxUSN: 8,
})
// Verify server has both clients' books
var svrBookJS, svrBookCSS, svrBookJS2, svrBookCSS2 database.Book
apitest.MustExec(t, env.ServerDB.Where("label = ?", "js").First(&svrBookJS), "finding server book 'js'")
apitest.MustExec(t, env.ServerDB.Where("label = ?", "css").First(&svrBookCSS), "finding server book 'css'")
apitest.MustExec(t, env.ServerDB.Where("label = ?", "js_2").First(&svrBookJS2), "finding server book 'js_2'")
apitest.MustExec(t, env.ServerDB.Where("label = ?", "css_2").First(&svrBookCSS2), "finding server book 'css_2'")
assert.Equal(t, svrBookJS.Label, "js", "server should have book 'js' (Client B)")
assert.Equal(t, svrBookCSS.Label, "css", "server should have book 'css' (Client B)")
assert.Equal(t, svrBookJS2.Label, "js_2", "server should have book 'js_2' (Client A renamed)")
assert.Equal(t, svrBookCSS2.Label, "css_2", "server should have book 'css_2' (Client A renamed)")
// Verify client has all books
var cliBookJS, cliBookCSS, cliBookJS2, cliBookCSS2 cliDatabase.Book
cliDatabase.MustScan(t, "finding client book 'js'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "js"), &cliBookJS.UUID, &cliBookJS.Label, &cliBookJS.USN)
cliDatabase.MustScan(t, "finding client book 'css'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "css"), &cliBookCSS.UUID, &cliBookCSS.Label, &cliBookCSS.USN)
cliDatabase.MustScan(t, "finding client book 'js_2'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "js_2"), &cliBookJS2.UUID, &cliBookJS2.Label, &cliBookJS2.USN)
cliDatabase.MustScan(t, "finding client book 'css_2'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "css_2"), &cliBookCSS2.UUID, &cliBookCSS2.Label, &cliBookCSS2.USN)
// Verify client UUIDs match server
assert.Equal(t, cliBookJS.UUID, svrBookJS.UUID, "client 'js' UUID should match server")
assert.Equal(t, cliBookCSS.UUID, svrBookCSS.UUID, "client 'css' UUID should match server")
assert.Equal(t, cliBookJS2.UUID, svrBookJS2.UUID, "client 'js_2' UUID should match server")
assert.Equal(t, cliBookCSS2.UUID, svrBookCSS2.UUID, "client 'css_2' UUID should match server")
// Verify all items have non-zero USN (synced successfully)
assert.NotEqual(t, cliBookJS.USN, 0, "client 'js' should have non-zero USN")
assert.NotEqual(t, cliBookCSS.USN, 0, "client 'css' should have non-zero USN")
assert.NotEqual(t, cliBookJS2.USN, 0, "client 'js_2' should have non-zero USN")
assert.NotEqual(t, cliBookCSS2.USN, 0, "client 'css_2' should have non-zero USN")
})
t.Run("sync to server A, then B, then back to A, then back to B", func(t *testing.T) {
// Test switching between two actual servers to verify:
// 1. Empty server detection works when switching to empty server
// 2. No false detection when switching back to non-empty servers
// 3. Both servers maintain independent state across multiple switches
env := setupTestEnv(t)
// Create Server A with its own database
serverA, serverDBA, err := setupTestServer(t, serverTime)
if err != nil {
t.Fatal(errors.Wrap(err, "setting up server A"))
}
defer serverA.Close()
// Create Server B with its own database
serverB, serverDBB, err := setupTestServer(t, serverTime)
if err != nil {
t.Fatal(errors.Wrap(err, "setting up server B"))
}
defer serverB.Close()
// Step 1: Set up user on Server A and sync
apiEndpointA := fmt.Sprintf("%s/api", serverA.URL)
userA := apitest.SetupUserData(serverDBA, "alice@example.com", "pass1234")
sessionA := apitest.SetupSession(serverDBA, userA)
cliDatabase.MustExec(t, "inserting session_key", env.DB, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKey, sessionA.Key)
cliDatabase.MustExec(t, "inserting session_key_expiry", env.DB, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKeyExpiry, sessionA.ExpiresAt.Unix())
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "--apiEndpoint", apiEndpointA)
// Verify sync to Server A succeeded
checkState(t, env.DB, userA, serverDBA, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 2: Switch to Server B (empty) and sync
apiEndpointB := fmt.Sprintf("%s/api", serverB.URL)
// Set up user on Server B
userB := apitest.SetupUserData(serverDBB, "alice@example.com", "pass1234")
sessionB := apitest.SetupSession(serverDBB, userB)
cliDatabase.MustExec(t, "updating session_key for B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.Key, consts.SystemSessionKey)
cliDatabase.MustExec(t, "updating session_key_expiry for B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.ExpiresAt.Unix(), consts.SystemSessionKeyExpiry)
// Should detect empty server and prompt
clitest.MustWaitDnoteCmd(t, env.CmdOpts, clitest.UserConfirmEmptyServerSync, cliBinaryName, "sync", "--apiEndpoint", apiEndpointB)
// Verify Server B now has data
checkState(t, env.DB, userB, serverDBB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 3: Switch back to Server A and sync
cliDatabase.MustExec(t, "updating session_key back to A", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionA.Key, consts.SystemSessionKey)
cliDatabase.MustExec(t, "updating session_key_expiry back to A", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionA.ExpiresAt.Unix(), consts.SystemSessionKeyExpiry)
// Should NOT trigger empty server detection (Server A has MaxUSN > 0)
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "--apiEndpoint", apiEndpointA)
// Verify Server A still has its data
checkState(t, env.DB, userA, serverDBA, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 4: Switch back to Server B and sync again
cliDatabase.MustExec(t, "updating session_key back to B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.Key, consts.SystemSessionKey)
cliDatabase.MustExec(t, "updating session_key_expiry back to B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.ExpiresAt.Unix(), consts.SystemSessionKeyExpiry)
// Should NOT trigger empty server detection (Server B now has MaxUSN > 0 from Step 2)
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "--apiEndpoint", apiEndpointB)
// Verify both servers maintain independent state
checkState(t, env.DB, userB, serverDBB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
})
}
func TestSync_FreshClientConcurrent(t *testing.T) {
// Test the core issue: Fresh client (never synced, lastMaxUSN=0) syncing to a server
// that already has data uploaded by another client.

73
pkg/e2e/sync/edge_cases_test.go Normal file
View file

@ -0,0 +1,73 @@
/* Copyright (C) 2019, 2020, 2021, 2022, 2023, 2024, 2025 Dnote contributors
*
* This file is part of Dnote.
*
* Dnote is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Dnote is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Dnote. If not, see <https://www.gnu.org/licenses/>.
*/
package sync
import (
"testing"
"github.com/dnote/dnote/pkg/assert"
"github.com/dnote/dnote/pkg/cli/consts"
cliDatabase "github.com/dnote/dnote/pkg/cli/database"
clitest "github.com/dnote/dnote/pkg/cli/testutils"
"github.com/google/uuid"
)
// TestSync_EmptyFragmentPreservesLastMaxUSN verifies that last_max_usn is not reset to 0
// when sync receives an empty response from the server.
//
// Scenario: Client has orphaned note (references non-existent book). During sync:
// 1. Downloads data successfully (last_max_usn=3)
// 2. Upload fails (orphaned note -> 500 error, triggers retry stepSync)
// 3. Retry stepSync gets 0 fragments (already at latest USN)
// 4. last_max_usn should stay at 3, not reset to 0
func TestSync_EmptyFragmentPreservesLastMaxUSN(t *testing.T) {
env := setupTestEnv(t)
user := setupUserAndLogin(t, env)
// Create data on server (max_usn=3)
bookUUID := apiCreateBook(t, env, user, "javascript", "creating book via API")
apiCreateNote(t, env, user, bookUUID, "note1 content", "creating note1 via API")
apiCreateNote(t, env, user, bookUUID, "note2 content", "creating note2 via API")
// Create orphaned note locally (will fail to upload)
orphanedNote := cliDatabase.Note{
UUID: uuid.New().String(),
BookUUID: uuid.New().String(), // non-existent book
Body: "orphaned note content",
AddedOn: 1234567890,
EditedOn: 0,
USN: 0,
Deleted: false,
Dirty: true,
}
if err := orphanedNote.Insert(env.DB); err != nil {
t.Fatal(err)
}
// Run sync (downloads data, upload fails, retry gets 0 fragments)
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync")
// Verify last_max_usn is preserved at 3, not reset to 0
var lastMaxUSN int
cliDatabase.MustScan(t, "finding system last_max_usn",
env.DB.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastMaxUSN),
&lastMaxUSN)
assert.Equal(t, lastMaxUSN, 3, "last_max_usn should be 3 after syncing")
}

449
pkg/e2e/sync/empty_server_test.go Normal file
View file

@ -0,0 +1,449 @@
/* Copyright (C) 2019, 2020, 2021, 2022, 2023, 2024, 2025 Dnote contributors
*
* This file is part of Dnote.
*
* Dnote is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Dnote is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Dnote. If not, see <https://www.gnu.org/licenses/>.
*/
package sync
import (
"fmt"
"io"
"strings"
"testing"
"github.com/dnote/dnote/pkg/assert"
"github.com/dnote/dnote/pkg/cli/consts"
cliDatabase "github.com/dnote/dnote/pkg/cli/database"
clitest "github.com/dnote/dnote/pkg/cli/testutils"
"github.com/dnote/dnote/pkg/server/database"
apitest "github.com/dnote/dnote/pkg/server/testutils"
"github.com/pkg/errors"
)
func TestSync_EmptyServer(t *testing.T) {
t.Run("sync to empty server after syncing to non-empty server", func(t *testing.T) {
// Test server data loss/wipe scenario (disaster recovery):
// Verify empty server detection works when the server loses all its data
env := setupTestEnv(t)
user := setupUserAndLogin(t, env)
// Step 1: Create local data and sync to server
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync")
// Verify sync succeeded
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 2: Switch to a completely new empty server
switchToEmptyServer(t, &env)
// Recreate user and session on new server
user = setupUserAndLogin(t, env)
// Step 3: Sync again - should detect empty server and prompt user
// User confirms with "y"
clitest.MustWaitDnoteCmd(t, env.CmdOpts, clitest.UserConfirmEmptyServerSync, cliBinaryName, "sync")
// Step 4: Verify data was uploaded to the empty server
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Verify the content is correct on both client and server
var cliNote1JS, cliNote1CSS cliDatabase.Note
var cliBookJS, cliBookCSS cliDatabase.Book
cliDatabase.MustScan(t, "finding cliNote1JS", env.DB.QueryRow("SELECT uuid, body FROM notes WHERE body = ?", "js1"), &cliNote1JS.UUID, &cliNote1JS.Body)
cliDatabase.MustScan(t, "finding cliNote1CSS", env.DB.QueryRow("SELECT uuid, body FROM notes WHERE body = ?", "css1"), &cliNote1CSS.UUID, &cliNote1CSS.Body)
cliDatabase.MustScan(t, "finding cliBookJS", env.DB.QueryRow("SELECT uuid, label FROM books WHERE label = ?", "js"), &cliBookJS.UUID, &cliBookJS.Label)
cliDatabase.MustScan(t, "finding cliBookCSS", env.DB.QueryRow("SELECT uuid, label FROM books WHERE label = ?", "css"), &cliBookCSS.UUID, &cliBookCSS.Label)
assert.Equal(t, cliNote1JS.Body, "js1", "js note body mismatch")
assert.Equal(t, cliNote1CSS.Body, "css1", "css note body mismatch")
assert.Equal(t, cliBookJS.Label, "js", "js book label mismatch")
assert.Equal(t, cliBookCSS.Label, "css", "css book label mismatch")
// Verify on server side
var serverNoteJS, serverNoteCSS database.Note
var serverBookJS, serverBookCSS database.Book
apitest.MustExec(t, env.ServerDB.Where("body = ?", "js1").First(&serverNoteJS), "finding server note js1")
apitest.MustExec(t, env.ServerDB.Where("body = ?", "css1").First(&serverNoteCSS), "finding server note css1")
apitest.MustExec(t, env.ServerDB.Where("label = ?", "js").First(&serverBookJS), "finding server book js")
apitest.MustExec(t, env.ServerDB.Where("label = ?", "css").First(&serverBookCSS), "finding server book css")
assert.Equal(t, serverNoteJS.Body, "js1", "server js note body mismatch")
assert.Equal(t, serverNoteCSS.Body, "css1", "server css note body mismatch")
assert.Equal(t, serverBookJS.Label, "js", "server js book label mismatch")
assert.Equal(t, serverBookCSS.Label, "css", "server css book label mismatch")
})
t.Run("user cancels empty server prompt", func(t *testing.T) {
env := setupTestEnv(t)
user := setupUserAndLogin(t, env)
// Step 1: Create local data and sync to server
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync")
// Verify initial sync succeeded
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 2: Switch to empty server
switchToEmptyServer(t, &env)
user = setupUserAndLogin(t, env)
// Step 3: Sync again but user cancels with "n"
output, err := clitest.WaitDnoteCmd(t, env.CmdOpts, clitest.UserCancelEmptyServerSync, cliBinaryName, "sync")
if err == nil {
t.Fatal("Expected sync to fail when user cancels, but it succeeded")
}
// Verify the prompt appeared
if !strings.Contains(output, clitest.PromptEmptyServer) {
t.Fatalf("Expected empty server warning in output, got: %s", output)
}
// Step 4: Verify local state unchanged (transaction rolled back)
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 0,
serverBookCount: 0,
serverUserMaxUSN: 0,
})
// Verify items still have original USN and dirty=false
var book cliDatabase.Book
var note cliDatabase.Note
cliDatabase.MustScan(t, "checking book state", env.DB.QueryRow("SELECT usn, dirty FROM books WHERE label = ?", "js"), &book.USN, &book.Dirty)
cliDatabase.MustScan(t, "checking note state", env.DB.QueryRow("SELECT usn, dirty FROM notes WHERE body = ?", "js1"), &note.USN, &note.Dirty)
assert.NotEqual(t, book.USN, 0, "book USN should not be reset")
assert.NotEqual(t, note.USN, 0, "note USN should not be reset")
assert.Equal(t, book.Dirty, false, "book should not be marked dirty")
assert.Equal(t, note.Dirty, false, "note should not be marked dirty")
})
t.Run("all local data is marked deleted - should not upload", func(t *testing.T) {
// Test edge case: Server MaxUSN=0, local MaxUSN>0, but all items are deleted=true
// Should NOT prompt because there's nothing to upload
env := setupTestEnv(t)
user := setupUserAndLogin(t, env)
// Step 1: Create local data and sync to server
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync")
// Verify initial sync succeeded
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 2: Delete all local notes and books (mark as deleted)
cliDatabase.MustExec(t, "marking all books deleted", env.DB, "UPDATE books SET deleted = 1")
cliDatabase.MustExec(t, "marking all notes deleted", env.DB, "UPDATE notes SET deleted = 1")
// Step 3: Switch to empty server
switchToEmptyServer(t, &env)
user = setupUserAndLogin(t, env)
// Step 4: Sync - should NOT prompt because bookCount=0 and noteCount=0 (counting only deleted=0)
// This should complete without user interaction
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync")
// Verify no data was uploaded (server still empty, but client still has deleted items)
// Check server is empty
var serverNoteCount, serverBookCount int64
apitest.MustExec(t, env.ServerDB.Model(&database.Note{}).Count(&serverNoteCount), "counting server notes")
apitest.MustExec(t, env.ServerDB.Model(&database.Book{}).Count(&serverBookCount), "counting server books")
assert.Equal(t, serverNoteCount, int64(0), "server should have no notes")
assert.Equal(t, serverBookCount, int64(0), "server should have no books")
// Check client still has the deleted items locally
var clientNoteCount, clientBookCount int
cliDatabase.MustScan(t, "counting client notes", env.DB.QueryRow("SELECT count(*) FROM notes WHERE deleted = 1"), &clientNoteCount)
cliDatabase.MustScan(t, "counting client books", env.DB.QueryRow("SELECT count(*) FROM books WHERE deleted = 1"), &clientBookCount)
assert.Equal(t, clientNoteCount, 2, "client should still have 2 deleted notes")
assert.Equal(t, clientBookCount, 2, "client should still have 2 deleted books")
// Verify lastMaxUSN was reset to 0
var lastMaxUSN int
cliDatabase.MustScan(t, "getting lastMaxUSN", env.DB.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastMaxUSN), &lastMaxUSN)
assert.Equal(t, lastMaxUSN, 0, "lastMaxUSN should be reset to 0")
})
t.Run("race condition - other client uploads first", func(t *testing.T) {
// This test exercises a race condition that can occur during sync:
// While Client A is waiting for user input, Client B uploads data to the server.
//
// The empty server scenario is the natural place to test this because
// an empty server detection triggers a prompt, at which point the test
// can make client B upload data. We trigger the race condition deterministically.
//
// Test flow:
// - Client A detects empty server and prompts user
// - While waiting for confirmation, Client B uploads the same data via API
// - Client A continues and handles the 409 conflict gracefully by:
// 1. Detecting the 409 error when trying to CREATE books that already exist
// 2. Running stepSync to pull the server's books (js, css)
// 3. mergeBook renames local conflicts (js→js_2, css→css_2)
// 4. Retrying sendChanges to upload the renamed books
// - Result: Both clients' data is preserved (4 books total)
env := setupTestEnv(t)
user := setupUserAndLogin(t, env)
// Step 1: Create local data and sync to establish lastMaxUSN > 0
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync")
// Verify initial sync succeeded
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 2: Switch to new empty server to simulate switching to empty server
switchToEmptyServer(t, &env)
// Create user on new server and login
user = setupUserAndLogin(t, env)
// Step 3: Trigger sync which will detect empty server and prompt user
// Inside the callback (before confirming), we simulate Client B uploading via API.
// We wait for the empty server prompt to ensure Client B uploads AFTER
// GetSyncState but BEFORE the sync decision, creating the race condition deterministically
raceCallback := func(stdout io.Reader, stdin io.WriteCloser) error {
// First, wait for the prompt to ensure Client A has obtained the sync state from the server.
clitest.MustWaitForPrompt(t, stdout, clitest.PromptEmptyServer)
// Now Client B uploads the same data via API (after Client A got the sync state from the server
// but before its sync decision)
// This creates the race condition: Client A thinks server is empty, but Client B uploads data
jsBookUUID := apiCreateBook(t, env, user, "js", "client B creating js book")
cssBookUUID := apiCreateBook(t, env, user, "css", "client B creating css book")
apiCreateNote(t, env, user, jsBookUUID, "js1", "client B creating js note")
apiCreateNote(t, env, user, cssBookUUID, "css1", "client B creating css note")
// Now user confirms
if _, err := io.WriteString(stdin, "y\n"); err != nil {
return errors.Wrap(err, "confirming sync")
}
return nil
}
// Step 4: Client A runs sync with race condition
// The 409 conflict is automatically handled:
// - When 409 is detected, isBehind flag is set
// - stepSync pulls Client B's data
// - mergeBook renames Client A's books to js_2, css_2
// - Renamed books are uploaded
// - Both clients' data is preserved.
clitest.MustWaitDnoteCmd(t, env.CmdOpts, raceCallback, cliBinaryName, "sync")
// Verify final state - both clients' data preserved
checkState(t, env.DB, user, env.ServerDB, systemState{
clientNoteCount: 4, // Both clients' notes
clientBookCount: 4, // js, css, js_2, css_2
clientLastMaxUSN: 8, // 4 from Client B + 4 from Client A's renamed books/notes
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 4,
serverBookCount: 4,
serverUserMaxUSN: 8,
})
// Verify server has both clients' books
var svrBookJS, svrBookCSS, svrBookJS2, svrBookCSS2 database.Book
apitest.MustExec(t, env.ServerDB.Where("label = ?", "js").First(&svrBookJS), "finding server book 'js'")
apitest.MustExec(t, env.ServerDB.Where("label = ?", "css").First(&svrBookCSS), "finding server book 'css'")
apitest.MustExec(t, env.ServerDB.Where("label = ?", "js_2").First(&svrBookJS2), "finding server book 'js_2'")
apitest.MustExec(t, env.ServerDB.Where("label = ?", "css_2").First(&svrBookCSS2), "finding server book 'css_2'")
assert.Equal(t, svrBookJS.Label, "js", "server should have book 'js' (Client B)")
assert.Equal(t, svrBookCSS.Label, "css", "server should have book 'css' (Client B)")
assert.Equal(t, svrBookJS2.Label, "js_2", "server should have book 'js_2' (Client A renamed)")
assert.Equal(t, svrBookCSS2.Label, "css_2", "server should have book 'css_2' (Client A renamed)")
// Verify client has all books
var cliBookJS, cliBookCSS, cliBookJS2, cliBookCSS2 cliDatabase.Book
cliDatabase.MustScan(t, "finding client book 'js'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "js"), &cliBookJS.UUID, &cliBookJS.Label, &cliBookJS.USN)
cliDatabase.MustScan(t, "finding client book 'css'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "css"), &cliBookCSS.UUID, &cliBookCSS.Label, &cliBookCSS.USN)
cliDatabase.MustScan(t, "finding client book 'js_2'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "js_2"), &cliBookJS2.UUID, &cliBookJS2.Label, &cliBookJS2.USN)
cliDatabase.MustScan(t, "finding client book 'css_2'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "css_2"), &cliBookCSS2.UUID, &cliBookCSS2.Label, &cliBookCSS2.USN)
// Verify client UUIDs match server
assert.Equal(t, cliBookJS.UUID, svrBookJS.UUID, "client 'js' UUID should match server")
assert.Equal(t, cliBookCSS.UUID, svrBookCSS.UUID, "client 'css' UUID should match server")
assert.Equal(t, cliBookJS2.UUID, svrBookJS2.UUID, "client 'js_2' UUID should match server")
assert.Equal(t, cliBookCSS2.UUID, svrBookCSS2.UUID, "client 'css_2' UUID should match server")
// Verify all items have non-zero USN (synced successfully)
assert.NotEqual(t, cliBookJS.USN, 0, "client 'js' should have non-zero USN")
assert.NotEqual(t, cliBookCSS.USN, 0, "client 'css' should have non-zero USN")
assert.NotEqual(t, cliBookJS2.USN, 0, "client 'js_2' should have non-zero USN")
assert.NotEqual(t, cliBookCSS2.USN, 0, "client 'css_2' should have non-zero USN")
})
t.Run("sync to server A, then B, then back to A, then back to B", func(t *testing.T) {
// Test switching between two actual servers to verify:
// 1. Empty server detection works when switching to empty server
// 2. No false detection when switching back to non-empty servers
// 3. Both servers maintain independent state across multiple switches
env := setupTestEnv(t)
// Create Server A with its own database
serverA, serverDBA, err := setupTestServer(t, serverTime)
if err != nil {
t.Fatal(errors.Wrap(err, "setting up server A"))
}
defer serverA.Close()
// Create Server B with its own database
serverB, serverDBB, err := setupTestServer(t, serverTime)
if err != nil {
t.Fatal(errors.Wrap(err, "setting up server B"))
}
defer serverB.Close()
// Step 1: Set up user on Server A and sync
apiEndpointA := fmt.Sprintf("%s/api", serverA.URL)
userA := apitest.SetupUserData(serverDBA, "alice@example.com", "pass1234")
sessionA := apitest.SetupSession(serverDBA, userA)
cliDatabase.MustExec(t, "inserting session_key", env.DB, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKey, sessionA.Key)
cliDatabase.MustExec(t, "inserting session_key_expiry", env.DB, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKeyExpiry, sessionA.ExpiresAt.Unix())
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1")
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "--apiEndpoint", apiEndpointA)
// Verify sync to Server A succeeded
checkState(t, env.DB, userA, serverDBA, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 2: Switch to Server B (empty) and sync
apiEndpointB := fmt.Sprintf("%s/api", serverB.URL)
// Set up user on Server B
userB := apitest.SetupUserData(serverDBB, "alice@example.com", "pass1234")
sessionB := apitest.SetupSession(serverDBB, userB)
cliDatabase.MustExec(t, "updating session_key for B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.Key, consts.SystemSessionKey)
cliDatabase.MustExec(t, "updating session_key_expiry for B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.ExpiresAt.Unix(), consts.SystemSessionKeyExpiry)
// Should detect empty server and prompt
clitest.MustWaitDnoteCmd(t, env.CmdOpts, clitest.UserConfirmEmptyServerSync, cliBinaryName, "sync", "--apiEndpoint", apiEndpointB)
// Verify Server B now has data
checkState(t, env.DB, userB, serverDBB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 3: Switch back to Server A and sync
cliDatabase.MustExec(t, "updating session_key back to A", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionA.Key, consts.SystemSessionKey)
cliDatabase.MustExec(t, "updating session_key_expiry back to A", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionA.ExpiresAt.Unix(), consts.SystemSessionKeyExpiry)
// Should NOT trigger empty server detection (Server A has MaxUSN > 0)
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "--apiEndpoint", apiEndpointA)
// Verify Server A still has its data
checkState(t, env.DB, userA, serverDBA, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
// Step 4: Switch back to Server B and sync again
cliDatabase.MustExec(t, "updating session_key back to B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.Key, consts.SystemSessionKey)
cliDatabase.MustExec(t, "updating session_key_expiry back to B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.ExpiresAt.Unix(), consts.SystemSessionKeyExpiry)
// Should NOT trigger empty server detection (Server B now has MaxUSN > 0 from Step 2)
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "--apiEndpoint", apiEndpointB)
// Verify both servers maintain independent state
checkState(t, env.DB, userB, serverDBB, systemState{
clientNoteCount: 2,
clientBookCount: 2,
clientLastMaxUSN: 4,
clientLastSyncAt: serverTime.Unix(),
serverNoteCount: 2,
serverBookCount: 2,
serverUserMaxUSN: 4,
})
})
}

59
pkg/e2e/sync/main_test.go Normal file
View file

@ -0,0 +1,59 @@
/* Copyright (C) 2019, 2020, 2021, 2022, 2023, 2024, 2025 Dnote contributors
*
* This file is part of Dnote.
*
* This file is part of Dnote.
*
* Dnote is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Dnote is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Dnote. If not, see <https://www.gnu.org/licenses/>.
*/
package sync
import (
"bytes"
"fmt"
"log"
"os"
"os/exec"
"testing"
"time"
"github.com/pkg/errors"
)
var cliBinaryName string
var serverTime = time.Date(2017, time.March, 14, 21, 15, 0, 0, time.UTC)
var testDir = "./tmp/"
func init() {
cliBinaryName = fmt.Sprintf("%s/test-cli", testDir)
}
func TestMain(m *testing.M) {
// Build CLI binary without hardcoded API endpoint
// Each test will create its own server and config file
cmd := exec.Command("go", "build", "--tags", "fts5", "-o", cliBinaryName, "github.com/dnote/dnote/pkg/cli")
var stderr bytes.Buffer
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Print(errors.Wrap(err, "building a CLI binary").Error())
log.Print(stderr.String())
os.Exit(1)
}
os.Exit(m.Run())
}

300
pkg/e2e/sync/testutils.go Normal file
View file

@ -0,0 +1,300 @@
/* Copyright (C) 2019, 2020, 2021, 2022, 2023, 2024, 2025 Dnote contributors
*
* This file is part of Dnote.
*
* Dnote is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Dnote is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Dnote. If not, see <https://www.gnu.org/licenses/>.
*/
package sync
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/dnote/dnote/pkg/assert"
"github.com/dnote/dnote/pkg/cli/consts"
cliDatabase "github.com/dnote/dnote/pkg/cli/database"
clitest "github.com/dnote/dnote/pkg/cli/testutils"
"github.com/dnote/dnote/pkg/clock"
"github.com/dnote/dnote/pkg/server/app"
"github.com/dnote/dnote/pkg/server/controllers"
"github.com/dnote/dnote/pkg/server/database"
"github.com/dnote/dnote/pkg/server/mailer"
apitest "github.com/dnote/dnote/pkg/server/testutils"
"github.com/pkg/errors"
"gorm.io/gorm"
)
// testEnv holds the test environment for a single test
type testEnv struct {
DB *cliDatabase.DB
CmdOpts clitest.RunDnoteCmdOptions
Server *httptest.Server
ServerDB *gorm.DB
TmpDir string
}
// setupTestEnv creates an isolated test environment with its own database and temp directory
func setupTestEnv(t *testing.T) testEnv {
tmpDir := t.TempDir()
// Create .dnote directory
dnoteDir := filepath.Join(tmpDir, consts.DnoteDirName)
if err := os.MkdirAll(dnoteDir, 0755); err != nil {
t.Fatal(errors.Wrap(err, "creating dnote directory"))
}
// Create database at the expected path
dbPath := filepath.Join(dnoteDir, consts.DnoteDBFileName)
db := cliDatabase.InitTestFileDBRaw(t, dbPath)
// Create server
server, serverDB := setupNewServer(t)
// Create config file with this server's endpoint
apiEndpoint := fmt.Sprintf("%s/api", server.URL)
updateConfigAPIEndpoint(t, tmpDir, apiEndpoint)
// Create command options with XDG paths pointing to temp dir
cmdOpts := clitest.RunDnoteCmdOptions{
Env: []string{
fmt.Sprintf("XDG_CONFIG_HOME=%s", tmpDir),
fmt.Sprintf("XDG_DATA_HOME=%s", tmpDir),
fmt.Sprintf("XDG_CACHE_HOME=%s", tmpDir),
},
}
return testEnv{
DB: db,
CmdOpts: cmdOpts,
Server: server,
ServerDB: serverDB,
TmpDir: tmpDir,
}
}
// setupTestServer creates a test server with its own database
func setupTestServer(t *testing.T, serverTime time.Time) (*httptest.Server, *gorm.DB, error) {
db := apitest.InitMemoryDB(t)
mockClock := clock.NewMock()
mockClock.SetNow(serverTime)
a := app.NewTest()
a.Clock = mockClock
a.EmailTemplates = mailer.Templates{}
a.EmailBackend = &apitest.MockEmailbackendImplementation{}
a.DB = db
server, err := controllers.NewServer(&a)
if err != nil {
return nil, nil, errors.Wrap(err, "initializing server")
}
return server, db, nil
}
// setupNewServer creates a new server and returns the server and database.
// This is useful when a test needs to switch to a new empty server.
func setupNewServer(t *testing.T) (*httptest.Server, *gorm.DB) {
server, serverDB, err := setupTestServer(t, serverTime)
if err != nil {
t.Fatal(errors.Wrap(err, "setting up new test server"))
}
t.Cleanup(func() { server.Close() })
return server, serverDB
}
// updateConfigAPIEndpoint updates the config file with the given API endpoint
func updateConfigAPIEndpoint(t *testing.T, tmpDir string, apiEndpoint string) {
dnoteDir := filepath.Join(tmpDir, consts.DnoteDirName)
configPath := filepath.Join(dnoteDir, consts.ConfigFilename)
configContent := fmt.Sprintf("apiEndpoint: %s\n", apiEndpoint)
if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil {
t.Fatal(errors.Wrap(err, "writing config file"))
}
}
// switchToEmptyServer closes the current server and creates a new empty server,
// updating the config file to point to it.
func switchToEmptyServer(t *testing.T, env *testEnv) {
// Close old server
env.Server.Close()
// Create new empty server
env.Server, env.ServerDB = setupNewServer(t)
// Update config file to point to new server
apiEndpoint := fmt.Sprintf("%s/api", env.Server.URL)
updateConfigAPIEndpoint(t, env.TmpDir, apiEndpoint)
}
// setupUser creates a test user in the server database
func setupUser(t *testing.T, env testEnv) database.User {
user := apitest.SetupUserData(env.ServerDB, "alice@example.com", "pass1234")
return user
}
// setupUserAndLogin creates a test user and logs them in on the CLI
func setupUserAndLogin(t *testing.T, env testEnv) database.User {
user := setupUser(t, env)
login(t, env.DB, env.ServerDB, user)
return user
}
// login logs in the user in CLI
func login(t *testing.T, db *cliDatabase.DB, serverDB *gorm.DB, user database.User) {
session := apitest.SetupSession(serverDB, user)
cliDatabase.MustExec(t, "inserting session_key", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKey, session.Key)
cliDatabase.MustExec(t, "inserting session_key_expiry", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKeyExpiry, session.ExpiresAt.Unix())
}
// apiCreateBook creates a book via the API and returns its UUID
func apiCreateBook(t *testing.T, env testEnv, user database.User, name, message string) string {
res := doHTTPReq(t, env, "POST", "/v3/books", fmt.Sprintf(`{"name": "%s"}`, name), message, user)
var resp controllers.CreateBookResp
if err := json.NewDecoder(res.Body).Decode(&resp); err != nil {
t.Fatal(errors.Wrap(err, "decoding payload for adding book"))
return ""
}
return resp.Book.UUID
}
// apiPatchBook updates a book via the API
func apiPatchBook(t *testing.T, env testEnv, user database.User, uuid, payload, message string) {
doHTTPReq(t, env, "PATCH", fmt.Sprintf("/v3/books/%s", uuid), payload, message, user)
}
// apiDeleteBook deletes a book via the API
func apiDeleteBook(t *testing.T, env testEnv, user database.User, uuid, message string) {
doHTTPReq(t, env, "DELETE", fmt.Sprintf("/v3/books/%s", uuid), "", message, user)
}
// apiCreateNote creates a note via the API and returns its UUID
func apiCreateNote(t *testing.T, env testEnv, user database.User, bookUUID, body, message string) string {
res := doHTTPReq(t, env, "POST", "/v3/notes", fmt.Sprintf(`{"book_uuid": "%s", "content": "%s"}`, bookUUID, body), message, user)
var resp controllers.CreateNoteResp
if err := json.NewDecoder(res.Body).Decode(&resp); err != nil {
t.Fatal(errors.Wrap(err, "decoding payload for adding note"))
return ""
}
return resp.Result.UUID
}
// apiPatchNote updates a note via the API
func apiPatchNote(t *testing.T, env testEnv, user database.User, noteUUID, payload, message string) {
doHTTPReq(t, env, "PATCH", fmt.Sprintf("/v3/notes/%s", noteUUID), payload, message, user)
}
// apiDeleteNote deletes a note via the API
func apiDeleteNote(t *testing.T, env testEnv, user database.User, noteUUID, message string) {
doHTTPReq(t, env, "DELETE", fmt.Sprintf("/v3/notes/%s", noteUUID), "", message, user)
}
// doHTTPReq performs an authenticated HTTP request and checks for errors
func doHTTPReq(t *testing.T, env testEnv, method, path, payload, message string, user database.User) *http.Response {
apiEndpoint := fmt.Sprintf("%s/api", env.Server.URL)
endpoint := fmt.Sprintf("%s%s", apiEndpoint, path)
req, err := http.NewRequest(method, endpoint, strings.NewReader(payload))
if err != nil {
panic(errors.Wrap(err, "constructing http request"))
}
res := apitest.HTTPAuthDo(t, env.ServerDB, req, user)
if res.StatusCode >= 400 {
bs, err := io.ReadAll(res.Body)
if err != nil {
panic(errors.Wrap(err, "parsing response body for error"))
}
t.Errorf("%s. HTTP status %d. Message: %s", message, res.StatusCode, string(bs))
}
return res
}
// setupFunc is a function that sets up test data and returns IDs for assertions
type setupFunc func(t *testing.T, env testEnv, user database.User) map[string]string
// assertFunc is a function that asserts the expected state after sync
type assertFunc func(t *testing.T, env testEnv, user database.User, ids map[string]string)
// testSyncCmd is a test helper that sets up a test environment, runs setup, syncs, and asserts
func testSyncCmd(t *testing.T, fullSync bool, setup setupFunc, assert assertFunc) {
env := setupTestEnv(t)
user := setupUserAndLogin(t, env)
ids := setup(t, env, user)
if fullSync {
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "-f")
} else {
clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync")
}
assert(t, env, user, ids)
}
// systemState represents the expected state of the sync system
type systemState struct {
clientNoteCount int
clientBookCount int
clientLastMaxUSN int
clientLastSyncAt int64
serverNoteCount int64
serverBookCount int64
serverUserMaxUSN int
}
// checkState compares the state of the client and the server with the given system state
func checkState(t *testing.T, clientDB *cliDatabase.DB, user database.User, serverDB *gorm.DB, expected systemState) {
var clientBookCount, clientNoteCount int
cliDatabase.MustScan(t, "counting client notes", clientDB.QueryRow("SELECT count(*) FROM notes"), &clientNoteCount)
cliDatabase.MustScan(t, "counting client books", clientDB.QueryRow("SELECT count(*) FROM books"), &clientBookCount)
assert.Equal(t, clientNoteCount, expected.clientNoteCount, "client note count mismatch")
assert.Equal(t, clientBookCount, expected.clientBookCount, "client book count mismatch")
var clientLastMaxUSN int
var clientLastSyncAt int64
cliDatabase.MustScan(t, "finding system last_max_usn", clientDB.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastMaxUSN), &clientLastMaxUSN)
cliDatabase.MustScan(t, "finding system last_sync_at", clientDB.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastSyncAt), &clientLastSyncAt)
assert.Equal(t, clientLastMaxUSN, expected.clientLastMaxUSN, "client last_max_usn mismatch")
assert.Equal(t, clientLastSyncAt, expected.clientLastSyncAt, "client last_sync_at mismatch")
var serverBookCount, serverNoteCount int64
apitest.MustExec(t, serverDB.Model(&database.Note{}).Count(&serverNoteCount), "counting server notes")
apitest.MustExec(t, serverDB.Model(&database.Book{}).Count(&serverBookCount), "counting api notes")
assert.Equal(t, serverNoteCount, expected.serverNoteCount, "server note count mismatch")
assert.Equal(t, serverBookCount, expected.serverBookCount, "server book count mismatch")
var serverUser database.User
apitest.MustExec(t, serverDB.Where("id = ?", user.ID).First(&serverUser), "finding user")
assert.Equal(t, serverUser.MaxUSN, expected.serverUserMaxUSN, "user max_usn mismatch")
}

2
pkg/server/controllers/notes.go
View file

@ -218,7 +218,7 @@ func (n *Notes) create(r *http.Request) (database.Note, error) {
var book database.Book
if err := n.app.DB.Where("uuid = ? AND user_id = ?", params.BookUUID, user.ID).First(&book).Error; err != nil {
return database.Note{}, errors.Wrap(err, "finding book")
return database.Note{}, errors.Wrapf(err, "finding book %s", params.BookUUID)
}
client := getClientType(r)