From d6afbd51a8fc3f380e360523959cdcdcaa962b5a Mon Sep 17 00:00:00 2001 From: Sung Date: Sat, 25 Oct 2025 22:50:01 -0700 Subject: [PATCH] Split sync test --- pkg/e2e/{sync_test.go => sync/basic_test.go} | 703 +------------------ pkg/e2e/sync/empty_server_test.go | 449 ++++++++++++ pkg/e2e/sync/main_test.go | 59 ++ pkg/e2e/sync/testutils.go | 300 ++++++++ 4 files changed, 809 insertions(+), 702 deletions(-) rename pkg/e2e/{sync_test.go => sync/basic_test.go} (86%) create mode 100644 pkg/e2e/sync/empty_server_test.go create mode 100644 pkg/e2e/sync/main_test.go create mode 100644 pkg/e2e/sync/testutils.go diff --git a/pkg/e2e/sync_test.go b/pkg/e2e/sync/basic_test.go similarity index 86% rename from pkg/e2e/sync_test.go rename to pkg/e2e/sync/basic_test.go index 10df15c9..4986229a 100644 --- a/pkg/e2e/sync_test.go +++ b/pkg/e2e/sync/basic_test.go @@ -16,307 +16,21 @@ * along with Dnote. If not, see . */ -package main +package sync import ( - "bytes" - "encoding/json" "fmt" - "io" - "log" - "net/http" - "net/http/httptest" "os" - "os/exec" - "path/filepath" - "strings" "testing" - "time" "github.com/dnote/dnote/pkg/assert" - "github.com/dnote/dnote/pkg/cli/consts" cliDatabase "github.com/dnote/dnote/pkg/cli/database" "github.com/dnote/dnote/pkg/cli/testutils" clitest "github.com/dnote/dnote/pkg/cli/testutils" - "github.com/dnote/dnote/pkg/clock" - "github.com/dnote/dnote/pkg/server/app" - "github.com/dnote/dnote/pkg/server/controllers" "github.com/dnote/dnote/pkg/server/database" - "github.com/dnote/dnote/pkg/server/mailer" apitest "github.com/dnote/dnote/pkg/server/testutils" - "github.com/pkg/errors" - "gorm.io/gorm" ) -var cliBinaryName string -var serverTime = time.Date(2017, time.March, 14, 21, 15, 0, 0, time.UTC) - -var testDir = "./tmp/.dnote" - -func init() { - cliBinaryName = fmt.Sprintf("%s/test/cli/test-cli", testDir) -} - -// testEnv holds the test environment for a single test -type testEnv struct { - DB *cliDatabase.DB - CmdOpts clitest.RunDnoteCmdOptions - Server *httptest.Server - ServerDB *gorm.DB - TmpDir string -} - -// setupTestEnv creates an isolated test environment with its own database and temp directory -func setupTestEnv(t *testing.T) testEnv { - tmpDir := t.TempDir() - - // Create .dnote directory - dnoteDir := filepath.Join(tmpDir, consts.DnoteDirName) - if err := os.MkdirAll(dnoteDir, 0755); err != nil { - t.Fatal(errors.Wrap(err, "creating dnote directory")) - } - - // Create database at the expected path - dbPath := filepath.Join(dnoteDir, consts.DnoteDBFileName) - db := cliDatabase.InitTestFileDBRaw(t, dbPath) - - // Create server - server, serverDB := setupNewServer(t) - - // Create config file with this server's endpoint - apiEndpoint := fmt.Sprintf("%s/api", server.URL) - updateConfigAPIEndpoint(t, tmpDir, apiEndpoint) - - // Create command options with XDG paths pointing to temp dir - cmdOpts := clitest.RunDnoteCmdOptions{ - Env: []string{ - fmt.Sprintf("XDG_CONFIG_HOME=%s", tmpDir), - fmt.Sprintf("XDG_DATA_HOME=%s", tmpDir), - fmt.Sprintf("XDG_CACHE_HOME=%s", tmpDir), - }, - } - - return testEnv{ - DB: db, - CmdOpts: cmdOpts, - Server: server, - ServerDB: serverDB, - TmpDir: tmpDir, - } -} - -// setupTestServer creates a test server with its own database -func setupTestServer(t *testing.T, serverTime time.Time) (*httptest.Server, *gorm.DB, error) { - db := apitest.InitMemoryDB(t) - - mockClock := clock.NewMock() - mockClock.SetNow(serverTime) - - a := app.NewTest() - a.Clock = mockClock - a.EmailTemplates = mailer.Templates{} - a.EmailBackend = &apitest.MockEmailbackendImplementation{} - a.DB = db - - server, err := controllers.NewServer(&a) - if err != nil { - return nil, nil, errors.Wrap(err, "initializing server") - } - - return server, db, nil -} - -// setupNewServer creates a new server and returns the server and database. -// This is useful when a test needs to switch to a new empty server. -func setupNewServer(t *testing.T) (*httptest.Server, *gorm.DB) { - server, serverDB, err := setupTestServer(t, serverTime) - if err != nil { - t.Fatal(errors.Wrap(err, "setting up new test server")) - } - t.Cleanup(func() { server.Close() }) - - return server, serverDB -} - -// updateConfigAPIEndpoint updates the config file with the given API endpoint -func updateConfigAPIEndpoint(t *testing.T, tmpDir string, apiEndpoint string) { - dnoteDir := filepath.Join(tmpDir, consts.DnoteDirName) - configPath := filepath.Join(dnoteDir, consts.ConfigFilename) - configContent := fmt.Sprintf("apiEndpoint: %s\n", apiEndpoint) - if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil { - t.Fatal(errors.Wrap(err, "writing config file")) - } -} - -// switchToEmptyServer closes the current server and creates a new empty server, -// updating the config file to point to it. -func switchToEmptyServer(t *testing.T, env *testEnv) { - // Close old server - env.Server.Close() - - // Create new empty server - env.Server, env.ServerDB = setupNewServer(t) - - // Update config file to point to new server - apiEndpoint := fmt.Sprintf("%s/api", env.Server.URL) - updateConfigAPIEndpoint(t, env.TmpDir, apiEndpoint) -} - -func TestMain(m *testing.M) { - // Build CLI binary without hardcoded API endpoint - // Each test will create its own server and config file - cmd := exec.Command("go", "build", "--tags", "fts5", "-o", cliBinaryName, "github.com/dnote/dnote/pkg/cli") - - var stderr bytes.Buffer - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - log.Print(errors.Wrap(err, "building a CLI binary").Error()) - log.Print(stderr.String()) - os.Exit(1) - } - - os.Exit(m.Run()) -} - -// helpers -func setupUser(t *testing.T, env testEnv) database.User { - user := apitest.SetupUserData(env.ServerDB, "alice@example.com", "pass1234") - - return user -} - -func setupUserAndLogin(t *testing.T, env testEnv) database.User { - user := setupUser(t, env) - login(t, env.DB, env.ServerDB, user) - - return user -} - -// log in the user in CLI -func login(t *testing.T, db *cliDatabase.DB, serverDB *gorm.DB, user database.User) { - session := apitest.SetupSession(serverDB, user) - - cliDatabase.MustExec(t, "inserting session_key", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKey, session.Key) - cliDatabase.MustExec(t, "inserting session_key_expiry", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKeyExpiry, session.ExpiresAt.Unix()) -} - -func apiCreateBook(t *testing.T, env testEnv, user database.User, name, message string) string { - res := doHTTPReq(t, env, "POST", "/v3/books", fmt.Sprintf(`{"name": "%s"}`, name), message, user) - - var resp controllers.CreateBookResp - if err := json.NewDecoder(res.Body).Decode(&resp); err != nil { - t.Fatal(errors.Wrap(err, "decoding payload for adding book")) - return "" - } - - return resp.Book.UUID -} - -func apiPatchBook(t *testing.T, env testEnv, user database.User, uuid, payload, message string) { - doHTTPReq(t, env, "PATCH", fmt.Sprintf("/v3/books/%s", uuid), payload, message, user) -} - -func apiDeleteBook(t *testing.T, env testEnv, user database.User, uuid, message string) { - doHTTPReq(t, env, "DELETE", fmt.Sprintf("/v3/books/%s", uuid), "", message, user) -} - -func apiCreateNote(t *testing.T, env testEnv, user database.User, bookUUID, body, message string) string { - res := doHTTPReq(t, env, "POST", "/v3/notes", fmt.Sprintf(`{"book_uuid": "%s", "content": "%s"}`, bookUUID, body), message, user) - - var resp controllers.CreateNoteResp - if err := json.NewDecoder(res.Body).Decode(&resp); err != nil { - t.Fatal(errors.Wrap(err, "decoding payload for adding note")) - return "" - } - - return resp.Result.UUID -} - -func apiPatchNote(t *testing.T, env testEnv, user database.User, noteUUID, payload, message string) { - doHTTPReq(t, env, "PATCH", fmt.Sprintf("/v3/notes/%s", noteUUID), payload, message, user) -} - -func apiDeleteNote(t *testing.T, env testEnv, user database.User, noteUUID, message string) { - doHTTPReq(t, env, "DELETE", fmt.Sprintf("/v3/notes/%s", noteUUID), "", message, user) -} - -func doHTTPReq(t *testing.T, env testEnv, method, path, payload, message string, user database.User) *http.Response { - apiEndpoint := fmt.Sprintf("%s/api", env.Server.URL) - endpoint := fmt.Sprintf("%s%s", apiEndpoint, path) - - req, err := http.NewRequest(method, endpoint, strings.NewReader(payload)) - if err != nil { - panic(errors.Wrap(err, "constructing http request")) - } - - res := apitest.HTTPAuthDo(t, env.ServerDB, req, user) - if res.StatusCode >= 400 { - bs, err := io.ReadAll(res.Body) - if err != nil { - panic(errors.Wrap(err, "parsing response body for error")) - } - - t.Errorf("%s. HTTP status %d. Message: %s", message, res.StatusCode, string(bs)) - } - - return res -} - -type setupFunc func(t *testing.T, env testEnv, user database.User) map[string]string -type assertFunc func(t *testing.T, env testEnv, user database.User, ids map[string]string) - -func testSyncCmd(t *testing.T, fullSync bool, setup setupFunc, assert assertFunc) { - env := setupTestEnv(t) - - user := setupUserAndLogin(t, env) - ids := setup(t, env, user) - - if fullSync { - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "-f") - } else { - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync") - } - - assert(t, env, user, ids) -} - -type systemState struct { - clientNoteCount int - clientBookCount int - clientLastMaxUSN int - clientLastSyncAt int64 - serverNoteCount int64 - serverBookCount int64 - serverUserMaxUSN int -} - -// checkState compares the state of the client and the server with the given system state -func checkState(t *testing.T, clientDB *cliDatabase.DB, user database.User, serverDB *gorm.DB, expected systemState) { - var clientBookCount, clientNoteCount int - cliDatabase.MustScan(t, "counting client notes", clientDB.QueryRow("SELECT count(*) FROM notes"), &clientNoteCount) - cliDatabase.MustScan(t, "counting client books", clientDB.QueryRow("SELECT count(*) FROM books"), &clientBookCount) - assert.Equal(t, clientNoteCount, expected.clientNoteCount, "client note count mismatch") - assert.Equal(t, clientBookCount, expected.clientBookCount, "client book count mismatch") - - var clientLastMaxUSN int - var clientLastSyncAt int64 - cliDatabase.MustScan(t, "finding system last_max_usn", clientDB.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastMaxUSN), &clientLastMaxUSN) - cliDatabase.MustScan(t, "finding system last_sync_at", clientDB.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastSyncAt), &clientLastSyncAt) - assert.Equal(t, clientLastMaxUSN, expected.clientLastMaxUSN, "client last_max_usn mismatch") - assert.Equal(t, clientLastSyncAt, expected.clientLastSyncAt, "client last_sync_at mismatch") - - var serverBookCount, serverNoteCount int64 - apitest.MustExec(t, serverDB.Model(&database.Note{}).Count(&serverNoteCount), "counting server notes") - apitest.MustExec(t, serverDB.Model(&database.Book{}).Count(&serverBookCount), "counting api notes") - assert.Equal(t, serverNoteCount, expected.serverNoteCount, "server note count mismatch") - assert.Equal(t, serverBookCount, expected.serverBookCount, "server book count mismatch") - var serverUser database.User - apitest.MustExec(t, serverDB.Where("id = ?", user.ID).First(&serverUser), "finding user") - assert.Equal(t, serverUser.MaxUSN, expected.serverUserMaxUSN, "user max_usn mismatch") -} - -// tests func TestSync_Empty(t *testing.T) { setup := func(t *testing.T, env testEnv, user database.User) map[string]string { return map[string]string{} @@ -3896,421 +3610,6 @@ func TestFullSync(t *testing.T) { }) } -func TestSync_EmptyServer(t *testing.T) { - t.Run("sync to empty server after syncing to non-empty server", func(t *testing.T) { - // Test server data loss/wipe scenario (disaster recovery): - // Verify empty server detection works when the server loses all its data - - env := setupTestEnv(t) - - user := setupUserAndLogin(t, env) - - // Step 1: Create local data and sync to server - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1") - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1") - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync") - - // Verify sync succeeded - checkState(t, env.DB, user, env.ServerDB, systemState{ - clientNoteCount: 2, - clientBookCount: 2, - clientLastMaxUSN: 4, - clientLastSyncAt: serverTime.Unix(), - serverNoteCount: 2, - serverBookCount: 2, - serverUserMaxUSN: 4, - }) - - // Step 2: Switch to a completely new empty server - switchToEmptyServer(t, &env) - - // Recreate user and session on new server - user = setupUserAndLogin(t, env) - - // Step 3: Sync again - should detect empty server and prompt user - // User confirms with "y" - clitest.MustWaitDnoteCmd(t, env.CmdOpts, clitest.UserConfirmEmptyServerSync, cliBinaryName, "sync") - - // Step 4: Verify data was uploaded to the empty server - checkState(t, env.DB, user, env.ServerDB, systemState{ - clientNoteCount: 2, - clientBookCount: 2, - clientLastMaxUSN: 4, - clientLastSyncAt: serverTime.Unix(), - serverNoteCount: 2, - serverBookCount: 2, - serverUserMaxUSN: 4, - }) - - // Verify the content is correct on both client and server - var cliNote1JS, cliNote1CSS cliDatabase.Note - var cliBookJS, cliBookCSS cliDatabase.Book - cliDatabase.MustScan(t, "finding cliNote1JS", env.DB.QueryRow("SELECT uuid, body FROM notes WHERE body = ?", "js1"), &cliNote1JS.UUID, &cliNote1JS.Body) - cliDatabase.MustScan(t, "finding cliNote1CSS", env.DB.QueryRow("SELECT uuid, body FROM notes WHERE body = ?", "css1"), &cliNote1CSS.UUID, &cliNote1CSS.Body) - cliDatabase.MustScan(t, "finding cliBookJS", env.DB.QueryRow("SELECT uuid, label FROM books WHERE label = ?", "js"), &cliBookJS.UUID, &cliBookJS.Label) - cliDatabase.MustScan(t, "finding cliBookCSS", env.DB.QueryRow("SELECT uuid, label FROM books WHERE label = ?", "css"), &cliBookCSS.UUID, &cliBookCSS.Label) - - assert.Equal(t, cliNote1JS.Body, "js1", "js note body mismatch") - assert.Equal(t, cliNote1CSS.Body, "css1", "css note body mismatch") - assert.Equal(t, cliBookJS.Label, "js", "js book label mismatch") - assert.Equal(t, cliBookCSS.Label, "css", "css book label mismatch") - - // Verify on server side - var serverNoteJS, serverNoteCSS database.Note - var serverBookJS, serverBookCSS database.Book - apitest.MustExec(t, env.ServerDB.Where("body = ?", "js1").First(&serverNoteJS), "finding server note js1") - apitest.MustExec(t, env.ServerDB.Where("body = ?", "css1").First(&serverNoteCSS), "finding server note css1") - apitest.MustExec(t, env.ServerDB.Where("label = ?", "js").First(&serverBookJS), "finding server book js") - apitest.MustExec(t, env.ServerDB.Where("label = ?", "css").First(&serverBookCSS), "finding server book css") - - assert.Equal(t, serverNoteJS.Body, "js1", "server js note body mismatch") - assert.Equal(t, serverNoteCSS.Body, "css1", "server css note body mismatch") - assert.Equal(t, serverBookJS.Label, "js", "server js book label mismatch") - assert.Equal(t, serverBookCSS.Label, "css", "server css book label mismatch") - }) - - t.Run("user cancels empty server prompt", func(t *testing.T) { - env := setupTestEnv(t) - - user := setupUserAndLogin(t, env) - - // Step 1: Create local data and sync to server - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1") - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1") - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync") - - // Verify initial sync succeeded - checkState(t, env.DB, user, env.ServerDB, systemState{ - clientNoteCount: 2, - clientBookCount: 2, - clientLastMaxUSN: 4, - clientLastSyncAt: serverTime.Unix(), - serverNoteCount: 2, - serverBookCount: 2, - serverUserMaxUSN: 4, - }) - - // Step 2: Switch to empty server - switchToEmptyServer(t, &env) - user = setupUserAndLogin(t, env) - - // Step 3: Sync again but user cancels with "n" - output, err := clitest.WaitDnoteCmd(t, env.CmdOpts, clitest.UserCancelEmptyServerSync, cliBinaryName, "sync") - if err == nil { - t.Fatal("Expected sync to fail when user cancels, but it succeeded") - } - - // Verify the prompt appeared - if !strings.Contains(output, clitest.PromptEmptyServer) { - t.Fatalf("Expected empty server warning in output, got: %s", output) - } - - // Step 4: Verify local state unchanged (transaction rolled back) - checkState(t, env.DB, user, env.ServerDB, systemState{ - clientNoteCount: 2, - clientBookCount: 2, - clientLastMaxUSN: 4, - clientLastSyncAt: serverTime.Unix(), - serverNoteCount: 0, - serverBookCount: 0, - serverUserMaxUSN: 0, - }) - - // Verify items still have original USN and dirty=false - var book cliDatabase.Book - var note cliDatabase.Note - cliDatabase.MustScan(t, "checking book state", env.DB.QueryRow("SELECT usn, dirty FROM books WHERE label = ?", "js"), &book.USN, &book.Dirty) - cliDatabase.MustScan(t, "checking note state", env.DB.QueryRow("SELECT usn, dirty FROM notes WHERE body = ?", "js1"), ¬e.USN, ¬e.Dirty) - - assert.NotEqual(t, book.USN, 0, "book USN should not be reset") - assert.NotEqual(t, note.USN, 0, "note USN should not be reset") - assert.Equal(t, book.Dirty, false, "book should not be marked dirty") - assert.Equal(t, note.Dirty, false, "note should not be marked dirty") - }) - - t.Run("all local data is marked deleted - should not upload", func(t *testing.T) { - // Test edge case: Server MaxUSN=0, local MaxUSN>0, but all items are deleted=true - // Should NOT prompt because there's nothing to upload - - env := setupTestEnv(t) - - user := setupUserAndLogin(t, env) - - // Step 1: Create local data and sync to server - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1") - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1") - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync") - - // Verify initial sync succeeded - checkState(t, env.DB, user, env.ServerDB, systemState{ - clientNoteCount: 2, - clientBookCount: 2, - clientLastMaxUSN: 4, - clientLastSyncAt: serverTime.Unix(), - serverNoteCount: 2, - serverBookCount: 2, - serverUserMaxUSN: 4, - }) - - // Step 2: Delete all local notes and books (mark as deleted) - cliDatabase.MustExec(t, "marking all books deleted", env.DB, "UPDATE books SET deleted = 1") - cliDatabase.MustExec(t, "marking all notes deleted", env.DB, "UPDATE notes SET deleted = 1") - - // Step 3: Switch to empty server - switchToEmptyServer(t, &env) - user = setupUserAndLogin(t, env) - - // Step 4: Sync - should NOT prompt because bookCount=0 and noteCount=0 (counting only deleted=0) - // This should complete without user interaction - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync") - - // Verify no data was uploaded (server still empty, but client still has deleted items) - // Check server is empty - var serverNoteCount, serverBookCount int64 - apitest.MustExec(t, env.ServerDB.Model(&database.Note{}).Count(&serverNoteCount), "counting server notes") - apitest.MustExec(t, env.ServerDB.Model(&database.Book{}).Count(&serverBookCount), "counting server books") - assert.Equal(t, serverNoteCount, int64(0), "server should have no notes") - assert.Equal(t, serverBookCount, int64(0), "server should have no books") - - // Check client still has the deleted items locally - var clientNoteCount, clientBookCount int - cliDatabase.MustScan(t, "counting client notes", env.DB.QueryRow("SELECT count(*) FROM notes WHERE deleted = 1"), &clientNoteCount) - cliDatabase.MustScan(t, "counting client books", env.DB.QueryRow("SELECT count(*) FROM books WHERE deleted = 1"), &clientBookCount) - assert.Equal(t, clientNoteCount, 2, "client should still have 2 deleted notes") - assert.Equal(t, clientBookCount, 2, "client should still have 2 deleted books") - - // Verify lastMaxUSN was reset to 0 - var lastMaxUSN int - cliDatabase.MustScan(t, "getting lastMaxUSN", env.DB.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastMaxUSN), &lastMaxUSN) - assert.Equal(t, lastMaxUSN, 0, "lastMaxUSN should be reset to 0") - }) - - t.Run("race condition - other client uploads first", func(t *testing.T) { - // This test exercises a race condition that can occur during sync: - // While Client A is waiting for user input, Client B uploads data to the server. - // - // The empty server scenario is the natural place to test this because - // an empty server detection triggers a prompt, at which point the test - // can make client B upload data. We trigger the race condition deterministically. - // - // Test flow: - // - Client A detects empty server and prompts user - // - While waiting for confirmation, Client B uploads the same data via API - // - Client A continues and handles the 409 conflict gracefully by: - // 1. Detecting the 409 error when trying to CREATE books that already exist - // 2. Running stepSync to pull the server's books (js, css) - // 3. mergeBook renames local conflicts (js→js_2, css→css_2) - // 4. Retrying sendChanges to upload the renamed books - // - Result: Both clients' data is preserved (4 books total) - - env := setupTestEnv(t) - - user := setupUserAndLogin(t, env) - - // Step 1: Create local data and sync to establish lastMaxUSN > 0 - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1") - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1") - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync") - - // Verify initial sync succeeded - checkState(t, env.DB, user, env.ServerDB, systemState{ - clientNoteCount: 2, - clientBookCount: 2, - clientLastMaxUSN: 4, - clientLastSyncAt: serverTime.Unix(), - serverNoteCount: 2, - serverBookCount: 2, - serverUserMaxUSN: 4, - }) - - // Step 2: Switch to new empty server to simulate switching to empty server - switchToEmptyServer(t, &env) - - // Create user on new server and login - user = setupUserAndLogin(t, env) - - // Step 3: Trigger sync which will detect empty server and prompt user - // Inside the callback (before confirming), we simulate Client B uploading via API. - // We wait for the empty server prompt to ensure Client B uploads AFTER - // GetSyncState but BEFORE the sync decision, creating the race condition deterministically - raceCallback := func(stdout io.Reader, stdin io.WriteCloser) error { - // First, wait for the prompt to ensure Client A has obtained the sync state from the server. - clitest.MustWaitForPrompt(t, stdout, clitest.PromptEmptyServer) - - // Now Client B uploads the same data via API (after Client A got the sync state from the server - // but before its sync decision) - // This creates the race condition: Client A thinks server is empty, but Client B uploads data - jsBookUUID := apiCreateBook(t, env, user, "js", "client B creating js book") - cssBookUUID := apiCreateBook(t, env, user, "css", "client B creating css book") - apiCreateNote(t, env, user, jsBookUUID, "js1", "client B creating js note") - apiCreateNote(t, env, user, cssBookUUID, "css1", "client B creating css note") - - // Now user confirms - if _, err := io.WriteString(stdin, "y\n"); err != nil { - return errors.Wrap(err, "confirming sync") - } - - return nil - } - - // Step 4: Client A runs sync with race condition - // The 409 conflict is automatically handled: - // - When 409 is detected, isBehind flag is set - // - stepSync pulls Client B's data - // - mergeBook renames Client A's books to js_2, css_2 - // - Renamed books are uploaded - // - Both clients' data is preserved. - clitest.MustWaitDnoteCmd(t, env.CmdOpts, raceCallback, cliBinaryName, "sync") - - // Verify final state - both clients' data preserved - checkState(t, env.DB, user, env.ServerDB, systemState{ - clientNoteCount: 4, // Both clients' notes - clientBookCount: 4, // js, css, js_2, css_2 - clientLastMaxUSN: 8, // 4 from Client B + 4 from Client A's renamed books/notes - clientLastSyncAt: serverTime.Unix(), - serverNoteCount: 4, - serverBookCount: 4, - serverUserMaxUSN: 8, - }) - - // Verify server has both clients' books - var svrBookJS, svrBookCSS, svrBookJS2, svrBookCSS2 database.Book - apitest.MustExec(t, env.ServerDB.Where("label = ?", "js").First(&svrBookJS), "finding server book 'js'") - apitest.MustExec(t, env.ServerDB.Where("label = ?", "css").First(&svrBookCSS), "finding server book 'css'") - apitest.MustExec(t, env.ServerDB.Where("label = ?", "js_2").First(&svrBookJS2), "finding server book 'js_2'") - apitest.MustExec(t, env.ServerDB.Where("label = ?", "css_2").First(&svrBookCSS2), "finding server book 'css_2'") - - assert.Equal(t, svrBookJS.Label, "js", "server should have book 'js' (Client B)") - assert.Equal(t, svrBookCSS.Label, "css", "server should have book 'css' (Client B)") - assert.Equal(t, svrBookJS2.Label, "js_2", "server should have book 'js_2' (Client A renamed)") - assert.Equal(t, svrBookCSS2.Label, "css_2", "server should have book 'css_2' (Client A renamed)") - - // Verify client has all books - var cliBookJS, cliBookCSS, cliBookJS2, cliBookCSS2 cliDatabase.Book - cliDatabase.MustScan(t, "finding client book 'js'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "js"), &cliBookJS.UUID, &cliBookJS.Label, &cliBookJS.USN) - cliDatabase.MustScan(t, "finding client book 'css'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "css"), &cliBookCSS.UUID, &cliBookCSS.Label, &cliBookCSS.USN) - cliDatabase.MustScan(t, "finding client book 'js_2'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "js_2"), &cliBookJS2.UUID, &cliBookJS2.Label, &cliBookJS2.USN) - cliDatabase.MustScan(t, "finding client book 'css_2'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "css_2"), &cliBookCSS2.UUID, &cliBookCSS2.Label, &cliBookCSS2.USN) - - // Verify client UUIDs match server - assert.Equal(t, cliBookJS.UUID, svrBookJS.UUID, "client 'js' UUID should match server") - assert.Equal(t, cliBookCSS.UUID, svrBookCSS.UUID, "client 'css' UUID should match server") - assert.Equal(t, cliBookJS2.UUID, svrBookJS2.UUID, "client 'js_2' UUID should match server") - assert.Equal(t, cliBookCSS2.UUID, svrBookCSS2.UUID, "client 'css_2' UUID should match server") - - // Verify all items have non-zero USN (synced successfully) - assert.NotEqual(t, cliBookJS.USN, 0, "client 'js' should have non-zero USN") - assert.NotEqual(t, cliBookCSS.USN, 0, "client 'css' should have non-zero USN") - assert.NotEqual(t, cliBookJS2.USN, 0, "client 'js_2' should have non-zero USN") - assert.NotEqual(t, cliBookCSS2.USN, 0, "client 'css_2' should have non-zero USN") - }) - - t.Run("sync to server A, then B, then back to A, then back to B", func(t *testing.T) { - // Test switching between two actual servers to verify: - // 1. Empty server detection works when switching to empty server - // 2. No false detection when switching back to non-empty servers - // 3. Both servers maintain independent state across multiple switches - - env := setupTestEnv(t) - - // Create Server A with its own database - serverA, serverDBA, err := setupTestServer(t, serverTime) - if err != nil { - t.Fatal(errors.Wrap(err, "setting up server A")) - } - defer serverA.Close() - - // Create Server B with its own database - serverB, serverDBB, err := setupTestServer(t, serverTime) - if err != nil { - t.Fatal(errors.Wrap(err, "setting up server B")) - } - defer serverB.Close() - - // Step 1: Set up user on Server A and sync - apiEndpointA := fmt.Sprintf("%s/api", serverA.URL) - - userA := apitest.SetupUserData(serverDBA, "alice@example.com", "pass1234") - sessionA := apitest.SetupSession(serverDBA, userA) - cliDatabase.MustExec(t, "inserting session_key", env.DB, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKey, sessionA.Key) - cliDatabase.MustExec(t, "inserting session_key_expiry", env.DB, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKeyExpiry, sessionA.ExpiresAt.Unix()) - - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1") - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1") - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "--apiEndpoint", apiEndpointA) - - // Verify sync to Server A succeeded - checkState(t, env.DB, userA, serverDBA, systemState{ - clientNoteCount: 2, - clientBookCount: 2, - clientLastMaxUSN: 4, - clientLastSyncAt: serverTime.Unix(), - serverNoteCount: 2, - serverBookCount: 2, - serverUserMaxUSN: 4, - }) - - // Step 2: Switch to Server B (empty) and sync - apiEndpointB := fmt.Sprintf("%s/api", serverB.URL) - - // Set up user on Server B - userB := apitest.SetupUserData(serverDBB, "alice@example.com", "pass1234") - sessionB := apitest.SetupSession(serverDBB, userB) - cliDatabase.MustExec(t, "updating session_key for B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.Key, consts.SystemSessionKey) - cliDatabase.MustExec(t, "updating session_key_expiry for B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.ExpiresAt.Unix(), consts.SystemSessionKeyExpiry) - - // Should detect empty server and prompt - clitest.MustWaitDnoteCmd(t, env.CmdOpts, clitest.UserConfirmEmptyServerSync, cliBinaryName, "sync", "--apiEndpoint", apiEndpointB) - - // Verify Server B now has data - checkState(t, env.DB, userB, serverDBB, systemState{ - clientNoteCount: 2, - clientBookCount: 2, - clientLastMaxUSN: 4, - clientLastSyncAt: serverTime.Unix(), - serverNoteCount: 2, - serverBookCount: 2, - serverUserMaxUSN: 4, - }) - - // Step 3: Switch back to Server A and sync - cliDatabase.MustExec(t, "updating session_key back to A", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionA.Key, consts.SystemSessionKey) - cliDatabase.MustExec(t, "updating session_key_expiry back to A", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionA.ExpiresAt.Unix(), consts.SystemSessionKeyExpiry) - - // Should NOT trigger empty server detection (Server A has MaxUSN > 0) - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "--apiEndpoint", apiEndpointA) - - // Verify Server A still has its data - checkState(t, env.DB, userA, serverDBA, systemState{ - clientNoteCount: 2, - clientBookCount: 2, - clientLastMaxUSN: 4, - clientLastSyncAt: serverTime.Unix(), - serverNoteCount: 2, - serverBookCount: 2, - serverUserMaxUSN: 4, - }) - - // Step 4: Switch back to Server B and sync again - cliDatabase.MustExec(t, "updating session_key back to B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.Key, consts.SystemSessionKey) - cliDatabase.MustExec(t, "updating session_key_expiry back to B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.ExpiresAt.Unix(), consts.SystemSessionKeyExpiry) - - // Should NOT trigger empty server detection (Server B now has MaxUSN > 0 from Step 2) - clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "--apiEndpoint", apiEndpointB) - - // Verify both servers maintain independent state - checkState(t, env.DB, userB, serverDBB, systemState{ - clientNoteCount: 2, - clientBookCount: 2, - clientLastMaxUSN: 4, - clientLastSyncAt: serverTime.Unix(), - serverNoteCount: 2, - serverBookCount: 2, - serverUserMaxUSN: 4, - }) - }) -} - func TestSync_FreshClientConcurrent(t *testing.T) { // Test the core issue: Fresh client (never synced, lastMaxUSN=0) syncing to a server // that already has data uploaded by another client. diff --git a/pkg/e2e/sync/empty_server_test.go b/pkg/e2e/sync/empty_server_test.go new file mode 100644 index 00000000..a35086f9 --- /dev/null +++ b/pkg/e2e/sync/empty_server_test.go @@ -0,0 +1,449 @@ +/* Copyright (C) 2019, 2020, 2021, 2022, 2023, 2024, 2025 Dnote contributors + * + * This file is part of Dnote. + * + * Dnote is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Dnote is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Dnote. If not, see . + */ + +package sync + +import ( + "fmt" + "io" + "strings" + "testing" + + "github.com/dnote/dnote/pkg/assert" + "github.com/dnote/dnote/pkg/cli/consts" + cliDatabase "github.com/dnote/dnote/pkg/cli/database" + clitest "github.com/dnote/dnote/pkg/cli/testutils" + "github.com/dnote/dnote/pkg/server/database" + apitest "github.com/dnote/dnote/pkg/server/testutils" + "github.com/pkg/errors" +) + +func TestSync_EmptyServer(t *testing.T) { + t.Run("sync to empty server after syncing to non-empty server", func(t *testing.T) { + // Test server data loss/wipe scenario (disaster recovery): + // Verify empty server detection works when the server loses all its data + + env := setupTestEnv(t) + + user := setupUserAndLogin(t, env) + + // Step 1: Create local data and sync to server + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1") + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1") + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync") + + // Verify sync succeeded + checkState(t, env.DB, user, env.ServerDB, systemState{ + clientNoteCount: 2, + clientBookCount: 2, + clientLastMaxUSN: 4, + clientLastSyncAt: serverTime.Unix(), + serverNoteCount: 2, + serverBookCount: 2, + serverUserMaxUSN: 4, + }) + + // Step 2: Switch to a completely new empty server + switchToEmptyServer(t, &env) + + // Recreate user and session on new server + user = setupUserAndLogin(t, env) + + // Step 3: Sync again - should detect empty server and prompt user + // User confirms with "y" + clitest.MustWaitDnoteCmd(t, env.CmdOpts, clitest.UserConfirmEmptyServerSync, cliBinaryName, "sync") + + // Step 4: Verify data was uploaded to the empty server + checkState(t, env.DB, user, env.ServerDB, systemState{ + clientNoteCount: 2, + clientBookCount: 2, + clientLastMaxUSN: 4, + clientLastSyncAt: serverTime.Unix(), + serverNoteCount: 2, + serverBookCount: 2, + serverUserMaxUSN: 4, + }) + + // Verify the content is correct on both client and server + var cliNote1JS, cliNote1CSS cliDatabase.Note + var cliBookJS, cliBookCSS cliDatabase.Book + cliDatabase.MustScan(t, "finding cliNote1JS", env.DB.QueryRow("SELECT uuid, body FROM notes WHERE body = ?", "js1"), &cliNote1JS.UUID, &cliNote1JS.Body) + cliDatabase.MustScan(t, "finding cliNote1CSS", env.DB.QueryRow("SELECT uuid, body FROM notes WHERE body = ?", "css1"), &cliNote1CSS.UUID, &cliNote1CSS.Body) + cliDatabase.MustScan(t, "finding cliBookJS", env.DB.QueryRow("SELECT uuid, label FROM books WHERE label = ?", "js"), &cliBookJS.UUID, &cliBookJS.Label) + cliDatabase.MustScan(t, "finding cliBookCSS", env.DB.QueryRow("SELECT uuid, label FROM books WHERE label = ?", "css"), &cliBookCSS.UUID, &cliBookCSS.Label) + + assert.Equal(t, cliNote1JS.Body, "js1", "js note body mismatch") + assert.Equal(t, cliNote1CSS.Body, "css1", "css note body mismatch") + assert.Equal(t, cliBookJS.Label, "js", "js book label mismatch") + assert.Equal(t, cliBookCSS.Label, "css", "css book label mismatch") + + // Verify on server side + var serverNoteJS, serverNoteCSS database.Note + var serverBookJS, serverBookCSS database.Book + apitest.MustExec(t, env.ServerDB.Where("body = ?", "js1").First(&serverNoteJS), "finding server note js1") + apitest.MustExec(t, env.ServerDB.Where("body = ?", "css1").First(&serverNoteCSS), "finding server note css1") + apitest.MustExec(t, env.ServerDB.Where("label = ?", "js").First(&serverBookJS), "finding server book js") + apitest.MustExec(t, env.ServerDB.Where("label = ?", "css").First(&serverBookCSS), "finding server book css") + + assert.Equal(t, serverNoteJS.Body, "js1", "server js note body mismatch") + assert.Equal(t, serverNoteCSS.Body, "css1", "server css note body mismatch") + assert.Equal(t, serverBookJS.Label, "js", "server js book label mismatch") + assert.Equal(t, serverBookCSS.Label, "css", "server css book label mismatch") + }) + + t.Run("user cancels empty server prompt", func(t *testing.T) { + env := setupTestEnv(t) + + user := setupUserAndLogin(t, env) + + // Step 1: Create local data and sync to server + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1") + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1") + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync") + + // Verify initial sync succeeded + checkState(t, env.DB, user, env.ServerDB, systemState{ + clientNoteCount: 2, + clientBookCount: 2, + clientLastMaxUSN: 4, + clientLastSyncAt: serverTime.Unix(), + serverNoteCount: 2, + serverBookCount: 2, + serverUserMaxUSN: 4, + }) + + // Step 2: Switch to empty server + switchToEmptyServer(t, &env) + user = setupUserAndLogin(t, env) + + // Step 3: Sync again but user cancels with "n" + output, err := clitest.WaitDnoteCmd(t, env.CmdOpts, clitest.UserCancelEmptyServerSync, cliBinaryName, "sync") + if err == nil { + t.Fatal("Expected sync to fail when user cancels, but it succeeded") + } + + // Verify the prompt appeared + if !strings.Contains(output, clitest.PromptEmptyServer) { + t.Fatalf("Expected empty server warning in output, got: %s", output) + } + + // Step 4: Verify local state unchanged (transaction rolled back) + checkState(t, env.DB, user, env.ServerDB, systemState{ + clientNoteCount: 2, + clientBookCount: 2, + clientLastMaxUSN: 4, + clientLastSyncAt: serverTime.Unix(), + serverNoteCount: 0, + serverBookCount: 0, + serverUserMaxUSN: 0, + }) + + // Verify items still have original USN and dirty=false + var book cliDatabase.Book + var note cliDatabase.Note + cliDatabase.MustScan(t, "checking book state", env.DB.QueryRow("SELECT usn, dirty FROM books WHERE label = ?", "js"), &book.USN, &book.Dirty) + cliDatabase.MustScan(t, "checking note state", env.DB.QueryRow("SELECT usn, dirty FROM notes WHERE body = ?", "js1"), ¬e.USN, ¬e.Dirty) + + assert.NotEqual(t, book.USN, 0, "book USN should not be reset") + assert.NotEqual(t, note.USN, 0, "note USN should not be reset") + assert.Equal(t, book.Dirty, false, "book should not be marked dirty") + assert.Equal(t, note.Dirty, false, "note should not be marked dirty") + }) + + t.Run("all local data is marked deleted - should not upload", func(t *testing.T) { + // Test edge case: Server MaxUSN=0, local MaxUSN>0, but all items are deleted=true + // Should NOT prompt because there's nothing to upload + + env := setupTestEnv(t) + + user := setupUserAndLogin(t, env) + + // Step 1: Create local data and sync to server + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1") + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1") + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync") + + // Verify initial sync succeeded + checkState(t, env.DB, user, env.ServerDB, systemState{ + clientNoteCount: 2, + clientBookCount: 2, + clientLastMaxUSN: 4, + clientLastSyncAt: serverTime.Unix(), + serverNoteCount: 2, + serverBookCount: 2, + serverUserMaxUSN: 4, + }) + + // Step 2: Delete all local notes and books (mark as deleted) + cliDatabase.MustExec(t, "marking all books deleted", env.DB, "UPDATE books SET deleted = 1") + cliDatabase.MustExec(t, "marking all notes deleted", env.DB, "UPDATE notes SET deleted = 1") + + // Step 3: Switch to empty server + switchToEmptyServer(t, &env) + user = setupUserAndLogin(t, env) + + // Step 4: Sync - should NOT prompt because bookCount=0 and noteCount=0 (counting only deleted=0) + // This should complete without user interaction + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync") + + // Verify no data was uploaded (server still empty, but client still has deleted items) + // Check server is empty + var serverNoteCount, serverBookCount int64 + apitest.MustExec(t, env.ServerDB.Model(&database.Note{}).Count(&serverNoteCount), "counting server notes") + apitest.MustExec(t, env.ServerDB.Model(&database.Book{}).Count(&serverBookCount), "counting server books") + assert.Equal(t, serverNoteCount, int64(0), "server should have no notes") + assert.Equal(t, serverBookCount, int64(0), "server should have no books") + + // Check client still has the deleted items locally + var clientNoteCount, clientBookCount int + cliDatabase.MustScan(t, "counting client notes", env.DB.QueryRow("SELECT count(*) FROM notes WHERE deleted = 1"), &clientNoteCount) + cliDatabase.MustScan(t, "counting client books", env.DB.QueryRow("SELECT count(*) FROM books WHERE deleted = 1"), &clientBookCount) + assert.Equal(t, clientNoteCount, 2, "client should still have 2 deleted notes") + assert.Equal(t, clientBookCount, 2, "client should still have 2 deleted books") + + // Verify lastMaxUSN was reset to 0 + var lastMaxUSN int + cliDatabase.MustScan(t, "getting lastMaxUSN", env.DB.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastMaxUSN), &lastMaxUSN) + assert.Equal(t, lastMaxUSN, 0, "lastMaxUSN should be reset to 0") + }) + + t.Run("race condition - other client uploads first", func(t *testing.T) { + // This test exercises a race condition that can occur during sync: + // While Client A is waiting for user input, Client B uploads data to the server. + // + // The empty server scenario is the natural place to test this because + // an empty server detection triggers a prompt, at which point the test + // can make client B upload data. We trigger the race condition deterministically. + // + // Test flow: + // - Client A detects empty server and prompts user + // - While waiting for confirmation, Client B uploads the same data via API + // - Client A continues and handles the 409 conflict gracefully by: + // 1. Detecting the 409 error when trying to CREATE books that already exist + // 2. Running stepSync to pull the server's books (js, css) + // 3. mergeBook renames local conflicts (js→js_2, css→css_2) + // 4. Retrying sendChanges to upload the renamed books + // - Result: Both clients' data is preserved (4 books total) + + env := setupTestEnv(t) + + user := setupUserAndLogin(t, env) + + // Step 1: Create local data and sync to establish lastMaxUSN > 0 + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1") + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1") + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync") + + // Verify initial sync succeeded + checkState(t, env.DB, user, env.ServerDB, systemState{ + clientNoteCount: 2, + clientBookCount: 2, + clientLastMaxUSN: 4, + clientLastSyncAt: serverTime.Unix(), + serverNoteCount: 2, + serverBookCount: 2, + serverUserMaxUSN: 4, + }) + + // Step 2: Switch to new empty server to simulate switching to empty server + switchToEmptyServer(t, &env) + + // Create user on new server and login + user = setupUserAndLogin(t, env) + + // Step 3: Trigger sync which will detect empty server and prompt user + // Inside the callback (before confirming), we simulate Client B uploading via API. + // We wait for the empty server prompt to ensure Client B uploads AFTER + // GetSyncState but BEFORE the sync decision, creating the race condition deterministically + raceCallback := func(stdout io.Reader, stdin io.WriteCloser) error { + // First, wait for the prompt to ensure Client A has obtained the sync state from the server. + clitest.MustWaitForPrompt(t, stdout, clitest.PromptEmptyServer) + + // Now Client B uploads the same data via API (after Client A got the sync state from the server + // but before its sync decision) + // This creates the race condition: Client A thinks server is empty, but Client B uploads data + jsBookUUID := apiCreateBook(t, env, user, "js", "client B creating js book") + cssBookUUID := apiCreateBook(t, env, user, "css", "client B creating css book") + apiCreateNote(t, env, user, jsBookUUID, "js1", "client B creating js note") + apiCreateNote(t, env, user, cssBookUUID, "css1", "client B creating css note") + + // Now user confirms + if _, err := io.WriteString(stdin, "y\n"); err != nil { + return errors.Wrap(err, "confirming sync") + } + + return nil + } + + // Step 4: Client A runs sync with race condition + // The 409 conflict is automatically handled: + // - When 409 is detected, isBehind flag is set + // - stepSync pulls Client B's data + // - mergeBook renames Client A's books to js_2, css_2 + // - Renamed books are uploaded + // - Both clients' data is preserved. + clitest.MustWaitDnoteCmd(t, env.CmdOpts, raceCallback, cliBinaryName, "sync") + + // Verify final state - both clients' data preserved + checkState(t, env.DB, user, env.ServerDB, systemState{ + clientNoteCount: 4, // Both clients' notes + clientBookCount: 4, // js, css, js_2, css_2 + clientLastMaxUSN: 8, // 4 from Client B + 4 from Client A's renamed books/notes + clientLastSyncAt: serverTime.Unix(), + serverNoteCount: 4, + serverBookCount: 4, + serverUserMaxUSN: 8, + }) + + // Verify server has both clients' books + var svrBookJS, svrBookCSS, svrBookJS2, svrBookCSS2 database.Book + apitest.MustExec(t, env.ServerDB.Where("label = ?", "js").First(&svrBookJS), "finding server book 'js'") + apitest.MustExec(t, env.ServerDB.Where("label = ?", "css").First(&svrBookCSS), "finding server book 'css'") + apitest.MustExec(t, env.ServerDB.Where("label = ?", "js_2").First(&svrBookJS2), "finding server book 'js_2'") + apitest.MustExec(t, env.ServerDB.Where("label = ?", "css_2").First(&svrBookCSS2), "finding server book 'css_2'") + + assert.Equal(t, svrBookJS.Label, "js", "server should have book 'js' (Client B)") + assert.Equal(t, svrBookCSS.Label, "css", "server should have book 'css' (Client B)") + assert.Equal(t, svrBookJS2.Label, "js_2", "server should have book 'js_2' (Client A renamed)") + assert.Equal(t, svrBookCSS2.Label, "css_2", "server should have book 'css_2' (Client A renamed)") + + // Verify client has all books + var cliBookJS, cliBookCSS, cliBookJS2, cliBookCSS2 cliDatabase.Book + cliDatabase.MustScan(t, "finding client book 'js'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "js"), &cliBookJS.UUID, &cliBookJS.Label, &cliBookJS.USN) + cliDatabase.MustScan(t, "finding client book 'css'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "css"), &cliBookCSS.UUID, &cliBookCSS.Label, &cliBookCSS.USN) + cliDatabase.MustScan(t, "finding client book 'js_2'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "js_2"), &cliBookJS2.UUID, &cliBookJS2.Label, &cliBookJS2.USN) + cliDatabase.MustScan(t, "finding client book 'css_2'", env.DB.QueryRow("SELECT uuid, label, usn FROM books WHERE label = ?", "css_2"), &cliBookCSS2.UUID, &cliBookCSS2.Label, &cliBookCSS2.USN) + + // Verify client UUIDs match server + assert.Equal(t, cliBookJS.UUID, svrBookJS.UUID, "client 'js' UUID should match server") + assert.Equal(t, cliBookCSS.UUID, svrBookCSS.UUID, "client 'css' UUID should match server") + assert.Equal(t, cliBookJS2.UUID, svrBookJS2.UUID, "client 'js_2' UUID should match server") + assert.Equal(t, cliBookCSS2.UUID, svrBookCSS2.UUID, "client 'css_2' UUID should match server") + + // Verify all items have non-zero USN (synced successfully) + assert.NotEqual(t, cliBookJS.USN, 0, "client 'js' should have non-zero USN") + assert.NotEqual(t, cliBookCSS.USN, 0, "client 'css' should have non-zero USN") + assert.NotEqual(t, cliBookJS2.USN, 0, "client 'js_2' should have non-zero USN") + assert.NotEqual(t, cliBookCSS2.USN, 0, "client 'css_2' should have non-zero USN") + }) + + t.Run("sync to server A, then B, then back to A, then back to B", func(t *testing.T) { + // Test switching between two actual servers to verify: + // 1. Empty server detection works when switching to empty server + // 2. No false detection when switching back to non-empty servers + // 3. Both servers maintain independent state across multiple switches + + env := setupTestEnv(t) + + // Create Server A with its own database + serverA, serverDBA, err := setupTestServer(t, serverTime) + if err != nil { + t.Fatal(errors.Wrap(err, "setting up server A")) + } + defer serverA.Close() + + // Create Server B with its own database + serverB, serverDBB, err := setupTestServer(t, serverTime) + if err != nil { + t.Fatal(errors.Wrap(err, "setting up server B")) + } + defer serverB.Close() + + // Step 1: Set up user on Server A and sync + apiEndpointA := fmt.Sprintf("%s/api", serverA.URL) + + userA := apitest.SetupUserData(serverDBA, "alice@example.com", "pass1234") + sessionA := apitest.SetupSession(serverDBA, userA) + cliDatabase.MustExec(t, "inserting session_key", env.DB, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKey, sessionA.Key) + cliDatabase.MustExec(t, "inserting session_key_expiry", env.DB, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKeyExpiry, sessionA.ExpiresAt.Unix()) + + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "js", "-c", "js1") + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "add", "css", "-c", "css1") + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "--apiEndpoint", apiEndpointA) + + // Verify sync to Server A succeeded + checkState(t, env.DB, userA, serverDBA, systemState{ + clientNoteCount: 2, + clientBookCount: 2, + clientLastMaxUSN: 4, + clientLastSyncAt: serverTime.Unix(), + serverNoteCount: 2, + serverBookCount: 2, + serverUserMaxUSN: 4, + }) + + // Step 2: Switch to Server B (empty) and sync + apiEndpointB := fmt.Sprintf("%s/api", serverB.URL) + + // Set up user on Server B + userB := apitest.SetupUserData(serverDBB, "alice@example.com", "pass1234") + sessionB := apitest.SetupSession(serverDBB, userB) + cliDatabase.MustExec(t, "updating session_key for B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.Key, consts.SystemSessionKey) + cliDatabase.MustExec(t, "updating session_key_expiry for B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.ExpiresAt.Unix(), consts.SystemSessionKeyExpiry) + + // Should detect empty server and prompt + clitest.MustWaitDnoteCmd(t, env.CmdOpts, clitest.UserConfirmEmptyServerSync, cliBinaryName, "sync", "--apiEndpoint", apiEndpointB) + + // Verify Server B now has data + checkState(t, env.DB, userB, serverDBB, systemState{ + clientNoteCount: 2, + clientBookCount: 2, + clientLastMaxUSN: 4, + clientLastSyncAt: serverTime.Unix(), + serverNoteCount: 2, + serverBookCount: 2, + serverUserMaxUSN: 4, + }) + + // Step 3: Switch back to Server A and sync + cliDatabase.MustExec(t, "updating session_key back to A", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionA.Key, consts.SystemSessionKey) + cliDatabase.MustExec(t, "updating session_key_expiry back to A", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionA.ExpiresAt.Unix(), consts.SystemSessionKeyExpiry) + + // Should NOT trigger empty server detection (Server A has MaxUSN > 0) + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "--apiEndpoint", apiEndpointA) + + // Verify Server A still has its data + checkState(t, env.DB, userA, serverDBA, systemState{ + clientNoteCount: 2, + clientBookCount: 2, + clientLastMaxUSN: 4, + clientLastSyncAt: serverTime.Unix(), + serverNoteCount: 2, + serverBookCount: 2, + serverUserMaxUSN: 4, + }) + + // Step 4: Switch back to Server B and sync again + cliDatabase.MustExec(t, "updating session_key back to B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.Key, consts.SystemSessionKey) + cliDatabase.MustExec(t, "updating session_key_expiry back to B", env.DB, "UPDATE system SET value = ? WHERE key = ?", sessionB.ExpiresAt.Unix(), consts.SystemSessionKeyExpiry) + + // Should NOT trigger empty server detection (Server B now has MaxUSN > 0 from Step 2) + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "--apiEndpoint", apiEndpointB) + + // Verify both servers maintain independent state + checkState(t, env.DB, userB, serverDBB, systemState{ + clientNoteCount: 2, + clientBookCount: 2, + clientLastMaxUSN: 4, + clientLastSyncAt: serverTime.Unix(), + serverNoteCount: 2, + serverBookCount: 2, + serverUserMaxUSN: 4, + }) + }) +} diff --git a/pkg/e2e/sync/main_test.go b/pkg/e2e/sync/main_test.go new file mode 100644 index 00000000..b5d517f5 --- /dev/null +++ b/pkg/e2e/sync/main_test.go @@ -0,0 +1,59 @@ +/* Copyright (C) 2019, 2020, 2021, 2022, 2023, 2024, 2025 Dnote contributors + * + * This file is part of Dnote. + * + * This file is part of Dnote. + * + * Dnote is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Dnote is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Dnote. If not, see . + */ + +package sync + +import ( + "bytes" + "fmt" + "log" + "os" + "os/exec" + "testing" + "time" + + "github.com/pkg/errors" +) + +var cliBinaryName string +var serverTime = time.Date(2017, time.March, 14, 21, 15, 0, 0, time.UTC) + +var testDir = "./tmp/" + +func init() { + cliBinaryName = fmt.Sprintf("%s/test-cli", testDir) +} + +func TestMain(m *testing.M) { + // Build CLI binary without hardcoded API endpoint + // Each test will create its own server and config file + cmd := exec.Command("go", "build", "--tags", "fts5", "-o", cliBinaryName, "github.com/dnote/dnote/pkg/cli") + + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + log.Print(errors.Wrap(err, "building a CLI binary").Error()) + log.Print(stderr.String()) + os.Exit(1) + } + + os.Exit(m.Run()) +} diff --git a/pkg/e2e/sync/testutils.go b/pkg/e2e/sync/testutils.go new file mode 100644 index 00000000..def7bc47 --- /dev/null +++ b/pkg/e2e/sync/testutils.go @@ -0,0 +1,300 @@ +/* Copyright (C) 2019, 2020, 2021, 2022, 2023, 2024, 2025 Dnote contributors + * + * This file is part of Dnote. + * + * Dnote is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Dnote is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Dnote. If not, see . + */ + +package sync + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/dnote/dnote/pkg/assert" + "github.com/dnote/dnote/pkg/cli/consts" + cliDatabase "github.com/dnote/dnote/pkg/cli/database" + clitest "github.com/dnote/dnote/pkg/cli/testutils" + "github.com/dnote/dnote/pkg/clock" + "github.com/dnote/dnote/pkg/server/app" + "github.com/dnote/dnote/pkg/server/controllers" + "github.com/dnote/dnote/pkg/server/database" + "github.com/dnote/dnote/pkg/server/mailer" + apitest "github.com/dnote/dnote/pkg/server/testutils" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +// testEnv holds the test environment for a single test +type testEnv struct { + DB *cliDatabase.DB + CmdOpts clitest.RunDnoteCmdOptions + Server *httptest.Server + ServerDB *gorm.DB + TmpDir string +} + +// setupTestEnv creates an isolated test environment with its own database and temp directory +func setupTestEnv(t *testing.T) testEnv { + tmpDir := t.TempDir() + + // Create .dnote directory + dnoteDir := filepath.Join(tmpDir, consts.DnoteDirName) + if err := os.MkdirAll(dnoteDir, 0755); err != nil { + t.Fatal(errors.Wrap(err, "creating dnote directory")) + } + + // Create database at the expected path + dbPath := filepath.Join(dnoteDir, consts.DnoteDBFileName) + db := cliDatabase.InitTestFileDBRaw(t, dbPath) + + // Create server + server, serverDB := setupNewServer(t) + + // Create config file with this server's endpoint + apiEndpoint := fmt.Sprintf("%s/api", server.URL) + updateConfigAPIEndpoint(t, tmpDir, apiEndpoint) + + // Create command options with XDG paths pointing to temp dir + cmdOpts := clitest.RunDnoteCmdOptions{ + Env: []string{ + fmt.Sprintf("XDG_CONFIG_HOME=%s", tmpDir), + fmt.Sprintf("XDG_DATA_HOME=%s", tmpDir), + fmt.Sprintf("XDG_CACHE_HOME=%s", tmpDir), + }, + } + + return testEnv{ + DB: db, + CmdOpts: cmdOpts, + Server: server, + ServerDB: serverDB, + TmpDir: tmpDir, + } +} + +// setupTestServer creates a test server with its own database +func setupTestServer(t *testing.T, serverTime time.Time) (*httptest.Server, *gorm.DB, error) { + db := apitest.InitMemoryDB(t) + + mockClock := clock.NewMock() + mockClock.SetNow(serverTime) + + a := app.NewTest() + a.Clock = mockClock + a.EmailTemplates = mailer.Templates{} + a.EmailBackend = &apitest.MockEmailbackendImplementation{} + a.DB = db + + server, err := controllers.NewServer(&a) + if err != nil { + return nil, nil, errors.Wrap(err, "initializing server") + } + + return server, db, nil +} + +// setupNewServer creates a new server and returns the server and database. +// This is useful when a test needs to switch to a new empty server. +func setupNewServer(t *testing.T) (*httptest.Server, *gorm.DB) { + server, serverDB, err := setupTestServer(t, serverTime) + if err != nil { + t.Fatal(errors.Wrap(err, "setting up new test server")) + } + t.Cleanup(func() { server.Close() }) + + return server, serverDB +} + +// updateConfigAPIEndpoint updates the config file with the given API endpoint +func updateConfigAPIEndpoint(t *testing.T, tmpDir string, apiEndpoint string) { + dnoteDir := filepath.Join(tmpDir, consts.DnoteDirName) + configPath := filepath.Join(dnoteDir, consts.ConfigFilename) + configContent := fmt.Sprintf("apiEndpoint: %s\n", apiEndpoint) + if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil { + t.Fatal(errors.Wrap(err, "writing config file")) + } +} + +// switchToEmptyServer closes the current server and creates a new empty server, +// updating the config file to point to it. +func switchToEmptyServer(t *testing.T, env *testEnv) { + // Close old server + env.Server.Close() + + // Create new empty server + env.Server, env.ServerDB = setupNewServer(t) + + // Update config file to point to new server + apiEndpoint := fmt.Sprintf("%s/api", env.Server.URL) + updateConfigAPIEndpoint(t, env.TmpDir, apiEndpoint) +} + +// setupUser creates a test user in the server database +func setupUser(t *testing.T, env testEnv) database.User { + user := apitest.SetupUserData(env.ServerDB, "alice@example.com", "pass1234") + + return user +} + +// setupUserAndLogin creates a test user and logs them in on the CLI +func setupUserAndLogin(t *testing.T, env testEnv) database.User { + user := setupUser(t, env) + login(t, env.DB, env.ServerDB, user) + + return user +} + +// login logs in the user in CLI +func login(t *testing.T, db *cliDatabase.DB, serverDB *gorm.DB, user database.User) { + session := apitest.SetupSession(serverDB, user) + + cliDatabase.MustExec(t, "inserting session_key", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKey, session.Key) + cliDatabase.MustExec(t, "inserting session_key_expiry", db, "INSERT INTO system (key, value) VALUES (?, ?)", consts.SystemSessionKeyExpiry, session.ExpiresAt.Unix()) +} + +// apiCreateBook creates a book via the API and returns its UUID +func apiCreateBook(t *testing.T, env testEnv, user database.User, name, message string) string { + res := doHTTPReq(t, env, "POST", "/v3/books", fmt.Sprintf(`{"name": "%s"}`, name), message, user) + + var resp controllers.CreateBookResp + if err := json.NewDecoder(res.Body).Decode(&resp); err != nil { + t.Fatal(errors.Wrap(err, "decoding payload for adding book")) + return "" + } + + return resp.Book.UUID +} + +// apiPatchBook updates a book via the API +func apiPatchBook(t *testing.T, env testEnv, user database.User, uuid, payload, message string) { + doHTTPReq(t, env, "PATCH", fmt.Sprintf("/v3/books/%s", uuid), payload, message, user) +} + +// apiDeleteBook deletes a book via the API +func apiDeleteBook(t *testing.T, env testEnv, user database.User, uuid, message string) { + doHTTPReq(t, env, "DELETE", fmt.Sprintf("/v3/books/%s", uuid), "", message, user) +} + +// apiCreateNote creates a note via the API and returns its UUID +func apiCreateNote(t *testing.T, env testEnv, user database.User, bookUUID, body, message string) string { + res := doHTTPReq(t, env, "POST", "/v3/notes", fmt.Sprintf(`{"book_uuid": "%s", "content": "%s"}`, bookUUID, body), message, user) + + var resp controllers.CreateNoteResp + if err := json.NewDecoder(res.Body).Decode(&resp); err != nil { + t.Fatal(errors.Wrap(err, "decoding payload for adding note")) + return "" + } + + return resp.Result.UUID +} + +// apiPatchNote updates a note via the API +func apiPatchNote(t *testing.T, env testEnv, user database.User, noteUUID, payload, message string) { + doHTTPReq(t, env, "PATCH", fmt.Sprintf("/v3/notes/%s", noteUUID), payload, message, user) +} + +// apiDeleteNote deletes a note via the API +func apiDeleteNote(t *testing.T, env testEnv, user database.User, noteUUID, message string) { + doHTTPReq(t, env, "DELETE", fmt.Sprintf("/v3/notes/%s", noteUUID), "", message, user) +} + +// doHTTPReq performs an authenticated HTTP request and checks for errors +func doHTTPReq(t *testing.T, env testEnv, method, path, payload, message string, user database.User) *http.Response { + apiEndpoint := fmt.Sprintf("%s/api", env.Server.URL) + endpoint := fmt.Sprintf("%s%s", apiEndpoint, path) + + req, err := http.NewRequest(method, endpoint, strings.NewReader(payload)) + if err != nil { + panic(errors.Wrap(err, "constructing http request")) + } + + res := apitest.HTTPAuthDo(t, env.ServerDB, req, user) + if res.StatusCode >= 400 { + bs, err := io.ReadAll(res.Body) + if err != nil { + panic(errors.Wrap(err, "parsing response body for error")) + } + + t.Errorf("%s. HTTP status %d. Message: %s", message, res.StatusCode, string(bs)) + } + + return res +} + +// setupFunc is a function that sets up test data and returns IDs for assertions +type setupFunc func(t *testing.T, env testEnv, user database.User) map[string]string + +// assertFunc is a function that asserts the expected state after sync +type assertFunc func(t *testing.T, env testEnv, user database.User, ids map[string]string) + +// testSyncCmd is a test helper that sets up a test environment, runs setup, syncs, and asserts +func testSyncCmd(t *testing.T, fullSync bool, setup setupFunc, assert assertFunc) { + env := setupTestEnv(t) + + user := setupUserAndLogin(t, env) + ids := setup(t, env, user) + + if fullSync { + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync", "-f") + } else { + clitest.RunDnoteCmd(t, env.CmdOpts, cliBinaryName, "sync") + } + + assert(t, env, user, ids) +} + +// systemState represents the expected state of the sync system +type systemState struct { + clientNoteCount int + clientBookCount int + clientLastMaxUSN int + clientLastSyncAt int64 + serverNoteCount int64 + serverBookCount int64 + serverUserMaxUSN int +} + +// checkState compares the state of the client and the server with the given system state +func checkState(t *testing.T, clientDB *cliDatabase.DB, user database.User, serverDB *gorm.DB, expected systemState) { + var clientBookCount, clientNoteCount int + cliDatabase.MustScan(t, "counting client notes", clientDB.QueryRow("SELECT count(*) FROM notes"), &clientNoteCount) + cliDatabase.MustScan(t, "counting client books", clientDB.QueryRow("SELECT count(*) FROM books"), &clientBookCount) + assert.Equal(t, clientNoteCount, expected.clientNoteCount, "client note count mismatch") + assert.Equal(t, clientBookCount, expected.clientBookCount, "client book count mismatch") + + var clientLastMaxUSN int + var clientLastSyncAt int64 + cliDatabase.MustScan(t, "finding system last_max_usn", clientDB.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastMaxUSN), &clientLastMaxUSN) + cliDatabase.MustScan(t, "finding system last_sync_at", clientDB.QueryRow("SELECT value FROM system WHERE key = ?", consts.SystemLastSyncAt), &clientLastSyncAt) + assert.Equal(t, clientLastMaxUSN, expected.clientLastMaxUSN, "client last_max_usn mismatch") + assert.Equal(t, clientLastSyncAt, expected.clientLastSyncAt, "client last_sync_at mismatch") + + var serverBookCount, serverNoteCount int64 + apitest.MustExec(t, serverDB.Model(&database.Note{}).Count(&serverNoteCount), "counting server notes") + apitest.MustExec(t, serverDB.Model(&database.Book{}).Count(&serverBookCount), "counting api notes") + assert.Equal(t, serverNoteCount, expected.serverNoteCount, "server note count mismatch") + assert.Equal(t, serverBookCount, expected.serverBookCount, "server book count mismatch") + var serverUser database.User + apitest.MustExec(t, serverDB.Where("id = ?", user.ID).First(&serverUser), "finding user") + assert.Equal(t, serverUser.MaxUSN, expected.serverUserMaxUSN, "user max_usn mismatch") +}