From 3992576fa19eceb2ad130c9d74d1fd873ce9a839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Helleu?= Date: Sat, 6 Apr 2024 17:18:25 +0200 Subject: [PATCH 1/7] fix typos in readme --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b3b4fea..8229094 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# Database Anonimizer +# Database Anonymizer **Database Anonymizer** is a tool written in GO that allows **anonymizing or deleting data from a MySQL or PostgreSQL database**. -It addresses various use cases such as **providing developers with an anonymized copy of a database** or **satisfying the need to anonymize or delete data in accordance with GDPR (General Data Protection Regulation) requirements. data protection)**, depending on the retention periods. defined in the processing register. +It addresses various use cases such as **providing developers with an anonymized copy of a database** or **satisfying the need to anonymize or delete data in accordance with GDPR** (General Data Protection Regulation), depending on the retention periods defined in the processing register. The project includes a vast array of fakers. It also enables data generation via Twig-written templates. You can specify precise rules for each table or global rules applied to all tables in your configuration. @@ -37,7 +37,7 @@ rules: delete: true ``` -### Exécution +### Run To display help, use `-h`: @@ -56,7 +56,7 @@ database-anonymizer --dsn "postgres://postgres:postgres@localhost:5432/test" --s #### Special fakers -- `"null"` : set `null` +- `"null"`: set `null` - `""` or `"_"`: do nothing #### Others From 75efdde05cad5b07ca0a7b78bec51b5f67420a82 Mon Sep 17 00:00:00 2001 From: Simon Vieille Date: Thu, 18 Jul 2024 10:02:33 +0200 Subject: [PATCH 2/7] add lorem fakers --- CHANGELOG.md | 9 +++++++++ README.fr.md | 6 ++++++ README.md | 6 ++++++ faker/faker.go | 16 ++++++++++++++++ go.mod | 1 + go.sum | 2 ++ 6 files changed, 40 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb80916..cf96c32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ ## [Unreleased] +## v1.2.0 +### Added +- add "lorem_word" faker +- add "lorem_sentence" faker +- add "lorem_paragraph" faker +- add "lorem_words_2" to "lorem_words_10" fakers +- add "lorem_sentences_2" to "lorem_sentences_10" fakers +- add "lorem_paragraphs_2" to "lorem_paragraphs_10" fakers + ## v1.1.0 ### Added - PostgreSQL: manage all number types diff --git a/README.fr.md b/README.fr.md index ccd6b73..737f7ee 100644 --- a/README.fr.md +++ b/README.fr.md @@ -61,6 +61,12 @@ database-anonymizer --dsn "postgres://postgres:postgres@localhost:5432/test" --s #### Les autres +- `"lorem_word"` +- `"lorem_sentence"` +- `"lorem_paragraph"` +- `"lorem_words_2"` to `"lorem_words_10"` +- `"lorem_sentences_2"` to `"lorem_sentences_10"` +- `"lorem_paragraphs_2"` to `"lorem_paragraphs_10"` - `"address"` - `"address_buildingnumber"` - `"address_city"` diff --git a/README.md b/README.md index 8229094..ab77546 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,12 @@ database-anonymizer --dsn "postgres://postgres:postgres@localhost:5432/test" --s #### Others +- `"lorem_word"` +- `"lorem_sentence"` +- `"lorem_paragraph"` +- `"lorem_words_2"` to `"lorem_words_10"` +- `"lorem_sentences_2"` to `"lorem_sentences_10"` +- `"lorem_paragraphs_2"` to `"lorem_paragraphs_10"` - `"address"` - `"address_buildingnumber"` - `"address_city"` diff --git a/faker/faker.go b/faker/faker.go index 2bd84b0..07d7436 100644 --- a/faker/faker.go +++ b/faker/faker.go @@ -2,7 +2,9 @@ package faker import ( "fmt" + "github.com/go-loremipsum/loremipsum" base_faker "github.com/jaswdr/faker" + "math/rand" "strconv" ) @@ -10,12 +12,26 @@ type FakeManager struct { Fakers map[string]func() string } +func NewLoremFaker() *loremipsum.LoremIpsum { + return loremipsum.NewWithSeed(rand.Int63()) +} + func NewFakeManager() FakeManager { manager := FakeManager{} datas := make(map[string]func() string) fake := base_faker.New() + datas["lorem_word"] = func() string { return NewLoremFaker().Word() } + datas["lorem_sentence"] = func() string { return NewLoremFaker().Sentence() } + datas["lorem_paragraph"] = func() string { return NewLoremFaker().Paragraph() } + + for i := 2; i <= 10; i++ { + datas[fmt.Sprintf("lorem_words_%d", i)] = func() string { return NewLoremFaker().Words(i) } + datas[fmt.Sprintf("lorem_sentences_%d", i)] = func() string { return NewLoremFaker().Sentences(i) } + datas[fmt.Sprintf("lorem_paragraphs_%d", i)] = func() string { return NewLoremFaker().Paragraphs(i) } + } + datas["address"] = func() string { return fake.Address().Address() } datas["address_buildingnumber"] = func() string { return fake.Address().BuildingNumber() } datas["address_city"] = func() string { return fake.Address().City() } diff --git a/go.mod b/go.mod index 08d1b50..42e6567 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect + github.com/go-loremipsum/loremipsum v1.1.3 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index e8c409e..f24f1ef 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-loremipsum/loremipsum v1.1.3 h1:ZRhA0ZmJ49lGe5HhWeMONr+iGftWDsHfrYBl5ktDXso= +github.com/go-loremipsum/loremipsum v1.1.3/go.mod h1:OJQjXdvwlG9hsyhmMQoT4HOm4DG4l62CYywebw0XBoo= github.com/go-sql-driver/mysql v1.8.0 h1:UtktXaU2Nb64z/pLiGIxY4431SJ4/dR5cjMmlVHgnT4= github.com/go-sql-driver/mysql v1.8.0/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/jaswdr/faker v1.19.1 h1:xBoz8/O6r0QAR8eEvKJZMdofxiRH+F0M/7MU9eNKhsM= From 50783c3a7584dad74b681e035bf6fe9d19f0c3f0 Mon Sep 17 00:00:00 2001 From: Simon Vieille Date: Thu, 18 Jul 2024 10:19:57 +0200 Subject: [PATCH 3/7] fix doc: replace delete with truncate --- README.fr.md | 4 ++-- README.md | 4 ++-- example.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.fr.md b/README.fr.md index 737f7ee..dba11e3 100644 --- a/README.fr.md +++ b/README.fr.md @@ -31,10 +31,10 @@ rules: name: company_name - table: access_log query: 'select * from access_log where date < (NOW() - INTERVAL 6 MONTH)' - delete: true + truncate: true - table: user_ip primary_key: [user_id, ip_id] - delete: true + truncate: true ``` ### Exécution diff --git a/README.md b/README.md index ab77546..dd8e5e1 100644 --- a/README.md +++ b/README.md @@ -31,10 +31,10 @@ rules: name: company_name - table: access_log query: 'select * from access_log where date < (NOW() - INTERVAL 6 MONTH)' - delete: true + truncate: true - table: user_ip primary_key: [user_id, ip_id] - delete: true + truncate: true ``` ### Run diff --git a/example.yml b/example.yml index da3b9e5..3a52a1a 100644 --- a/example.yml +++ b/example.yml @@ -16,7 +16,7 @@ rules: name: company_name - table: access_log query: 'select * from access_log where date < (NOW() - INTERVAL 6 MONTH)' - delete: true + truncate: true - table: user_ip primary_key: [user_id, ip_id] - delete: true + truncate: true From 6c9e1cfd06b8704afc844c67ee4a430991066d54 Mon Sep 17 00:00:00 2001 From: Simon Vieille Date: Thu, 18 Jul 2024 15:26:45 +0200 Subject: [PATCH 4/7] add default type of string --- CHANGELOG.md | 4 ++++ database/database.go | 1 + 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf96c32..ba46f62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## [Unreleased] +## v1.2.1 +### Fixed +- add default type of string + ## v1.2.0 ### Added - add "lorem_word" faker diff --git a/database/database.go b/database/database.go index 9998c6a..6a1c699 100644 --- a/database/database.go +++ b/database/database.go @@ -73,6 +73,7 @@ func GetRows(db *sql.DB, query, table, dbType string) map[int]map[string]data.Da value := values[i] d := data.Data{ IsVirtual: false, + IsString: true, IsNull: value == nil, } From 6754905f2451f30517a661ae423435be04ded593 Mon Sep 17 00:00:00 2001 From: Simon Vieille Date: Thu, 18 Jul 2024 17:03:32 +0200 Subject: [PATCH 5/7] add column escaper function --- app/app.go | 4 ++-- database/database.go | 10 +++++++--- database/database_test.go | 10 ++++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/app/app.go b/app/app.go index f167eeb..acf4e44 100644 --- a/app/app.go +++ b/app/app.go @@ -170,7 +170,7 @@ func (a *App) UpdateRows(c config.SchemaConfigAction, globalColumns map[string]s updates = append(updates, database.GetNamedParameter(a.DbConfig.Type, col, len(values)+1)) values[len(values)+1] = value.FinalValue() } else { - updates = append(updates, fmt.Sprintf("%s=%s", col, value.FinalValue())) + updates = append(updates, fmt.Sprintf("%s=%s", database.EscapeColumn(a.DbConfig.Type, col), value.FinalValue())) } } } @@ -179,7 +179,7 @@ func (a *App) UpdateRows(c config.SchemaConfigAction, globalColumns map[string]s value := row[col] if !value.IsString || value.IsNull { - pkeys = append(pkeys, fmt.Sprintf("%s=%s", col, value.FinalValue())) + pkeys = append(pkeys, fmt.Sprintf("%s=%s", database.EscapeColumn(a.DbConfig.Type, col), value.FinalValue())) } else { pkeys = append(pkeys, database.GetNamedParameter(a.DbConfig.Type, col, len(values)+1)) values[len(values)+1] = value.FinalValue() diff --git a/database/database.go b/database/database.go index 6a1c699..94b052b 100644 --- a/database/database.go +++ b/database/database.go @@ -15,12 +15,16 @@ func EscapeTable(dbType, table string) string { return fmt.Sprintf("\"%s\"", table) } +func EscapeColumn(dbType, col string) string { + return EscapeTable(dbType, col) +} + func GetNamedParameter(dbType, col string, number int) string { if dbType == "mysql" { - return fmt.Sprintf("%s=?", col) + return fmt.Sprintf("%s=?", EscapeColumn(col)) } - return fmt.Sprintf("%s=$%d", col, number) + return fmt.Sprintf("%s=$%d", EscapeColumn(col), number) } func IsPgNumberType(value string) bool { @@ -80,7 +84,7 @@ func GetRows(db *sql.DB, query, table, dbType string) map[int]map[string]data.Da if value != nil { if dbType == "postgres" { if len(columnsTypes[col]) == 0 { - typeQuery := fmt.Sprintf("SELECT pg_typeof(%s) as value FROM %s", col, EscapeTable(dbType, table)) + typeQuery := fmt.Sprintf("SELECT pg_typeof(%s) as value FROM %s", EscapeColumn(dbType, col), EscapeTable(dbType, table)) db.QueryRow(typeQuery).Scan(&typeValue) columnsTypes[col] = typeValue } diff --git a/database/database_test.go b/database/database_test.go index 75cc8e8..343d675 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -14,6 +14,16 @@ func TestEscapeTable(t *testing.T) { } } +func TestEscapeColumn(t *testing.T) { + if EscapeColumn("mysql", "foo") != "`foo`" { + t.Fatalf("TestEscapeColumn: mysql check failed") + } + + if EscapeTable("postgres", "foo") != "\"foo\"" { + t.Fatalf("TestEscapeColumn: postgres check failed") + } +} + func TestGetNamedParameter(t *testing.T) { if GetNamedParameter("mysql", "foo", 1) != "foo=?" { t.Fatalf("TestGetNamedParameter: mysql check failed") From 5eb3701f9ff05f0d103e5e2b72a21b3d6c2d1311 Mon Sep 17 00:00:00 2001 From: Simon Vieille Date: Thu, 18 Jul 2024 17:05:30 +0200 Subject: [PATCH 6/7] add column escaper function --- database/database.go | 4 ++-- database/database_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/database/database.go b/database/database.go index 94b052b..b819e4a 100644 --- a/database/database.go +++ b/database/database.go @@ -21,10 +21,10 @@ func EscapeColumn(dbType, col string) string { func GetNamedParameter(dbType, col string, number int) string { if dbType == "mysql" { - return fmt.Sprintf("%s=?", EscapeColumn(col)) + return fmt.Sprintf("%s=?", EscapeColumn(dbType, col)) } - return fmt.Sprintf("%s=$%d", EscapeColumn(col), number) + return fmt.Sprintf("%s=$%d", EscapeColumn(dbType, col), number) } func IsPgNumberType(value string) bool { diff --git a/database/database_test.go b/database/database_test.go index 343d675..81b9f67 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -19,7 +19,7 @@ func TestEscapeColumn(t *testing.T) { t.Fatalf("TestEscapeColumn: mysql check failed") } - if EscapeTable("postgres", "foo") != "\"foo\"" { + if EscapeColumn("postgres", "foo") != "\"foo\"" { t.Fatalf("TestEscapeColumn: postgres check failed") } } From 23f432ea0222c1d7275b8bcf7b34e73833c4abcf Mon Sep 17 00:00:00 2001 From: Simon Vieille Date: Thu, 18 Jul 2024 17:07:33 +0200 Subject: [PATCH 7/7] fix tests --- database/database_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/database_test.go b/database/database_test.go index 81b9f67..b51c1a4 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -25,11 +25,11 @@ func TestEscapeColumn(t *testing.T) { } func TestGetNamedParameter(t *testing.T) { - if GetNamedParameter("mysql", "foo", 1) != "foo=?" { + if GetNamedParameter("mysql", "foo", 1) != "`foo`=?" { t.Fatalf("TestGetNamedParameter: mysql check failed") } - if GetNamedParameter("postgres", "foo", 1) != "foo=$1" { + if GetNamedParameter("postgres", "foo", 1) != "\"foo\"=$1" { t.Fatalf("TestGetNamedParameter: postgres check failed") } }