flashcode/database-anonymizer
1
0
Fork 0
forked from deblan/database-anonymizer
Code Pull requests Activity

manager all number types

Browse source 
add 'null' faker

update readme
This commit is contained in:
Simon Vieille 2024-04-02 19:24:50 +02:00
parent b8f3f59211
commit 628308e2fc
Signed by untrusted user: deblan
GPG key ID: 579388D585F70417
6 changed files with 297 additions and 257 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

257
README.fr.md
View file

@ -54,128 +54,135 @@ database-anonymizer --dsn "postgres://postgres:postgres@localhost:5432/test" --s
### Liste des générateurs ### Liste des générateurs
- address #### Générateurs spéciaux
- address_buildingnumber
- address_city - `"null"` : remplace la valeur par `null`
- address_cityprefix - `""` ou `"_"`: n'applique aucun changement
- address_citysuffix
- address_country #### Les autres
- address_countryabbr
- address_countrycode - `"address"`
- address_latitude - `"address_buildingnumber"`
- address_longitude - `"address_city"`
- address_postcode - `"address_cityprefix"`
- address_secondaryaddress - `"address_citysuffix"`
- address_state - `"address_country"`
- address_stateabbr - `"address_countryabbr"`
- address_streetaddress - `"address_countrycode"`
- address_streetname - `"address_latitude"`
- address_streetsuffix - `"address_longitude"`
- app_name - `"address_postcode"`
- app_version - `"address_secondaryaddress"`
- beer_alcohol - `"address_state"`
- beer_blg - `"address_stateabbr"`
- beer_hop - `"address_streetaddress"`
- beer_ibu - `"address_streetname"`
- beer_malt - `"address_streetsuffix"`
- beer_name - `"app_name"`
- beer_style - `"app_version"`
- blood_name - `"beer_alcohol"`
- boolean_bool - `"beer_blg"`
- car_category - `"beer_hop"`
- car_fueltype - `"beer_ibu"`
- car_maker - `"beer_malt"`
- car_model - `"beer_name"`
- car_plate - `"beer_style"`
- car_transmissiongear - `"blood_name"`
- color_css - `"boolean_bool"`
- color_colorname - `"car_category"`
- color_hex - `"car_fueltype"`
- color_rgb - `"car_maker"`
- color_safecolorname - `"car_model"`
- company_bs - `"car_plate"`
- company_catchphrase - `"car_transmissiongear"`
- company_ein - `"color_css"`
- company_jobtitle - `"color_colorname"`
- company_name - `"color_hex"`
- company_suffix - `"color_rgb"`
- crypto_bech32address - `"color_safecolorname"`
- crypto_bitcoinaddress - `"company_bs"`
- crypto_etheriumaddress - `"company_catchphrase"`
- crypto_p2pkhaddress - `"company_ein"`
- crypto_p2shaddress - `"company_jobtitle"`
- currency_code - `"company_name"`
- currency_country - `"company_suffix"`
- currency_currency - `"crypto_bech32address"`
- currency_number - `"crypto_bitcoinaddress"`
- emoji_emoji - `"crypto_etheriumaddress"`
- emoji_emojicode - `"crypto_p2pkhaddress"`
- file_extension - `"crypto_p2shaddress"`
- file_filenamewithextension - `"currency_code"`
- food_fruit - `"currency_country"`
- food_vegetable - `"currency_currency"`
- gamer_tag - `"currency_number"`
- gender_abbr - `"emoji_emoji"`
- gender_name - `"emoji_emojicode"`
- genre_name - `"file_extension"`
- internet_companyemail - `"file_filenamewithextension"`
- internet_domain - `"food_fruit"`
- internet_email - `"food_vegetable"`
- internet_freeemail - `"gamer_tag"`
- internet_freeemaildomain - `"gender_abbr"`
- internet_httpmethod - `"gender_name"`
- internet_ipv4 - `"genre_name"`
- internet_ipv6 - `"internet_companyemail"`
- internet_localipv4 - `"internet_domain"`
- internet_macaddress - `"internet_email"`
- internet_password - `"internet_freeemail"`
- internet_query - `"internet_freeemaildomain"`
- internet_safeemail - `"internet_httpmethod"`
- internet_slug - `"internet_ipv4"`
- internet_statuscode - `"internet_ipv6"`
- internet_statuscodemessage - `"internet_localipv4"`
- internet_statuscodewithmessage - `"internet_macaddress"`
- internet_tld - `"internet_password"`
- internet_url - `"internet_query"`
- internet_user - `"internet_safeemail"`
- language_language - `"internet_slug"`
- language_languageabbr - `"internet_statuscode"`
- language_programminglanguage - `"internet_statuscodemessage"`
- mimetype_mimetype - `"internet_statuscodewithmessage"`
- music_genre - `"internet_tld"`
- music_name - `"internet_url"`
- payment_creditcardexpirationdatestring - `"internet_user"`
- payment_creditcardnumber - `"language_language"`
- payment_creditcardtype - `"language_languageabbr"`
- person_firstname - `"language_programminglanguage"`
- person_firstnamefemale - `"mimetype_mimetype"`
- person_firstnamemale - `"music_genre"`
- person_gender - `"music_name"`
- person_lastname - `"payment_creditcardexpirationdatestring"`
- person_name - `"payment_creditcardnumber"`
- person_namefemale - `"payment_creditcardtype"`
- person_namemale - `"person_firstname"`
- person_ssn - `"person_firstnamefemale"`
- person_suffix - `"person_firstnamemale"`
- person_title - `"person_gender"`
- pet_cat - `"person_lastname"`
- pet_dog - `"person_name"`
- pet_name - `"person_namefemale"`
- phone_areacode - `"person_namemale"`
- phone_e164number - `"person_ssn"`
- phone_exchangecode - `"person_suffix"`
- phone_number - `"person_title"`
- phone_tollfreeareacode - `"pet_cat"`
- phone_toolfreenumber - `"pet_dog"`
- time_ampm - `"pet_name"`
- time_century - `"phone_areacode"`
- time_dayofmonth - `"phone_e164number"`
- time_monthname - `"phone_exchangecode"`
- time_timezone - `"phone_number"`
- time_year - `"phone_tollfreeareacode"`
- useragent_chrome - `"phone_toolfreenumber"`
- useragent_firefox - `"time_ampm"`
- useragent_internetexplorer - `"time_century"`
- useragent_opera - `"time_dayofmonth"`
- useragent_safari - `"time_monthname"`
- useragent_useragent - `"time_timezone"`
- `"time_year"`
- `"useragent_chrome"`
- `"useragent_firefox"`
- `"useragent_internetexplorer"`
- `"useragent_opera"`
- `"useragent_safari"`
- `"useragent_useragent"`

257
README.md
View file

@ -54,128 +54,135 @@ database-anonymizer --dsn "postgres://postgres:postgres@localhost:5432/test" --s
### List of fakers ### List of fakers
- address #### Special fakers
- address_buildingnumber
- address_city - `"null"` : set `null`
- address_cityprefix - `""` or `"_"`: do nothing
- address_citysuffix
- address_country #### Others
- address_countryabbr
- address_countrycode - `"address"`
- address_latitude - `"address_buildingnumber"`
- address_longitude - `"address_city"`
- address_postcode - `"address_cityprefix"`
- address_secondaryaddress - `"address_citysuffix"`
- address_state - `"address_country"`
- address_stateabbr - `"address_countryabbr"`
- address_streetaddress - `"address_countrycode"`
- address_streetname - `"address_latitude"`
- address_streetsuffix - `"address_longitude"`
- app_name - `"address_postcode"`
- app_version - `"address_secondaryaddress"`
- beer_alcohol - `"address_state"`
- beer_blg - `"address_stateabbr"`
- beer_hop - `"address_streetaddress"`
- beer_ibu - `"address_streetname"`
- beer_malt - `"address_streetsuffix"`
- beer_name - `"app_name"`
- beer_style - `"app_version"`
- blood_name - `"beer_alcohol"`
- boolean_bool - `"beer_blg"`
- car_category - `"beer_hop"`
- car_fueltype - `"beer_ibu"`
- car_maker - `"beer_malt"`
- car_model - `"beer_name"`
- car_plate - `"beer_style"`
- car_transmissiongear - `"blood_name"`
- color_css - `"boolean_bool"`
- color_colorname - `"car_category"`
- color_hex - `"car_fueltype"`
- color_rgb - `"car_maker"`
- color_safecolorname - `"car_model"`
- company_bs - `"car_plate"`
- company_catchphrase - `"car_transmissiongear"`
- company_ein - `"color_css"`
- company_jobtitle - `"color_colorname"`
- company_name - `"color_hex"`
- company_suffix - `"color_rgb"`
- crypto_bech32address - `"color_safecolorname"`
- crypto_bitcoinaddress - `"company_bs"`
- crypto_etheriumaddress - `"company_catchphrase"`
- crypto_p2pkhaddress - `"company_ein"`
- crypto_p2shaddress - `"company_jobtitle"`
- currency_code - `"company_name"`
- currency_country - `"company_suffix"`
- currency_currency - `"crypto_bech32address"`
- currency_number - `"crypto_bitcoinaddress"`
- emoji_emoji - `"crypto_etheriumaddress"`
- emoji_emojicode - `"crypto_p2pkhaddress"`
- file_extension - `"crypto_p2shaddress"`
- file_filenamewithextension - `"currency_code"`
- food_fruit - `"currency_country"`
- food_vegetable - `"currency_currency"`
- gamer_tag - `"currency_number"`
- gender_abbr - `"emoji_emoji"`
- gender_name - `"emoji_emojicode"`
- genre_name - `"file_extension"`
- internet_companyemail - `"file_filenamewithextension"`
- internet_domain - `"food_fruit"`
- internet_email - `"food_vegetable"`
- internet_freeemail - `"gamer_tag"`
- internet_freeemaildomain - `"gender_abbr"`
- internet_httpmethod - `"gender_name"`
- internet_ipv4 - `"genre_name"`
- internet_ipv6 - `"internet_companyemail"`
- internet_localipv4 - `"internet_domain"`
- internet_macaddress - `"internet_email"`
- internet_password - `"internet_freeemail"`
- internet_query - `"internet_freeemaildomain"`
- internet_safeemail - `"internet_httpmethod"`
- internet_slug - `"internet_ipv4"`
- internet_statuscode - `"internet_ipv6"`
- internet_statuscodemessage - `"internet_localipv4"`
- internet_statuscodewithmessage - `"internet_macaddress"`
- internet_tld - `"internet_password"`
- internet_url - `"internet_query"`
- internet_user - `"internet_safeemail"`
- language_language - `"internet_slug"`
- language_languageabbr - `"internet_statuscode"`
- language_programminglanguage - `"internet_statuscodemessage"`
- mimetype_mimetype - `"internet_statuscodewithmessage"`
- music_genre - `"internet_tld"`
- music_name - `"internet_url"`
- payment_creditcardexpirationdatestring - `"internet_user"`
- payment_creditcardnumber - `"language_language"`
- payment_creditcardtype - `"language_languageabbr"`
- person_firstname - `"language_programminglanguage"`
- person_firstnamefemale - `"mimetype_mimetype"`
- person_firstnamemale - `"music_genre"`
- person_gender - `"music_name"`
- person_lastname - `"payment_creditcardexpirationdatestring"`
- person_name - `"payment_creditcardnumber"`
- person_namefemale - `"payment_creditcardtype"`
- person_namemale - `"person_firstname"`
- person_ssn - `"person_firstnamefemale"`
- person_suffix - `"person_firstnamemale"`
- person_title - `"person_gender"`
- pet_cat - `"person_lastname"`
- pet_dog - `"person_name"`
- pet_name - `"person_namefemale"`
- phone_areacode - `"person_namemale"`
- phone_e164number - `"person_ssn"`
- phone_exchangecode - `"person_suffix"`
- phone_number - `"person_title"`
- phone_tollfreeareacode - `"pet_cat"`
- phone_toolfreenumber - `"pet_dog"`
- time_ampm - `"pet_name"`
- time_century - `"phone_areacode"`
- time_dayofmonth - `"phone_e164number"`
- time_monthname - `"phone_exchangecode"`
- time_timezone - `"phone_number"`
- time_year - `"phone_tollfreeareacode"`
- useragent_chrome - `"phone_toolfreenumber"`
- useragent_firefox - `"time_ampm"`
- useragent_internetexplorer - `"time_century"`
- useragent_opera - `"time_dayofmonth"`
- useragent_safari - `"time_monthname"`
- useragent_useragent - `"time_timezone"`
- `"time_year"`
- `"useragent_chrome"`
- `"useragent_firefox"`
- `"useragent_internetexplorer"`
- `"useragent_opera"`
- `"useragent_safari"`
- `"useragent_useragent"`

6
app/app.go
View file

@ -65,7 +65,7 @@ func (a *App) TruncateTable(c config.SchemaConfigAction) error {
values := make(map[int]string) values := make(map[int]string)
for _, col := range c.PrimaryKey { for _, col := range c.PrimaryKey {
if !row[col].IsString { if !row[col].IsString || row[col].IsNull {
value := row[col] value := row[col]
pkeys = append(pkeys, fmt.Sprintf("%s=%s", col, value.FinalValue())) pkeys = append(pkeys, fmt.Sprintf("%s=%s", col, value.FinalValue()))
} else { } else {
@ -166,7 +166,7 @@ func (a *App) UpdateRows(c config.SchemaConfigAction, globalColumns map[string]s
for col, value := range row { for col, value := range row {
if value.IsUpdated && !value.IsVirtual { if value.IsUpdated && !value.IsVirtual {
if value.IsString { if value.IsString && !value.IsNull {
updates = append(updates, database.GetNamedParameter(a.DbConfig.Type, col, len(values)+1)) updates = append(updates, database.GetNamedParameter(a.DbConfig.Type, col, len(values)+1))
values[len(values)+1] = value.FinalValue() values[len(values)+1] = value.FinalValue()
} else { } else {
@ -178,7 +178,7 @@ func (a *App) UpdateRows(c config.SchemaConfigAction, globalColumns map[string]s
for _, col := range c.PrimaryKey { for _, col := range c.PrimaryKey {
value := row[col] value := row[col]
if !value.IsString { if !value.IsString || value.IsNull {
pkeys = append(pkeys, fmt.Sprintf("%s=%s", col, value.FinalValue())) pkeys = append(pkeys, fmt.Sprintf("%s=%s", col, value.FinalValue()))
} else { } else {
pkeys = append(pkeys, database.GetNamedParameter(a.DbConfig.Type, col, len(values)+1)) pkeys = append(pkeys, database.GetNamedParameter(a.DbConfig.Type, col, len(values)+1))

11
data/data.go
View file

@ -17,7 +17,7 @@ type Data struct {
IsPrimaryKey bool IsPrimaryKey bool
IsUpdated bool IsUpdated bool
IsInteger bool IsNumber bool
IsBoolean bool IsBoolean bool
IsString bool IsString bool
IsNull bool IsNull bool
@ -70,6 +70,14 @@ func (d *Data) Update(row map[string]Data, manager faker.FakeManager) {
return return
} }
if d.Faker == "null" {
d.Value = ""
d.IsUpdated = true
d.IsNull = true
return
}
if d.IsTwigExpression() { if d.IsTwigExpression() {
env := twig.New(nil) env := twig.New(nil)
params := map[string]stick.Value{} params := map[string]stick.Value{}
@ -85,6 +93,7 @@ func (d *Data) Update(row map[string]Data, manager faker.FakeManager) {
d.Value = buf.String() d.Value = buf.String()
d.IsUpdated = true d.IsUpdated = true
d.IsNull = d.Value == ""
return return
} }

21
database/database.go
View file

@ -23,6 +23,23 @@ func GetNamedParameter(dbType, col string, number int) string {
return fmt.Sprintf("%s=$%d", col, number) return fmt.Sprintf("%s=$%d", col, number)
} }
func IsPgNumberType(value string) bool {
switch value {
case
"smallint",
"integer",
"bigint",
"decimal",
"numeric",
"real",
"double precision":
return true
}
return false
}
func GetRows(db *sql.DB, query, table, dbType string) map[int]map[string]data.Data { func GetRows(db *sql.DB, query, table, dbType string) map[int]map[string]data.Data {
rows, err := db.Query(query) rows, err := db.Query(query)
defer rows.Close() defer rows.Close()
@ -69,9 +86,9 @@ func GetRows(db *sql.DB, query, table, dbType string) map[int]map[string]data.Da
dataType := columnsTypes[col] dataType := columnsTypes[col]
d.IsInteger = dataType == "integer" d.IsNumber = IsPgNumberType(dataType)
d.IsBoolean = dataType == "boolean" d.IsBoolean = dataType == "boolean"
d.IsString = !d.IsBoolean && !d.IsInteger d.IsString = !d.IsBoolean && !d.IsNumber
} else { } else {
d.IsString = true d.IsString = true
} }

2
faker/faker.go
View file

@ -154,7 +154,7 @@ func NewFakeManager() FakeManager {
} }
func (f *FakeManager) IsValidFaker(name string) bool { func (f *FakeManager) IsValidFaker(name string) bool {
if name == "" || name == "_" { if name == "" || name == "_" || name == "null" {
return true return true
} }