deblan/database-anonymizer
deblan/database-anonymizer
1
0
Fork 1
Code Issues Pull requests Packages Projects Releases 2 Wiki Activity

manager all number types

Browse source 
add 'null' faker

update readme
This commit is contained in:
Simon Vieille 2024-04-02 19:24:50 +02:00
parent b8f3f59211
commit 628308e2fc
Signed by: deblan
GPG key ID: 579388D585F70417
6 changed files with 297 additions and 257 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

257
README.fr.md
View file

@ -54,128 +54,135 @@ database-anonymizer --dsn "postgres://postgres:postgres@localhost:5432/test" --s
### Liste des générateurs
- address
- address_buildingnumber
- address_city
- address_cityprefix
- address_citysuffix
- address_country
- address_countryabbr
- address_countrycode
- address_latitude
- address_longitude
- address_postcode
- address_secondaryaddress
- address_state
- address_stateabbr
- address_streetaddress
- address_streetname
- address_streetsuffix
- app_name
- app_version
- beer_alcohol
- beer_blg
- beer_hop
- beer_ibu
- beer_malt
- beer_name
- beer_style
- blood_name
- boolean_bool
- car_category
- car_fueltype
- car_maker
- car_model
- car_plate
- car_transmissiongear
- color_css
- color_colorname
- color_hex
- color_rgb
- color_safecolorname
- company_bs
- company_catchphrase
- company_ein
- company_jobtitle
- company_name
- company_suffix
- crypto_bech32address
- crypto_bitcoinaddress
- crypto_etheriumaddress
- crypto_p2pkhaddress
- crypto_p2shaddress
- currency_code
- currency_country
- currency_currency
- currency_number
- emoji_emoji
- emoji_emojicode
- file_extension
- file_filenamewithextension
- food_fruit
- food_vegetable
- gamer_tag
- gender_abbr
- gender_name
- genre_name
- internet_companyemail
- internet_domain
- internet_email
- internet_freeemail
- internet_freeemaildomain
- internet_httpmethod
- internet_ipv4
- internet_ipv6
- internet_localipv4
- internet_macaddress
- internet_password
- internet_query
- internet_safeemail
- internet_slug
- internet_statuscode
- internet_statuscodemessage
- internet_statuscodewithmessage
- internet_tld
- internet_url
- internet_user
- language_language
- language_languageabbr
- language_programminglanguage
- mimetype_mimetype
- music_genre
- music_name
- payment_creditcardexpirationdatestring
- payment_creditcardnumber
- payment_creditcardtype
- person_firstname
- person_firstnamefemale
- person_firstnamemale
- person_gender
- person_lastname
- person_name
- person_namefemale
- person_namemale
- person_ssn
- person_suffix
- person_title
- pet_cat
- pet_dog
- pet_name
- phone_areacode
- phone_e164number
- phone_exchangecode
- phone_number
- phone_tollfreeareacode
- phone_toolfreenumber
- time_ampm
- time_century
- time_dayofmonth
- time_monthname
- time_timezone
- time_year
- useragent_chrome
- useragent_firefox
- useragent_internetexplorer
- useragent_opera
- useragent_safari
- useragent_useragent
#### Générateurs spéciaux
- `"null"` : remplace la valeur par `null`
- `""` ou `"_"`: n'applique aucun changement
#### Les autres
- `"address"`
- `"address_buildingnumber"`
- `"address_city"`
- `"address_cityprefix"`
- `"address_citysuffix"`
- `"address_country"`
- `"address_countryabbr"`
- `"address_countrycode"`
- `"address_latitude"`
- `"address_longitude"`
- `"address_postcode"`
- `"address_secondaryaddress"`
- `"address_state"`
- `"address_stateabbr"`
- `"address_streetaddress"`
- `"address_streetname"`
- `"address_streetsuffix"`
- `"app_name"`
- `"app_version"`
- `"beer_alcohol"`
- `"beer_blg"`
- `"beer_hop"`
- `"beer_ibu"`
- `"beer_malt"`
- `"beer_name"`
- `"beer_style"`
- `"blood_name"`
- `"boolean_bool"`
- `"car_category"`
- `"car_fueltype"`
- `"car_maker"`
- `"car_model"`
- `"car_plate"`
- `"car_transmissiongear"`
- `"color_css"`
- `"color_colorname"`
- `"color_hex"`
- `"color_rgb"`
- `"color_safecolorname"`
- `"company_bs"`
- `"company_catchphrase"`
- `"company_ein"`
- `"company_jobtitle"`
- `"company_name"`
- `"company_suffix"`
- `"crypto_bech32address"`
- `"crypto_bitcoinaddress"`
- `"crypto_etheriumaddress"`
- `"crypto_p2pkhaddress"`
- `"crypto_p2shaddress"`
- `"currency_code"`
- `"currency_country"`
- `"currency_currency"`
- `"currency_number"`
- `"emoji_emoji"`
- `"emoji_emojicode"`
- `"file_extension"`
- `"file_filenamewithextension"`
- `"food_fruit"`
- `"food_vegetable"`
- `"gamer_tag"`
- `"gender_abbr"`
- `"gender_name"`
- `"genre_name"`
- `"internet_companyemail"`
- `"internet_domain"`
- `"internet_email"`
- `"internet_freeemail"`
- `"internet_freeemaildomain"`
- `"internet_httpmethod"`
- `"internet_ipv4"`
- `"internet_ipv6"`
- `"internet_localipv4"`
- `"internet_macaddress"`
- `"internet_password"`
- `"internet_query"`
- `"internet_safeemail"`
- `"internet_slug"`
- `"internet_statuscode"`
- `"internet_statuscodemessage"`
- `"internet_statuscodewithmessage"`
- `"internet_tld"`
- `"internet_url"`
- `"internet_user"`
- `"language_language"`
- `"language_languageabbr"`
- `"language_programminglanguage"`
- `"mimetype_mimetype"`
- `"music_genre"`
- `"music_name"`
- `"payment_creditcardexpirationdatestring"`
- `"payment_creditcardnumber"`
- `"payment_creditcardtype"`
- `"person_firstname"`
- `"person_firstnamefemale"`
- `"person_firstnamemale"`
- `"person_gender"`
- `"person_lastname"`
- `"person_name"`
- `"person_namefemale"`
- `"person_namemale"`
- `"person_ssn"`
- `"person_suffix"`
- `"person_title"`
- `"pet_cat"`
- `"pet_dog"`
- `"pet_name"`
- `"phone_areacode"`
- `"phone_e164number"`
- `"phone_exchangecode"`
- `"phone_number"`
- `"phone_tollfreeareacode"`
- `"phone_toolfreenumber"`
- `"time_ampm"`
- `"time_century"`
- `"time_dayofmonth"`
- `"time_monthname"`
- `"time_timezone"`
- `"time_year"`
- `"useragent_chrome"`
- `"useragent_firefox"`
- `"useragent_internetexplorer"`
- `"useragent_opera"`
- `"useragent_safari"`
- `"useragent_useragent"`

257
README.md
View file

@ -54,128 +54,135 @@ database-anonymizer --dsn "postgres://postgres:postgres@localhost:5432/test" --s
### List of fakers
- address
- address_buildingnumber
- address_city
- address_cityprefix
- address_citysuffix
- address_country
- address_countryabbr
- address_countrycode
- address_latitude
- address_longitude
- address_postcode
- address_secondaryaddress
- address_state
- address_stateabbr
- address_streetaddress
- address_streetname
- address_streetsuffix
- app_name
- app_version
- beer_alcohol
- beer_blg
- beer_hop
- beer_ibu
- beer_malt
- beer_name
- beer_style
- blood_name
- boolean_bool
- car_category
- car_fueltype
- car_maker
- car_model
- car_plate
- car_transmissiongear
- color_css
- color_colorname
- color_hex
- color_rgb
- color_safecolorname
- company_bs
- company_catchphrase
- company_ein
- company_jobtitle
- company_name
- company_suffix
- crypto_bech32address
- crypto_bitcoinaddress
- crypto_etheriumaddress
- crypto_p2pkhaddress
- crypto_p2shaddress
- currency_code
- currency_country
- currency_currency
- currency_number
- emoji_emoji
- emoji_emojicode
- file_extension
- file_filenamewithextension
- food_fruit
- food_vegetable
- gamer_tag
- gender_abbr
- gender_name
- genre_name
- internet_companyemail
- internet_domain
- internet_email
- internet_freeemail
- internet_freeemaildomain
- internet_httpmethod
- internet_ipv4
- internet_ipv6
- internet_localipv4
- internet_macaddress
- internet_password
- internet_query
- internet_safeemail
- internet_slug
- internet_statuscode
- internet_statuscodemessage
- internet_statuscodewithmessage
- internet_tld
- internet_url
- internet_user
- language_language
- language_languageabbr
- language_programminglanguage
- mimetype_mimetype
- music_genre
- music_name
- payment_creditcardexpirationdatestring
- payment_creditcardnumber
- payment_creditcardtype
- person_firstname
- person_firstnamefemale
- person_firstnamemale
- person_gender
- person_lastname
- person_name
- person_namefemale
- person_namemale
- person_ssn
- person_suffix
- person_title
- pet_cat
- pet_dog
- pet_name
- phone_areacode
- phone_e164number
- phone_exchangecode
- phone_number
- phone_tollfreeareacode
- phone_toolfreenumber
- time_ampm
- time_century
- time_dayofmonth
- time_monthname
- time_timezone
- time_year
- useragent_chrome
- useragent_firefox
- useragent_internetexplorer
- useragent_opera
- useragent_safari
- useragent_useragent
#### Special fakers
- `"null"` : set `null`
- `""` or `"_"`: do nothing
#### Others
- `"address"`
- `"address_buildingnumber"`
- `"address_city"`
- `"address_cityprefix"`
- `"address_citysuffix"`
- `"address_country"`
- `"address_countryabbr"`
- `"address_countrycode"`
- `"address_latitude"`
- `"address_longitude"`
- `"address_postcode"`
- `"address_secondaryaddress"`
- `"address_state"`
- `"address_stateabbr"`
- `"address_streetaddress"`
- `"address_streetname"`
- `"address_streetsuffix"`
- `"app_name"`
- `"app_version"`
- `"beer_alcohol"`
- `"beer_blg"`
- `"beer_hop"`
- `"beer_ibu"`
- `"beer_malt"`
- `"beer_name"`
- `"beer_style"`
- `"blood_name"`
- `"boolean_bool"`
- `"car_category"`
- `"car_fueltype"`
- `"car_maker"`
- `"car_model"`
- `"car_plate"`
- `"car_transmissiongear"`
- `"color_css"`
- `"color_colorname"`
- `"color_hex"`
- `"color_rgb"`
- `"color_safecolorname"`
- `"company_bs"`
- `"company_catchphrase"`
- `"company_ein"`
- `"company_jobtitle"`
- `"company_name"`
- `"company_suffix"`
- `"crypto_bech32address"`
- `"crypto_bitcoinaddress"`
- `"crypto_etheriumaddress"`
- `"crypto_p2pkhaddress"`
- `"crypto_p2shaddress"`
- `"currency_code"`
- `"currency_country"`
- `"currency_currency"`
- `"currency_number"`
- `"emoji_emoji"`
- `"emoji_emojicode"`
- `"file_extension"`
- `"file_filenamewithextension"`
- `"food_fruit"`
- `"food_vegetable"`
- `"gamer_tag"`
- `"gender_abbr"`
- `"gender_name"`
- `"genre_name"`
- `"internet_companyemail"`
- `"internet_domain"`
- `"internet_email"`
- `"internet_freeemail"`
- `"internet_freeemaildomain"`
- `"internet_httpmethod"`
- `"internet_ipv4"`
- `"internet_ipv6"`
- `"internet_localipv4"`
- `"internet_macaddress"`
- `"internet_password"`
- `"internet_query"`
- `"internet_safeemail"`
- `"internet_slug"`
- `"internet_statuscode"`
- `"internet_statuscodemessage"`
- `"internet_statuscodewithmessage"`
- `"internet_tld"`
- `"internet_url"`
- `"internet_user"`
- `"language_language"`
- `"language_languageabbr"`
- `"language_programminglanguage"`
- `"mimetype_mimetype"`
- `"music_genre"`
- `"music_name"`
- `"payment_creditcardexpirationdatestring"`
- `"payment_creditcardnumber"`
- `"payment_creditcardtype"`
- `"person_firstname"`
- `"person_firstnamefemale"`
- `"person_firstnamemale"`
- `"person_gender"`
- `"person_lastname"`
- `"person_name"`
- `"person_namefemale"`
- `"person_namemale"`
- `"person_ssn"`
- `"person_suffix"`
- `"person_title"`
- `"pet_cat"`
- `"pet_dog"`
- `"pet_name"`
- `"phone_areacode"`
- `"phone_e164number"`
- `"phone_exchangecode"`
- `"phone_number"`
- `"phone_tollfreeareacode"`
- `"phone_toolfreenumber"`
- `"time_ampm"`
- `"time_century"`
- `"time_dayofmonth"`
- `"time_monthname"`
- `"time_timezone"`
- `"time_year"`
- `"useragent_chrome"`
- `"useragent_firefox"`
- `"useragent_internetexplorer"`
- `"useragent_opera"`
- `"useragent_safari"`
- `"useragent_useragent"`

6
app/app.go
View file

@ -65,7 +65,7 @@ func (a *App) TruncateTable(c config.SchemaConfigAction) error {
values := make(map[int]string)
for _, col := range c.PrimaryKey {
if !row[col].IsString {
if !row[col].IsString || row[col].IsNull {
value := row[col]
pkeys = append(pkeys, fmt.Sprintf("%s=%s", col, value.FinalValue()))
} else {
@ -166,7 +166,7 @@ func (a *App) UpdateRows(c config.SchemaConfigAction, globalColumns map[string]s
for col, value := range row {
if value.IsUpdated && !value.IsVirtual {
if value.IsString {
if value.IsString && !value.IsNull {
updates = append(updates, database.GetNamedParameter(a.DbConfig.Type, col, len(values)+1))
values[len(values)+1] = value.FinalValue()
} else {
@ -178,7 +178,7 @@ func (a *App) UpdateRows(c config.SchemaConfigAction, globalColumns map[string]s
for _, col := range c.PrimaryKey {
value := row[col]
if !value.IsString {
if !value.IsString || value.IsNull {
pkeys = append(pkeys, fmt.Sprintf("%s=%s", col, value.FinalValue()))
} else {
pkeys = append(pkeys, database.GetNamedParameter(a.DbConfig.Type, col, len(values)+1))

11
data/data.go
View file

@ -17,7 +17,7 @@ type Data struct {
IsPrimaryKey bool
IsUpdated bool
IsInteger bool
IsNumber bool
IsBoolean bool
IsString bool
IsNull bool
@ -70,6 +70,14 @@ func (d *Data) Update(row map[string]Data, manager faker.FakeManager) {
return
}
if d.Faker == "null" {
d.Value = ""
d.IsUpdated = true
d.IsNull = true
return
}
if d.IsTwigExpression() {
env := twig.New(nil)
params := map[string]stick.Value{}
@ -85,6 +93,7 @@ func (d *Data) Update(row map[string]Data, manager faker.FakeManager) {
d.Value = buf.String()
d.IsUpdated = true
d.IsNull = d.Value == ""
return
}

21
database/database.go
View file

@ -23,6 +23,23 @@ func GetNamedParameter(dbType, col string, number int) string {
return fmt.Sprintf("%s=$%d", col, number)
}
func IsPgNumberType(value string) bool {
switch value {
case
"smallint",
"integer",
"bigint",
"decimal",
"numeric",
"real",
"double precision":
return true
}
return false
}
func GetRows(db *sql.DB, query, table, dbType string) map[int]map[string]data.Data {
rows, err := db.Query(query)
defer rows.Close()
@ -69,9 +86,9 @@ func GetRows(db *sql.DB, query, table, dbType string) map[int]map[string]data.Da
dataType := columnsTypes[col]
d.IsInteger = dataType == "integer"
d.IsNumber = IsPgNumberType(dataType)
d.IsBoolean = dataType == "boolean"
d.IsString = !d.IsBoolean && !d.IsInteger
d.IsString = !d.IsBoolean && !d.IsNumber
} else {
d.IsString = true
}

2
faker/faker.go
View file

@ -154,7 +154,7 @@ func NewFakeManager() FakeManager {
}
func (f *FakeManager) IsValidFaker(name string) bool {
if name == "" || name == "_" {
if name == "" || name == "_" || name == "null" {
return true
}