From a6f0bf26d4eec612c138a586f0b43d579d0c5350 Mon Sep 17 00:00:00 2001 From: Khanh Ngo Date: Mon, 11 Jun 2018 10:58:47 +0700 Subject: [PATCH 1/4] Use Flask-Migrate for db migration --- .gitignore | 1 + app/__init__.py | 2 + app/models.py | 3 +- create_db.py | 121 ------------------------------------------------ db_downgrade.py | 8 ---- db_migrate.py | 17 ------- db_upgrade.py | 7 --- 7 files changed, 5 insertions(+), 154 deletions(-) delete mode 100755 create_db.py delete mode 100755 db_downgrade.py delete mode 100755 db_migrate.py delete mode 100755 db_upgrade.py diff --git a/.gitignore b/.gitignore index bd129d0..8acb3db 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ advanced_settings.json idp.crt log.txt +migrations/* db_repository/* upload/avatar/* tmp/* diff --git a/app/__init__.py b/app/__init__.py index c309b78..20dcab9 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -2,6 +2,7 @@ from werkzeug.contrib.fixers import ProxyFix from flask import Flask, request, session, redirect, url_for from flask_login import LoginManager from flask_sqlalchemy import SQLAlchemy +from flask_migrate import Migrate app = Flask(__name__) app.config.from_object('config') @@ -14,6 +15,7 @@ logging = logger('powerdns-admin', app.config['LOG_LEVEL'], app.config['LOG_FILE login_manager = LoginManager() login_manager.init_app(app) db = SQLAlchemy(app) +migrate = Migrate(app, db) # used for flask-migrate def enable_github_oauth(GITHUB_ENABLE): if not GITHUB_ENABLE: diff --git a/app/models.py b/app/models.py index 8d01a4f..0938c14 100644 --- a/app/models.py +++ b/app/models.py @@ -1389,7 +1389,8 @@ class Server(object): class History(db.Model): id = db.Column(db.Integer, primary_key = True) msg = db.Column(db.String(256)) - detail = db.Column(db.Text().with_variant(db.Text(length=2**24-2), 'mysql')) + # detail = db.Column(db.Text().with_variant(db.Text(length=2**24-2), 'mysql')) + detail = db.Column(db.Text()) created_by = db.Column(db.String(128)) created_on = db.Column(db.DateTime, default=datetime.utcnow) diff --git a/create_db.py b/create_db.py deleted file mode 100755 index 952f04e..0000000 --- a/create_db.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import time -import os.path -import traceback - -from migrate.versioning import api -from config import SQLALCHEMY_DATABASE_URI -from config import SQLALCHEMY_MIGRATE_REPO -from app import db -from app.models import Role, Setting, DomainTemplate - - -def start(): - wait_time = get_waittime_from_env() - - if not connect_db(wait_time): - print("ERROR: Couldn't connect to database server") - exit(1) - - init_records() - -def get_waittime_from_env(): - return int(os.environ.get('WAITFOR_DB', 1)) - -def connect_db(wait_time): - for i in range(0, wait_time): - print("INFO: Wait for database server") - sys.stdout.flush() - try: - db.create_all() - return True - except: - traceback.print_exc() - time.sleep(1) - - return False - -def init_roles(db, role_names): - - # Get key name of data - name_of_roles = [r.name for r in role_names] - - # Query to get current data - rows = db.session.query(Role).filter(Role.name.in_(name_of_roles)).all() - name_of_rows = [r.name for r in rows] - - # Check which data that need to insert - roles = [r for r in role_names if r.name not in name_of_rows] - - # Insert data - for role in roles: - db.session.add(role) - -def init_settings(db, setting_names): - - # Get key name of data - name_of_settings = [r.name for r in setting_names] - - # Query to get current data - rows = db.session.query(Setting).filter(Setting.name.in_(name_of_settings)).all() - - # Check which data that need to insert - name_of_rows = [r.name for r in rows] - settings = [r for r in setting_names if r.name not in name_of_rows] - - # Insert data - for setting in settings: - db.session.add(setting) - - -def init_domain_templates(db, domain_template_names): - - # Get key name of data - name_of_domain_templates = map(lambda r: r.name, domain_template_names) - - # Query to get current data - rows = db.session.query(DomainTemplate).filter(DomainTemplate.name.in_(name_of_domain_templates)).all() - - # Check which data that need to insert - name_of_rows = map(lambda r: r.name, rows) - domain_templates = filter(lambda r: r.name not in name_of_rows, domain_template_names) - - # Insert data - for domain_template in domain_templates: - db.session.add(domain_template) - -def init_records(): - # Create initial user roles and turn off maintenance mode - init_roles(db, [ - Role('Administrator', 'Administrator'), - Role('User', 'User') - ]) - init_settings(db, [ - Setting('maintenance', 'False'), - Setting('fullscreen_layout', 'True'), - Setting('record_helper', 'True'), - Setting('login_ldap_first', 'True'), - Setting('default_record_table_size', '15'), - Setting('default_domain_table_size', '10'), - Setting('auto_ptr','False') - ]) - # TODO: add sample records to sample templates - init_domain_templates(db, [ - DomainTemplate('basic_template_1', 'Basic Template #1'), - DomainTemplate('basic_template_2', 'Basic Template #2'), - DomainTemplate('basic_template_3', 'Basic Template #3') - ]) - db_commit = db.session.commit() - commit_version_control(db_commit) - -def commit_version_control(db_commit): - if not os.path.exists(SQLALCHEMY_MIGRATE_REPO): - api.create(SQLALCHEMY_MIGRATE_REPO, 'database repository') - api.version_control(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) - elif db_commit is not None: - api.version_control(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO, api.version(SQLALCHEMY_MIGRATE_REPO)) - -if __name__ == '__main__': - start() diff --git a/db_downgrade.py b/db_downgrade.py deleted file mode 100755 index fa5866e..0000000 --- a/db_downgrade.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python3 -from migrate.versioning import api -from config import SQLALCHEMY_DATABASE_URI -from config import SQLALCHEMY_MIGRATE_REPO -v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) -api.downgrade(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO, v - 1) -v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) -print('Current database version: ' + str(v)) diff --git a/db_migrate.py b/db_migrate.py deleted file mode 100755 index 7d0869e..0000000 --- a/db_migrate.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python3 -import imp -from migrate.versioning import api -from app import db -from config import SQLALCHEMY_DATABASE_URI -from config import SQLALCHEMY_MIGRATE_REPO -v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) -migration = SQLALCHEMY_MIGRATE_REPO + ('/versions/%03d_migration.py' % (v+1)) -tmp_module = imp.new_module('old_model') -old_model = api.create_model(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) -exec(old_model, tmp_module.__dict__) -script = api.make_update_script_for_model(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO, tmp_module.meta, db.metadata) -open(migration, "wt").write(script) -api.upgrade(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) -v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) -print('New migration saved as ' + migration) -print('Current database version: ' + str(v)) diff --git a/db_upgrade.py b/db_upgrade.py deleted file mode 100755 index 515b309..0000000 --- a/db_upgrade.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python3 -from migrate.versioning import api -from config import SQLALCHEMY_DATABASE_URI -from config import SQLALCHEMY_MIGRATE_REPO -api.upgrade(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) -v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) -print('Current database version: ' + str(v)) From ab0124b3e24c8fadd92aee06c32a598f9f79b975 Mon Sep 17 00:00:00 2001 From: Khanh Ngo Date: Mon, 11 Jun 2018 11:11:43 +0700 Subject: [PATCH 2/4] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 555f16c..ef90853 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,8 @@ Flask-WTF==0.14.2 Flask-Login==0.4.1 Flask-OAuthlib==0.9.4 Flask-SQLAlchemy==2.3.2 +Flask-Migrate==2.1.1 SQLAlchemy==1.2.5 -sqlalchemy-migrate==0.10.0 mysqlclient==1.3.12 configobj==5.0.6 bcrypt==3.1.4 From 232d39cca085af6d098f5b22bd54e887a0b111ad Mon Sep 17 00:00:00 2001 From: Khanh Ngo Date: Mon, 11 Jun 2018 14:12:04 +0700 Subject: [PATCH 3/4] Adjustment in Docker stuff to work with new migration. Add init_data.py script to initialize table data --- docker-compose.yml | 1 + docker/PowerDNS-Admin/Dockerfile | 8 ++++++- docker/PowerDNS-Admin/entrypoint.sh | 13 ++++++++++- init_data.py | 36 +++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 2 deletions(-) create mode 100755 init_data.py diff --git a/docker-compose.yml b/docker-compose.yml index 4c9ec5c..08925ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,7 @@ services: - PDA_DB_PASSWORD=${PDA_DB_PASSWORD} - PDNS_HOST=${PDNS_HOST} - PDNS_API_KEY=${PDNS_API_KEY} + - FLASK_APP=/powerdns-admin/app/__init__.py depends_on: powerdns-admin-mysql: condition: service_healthy diff --git a/docker/PowerDNS-Admin/Dockerfile b/docker/PowerDNS-Admin/Dockerfile index bb6c51f..09b37dc 100644 --- a/docker/PowerDNS-Admin/Dockerfile +++ b/docker/PowerDNS-Admin/Dockerfile @@ -1,11 +1,17 @@ FROM ubuntu:latest -MAINTAINER Khanh Ngo "ngokhanhit@gmail.com" +MAINTAINER Khanh Ngo "k@ndk.name" ARG ENVIRONMENT=development ENV ENVIRONMENT=${ENVIRONMENT} WORKDIR /powerdns-admin RUN apt-get update -y + +RUN apt-get install -y locales locales-all +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 + RUN apt-get install -y python3-pip python3-dev supervisor # lib for building mysql db driver diff --git a/docker/PowerDNS-Admin/entrypoint.sh b/docker/PowerDNS-Admin/entrypoint.sh index f041308..7bdb0b0 100755 --- a/docker/PowerDNS-Admin/entrypoint.sh +++ b/docker/PowerDNS-Admin/entrypoint.sh @@ -1,3 +1,14 @@ #!/bin/sh -cd /powerdns-admin && ./create_db.py + +if [ ! -d "/powerdns-admin/migrations" ]; then + /usr/local/bin/flask db init --directory /powerdns-admin/migrations + /usr/local/bin/flask db migrate -m "Init DB" --directory /powerdns-admin/migrations + /usr/local/bin/flask db upgrade --directory /powerdns-admin/migrations + cd /powerdns-admin && ./init_data.py + +else + /usr/local/bin/flask db migrate -m "Upgrade BD Schema" --directory /powerdns-admin/migrations + /usr/local/bin/flask db upgrade --directory /powerdns-admin/migrations +fi + /usr/bin/supervisord -c /etc/supervisord.conf diff --git a/init_data.py b/init_data.py new file mode 100755 index 0000000..dbd3277 --- /dev/null +++ b/init_data.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +from app import app, db +from app.models import Role, Setting, DomainTemplate + +admin_role = Role(name='Administrator', description='Administrator') +user_role = Role(name='User', description='User') + +setting_1 = Setting(name='maintenance', value='False') +setting_2 = Setting(name='fullscreen_layout', value='True') +setting_3 = Setting(name='record_helper', value='True') +setting_4 = Setting(name='login_ldap_first', value='True') +setting_5 = Setting(name='default_record_table_size', value='15') +setting_6 = Setting(name='default_domain_table_size', value='10') +setting_7 = Setting(name='auto_ptr', value='False') + +template_1 = DomainTemplate(name='basic_template_1', description='Basic Template #1') +template_2 = DomainTemplate(name='basic_template_2', description='Basic Template #2') +template_3 = DomainTemplate(name='basic_template_3', description='Basic Template #3') + +db.session.add(admin_role) +db.session.add(user_role) + +db.session.add(setting_1) +db.session.add(setting_2) +db.session.add(setting_3) +db.session.add(setting_4) +db.session.add(setting_5) +db.session.add(setting_6) +db.session.add(setting_7) + +db.session.add(template_1) +db.session.add(template_2) +db.session.add(template_3) + +db.session.commit() From 646166bbd7a168d15f08f8f901d4fdca2c600644 Mon Sep 17 00:00:00 2001 From: Khanh Ngo Date: Wed, 13 Jun 2018 09:55:57 +0700 Subject: [PATCH 4/4] Update sqlite db file path in config template --- config_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config_template.py b/config_template.py index 9d25fd9..9c2d8fd 100644 --- a/config_template.py +++ b/config_template.py @@ -31,7 +31,7 @@ SQLA_DB_NAME = 'powerdnsadmin' #SQLALCHEMY_DATABASE_URI = 'mysql://'+SQLA_DB_USER+':'\ # +SQLA_DB_PASSWORD+'@'+SQLA_DB_HOST+'/'+SQLA_DB_NAME #SQLite -SQLALCHEMY_DATABASE_URI = 'sqlite:///pdns.db' +SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir, 'pdns.db') SQLALCHEMY_MIGRATE_REPO = os.path.join(basedir, 'db_repository') SQLALCHEMY_TRACK_MODIFICATIONS = True