From 4e4fc39b6984cef9dff458364384f9faaf67252f Mon Sep 17 00:00:00 2001 From: Francois Date: Sun, 10 Mar 2019 00:19:08 +0100 Subject: [PATCH] conf files --- Makefile | 94 +++++++++---------------------------- config/config.template.yaml | 55 ++++++++++++++++------ requirements.txt | 24 ++++------ 3 files changed, 73 insertions(+), 100 deletions(-) diff --git a/Makefile b/Makefile index fadd0be..80a9cb9 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,6 @@ ################################################################################# PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) -BUCKET = [OPTIONAL] your-bucket-for-syncing-data (do not include 's3://') PROFILE = default PROJECT_NAME = smartissposts PYTHON_INTERPRETER = python @@ -21,11 +20,33 @@ endif # COMMANDS # ################################################################################# +## Delete all compiled Python files +clean: + find . -path ./mysql -prune -o -type f -name "*.py[co]" -exec rm {} + + find . -path ./mysql -prune -o -type d -name "__pycache__" -exec rm {} + + +## Test python environment is setup correctly +test_environment: + $(PYTHON_INTERPRETER) test_environment.py + +## Write requirements.txt +pipreqs: + pipreqs --force $(PROJECT_DIR) + ## Install Python Dependencies requirements: test_environment pip install -U pip setuptools wheel pip install -r requirements.txt +## Write config template +config_template: + $(PYTHON_INTERPRETER) iss/tools/config_template.py + + +################################################################################# +# PROJECT RULES # +################################################################################# + ## Make Dataset data: requirements $(PYTHON_INTERPRETER) src/data/make_dataset.py @@ -34,77 +55,6 @@ data: requirements sync_collections: iss/data/sync_collections.sh $(PYTHON_INTERPRETER) iss/data/sync_collections.sh -## Resize collection -resize_collections: src/data/resize_collections.py - $(PYTHON_INTERPRETER) src/data/resize_collections.py - touch resize_collections - -## Build TFRecords -build_tfrecords: sync_collections resize_collections src/features/build_tfrecords.py - $(PYTHON_INTERPRETER) src/features/build_tfrecords.py - touch build_tfrecords - -## Train model -train_model: build_tfrecords - $(PYTHON_INTERPRETER) src/models/train_model.py - -## test -draft: - $(PYTHON_INTERPRETER) src/models/data_loader.py - - -## Delete all compiled Python files -clean: - find . -path ./mysql -prune -o -type f -name "*.py[co]" -exec rm {} + - find . -path ./mysql -prune -o -type d -name "__pycache__" -exec rm {} + - -## Lint using flake8 -lint: - flake8 src - -## Upload Data to S3 -sync_data_to_s3: -ifeq (default,$(PROFILE)) - aws s3 sync data/ s3://$(BUCKET)/data/ -else - aws s3 sync data/ s3://$(BUCKET)/data/ --profile $(PROFILE) -endif - -## Download Data from S3 -sync_data_from_s3: -ifeq (default,$(PROFILE)) - aws s3 sync s3://$(BUCKET)/data/ data/ -else - aws s3 sync s3://$(BUCKET)/data/ data/ --profile $(PROFILE) -endif - -## Set up python interpreter environment -create_environment: -ifeq (True,$(HAS_CONDA)) - @echo ">>> Detected conda, creating conda environment." -ifeq (3,$(findstring 3,$(PYTHON_INTERPRETER))) - conda create --name $(PROJECT_NAME) python=3 -else - conda create --name $(PROJECT_NAME) python=2.7 -endif - @echo ">>> New conda env created. Activate with:\nsource activate $(PROJECT_NAME)" -else - @pip install -q virtualenv virtualenvwrapper - @echo ">>> Installing virtualenvwrapper if not already intalled.\nMake sure the following lines are in shell startup file\n\ - export WORKON_HOME=$$HOME/.virtualenvs\nexport PROJECT_HOME=$$HOME/Devel\nsource /usr/local/bin/virtualenvwrapper.sh\n" - @bash -c "source `which virtualenvwrapper.sh`;mkvirtualenv $(PROJECT_NAME) --python=$(PYTHON_INTERPRETER)" - @echo ">>> New virtualenv created. Activate with:\nworkon $(PROJECT_NAME)" -endif - -## Test python environment is setup correctly -test_environment: - $(PYTHON_INTERPRETER) test_environment.py - -################################################################################# -# PROJECT RULES # -################################################################################# - - ################################################################################# # Self Documenting Commands # diff --git a/config/config.template.yaml b/config/config.template.yaml index 1a553c1..c0ab64b 100644 --- a/config/config.template.yaml +++ b/config/config.template.yaml @@ -1,21 +1,48 @@ -version: 1 +directory: + autoencoder: + base: XXX + test: XXX + train: XXX + valid: XXX + collections: XXX +models: + simple: + batch_size: XXX + callbacks: + checkpoint: + directory: XXX + period: XXX + verbose: XXX + csv_logger: + append: XXX + directory: XXX + display_picture: + epoch_laps: XXX + epochs: XXX + initial_epoch: XXX + input_channel: XXX + input_height: XXX + input_width: XXX + learning_rate: XXX + model_name: XXX + save_directory: XXX + steps_per_epoch: XXX + use_multiprocessing: XXX + validation_freq: XXX + validation_steps: XXX + verbose: XXX + workers: XXX mysql: database: name: XXX - user: XXX password: XXX + port: XXX server: XXX - port: 3306 -directory: - collections: 'path/to/dir/collection' - autoencoder: - base: 'path/to/dir/collection' - train: 'path/to/dir/collection' - test: 'path/to/dir/collection' - valid: 'path/to/dir/collection' + user: XXX training: - seed: 37672 proportions: - train: 0.7 - test: 0.2 - valid: 0.1 \ No newline at end of file + test: XXX + train: XXX + valid: XXX + seed: XXX +version: XXX diff --git a/requirements.txt b/requirements.txt index bdcde80..0df7daf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,10 @@ -# local package --e . - -# external requirements -click -Sphinx -coverage -awscli -flake8 -python-dotenv>=0.5.1 - - -# backwards compatibility -pathlib2 +setuptools==40.8.0 +Click==7.0 +numpy==1.13.3 +pandas==0.23.4 +tensorflow==1.12.0 +Keras==2.2.4 +ipython==7.3.0 +Pillow==5.4.1 +python-dotenv==0.10.1 +PyYAML==3.13