mirror of
https://github.com/prise6/smart-iss-posts
synced 2024-04-24 18:20:28 +02:00
conf files
This commit is contained in:
Francois
parent
f62ee67730
commit
4e4fc39b69
94
Makefile
94
Makefile
|
|
@ -5,7 +5,6 @@
|
|||
#################################################################################
|
||||
|
||||
PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
|
||||
BUCKET = [OPTIONAL] your-bucket-for-syncing-data (do not include 's3://')
|
||||
PROFILE = default
|
||||
PROJECT_NAME = smartissposts
|
||||
PYTHON_INTERPRETER = python
|
||||
|
|
@ -21,11 +20,33 @@ endif
|
|||
# COMMANDS #
|
||||
#################################################################################
|
||||
|
||||
## Delete all compiled Python files
|
||||
clean:
|
||||
find . -path ./mysql -prune -o -type f -name "*.py[co]" -exec rm {} +
|
||||
find . -path ./mysql -prune -o -type d -name "__pycache__" -exec rm {} +
|
||||
|
||||
## Test python environment is setup correctly
|
||||
test_environment:
|
||||
$(PYTHON_INTERPRETER) test_environment.py
|
||||
|
||||
## Write requirements.txt
|
||||
pipreqs:
|
||||
pipreqs --force $(PROJECT_DIR)
|
||||
|
||||
## Install Python Dependencies
|
||||
requirements: test_environment
|
||||
pip install -U pip setuptools wheel
|
||||
pip install -r requirements.txt
|
||||
|
||||
## Write config template
|
||||
config_template:
|
||||
$(PYTHON_INTERPRETER) iss/tools/config_template.py
|
||||
|
||||
|
||||
#################################################################################
|
||||
# PROJECT RULES #
|
||||
#################################################################################
|
||||
|
||||
## Make Dataset
|
||||
data: requirements
|
||||
$(PYTHON_INTERPRETER) src/data/make_dataset.py
|
||||
|
|
@ -34,77 +55,6 @@ data: requirements
|
|||
sync_collections: iss/data/sync_collections.sh
|
||||
$(PYTHON_INTERPRETER) iss/data/sync_collections.sh
|
||||
|
||||
## Resize collection
|
||||
resize_collections: src/data/resize_collections.py
|
||||
$(PYTHON_INTERPRETER) src/data/resize_collections.py
|
||||
touch resize_collections
|
||||
|
||||
## Build TFRecords
|
||||
build_tfrecords: sync_collections resize_collections src/features/build_tfrecords.py
|
||||
$(PYTHON_INTERPRETER) src/features/build_tfrecords.py
|
||||
touch build_tfrecords
|
||||
|
||||
## Train model
|
||||
train_model: build_tfrecords
|
||||
$(PYTHON_INTERPRETER) src/models/train_model.py
|
||||
|
||||
## test
|
||||
draft:
|
||||
$(PYTHON_INTERPRETER) src/models/data_loader.py
|
||||
|
||||
|
||||
## Delete all compiled Python files
|
||||
clean:
|
||||
find . -path ./mysql -prune -o -type f -name "*.py[co]" -exec rm {} +
|
||||
find . -path ./mysql -prune -o -type d -name "__pycache__" -exec rm {} +
|
||||
|
||||
## Lint using flake8
|
||||
lint:
|
||||
flake8 src
|
||||
|
||||
## Upload Data to S3
|
||||
sync_data_to_s3:
|
||||
ifeq (default,$(PROFILE))
|
||||
aws s3 sync data/ s3://$(BUCKET)/data/
|
||||
else
|
||||
aws s3 sync data/ s3://$(BUCKET)/data/ --profile $(PROFILE)
|
||||
endif
|
||||
|
||||
## Download Data from S3
|
||||
sync_data_from_s3:
|
||||
ifeq (default,$(PROFILE))
|
||||
aws s3 sync s3://$(BUCKET)/data/ data/
|
||||
else
|
||||
aws s3 sync s3://$(BUCKET)/data/ data/ --profile $(PROFILE)
|
||||
endif
|
||||
|
||||
## Set up python interpreter environment
|
||||
create_environment:
|
||||
ifeq (True,$(HAS_CONDA))
|
||||
@echo ">>> Detected conda, creating conda environment."
|
||||
ifeq (3,$(findstring 3,$(PYTHON_INTERPRETER)))
|
||||
conda create --name $(PROJECT_NAME) python=3
|
||||
else
|
||||
conda create --name $(PROJECT_NAME) python=2.7
|
||||
endif
|
||||
@echo ">>> New conda env created. Activate with:\nsource activate $(PROJECT_NAME)"
|
||||
else
|
||||
@pip install -q virtualenv virtualenvwrapper
|
||||
@echo ">>> Installing virtualenvwrapper if not already intalled.\nMake sure the following lines are in shell startup file\n\
|
||||
export WORKON_HOME=$$HOME/.virtualenvs\nexport PROJECT_HOME=$$HOME/Devel\nsource /usr/local/bin/virtualenvwrapper.sh\n"
|
||||
@bash -c "source `which virtualenvwrapper.sh`;mkvirtualenv $(PROJECT_NAME) --python=$(PYTHON_INTERPRETER)"
|
||||
@echo ">>> New virtualenv created. Activate with:\nworkon $(PROJECT_NAME)"
|
||||
endif
|
||||
|
||||
## Test python environment is setup correctly
|
||||
test_environment:
|
||||
$(PYTHON_INTERPRETER) test_environment.py
|
||||
|
||||
#################################################################################
|
||||
# PROJECT RULES #
|
||||
#################################################################################
|
||||
|
||||
|
||||
|
||||
#################################################################################
|
||||
# Self Documenting Commands #
|
||||
|
|
|
|||
|
|
@ -1,21 +1,48 @@
|
|||
version: 1
|
||||
directory:
|
||||
autoencoder:
|
||||
base: XXX
|
||||
test: XXX
|
||||
train: XXX
|
||||
valid: XXX
|
||||
collections: XXX
|
||||
models:
|
||||
simple:
|
||||
batch_size: XXX
|
||||
callbacks:
|
||||
checkpoint:
|
||||
directory: XXX
|
||||
period: XXX
|
||||
verbose: XXX
|
||||
csv_logger:
|
||||
append: XXX
|
||||
directory: XXX
|
||||
display_picture:
|
||||
epoch_laps: XXX
|
||||
epochs: XXX
|
||||
initial_epoch: XXX
|
||||
input_channel: XXX
|
||||
input_height: XXX
|
||||
input_width: XXX
|
||||
learning_rate: XXX
|
||||
model_name: XXX
|
||||
save_directory: XXX
|
||||
steps_per_epoch: XXX
|
||||
use_multiprocessing: XXX
|
||||
validation_freq: XXX
|
||||
validation_steps: XXX
|
||||
verbose: XXX
|
||||
workers: XXX
|
||||
mysql:
|
||||
database:
|
||||
name: XXX
|
||||
user: XXX
|
||||
password: XXX
|
||||
port: XXX
|
||||
server: XXX
|
||||
port: 3306
|
||||
directory:
|
||||
collections: 'path/to/dir/collection'
|
||||
autoencoder:
|
||||
base: 'path/to/dir/collection'
|
||||
train: 'path/to/dir/collection'
|
||||
test: 'path/to/dir/collection'
|
||||
valid: 'path/to/dir/collection'
|
||||
user: XXX
|
||||
training:
|
||||
seed: 37672
|
||||
proportions:
|
||||
train: 0.7
|
||||
test: 0.2
|
||||
valid: 0.1
|
||||
test: XXX
|
||||
train: XXX
|
||||
valid: XXX
|
||||
seed: XXX
|
||||
version: XXX
|
||||
|
|
|
|||
|
|
@ -1,14 +1,10 @@
|
|||
# local package
|
||||
-e .
|
||||
|
||||
# external requirements
|
||||
click
|
||||
Sphinx
|
||||
coverage
|
||||
awscli
|
||||
flake8
|
||||
python-dotenv>=0.5.1
|
||||
|
||||
|
||||
# backwards compatibility
|
||||
pathlib2
|
||||
setuptools==40.8.0
|
||||
Click==7.0
|
||||
numpy==1.13.3
|
||||
pandas==0.23.4
|
||||
tensorflow==1.12.0
|
||||
Keras==2.2.4
|
||||
ipython==7.3.0
|
||||
Pillow==5.4.1
|
||||
python-dotenv==0.10.1
|
||||
PyYAML==3.13
|
||||
|
|
|
|||
Loading…
Reference in a new issue