mirror of
https://github.com/prise6/smart-iss-posts
synced 2024-05-06 07:33:09 +02:00
conf files
This commit is contained in:
parent
f62ee67730
commit
4e4fc39b69
94
Makefile
94
Makefile
|
@ -5,7 +5,6 @@
|
||||||
#################################################################################
|
#################################################################################
|
||||||
|
|
||||||
PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
|
PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
|
||||||
BUCKET = [OPTIONAL] your-bucket-for-syncing-data (do not include 's3://')
|
|
||||||
PROFILE = default
|
PROFILE = default
|
||||||
PROJECT_NAME = smartissposts
|
PROJECT_NAME = smartissposts
|
||||||
PYTHON_INTERPRETER = python
|
PYTHON_INTERPRETER = python
|
||||||
|
@ -21,11 +20,33 @@ endif
|
||||||
# COMMANDS #
|
# COMMANDS #
|
||||||
#################################################################################
|
#################################################################################
|
||||||
|
|
||||||
|
## Delete all compiled Python files
|
||||||
|
clean:
|
||||||
|
find . -path ./mysql -prune -o -type f -name "*.py[co]" -exec rm {} +
|
||||||
|
find . -path ./mysql -prune -o -type d -name "__pycache__" -exec rm {} +
|
||||||
|
|
||||||
|
## Test python environment is setup correctly
|
||||||
|
test_environment:
|
||||||
|
$(PYTHON_INTERPRETER) test_environment.py
|
||||||
|
|
||||||
|
## Write requirements.txt
|
||||||
|
pipreqs:
|
||||||
|
pipreqs --force $(PROJECT_DIR)
|
||||||
|
|
||||||
## Install Python Dependencies
|
## Install Python Dependencies
|
||||||
requirements: test_environment
|
requirements: test_environment
|
||||||
pip install -U pip setuptools wheel
|
pip install -U pip setuptools wheel
|
||||||
pip install -r requirements.txt
|
pip install -r requirements.txt
|
||||||
|
|
||||||
|
## Write config template
|
||||||
|
config_template:
|
||||||
|
$(PYTHON_INTERPRETER) iss/tools/config_template.py
|
||||||
|
|
||||||
|
|
||||||
|
#################################################################################
|
||||||
|
# PROJECT RULES #
|
||||||
|
#################################################################################
|
||||||
|
|
||||||
## Make Dataset
|
## Make Dataset
|
||||||
data: requirements
|
data: requirements
|
||||||
$(PYTHON_INTERPRETER) src/data/make_dataset.py
|
$(PYTHON_INTERPRETER) src/data/make_dataset.py
|
||||||
|
@ -34,77 +55,6 @@ data: requirements
|
||||||
sync_collections: iss/data/sync_collections.sh
|
sync_collections: iss/data/sync_collections.sh
|
||||||
$(PYTHON_INTERPRETER) iss/data/sync_collections.sh
|
$(PYTHON_INTERPRETER) iss/data/sync_collections.sh
|
||||||
|
|
||||||
## Resize collection
|
|
||||||
resize_collections: src/data/resize_collections.py
|
|
||||||
$(PYTHON_INTERPRETER) src/data/resize_collections.py
|
|
||||||
touch resize_collections
|
|
||||||
|
|
||||||
## Build TFRecords
|
|
||||||
build_tfrecords: sync_collections resize_collections src/features/build_tfrecords.py
|
|
||||||
$(PYTHON_INTERPRETER) src/features/build_tfrecords.py
|
|
||||||
touch build_tfrecords
|
|
||||||
|
|
||||||
## Train model
|
|
||||||
train_model: build_tfrecords
|
|
||||||
$(PYTHON_INTERPRETER) src/models/train_model.py
|
|
||||||
|
|
||||||
## test
|
|
||||||
draft:
|
|
||||||
$(PYTHON_INTERPRETER) src/models/data_loader.py
|
|
||||||
|
|
||||||
|
|
||||||
## Delete all compiled Python files
|
|
||||||
clean:
|
|
||||||
find . -path ./mysql -prune -o -type f -name "*.py[co]" -exec rm {} +
|
|
||||||
find . -path ./mysql -prune -o -type d -name "__pycache__" -exec rm {} +
|
|
||||||
|
|
||||||
## Lint using flake8
|
|
||||||
lint:
|
|
||||||
flake8 src
|
|
||||||
|
|
||||||
## Upload Data to S3
|
|
||||||
sync_data_to_s3:
|
|
||||||
ifeq (default,$(PROFILE))
|
|
||||||
aws s3 sync data/ s3://$(BUCKET)/data/
|
|
||||||
else
|
|
||||||
aws s3 sync data/ s3://$(BUCKET)/data/ --profile $(PROFILE)
|
|
||||||
endif
|
|
||||||
|
|
||||||
## Download Data from S3
|
|
||||||
sync_data_from_s3:
|
|
||||||
ifeq (default,$(PROFILE))
|
|
||||||
aws s3 sync s3://$(BUCKET)/data/ data/
|
|
||||||
else
|
|
||||||
aws s3 sync s3://$(BUCKET)/data/ data/ --profile $(PROFILE)
|
|
||||||
endif
|
|
||||||
|
|
||||||
## Set up python interpreter environment
|
|
||||||
create_environment:
|
|
||||||
ifeq (True,$(HAS_CONDA))
|
|
||||||
@echo ">>> Detected conda, creating conda environment."
|
|
||||||
ifeq (3,$(findstring 3,$(PYTHON_INTERPRETER)))
|
|
||||||
conda create --name $(PROJECT_NAME) python=3
|
|
||||||
else
|
|
||||||
conda create --name $(PROJECT_NAME) python=2.7
|
|
||||||
endif
|
|
||||||
@echo ">>> New conda env created. Activate with:\nsource activate $(PROJECT_NAME)"
|
|
||||||
else
|
|
||||||
@pip install -q virtualenv virtualenvwrapper
|
|
||||||
@echo ">>> Installing virtualenvwrapper if not already intalled.\nMake sure the following lines are in shell startup file\n\
|
|
||||||
export WORKON_HOME=$$HOME/.virtualenvs\nexport PROJECT_HOME=$$HOME/Devel\nsource /usr/local/bin/virtualenvwrapper.sh\n"
|
|
||||||
@bash -c "source `which virtualenvwrapper.sh`;mkvirtualenv $(PROJECT_NAME) --python=$(PYTHON_INTERPRETER)"
|
|
||||||
@echo ">>> New virtualenv created. Activate with:\nworkon $(PROJECT_NAME)"
|
|
||||||
endif
|
|
||||||
|
|
||||||
## Test python environment is setup correctly
|
|
||||||
test_environment:
|
|
||||||
$(PYTHON_INTERPRETER) test_environment.py
|
|
||||||
|
|
||||||
#################################################################################
|
|
||||||
# PROJECT RULES #
|
|
||||||
#################################################################################
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#################################################################################
|
#################################################################################
|
||||||
# Self Documenting Commands #
|
# Self Documenting Commands #
|
||||||
|
|
|
@ -1,21 +1,48 @@
|
||||||
version: 1
|
directory:
|
||||||
|
autoencoder:
|
||||||
|
base: XXX
|
||||||
|
test: XXX
|
||||||
|
train: XXX
|
||||||
|
valid: XXX
|
||||||
|
collections: XXX
|
||||||
|
models:
|
||||||
|
simple:
|
||||||
|
batch_size: XXX
|
||||||
|
callbacks:
|
||||||
|
checkpoint:
|
||||||
|
directory: XXX
|
||||||
|
period: XXX
|
||||||
|
verbose: XXX
|
||||||
|
csv_logger:
|
||||||
|
append: XXX
|
||||||
|
directory: XXX
|
||||||
|
display_picture:
|
||||||
|
epoch_laps: XXX
|
||||||
|
epochs: XXX
|
||||||
|
initial_epoch: XXX
|
||||||
|
input_channel: XXX
|
||||||
|
input_height: XXX
|
||||||
|
input_width: XXX
|
||||||
|
learning_rate: XXX
|
||||||
|
model_name: XXX
|
||||||
|
save_directory: XXX
|
||||||
|
steps_per_epoch: XXX
|
||||||
|
use_multiprocessing: XXX
|
||||||
|
validation_freq: XXX
|
||||||
|
validation_steps: XXX
|
||||||
|
verbose: XXX
|
||||||
|
workers: XXX
|
||||||
mysql:
|
mysql:
|
||||||
database:
|
database:
|
||||||
name: XXX
|
name: XXX
|
||||||
user: XXX
|
|
||||||
password: XXX
|
password: XXX
|
||||||
|
port: XXX
|
||||||
server: XXX
|
server: XXX
|
||||||
port: 3306
|
user: XXX
|
||||||
directory:
|
|
||||||
collections: 'path/to/dir/collection'
|
|
||||||
autoencoder:
|
|
||||||
base: 'path/to/dir/collection'
|
|
||||||
train: 'path/to/dir/collection'
|
|
||||||
test: 'path/to/dir/collection'
|
|
||||||
valid: 'path/to/dir/collection'
|
|
||||||
training:
|
training:
|
||||||
seed: 37672
|
|
||||||
proportions:
|
proportions:
|
||||||
train: 0.7
|
test: XXX
|
||||||
test: 0.2
|
train: XXX
|
||||||
valid: 0.1
|
valid: XXX
|
||||||
|
seed: XXX
|
||||||
|
version: XXX
|
||||||
|
|
|
@ -1,14 +1,10 @@
|
||||||
# local package
|
setuptools==40.8.0
|
||||||
-e .
|
Click==7.0
|
||||||
|
numpy==1.13.3
|
||||||
# external requirements
|
pandas==0.23.4
|
||||||
click
|
tensorflow==1.12.0
|
||||||
Sphinx
|
Keras==2.2.4
|
||||||
coverage
|
ipython==7.3.0
|
||||||
awscli
|
Pillow==5.4.1
|
||||||
flake8
|
python-dotenv==0.10.1
|
||||||
python-dotenv>=0.5.1
|
PyYAML==3.13
|
||||||
|
|
||||||
|
|
||||||
# backwards compatibility
|
|
||||||
pathlib2
|
|
||||||
|
|
Loading…
Reference in a new issue