mirror of
https://github.com/prise6/smart-iss-posts
synced 2024-06-08 00:32:12 +02:00
tests dans les notebooks
This commit is contained in:
parent
18aaf633a2
commit
f86bb1fc53
|
@ -1,699 +0,0 @@
|
||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 132,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import pandas as pd"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 133,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"refs = pd.read_csv('../data/external/refs/references_labels.csv')"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 103,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"text/html": [
|
|
||||||
"<div>\n",
|
|
||||||
"<style scoped>\n",
|
|
||||||
" .dataframe tbody tr th:only-of-type {\n",
|
|
||||||
" vertical-align: middle;\n",
|
|
||||||
" }\n",
|
|
||||||
"\n",
|
|
||||||
" .dataframe tbody tr th {\n",
|
|
||||||
" vertical-align: top;\n",
|
|
||||||
" }\n",
|
|
||||||
"\n",
|
|
||||||
" .dataframe thead th {\n",
|
|
||||||
" text-align: right;\n",
|
|
||||||
" }\n",
|
|
||||||
"</style>\n",
|
|
||||||
"<table border=\"1\" class=\"dataframe\">\n",
|
|
||||||
" <thead>\n",
|
|
||||||
" <tr style=\"text-align: right;\">\n",
|
|
||||||
" <th></th>\n",
|
|
||||||
" <th>image</th>\n",
|
|
||||||
" <th>label</th>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" </thead>\n",
|
|
||||||
" <tbody>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>0</th>\n",
|
|
||||||
" <td>20170416-012001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>1</th>\n",
|
|
||||||
" <td>20170416-012001.jpg</td>\n",
|
|
||||||
" <td>3</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>2</th>\n",
|
|
||||||
" <td>20170416-013001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>3</th>\n",
|
|
||||||
" <td>20170416-014001.jpg</td>\n",
|
|
||||||
" <td>2</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>4</th>\n",
|
|
||||||
" <td>20170416-014001.jpg</td>\n",
|
|
||||||
" <td>9</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>5</th>\n",
|
|
||||||
" <td>20170416-015001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>6</th>\n",
|
|
||||||
" <td>20170416-015001.jpg</td>\n",
|
|
||||||
" <td>3</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>7</th>\n",
|
|
||||||
" <td>20170416-015001.jpg</td>\n",
|
|
||||||
" <td>6</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>8</th>\n",
|
|
||||||
" <td>20170416-015001.jpg</td>\n",
|
|
||||||
" <td>9</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>9</th>\n",
|
|
||||||
" <td>20170416-025001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>10</th>\n",
|
|
||||||
" <td>20170416-025001.jpg</td>\n",
|
|
||||||
" <td>8</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>11</th>\n",
|
|
||||||
" <td>20170416-030001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>12</th>\n",
|
|
||||||
" <td>20170416-030001.jpg</td>\n",
|
|
||||||
" <td>3</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>13</th>\n",
|
|
||||||
" <td>20170416-030001.jpg</td>\n",
|
|
||||||
" <td>6</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>14</th>\n",
|
|
||||||
" <td>20170416-030001.jpg</td>\n",
|
|
||||||
" <td>9</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>15</th>\n",
|
|
||||||
" <td>20170416-032001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>16</th>\n",
|
|
||||||
" <td>20170416-044001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>17</th>\n",
|
|
||||||
" <td>20170416-044001.jpg</td>\n",
|
|
||||||
" <td>8</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>18</th>\n",
|
|
||||||
" <td>20170416-045001.jpg</td>\n",
|
|
||||||
" <td>2</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>19</th>\n",
|
|
||||||
" <td>20170416-045001.jpg</td>\n",
|
|
||||||
" <td>4</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>20</th>\n",
|
|
||||||
" <td>20170416-050001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>21</th>\n",
|
|
||||||
" <td>20170416-050001.jpg</td>\n",
|
|
||||||
" <td>3</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>22</th>\n",
|
|
||||||
" <td>20170416-050001.jpg</td>\n",
|
|
||||||
" <td>6</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>23</th>\n",
|
|
||||||
" <td>20170416-050001.jpg</td>\n",
|
|
||||||
" <td>9</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>24</th>\n",
|
|
||||||
" <td>20170416-051002.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>25</th>\n",
|
|
||||||
" <td>20170416-051002.jpg</td>\n",
|
|
||||||
" <td>3</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>26</th>\n",
|
|
||||||
" <td>20170416-051002.jpg</td>\n",
|
|
||||||
" <td>7</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>27</th>\n",
|
|
||||||
" <td>20170416-051002.jpg</td>\n",
|
|
||||||
" <td>8</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>28</th>\n",
|
|
||||||
" <td>20170416-062001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>29</th>\n",
|
|
||||||
" <td>20170416-062001.jpg</td>\n",
|
|
||||||
" <td>3</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>...</th>\n",
|
|
||||||
" <td>...</td>\n",
|
|
||||||
" <td>...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>646</th>\n",
|
|
||||||
" <td>20180415-022001.jpg</td>\n",
|
|
||||||
" <td>5</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>647</th>\n",
|
|
||||||
" <td>20180415-022001.jpg</td>\n",
|
|
||||||
" <td>9</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>648</th>\n",
|
|
||||||
" <td>20170803-210001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>649</th>\n",
|
|
||||||
" <td>20170803-210001.jpg</td>\n",
|
|
||||||
" <td>3</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>650</th>\n",
|
|
||||||
" <td>20170803-210001.jpg</td>\n",
|
|
||||||
" <td>5</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>651</th>\n",
|
|
||||||
" <td>20170803-210001.jpg</td>\n",
|
|
||||||
" <td>9</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>652</th>\n",
|
|
||||||
" <td>20180101-174001.jpg</td>\n",
|
|
||||||
" <td>2</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>653</th>\n",
|
|
||||||
" <td>20180101-174001.jpg</td>\n",
|
|
||||||
" <td>3</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>654</th>\n",
|
|
||||||
" <td>20180101-174001.jpg</td>\n",
|
|
||||||
" <td>9</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>655</th>\n",
|
|
||||||
" <td>20180327-112001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>656</th>\n",
|
|
||||||
" <td>20180327-112001.jpg</td>\n",
|
|
||||||
" <td>8</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>657</th>\n",
|
|
||||||
" <td>20180418-060001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>658</th>\n",
|
|
||||||
" <td>20170813-034001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>659</th>\n",
|
|
||||||
" <td>20170813-034001.jpg</td>\n",
|
|
||||||
" <td>3</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>660</th>\n",
|
|
||||||
" <td>20170813-034001.jpg</td>\n",
|
|
||||||
" <td>4</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>661</th>\n",
|
|
||||||
" <td>20170716-094001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>662</th>\n",
|
|
||||||
" <td>20170716-094001.jpg</td>\n",
|
|
||||||
" <td>4</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>663</th>\n",
|
|
||||||
" <td>20180501-120001.jpg</td>\n",
|
|
||||||
" <td>2</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>664</th>\n",
|
|
||||||
" <td>20180501-120001.jpg</td>\n",
|
|
||||||
" <td>3</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>665</th>\n",
|
|
||||||
" <td>20180501-120001.jpg</td>\n",
|
|
||||||
" <td>4</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>666</th>\n",
|
|
||||||
" <td>20170913-012001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>667</th>\n",
|
|
||||||
" <td>20170726-033001.jpg</td>\n",
|
|
||||||
" <td>2</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>668</th>\n",
|
|
||||||
" <td>20171216-170001.jpg</td>\n",
|
|
||||||
" <td>2</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>669</th>\n",
|
|
||||||
" <td>20171224-144001.jpg</td>\n",
|
|
||||||
" <td>2</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>670</th>\n",
|
|
||||||
" <td>20171224-144001.jpg</td>\n",
|
|
||||||
" <td>3</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>671</th>\n",
|
|
||||||
" <td>20170423-003001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>672</th>\n",
|
|
||||||
" <td>20170423-003001.jpg</td>\n",
|
|
||||||
" <td>3</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>673</th>\n",
|
|
||||||
" <td>20170424-092001.jpg</td>\n",
|
|
||||||
" <td>1</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>674</th>\n",
|
|
||||||
" <td>20170424-092001.jpg</td>\n",
|
|
||||||
" <td>3</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>675</th>\n",
|
|
||||||
" <td>20170424-092001.jpg</td>\n",
|
|
||||||
" <td>8</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" </tbody>\n",
|
|
||||||
"</table>\n",
|
|
||||||
"<p>676 rows × 2 columns</p>\n",
|
|
||||||
"</div>"
|
|
||||||
],
|
|
||||||
"text/plain": [
|
|
||||||
" image label\n",
|
|
||||||
"0 20170416-012001.jpg 1\n",
|
|
||||||
"1 20170416-012001.jpg 3\n",
|
|
||||||
"2 20170416-013001.jpg 1\n",
|
|
||||||
"3 20170416-014001.jpg 2\n",
|
|
||||||
"4 20170416-014001.jpg 9\n",
|
|
||||||
"5 20170416-015001.jpg 1\n",
|
|
||||||
"6 20170416-015001.jpg 3\n",
|
|
||||||
"7 20170416-015001.jpg 6\n",
|
|
||||||
"8 20170416-015001.jpg 9\n",
|
|
||||||
"9 20170416-025001.jpg 1\n",
|
|
||||||
"10 20170416-025001.jpg 8\n",
|
|
||||||
"11 20170416-030001.jpg 1\n",
|
|
||||||
"12 20170416-030001.jpg 3\n",
|
|
||||||
"13 20170416-030001.jpg 6\n",
|
|
||||||
"14 20170416-030001.jpg 9\n",
|
|
||||||
"15 20170416-032001.jpg 1\n",
|
|
||||||
"16 20170416-044001.jpg 1\n",
|
|
||||||
"17 20170416-044001.jpg 8\n",
|
|
||||||
"18 20170416-045001.jpg 2\n",
|
|
||||||
"19 20170416-045001.jpg 4\n",
|
|
||||||
"20 20170416-050001.jpg 1\n",
|
|
||||||
"21 20170416-050001.jpg 3\n",
|
|
||||||
"22 20170416-050001.jpg 6\n",
|
|
||||||
"23 20170416-050001.jpg 9\n",
|
|
||||||
"24 20170416-051002.jpg 1\n",
|
|
||||||
"25 20170416-051002.jpg 3\n",
|
|
||||||
"26 20170416-051002.jpg 7\n",
|
|
||||||
"27 20170416-051002.jpg 8\n",
|
|
||||||
"28 20170416-062001.jpg 1\n",
|
|
||||||
"29 20170416-062001.jpg 3\n",
|
|
||||||
".. ... ...\n",
|
|
||||||
"646 20180415-022001.jpg 5\n",
|
|
||||||
"647 20180415-022001.jpg 9\n",
|
|
||||||
"648 20170803-210001.jpg 1\n",
|
|
||||||
"649 20170803-210001.jpg 3\n",
|
|
||||||
"650 20170803-210001.jpg 5\n",
|
|
||||||
"651 20170803-210001.jpg 9\n",
|
|
||||||
"652 20180101-174001.jpg 2\n",
|
|
||||||
"653 20180101-174001.jpg 3\n",
|
|
||||||
"654 20180101-174001.jpg 9\n",
|
|
||||||
"655 20180327-112001.jpg 1\n",
|
|
||||||
"656 20180327-112001.jpg 8\n",
|
|
||||||
"657 20180418-060001.jpg 1\n",
|
|
||||||
"658 20170813-034001.jpg 1\n",
|
|
||||||
"659 20170813-034001.jpg 3\n",
|
|
||||||
"660 20170813-034001.jpg 4\n",
|
|
||||||
"661 20170716-094001.jpg 1\n",
|
|
||||||
"662 20170716-094001.jpg 4\n",
|
|
||||||
"663 20180501-120001.jpg 2\n",
|
|
||||||
"664 20180501-120001.jpg 3\n",
|
|
||||||
"665 20180501-120001.jpg 4\n",
|
|
||||||
"666 20170913-012001.jpg 1\n",
|
|
||||||
"667 20170726-033001.jpg 2\n",
|
|
||||||
"668 20171216-170001.jpg 2\n",
|
|
||||||
"669 20171224-144001.jpg 2\n",
|
|
||||||
"670 20171224-144001.jpg 3\n",
|
|
||||||
"671 20170423-003001.jpg 1\n",
|
|
||||||
"672 20170423-003001.jpg 3\n",
|
|
||||||
"673 20170424-092001.jpg 1\n",
|
|
||||||
"674 20170424-092001.jpg 3\n",
|
|
||||||
"675 20170424-092001.jpg 8\n",
|
|
||||||
"\n",
|
|
||||||
"[676 rows x 2 columns]"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"execution_count": 103,
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "execute_result"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"refs"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 155,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"refs2 = refs.pivot_table(index=\"image\", columns=\"label\", aggfunc=len, fill_value=0).reset_index()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 156,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"text/plain": [
|
|
||||||
"Index(['image', 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype='object', name='label')"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"execution_count": 156,
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "execute_result"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"refs2.columns"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 105,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"refs2.index.name = None"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 136,
|
|
||||||
"metadata": {
|
|
||||||
"scrolled": true
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"ename": "ValueError",
|
|
||||||
"evalue": "Length mismatch: Expected axis has 10 elements, new values have 9 elements",
|
|
||||||
"output_type": "error",
|
|
||||||
"traceback": [
|
|
||||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
||||||
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
|
|
||||||
"\u001b[0;32m<ipython-input-136-d55c82b68b48>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrefs2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"label\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcol\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrefs2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcol\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'image'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
|
|
||||||
"\u001b[0;32m~/anaconda3/envs/py35/lib/python3.5/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__setattr__\u001b[0;34m(self, name, value)\u001b[0m\n\u001b[1;32m 3625\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3626\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3627\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3628\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3629\u001b[0m \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
|
||||||
"\u001b[0;32mpandas/_libs/properties.pyx\u001b[0m in \u001b[0;36mpandas._libs.properties.AxisProperty.__set__\u001b[0;34m()\u001b[0m\n",
|
|
||||||
"\u001b[0;32m~/anaconda3/envs/py35/lib/python3.5/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_set_axis\u001b[0;34m(self, axis, labels)\u001b[0m\n\u001b[1;32m 557\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 558\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_set_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 559\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 560\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_clear_item_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 561\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
|
|
||||||
"\u001b[0;32m~/anaconda3/envs/py35/lib/python3.5/site-packages/pandas/core/internals.py\u001b[0m in \u001b[0;36mset_axis\u001b[0;34m(self, axis, new_labels)\u001b[0m\n\u001b[1;32m 3072\u001b[0m raise ValueError('Length mismatch: Expected axis has %d elements, '\n\u001b[1;32m 3073\u001b[0m \u001b[0;34m'new values have %d elements'\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3074\u001b[0;31m (old_len, new_len))\n\u001b[0m\u001b[1;32m 3075\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3076\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_labels\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
|
||||||
"\u001b[0;31mValueError\u001b[0m: Length mismatch: Expected axis has 10 elements, new values have 9 elements"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"refs2.columns = [\"label\" + str(col) for col in refs2.columns.tolist() if col != 'image']"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 157,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"refs2.rename(columns=lambda x: \"label\" + str(x) if x != 'image' else 'image', inplace=True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 163,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"text/plain": [
|
|
||||||
"RangeIndex(start=0, stop=320, step=1)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"execution_count": 163,
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "execute_result"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"refs2.index"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 164,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"label\n",
|
|
||||||
"label1 1\n",
|
|
||||||
"label2 0\n",
|
|
||||||
"Name: 0, dtype: object\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"for i, row in refs2.iterrows():\n",
|
|
||||||
" print(row[{'label1', 'label2'}])\n",
|
|
||||||
" break"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 126,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"text/plain": [
|
|
||||||
"320"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"execution_count": 126,
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "execute_result"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"refs2.index.size"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 127,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"text/plain": [
|
|
||||||
"(676, 2)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"execution_count": 127,
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "execute_result"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"refs.shape"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 131,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"text/plain": [
|
|
||||||
"<bound method NDFrame.describe of label image 1 2 3 4 5 6 7 8 9\n",
|
|
||||||
"0 20170416-012001.jpg 1 0 1 0 0 0 0 0 0\n",
|
|
||||||
"1 20170416-013001.jpg 1 0 0 0 0 0 0 0 0\n",
|
|
||||||
"2 20170416-014001.jpg 0 1 0 0 0 0 0 0 1\n",
|
|
||||||
"3 20170416-015001.jpg 1 0 1 0 0 1 0 0 1\n",
|
|
||||||
"4 20170416-025001.jpg 1 0 0 0 0 0 0 1 0\n",
|
|
||||||
"5 20170416-030001.jpg 1 0 1 0 0 1 0 0 1\n",
|
|
||||||
"6 20170416-032001.jpg 1 0 0 0 0 0 0 0 0\n",
|
|
||||||
"7 20170416-044001.jpg 1 0 0 0 0 0 0 1 0\n",
|
|
||||||
"8 20170416-045001.jpg 0 1 0 1 0 0 0 0 0\n",
|
|
||||||
"9 20170416-050001.jpg 1 0 1 0 0 1 0 0 1\n",
|
|
||||||
"10 20170416-051002.jpg 1 0 1 0 0 0 1 1 0\n",
|
|
||||||
"11 20170416-062001.jpg 1 0 1 0 0 1 0 0 1\n",
|
|
||||||
"12 20170416-063001.jpg 1 0 0 0 0 0 0 1 0\n",
|
|
||||||
"13 20170416-073001.jpg 1 0 1 0 1 0 0 0 1\n",
|
|
||||||
"14 20170416-080001.jpg 0 1 0 1 0 0 0 0 0\n",
|
|
||||||
"15 20170416-081001.jpg 1 0 1 0 0 1 0 0 1\n",
|
|
||||||
"16 20170416-090001.jpg 1 0 0 0 0 0 1 1 0\n",
|
|
||||||
"17 20170416-094002.jpg 1 0 0 0 0 0 0 0 0\n",
|
|
||||||
"18 20170416-104001.jpg 1 0 1 0 0 1 0 0 1\n",
|
|
||||||
"19 20170416-105002.jpg 1 0 1 0 0 0 0 0 0\n",
|
|
||||||
"20 20170416-110001.jpg 1 0 0 1 0 0 0 1 0\n",
|
|
||||||
"21 20170416-111002.jpg 1 0 1 0 0 1 0 0 1\n",
|
|
||||||
"22 20170416-120001.jpg 0 0 1 0 1 0 0 0 1\n",
|
|
||||||
"23 20170416-121001.jpg 1 0 0 0 0 0 1 0 0\n",
|
|
||||||
"24 20170416-123001.jpg 1 0 1 1 0 1 0 0 0\n",
|
|
||||||
"25 20170416-134001.jpg 1 0 1 0 1 0 0 0 0\n",
|
|
||||||
"26 20170416-151001.jpg 0 0 1 0 1 0 0 0 0\n",
|
|
||||||
"27 20170416-152001.jpg 1 0 0 1 0 0 0 1 0\n",
|
|
||||||
"28 20170416-153001.jpg 0 1 0 0 0 0 0 0 0\n",
|
|
||||||
"29 20170416-154002.jpg 1 0 1 0 0 1 0 0 0\n",
|
|
||||||
".. ... .. .. .. .. .. .. .. .. ..\n",
|
|
||||||
"290 20180406-232001.jpg 1 0 1 0 0 1 0 0 0\n",
|
|
||||||
"291 20180408-060001.jpg 1 0 1 0 0 1 0 0 0\n",
|
|
||||||
"292 20180409-222001.jpg 1 0 1 0 0 0 0 0 0\n",
|
|
||||||
"293 20180410-164001.jpg 1 0 0 0 0 0 0 1 0\n",
|
|
||||||
"294 20180411-174001.jpg 1 0 1 1 0 0 0 0 0\n",
|
|
||||||
"295 20180412-170001.jpg 1 0 0 0 0 0 0 0 0\n",
|
|
||||||
"296 20180415-002001.jpg 1 0 1 0 0 0 0 1 0\n",
|
|
||||||
"297 20180415-022001.jpg 1 0 0 0 1 0 0 0 1\n",
|
|
||||||
"298 20180415-034001.jpg 1 0 0 0 0 0 0 0 0\n",
|
|
||||||
"299 20180415-204001.jpg 1 0 0 1 0 0 0 0 0\n",
|
|
||||||
"300 20180415-232001.jpg 0 1 1 0 0 0 0 0 1\n",
|
|
||||||
"301 20180416-024001.jpg 1 0 1 0 0 0 0 0 0\n",
|
|
||||||
"302 20180417-000001.jpg 1 0 1 0 0 0 0 0 0\n",
|
|
||||||
"303 20180418-060001.jpg 1 0 0 0 0 0 0 0 0\n",
|
|
||||||
"304 20180418-222001.jpg 1 0 1 0 0 0 0 0 0\n",
|
|
||||||
"305 20180419-032001.jpg 1 0 1 0 0 0 0 1 0\n",
|
|
||||||
"306 20180419-154001.jpg 0 1 1 0 0 0 0 0 1\n",
|
|
||||||
"307 20180419-200001.jpg 0 1 1 0 0 0 0 0 0\n",
|
|
||||||
"308 20180420-204001.jpg 0 0 1 0 1 0 0 0 0\n",
|
|
||||||
"309 20180424-010001.jpg 1 0 0 0 0 0 0 0 0\n",
|
|
||||||
"310 20180424-052001.jpg 1 0 1 1 0 0 0 0 0\n",
|
|
||||||
"311 20180425-122001.jpg 0 1 1 1 0 0 0 1 0\n",
|
|
||||||
"312 20180429-042001.jpg 0 1 1 0 0 0 0 0 1\n",
|
|
||||||
"313 20180429-070001.jpg 0 0 0 0 1 0 1 0 0\n",
|
|
||||||
"314 20180501-120001.jpg 0 1 1 1 0 0 0 0 0\n",
|
|
||||||
"315 20180506-172001.jpg 1 0 1 0 0 1 0 0 0\n",
|
|
||||||
"316 20180506-232001.jpg 1 0 0 0 0 0 0 0 0\n",
|
|
||||||
"317 20180510-184001.jpg 1 0 1 0 0 0 0 0 0\n",
|
|
||||||
"318 20180511-022001.jpg 0 1 0 0 0 0 0 0 0\n",
|
|
||||||
"319 20180511-062001.jpg 1 0 0 1 0 0 0 0 0\n",
|
|
||||||
"\n",
|
|
||||||
"[320 rows x 10 columns]>"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"execution_count": 131,
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "execute_result"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"refs2."
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.5.4"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 2
|
|
||||||
}
|
|
518
notebooks/test_collection.ipynb
Normal file
518
notebooks/test_collection.ipynb
Normal file
File diff suppressed because one or more lines are too long
207
notebooks/test_config.ipynb
Normal file
207
notebooks/test_config.ipynb
Normal file
|
@ -0,0 +1,207 @@
|
||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load_ext autoreload\n",
|
||||||
|
"%autoreload 2"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Collecting python-dotenv\n",
|
||||||
|
" Downloading https://files.pythonhosted.org/packages/8c/14/501508b016e7b1ad0eb91bba581e66ad9bfc7c66fcacbb580eaf9bc38458/python_dotenv-0.10.1-py2.py3-none-any.whl\n",
|
||||||
|
"Installing collected packages: python-dotenv\n",
|
||||||
|
"Successfully installed python-dotenv-0.10.1\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"!pip install python-dotenv"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"/opt/conda/bin/python\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"!which python"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 13,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from dotenv import find_dotenv, load_dotenv\n",
|
||||||
|
"import os\n",
|
||||||
|
"import yaml\n",
|
||||||
|
"from iss.tools.config import Config"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"load_dotenv(find_dotenv())"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"!export PROJECT_DIR=\"/home/jovyan/work/\""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"'dev'"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"os.getenv(\"MODE\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 8,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"'/home/jovyan/work/'"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 8,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"os.getenv(\"PROJECT_DIR\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 9,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"os.getenv(\"TEST\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 14,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"/home/jovyan/work/\n",
|
||||||
|
"/home/jovyan/work/config/config_dev.yaml\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"cfg = Config()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 15,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"1"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 15,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"cfg.get('version')"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.6.8"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
316
notebooks/test_mysql.ipynb
Normal file
316
notebooks/test_mysql.ipynb
Normal file
|
@ -0,0 +1,316 @@
|
||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"os.chdir(os.getcwd() + '/..')"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load_ext autoreload\n",
|
||||||
|
"%autoreload 2"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from iss.data.DataBaseManager import DataBaseManager"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Requirement already satisfied: MySQL-connector-python in /opt/conda/lib/python3.6/site-packages (8.0.15)\n",
|
||||||
|
"Requirement already satisfied: protobuf>=3.0.0 in /opt/conda/lib/python3.6/site-packages (from MySQL-connector-python) (3.6.1)\n",
|
||||||
|
"Requirement already satisfied: six>=1.9 in /opt/conda/lib/python3.6/site-packages (from protobuf>=3.0.0->MySQL-connector-python) (1.12.0)\n",
|
||||||
|
"Requirement already satisfied: setuptools in /opt/conda/lib/python3.6/site-packages (from protobuf>=3.0.0->MySQL-connector-python) (40.8.0)\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"!pip install MySQL-connector-python"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import mysql.connector\n",
|
||||||
|
"from iss.tools.config import Config\n",
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"import datetime as dt\n",
|
||||||
|
"import time\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"from iss.data.DataBaseManager import DataBaseManager"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"cfg = Config()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"mydb = mysql.connector.connect(\n",
|
||||||
|
" host = cfg.get('mysql')['database']['server'],\n",
|
||||||
|
" user = cfg.get('mysql')['database']['user'],\n",
|
||||||
|
" passwd = cfg.get('mysql')['database']['password'],\n",
|
||||||
|
" database = cfg.get('mysql')['database']['name'],\n",
|
||||||
|
" port = cfg.get('mysql')['database']['port']\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"history = pd.read_csv(\"./data/raw/history/history.txt\", sep=\";\", names=['latitude', 'longitude', 'id', 'location'])"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 8,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"0 2018-05-13 15:40:01\n",
|
||||||
|
"1 2018-05-13 16:00:01\n",
|
||||||
|
"2 2018-05-13 17:20:01\n",
|
||||||
|
"3 2018-05-13 17:40:01\n",
|
||||||
|
"4 2018-05-13 18:40:01\n",
|
||||||
|
"Name: id, dtype: object"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 8,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"pd.to_datetime(history.id, format=\"%Y%m%d-%H%M%S\").dt.strftime(\"%Y-%m-%d %H:%M:%S\").head()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 29,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"history['timestamp'] = pd.to_datetime(history.id, format=\"%Y%m%d-%H%M%S\").dt.strftime(\"%Y-%m-%d %H:%M:%S\")\n",
|
||||||
|
"history.fillna('NULL', inplace=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 31,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"history = history[['latitude', 'longitude', 'id', 'timestamp', 'location']]\n",
|
||||||
|
"history_tuple = [tuple(x) for x in history.values]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 32,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"[('NULL', 'NULL', '20190228-180001', '2019-02-28 18:00:01', 'NULL'),\n",
|
||||||
|
" ('NULL', 'NULL', '20190301-020001', '2019-03-01 02:00:01', 'NULL'),\n",
|
||||||
|
" (-50.906848280632,\n",
|
||||||
|
" -129.67152170575,\n",
|
||||||
|
" '20190301-040001',\n",
|
||||||
|
" '2019-03-01 04:00:01',\n",
|
||||||
|
" 'South Pacific Ocean'),\n",
|
||||||
|
" (46.224697680948,\n",
|
||||||
|
" 38.508590038953,\n",
|
||||||
|
" '20190301-080001',\n",
|
||||||
|
" '2019-03-01 08:00:01',\n",
|
||||||
|
" 'Russia'),\n",
|
||||||
|
" (48.272215894114,\n",
|
||||||
|
" -115.83481468293002,\n",
|
||||||
|
" '20190301-140001',\n",
|
||||||
|
" '2019-03-01 14:00:01',\n",
|
||||||
|
" 'United States'),\n",
|
||||||
|
" (1.6470165175667,\n",
|
||||||
|
" -29.550114288583003,\n",
|
||||||
|
" '20190301-160001',\n",
|
||||||
|
" '2019-03-01 16:00:01',\n",
|
||||||
|
" 'North Atlantic Ocean'),\n",
|
||||||
|
" (32.938297820693,\n",
|
||||||
|
" -150.40851274645001,\n",
|
||||||
|
" '20190301-220001',\n",
|
||||||
|
" '2019-03-01 22:00:01',\n",
|
||||||
|
" 'North Pacific Ocean'),\n",
|
||||||
|
" (-44.190661647858,\n",
|
||||||
|
" -100.34112806661,\n",
|
||||||
|
" '20190302-000001',\n",
|
||||||
|
" '2019-03-02 00:00:01',\n",
|
||||||
|
" 'South Pacific Ocean'),\n",
|
||||||
|
" (-16.432920558626,\n",
|
||||||
|
" 131.67746802656,\n",
|
||||||
|
" '20190302-060001',\n",
|
||||||
|
" '2019-03-02 06:00:01',\n",
|
||||||
|
" 'Australia'),\n",
|
||||||
|
" (15.985374697618001,\n",
|
||||||
|
" 13.727633100791,\n",
|
||||||
|
" '20190302-120001',\n",
|
||||||
|
" '2019-03-02 12:00:01',\n",
|
||||||
|
" 'Niger'),\n",
|
||||||
|
" (-33.152964317853,\n",
|
||||||
|
" -64.290916318981,\n",
|
||||||
|
" '20190302-200001',\n",
|
||||||
|
" '2019-03-02 20:00:01',\n",
|
||||||
|
" 'Argentina'),\n",
|
||||||
|
" (-1.9700946508908002,\n",
|
||||||
|
" 174.83004739548,\n",
|
||||||
|
" '20190303-020001',\n",
|
||||||
|
" '2019-03-03 02:00:01',\n",
|
||||||
|
" 'South Pacific Ocean'),\n",
|
||||||
|
" (30.034749186781003,\n",
|
||||||
|
" 54.744294350939995,\n",
|
||||||
|
" '20190303-080001',\n",
|
||||||
|
" '2019-03-03 08:00:01',\n",
|
||||||
|
" 'Iran'),\n",
|
||||||
|
" (-46.295462250407,\n",
|
||||||
|
" 106.72882908036999,\n",
|
||||||
|
" '20190303-100001',\n",
|
||||||
|
" '2019-03-03 10:00:01',\n",
|
||||||
|
" 'Indian Ocean'),\n",
|
||||||
|
" (50.937467506234,\n",
|
||||||
|
" -85.532585394465,\n",
|
||||||
|
" '20190303-140001',\n",
|
||||||
|
" '2019-03-03 14:00:01',\n",
|
||||||
|
" 'Canada'),\n",
|
||||||
|
" (-19.534658922649,\n",
|
||||||
|
" -23.957895905650002,\n",
|
||||||
|
" '20190303-160001',\n",
|
||||||
|
" '2019-03-03 16:00:01',\n",
|
||||||
|
" 'South Atlantic Ocean'),\n",
|
||||||
|
" (12.864013306513,\n",
|
||||||
|
" -142.02470134325,\n",
|
||||||
|
" '20190303-220001',\n",
|
||||||
|
" '2019-03-03 22:00:01',\n",
|
||||||
|
" 'North Pacific Ocean'),\n",
|
||||||
|
" (-51.795754684285,\n",
|
||||||
|
" -72.822090610609,\n",
|
||||||
|
" '20190304-000001',\n",
|
||||||
|
" '2019-03-04 00:00:01',\n",
|
||||||
|
" 'South Pacific Ocean'),\n",
|
||||||
|
" (-35.859523196283,\n",
|
||||||
|
" 141.0150238206,\n",
|
||||||
|
" '20190304-060001',\n",
|
||||||
|
" '2019-03-04 06:00:01',\n",
|
||||||
|
" 'Australia'),\n",
|
||||||
|
" (-5.2636362719691,\n",
|
||||||
|
" 19.004947687717,\n",
|
||||||
|
" '20190304-120001',\n",
|
||||||
|
" '2019-03-04 12:00:01',\n",
|
||||||
|
" 'Democratic Republic of the Congo')]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 32,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"history_tuple[3850:]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 33,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"dbm = DataBaseManager(mydb, cfg)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 34,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"dbm.createPicturesTable(force=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 35,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"3870"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 35,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"dbm.insertRowPictures(history_tuple)"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.6.8"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
Loading…
Reference in a new issue