diff --git a/06_classifiers_overview-part_2.ipynb b/06_classifiers_overview-part_2.ipynb
index 4e70f84e4094b51ce495520b498a3e9a7436fa82..2f229f6043c666ec5c27c3447fcda4dfb5878630 100644
--- a/06_classifiers_overview-part_2.ipynb
+++ b/06_classifiers_overview-part_2.ipynb
@@ -316,7 +316,6 @@
"from sklearn.preprocessing import StandardScaler\n",
"from sklearn.tree import DecisionTreeClassifier, plot_tree\n",
"\n",
- "\n",
"df = pd.read_csv(\"data/beers.csv\")\n",
"features = df.iloc[:, :-1]\n",
"labelv = df.iloc[:, -1]\n",
@@ -722,7 +721,13 @@
"source": [
"## Coding session\n",
"\n",
- "For the beers data compare mean cross validation accuracy, precision, recall and f1 scores for all classifiers shown so far. Try to squeeze better than default performance out of the classifiers by tuning their hyperparameters. Which ones perform best?"
+ "Apply all classifiers seen so far to the MNIST images dataset (`from sklearn.datasets import load_digits`). To that end:\n",
+ "\n",
+ "1. split your data into training and a test \"holdout\" data (`from sklearn.model_selection import train_test_split`) in 80:20 proporition; stratify the 10 target classes (`stratify=...`);\n",
+ "2. compare cross validation mean f1 scores; use the stratified k-fold CV strategy (`from sklearn.model_selection import cross_val_score, StratifiedKFold`; attention: this is a non-binary multi-class problem, you will have to use an adjusted f1 score, e.g. unweighted per class mean via `scoring=\"f1_macro\"` keyword arg - see [the `scoring` parameter predefined values](https://scikit-learn.org/stable/modules/model_evaluation.html#the-scoring-parameter-defining-model-evaluation-rules));\n",
+ "3. train the final model on the whole dataset and, use the test \"holdout\" data to repot a final model f1 performance; report also accuracy, precision and recall scores (`from sklearn.metrics import classification_report`);\n",
+ "4. next, try to manually tune hyperparameters to minimize the train test gap and to squeeze out at least 90% cross validation f1 score performance out of each classifier; try using PCA preprocessing (`sklearn.pipeline.make_pipeline`, `sklearn.decomposition.PCA`); what about data scaling? which models are most effective and easiest to tune manually?\n",
+ "5. optionally, once you get a feel of good preprocessing and classifier hyperparameters, define parameters ranges and apply a CV-based search to find optimal values for the hyperparameters (`from sklearn.model_selection import GridSearchCV, RandomizedSearchCV`)."
]
},
{
@@ -731,23 +736,50 @@
"metadata": {},
"outputs": [],
"source": [
+ "import numpy as np\n",
+ "from sklearn.datasets import load_digits\n",
+ "from sklearn.decomposition import PCA\n",
+ "\n",
+ "# classifiers\n",
"from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier\n",
"from sklearn.linear_model import LogisticRegression\n",
- "from sklearn.model_selection import cross_val_score\n",
+ "from sklearn.metrics import classification_report\n",
+ "from sklearn.model_selection import StratifiedKFold, cross_val_score, train_test_split\n",
"from sklearn.pipeline import make_pipeline\n",
- "from sklearn.preprocessing import StandardScaler\n",
"from sklearn.svm import SVC, LinearSVC\n",
"from sklearn.tree import DecisionTreeClassifier\n",
"\n",
"\n",
- "df = pd.read_csv(\"data/beers.csv\")\n",
- "features = df.iloc[:, :-1]\n",
- "labelv = df.iloc[:, -1]\n",
- "\n",
- "# ...\n",
- "# classifier = ...\n",
- "# pipeline = make_pipeline(StandardScaler(), classifier)\n",
- "# scores = cross_val_score(pipeline, features, labelv, scoring=\"f1\", cv=5)\n",
+ "digits = load_digits()\n",
+ "\n",
+ "labels = digits.target\n",
+ "n_samples = len(labels)\n",
+ "\n",
+ "# flatten images of shape N_SAMPLES x 8 x 8 to N_SAMPLES x 64:\n",
+ "print(\"digits data set shape:\", digits.images.shape)\n",
+ "features = digits.images.reshape((n_samples, -1))\n",
+ "print(\"feature matrix shape:\", features.shape)\n",
+ "\n",
+ "# (\n",
+ "# features_train,\n",
+ "# features_test,\n",
+ "# labels_train,\n",
+ "# labels_test,\n",
+ "# ) = train_test_split(..., test_size=..., stratify=..., random_state=42)\n",
+ "#\n",
+ "# classifiers = [\n",
+ "# LogisticRegression(...),\n",
+ "# LinearSVC(...),\n",
+ "# SVC(...),\n",
+ "# DecisionTreeClassifier(..., random_state=42), # rather won't do very well when not used in an ensemble method\n",
+ "# RandomForestClassifier(..., random_state=42),\n",
+ "# AdaBoostClassifier(..., random_state=42),\n",
+ "# ]\n",
+ "#\n",
+ "# pipeline = make_pipeline(...)\n",
+ "#\n",
+ "# cross_validator = StratifiedKFold(...)\n",
+ "# scores = cross_val_score(..., scoring=..., cv=...)\n",
"# ..."
]
},
@@ -762,40 +794,92 @@
"outputs": [],
"source": [
"# SOLUTION\n",
+ "import numpy as np\n",
+ "from sklearn.datasets import load_digits\n",
+ "from sklearn.decomposition import PCA\n",
+ "\n",
+ "# classifiers\n",
"from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier\n",
"from sklearn.linear_model import LogisticRegression\n",
- "from sklearn.model_selection import cross_val_score\n",
+ "from sklearn.metrics import classification_report\n",
+ "from sklearn.model_selection import StratifiedKFold, cross_val_score, train_test_split\n",
"from sklearn.pipeline import make_pipeline\n",
- "from sklearn.preprocessing import StandardScaler\n",
"from sklearn.svm import SVC, LinearSVC\n",
"from sklearn.tree import DecisionTreeClassifier\n",
"\n",
"\n",
+ "digits = load_digits()\n",
+ "\n",
+ "labels = digits.target\n",
+ "n_samples = len(labels)\n",
+ "\n",
+ "# flatten images of shape N_SAMPLES x 8 x 8 to N_SAMPLES x 64:\n",
+ "print(\"digits data set shape:\", digits.images.shape)\n",
+ "features = digits.images.reshape((n_samples, -1))\n",
+ "print(\"feature matrix shape:\", features.shape)\n",
+ "\n",
+ "(\n",
+ " features_train,\n",
+ " features_test,\n",
+ " labels_train,\n",
+ " labels_test,\n",
+ ") = train_test_split(features, labels, test_size=0.2, stratify=labels, random_state=42)\n",
+ "\n",
"classifiers = [\n",
- " LogisticRegression(C=100),\n",
- " LinearSVC(C=10, max_iter=30000),\n",
- " SVC(C=30, gamma=0.1),\n",
- " DecisionTreeClassifier(max_depth=7, random_state=0),\n",
+ " LogisticRegression(C=10, penalty=\"l1\", solver=\"liblinear\", max_iter=10_000),\n",
+ " LinearSVC(C=1, penalty=\"l1\", dual=False, max_iter=50_000),\n",
+ " SVC(C=10, gamma=0.001),\n",
+ " DecisionTreeClassifier(\n",
+ " max_depth=9, random_state=42\n",
+ " ), # rather won't do very well when not used in an ensemble method\n",
" RandomForestClassifier(\n",
- " max_depth=4,\n",
- " n_estimators=10,\n",
- " max_features=2,\n",
- " random_state=0,\n",
+ " max_depth=6,\n",
+ " n_estimators=50,\n",
+ " max_features=\"log2\",\n",
+ " random_state=42,\n",
+ " ),\n",
+ " AdaBoostClassifier(\n",
+ " base_estimator=DecisionTreeClassifier(\n",
+ " max_depth=3\n",
+ " ), # in CV search use: adaboostclassifier__base_estimator__max_depth\n",
+ " n_estimators=100,\n",
+ " algorithm=\"SAMME\", # works better than default for a multi-class problem\n",
+ " random_state=42,\n",
" ),\n",
- " AdaBoostClassifier(n_estimators=20, random_state=0),\n",
"]\n",
"\n",
- "df = pd.read_csv(\"data/beers.csv\")\n",
- "features = df.iloc[:, :-1]\n",
- "labelv = df.iloc[:, -1]\n",
+ "# We do already account for class imbalances in train/test and CV splits via stratification\n",
+ "# => take an unweighted f1 score per class mean (\"f1_macro\"), thus, keeping f1 score in between\n",
+ "# the precision and recall scores.\n",
+ "scoring = \"f1_macro\"\n",
+ "pca_n_components = features.shape[1] // 3 # accuracy-wise does not add much but speeds up training\n",
+ "cv_n_splits = 10\n",
"\n",
"for classifier in classifiers:\n",
- " print(classifier.__class__.__name__)\n",
- " pipeline = make_pipeline(StandardScaler(), classifier)\n",
- " for scoring in [\"accuracy\", \"precision\", \"recall\", \"f1\"]:\n",
- " scores = cross_val_score(pipeline, features, labelv, scoring=scoring, cv=5)\n",
- " print(f\"\\t5-fold CV mean {scoring}: {scores.mean():.2f} +/- {scores.std():.2f}\")\n",
- " print()"
+ " print()\n",
+ " print(f\"#### {classifier}\")\n",
+ " pipeline = make_pipeline(PCA(n_components=pca_n_components), classifier)\n",
+ " cross_validator = StratifiedKFold(\n",
+ " n_splits=cv_n_splits, shuffle=True, random_state=42\n",
+ " )\n",
+ " scores = cross_val_score(\n",
+ " pipeline, features_train, labels_train, scoring=scoring, cv=cross_validator\n",
+ " )\n",
+ " print()\n",
+ " print(f\"train {cv_n_splits}-fold CV mean {scoring}:\")\n",
+ " print(f\"\\t{scores.mean():.2f} +/- {scores.std():.2f}\")\n",
+ " print()\n",
+ " # the final model - train on the whole dataset\n",
+ " classifier.fit(features_train, labels_train)\n",
+ " # the final \"holdout\" test\n",
+ " print(f\"test score:\")\n",
+ " predicted_test = classifier.predict(features_test)\n",
+ " print(\n",
+ " classification_report(\n",
+ " labels_test,\n",
+ " predicted_test,\n",
+ " )\n",
+ " )"
]
},
{