.. DO NOT EDIT. .. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY. .. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE: .. "auto_examples/simulated/plot_classifier_comparison.py" .. LINE NUMBERS ARE GIVEN BELOW. .. only:: html .. note:: :class: sphx-glr-download-link-note :ref:`Go to the end ` to download the full example code. .. rst-class:: sphx-glr-example-title .. _sphx_glr_auto_examples_simulated_plot_classifier_comparison.py: =============================================================================== Classifier comparison =============================================================================== A comparison of several classifiers on low-dimensional synthetic datasets, adapted to SPD matrices from [1]_. The point of this example is to illustrate the nature of decision boundaries of different classifiers, used with different metrics [2]_. This should be taken with a grain of salt, as the intuition conveyed by these examples does not necessarily carry over to real datasets. The 3D plots show training matrices in solid colors and testing matrices semi-transparent. The lower right shows the classification accuracy on the test set. .. GENERATED FROM PYTHON SOURCE LINES 18-38 .. code-block:: Python # Authors: Quentin Barthélemy # # License: BSD (3-clause) from functools import partial from time import time import matplotlib.pyplot as plt from matplotlib.colors import ListedColormap import numpy as np from sklearn.model_selection import train_test_split from pyriemann.classification import ( MDM, KNearestNeighbor, SVC, ) from pyriemann.datasets import make_matrices, make_gaussian_blobs .. GENERATED FROM PYTHON SOURCE LINES 39-179 .. code-block:: Python @partial(np.vectorize, excluded=["clf"]) def get_proba(cov_00, cov_01, cov_11, clf): cov = np.array([[cov_00, cov_01], [cov_01, cov_11]]) with np.testing.suppress_warnings() as sup: sup.filter(RuntimeWarning) return clf.predict_proba(cov[np.newaxis, ...])[0, 1] def plot_classifiers(metric): figure = plt.figure(figsize=(12, 10)) figure.suptitle(f"Compare classifiers with metric='{metric}'", fontsize=16) i = 1 # iterate over datasets for ds_cnt, (X, y) in enumerate(datasets): # split dataset into training and test part X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.4, random_state=42 ) x_min, x_max = X[:, 0, 0].min(), X[:, 0, 0].max() y_min, y_max = X[:, 0, 1].min(), X[:, 0, 1].max() z_min, z_max = X[:, 1, 1].min(), X[:, 1, 1].max() # just plot the dataset first ax = plt.subplot(n_datasets, n_classifs + 1, i, projection="3d") if ds_cnt == 0: ax.set_title("Input data") # Plot the training points ax.scatter( X_train[:, 0, 0], X_train[:, 0, 1], X_train[:, 1, 1], c=y_train, cmap=cm_bright, edgecolors="k" ) # Plot the testing points ax.scatter( X_test[:, 0, 0], X_test[:, 0, 1], X_test[:, 1, 1], c=y_test, cmap=cm_bright, alpha=0.6, edgecolors="k" ) ax.set_xlim(x_min, x_max) ax.set_ylim(y_min, y_max) ax.set_zlim(z_min, z_max) ax.set_xticklabels(()) ax.set_yticklabels(()) ax.set_zticklabels(()) i += 1 rx = np.arange(x_min, x_max, (x_max - x_min) / 50) ry = np.arange(y_min, y_max, (y_max - y_min) / 50) rz = np.arange(z_min, z_max, (z_max - z_min) / 50) print(f"Dataset n°{ds_cnt+1}") # iterate over classifiers for name, clf in zip(names, classifiers): ax = plt.subplot(n_datasets, n_classifs + 1, i, projection="3d") clf.set_params(**{"metric": metric}) t0 = time() clf.fit(X_train, y_train) t1 = time() - t0 t0 = time() score = clf.score(X_test, y_test) t2 = time() - t0 print( f" {name}:\n training time={t1:.5f}\n test time ={t2:.5f}" ) # Plot the decision boundaries for horizontal 2D planes going # through the mean value of the third coordinates xx, yy = np.meshgrid(rx, ry) zz = get_proba(xx, yy, X[:, 1, 1].mean()*np.ones_like(xx), clf=clf) zz = np.ma.masked_where(~np.isfinite(zz), zz) ax.contourf(xx, yy, zz, zdir="z", offset=z_min, cmap=cm, alpha=0.5) xx, zz = np.meshgrid(rx, rz) yy = get_proba(xx, X[:, 0, 1].mean()*np.ones_like(xx), zz, clf=clf) yy = np.ma.masked_where(~np.isfinite(yy), yy) ax.contourf(xx, yy, zz, zdir="y", offset=y_max, cmap=cm, alpha=0.5) yy, zz = np.meshgrid(ry, rz) xx = get_proba(X[:, 0, 0].mean()*np.ones_like(yy), yy, zz, clf=clf) xx = np.ma.masked_where(~np.isfinite(xx), xx) ax.contourf(xx, yy, zz, zdir="x", offset=x_min, cmap=cm, alpha=0.5) # Plot the training points ax.scatter( X_train[:, 0, 0], X_train[:, 0, 1], X_train[:, 1, 1], c=y_train, cmap=cm_bright, edgecolors="k" ) # Plot the testing points ax.scatter( X_test[:, 0, 0], X_test[:, 0, 1], X_test[:, 1, 1], c=y_test, cmap=cm_bright, edgecolors="k", alpha=0.6 ) if ds_cnt == 0: ax.set_title(name) ax.text( 1.3 * x_max, y_min, z_min, ("%.2f" % score).lstrip("0"), size=15, horizontalalignment="right", verticalalignment="bottom" ) ax.set_xlim(x_min, x_max) ax.set_ylim(y_min, y_max) ax.set_zlim(z_min, z_max) ax.set_xticks(()) ax.set_yticks(()) ax.set_zticks(()) i += 1 plt.show() .. GENERATED FROM PYTHON SOURCE LINES 180-182 Classifiers and Datasets ------------------------ .. GENERATED FROM PYTHON SOURCE LINES 182-237 .. code-block:: Python names = [ "MDM", "k-NN", "SVC", ] classifiers = [ MDM(), KNearestNeighbor(n_neighbors=3), SVC(probability=True), ] n_classifs = len(classifiers) rs = np.random.RandomState(2022) n_matrices, n_channels = 50, 2 y = np.concatenate([np.zeros(n_matrices), np.ones(n_matrices)]) datasets = [ ( np.concatenate([ make_matrices( n_matrices, n_channels, "spd", rs, evals_low=10, evals_high=14 ), make_matrices( n_matrices, n_channels, "spd", rs, evals_low=13, evals_high=17 ) ]), y ), ( np.concatenate([ make_matrices( n_matrices, n_channels, "spd", rs, evals_low=10, evals_high=14 ), make_matrices( n_matrices, n_channels, "spd", rs, evals_low=11, evals_high=15 ) ]), y ), make_gaussian_blobs( 2*n_matrices, n_channels, random_state=rs, class_sep=1., class_disp=.5, n_jobs=4 ), make_gaussian_blobs( 2*n_matrices, n_channels, random_state=rs, class_sep=.5, class_disp=.5, n_jobs=4 ) ] n_datasets = len(datasets) cm = plt.cm.RdBu cm_bright = ListedColormap(["#FF0000", "#0000FF"]) .. GENERATED FROM PYTHON SOURCE LINES 238-240 Classifiers with Riemannian metric ---------------------------------- .. GENERATED FROM PYTHON SOURCE LINES 240-244 .. code-block:: Python plot_classifiers("riemann") .. image-sg:: /auto_examples/simulated/images/sphx_glr_plot_classifier_comparison_001.png :alt: Compare classifiers with metric='riemann', Input data, MDM, k-NN, SVC :srcset: /auto_examples/simulated/images/sphx_glr_plot_classifier_comparison_001.png :class: sphx-glr-single-img .. rst-class:: sphx-glr-script-out .. code-block:: none Dataset n°1 MDM: training time=0.00124 test time =0.00278 k-NN: training time=0.00005 test time =0.05967 SVC: training time=0.00219 test time =0.00100 Dataset n°2 MDM: training time=0.00137 test time =0.00257 k-NN: training time=0.00003 test time =0.06048 SVC: training time=0.00245 test time =0.00108 Dataset n°3 MDM: training time=0.00229 test time =0.00457 k-NN: training time=0.00003 test time =0.23168 SVC: training time=0.00340 test time =0.00111 Dataset n°4 MDM: training time=0.00266 test time =0.00466 k-NN: training time=0.00003 test time =0.23179 SVC: training time=0.00343 test time =0.00113 .. GENERATED FROM PYTHON SOURCE LINES 245-247 Classifiers with Log-Euclidean metric ------------------------------------- .. GENERATED FROM PYTHON SOURCE LINES 247-251 .. code-block:: Python plot_classifiers("logeuclid") .. image-sg:: /auto_examples/simulated/images/sphx_glr_plot_classifier_comparison_002.png :alt: Compare classifiers with metric='logeuclid', Input data, MDM, k-NN, SVC :srcset: /auto_examples/simulated/images/sphx_glr_plot_classifier_comparison_002.png :class: sphx-glr-single-img .. rst-class:: sphx-glr-script-out .. code-block:: none Dataset n°1 MDM: training time=0.00047 test time =0.00430 k-NN: training time=0.00004 test time =0.10280 SVC: training time=0.00141 test time =0.00100 Dataset n°2 MDM: training time=0.00038 test time =0.00397 k-NN: training time=0.00003 test time =0.10297 SVC: training time=0.00148 test time =0.00098 Dataset n°3 MDM: training time=0.00045 test time =0.00748 k-NN: training time=0.00004 test time =0.40898 SVC: training time=0.00179 test time =0.00121 Dataset n°4 MDM: training time=0.00053 test time =0.00741 k-NN: training time=0.00003 test time =0.40293 SVC: training time=0.00189 test time =0.00112 .. GENERATED FROM PYTHON SOURCE LINES 252-254 Classifiers with Euclidean metric --------------------------------- .. GENERATED FROM PYTHON SOURCE LINES 254-258 .. code-block:: Python plot_classifiers("euclid") .. image-sg:: /auto_examples/simulated/images/sphx_glr_plot_classifier_comparison_003.png :alt: Compare classifiers with metric='euclid', Input data, MDM, k-NN, SVC :srcset: /auto_examples/simulated/images/sphx_glr_plot_classifier_comparison_003.png :class: sphx-glr-single-img .. rst-class:: sphx-glr-script-out .. code-block:: none Dataset n°1 MDM: training time=0.00021 test time =0.00101 k-NN: training time=0.00004 test time =0.01392 SVC: training time=0.00096 test time =0.00082 Dataset n°2 MDM: training time=0.00020 test time =0.00096 k-NN: training time=0.00004 test time =0.01419 SVC: training time=0.00127 test time =0.00075 Dataset n°3 MDM: training time=0.00019 test time =0.00142 k-NN: training time=0.00003 test time =0.05170 SVC: training time=0.00130 test time =0.00077 Dataset n°4 MDM: training time=0.00020 test time =0.00140 k-NN: training time=0.00003 test time =0.05270 SVC: training time=0.00177 test time =0.00082 .. GENERATED FROM PYTHON SOURCE LINES 259-267 References ---------- .. [1] https://scikit-learn.org/stable/auto_examples/classification/plot_classifier_comparison.html # noqa .. [2] `Review of Riemannian distances and divergences, applied to SSVEP-based BCI `_ S. Chevallier, E. K. Kalunga, Q. Barthélemy, E. Monacelli. Neuroinformatics, Springer, 2021, 19 (1), pp.93-106 .. rst-class:: sphx-glr-timing **Total running time of the script:** (4 minutes 29.702 seconds) .. _sphx_glr_download_auto_examples_simulated_plot_classifier_comparison.py: .. only:: html .. container:: sphx-glr-footer sphx-glr-footer-example .. container:: sphx-glr-download sphx-glr-download-jupyter :download:`Download Jupyter notebook: plot_classifier_comparison.ipynb ` .. container:: sphx-glr-download sphx-glr-download-python :download:`Download Python source code: plot_classifier_comparison.py ` .. container:: sphx-glr-download sphx-glr-download-zip :download:`Download zipped: plot_classifier_comparison.zip ` .. only:: html .. rst-class:: sphx-glr-signature `Gallery generated by Sphinx-Gallery `_