Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
probml
GitHub Repository: probml/pyprobml
 Path: blob/master/internal/py_to_notebook_book1.ipynb
1191 views
Kernel: Python [conda env:jax_cpu]

Main Workflow

from time import time

init = time()

import re
import os
import sys
import json
import yaml
from functools import reduce
from collections import ChainMap
import subprocess

import pandas as pd
from glob import glob
import nbformat

import jax

Load old colab notebook names

old_nb_files = glob("../../pml-book/pml1/figure_notebooks/*")
old_nb_files[:2]
['../../pml-book/pml1/figure_notebooks/chapter10_logistic_regression_figures.ipynb', '../../pml-book/pml1/figure_notebooks/chapter4_statistics_figures.ipynb']

Parse script names from colab notebooks

new_nb_path = "../notebooks/book1/"
scripts_path = "../scripts/"

def get_fig_wise_scripts(cells):
    prev_cell, cell = cells
    scripts = re.findall("\[(\S*?\.py)\]\(http", cell["source"])

    if scripts:
        fig_num = re.findall("## Figure (.*?):", prev_cell["source"])[0]
        fig_num = ".".join([fig_num.split(".")[0].zfill(2), fig_num.split(".")[1].zfill(2)])
        return {fig_num: scripts}


def process_notebook(file_name):
    chap_num, chap_name = file_name.split("/")[-1].split(".")[0].split("_", 1)
    chap_num = chap_num.replace("chapter", "").zfill(2)
    chap_name = chap_name.replace("_figures", "")
    nb = nbformat.read(file_name, as_version=4)

    scripts = map(get_fig_wise_scripts, zip(nb["cells"], nb["cells"][1:]))
    scripts = filter(None, scripts)
    # https://stackoverflow.com/a/15714097
    scripts = reduce(lambda x, y: x.update(y) or x, scripts, {})
    return {f"{chap_num}_{chap_name}": scripts}


master_metadata = map(process_notebook, old_nb_files)
master_metadata = reduce(lambda x, y: x.update(y) or x, master_metadata, {})

scripts = list(set(jax.tree_leaves(master_metadata)))
print(f"Found {len(set(scripts))} unique scripts")

# Check appendix to see full output mapping
Found 164 unique scripts

Process the code

Ways to import modules in python

  • import foo

  • import foo as bar

  • import foo.bar

  • import foo.bar as bar

  • from foo import bar

  • from foo import *

  • from foo.bar import baz

  • from foo.bar import baz as qux

def get_module(line):
    line = line.rstrip()
    import_kw = None

    if line.lstrip().startswith("import "):
        import_kw = "import "
    elif line.lstrip().startswith("from "):
        import_kw = "from "

    if import_kw:
        module = line.lstrip()[len(import_kw) :].split(" ")[0].split(".")[0]
        return module, import_kw
    return (None, None)


def get_modules_from_script(file_name):
    try:
        with open(os.path.join(scripts_path, file_name)) as f:
            code = f.read()
        codelines = code.split("\n")
        modules = set(filter(None, map(lambda x: get_module(x)[0], codelines)))
        return modules
    except FileNotFoundError:
        print(f"{file_name} not found")

INBUILT_MODULES = [
    "__future__",
    "collections",
    "functools",
    "io",
    "itertools",
    "math",
    "os",
    "pathlib",
    "pprint",
    "random",
    "sys",
    "time",
    "timeit",
    "warnings",
    "mpl_toolkits",
]
REMOVE_MODULES = ["superimport"]
SCRIPT_MODULES = [
    "rvm_regressor",
    "gmm_lib",
    "rvm_classifier",
    "gauss_utils",
    "prefit_voting_classifier",
    "mix_bernoulli_lib",
    "fisher_lda_fit",
]
TRANSFORM_MODULES = {"PIL": "pillow", "tensorflow_probability": "tensorflow-probability", "sklearn": "scikit-learn"}
with open("../requirements.txt") as f:
    REQ_MODULES = f.read().strip().split("\n")
# TODO: Replace import pyprobml_utils with probml_utils

Corrected scripts:

  • Figure 2.5: typo_fix: changed anscobmes_quartet.py to anscombes_quartet.py

  • Figure 3.13: name_change: changed mix_ber_em_mnist.py to mix_bernoulli_em_mnist.py

  • Figure 4.17: missing: gaussInferParamsMean2d.py is not present in scripts folder (changed to gauss_infer_2d.py)

  • Figure 9.5: name_change: changed fisher_vowel_demo.py to fisher_discrim_vowel.py

all_modules = reduce(
    lambda x, y: x.union(y) or x, filter(None, map(get_modules_from_script, jax.tree_leaves(master_metadata)))
)

check_modules = all_modules - set(INBUILT_MODULES) - set(SCRIPT_MODULES) - set(REQ_MODULES) - set(REMOVE_MODULES)

for module in check_modules:
    try:
        if module in TRANSFORM_MODULES:
            module_install = TRANSFORM_MODULES[module]
        else:
            module_install = module
        exec(f"import {module}")
    except Exception as e:
        print(e)
        print(module, "failed")
No module named 'pyprobml_utils' pyprobml_utils failed 
def get_white_space(line):
    space = 0
    while line[0] == " ":
        line = line[1:]
        space += 1
    return space * " "


def convert_py_to_ipynb(file_name, chapter, fig_num, prev=""):
    chap_num, _ = chapter.split("_", 1)
    current_modules = set()
    new_lines = []
    notebook = nbformat.v4.new_notebook()

    with open(os.path.join(scripts_path, file_name)) as f:
        code = f.read().strip()
    codelines = code.split("\n")
    for line in codelines:
        # Ignore superimport
        if line.strip().startswith("import superimport"):
            continue

        # consistently use savefig only
        line = line.replace("save_fig", "savefig")

        # change folder path
        line = line.replace("../figures", "figures")

        # Change pyprobml_utils to probml_utils
        if "pyprobml_utils" in line:
            line = line.replace("pyprobml_utils", "probml_utils")
            current_modules.add("probml_utils")

        # Check if the line is an import command
        module, import_kw = get_module(line)
        if module:
            if module in SCRIPT_MODULES:
                if import_kw == "import ":
                    if " as " in line:
                        line = line.replace(f"{module}", f"probml_utils.{module}", 1)
                    else:
                        line = line.replace(f"{module}", f"probml_utils.{module} as {module}", 1)
                elif import_kw == "from ":
                    line = line.replace(f"{module}", f"probml_utils.{module}", 1)
                else:
                    raise NameError()
            elif module not in INBUILT_MODULES + REQ_MODULES + list(current_modules):
                current_modules.add(module)
                module_install = TRANSFORM_MODULES[module] if module in TRANSFORM_MODULES else module
                space = get_white_space(line)
                line = f"{space}try:\n    {space}{line}\n{space}except ModuleNotFoundError:\n    {space}%pip install {module_install}\n    {space}{line}"

        new_lines.append(line)
    new_code = "\n".join(new_lines) + "\n"
    if len(prev) == 0:
        notebook["cells"].append(nbformat.v4.new_code_cell(new_code))
    else:
        notebook["cells"].append(nbformat.v4.new_markdown_cell(prev))

    save_path = f"../notebooks/book1/{chap_num}"
    if not os.path.exists(save_path):
        os.makedirs(save_path)
    nbformat.write(notebook, os.path.join(save_path, f"{file_name.replace('.py', '.ipynb')}"))
    print(f"{file_name} saved")

Convert

global_store = []
global_chap = []
repo_path = "https://github.com/probml/pyprobml/tree/master/notebooks/book1"
for chapter in sorted(master_metadata):
    local_store = []
    for fig_num, script_names in master_metadata[chapter].items():
        for script_name in script_names:
            print(f"Processing: chapter {chapter}, figure {fig_num}, script_name {script_name}")
            prev = ""
            if script_name in global_store:
                if script_name not in local_store:
                    idx = global_store.index(script_name)
                    chap_num = global_chap[idx]
                    prev = f"Source of this notebook is here:  {repo_path}/{chap_num}/{script_name.replace('.py', '.ipynb')}"
                    print("##### PREV triggered. duplicate of", chap_num, script_name)
            else:
                global_store.append(script_name)
                global_chap.append(chapter.split("_", 1)[0])
                local_store.append(script_name)

            convert_py_to_ipynb(script_name, chapter, fig_num, prev)
Processing: chapter 01_introduction, figure 01.03, script_name iris_plot.py iris_plot.py saved Processing: chapter 01_introduction, figure 01.05, script_name linreg_residuals_plot.py linreg_residuals_plot.py saved Processing: chapter 01_introduction, figure 01.06, script_name linreg_2d_surface_demo.py linreg_2d_surface_demo.py saved Processing: chapter 01_introduction, figure 01.07, script_name linreg_poly_vs_degree.py linreg_poly_vs_degree.py saved Processing: chapter 01_introduction, figure 01.08, script_name iris_kmeans.py iris_kmeans.py saved Processing: chapter 01_introduction, figure 01.09, script_name iris_pca.py iris_pca.py saved Processing: chapter 01_introduction, figure 01.12, script_name mnist_viz_tf.py mnist_viz_tf.py saved Processing: chapter 01_introduction, figure 01.12, script_name emnist_viz_pytorch.py emnist_viz_pytorch.py saved Processing: chapter 01_introduction, figure 01.13, script_name fashion_viz_tf.py fashion_viz_tf.py saved Processing: chapter 01_introduction, figure 01.13, script_name cifar_viz_tf.py cifar_viz_tf.py saved Processing: chapter 02_probability_univariate_models, figure 02.01, script_name discrete_prob_dist_plot.py discrete_prob_dist_plot.py saved Processing: chapter 02_probability_univariate_models, figure 02.02, script_name gauss_plot.py gauss_plot.py saved Processing: chapter 02_probability_univariate_models, figure 02.02, script_name quantile_plot.py quantile_plot.py saved Processing: chapter 02_probability_univariate_models, figure 02.04, script_name bimodal_dist_plot.py bimodal_dist_plot.py saved Processing: chapter 02_probability_univariate_models, figure 02.05, script_name anscombes_quartet.py anscombes_quartet.py saved Processing: chapter 02_probability_univariate_models, figure 02.06, script_name datasaurus_dozen.py datasaurus_dozen.py saved Processing: chapter 02_probability_univariate_models, figure 02.09, script_name binom_dist_plot.py binom_dist_plot.py saved Processing: chapter 02_probability_univariate_models, figure 02.10, script_name activation_fun_plot.py activation_fun_plot.py saved Processing: chapter 02_probability_univariate_models, figure 02.11, script_name iris_logreg.py iris_logreg.py saved Processing: chapter 02_probability_univariate_models, figure 02.12, script_name softmax_plot.py softmax_plot.py saved Processing: chapter 02_probability_univariate_models, figure 02.13, script_name iris_logreg.py iris_logreg.py saved Processing: chapter 02_probability_univariate_models, figure 02.14, script_name linreg_1d_hetero_tfp.py linreg_1d_hetero_tfp.py saved Processing: chapter 02_probability_univariate_models, figure 02.15, script_name student_laplace_pdf_plot.py student_laplace_pdf_plot.py saved Processing: chapter 02_probability_univariate_models, figure 02.16, script_name robust_pdf_plot.py robust_pdf_plot.py saved Processing: chapter 02_probability_univariate_models, figure 02.17, script_name beta_dist_plot.py beta_dist_plot.py saved Processing: chapter 02_probability_univariate_models, figure 02.17, script_name gamma_dist_plot.py gamma_dist_plot.py saved Processing: chapter 02_probability_univariate_models, figure 02.23, script_name centralLimitDemo.py centralLimitDemo.py saved Processing: chapter 02_probability_univariate_models, figure 02.24, script_name change_of_vars_demo1d.py change_of_vars_demo1d.py saved Processing: chapter 03_probability_multivariate_models, figure 03.05, script_name gauss_plot_2d.py gauss_plot_2d.py saved Processing: chapter 03_probability_multivariate_models, figure 03.06, script_name gauss_plot_2d.py gauss_plot_2d.py saved Processing: chapter 03_probability_multivariate_models, figure 03.07, script_name gauss_imputation_known_params_demo.py gauss_imputation_known_params_demo.py saved Processing: chapter 03_probability_multivariate_models, figure 03.08, script_name gauss_infer_1d.py gauss_infer_1d.py saved Processing: chapter 03_probability_multivariate_models, figure 03.09, script_name gauss_infer_2d.py gauss_infer_2d.py saved Processing: chapter 03_probability_multivariate_models, figure 03.10, script_name sensor_fusion_2d.py sensor_fusion_2d.py saved Processing: chapter 03_probability_multivariate_models, figure 03.11, script_name gmm_plot_demo.py gmm_plot_demo.py saved Processing: chapter 03_probability_multivariate_models, figure 03.12, script_name gmm_2d.py gmm_2d.py saved Processing: chapter 03_probability_multivariate_models, figure 03.13, script_name mix_bernoulli_em_mnist.py mix_bernoulli_em_mnist.py saved Processing: chapter 04_statistics, figure 04.01, script_name iris_cov_mat.py iris_cov_mat.py saved Processing: chapter 04_statistics, figure 04.02, script_name hinge_loss_plot.py hinge_loss_plot.py saved Processing: chapter 04_statistics, figure 04.03, script_name ema_demo.py ema_demo.py saved Processing: chapter 04_statistics, figure 04.04, script_name shrinkcov_plots.py shrinkcov_plots.py saved Processing: chapter 04_statistics, figure 04.05, script_name linreg_poly_ridge.py linreg_poly_ridge.py saved Processing: chapter 04_statistics, figure 04.07, script_name polyfitRidgeCV.py polyfitRidgeCV.py saved Processing: chapter 04_statistics, figure 04.08, script_name imdb_mlp_bow_tf.py imdb_mlp_bow_tf.py saved Processing: chapter 04_statistics, figure 04.09, script_name linreg_poly_vs_n.py linreg_poly_vs_n.py saved Processing: chapter 04_statistics, figure 04.10, script_name beta_binom_post_plot.py beta_binom_post_plot.py saved Processing: chapter 04_statistics, figure 04.12, script_name beta_binom_post_pred_plot.py beta_binom_post_pred_plot.py saved Processing: chapter 04_statistics, figure 04.13, script_name mixbetademo.py mixbetademo.py saved Processing: chapter 04_statistics, figure 04.14, script_name dirichlet_3d_triangle_plot.py dirichlet_3d_triangle_plot.py saved Processing: chapter 04_statistics, figure 04.14, script_name dirichlet_3d_spiky_plot.py dirichlet_3d_spiky_plot.py saved Processing: chapter 04_statistics, figure 04.15, script_name dirichlet_samples_plot.py dirichlet_samples_plot.py saved Processing: chapter 04_statistics, figure 04.16, script_name gauss_infer_1d.py ##### PREV triggered. duplicate of 03 gauss_infer_1d.py gauss_infer_1d.py saved Processing: chapter 04_statistics, figure 04.17, script_name gauss_infer_2d.py ##### PREV triggered. duplicate of 03 gauss_infer_2d.py gauss_infer_2d.py saved Processing: chapter 04_statistics, figure 04.18, script_name betaHPD.py betaHPD.py saved Processing: chapter 04_statistics, figure 04.19, script_name postDensityIntervals.py postDensityIntervals.py saved Processing: chapter 04_statistics, figure 04.20, script_name logreg_iris_1d.py logreg_iris_1d.py saved Processing: chapter 04_statistics, figure 04.20, script_name logreg_iris_bayes_1d_pymc3.py logreg_iris_bayes_1d_pymc3.py saved Processing: chapter 04_statistics, figure 04.22, script_name beta_binom_approx_post_pymc3.py beta_binom_approx_post_pymc3.py saved Processing: chapter 04_statistics, figure 04.23, script_name bootstrapDemoBer.py bootstrapDemoBer.py saved Processing: chapter 04_statistics, figure 04.24, script_name samplingDistributionGaussianShrinkage.py samplingDistributionGaussianShrinkage.py saved Processing: chapter 04_statistics, figure 04.25, script_name biasVarModelComplexity3.py biasVarModelComplexity3.py saved Processing: chapter 05_decision_theory, figure 05.02, script_name roc_plot.py roc_plot.py saved Processing: chapter 05_decision_theory, figure 05.02, script_name pr_plot.py pr_plot.py saved Processing: chapter 05_decision_theory, figure 05.03, script_name huberLossPlot.py huberLossPlot.py saved Processing: chapter 05_decision_theory, figure 05.04, script_name coins_model_sel_demo.py coins_model_sel_demo.py saved Processing: chapter 05_decision_theory, figure 05.05, script_name linreg_eb_modelsel_vs_n.py linreg_eb_modelsel_vs_n.py saved Processing: chapter 05_decision_theory, figure 05.06, script_name linreg_eb_modelsel_vs_n.py linreg_eb_modelsel_vs_n.py saved Processing: chapter 05_decision_theory, figure 05.08, script_name riskFnGauss.py riskFnGauss.py saved Processing: chapter 05_decision_theory, figure 05.10, script_name neymanPearson2.py neymanPearson2.py saved Processing: chapter 05_decision_theory, figure 05.10, script_name twoPowerCurves.py twoPowerCurves.py saved Processing: chapter 06_information_theory, figure 06.01, script_name bernoulli_entropy_fig.py bernoulli_entropy_fig.py saved Processing: chapter 06_information_theory, figure 06.02, script_name seq_logo_demo.py seq_logo_demo.py saved Processing: chapter 06_information_theory, figure 06.03, script_name KLfwdReverseMixGauss.py KLfwdReverseMixGauss.py saved Processing: chapter 07_linear_algebra, figure 07.06, script_name gaussEvec.py gaussEvec.py saved Processing: chapter 07_linear_algebra, figure 07.07, script_name height_weight_whiten_plot.py height_weight_whiten_plot.py saved Processing: chapter 07_linear_algebra, figure 07.09, script_name svd_image_demo.py svd_image_demo.py saved Processing: chapter 07_linear_algebra, figure 07.10, script_name svd_image_demo.py svd_image_demo.py saved Processing: chapter 08_optimization, figure 08.01, script_name extrema_fig_1d.py extrema_fig_1d.py saved Processing: chapter 08_optimization, figure 08.01, script_name saddle.py saddle.py saved Processing: chapter 08_optimization, figure 08.07, script_name smooth-vs-nonsmooth-1d.py smooth-vs-nonsmooth-1d.py saved Processing: chapter 08_optimization, figure 08.11, script_name steepestDescentDemo.py steepestDescentDemo.py saved Processing: chapter 08_optimization, figure 08.12, script_name lineSearchConditionNum.py lineSearchConditionNum.py saved Processing: chapter 08_optimization, figure 08.14, script_name newtonsMethodMinQuad.py newtonsMethodMinQuad.py saved Processing: chapter 08_optimization, figure 08.14, script_name newtonsMethodNonConvex.py newtonsMethodNonConvex.py saved Processing: chapter 08_optimization, figure 08.16, script_name lms_demo.py lms_demo.py saved Processing: chapter 08_optimization, figure 08.18, script_name learning_rate_plot.py learning_rate_plot.py saved Processing: chapter 08_optimization, figure 08.23, script_name emLogLikelihoodMax.py emLogLikelihoodMax.py saved Processing: chapter 08_optimization, figure 08.25, script_name mix_gauss_demo_faithful.py mix_gauss_demo_faithful.py saved Processing: chapter 08_optimization, figure 08.26, script_name mix_gauss_singularity.py mix_gauss_singularity.py saved Processing: chapter 08_optimization, figure 08.26, script_name mix_gauss_mle_vs_map.py mix_gauss_mle_vs_map.py saved Processing: chapter 08_optimization, figure 08.27, script_name gmm_lik_surface_plot.py gmm_lik_surface_plot.py saved Processing: chapter 09_linear_discriminant_analysis, figure 09.01, script_name discrim_analysis_dboundaries_plot2.py discrim_analysis_dboundaries_plot2.py saved Processing: chapter 09_linear_discriminant_analysis, figure 09.02, script_name discrim_analysis_dboundaries_plot2.py discrim_analysis_dboundaries_plot2.py saved Processing: chapter 09_linear_discriminant_analysis, figure 09.04, script_name fisher_lda_demo.py fisher_lda_demo.py saved Processing: chapter 09_linear_discriminant_analysis, figure 09.05, script_name fisher_discrim_vowel.py fisher_discrim_vowel.py saved Processing: chapter 09_linear_discriminant_analysis, figure 09.08, script_name generativeVsDiscrim.py generativeVsDiscrim.py saved Processing: chapter 10_logistic_regression, figure 10.01, script_name iris_logreg.py ##### PREV triggered. duplicate of 02 iris_logreg.py iris_logreg.py saved Processing: chapter 10_logistic_regression, figure 10.02, script_name sigmoid_2d_plot.py sigmoid_2d_plot.py saved Processing: chapter 10_logistic_regression, figure 10.04, script_name logreg_poly_demo.py logreg_poly_demo.py saved Processing: chapter 10_logistic_regression, figure 10.05, script_name iris_logreg_loss_surface.py iris_logreg_loss_surface.py saved Processing: chapter 10_logistic_regression, figure 10.06, script_name logreg_poly_demo.py logreg_poly_demo.py saved Processing: chapter 10_logistic_regression, figure 10.07, script_name logreg_multiclass_demo.py logreg_multiclass_demo.py saved Processing: chapter 10_logistic_regression, figure 10.10, script_name logreg_iris_bayes_robust_1d_pymc3.py logreg_iris_bayes_robust_1d_pymc3.py saved Processing: chapter 10_logistic_regression, figure 10.13, script_name logreg_laplace_demo.py logreg_laplace_demo.py saved Processing: chapter 10_logistic_regression, figure 10.14, script_name logreg_laplace_demo.py logreg_laplace_demo.py saved Processing: chapter 11_linear_regression, figure 11.01, script_name linreg_poly_vs_degree.py ##### PREV triggered. duplicate of 01 linreg_poly_vs_degree.py linreg_poly_vs_degree.py saved Processing: chapter 11_linear_regression, figure 11.02, script_name linreg_contours_sse_plot.py linreg_contours_sse_plot.py saved Processing: chapter 11_linear_regression, figure 11.04, script_name linregOnlineDemo.py linregOnlineDemo.py saved Processing: chapter 11_linear_regression, figure 11.05, script_name linreg_poly_vs_degree.py ##### PREV triggered. duplicate of 01 linreg_poly_vs_degree.py linreg_poly_vs_degree.py saved Processing: chapter 11_linear_regression, figure 11.06, script_name linreg_poly_vs_degree.py ##### PREV triggered. duplicate of 01 linreg_poly_vs_degree.py linreg_poly_vs_degree.py saved Processing: chapter 11_linear_regression, figure 11.07, script_name geom_ridge.py geom_ridge.py saved Processing: chapter 11_linear_regression, figure 11.10, script_name ridgePathProstate.py ridgePathProstate.py saved Processing: chapter 11_linear_regression, figure 11.10, script_name lassoPathProstate.py lassoPathProstate.py saved Processing: chapter 11_linear_regression, figure 11.11, script_name prostate_comparison.py prostate_comparison.py saved Processing: chapter 11_linear_regression, figure 11.12, script_name prostate_comparison.py prostate_comparison.py saved Processing: chapter 11_linear_regression, figure 11.13, script_name sparse_sensing_demo.py sparse_sensing_demo.py saved Processing: chapter 11_linear_regression, figure 11.14, script_name groupLassoDemo.py groupLassoDemo.py saved Processing: chapter 11_linear_regression, figure 11.15, script_name groupLassoDemo.py groupLassoDemo.py saved Processing: chapter 11_linear_regression, figure 11.16, script_name splines_basis_weighted.py splines_basis_weighted.py saved Processing: chapter 11_linear_regression, figure 11.17, script_name splines_basis_heatmap.py splines_basis_heatmap.py saved Processing: chapter 11_linear_regression, figure 11.18, script_name splines_cherry_blossoms.py splines_cherry_blossoms.py saved Processing: chapter 11_linear_regression, figure 11.19, script_name linregRobustDemoCombined.py linregRobustDemoCombined.py saved Processing: chapter 11_linear_regression, figure 11.19, script_name huberLossPlot.py ##### PREV triggered. duplicate of 05 huberLossPlot.py huberLossPlot.py saved Processing: chapter 11_linear_regression, figure 11.20, script_name linreg_2d_bayes_demo.py linreg_2d_bayes_demo.py saved Processing: chapter 11_linear_regression, figure 11.21, script_name linreg_post_pred_plot.py linreg_post_pred_plot.py saved Processing: chapter 11_linear_regression, figure 11.22, script_name linreg_2d_bayes_centering_pymc3.py linreg_2d_bayes_centering_pymc3.py saved Processing: chapter 11_linear_regression, figure 11.23, script_name multi_collinear_legs_numpyro.py multi_collinear_legs_numpyro.py saved Processing: chapter 11_linear_regression, figure 11.24, script_name multi_collinear_legs_numpyro.py multi_collinear_legs_numpyro.py saved Processing: chapter 13_neural_networks_for_structured_data, figure 13.01, script_name xor_heaviside.py xor_heaviside.py saved Processing: chapter 13_neural_networks_for_structured_data, figure 13.02, script_name activation_fun_plot.py ##### PREV triggered. duplicate of 02 activation_fun_plot.py activation_fun_plot.py saved Processing: chapter 13_neural_networks_for_structured_data, figure 13.21, script_name logregXorDemo.py logregXorDemo.py saved Processing: chapter 13_neural_networks_for_structured_data, figure 13.22, script_name linregRbfDemo.py linregRbfDemo.py saved Processing: chapter 13_neural_networks_for_structured_data, figure 13.23, script_name mixexpDemoOneToMany.py mixexpDemoOneToMany.py saved Processing: chapter 16_exemplar-based_methods, figure 16.01, script_name knn_voronoi_plot.py knn_voronoi_plot.py saved Processing: chapter 16_exemplar-based_methods, figure 16.02, script_name knn_classify_demo.py knn_classify_demo.py saved Processing: chapter 16_exemplar-based_methods, figure 16.03, script_name curse_dimensionality_plot.py curse_dimensionality_plot.py saved Processing: chapter 16_exemplar-based_methods, figure 16.08, script_name smoothingKernelPlot.py smoothingKernelPlot.py saved Processing: chapter 16_exemplar-based_methods, figure 16.09, script_name parzen_window_demo2.py parzen_window_demo2.py saved Processing: chapter 16_exemplar-based_methods, figure 16.10, script_name kernelRegressionDemo.py kernelRegressionDemo.py saved Processing: chapter 17_kernel_methods, figure 17.01, script_name gprDemoArd.py gprDemoArd.py saved Processing: chapter 17_kernel_methods, figure 17.02, script_name gpKernelPlot.py gpKernelPlot.py saved Processing: chapter 17_kernel_methods, figure 17.03, script_name gpKernelPlot.py gpKernelPlot.py saved Processing: chapter 17_kernel_methods, figure 17.07, script_name gprDemoNoiseFree.py gprDemoNoiseFree.py saved Processing: chapter 17_kernel_methods, figure 17.08, script_name gprDemoChangeHparams.py gprDemoChangeHparams.py saved Processing: chapter 17_kernel_methods, figure 17.09, script_name gpr_demo_marglik.py gpr_demo_marglik.py saved Processing: chapter 17_kernel_methods, figure 17.10, script_name gp_classify_iris_1d_pymc3.py gp_classify_iris_1d_pymc3.py saved Processing: chapter 17_kernel_methods, figure 17.11, script_name gp_classify_spaceflu_1d_pymc3.py gp_classify_spaceflu_1d_pymc3.py saved Processing: chapter 17_kernel_methods, figure 17.14, script_name svm_classifier_feature_scaling.py svm_classifier_feature_scaling.py saved Processing: chapter 17_kernel_methods, figure 17.17, script_name svm_classifier_2d.py svm_classifier_2d.py saved Processing: chapter 17_kernel_methods, figure 17.18, script_name svmCgammaDemo.py svmCgammaDemo.py saved Processing: chapter 17_kernel_methods, figure 17.19, script_name huberLossPlot.py ##### PREV triggered. duplicate of 05 huberLossPlot.py huberLossPlot.py saved Processing: chapter 17_kernel_methods, figure 17.20, script_name svm_regression_1d.py svm_regression_1d.py saved Processing: chapter 17_kernel_methods, figure 17.21, script_name kernelBinaryClassifDemo.py kernelBinaryClassifDemo.py saved Processing: chapter 17_kernel_methods, figure 17.22, script_name rvm_regression_1d.py rvm_regression_1d.py saved Processing: chapter 17_kernel_methods, figure 17.23, script_name rvm_regression_1d.py rvm_regression_1d.py saved Processing: chapter 18_trees_forests_bagging_and_boosting, figure 18.01, script_name regtreeSurfaceDemo.py regtreeSurfaceDemo.py saved Processing: chapter 18_trees_forests_bagging_and_boosting, figure 18.03, script_name dtree_sensitivity.py dtree_sensitivity.py saved Processing: chapter 18_trees_forests_bagging_and_boosting, figure 18.04, script_name bagging_trees.py bagging_trees.py saved Processing: chapter 18_trees_forests_bagging_and_boosting, figure 18.04, script_name rf_demo_2d.py rf_demo_2d.py saved Processing: chapter 18_trees_forests_bagging_and_boosting, figure 18.05, script_name spam_tree_ensemble_compare.py spam_tree_ensemble_compare.py saved Processing: chapter 18_trees_forests_bagging_and_boosting, figure 18.06, script_name boosted_regr_trees.py boosted_regr_trees.py saved Processing: chapter 18_trees_forests_bagging_and_boosting, figure 18.07, script_name hinge_loss_plot.py ##### PREV triggered. duplicate of 04 hinge_loss_plot.py hinge_loss_plot.py saved Processing: chapter 18_trees_forests_bagging_and_boosting, figure 18.08, script_name rf_feature_importance_mnist.py rf_feature_importance_mnist.py saved Processing: chapter 18_trees_forests_bagging_and_boosting, figure 18.09, script_name spam_tree_ensemble_interpret.py spam_tree_ensemble_interpret.py saved Processing: chapter 18_trees_forests_bagging_and_boosting, figure 18.10, script_name spam_tree_ensemble_interpret.py spam_tree_ensemble_interpret.py saved Processing: chapter 20_dimensionality_reduction, figure 20.01, script_name pcaDemo2d.py pcaDemo2d.py saved Processing: chapter 20_dimensionality_reduction, figure 20.02, script_name pca_digits.py pca_digits.py saved Processing: chapter 20_dimensionality_reduction, figure 20.03, script_name pcaImageDemo.py pcaImageDemo.py saved Processing: chapter 20_dimensionality_reduction, figure 20.04, script_name pca_projected_variance.py pca_projected_variance.py saved Processing: chapter 20_dimensionality_reduction, figure 20.05, script_name pcaStandardization.py pcaStandardization.py saved Processing: chapter 20_dimensionality_reduction, figure 20.06, script_name pcaOverfitDemo.py pcaOverfitDemo.py saved Processing: chapter 20_dimensionality_reduction, figure 20.07, script_name pcaOverfitDemo.py pcaOverfitDemo.py saved Processing: chapter 20_dimensionality_reduction, figure 20.08, script_name pcaOverfitDemo.py pcaOverfitDemo.py saved Processing: chapter 20_dimensionality_reduction, figure 20.10, script_name pcaEmStepByStep.py pcaEmStepByStep.py saved Processing: chapter 20_dimensionality_reduction, figure 20.12, script_name mixPpcaDemo.py mixPpcaDemo.py saved Processing: chapter 20_dimensionality_reduction, figure 20.13, script_name binary_fa_demo.py binary_fa_demo.py saved Processing: chapter 20_dimensionality_reduction, figure 20.30, script_name manifold_swiss_sklearn.py manifold_swiss_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.30, script_name manifold_digits_sklearn.py manifold_digits_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.31, script_name manifold_swiss_sklearn.py manifold_swiss_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.31, script_name manifold_digits_sklearn.py manifold_digits_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.33, script_name manifold_swiss_sklearn.py manifold_swiss_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.33, script_name manifold_digits_sklearn.py manifold_digits_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.34, script_name manifold_swiss_sklearn.py manifold_swiss_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.35, script_name kpcaScholkopf.py kpcaScholkopf.py saved Processing: chapter 20_dimensionality_reduction, figure 20.36, script_name manifold_swiss_sklearn.py manifold_swiss_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.36, script_name manifold_digits_sklearn.py manifold_digits_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.37, script_name manifold_swiss_sklearn.py manifold_swiss_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.37, script_name manifold_digits_sklearn.py manifold_digits_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.38, script_name manifold_swiss_sklearn.py manifold_swiss_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.38, script_name manifold_digits_sklearn.py manifold_digits_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.41, script_name manifold_swiss_sklearn.py manifold_swiss_sklearn.py saved Processing: chapter 20_dimensionality_reduction, figure 20.41, script_name manifold_digits_sklearn.py manifold_digits_sklearn.py saved Processing: chapter 21_clustering, figure 21.02, script_name agglomDemo.py agglomDemo.py saved Processing: chapter 21_clustering, figure 21.04, script_name hclust_yeast_demo.py hclust_yeast_demo.py saved Processing: chapter 21_clustering, figure 21.05, script_name yeast_data_viz.py yeast_data_viz.py saved Processing: chapter 21_clustering, figure 21.06, script_name hclust_yeast_demo.py hclust_yeast_demo.py saved Processing: chapter 21_clustering, figure 21.07, script_name kmeans_voronoi.py kmeans_voronoi.py saved Processing: chapter 21_clustering, figure 21.08, script_name kmeans_yeast_demo.py kmeans_yeast_demo.py saved Processing: chapter 21_clustering, figure 21.09, script_name vqDemo.py vqDemo.py saved Processing: chapter 21_clustering, figure 21.10, script_name kmeans_minibatch.py kmeans_minibatch.py saved Processing: chapter 21_clustering, figure 21.11, script_name kmeans_silhouette.py kmeans_silhouette.py saved Processing: chapter 21_clustering, figure 21.11, script_name gmm_2d.py ##### PREV triggered. duplicate of 03 gmm_2d.py gmm_2d.py saved Processing: chapter 21_clustering, figure 21.11, script_name kmeans_silhouette.py kmeans_silhouette.py saved Processing: chapter 21_clustering, figure 21.12, script_name kmeans_silhouette.py kmeans_silhouette.py saved Processing: chapter 21_clustering, figure 21.13, script_name kmeans_silhouette.py kmeans_silhouette.py saved Processing: chapter 21_clustering, figure 21.14, script_name gmm_2d.py ##### PREV triggered. duplicate of 03 gmm_2d.py gmm_2d.py saved Processing: chapter 21_clustering, figure 21.15, script_name gmm_identifiability_pymc3.py gmm_identifiability_pymc3.py saved Processing: chapter 21_clustering, figure 21.16, script_name gmm_identifiability_pymc3.py gmm_identifiability_pymc3.py saved Processing: chapter 21_clustering, figure 21.19, script_name spectral_clustering_demo.py spectral_clustering_demo.py saved 
print("Total notebooks:", len(glob("../notebooks/book1/*/*.ipynb")))
Total notebooks: 174

Save metadata

pd.to_pickle(master_metadata, "metadata_book1.pkl")

print("Everything is done in", time() - init, "seconds")
Everything is done in 8.67478609085083 seconds

Appendix

Chapter wise figure number map with scripts

def print_names(key):
    print(f"Chapter_{key}")
    print(yaml.dump(master_metadata[key]))


list(map(print_names, sorted(master_metadata)));
Chapter_01_introduction '01.03': - iris_plot.py '01.05': - linreg_residuals_plot.py '01.06': - linreg_2d_surface_demo.py '01.07': - linreg_poly_vs_degree.py '01.08': - iris_kmeans.py '01.09': - iris_pca.py '01.12': - mnist_viz_tf.py - emnist_viz_pytorch.py '01.13': - fashion_viz_tf.py - cifar_viz_tf.py Chapter_02_probability_univariate_models '02.01': - discrete_prob_dist_plot.py '02.02': - gauss_plot.py - quantile_plot.py '02.04': - bimodal_dist_plot.py '02.05': - anscombes_quartet.py '02.06': - datasaurus_dozen.py '02.09': - binom_dist_plot.py '02.10': - activation_fun_plot.py '02.11': - iris_logreg.py '02.12': - softmax_plot.py '02.13': - iris_logreg.py '02.14': - linreg_1d_hetero_tfp.py '02.15': - student_laplace_pdf_plot.py '02.16': - robust_pdf_plot.py '02.17': - beta_dist_plot.py - gamma_dist_plot.py '02.23': - centralLimitDemo.py '02.24': - change_of_vars_demo1d.py Chapter_03_probability_multivariate_models '03.05': - gauss_plot_2d.py '03.06': - gauss_plot_2d.py '03.07': - gauss_imputation_known_params_demo.py '03.08': - gauss_infer_1d.py '03.09': - gauss_infer_2d.py '03.10': - sensor_fusion_2d.py '03.11': - gmm_plot_demo.py '03.12': - gmm_2d.py '03.13': - mix_bernoulli_em_mnist.py Chapter_04_statistics '04.01': - iris_cov_mat.py '04.02': - hinge_loss_plot.py '04.03': - ema_demo.py '04.04': - shrinkcov_plots.py '04.05': - linreg_poly_ridge.py '04.07': - polyfitRidgeCV.py '04.08': - imdb_mlp_bow_tf.py '04.09': - linreg_poly_vs_n.py '04.10': - beta_binom_post_plot.py '04.12': - beta_binom_post_pred_plot.py '04.13': - mixbetademo.py '04.14': - dirichlet_3d_triangle_plot.py - dirichlet_3d_spiky_plot.py '04.15': - dirichlet_samples_plot.py '04.16': - gauss_infer_1d.py '04.17': - gauss_infer_2d.py '04.18': - betaHPD.py '04.19': - postDensityIntervals.py '04.20': - logreg_iris_1d.py - logreg_iris_bayes_1d_pymc3.py '04.22': - beta_binom_approx_post_pymc3.py '04.23': - bootstrapDemoBer.py '04.24': - samplingDistributionGaussianShrinkage.py '04.25': - biasVarModelComplexity3.py Chapter_05_decision_theory '05.02': - roc_plot.py - pr_plot.py '05.03': - huberLossPlot.py '05.04': - coins_model_sel_demo.py '05.05': - linreg_eb_modelsel_vs_n.py '05.06': - linreg_eb_modelsel_vs_n.py '05.08': - riskFnGauss.py '05.10': - neymanPearson2.py - twoPowerCurves.py Chapter_06_information_theory '06.01': - bernoulli_entropy_fig.py '06.02': - seq_logo_demo.py '06.03': - KLfwdReverseMixGauss.py Chapter_07_linear_algebra '07.06': - gaussEvec.py '07.07': - height_weight_whiten_plot.py '07.09': - svd_image_demo.py '07.10': - svd_image_demo.py Chapter_08_optimization '08.01': - extrema_fig_1d.py - saddle.py '08.07': - smooth-vs-nonsmooth-1d.py '08.11': - steepestDescentDemo.py '08.12': - lineSearchConditionNum.py '08.14': - newtonsMethodMinQuad.py - newtonsMethodNonConvex.py '08.16': - lms_demo.py '08.18': - learning_rate_plot.py '08.23': - emLogLikelihoodMax.py '08.25': - mix_gauss_demo_faithful.py '08.26': - mix_gauss_singularity.py - mix_gauss_mle_vs_map.py '08.27': - gmm_lik_surface_plot.py Chapter_09_linear_discriminant_analysis '09.01': - discrim_analysis_dboundaries_plot2.py '09.02': - discrim_analysis_dboundaries_plot2.py '09.04': - fisher_lda_demo.py '09.05': - fisher_discrim_vowel.py '09.08': - generativeVsDiscrim.py Chapter_10_logistic_regression '10.01': - iris_logreg.py '10.02': - sigmoid_2d_plot.py '10.04': - logreg_poly_demo.py '10.05': - iris_logreg_loss_surface.py '10.06': - logreg_poly_demo.py '10.07': - logreg_multiclass_demo.py '10.10': - logreg_iris_bayes_robust_1d_pymc3.py '10.13': - logreg_laplace_demo.py '10.14': - logreg_laplace_demo.py Chapter_11_linear_regression '11.01': - linreg_poly_vs_degree.py '11.02': - linreg_contours_sse_plot.py '11.04': - linregOnlineDemo.py '11.05': - linreg_poly_vs_degree.py '11.06': - linreg_poly_vs_degree.py '11.07': - geom_ridge.py '11.10': - ridgePathProstate.py - lassoPathProstate.py '11.11': - prostate_comparison.py '11.12': - prostate_comparison.py '11.13': - sparse_sensing_demo.py '11.14': - groupLassoDemo.py '11.15': - groupLassoDemo.py '11.16': - splines_basis_weighted.py '11.17': - splines_basis_heatmap.py '11.18': - splines_cherry_blossoms.py '11.19': - linregRobustDemoCombined.py - huberLossPlot.py '11.20': - linreg_2d_bayes_demo.py '11.21': - linreg_post_pred_plot.py '11.22': - linreg_2d_bayes_centering_pymc3.py '11.23': - multi_collinear_legs_numpyro.py '11.24': - multi_collinear_legs_numpyro.py Chapter_12_generalized_linear_models {} Chapter_13_neural_networks_for_structured_data '13.01': - xor_heaviside.py '13.02': - activation_fun_plot.py '13.21': - logregXorDemo.py '13.22': - linregRbfDemo.py '13.23': - mixexpDemoOneToMany.py Chapter_14_neural_networks_for_images {} Chapter_15_neural_networks_for_sequences {} Chapter_16_exemplar-based_methods '16.01': - knn_voronoi_plot.py '16.02': - knn_classify_demo.py '16.03': - curse_dimensionality_plot.py '16.08': - smoothingKernelPlot.py '16.09': - parzen_window_demo2.py '16.10': - kernelRegressionDemo.py Chapter_17_kernel_methods '17.01': - gprDemoArd.py '17.02': - gpKernelPlot.py '17.03': - gpKernelPlot.py '17.07': - gprDemoNoiseFree.py '17.08': - gprDemoChangeHparams.py '17.09': - gpr_demo_marglik.py '17.10': - gp_classify_iris_1d_pymc3.py '17.11': - gp_classify_spaceflu_1d_pymc3.py '17.14': - svm_classifier_feature_scaling.py '17.17': - svm_classifier_2d.py '17.18': - svmCgammaDemo.py '17.19': - huberLossPlot.py '17.20': - svm_regression_1d.py '17.21': - kernelBinaryClassifDemo.py '17.22': - rvm_regression_1d.py '17.23': - rvm_regression_1d.py Chapter_18_trees_forests_bagging_and_boosting '18.01': - regtreeSurfaceDemo.py '18.03': - dtree_sensitivity.py '18.04': - bagging_trees.py - rf_demo_2d.py '18.05': - spam_tree_ensemble_compare.py '18.06': - boosted_regr_trees.py '18.07': - hinge_loss_plot.py '18.08': - rf_feature_importance_mnist.py '18.09': - spam_tree_ensemble_interpret.py '18.10': - spam_tree_ensemble_interpret.py Chapter_19_learning_with_fewer_labeled_examples {} Chapter_20_dimensionality_reduction '20.01': - pcaDemo2d.py '20.02': - pca_digits.py '20.03': - pcaImageDemo.py '20.04': - pca_projected_variance.py '20.05': - pcaStandardization.py '20.06': - pcaOverfitDemo.py '20.07': - pcaOverfitDemo.py '20.08': - pcaOverfitDemo.py '20.10': - pcaEmStepByStep.py '20.12': - mixPpcaDemo.py '20.13': - binary_fa_demo.py '20.30': - manifold_swiss_sklearn.py - manifold_digits_sklearn.py '20.31': - manifold_swiss_sklearn.py - manifold_digits_sklearn.py '20.33': - manifold_swiss_sklearn.py - manifold_digits_sklearn.py '20.34': - manifold_swiss_sklearn.py '20.35': - kpcaScholkopf.py '20.36': - manifold_swiss_sklearn.py - manifold_digits_sklearn.py '20.37': - manifold_swiss_sklearn.py - manifold_digits_sklearn.py '20.38': - manifold_swiss_sklearn.py - manifold_digits_sklearn.py '20.41': - manifold_swiss_sklearn.py - manifold_digits_sklearn.py Chapter_21_clustering '21.02': - agglomDemo.py '21.04': - hclust_yeast_demo.py '21.05': - yeast_data_viz.py '21.06': - hclust_yeast_demo.py '21.07': - kmeans_voronoi.py '21.08': - kmeans_yeast_demo.py '21.09': - vqDemo.py '21.10': - kmeans_minibatch.py '21.11': - kmeans_silhouette.py - gmm_2d.py - kmeans_silhouette.py '21.12': - kmeans_silhouette.py '21.13': - kmeans_silhouette.py '21.14': - gmm_2d.py '21.15': - gmm_identifiability_pymc3.py '21.16': - gmm_identifiability_pymc3.py '21.19': - spectral_clustering_demo.py Chapter_22_recommender_systems {} Chapter_23_graph_embeddings {}