Approximation Locale

1 décembre 2024

Entraîner un réseau de neurones profond est essentiellement une tâche de compression. Nous voulons représenter notre distribution de données d’entraînement comme une fonction paramétrée par un ensemble de matrices. Plus la distribution est complexe, plus nous avons besoin de paramètres. La raison d’approximer la distribution entière est de pouvoir propager n’importe quel point valide lors de l’inférence en utilisant le même modèle, avec les mêmes poids. Mais que se passerait-il si notre modèle était entraîné à la volée, lors de l’inférence ? Alors, en propageant $x$ , nous n’aurions besoin de modéliser que la distribution locale autour de $x$ . Comme la région locale devrait avoir une dimensionnalité inférieure à celle de l’ensemble d’entraînement complet, un modèle bien plus simple suffirait !

C’est l’idée derrière l’approximation locale ou la régression locale. Considérons une tâche de régression simple.

Tâche

Nous disposons de $100$ échantillons des données suivantes :

Y = sin (4 X) + ϵ

où

ϵ \sim N (0, \frac{1}{3})

Code de tracé

from pathlib import Path

import numpy as np
import plotly.graph_objects as go

# Generate data
np.random.seed(42)
n_points = 100
X = np.random.uniform(0, 1, n_points)
epsilon = np.random.normal(0, 1 / 3, n_points)
Y = np.sin(4 * X) + epsilon

# True function
x_true = np.linspace(0, 1, 500)
y_true = np.sin(4 * x_true)

# Create the plot
fig = go.Figure()

# Add scatter points for noisy data
fig.add_trace(
    go.Scatter(
        x=X,
        y=Y,
        mode="markers",
        name="Noisy Data",
        marker=dict(color="gray"),
    )
)

# Add true function
fig.add_trace(
    go.Scatter(
        x=x_true,
        y=y_true,
        mode="lines",
        name="True Function",
        line=dict(color="red"),
    )
)

# Update layout shared across themes
fig.update_layout(
    autosize=True,
    title="Data",
    xaxis_title="X",
    yaxis_title="Y",
)

# Theme configuration aligning with dario.css colors
themes = [
    {
        "name": "light",
        "template": "plotly_white",
        "font_color": "#141413",
        "background": "#f0efea",
        "axis_color": "#141413",
        "gridcolor": "rgba(20, 20, 19, 0.2)",
    },
    {
        "name": "dark",
        "template": "plotly_dark",
        "font_color": "#f0efea",
        "background": "#141413",
        "axis_color": "#f0efea",
        "gridcolor": "rgba(240, 239, 234, 0.2)",
    },
]

output_dir = Path(__file__).resolve().parents[3] / "static"
output_dir.mkdir(parents=True, exist_ok=True)

for theme in themes:
    themed_fig = go.Figure(fig)
    themed_fig.update_layout(
        template=theme["template"],
        font=dict(color=theme["font_color"]),
        paper_bgcolor=theme["background"],
        plot_bgcolor=theme["background"],
    )
    themed_fig.update_xaxes(
        showline=True,
        linecolor=theme["axis_color"],
        tickcolor=theme["axis_color"],
        tickfont=dict(color=theme["axis_color"]),
        title_font=dict(color=theme["axis_color"]),
        gridcolor=theme["gridcolor"],
        zeroline=False,
    )
    themed_fig.update_yaxes(
        showline=True,
        linecolor=theme["axis_color"],
        tickcolor=theme["axis_color"],
        tickfont=dict(color=theme["axis_color"]),
        title_font=dict(color=theme["axis_color"]),
        gridcolor=theme["gridcolor"],
        zeroline=False,
    )

    filename = output_dir / f"local_approximation_data_{theme['name']}.html"
    themed_fig.write_html(filename)
    print(f"Saved plot to {filename}")

# Show the plot
fig.show()

Nous notons l’ensemble de données $D$ qui consiste en des échantillons $(x_{i}, y_{i}) \in D$ .

Notre tâche est d’ajuster une courbe raisonnable à travers les données, qui correspond approximativement à la fonction réelle. Notons cette courbe $\hat{f}$ .

K Plus Proches Voisins

Étant donné un certain $x$ , une approche consiste à prendre les $k$ valeurs les plus proches $x_{i}$ de $x$ , et à faire la moyenne de leurs valeurs $y_{i}$ comme estimation. C’est-à-dire,

\hat{f} (x) = Moy (y_{i} ∣ x_{i} \in N_{k} (x))

où $N_{k} (x)$ désigne les $k$ points les plus proches de $x$ .

Code de tracé

from pathlib import Path

import numpy as np
import plotly.graph_objects as go

# Generate data
np.random.seed(42)
n_points = 100
X = np.random.uniform(0, 1, n_points)
epsilon = np.random.normal(0, 1 / 3, n_points)
Y = np.sin(4 * X) + epsilon

# True function
x_true = np.linspace(0, 1, 500)
y_true = np.sin(4 * x_true)

# k-NN for a range of k
x_curve = np.arange(0, 1, 0.01)
k_range = range(1, 21)
y_curves_knn = {}

for k in k_range:
    y_curve = []
    for x in x_curve:
        distances = np.square(X - x)
        nearest_indices = np.argsort(distances)[:k]
        y_curve.append(np.mean(Y[nearest_indices]))
    y_curves_knn[k] = y_curve

# Create the Plotly figure
fig = go.Figure()

# Add static traces
fig.add_trace(
    go.Scatter(x=X, y=Y, mode="markers", name="Noisy Data", marker=dict(color="gray"))
)

fig.add_trace(
    go.Scatter(
        x=x_true, y=y_true, mode="lines", name="True Function", line=dict(color="red")
    )
)

# Add the first k-NN curve (k=13, the default slider position)
initial_k = 13
fig.add_trace(
    go.Scatter(
        x=x_curve,
        y=y_curves_knn[initial_k],
        mode="lines",
        name="k-NN Curve",
        line=dict(color="yellow"),
    )
)

# Define slider steps
steps = []
for k in k_range:
    step = dict(
        method="update",
        args=[
            {"y": [Y, y_true, y_curves_knn[k]]},  # Update y-data for the traces
            {
                "title": f"Interactive k-NN Curve with Slider for k = {k}"
            },  # Update the title dynamically
        ],
        label=f"{k}",
    )
    steps.append(step)

# Add slider to the layout
sliders = [
    dict(
        active=initial_k - 1,
        currentvalue={"prefix": "k = "},
        pad={"t": 50},
        steps=steps,
    )
]

fig.update_layout(
    sliders=sliders,
    autosize=True,
    title=f"Interactive k-NN Curve with Slider for k = {initial_k}",
    xaxis_title="X",
    yaxis_title="Y",
)

themes = [
    {
        "name": "light",
        "template": "plotly_white",
        "font_color": "#141413",
        "background": "#f0efea",
        "axis_color": "#141413",
        "gridcolor": "rgba(20, 20, 19, 0.2)",
    },
    {
        "name": "dark",
        "template": "plotly_dark",
        "font_color": "#f0efea",
        "background": "#141413",
        "axis_color": "#f0efea",
        "gridcolor": "rgba(240, 239, 234, 0.2)",
    },
]

output_dir = Path(__file__).resolve().parents[3] / "static"
output_dir.mkdir(parents=True, exist_ok=True)

for theme in themes:
    themed_fig = go.Figure(fig)
    themed_fig.update_layout(
        template=theme["template"],
        font=dict(color=theme["font_color"]),
        paper_bgcolor=theme["background"],
        plot_bgcolor=theme["background"],
    )
    themed_fig.update_xaxes(
        showline=True,
        linecolor=theme["axis_color"],
        tickcolor=theme["axis_color"],
        tickfont=dict(color=theme["axis_color"]),
        title_font=dict(color=theme["axis_color"]),
        gridcolor=theme["gridcolor"],
        zeroline=False,
    )
    themed_fig.update_yaxes(
        showline=True,
        linecolor=theme["axis_color"],
        tickcolor=theme["axis_color"],
        tickfont=dict(color=theme["axis_color"]),
        title_font=dict(color=theme["axis_color"]),
        gridcolor=theme["gridcolor"],
        zeroline=False,
    )

    html_path = output_dir / f"knn_slider_{theme['name']}.html"
    themed_fig.write_html(html_path)
    print(f"Saved interactive plot to {html_path}")

# Show the plot
fig.show()

Vous pouvez voir en utilisant le curseur qu’un $k$ plus grand donne une courbe plus lisse, mais les courbes avec un $k$ faible intègrent un certain bruit. Aux extrêmes, $k = 1$ suit exactement les données d’entraînement et $k = 100$ donne une moyenne globale plate.

Régression à noyau de Nadaraya–Watson

Au lieu de limiter votre sous-ensemble de données à $k$ points, vous pourriez plutôt considérer tous les points de l’ensemble, mais pondérer la contribution de chaque point en fonction de sa proximité à $x$ . Considérez le modèle

\hat{f} (x) = \frac{\sum _{i = 1}^{∣ D ∣} K _{λ} ( x , x _{i} ) y _{i}}{\sum _{i = 1}^{∣ D ∣} K _{λ} ( x , x _{i} )}

où $K_{λ}$ est un noyau, que nous utiliserons comme métrique de proximité.

K_{λ} (x_{0}, x) = D (\frac{∣ x - x _{0} ∣}{λ})

Cette fonction est paramétrée par $λ$ , appelé la largeur de bande, qui contrôle la plage de valeurs de $x$ dans les données qui jouent un rôle dans la sortie de $\hat{f}$ . Cela devient clair si nous traçons ces fonctions.

Fonctions de Noyau

Ce qui est tracé ci-dessous est

f (x) = α K_{λ, D} (0, x)

où $α$ assure que $f$ s’intègre à $1$ sur son support.

D (u) = ⎩ ⎨ ⎧ \frac{3}{4} (1 - u^{2}) 0 si ∣ u ∣ \leq 1, si ∣ u ∣ > 1.

D (u) = ⎩ ⎨ ⎧ (1 - ∣ u ∣^{3})^{3} 0 si ∣ u ∣ \leq 1, si ∣ u ∣ > 1.

D (u) = \frac{1}{2 π} e^{- \frac{1}{2} u^{2}} .

Code de tracé

from pathlib import Path

import numpy as np
import plotly.graph_objects as go
from scipy.integrate import quad


# Define kernel functions
def epanechnikov_kernel(u):
    return np.maximum(0, 0.75 * (1 - u**2))


def tricube_kernel(u):
    return np.maximum(0, (1 - np.abs(u) ** 3) ** 3)


def gaussian_kernel(u):
    return np.exp(-0.5 * u**2) / np.sqrt(2 * np.pi)


def renormalized_kernel(kernel_func, u_range, bandwidth):
    def kernel_with_lambda(u):
        scaled_u = u / bandwidth
        normalization_factor, _ = quad(lambda v: kernel_func(v / bandwidth), *u_range)
        return kernel_func(scaled_u) / normalization_factor

    return kernel_with_lambda


# Kernel function plot generator
def generate_kernel_plot(
    kernel_name, kernel_func, x_range, u_range, lambda_values, y_range
):
    fig = go.Figure()

    # Initial lambda
    initial_lambda = lambda_values[len(lambda_values) // 2]

    # Generate initial kernel curve
    x = np.linspace(*x_range, 500)
    kernel_with_lambda = renormalized_kernel(kernel_func, u_range, initial_lambda)
    y = kernel_with_lambda(x)
    fig.add_trace(
        go.Scatter(
            x=x,
            y=y,
            mode="lines",
            name=f"{kernel_name} Kernel (λ={initial_lambda:.2f})",
            line=dict(color="green"),
        )
    )

    # Create frames for the slider
    frames = []
    for bandwidth in lambda_values:
        kernel_with_lambda = renormalized_kernel(kernel_func, u_range, bandwidth)
        y = kernel_with_lambda(x)
        frames.append(
            go.Frame(
                data=[
                    go.Scatter(
                        x=x,
                        y=y,
                        mode="lines",
                        name=f"{kernel_name} Kernel (λ={bandwidth:.2f})",
                        line=dict(color="green"),
                    )
                ],
                name=f"{bandwidth:.2f}",
            )
        )

    # Add frames to the figure
    fig.frames = frames

    # Add slider
    sliders = [
        {
            "active": len(lambda_values) // 2,
            "currentvalue": {"prefix": "Bandwidth λ: "},
            "steps": [
                {
                    "args": [
                        [f"{bandwidth:.2f}"],
                        {"frame": {"duration": 0, "redraw": True}, "mode": "immediate"},
                    ],
                    "label": f"{bandwidth:.2f}",
                    "method": "animate",
                }
                for bandwidth in lambda_values
            ],
        }
    ]

    # Update layout
    fig.update_layout(
        title=f"{kernel_name} Kernel Function",
        xaxis_title="u",
        yaxis_title="K(u)",
        yaxis_range=y_range,
        sliders=sliders,
        autosize=True,
        updatemenus=[
            {
                "buttons": [
                    # {
                    #     "args": [
                    #         None,
                    #         {
                    #             "frame": {"duration": 500, "redraw": True},
                    #             "fromcurrent": True,
                    #         },
                    #     ],
                    #     "label": "Play",
                    #     "method": "animate",
                    # },
                    # {
                    #     "args": [
                    #         [None],
                    #         {
                    #             "frame": {"duration": 0, "redraw": True},
                    #             "mode": "immediate",
                    #         },
                    #     ],
                    #     "label": "Pause",
                    #     "method": "animate",
                    # },
                ],
                "direction": "left",
                "pad": {"r": 10, "t": 87},
                "showactive": False,
                "type": "buttons",
                "x": 0.1,
                "xanchor": "right",
                "y": 0,
                "yanchor": "top",
            }
        ],
    )

    return fig


# Kernel functions
kernels = {
    "Epanechnikov": epanechnikov_kernel,
    "Tricube": tricube_kernel,
    "Gaussian": gaussian_kernel,
}

# Parameters
x_range_plot = (-3, 3)  # Range of u values for the plot
u_range_integration = (-3, 3)  # Range for normalization
lambda_values = np.linspace(0.01, 2, 20)  # Linear lambda values from 0.01 to 2
y_range_plot = (0, 1.5)  # Adjusted range to fit the normalized functions

# Generate and display plots for each kernel
themes = [
    {
        "name": "light",
        "template": "plotly_white",
        "font_color": "#141413",
        "background": "#f0efea",
        "axis_color": "#141413",
        "gridcolor": "rgba(20, 20, 19, 0.2)",
    },
    {
        "name": "dark",
        "template": "plotly_dark",
        "font_color": "#f0efea",
        "background": "#141413",
        "axis_color": "#f0efea",
        "gridcolor": "rgba(240, 239, 234, 0.2)",
    },
]

output_dir = Path(__file__).resolve().parents[3] / "static"
output_dir.mkdir(parents=True, exist_ok=True)

for kernel_name, kernel_func in kernels.items():
    fig = generate_kernel_plot(
        kernel_name,
        kernel_func,
        x_range_plot,
        u_range_integration,
        lambda_values,
        y_range_plot,
    )

    # Save themed figures to HTML files
    for theme in themes:
        themed_fig = go.Figure(fig)
        themed_fig.update_layout(
            template=theme["template"],
            font=dict(color=theme["font_color"]),
            paper_bgcolor=theme["background"],
            plot_bgcolor=theme["background"],
        )
        themed_fig.update_xaxes(
            showline=True,
            linecolor=theme["axis_color"],
            tickcolor=theme["axis_color"],
            tickfont=dict(color=theme["axis_color"]),
            title_font=dict(color=theme["axis_color"]),
            gridcolor=theme["gridcolor"],
            zeroline=False,
        )
        themed_fig.update_yaxes(
            showline=True,
            linecolor=theme["axis_color"],
            tickcolor=theme["axis_color"],
            tickfont=dict(color=theme["axis_color"]),
            title_font=dict(color=theme["axis_color"]),
            gridcolor=theme["gridcolor"],
            zeroline=False,
        )

        filename = (
            output_dir
            / f"{kernel_name}_dynamic_normalization_kernel_function_{theme['name']}.html"
        )
        themed_fig.write_html(filename, auto_play=False)
        print(f"Saved {kernel_name} kernel plot to {filename}")

    # Show the figure
    fig.show()

Résultats

Nous traçons maintenant les résultats pour chacune des fonctions de noyau. Chaque graphique possède un curseur $λ$ , qui contrôle la sortie en temps réel.

Code de tracé

from pathlib import Path

import numpy as np
import plotly.graph_objects as go


# Define kernel functions
def epanechnikov_kernel(u):
    return np.maximum(0, 0.75 * (1 - u**2))


def tricube_kernel(u):
    return np.maximum(0, (1 - np.abs(u) ** 3) ** 3)


def gaussian_kernel(u):
    return np.exp(-0.5 * u**2) / np.sqrt(2 * np.pi)


# Kernel regression function
def kernel_regression(X, Y, x_curve, kernel_func, bandwidth):
    y_curve = []
    for x in x_curve:
        distances = np.abs(X - x) / bandwidth
        weights = kernel_func(distances)
        weighted_average = (
            np.sum(weights * Y) / np.sum(weights) if np.sum(weights) > 0 else 0
        )
        y_curve.append(weighted_average)
    return y_curve


# Generate data
np.random.seed(42)
n_points = 100
X = np.random.uniform(0, 1, n_points)
epsilon = np.random.normal(0, 1 / 3, n_points)
Y = np.sin(4 * X) + epsilon

# True curve
x_true = np.linspace(0, 1, 500)
y_true = np.sin(4 * x_true)

# Points for kernel estimation
x_curve = x_true

# Kernel functions
kernels = {
    "Epanechnikov": epanechnikov_kernel,
    "Tricube": tricube_kernel,
    "Gaussian": gaussian_kernel,
}

# Range of bandwidths for the slider in logspace
lambda_values = np.logspace(-2, 0, 20)  # From 0.01 to 1

# Generate separate plots for each kernel
themes = [
    {
        "name": "light",
        "template": "plotly_white",
        "font_color": "#141413",
        "background": "#f0efea",
        "axis_color": "#141413",
        "gridcolor": "rgba(20, 20, 19, 0.2)",
    },
    {
        "name": "dark",
        "template": "plotly_dark",
        "font_color": "#f0efea",
        "background": "#141413",
        "axis_color": "#f0efea",
        "gridcolor": "rgba(240, 239, 234, 0.2)",
    },
]

output_dir = Path(__file__).resolve().parents[3] / "static"
output_dir.mkdir(parents=True, exist_ok=True)

# Generate separate plots for each kernel
for kernel_name, kernel_func in kernels.items():
    fig = go.Figure()

    # Add scatter points for noisy data
    fig.add_trace(
        go.Scatter(
            x=X, y=Y, mode="markers", name="Noisy Data", marker=dict(color="gray")
        )
    )

    # Add true function
    fig.add_trace(
        go.Scatter(
            x=x_true,
            y=y_true,
            mode="lines",
            name="True Function",
            line=dict(color="red"),
        )
    )

    # Add initial kernel curve
    initial_bandwidth = lambda_values[0]
    y_curve = kernel_regression(X, Y, x_curve, kernel_func, initial_bandwidth)
    fig.add_trace(
        go.Scatter(
            x=x_curve,
            y=y_curve,
            mode="lines",
            name=f"Nadaraya-Watson ({kernel_name})",
            line=dict(color="green"),
        )
    )

    # Create frames for the slider
    frames = []
    for bandwidth in lambda_values:
        y_curve = kernel_regression(X, Y, x_curve, kernel_func, bandwidth)
        frames.append(
            go.Frame(
                data=[
                    go.Scatter(
                        x=X,
                        y=Y,
                        mode="markers",
                        name="Noisy Data",
                        marker=dict(color="gray"),
                    ),
                    go.Scatter(
                        x=x_true,
                        y=y_true,
                        mode="lines",
                        name="True Function",
                        line=dict(color="red"),
                    ),
                    go.Scatter(
                        x=x_curve,
                        y=y_curve,
                        mode="lines",
                        name=f"Nadaraya-Watson ({kernel_name})",
                        line=dict(color="green"),
                    ),
                ],
                name=f"{bandwidth:.2f}",
            )
        )

    # Add frames to the figure
    fig.frames = frames

    # Add slider
    sliders = [
        {
            "active": 0,
            "currentvalue": {"prefix": "Bandwidth λ: "},
            "steps": [
                {
                    "args": [
                        [f"{bandwidth:.2f}"],
                        {"frame": {"duration": 0, "redraw": True}, "mode": "immediate"},
                    ],
                    "label": f"{bandwidth:.2f}",
                    "method": "animate",
                }
                for bandwidth in lambda_values
            ],
        }
    ]

    # Update layout
    fig.update_layout(
        autosize=True,
        title=f"Nadaraya-Watson Kernel Regression ({kernel_name} Kernel)",
        xaxis_title="X",
        yaxis_title="Y",
        sliders=sliders,
        updatemenus=[
            {
                "buttons": [
                    {
                        "args": [
                            None,
                            {
                                "frame": {"duration": 500, "redraw": True},
                                "fromcurrent": True,
                            },
                        ],
                        "label": "Play",
                        "method": "animate",
                    },
                    {
                        "args": [
                            [None],
                            {
                                "frame": {"duration": 0, "redraw": True},
                                "mode": "immediate",
                            },
                        ],
                        "label": "Pause",
                        "method": "animate",
                    },
                ],
                "direction": "left",
                "pad": {"r": 10, "t": 87},
                "showactive": False,
                "type": "buttons",
                "x": 0.1,
                "xanchor": "right",
                "y": 0,
                "yanchor": "top",
            }
        ],
    )

    # Save the figure to an HTML file per theme
    for theme in themes:
        themed_fig = go.Figure(fig)
        themed_fig.update_layout(
            template=theme["template"],
            font=dict(color=theme["font_color"]),
            paper_bgcolor=theme["background"],
            plot_bgcolor=theme["background"],
        )
        themed_fig.update_xaxes(
            showline=True,
            linecolor=theme["axis_color"],
            tickcolor=theme["axis_color"],
            tickfont=dict(color=theme["axis_color"]),
            title_font=dict(color=theme["axis_color"]),
            gridcolor=theme["gridcolor"],
            zeroline=False,
        )
        themed_fig.update_yaxes(
            showline=True,
            linecolor=theme["axis_color"],
            tickcolor=theme["axis_color"],
            tickfont=dict(color=theme["axis_color"]),
            title_font=dict(color=theme["axis_color"]),
            gridcolor=theme["gridcolor"],
            zeroline=False,
        )

        filename = output_dir / f"{kernel_name}_kernel_regression_{theme['name']}.html"
        themed_fig.write_html(filename, auto_play=False)
        print(f"Saved {kernel_name} kernel plot to {filename}")

    # Show the figure
    fig.show()

Nous voyons qu’une simple moyenne pondérée des données permet de modéliser une sinusoïde assez bien.

Régression Linéaire Locale

Dans la régression à noyau de Nadaraya-Watson, nous prenons une moyenne pondérée dans un voisinage défini par la fonction noyau $K_{λ}$ . Un problème potentiel avec cette approche est l’interpolation lisse au sein des voisinages locaux, car nous ne supposons pas réellement que la région suit un modèle particulier.

Et si nous supposons que chaque région est localement linéaire ? Alors, nous pourrions résoudre l’ajustement des moindres carrés et interpoler librement !

Région : $k$-NN

Définissons notre région locale comme les $k$ plus proches voisins de notre entrée. Soit $X = [N_{k} (x_{0}) 1]$ et $Y$ les valeurs $y$ correspondantes. Les coefficients de l’ajustement des moindres carrés sont

β = (X^{⊤} X)^{- 1} XY

Code de tracé

from pathlib import Path

import numpy as np
import plotly.graph_objects as go

# Generate data
np.random.seed(42)
n_points = 100
X = np.random.uniform(0, 1, n_points)
epsilon = np.random.normal(0, 1 / 3, n_points)
Y = np.sin(4 * X) + epsilon

# True function
x_true = np.linspace(0, 1, 500)
y_true = np.sin(4 * x_true)


# k-NN Local Linear Regression
def knn_linear_regression(X, Y, x_curve, k_range):
    y_curves = {}
    for k in k_range:
        y_curve = []
        for x in x_curve:
            # Find k nearest neighbors
            distances = np.abs(X - x)
            nearest_indices = np.argsort(distances)[:k]

            # Select k nearest neighbors
            X_knn = X[nearest_indices]
            Y_knn = Y[nearest_indices]

            # Create design matrix for k-nearest neighbors
            X_design = np.vstack((np.ones_like(X_knn), X_knn)).T

            # Solve for beta using ordinary least squares
            beta = np.linalg.pinv(X_design.T @ X_design) @ X_design.T @ Y_knn

            # Predict y-value
            y_curve.append(beta[0] + beta[1] * x)
        y_curves[k] = y_curve
    return y_curves


# Common variables
x_curve = np.arange(0, 1, 0.01)
k_range = range(1, 21)  # Values of k from 1 to 20
initial_k = 10  # Default value of k

# Compute LLR using k-NN
y_curves_knn = knn_linear_regression(X, Y, x_curve, k_range)

# Create the Plotly figure
fig = go.Figure()

# Add static traces
fig.add_trace(
    go.Scatter(x=X, y=Y, mode="markers", name="Noisy Data", marker=dict(color="gray"))
)

fig.add_trace(
    go.Scatter(
        x=x_true, y=y_true, mode="lines", name="True Function", line=dict(color="red")
    )
)

# Add the first k-NN curve (k=initial_k)
fig.add_trace(
    go.Scatter(
        x=x_curve,
        y=y_curves_knn[initial_k],
        mode="lines",
        name="k-NN Curve",
        line=dict(color="yellow"),
    )
)

# Define slider steps
steps = []
for k in k_range:
    step = dict(
        method="update",
        args=[
            {"y": [Y, y_true, y_curves_knn[k]]},  # Update y-data for the traces
            {
                "title": f"k-NN Local Linear Regression Curve with k = {k}"
            },  # Update the title dynamically
        ],
        label=f"{k}",
    )
    steps.append(step)

# Add slider to the layout
sliders = [
    dict(
        active=k_range.index(initial_k),  # Use the index of initial_k
        currentvalue={"prefix": "k = "},
        pad={"t": 50},
        steps=steps,
    )
]

fig.update_layout(
    autosize=True,
    sliders=sliders,
    title=f"k-NN Local Linear Regression Curve with k = {initial_k}",
    xaxis_title="X",
    yaxis_title="Y",
)

themes = [
    {
        "name": "light",
        "template": "plotly_white",
        "font_color": "#141413",
        "background": "#f0efea",
        "axis_color": "#141413",
        "gridcolor": "rgba(20, 20, 19, 0.2)",
    },
    {
        "name": "dark",
        "template": "plotly_dark",
        "font_color": "#f0efea",
        "background": "#141413",
        "axis_color": "#f0efea",
        "gridcolor": "rgba(240, 239, 234, 0.2)",
    },
]

output_dir = Path(__file__).resolve().parents[3] / "static"
output_dir.mkdir(parents=True, exist_ok=True)

for theme in themes:
    themed_fig = go.Figure(fig)
    themed_fig.update_layout(
        template=theme["template"],
        font=dict(color=theme["font_color"]),
        paper_bgcolor=theme["background"],
        plot_bgcolor=theme["background"],
    )
    themed_fig.update_xaxes(
        showline=True,
        linecolor=theme["axis_color"],
        tickcolor=theme["axis_color"],
        tickfont=dict(color=theme["axis_color"]),
        title_font=dict(color=theme["axis_color"]),
        gridcolor=theme["gridcolor"],
        zeroline=False,
    )
    themed_fig.update_yaxes(
        showline=True,
        linecolor=theme["axis_color"],
        tickcolor=theme["axis_color"],
        tickfont=dict(color=theme["axis_color"]),
        title_font=dict(color=theme["axis_color"]),
        gridcolor=theme["gridcolor"],
        zeroline=False,
    )

    html_path = output_dir / f"knn_slider_llr_{theme['name']}.html"
    themed_fig.write_html(html_path)
    print(f"Saved interactive k-NN plot to {html_path}")

# Show the plot
fig.show()

Nous voyons que la sortie peut être assez irrégulière pour de petites valeurs de $k$ .

Région : Fonction de Noyau

Peut-être pouvons-nous réutiliser certaines idées du noyau de Nadaraya-Watson. Nous aimerions considérer tous les points de l’ensemble d’entraînement à des degrés divers, avec des poids plus élevés à l’intérieur de la région locale et des poids plus faibles à l’extérieur.

Pour cela, nous pouvons utiliser l’objectif des moindres carrés pondérés, avec les poids $W (x_{0}) = diag (K_{λ} (x_{0}, x_{i}))$ . Cela donne la solution suivante :

β = (X^{⊤} WX)^{- 1} X^{⊤} WY

Tracé des résultats pour diverses fonctions de noyau $D$ :

Code de tracé

from pathlib import Path

import numpy as np
import plotly.graph_objects as go

# Generate data
np.random.seed(42)
n_points = 100
X = np.random.uniform(0, 1, n_points)
epsilon = np.random.normal(0, 1 / 3, n_points)
Y = np.sin(4 * X) + epsilon

# True function
x_true = np.linspace(0, 1, 500)
y_true = np.sin(4 * x_true)


# Kernels
def gaussian_kernel(u):
    return np.exp(-0.5 * u**2)


def epanechnikov_kernel(u):
    return np.maximum(0, 1 - u**2)


def tricube_kernel(u):
    return np.maximum(0, (1 - np.abs(u) ** 3) ** 3)


# Local Linear Regression for a specific kernel
def local_linear_regression(X, Y, x_curve, bandwidths, kernel):
    y_curves = {}
    for λ in bandwidths:
        λ_rounded = round(λ, 2)
        y_curve = []
        for x in x_curve:
            # Calculate weights using the specified kernel
            distances = (X - x) / λ
            weights = kernel(distances)
            W = np.diag(weights)

            # Create design matrix
            X_design = np.vstack((np.ones_like(X), X)).T

            # Solve for beta using weighted least squares
            beta = np.linalg.pinv(X_design.T @ W @ X_design) @ X_design.T @ W @ Y

            # Predict y-value
            y_curve.append(beta[0] + beta[1] * x)
        y_curves[λ_rounded] = y_curve
    return y_curves


# Common variables
x_curve = np.arange(0, 1, 0.01)
bandwidths = np.linspace(0.05, 0.5, 20)
initial_λ = bandwidths[len(bandwidths) // 2]

# Generate plots for each kernel
kernels = {
    "Gaussian Kernel": gaussian_kernel,
    "Epanechnikov Kernel": epanechnikov_kernel,
    "Tricube Kernel": tricube_kernel,
}
plots = []

for kernel_name, kernel_func in kernels.items():
    # Compute LLR with the specified kernel
    y_curves = local_linear_regression(X, Y, x_curve, bandwidths, kernel_func)

    # Create the Plotly figure
    fig = go.Figure()

    # Add static traces
    fig.add_trace(
        go.Scatter(
            x=X, y=Y, mode="markers", name="Noisy Data", marker=dict(color="gray")
        )
    )

    fig.add_trace(
        go.Scatter(
            x=x_true,
            y=y_true,
            mode="lines",
            name="True Function",
            line=dict(color="red"),
        )
    )

    # Add the first LLR curve (using the middle value of bandwidths)
    fig.add_trace(
        go.Scatter(
            x=x_curve,
            y=y_curves[round(initial_λ, 2)],
            mode="lines",
            name=f"{kernel_name} Curve",
            line=dict(color="yellow"),
        )
    )

    # Define slider steps
    steps = []
    for λ in bandwidths:
        λ_rounded = round(λ, 2)
        step = dict(
            method="update",
            args=[
                {"y": [Y, y_true, y_curves[λ_rounded]]},  # Update y-data for the traces
                {
                    "title": f"LLR: {kernel_name} with Bandwidth λ = {λ_rounded}"
                },  # Update the title dynamically
            ],
            label=f"{λ_rounded}",
        )
        steps.append(step)

    # Add slider to the layout
    sliders = [
        dict(
            active=len(bandwidths) // 2,  # Use the index of the middle bandwidth
            currentvalue={"prefix": "λ = "},
            pad={"t": 50},
            steps=steps,
        )
    ]

    fig.update_layout(
        autosize=True,
        sliders=sliders,
        title=f"LLR: {kernel_name} with Bandwidth λ = {round(initial_λ, 2)}",
        xaxis_title="X",
        yaxis_title="Y",
    )

    plots.append(fig)

# Show and save the plots with themed backgrounds
themes = [
    {
        "name": "light",
        "template": "plotly_white",
        "font_color": "#141413",
        "background": "#f0efea",
        "axis_color": "#141413",
        "gridcolor": "rgba(20, 20, 19, 0.2)",
    },
    {
        "name": "dark",
        "template": "plotly_dark",
        "font_color": "#f0efea",
        "background": "#141413",
        "axis_color": "#f0efea",
        "gridcolor": "rgba(240, 239, 234, 0.2)",
    },
]

output_dir = Path(__file__).resolve().parents[3] / "static"
output_dir.mkdir(parents=True, exist_ok=True)

for kernel_name, fig in zip(kernels.keys(), plots):
    fig.show()
    for theme in themes:
        themed_fig = go.Figure(fig)
        themed_fig.update_layout(
            template=theme["template"],
            font=dict(color=theme["font_color"]),
            paper_bgcolor=theme["background"],
            plot_bgcolor=theme["background"],
        )
        themed_fig.update_xaxes(
            showline=True,
            linecolor=theme["axis_color"],
            tickcolor=theme["axis_color"],
            tickfont=dict(color=theme["axis_color"]),
            title_font=dict(color=theme["axis_color"]),
            gridcolor=theme["gridcolor"],
            zeroline=False,
        )
        themed_fig.update_yaxes(
            showline=True,
            linecolor=theme["axis_color"],
            tickcolor=theme["axis_color"],
            tickfont=dict(color=theme["axis_color"]),
            title_font=dict(color=theme["axis_color"]),
            gridcolor=theme["gridcolor"],
            zeroline=False,
        )

        filename = (
            output_dir
            / f"llr_{kernel_name.lower().replace(' ', '_')}_{theme['name']}.html"
        )
        themed_fig.write_html(filename)
        print(f"Saved interactive plot for {kernel_name} to {filename}")

Je pense que les résultats semblent beaucoup plus lisses !

Références

The Elements of Statistical Learning - Hastie, Tibshirani, et Friedman (2009). Un guide complet sur l’exploration de données, l’inférence et la prédiction. En savoir plus.

←

Les Mécanismes de l'Attention Causale

Modèles de Mélange Gaussien Interactifs

→