Visualizations of activation functions

[14]:

from pathlib import Path

import plotly.graph_objects as go
import plotly.io as pio
import torch
from plotly.graph_objs import Figure
from plotly.io import write_image
from torch import sigmoid
from torch.nn.functional import softplus

from pytorch_gum_uncertainty_propagation.functionals import quadlu

Plot with Plotly

Preparations

[15]:

def export_plotly_to_pdf(fig: Figure, filename: str):
    """Export figure to pdf at the right location right away"""
    PATH_TO_IMAGES = Path(
        "/home/bjorn/code/GUM-compliant_neural_network_"
        "robustness_verification/src/thesis/images"
    )
    write_image(fig, PATH_TO_IMAGES.joinpath(filename))
    return filename

Set up shared Plotly layout parameters

[16]:

pio.templates.default = "plotly"
color_blind_seque = ["#2e2b83", "#7e3e69", "#a56753", "#b7a03c", "#abe600"]
common_font = dict(font=dict(family="Serif", size=15))
common_tlr_margin_for_layout = dict(
    margin_t=5,
    margin_l=63,
    margin_r=63,
)
common_tlr_margin_for_layout = dict(
    margin_t=5,
    margin_l=63,
    margin_r=63,
)
common_bottom_axis_title_margins = dict(
    margin_b=52,
)

[17]:

shared_layout = dict(
    **common_font,
    xaxis_title="$x$",
    yaxis_title="$y$",
    height=400,
    width=400,
    autosize=False,
    showlegend=True,
)
shared_ratio = dict(
    scaleanchor="x",
    scaleratio=1,
)
shared_title_params = dict(
    y=0.9,
    x=0.5,
    xanchor="center",
    yanchor="top",
)

Graph of QuadLU

[18]:

alphas = [1 / 8 * 2**exponent for exponent in range(3, -1, -1)]
x_quadlu_tensor = torch.linspace(-2.0, 1.0, 100)
alphas_and_ys = {}
for alpha in alphas:
    alphas_and_ys[alpha] = quadlu(x_quadlu_tensor, alpha).detach().numpy()
x_quadlu = x_quadlu_tensor.detach().numpy()

[19]:

fig_quadlu = go.Figure()
for idx, (alpha, y_quadlu) in enumerate(alphas_and_ys.items()):
    fig_quadlu.add_trace(
        go.Scatter(
            x=x_quadlu,
            y=y_quadlu,
            mode="lines",
            name=f"$\operatorname{{QuadLU}}_{{\\frac{{1}}{{{int(1 / alpha)}}}}}$",
            marker_color=color_blind_seque[-idx - 2],
        )
    )
    x_alphas = [-alpha, -alpha, alpha, alpha, -alpha]
    y_alphas = [-0.15, -0.05, -0.05, -0.15, -0.15]
    fig_quadlu.add_trace(
        go.Scatter(
            x=x_alphas,
            y=y_alphas,
            mode="none",
            fill="toself",
            name=f"$[-\\frac{{1}}{{{int(1 / alpha)}}}, \\frac{{1}}{{{int(1 / alpha)}}}]$",
            fillcolor=color_blind_seque[-idx - 2],
        )
    )

fig_quadlu.update_layout(
    legend=dict(yanchor="auto", y=0.97, xanchor="auto", x=0.1, **common_font),
    **common_bottom_axis_title_margins,
    **common_tlr_margin_for_layout,
    **shared_layout,
)
fig_quadlu.update_xaxes(
    dict(range=[x_quadlu.min(), x_quadlu.max()]), constrain="domain"
)
fig_quadlu.update_yaxes(**shared_ratio)
fig_quadlu.show()

[20]:

quadlus_filename = "quadlus_graphs.pdf"
path_to_pdf = export_plotly_to_pdf(fig_quadlu, quadlus_filename)

Graph of softplus

[21]:

betas = [1 / 8 * 2**exponent for exponent in range(5)]
x_softplus_tensor = torch.linspace(-3.0, 7, 100)
betas_and_ys = {}
for beta in betas:
    betas_and_ys[beta] = softplus(x_softplus_tensor, beta).detach().numpy()
x_softplus = x_softplus_tensor.detach().numpy()

[22]:

fig_softplus = go.Figure()
for idx, (beta, y_softplus) in enumerate(betas_and_ys.items()):
    fig_softplus.add_trace(
        go.Scatter(
            x=x_softplus,
            y=y_softplus,
            mode="lines",
            name=f"$\operatorname{{softplus}}_{{\\frac{{1}}{int(1 / beta)}}}$"
            if beta < 1
            else f"$\operatorname{{softplus}}_{{{int(beta)}}}$",
            marker_color=color_blind_seque[-idx - 1],
        )
    )

fig_softplus.update_layout(
    legend=dict(yanchor="auto", y=1, xanchor="auto", x=0.0, **common_font),
    **common_bottom_axis_title_margins,
    **common_tlr_margin_for_layout,
    **shared_layout,
)
fig_softplus.update_xaxes(
    dict(range=[x_softplus.min(), x_softplus.max()], constrain="domain")
)
fig_softplus.update_yaxes(**shared_ratio)
fig_softplus.show()

[23]:

softpluss_filename = "softpluss_graphs.pdf"
path_to_pdf = export_plotly_to_pdf(fig_softplus, softpluss_filename)

Graph of sigmoid

[24]:

x_sigmoid_tensor = torch.linspace(-4.0, 4.0, 100)
y_sigmoid = sigmoid(x_sigmoid_tensor).detach().numpy()
x_sigmoid = x_sigmoid_tensor.detach().numpy()

[25]:

fig_sigmoid = go.Figure()
fig_sigmoid.add_trace(
    go.Scatter(
        x=x_sigmoid,
        y=y_sigmoid,
        mode="lines",
        name=f"$\\sigma$",
        marker_color=color_blind_seque[1],
    )
)

fig_sigmoid.update_layout(
    legend=dict(yanchor="auto", y=1, xanchor="auto", x=0.0, **common_font),
    **common_bottom_axis_title_margins,
    **common_tlr_margin_for_layout,
    **shared_layout,
)
fig_sigmoid.update_xaxes(range=[x_sigmoid.min(), x_sigmoid.max()], constrain="domain")
fig_sigmoid.update_yaxes(range=[-0.5, 1.5])
fig_sigmoid.show()

[26]:

sigmoid_filename = "sigmoid_graphs.pdf"
path_to_pdf = export_plotly_to_pdf(fig_sigmoid, sigmoid_filename)