omnibias.keras.activations.tempered
Keras 3 operator layers and activation paths implemented with keras.ops for TensorFlow, JAX, or Torch.
import omnibias.keras.activations.temperedModule contract & mathematical conventions
Beta-tempered smooth surrogates of the hard non-smooth activations.
Keras (keras.ops) twin of omnibias.torch.activations.tempered. Built
on the backend-neutral omnibias.core.spec.tempered combinator, so every
order matches the torch / jax towers by construction. beta -> inf recovers
the hard activation (softplus -> relu, sigmoid -> Heaviside,
tanh -> sign); beta may be a float or a tensor for a learnable
temperature.
Shared formulas do not imply universal floating-point identity. Verify dtype, shape, derivative-order support, and the stated numerical assumptions for the path you use.
CONSTANT
SOFT_RELU
omnibias.keras.activations.tempered.SOFT_RELUSource ↗SOFT_RELU = register_activation(make_soft_relu_spec())This exported constant is defined by the expression above. Its exact value and annotation come from the pinned source; use it with the module contract.
Read the implementation
SOFT_RELU = register_activation(make_soft_relu_spec())CONSTANT
SOFT_STEP
omnibias.keras.activations.tempered.SOFT_STEPSource ↗SOFT_STEP = register_activation(make_soft_step_spec())This exported constant is defined by the expression above. Its exact value and annotation come from the pinned source; use it with the module contract.
Read the implementation
SOFT_STEP = register_activation(make_soft_step_spec())FUNCTION
make_soft_leaky_relu_spec
omnibias.keras.activations.tempered.make_soft_leaky_relu_specSource ↗make_soft_leaky_relu_spec(negative_slope: float=0.01, beta: float=_DEFAULT_BETA, *, name: str='soft_leaky_relu') -> ActivationSpec[Any]Smooth leaky ReLU alpha*z + (1-alpha)*soft_relu(z; beta) (all orders).
Parameters
| Name | Type & default | Contract |
|---|---|---|
negative_slopepositional-or-keyword | floatDefault: 0.01 | See the operation contract and implementation below. |
betapositional-or-keyword | floatDefault: _DEFAULT_BETA | See the operation contract and implementation below. |
namekeyword-only | strDefault: 'soft_leaky_relu' | See the operation contract and implementation below. |
Returns
ActivationSpec[Any]Read the implementation
def make_soft_leaky_relu_spec(
negative_slope: float = 0.01,
beta: float = _DEFAULT_BETA,
*,
name: str = "soft_leaky_relu",
) -> ActivationSpec[Any]:
"""Smooth leaky ReLU ``alpha*z + (1-alpha)*soft_relu(z; beta)`` (all orders)."""
alpha = negative_slope
one_minus_alpha = 1.0 - alpha
def _soft_relu_nth(z: Any, n: int) -> Any:
return (beta ** (n - 1)) * softplus_nth_derivative(beta * z, n)
def fwd(z: Any) -> Any:
return alpha * z + one_minus_alpha * (ops.softplus(beta * z) / beta)
def fp(z: Any, n: int) -> Any:
if n < 0:
raise ValueError(f"order n must be >= 0, got {n}.")
if n == 0:
return fwd(z)
soft = one_minus_alpha * _soft_relu_nth(z, n)
if n == 1:
return alpha * ops.ones_like(z) + soft
return soft
def deriv(z: Any) -> Any:
return fp(z, 1)
return ActivationSpec(
name=name,
forward=fwd,
derivative=deriv,
fastpath=fp,
riccati_polynomial=None,
noise_model="none",
operator_role=(
"Smooth leaky ReLU: alpha*z + (1-alpha)*softplus(beta z)/beta "
"-> leaky_relu as beta -> inf."
),
)FUNCTION
make_soft_relu_spec
omnibias.keras.activations.tempered.make_soft_relu_specSource ↗make_soft_relu_spec(beta: float=_DEFAULT_BETA, *, name: str='soft_relu') -> ActivationSpec[Any]Smooth ReLU surrogate softplus(beta z) / beta (all-orders tower).
Parameters
| Name | Type & default | Contract |
|---|---|---|
betapositional-or-keyword | floatDefault: _DEFAULT_BETA | See the operation contract and implementation below. |
namekeyword-only | strDefault: 'soft_relu' | See the operation contract and implementation below. |
Returns
ActivationSpec[Any]Read the implementation
def make_soft_relu_spec(
beta: float = _DEFAULT_BETA, *, name: str = "soft_relu"
) -> ActivationSpec[Any]:
"""Smooth ReLU surrogate ``softplus(beta z) / beta`` (all-orders tower)."""
return tempered(
SOFTPLUS,
beta,
scale="one_over_beta",
name=name,
operator_role=(
"Smooth ReLU: softplus(beta z)/beta -> relu as beta -> inf; "
"the n=2 bump -> delta."
),
limit_neg_inf=0.0,
)FUNCTION
make_soft_step_spec
omnibias.keras.activations.tempered.make_soft_step_specSource ↗make_soft_step_spec(beta: float=_DEFAULT_BETA, *, name: str='soft_step') -> ActivationSpec[Any]Smooth step surrogate sigmoid(beta z) (all-orders tower).
Parameters
| Name | Type & default | Contract |
|---|---|---|
betapositional-or-keyword | floatDefault: _DEFAULT_BETA | See the operation contract and implementation below. |
namekeyword-only | strDefault: 'soft_step' | See the operation contract and implementation below. |
Returns
ActivationSpec[Any]Read the implementation
def make_soft_step_spec(
beta: float = _DEFAULT_BETA, *, name: str = "soft_step"
) -> ActivationSpec[Any]:
"""Smooth step surrogate ``sigmoid(beta z)`` (all-orders tower)."""
return tempered(
SIGMOID,
beta,
scale="unit",
name=name,
aliases=("soft_heaviside",) if name == "soft_step" else (),
operator_role="Smooth Heaviside: sigmoid(beta z) -> step as beta -> inf.",
limit_pos_inf=1.0,
limit_neg_inf=0.0,
)FUNCTION
make_swish_spec
omnibias.keras.activations.tempered.make_swish_specSource ↗make_swish_spec(beta: float=_DEFAULT_BETA, *, name: str='swish') -> ActivationSpec[Any]Swish z * sigmoid(beta z) with tunable beta (silu is beta=1).
Parameters
| Name | Type & default | Contract |
|---|---|---|
betapositional-or-keyword | floatDefault: _DEFAULT_BETA | See the operation contract and implementation below. |
namekeyword-only | strDefault: 'swish' | See the operation contract and implementation below. |
Returns
ActivationSpec[Any]Read the implementation
def make_swish_spec(beta: float = _DEFAULT_BETA, *, name: str = "swish") -> ActivationSpec[Any]:
"""Swish ``z * sigmoid(beta z)`` with tunable ``beta`` (``silu`` is beta=1)."""
def fwd(z: Any) -> Any:
return z * ops.sigmoid(beta * z)
def fp(z: Any, n: int) -> Any:
if n < 0:
raise ValueError(f"order n must be >= 0, got {n}.")
if n == 0:
return fwd(z)
bz = beta * z
term_a = z * (beta**n) * sigmoid_nth_derivative(bz, n)
term_b = n * (beta ** (n - 1)) * sigmoid_nth_derivative(bz, n - 1)
return term_a + term_b
def deriv(z: Any) -> Any:
return fp(z, 1)
return ActivationSpec(
name=name,
forward=fwd,
derivative=deriv,
fastpath=fp,
riccati_polynomial=None,
noise_model="none",
operator_role="Swish: z * sigmoid(beta z); exact all-orders (silu is beta=1).",
)