langml.layers

Submodules

Package Contents

Classes

CRF

LayerNorm

AbsolutePositionEmbedding

SineCosinePositionEmbedding

Sine Cosine Position Embedding.

ScaleOffset

Scale Offset

ConditionalLayerNormalization

Conditional Layer Normalization

SelfAttention

SelfAdditiveAttention

ScaledDotProductAttention

ScaledDotProductAttention

MultiHeadAttention

MultiHeadAttention

GatedAttentionUnit

Gated Attention Unit

Attributes

TF_KERAS

custom_objects

langml.layers.TF_KERAS[source]
class langml.layers.CRF(output_dim: int, sparse_target: bool = True, **kwargs)[source]

Bases: tensorflow.keras.layers.Layer

build(self, input_shape: langml.tensor_typing.Tensors)
compute_mask(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None)
call(self, inputs: langml.tensor_typing.Tensors, sequence_lengths: Optional[langml.tensor_typing.Tensors] = None, training: Optional[Union[bool, int]] = None, mask: Optional[langml.tensor_typing.Tensors] = None, **kwargs) → langml.tensor_typing.Tensors
property loss(self) → Callable
property accuracy(self) → Callable
compute_output_shape(self, input_shape: langml.tensor_typing.Tensors) → langml.tensor_typing.Tensors
property trans(self) → langml.tensor_typing.Tensors

transition parameters

get_config(self) → dict
static get_custom_objects() → dict
class langml.layers.LayerNorm(center: bool = True, scale: bool = True, epsilon: float = 1e-07, gamma_initializer: langml.tensor_typing.Initializer = 'ones', gamma_regularizer: Optional[langml.tensor_typing.Regularizer] = None, gamma_constraint: Optional[langml.tensor_typing.Constraint] = None, beta_initializer: langml.tensor_typing.Initializer = 'zeros', beta_regularizer: Optional[langml.tensor_typing.Regularizer] = None, beta_constraint: Optional[langml.tensor_typing.Constraint] = None, **kwargs)[source]

Bases: tensorflow.keras.layers.Layer

get_config(self) → dict
build(self, input_shape: langml.tensor_typing.Tensors)
call(self, inputs: langml.tensor_typing.Tensors, **kwargs) → langml.tensor_typing.Tensors
compute_mask(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None) → Union[langml.tensor_typing.Tensors, None]
static get_custom_objects() → dict
compute_output_shape(self, input_shape: langml.tensor_typing.Tensors) → langml.tensor_typing.Tensors
class langml.layers.AbsolutePositionEmbedding(input_dim: int, output_dim: int, mode: str = 'add', embeddings_initializer: langml.tensor_typing.Initializer = 'uniform', embeddings_regularizer: Optional[langml.tensor_typing.Regularizer] = None, embeddings_constraint: Optional[langml.tensor_typing.Constraint] = None, mask_zero: bool = False, **kwargs)[source]

Bases: langml.L.Layer

get_config(self) → dict
static get_custom_objects() → dict
build(self, input_shape: langml.tensor_typing.Tensors)
compute_mask(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None) → langml.tensor_typing.Tensors
compute_output_shape(self, input_shape: langml.tensor_typing.Tensors) → langml.tensor_typing.Tensors
call(self, inputs: langml.tensor_typing.Tensors, **kwargs) → langml.tensor_typing.Tensors
class langml.layers.SineCosinePositionEmbedding(mode: str = 'add', output_dim: Optional[int] = None, **kwargs)[source]

Bases: langml.L.Layer

Sine Cosine Position Embedding. https://arxiv.org/pdf/1706.03762

get_config(self)
static get_custom_objects() → dict
compute_mask(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None) → langml.tensor_typing.Tensors
compute_output_shape(self, input_shape: langml.tensor_typing.Tensors) → langml.tensor_typing.Tensors
call(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None, **kwargs) → langml.tensor_typing.Tensors
class langml.layers.ScaleOffset(scale: bool = True, offset: bool = True, **kwargs)[source]

Bases: langml.L.Layer

Scale Offset

get_config(self)
build(self, input_shape: langml.tensor_typing.Tensors)
compute_mask(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None)
call(self, inputs: langml.tensor_typing.Tensors) → langml.tensor_typing.Tensors
compute_output_shape(self, input_shape: langml.tensor_typing.Tensors) → langml.tensor_typing.Tensors
static get_custom_objects() → dict
class langml.layers.ConditionalLayerNormalization(center: bool = True, epsilon: Optional[float] = None, scale: bool = True, offset: bool = True, **kwargs)[source]

Bases: langml.L.Layer

Conditional Layer Normalization https://arxiv.org/abs/2108.00449

get_config(self)
build(self, input_shapes: langml.tensor_typing.Tensors)
compute_mask(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None)
call(self, inputs: List[langml.tensor_typing.Tensors]) → langml.tensor_typing.Tensors
compute_output_shape(self, input_shape: langml.tensor_typing.Tensors) → langml.tensor_typing.Tensors
static get_custom_objects() → dict
class langml.layers.SelfAttention(attention_units: Optional[int] = None, return_attention: bool = False, is_residual: bool = False, attention_activation: langml.tensor_typing.Activation = 'relu', attention_epsilon: float = 10000000000.0, kernel_initializer: langml.tensor_typing.Initializer = 'glorot_normal', kernel_regularizer: Optional[langml.tensor_typing.Regularizer] = None, kernel_constraint: Optional[langml.tensor_typing.Constraint] = None, bias_initializer: langml.tensor_typing.Initializer = 'zeros', bias_regularizer: Optional[langml.tensor_typing.Regularizer] = None, bias_constraint: Optional[langml.tensor_typing.Constraint] = None, use_attention_bias: bool = True, attention_penalty_weight: float = 0.0, **kwargs)[source]

Bases: tensorflow.keras.layers.Layer

get_config(self) → dict
build(self, input_shape: langml.tensor_typing.Tensors)
call(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None, **kwargs) → Union[List[langml.tensor_typing.Tensors], langml.tensor_typing.Tensors]
compute_mask(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None) → Union[List[Union[langml.tensor_typing.Tensors, None]], langml.tensor_typing.Tensors]
_attention_penalty(self, attention: langml.tensor_typing.Tensors) → langml.tensor_typing.Tensors
static get_custom_objects() → dict
compute_output_shape(self, input_shape: langml.tensor_typing.Tensors) → Union[List[langml.tensor_typing.Tensors], langml.tensor_typing.Tensors]
class langml.layers.SelfAdditiveAttention(attention_units: Optional[int] = None, return_attention: bool = False, is_residual: bool = False, attention_activation: langml.tensor_typing.Activation = 'relu', attention_epsilon: float = 10000000000.0, kernel_initializer: langml.tensor_typing.Initializer = 'glorot_normal', kernel_regularizer: Optional[langml.tensor_typing.Regularizer] = None, kernel_constraint: Optional[langml.tensor_typing.Constraint] = None, bias_initializer: langml.tensor_typing.Initializer = 'zeros', bias_regularizer: Optional[langml.tensor_typing.Regularizer] = None, bias_constraint: Optional[langml.tensor_typing.Constraint] = None, use_attention_bias: bool = True, attention_penalty_weight: float = 0.0, **kwargs)[source]

Bases: tensorflow.keras.layers.Layer

get_config(self) → dict
build(self, input_shape: langml.tensor_typing.Tensors)
call(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None, **kwargs) → Union[List[langml.tensor_typing.Tensors], langml.tensor_typing.Tensors]
compute_mask(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None) → Union[List[Union[langml.tensor_typing.Tensors, None]], langml.tensor_typing.Tensors]
_attention_penalty(self, attention: langml.tensor_typing.Tensors) → langml.tensor_typing.Tensors
static get_custom_objects() → dict
compute_output_shape(self, input_shape: langml.tensor_typing.Tensors) → Union[List[langml.tensor_typing.Tensors], langml.tensor_typing.Tensors]
class langml.layers.ScaledDotProductAttention(return_attention: bool = False, history_only: bool = False, **kwargs)[source]

Bases: tensorflow.keras.layers.Layer

ScaledDotProductAttention

$Attention(Q, K, V) = softmax(frac{Q K^T}{sqrt{d_k}}) V$

https://arxiv.org/pdf/1706.03762.pdf

get_config(self) → dict
call(self, inputs: langml.tensor_typing.Tensors, mask: Optional[Union[langml.tensor_typing.Tensors, List[langml.tensor_typing.Tensors]]] = None, **kwargs) → Union[List[langml.tensor_typing.Tensors], langml.tensor_typing.Tensors]
compute_mask(self, inputs: langml.tensor_typing.Tensors, mask: Optional[Union[langml.tensor_typing.Tensors, List[langml.tensor_typing.Tensors]]] = None) → Union[List[Union[langml.tensor_typing.Tensors, None]], langml.tensor_typing.Tensors]
static get_custom_objects() → dict
compute_output_shape(self, input_shape: Union[langml.tensor_typing.Tensors, List[langml.tensor_typing.Tensors]]) → Union[List[langml.tensor_typing.Tensors], langml.tensor_typing.Tensors]
class langml.layers.MultiHeadAttention(head_num: int, return_attention: bool = False, attention_activation: langml.tensor_typing.Activation = 'relu', kernel_initializer: langml.tensor_typing.Initializer = 'glorot_normal', kernel_regularizer: Optional[langml.tensor_typing.Regularizer] = None, kernel_constraint: Optional[langml.tensor_typing.Constraint] = None, bias_initializer: langml.tensor_typing.Initializer = 'zeros', bias_regularizer: Optional[langml.tensor_typing.Regularizer] = None, bias_constraint: Optional[langml.tensor_typing.Constraint] = None, use_attention_bias: bool = True, history_only: bool = False, **kwargs)[source]

Bases: tensorflow.keras.layers.Layer

MultiHeadAttention https://arxiv.org/pdf/1706.03762.pdf

get_config(self) → dict
build(self, input_shape: langml.tensor_typing.Tensors)
static _reshape_to_batches(x, head_num)
static _reshape_attention_from_batches(x, head_num)
static _reshape_from_batches(x, head_num)
static _reshape_mask(mask, head_num)
call(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None, **kwargs) → langml.tensor_typing.Tensors
static get_custom_objects() → dict
compute_mask(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None) → Union[List[Union[langml.tensor_typing.Tensors, None]], langml.tensor_typing.Tensors]
compute_output_shape(self, input_shape: Union[langml.tensor_typing.Tensors, List[langml.tensor_typing.Tensors]]) → Union[List[langml.tensor_typing.Tensors], langml.tensor_typing.Tensors]
class langml.layers.GatedAttentionUnit(attention_units: int, attention_activation: langml.tensor_typing.Activation = 'relu', attention_normalizer: langml.tensor_typing.Activation = relu2, attention_epsilon: float = 10000000000.0, kernel_initializer: langml.tensor_typing.Initializer = 'glorot_normal', kernel_regularizer: Optional[langml.tensor_typing.Regularizer] = None, kernel_constraint: Optional[langml.tensor_typing.Constraint] = None, bias_initializer: langml.tensor_typing.Initializer = 'zeros', bias_regularizer: Optional[langml.tensor_typing.Regularizer] = None, bias_constraint: Optional[langml.tensor_typing.Constraint] = None, use_attention_bias: bool = True, use_attention_scale: bool = True, use_relative_position: bool = True, use_offset: bool = True, use_scale: bool = True, is_residual: bool = True, **kwargs)[source]

Bases: tensorflow.keras.layers.Layer

Gated Attention Unit https://arxiv.org/abs/2202.10447

get_config(self) → dict
build(self, input_shape: langml.tensor_typing.Tensors)
apply_rotary_position_embeddings(self, sinusoidal: langml.tensor_typing.Tensors, *tensors)

apply RoPE modified from: https://github.com/bojone/bert4keras/blob/master/bert4keras/backend.py#L310

attn(self, x: langml.tensor_typing.Tensors, v: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None) → langml.tensor_typing.Tensors
call(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None, **kwargs) → langml.tensor_typing.Tensors
compute_mask(self, inputs: langml.tensor_typing.Tensors, mask: Optional[langml.tensor_typing.Tensors] = None) → langml.tensor_typing.Tensors
compute_output_shape(self, input_shape: langml.tensor_typing.Tensors) → langml.tensor_typing.Tensors
static get_custom_objects() → dict
langml.layers.custom_objects[source]