Keras Basics

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, losses, metrics, optimizers, callbacks

model = keras.Sequential([
    layers.Input(shape=(784,)),
    layers.Dense(256, activation='relu'),
    layers.Dropout(0.3),
    layers.Dense(10, activation='softmax'),
], name='mlp')

# Add layers later
model.add(layers.Dense(64, activation='relu'))
model.pop()   # remove last layer

inputs = keras.Input(shape=(32,), name='my_input')
x = layers.Dense(64, activation='relu')(inputs)
x = layers.Dense(64, activation='relu')(x)
outputs = layers.Dense(10, activation='softmax')(x)

model = keras.Model(inputs=inputs, outputs=outputs, name='my_model')

class MyModel(keras.Model):
    def __init__(self):
        super().__init__()
        self.dense1 = layers.Dense(64, activation='relu')
        self.dense2 = layers.Dense(10, activation='softmax')

    def call(self, inputs, training=False):
        x = self.dense1(inputs)
        if training:
            x = tf.nn.dropout(x, rate=0.3)
        return self.dense2(x)

model = MyModel()

model.summary()                    # layer table with param counts
keras.utils.plot_model(model, show_shapes=True)   # requires pydot

model.layers                       # list of Layer objects
model.get_layer('dense')           # by name
model.inputs                       # input tensors
model.outputs                      # output tensors
model.count_params()
model.trainable_variables          # list of tf.Variable
model.non_trainable_variables

d = layers.Dense(64, activation='relu')

# Before build: no weights yet
d.build((None, 32))   # input_shape
d.weights             # [kernel, bias]
d.trainable_weights
d.non_trainable_weights
d.kernel              # shorthand for Dense
d.bias

# Freeze a layer
d.trainable = False

# Pass as string
layers.Dense(64, activation='relu')

# Pass as function
layers.Dense(64, activation=tf.nn.relu)

# Pass as callable instance
layers.Dense(64, activation=keras.activations.relu)

# Separate activation layer
layers.Activation('gelu')
layers.LeakyReLU(negative_slope=0.1)
layers.PReLU()   # learned alpha

layers.Dense(64,
    kernel_initializer='glorot_uniform',   # default
    bias_initializer='zeros',
)

# Common initializers (string or instance)
'glorot_uniform'      # Xavier uniform — default for Dense
'glorot_normal'       # Xavier normal
'he_uniform'          # Kaiming uniform — good for ReLU
'he_normal'           # Kaiming normal
'lecun_uniform'
'orthogonal'
'truncated_normal'

# As objects (allows extra args)
layers.Dense(64, kernel_initializer=keras.initializers.HeNormal(seed=42))

from tensorflow.keras import regularizers

layers.Dense(64,
    kernel_regularizer=regularizers.L2(0.01),
    bias_regularizer=regularizers.L1(0.001),
    activity_regularizer=regularizers.L1L2(l1=1e-5, l2=1e-4),
)

from tensorflow.keras import constraints

layers.Dense(64,
    kernel_constraint=constraints.MaxNorm(max_value=2.0),
    bias_constraint=constraints.NonNeg(),
)

# Available constraints
constraints.MaxNorm(max_value, axis=0)
constraints.MinMaxNorm(min_value=0.0, max_value=1.0)
constraints.NonNeg()
constraints.UnitNorm(axis=0)

class LinearWithBias(keras.Layer):
    def __init__(self, units, **kwargs):
        super().__init__(**kwargs)
        self.units = units

    def build(self, input_shape):
        self.w = self.add_weight(
            shape=(input_shape[-1], self.units),
            initializer='glorot_uniform',
            trainable=True,
            name='kernel',
        )
        self.b = self.add_weight(
            shape=(self.units,),
            initializer='zeros',
            trainable=True,
            name='bias',
        )

    def call(self, inputs):
        return tf.matmul(inputs, self.w) + self.b

    def get_config(self):               # required for serialization
        config = super().get_config()
        config.update({'units': self.units})
        return config

keras.mixed_precision.set_global_policy('mixed_float16')

# Now Dense layers use float16 compute but float32 weights
model = keras.Sequential([
    layers.Dense(256, activation='relu'),
    layers.Dense(10, activation='softmax', dtype='float32'),   # keep output float32
])

optimizer = keras.mixed_precision.LossScaleOptimizer(
    keras.optimizers.Adam(1e-3)
)