tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim)

pip install tensorflow numpy

import tensorflow as tf
import numpy as np

# Load text data
with open("your_dataset.txt", "r", encoding="utf-8") as f:
    text = f.read()

# Create character-level vocabulary
vocab = sorted(set(text))
char2idx = {u: i for i, u in enumerate(vocab)}
idx2char = np.array(vocab)

# Convert to int
text_as_int = np.array([char2idx[c] for c in text])

# Define sequence length
seq_length = 100
examples_per_epoch = len(text) // seq_length

# Create training dataset
char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)
sequences = char_dataset.batch(seq_length+1, drop_remainder=True)

def split_input_target(chunk):
    input_text = chunk[:-1]
    target_text = chunk[1:]
    return input_text, target_text

dataset = sequences.map(split_input_target)

# Batch size
BATCH_SIZE = 64
BUFFER_SIZE = 10000
dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)

from tensorflow.keras import layers

# Model config
vocab_size = len(vocab)
embedding_dim = 256
rnn_units = 1024

def build_model(vocab_size, embedding_dim, rnn_units, batch_size):
    model = tf.keras.Sequential([
        layers.Embedding(vocab_size, embedding_dim,
                         batch_input_shape=[batch_size, None]),
        layers.GRU(rnn_units,
                   return_sequences=True,
                   stateful=True,
                   recurrent_initializer='glorot_uniform'),
        layers.Dense(vocab_size)
    ])
    return model

model = build_model(vocab_size, embedding_dim, rnn_units, BATCH_SIZE)
model.summary()

def loss(labels, logits):
    return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)

model.compile(optimizer='adam', loss=loss)

# Checkpoint callback
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
    filepath="./checkpoints/ckpt_{epoch}",
    save_weights_only=True
)

EPOCHS = 10
history = model.fit(dataset, epochs=EPOCHS, callbacks=[checkpoint_callback])

model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)
model.load_weights(tf.train.latest_checkpoint('./checkpoints'))
model.build(tf.TensorShape([1, None]))

def generate_text(model, start_string, num_generate=500, temperature=1.0):
    input_eval = [char2idx[s] for s in start_string]
    input_eval = tf.expand_dims(input_eval, 0)
    text_generated = []

    model.reset_states()
    for _ in range(num_generate):
        predictions = model(input_eval)
        predictions = predictions[:, -1, :] / temperature
        predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()

        input_eval = tf.expand_dims([predicted_id], 0)
        text_generated.append(idx2char[predicted_id])

    return start_string + ''.join(text_generated)

print(generate_text(model, start_string="Once upon a time, "))

pip install tensorflow numpy

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import numpy as np

# Load your dataset
with open("your_dataset.txt", "r", encoding="utf-8") as f:
    text = f.read()

# Build vocabulary
vocab = sorted(set(text))
char2idx = {u: i for i, u in enumerate(vocab)}
idx2char = np.array(vocab)

# Encode text
text_as_int = np.array([char2idx[c] for c in text])

# Create input-target pairs
seq_length = 128
examples_per_epoch = len(text_as_int) // seq_length

char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)
sequences = char_dataset.batch(seq_length + 1, drop_remainder=True)

def split_input_target(chunk):
    input_text = chunk[:-1]
    target_text = chunk[1:]
    return input_text, target_text

dataset = sequences.map(split_input_target)
BATCH_SIZE = 64
BUFFER_SIZE = 10000

dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)

class PositionalEmbedding(layers.Layer):
    def __init__(self, vocab_size, d_model, max_len):
        super().__init__()
        self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=d_model)
        self.pos_emb = layers.Embedding(input_dim=max_len, output_dim=d_model)

    def call(self, x):
        maxlen = tf.shape(x)[-1]
        positions = tf.range(start=0, limit=maxlen, delta=1)
        positions = self.pos_emb(positions)
        x = self.token_emb(x)
        return x + positions

class CausalSelfAttention(layers.Layer):
    def __init__(self, d_model, num_heads):
        super().__init__()
        self.mha = layers.MultiHeadAttention(num_heads=num_heads, key_dim=d_model)
        self.layernorm = layers.LayerNormalization(epsilon=1e-6)
        self.dropout = layers.Dropout(0.1)

    def call(self, x, training):
        attn_output = self.mha(query=x, value=x, key=x, attention_mask=self._causal_mask(tf.shape(x)[1]))
        attn_output = self.dropout(attn_output, training=training)
        return self.layernorm(x + attn_output)

    def _causal_mask(self, size):
        i = tf.range(size)[:, None]
        j = tf.range(size)
        mask = tf.cast(i >= j, dtype=tf.int32)
        return mask[None, None, :, :]

class FeedForward(layers.Layer):
    def __init__(self, d_model, d_ff):
        super().__init__()
        self.seq = keras.Sequential([
            layers.Dense(d_ff, activation='relu'),
            layers.Dense(d_model),
            layers.Dropout(0.1)
        ])
        self.layernorm = layers.LayerNormalization(epsilon=1e-6)

    def call(self, x, training):
        out = self.seq(x, training=training)
        return self.layernorm(x + out)

class TransformerBlock(layers.Layer):
    def __init__(self, d_model, num_heads, d_ff):
        super().__init__()
        self.att = CausalSelfAttention(d_model, num_heads)
        self.ff = FeedForward(d_model, d_ff)

    def call(self, x, training):
        x = self.att(x, training=training)
        x = self.ff(x, training=training)
        return x

def build_gpt_model(vocab_size, seq_len, d_model=256, num_heads=4, d_ff=512, num_layers=4):
    inputs = layers.Input(shape=(seq_len,))
    x = PositionalEmbedding(vocab_size, d_model, seq_len)(inputs)

    for _ in range(num_layers):
        x = TransformerBlock(d_model, num_heads, d_ff)(x)

    outputs = layers.Dense(vocab_size)(x)
    return keras.Model(inputs=inputs, outputs=outputs)

model = build_gpt_model(
    vocab_size=len(vocab),
    seq_len=seq_length,
    d_model=256,
    num_heads=4,
    d_ff=512,
    num_layers=4
)

loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)

model.compile(optimizer="adam", loss=loss_fn)
model.summary()

EPOCHS = 10
model.fit(dataset, epochs=EPOCHS)

def generate_text(model, start_string, gen_length=500, temperature=1.0):
    input_indices = [char2idx[s] for s in start_string]
    input_tensor = tf.expand_dims(input_indices, 0)

    generated = []
    for _ in range(gen_length):
        input_padded = tf.keras.preprocessing.sequence.pad_sequences(input_tensor, maxlen=seq_length, padding='pre')
        predictions = model(tf.convert_to_tensor(input_padded))[:, -1, :]
        predictions /= temperature
        predicted_id = tf.random.categorical(predictions, num_samples=1)[0, 0].numpy()

        input_tensor = tf.concat([input_tensor, [[predicted_id]]], axis=-1)
        generated.append(idx2char[predicted_id])

    return start_string + ''.join(generated)

print(generate_text(model, start_string="Once upon a time, "))

llm_gpt_keras/
├── data/
│   └── your_dataset.txt
├── model/
│   ├── transformer_blocks.py
│   ├── gpt_model.py
├── train.py
├── generate.py
├── requirements.txt

tensorflow>=2.10
numpy

import tensorflow as tf
from tensorflow.keras import layers

class PositionalEmbedding(layers.Layer):
    def __init__(self, vocab_size, d_model, max_len):
        super().__init__()
        self.token_emb = layers.Embedding(vocab_size, d_model)
        self.pos_emb = layers.Embedding(max_len, d_model)

    def call(self, x):
        positions = tf.range(start=0, limit=tf.shape(x)[-1], delta=1)
        positions = self.pos_emb(positions)
        x = self.token_emb(x)
        return x + positions

class CausalSelfAttention(layers.Layer):
    def __init__(self, d_model, num_heads):
        super().__init__()
        self.mha = layers.MultiHeadAttention(num_heads=num_heads, key_dim=d_model)
        self.dropout = layers.Dropout(0.1)
        self.layernorm = layers.LayerNormalization(epsilon=1e-6)

    def call(self, x, training):
        attn = self.mha(query=x, value=x, key=x, attention_mask=self._causal_mask(tf.shape(x)[1]))
        x = self.layernorm(x + self.dropout(attn, training=training))
        return x

    def _causal_mask(self, size):
        i = tf.range(size)[:, None]
        j = tf.range(size)
        mask = tf.cast(i >= j, dtype=tf.int32)
        return mask[None, None, :, :]

class FeedForward(layers.Layer):
    def __init__(self, d_model, d_ff):
        super().__init__()
        self.ff = tf.keras.Sequential([
            layers.Dense(d_ff, activation='relu'),
            layers.Dense(d_model),
            layers.Dropout(0.1)
        ])
        self.layernorm = layers.LayerNormalization(epsilon=1e-6)

    def call(self, x, training):
        return self.layernorm(x + self.ff(x, training=training))

class TransformerBlock(layers.Layer):
    def __init__(self, d_model, num_heads, d_ff):
        super().__init__()
        self.att = CausalSelfAttention(d_model, num_heads)
        self.ff = FeedForward(d_model, d_ff)

    def call(self, x, training):
        x = self.att(x, training)
        x = self.ff(x, training)
        return x

from tensorflow.keras import layers, Model, Input
from model.transformer_blocks import PositionalEmbedding, TransformerBlock

def build_gpt_model(vocab_size, seq_len, d_model=256, num_heads=4, d_ff=512, num_layers=4):
    inputs = Input(shape=(seq_len,))
    x = PositionalEmbedding(vocab_size, d_model, seq_len)(inputs)

    for _ in range(num_layers):
        x = TransformerBlock(d_model, num_heads, d_ff)(x)

    outputs = layers.Dense(vocab_size)(x)
    return Model(inputs, outputs)

import tensorflow as tf
import numpy as np
from model.gpt_model import build_gpt_model

# Load data
with open("data/your_dataset.txt", "r", encoding="utf-8") as f:
    text = f.read()

vocab = sorted(set(text))
char2idx = {u: i for i, u in enumerate(vocab)}
idx2char = np.array(vocab)
text_as_int = np.array([char2idx[c] for c in text])

seq_length = 128
examples_per_epoch = len(text_as_int) // (seq_length + 1)
char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)
sequences = char_dataset.batch(seq_length + 1, drop_remainder=True)

def split_input_target(chunk):
    return chunk[:-1], chunk[1:]

dataset = sequences.map(split_input_target).shuffle(10000).batch(64, drop_remainder=True)

# Model setup
model = build_gpt_model(
    vocab_size=len(vocab),
    seq_len=seq_length,
    d_model=256,
    num_heads=4,
    d_ff=512,
    num_layers=4
)

model.compile(optimizer="adam", loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True))
model.fit(dataset, epochs=10)
model.save_weights("checkpoints/gpt_small.h5")

import tensorflow as tf
from model.gpt_model import build_gpt_model
import numpy as np

# Load vocabulary
with open("data/your_dataset.txt", "r", encoding="utf-8") as f:
    text = f.read()

vocab = sorted(set(text))
char2idx = {u: i for i, u in enumerate(vocab)}
idx2char = np.array(vocab)

seq_length = 128
model = build_gpt_model(len(vocab), seq_length)
model.load_weights("checkpoints/gpt_small.h5")

def generate_text(start_string, num_generate=500, temperature=1.0):
    input_eval = [char2idx[c] for c in start_string]
    input_eval = tf.expand_dims(input_eval, 0)
    generated = []

    for _ in range(num_generate):
        input_padded = tf.keras.preprocessing.sequence.pad_sequences(
            input_eval, maxlen=seq_length, padding='pre'
        )
        predictions = model(tf.convert_to_tensor(input_padded))[:, -1, :]
        predictions = predictions / temperature
        predicted_id = tf.random.categorical(predictions, num_samples=1)[0, 0].numpy()
        input_eval = tf.concat([input_eval, [[predicted_id]]], axis=-1)
        generated.append(idx2char[predicted_id])

    return start_string + ''.join(generated)

print(generate_text("Once upon a time, "))