Practice Questions — Large Language Models - GPT, BERT, and Beyond

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("gpt2")
tokens = tokenizer.tokenize("Hello world")
print(len(tokens))
print(tokens)

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("gpt2")
ids = tokenizer.encode("AI is great")
print(type(ids))
print(tokenizer.decode(ids))

temperatures = [0.0, 0.5, 1.0, 2.0]
for t in temperatures:
    if t == 0.0:
        print(f"Temp {t}: greedy (deterministic)")
    elif t < 1.0:
        print(f"Temp {t}: sharper distribution")
    elif t == 1.0:
        print(f"Temp {t}: original distribution")
    else:
        print(f"Temp {t}: flatter distribution")

models = {
    "BERT": "encoder-only",
    "GPT": "decoder-only",
    "T5": "encoder-decoder"
}
for name, arch in models.items():
    print(f"{name}: {arch}")

gpt_versions = [
    ("GPT-1", "117M"),
    ("GPT-2", "1.5B"),
    ("GPT-3", "175B"),
    ("GPT-4", "Multimodal")
]
for name, info in gpt_versions:
    print(f"{name}: {info}")

import math

def softmax_with_temperature(logits, temperature):
    if temperature == 0:
        result = [0.0] * len(logits)
        result[logits.index(max(logits))] = 1.0
        return result
    scaled = [x / temperature for x in logits]
    exp_scaled = [math.exp(x) for x in scaled]
    total = sum(exp_scaled)
    return [round(x / total, 3) for x in exp_scaled]

logits = [2.0, 1.0, 0.5]
print(softmax_with_temperature(logits, 0))
print(softmax_with_temperature(logits, 1.0))
print(softmax_with_temperature(logits, 0.5))

def top_k_filter(probs, k):
    sorted_probs = sorted(enumerate(probs), key=lambda x: x[1], reverse=True)
    top_k = sorted_probs[:k]
    total = sum(p for _, p in top_k)
    return [(idx, round(p/total, 3)) for idx, p in top_k]

probs = [0.4, 0.3, 0.15, 0.1, 0.05]
result = top_k_filter(probs, 3)
for idx, p in result:
    print(f"Token {idx}: {p}")

def top_p_filter(probs, p):
    sorted_probs = sorted(enumerate(probs), key=lambda x: x[1], reverse=True)
    cumulative = 0
    selected = []
    for idx, prob in sorted_probs:
        cumulative += prob
        selected.append((idx, prob))
        if cumulative >= p:
            break
    return len(selected)

probs = [0.4, 0.3, 0.15, 0.1, 0.05]
print(top_p_filter(probs, 0.5))
print(top_p_filter(probs, 0.7))
print(top_p_filter(probs, 0.9))

def count_parameters(total_params, lora_rank, num_layers, hidden_dim):
    # LoRA adds two matrices per layer: A (hidden_dim x rank) and B (rank x hidden_dim)
    lora_params = num_layers * 2 * hidden_dim * lora_rank
    percentage = (lora_params / total_params) * 100
    return lora_params, round(percentage, 2)

# GPT-2 small: 124M params, 12 layers, hidden_dim=768
lora_p, pct = count_parameters(124_000_000, 8, 12, 768)
print(f"LoRA params: {lora_p:,}")
print(f"Percentage: {pct}%")

def simulate_bpe(text, num_merges):
    # Start with character-level tokens
    tokens = list(text.replace(" ", "_ ").strip())
    print(f"Initial: {tokens}")
    
    for i in range(num_merges):
        # Count adjacent pairs
        pairs = {}
        for j in range(len(tokens) - 1):
            pair = (tokens[j], tokens[j+1])
            pairs[pair] = pairs.get(pair, 0) + 1
        
        if not pairs:
            break
        
        # Find most frequent pair
        best = max(pairs, key=pairs.get)
        merged = best[0] + best[1]
        
        # Merge all occurrences
        new_tokens = []
        j = 0
        while j < len(tokens):
            if j < len(tokens) - 1 and (tokens[j], tokens[j+1]) == best:
                new_tokens.append(merged)
                j += 2
            else:
                new_tokens.append(tokens[j])
                j += 1
        tokens = new_tokens
        print(f"Merge {i+1}: {best} -> '{merged}' | {tokens}")
    
    return tokens

result = simulate_bpe("ab ab ab cd", 2)
print(f"Final: {result}")

def few_shot_prompt(examples, query):
    prompt = "Classify the sentiment as Positive or Negative.\n\n"
    for text, label in examples:
        prompt += f"Text: {text}\nSentiment: {label}\n\n"
    prompt += f"Text: {query}\nSentiment:"
    return prompt

examples = [
    ("I love this product", "Positive"),
    ("Terrible experience", "Negative"),
    ("Absolutely wonderful", "Positive")
]

prompt = few_shot_prompt(examples, "Not worth the money")
lines = prompt.strip().split("\n")
print(f"Total lines: {len(lines)}")
print(f"Num examples: {len(examples)}")
print(f"Last line: {lines[-1]}")

class SimpleTokenizer:
    def __init__(self):
        self.vocab = {}
        self.next_id = 0
    
    def add_token(self, token):
        if token not in self.vocab:
            self.vocab[token] = self.next_id
            self.next_id += 1
    
    def encode(self, text):
        tokens = text.lower().split()
        return [self.vocab.get(t, -1) for t in tokens]
    
    def decode(self, ids):
        reverse = {v: k for k, v in self.vocab.items()}
        return " ".join(reverse.get(i, "[UNK]") for i in ids)

tok = SimpleTokenizer()
for word in ["hello", "world", "machine", "learning"]:
    tok.add_token(word)

print(tok.encode("hello machine learning"))
print(tok.encode("hello unknown world"))
print(tok.decode([0, 3, 2]))

pipeline_tasks = ["text-generation", "sentiment-analysis", "summarization", "ner", "question-answering"]
for task in pipeline_tasks:
    print(f"pipeline('{task}')")
print(f"Total tasks: {len(pipeline_tasks)}")

def compare_approaches(task, data_available, compute_available):
    if data_available > 1000 and compute_available:
        return "fine-tuning"
    elif data_available > 5:
        return "few-shot prompting"
    elif data_available > 0:
        return "one-shot prompting"
    else:
        return "zero-shot prompting"

scenarios = [
    ("Sentiment", 5000, True),
    ("Translation", 8, False),
    ("Summarization", 0, False),
    ("NER", 1, False)
]

for task, data, compute in scenarios:
    approach = compare_approaches(task, data, compute)
    print(f"{task:15s}: {approach}")

bert_tasks = ["Classification", "NER", "QA", "Similarity"]
gpt_tasks = ["Text Generation", "Completion", "Chat", "Code"]

print(f"BERT tasks: {len(bert_tasks)}")
print(f"GPT tasks: {len(gpt_tasks)}")
print(f"All tasks: {len(bert_tasks + gpt_tasks)}")

approach = {"fine-tuning": "updates weights", "prompting": "no weight change"}
for method, desc in approach.items():
    print(f"{method}: {desc}")

def estimate_memory_gb(params_billions, bytes_per_param):
    return round(params_billions * bytes_per_param, 1)

# FP32 = 4 bytes, FP16 = 2 bytes, INT8 = 1 byte, INT4 = 0.5 bytes
model_size = 7  # 7B parameter model

for precision, bpp in [("FP32", 4), ("FP16", 2), ("INT8", 1), ("INT4", 0.5)]:
    mem = estimate_memory_gb(model_size, bpp)
    print(f"{precision}: {mem} GB")

def zero_shot_vs_few_shot(num_examples):
    if num_examples == 0:
        return "zero-shot"
    elif num_examples == 1:
        return "one-shot"
    else:
        return "few-shot"

for n in [0, 1, 3, 5]:
    print(f"{n} examples: {zero_shot_vs_few_shot(n)}")

class LoRALayer:
    def __init__(self, in_dim, out_dim, rank):
        self.rank = rank
        self.a_shape = (in_dim, rank)
        self.b_shape = (rank, out_dim)
        self.trainable = in_dim * rank + rank * out_dim
        self.frozen = in_dim * out_dim
    
    def __repr__(self):
        pct = round(100 * self.trainable / (self.frozen + self.trainable), 2)
        return f"LoRA(rank={self.rank}, trainable={self.trainable}, frozen={self.frozen}, {pct}%)"

layer = LoRALayer(768, 768, 8)
print(layer)

import math

def perplexity(loss):
    return round(math.exp(loss), 2)

# Lower loss = lower perplexity = better model
losses = [4.5, 3.2, 2.1, 1.5]
for loss in losses:
    ppl = perplexity(loss)
    print(f"Loss: {loss} -> Perplexity: {ppl}")

def simulate_autoregressive(vocab, seed_tokens, steps):
    # Simulated next-token prediction (deterministic for demo)
    transitions = {
        "the": "model",
        "model": "predicts",
        "predicts": "the",
        "AI": "is",
        "is": "powerful",
        "powerful": "."
    }
    tokens = list(seed_tokens)
    for _ in range(steps):
        last = tokens[-1]
        next_token = transitions.get(last, "[END]")
        tokens.append(next_token)
    return " ".join(tokens)

print(simulate_autoregressive(None, ["the"], 5))
print(simulate_autoregressive(None, ["AI"], 3))

hf_components = ["AutoTokenizer", "AutoModel", "pipeline", "Trainer", "TrainingArguments"]
for comp in hf_components:
    print(f"from transformers import {comp}")
print(f"\nTotal imports: {len(hf_components)}")

def token_count_estimate(text, chars_per_token=4):
    """Rough estimate: ~4 characters per token for English text."""
    return len(text) // chars_per_token

texts = [
    "Hello world",
    "Machine learning is a subset of artificial intelligence",
    "The quick brown fox jumps over the lazy dog" * 10
]

for text in texts:
    tokens = token_count_estimate(text)
    print(f"Chars: {len(text):4d} | ~Tokens: {tokens:4d} | Text: {text[:40]}...")

def compute_attention_memory(seq_len, hidden_dim, num_heads, dtype_bytes=2):
    """Estimate memory for self-attention computation."""
    # Attention matrix: seq_len x seq_len per head
    attention_memory = seq_len * seq_len * num_heads * dtype_bytes
    # Q, K, V projections
    qkv_memory = 3 * seq_len * hidden_dim * dtype_bytes
    total_mb = (attention_memory + qkv_memory) / (1024 * 1024)
    return round(total_mb, 2)

for seq_len in [512, 2048, 8192, 32768]:
    mem = compute_attention_memory(seq_len, 4096, 32)
    print(f"Seq length {seq_len:6d}: ~{mem:8.2f} MB attention memory")