Generative AI और LLM Basics

# a toy tokenizer: split into words and give each new word an id
sentence = "the shop sells fresh sweets and the shop opens early"
vocab = {}
ids = []
for word in sentence.split():
    if word not in vocab:
        vocab[word] = len(vocab)
    ids.append(vocab[word])

print("tokens:", sentence.split())
print("ids   :", ids)
print("vocabulary size:", len(vocab))

# bytes show why non-English text is “longer” for machines
for text in ["sweet", "मिठाई"]:
    print(f"{text!r}: {len(text)} characters, {len(text.encode('utf-8'))} UTF-8 bytes")

import random
from collections import Counter, defaultdict

corpus = "the shop opens at nine . the shop sells fresh sweets . fresh sweets are made every morning . the owner makes kaju katli every morning . customers buy fresh kaju katli . the shop closes at nine at night . customers love the shop ."
words = corpus.split()

# 1. TRAINING: count next-word frequencies
next_counts = defaultdict(Counter)
for current, nxt in zip(words, words[1:]):
    next_counts[current][nxt] += 1

print("after 'fresh':", dict(next_counts["fresh"]))
print("after 'the'  :", dict(next_counts["the"]))

# 2. GENERATION: repeatedly sample the next word
def generate(start, length=10, seed=1):
    rng = random.Random(seed)
    out = [start]
    for _ in range(length):
        options = next_counts[out[-1]]
        if not options:
            break
        out.append(rng.choices(list(options), weights=options.values())[0])
    return " ".join(out)

for seed in (1, 2, 3):
    print("→", generate("the", seed=seed))

import math

next_words = ["sweets", "kaju", "snacks", "flowers"]
scores = [3.0, 2.2, 1.0, -1.0]          # model scores (logits) after "the shop sells fresh"

def softmax(values, temperature):
    exps = [math.exp(v / temperature) for v in values]
    total = sum(exps)
    return [e / total for e in exps]

for t in (0.2, 1.0, 2.0):
    probs = softmax(scores, t)
    row = "  ".join(f"{w}={p:.0%}" for w, p in zip(next_words, probs))
    print(f"temperature {t}: {row}")

documents = {
    "fees": "School fees are due on the 10th of every month. A late fee of Rs 100 applies after the 15th.",
    "timing": "Classes run from 8:00 am to 2:00 pm, Monday to Saturday.",
    "transport": "Bus service is available for Khurja and Bulandshahr routes at Rs 900 per month.",
}

def retrieve(question, k=1):
    q = set(question.lower().replace("?", "").split())
    scored = sorted(documents.items(), key=lambda d: -len(q & set(d[1].lower().split())))
    return [text for _, text in scored[:k]]

question = "What is the late fee if I pay after the 15th?"
context = retrieve(question)[0]
prompt = f"""Answer using ONLY the context. If the answer is not there, say you don't know.
Context: {context}
Question: {question}"""
print(prompt)

# pip install anthropic       (set ANTHROPIC_API_KEY and LLM_MODEL in your environment)
import os
import anthropic

client = anthropic.Anthropic()                      # reads ANTHROPIC_API_KEY automatically
reply = client.messages.create(
    model=os.environ["LLM_MODEL"],                  # e.g. the current model name from the docs
    max_tokens=300,
    temperature=0.3,
    system="You are a patient tutor for Class 10 students. Answer in simple Hindi and English.",
    messages=[{"role": "user", "content": "Explain photosynthesis in 5 short points."}],
)
print(reply.content[0].text)