git clone https://github.com/SDAIAAcademy/applied-nlp
cd applied-nlp/labs
uv sync                 # creates .venv from pyproject
uv run python prepare_data.py
uv run jupyter lab

from transformers import AutoTokenizer

tok = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-base")
batch = tok(
    ["Where is my order?", "أين طلبي؟ لم يصل بعد"],
    padding=True,          # pad to longest in batch
    truncation=True,       # cut at max_length
    max_length=32,
    return_tensors="pt",
)
batch["input_ids"].shape       # torch.Size([2, n])
batch["attention_mask"][0]     # 1 = real token, 0 = pad
tok.convert_ids_to_tokens(batch["input_ids"][1])
tok.model_max_length           # 512

• Mixing a model with another model's tokenizer
• Truncating the important part (e.g., the actual question at the end of a long email)
• Estimating cost with word counts instead of token counts
• Over-normalizing input to an LLM (removing diacritics or casing it relies on)

• Forgetting the chat template for instruction models (Day 2)
• Losing character offsets needed to highlight extracted spans
• Assuming the context limit applies only to input. Output tokens count too.
• Not testing on dialect, typos, emoji and code-switched text

• Exact identifiers: order #A-55821, product codes, IBANs. Embeddings blur these, so add keyword search (BM25) and combine → hybrid.
• Negation: "I don't want a refund" sits close to "I want a refund".
• Length mismatch: a 5-word query vs. a 2-page document; chunk the documents (Day 2).

• Domain jargon the model never saw → evaluate, then consider fine-tuning embeddings.
• Cross-lingual gaps: dialect queries against MSA or English documents. Test them explicitly.
• Stale index: new policies never re-embedded.

from transformers import AutoTokenizer, AutoModel
import torch

name = "google-bert/bert-base-multilingual-cased"
tok = AutoTokenizer.from_pretrained(name)
model = AutoModel.from_pretrained(name, output_attentions=True)

enc = tok("You charged me twice for one order", return_tensors="pt")
with torch.no_grad():
    out = model(**enc)

out.last_hidden_state.shape  # (batch, seq_len, hidden) = (1, n, 768)
len(out.attentions)          # 12 layers
out.attentions[0].shape      # (1, heads=12, n, n)

import torch
from transformers import pipeline

clf = pipeline(
    "zero-shot-classification",
    model="MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7",
    dtype=torch.float32,   # checkpoint is fp16: slow on CPU
)
labels = ["refund", "delivery", "payment problem",
          "account access", "order changes", "invoice"]
clf("لقد تم خصم المبلغ مرتين من بطاقتي",
    candidate_labels=labels,
    hypothesis_template="This customer message is about {}.")
# {'labels': [...], 'scores': [...]}  label wording changes the ranking

from sentence_transformers import SentenceTransformer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report

enc = SentenceTransformer("intfloat/multilingual-e5-small")
X_tr = enc.encode(["query: " + t for t in train.text], normalize_embeddings=True)
X_te = enc.encode(["query: " + t for t in test.text],  normalize_embeddings=True)

clf = LogisticRegression(max_iter=2000, C=4.0)
clf.fit(X_tr, train.intent)

pred = clf.predict(X_te)
proba = clf.predict_proba(X_te).max(axis=1)   # confidence
print(classification_report(test.intent, pred, digits=3))