Small Language Models with Hugging Face transformers Library + smolLM3

# Python 3.10 or newer required
python --version

# Create and activate a virtual environment
python -m venv smollm-env
source smollm-env/bin/activate       # macOS / Linux
smollm-env\Scripts\activate          # Windows

# Install all dependencies
pip install \
  "transformers>=4.53.0" \
  "torch>=2.3.0" \
  "accelerate>=0.30.0" \
  "bitsandbytes>=0.43.0" \
  "sentencepiece" \
  "trl>=0.9.0" \
  "peft>=0.11.0" \
  "datasets>=2.19.0"

# device_check.py
# Run this before anything else to confirm your setup and pick the right dtype.

def detect_device():
    """
    Detect the best available compute device.
    Returns (device_str, dtype_str, load_kwargs) for use with from_pretrained.
    """
    try:
        import torch
    except ImportError:
        raise RuntimeError("PyTorch not found. Install with: pip install torch")

    if torch.cuda.is_available():
        vram_gb = torch.cuda.get_device_properties(0).total_memory / 1e9
        print(f"CUDA GPU detected: {torch.cuda.get_device_name(0)} ({vram_gb:.1f} GB VRAM)")
        # bfloat16 is recommended for SmolLM3 -- it's the training dtype
        return "cuda", torch.bfloat16, {"device_map": "auto", "torch_dtype": torch.bfloat16}

    elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
        print("Apple Silicon MPS detected")
        # MPS supports float16 but not all bfloat16 ops -- use float16 on Apple Silicon
        return "mps", torch.float16, {"device_map": "mps", "torch_dtype": torch.float16}

    else:
        print("No GPU found -- running on CPU (slower but functional)")
        return "cpu", torch.float32, {"device_map": "cpu", "torch_dtype": torch.float32}


if __name__ == "__main__":
    device, dtype, kwargs = detect_device()
    print(f"Device : {device}")
    print(f"Dtype  : {dtype}")
    print(f"Kwargs : {kwargs}")

python device_check.py

CUDA GPU detected: NVIDIA GeForce RTX 3060 (12.0 GB VRAM)
Device : cuda
Dtype  : torch.bfloat16
Kwargs : {'device_map': 'auto', 'torch_dtype': torch.bfloat16}

# first_inference.py
# Prerequisites: transformers>=4.53.0, torch, accelerate
# Run: python first_inference.py

import re
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL_ID = "HuggingFaceTB/SmolLM3-3B"

# ── 1. Load tokenizer and model ───────────────────────────────────────────────

print(f"Loading {MODEL_ID}...")

tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    torch_dtype=torch.bfloat16,    # Match the training dtype; use float16 on Apple Silicon
    device_map="auto",             # Spreads across all available GPUs, or CPU if none
)
model.eval()

print(f"Model loaded on: {model.device}")

# ── 2. Generation helper ──────────────────────────────────────────────────────

def generate(messages: list[dict], max_new_tokens: int = 512) -> str:
    """
    Apply the SmolLM3 chat template, tokenize, generate, and decode.
    Strips the ... block from the output automatically
    so callers always receive the final answer only.
    """
    # apply_chat_template formats messages using SmolLM3's built-in chat template
    text = tokenizer.apply_chat_template(
        messages,
        tokenize=False,
        add_generation_prompt=True,
    )
    inputs = tokenizer(text, return_tensors="pt").to(model.device)

    with torch.no_grad():
        output_ids = model.generate(
            **inputs,
            max_new_tokens=max_new_tokens,
            temperature=0.6,   # Recommended by the SmolLM3 team for balanced output
            top_p=0.95,        # Nucleus sampling -- keeps output focused without being repetitive
            do_sample=True,
        )

    # Decode only the newly generated tokens, not the input prompt
    new_tokens = output_ids[0][inputs["input_ids"].shape[-1]:]
    raw = tokenizer.decode(new_tokens, skip_special_tokens=True)

    # Strip the chain-of-thought block if present.
    # In think mode the model prefixes its response with ....
    # Callers usually only need the final answer that follows.
    final = re.sub(r".*?", "", raw, flags=re.DOTALL).strip()
    return final


# ── 3. Compare think vs no_think on the same prompt ──────────────────────────

prompt = "A customer is charged twice for the same order. What are three concrete steps support should take?"

# no_think: fast, direct answer -- good for high-throughput classification and replies
no_think_messages = [
    {"role": "system", "content": "/no_think"},
    {"role": "user",   "content": prompt},
]

# think: reasoning trace before answer -- good for complex decisions and edge cases
think_messages = [
    {"role": "system", "content": "/think"},
    {"role": "user",   "content": prompt},
]

print("\n── no_think mode ──")
print(generate(no_think_messages, max_new_tokens=256))

print("\n── think mode ──")
print(generate(think_messages, max_new_tokens=512))

python first_inference.py

# ticket_router.py
# Prerequisites: transformers>=4.53.0, torch, accelerate
# Run: python ticket_router.py

import re
import json
import torch
from dataclasses import dataclass
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL_ID      = "HuggingFaceTB/SmolLM3-3B"
ESCALATE_AT   = 0.70   # Tickets with confidence below this go to a human agent

# ── Data class for a routing result ──────────────────────────────────────────

@dataclass
class RoutingResult:
    ticket: str
    category: str           # billing | technical | account | general
    confidence: float       # 0.0-1.0 self-reported by the model
    reply: str              # Generated in the same language as the ticket
    escalate: bool          # True when confidence < ESCALATE_AT
    raw_output: str         # Full model output for debugging


# ── System prompt ─────────────────────────────────────────────────────────────

SYSTEM_PROMPT = """You are a multilingual customer support router for a SaaS company.
Your job is to classify support tickets and draft a helpful, professional reply.

Rules:
- Detect the language of the ticket automatically.
- Classify into EXACTLY ONE of: billing, technical, account, general.
- Reply in the SAME language as the ticket.
- Rate your confidence honestly from 0.0 to 1.0. Low confidence means the ticket is ambiguous or outside your knowledge.
- Respond ONLY with a single JSON object -- no preamble, no explanation outside the JSON.

Required format:
{"category": "", "confidence": <0.0-1.0>, "reply": ""}"""


# ── Router class ──────────────────────────────────────────────────────────────

class TicketRouter:
    def __init__(self, model_id: str = MODEL_ID):
        print(f"Loading {model_id}...")
        self.tokenizer = AutoTokenizer.from_pretrained(model_id)
        self.model = AutoModelForCausalLM.from_pretrained(
            model_id,
            torch_dtype=torch.bfloat16,
            device_map="auto",
        )
        self.model.eval()
        print(f"Ready on {self.model.device}")

    def _call_model(self, ticket: str) -> str:
        """
        Format the ticket into a chat message, run inference in no_think mode
        (faster for classification), and return the raw decoded output.
        """
        messages = [
            {"role": "system", "content": SYSTEM_PROMPT},
            {"role": "user",   "content": ticket},
        ]
        text = self.tokenizer.apply_chat_template(
            messages,
            tokenize=False,
            add_generation_prompt=True,
            enable_thinking=False,   # Fast path -- no chain-of-thought for routine classification
        )
        inputs = self.tokenizer(text, return_tensors="pt").to(self.model.device)

        with torch.no_grad():
            output_ids = self.model.generate(
                **inputs,
                max_new_tokens=256,
                temperature=0.3,   # Lower temp for classification -- more deterministic output
                top_p=0.9,
                do_sample=True,
            )

        new_tokens = output_ids[0][inputs["input_ids"].shape[-1]:]
        return self.tokenizer.decode(new_tokens, skip_special_tokens=True).strip()

    def _parse_output(self, raw: str) -> dict:
        """
        Extract the JSON object from the model's output.
        Falls back to a default 'general' category with zero confidence if parsing fails.
        This prevents a JSON parse failure from crashing the pipeline.
        """
        # Find any JSON object in the output, even if surrounded by stray text
        match = re.search(r"\{.*?\}", raw, re.DOTALL)
        if not match:
            return {"category": "general", "confidence": 0.0, "reply": raw}
        try:
            return json.loads(match.group())
        except json.JSONDecodeError:
            return {"category": "general", "confidence": 0.0, "reply": raw}

    def route(self, ticket: str) -> RoutingResult:
        """
        Route a single ticket. Returns a RoutingResult with classification,
        confidence, reply, and escalation flag.
        """
        raw = self._call_model(ticket)
        parsed = self._parse_output(raw)

        category   = parsed.get("category", "general")
        confidence = float(parsed.get("confidence", 0.0))
        reply      = parsed.get("reply", "Thank you for reaching out. We will follow up shortly.")

        return RoutingResult(
            ticket=ticket,
            category=category,
            confidence=confidence,
            reply=reply,
            escalate=confidence < ESCALATE_AT,
            raw_output=raw,
        )

    def route_batch(self, tickets: list[str]) -> list[RoutingResult]:
        """Route a list of tickets sequentially. Returns results in input order."""
        return [self.route(t) for t in tickets]


# ── Run it ────────────────────────────────────────────────────────────────────

if __name__ == "__main__":
    router = TicketRouter()

    test_tickets = [
        "I was charged twice for my subscription this month. Please refund the duplicate charge.",
        "L'application se bloque chaque fois que j'essaie d'exporter un fichier PDF.",   # French
        "No puedo iniciar sesión en mi cuenta desde hace dos días.",                     # Spanish
        "Die Rechnung für März fehlt in meinem Abrechnungsbereich.",                     # German
        "Il mio abbonamento non si rinnova automaticamente nonostante il pagamento.",    # Italian
    ]

    print("\n" + "=" * 70)
    results = router.route_batch(test_tickets)

    for r in results:
        flag = "🔴 ESCALATE" if r.escalate else "🟢 AUTO"
        print(f"\n{flag}")
        print(f"Ticket     : {r.ticket[:70]}...")
        print(f"Category   : {r.category}")
        print(f"Confidence : {r.confidence:.2f}")
        print(f"Reply      : {r.reply[:100]}...")

    escalated = [r for r in results if r.escalate]
    print(f"\n{'─'*70}")
    print(f"Total tickets : {len(results)}")
    print(f"Auto-routed   : {len(results) - len(escalated)}")
    print(f"Escalated     : {len(escalated)}")

python ticket_router.py

# tool_calling.py
# Prerequisites: transformers>=4.53.0, torch, accelerate
# Run: python tool_calling.py

import re
import json
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL_ID  = "HuggingFaceTB/SmolLM3-3B"
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
model     = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, torch_dtype=torch.bfloat16, device_map="auto"
)
model.eval()

# ── Tool definition ───────────────────────────────────────────────────────────
# SmolLM3 accepts tool definitions as JSON Schema objects under xml_tools.
# The model uses the name and description to decide when to call the tool.
# The parameters schema tells it what arguments to include in the call.

TOOLS = [
    {
        "name": "lookup_order_status",
        "description": (
            "Look up the current status, estimated delivery date, and carrier "
            "for a specific customer order. Call this when the customer mentions "
            "an order number or asks where their order is."
        ),
        "parameters": {
            "type": "object",
            "properties": {
                "order_id": {
                    "type": "string",
                    "description": "The order ID, usually in the format ORD-XXXXXX."
                }
            },
            "required": ["order_id"]
        }
    }
]

# ── Simulated order database ──────────────────────────────────────────────────

def lookup_order_status(order_id: str) -> dict:
    """
    In production, replace this with a real database or API call.
    Returns a dict the model can read and summarize for the customer.
    """
    database = {
        "ORD-4821": {"status": "shipped",    "eta": "June 18, 2026", "carrier": "DHL"},
        "ORD-3307": {"status": "processing", "eta": "June 20, 2026", "carrier": None},
        "ORD-1190": {"status": "delivered",  "eta": None,            "carrier": "FedEx"},
    }
    return database.get(order_id, {"status": "not_found", "eta": None, "carrier": None})

# ── Tool call parser ──────────────────────────────────────────────────────────

def parse_tool_call(output: str):
    """
    Extract a tool call from the model's output.
    SmolLM3 emits: {"name": "...", "arguments": {...}}
    Returns (tool_name, arguments) or (None, None) if no tool call is present.
    """
    match = re.search(r"(.*?)", output, re.DOTALL)
    if not match:
        return None, None
    try:
        payload = json.loads(match.group(1).strip())
        return payload.get("name"), payload.get("arguments", {})
    except json.JSONDecodeError:
        return None, None

# ── Full tool-call round trip ─────────────────────────────────────────────────

def respond_with_tools(user_message: str) -> str:
    """
    Full agentic loop:
    1. Send user message + tool definitions to the model.
    2. If the model emits a tool call, execute it and inject the result.
    3. Generate the final customer-facing response.
    """
    # Turn 1: give the model the user message and available tools
    messages = [{"role": "user", "content": user_message}]

    inputs = tokenizer.apply_chat_template(
        messages,
        xml_tools=TOOLS,           # Pass tool definitions here
        enable_thinking=False,
        add_generation_prompt=True,
        tokenize=True,
        return_tensors="pt",
    ).to(model.device)

    with torch.no_grad():
        output_ids = model.generate(
            inputs, max_new_tokens=256, temperature=0.3, top_p=0.9, do_sample=True
        )
    turn1 = tokenizer.decode(
        output_ids[0][inputs.shape[-1]:], skip_special_tokens=True
    )

    # Check if the model wants to call a tool
    tool_name, tool_args = parse_tool_call(turn1)

    if tool_name == "lookup_order_status":
        # Execute the real function
        tool_result = lookup_order_status(**tool_args)
        print(f"  [Tool called] {tool_name}({tool_args}) → {tool_result}")

        # Turn 2: inject the tool result and ask for the final response
        messages += [
            {"role": "assistant", "content": turn1},
            {"role": "tool",      "content": json.dumps(tool_result), "name": tool_name},
        ]
        inputs2 = tokenizer.apply_chat_template(
            messages,
            xml_tools=TOOLS,
            enable_thinking=False,
            add_generation_prompt=True,
            tokenize=True,
            return_tensors="pt",
        ).to(model.device)

        with torch.no_grad():
            output_ids2 = model.generate(
                inputs2, max_new_tokens=256, temperature=0.3, top_p=0.9, do_sample=True
            )
        return tokenizer.decode(
            output_ids2[0][inputs2.shape[-1]:], skip_special_tokens=True
        ).strip()

    # No tool call -- model answered directly
    return turn1.strip()


# ── Test it ───────────────────────────────────────────────────────────────────

if __name__ == "__main__":
    queries = [
        "Where is my order ORD-4821? It's been a week.",
        "My order ORD-3307 hasn't shipped yet -- what's the status?",
        "I just want to change my email address.",  # No tool needed
    ]

    for query in queries:
        print(f"\nCustomer : {query}")
        response = respond_with_tools(query)
        print(f"Agent    : {response}")

python tool_calling.py

# finetune.py
# Additional prerequisites: pip install trl>=0.9.0 peft>=0.11.0 datasets>=2.19.0
# Run: python finetune.py
# Time: ~8-12 minutes on an RTX 3060 for 3 epochs over 50 examples

import json
import torch
from datasets import Dataset
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model
from trl import SFTTrainer, SFTConfig

MODEL_ID   = "HuggingFaceTB/SmolLM3-3B"
OUTPUT_DIR = "./smollm3-ticket-router"

# ── System prompt (same as the inference router) ──────────────────────────────

SYSTEM_PROMPT = """You are a multilingual customer support router for a SaaS company.
Classify the support ticket and generate a helpful reply in the same language as the ticket.
Respond ONLY with JSON: {"category": "", "confidence": <0.0-1.0>, "reply": ""}"""

# ── Training data ─────────────────────────────────────────────────────────────
# In production you would load hundreds of real labelled tickets.
# This minimal set demonstrates the format -- expand with your real data.

raw_examples = [
    ("I was charged twice for my subscription.", "billing",
     "We're sorry for the duplicate charge. Our billing team will review and issue a refund within 3-5 business days."),
    ("The app crashes every time I try to export a PDF.", "technical",
     "We apologize for the inconvenience. Our engineering team has been notified and will investigate."),
    ("I can't log into my account since yesterday.", "account",
     "We're sorry you're having trouble. Please try resetting your password. If the issue continues, we'll escalate to our account team."),
    ("Die App stürzt beim Exportieren von PDFs ab.", "technical",
     "Wir entschuldigen uns für die Unannehmlichkeiten. Unser Technikteam wurde benachrichtigt und untersucht das Problem."),
    ("L'application se bloque quand j'exporte un fichier.", "technical",
     "Nous nous excusons pour la gêne occasionnée. Notre équipe technique a été informée et travaille sur ce problème."),
    ("My March invoice is missing from the billing section.", "billing",
     "Thank you for flagging this. Our billing team will locate your March invoice and resend it within 24 hours."),
    ("No puedo iniciar sesión desde ayer por la noche.", "account",
     "Lamentamos el problema de acceso. Por favor, restablezca su contraseña. Si el problema persiste, escalaremos su caso."),
    ("How do I upgrade my plan to the Pro tier?", "general",
     "You can upgrade to Pro directly from Settings → Subscription. The new rate applies from your next billing cycle."),
]

def format_example(ticket: str, category: str, reply: str) -> dict:
    """
    Format a single example into the SmolLM3 messages format.
    The assistant turn contains the target JSON the model should learn to produce.
    """
    return {
        "messages": [
            {"role": "system",    "content": SYSTEM_PROMPT},
            {"role": "user",      "content": ticket},
            {"role": "assistant", "content": json.dumps({
                "category": category, "confidence": 0.95, "reply": reply
            })},
        ]
    }

dataset = Dataset.from_list([format_example(*ex) for ex in raw_examples])

# ── Tokenizer ─────────────────────────────────────────────────────────────────

tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
tokenizer.pad_token = tokenizer.eos_token   # SmolLM3 has no separate pad token

# ── Model (4-bit quantized base for QLoRA) ────────────────────────────────────

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
)
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    quantization_config=bnb_config,
    device_map="auto",
)

# ── LoRA config ───────────────────────────────────────────────────────────────
# We target the attention and MLP projection layers -- these carry the most
# task-specific signal and give the best accuracy/parameter trade-off.

lora_config = LoraConfig(
    r=16,              # Rank of the LoRA update matrices -- higher = more expressive, more memory
    lora_alpha=32,     # Scaling factor; conventionally set to 2*r
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",   # Attention projections
        "gate_proj", "up_proj", "down_proj",        # MLP projections (SwiGLU)
    ],
)
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
# Expected: trainable params: ~13M (0.4% of 3B total)

# ── Training config ───────────────────────────────────────────────────────────

sft_config = SFTConfig(
    output_dir=OUTPUT_DIR,
    num_train_epochs=3,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,   # Effective batch size = 8
    learning_rate=2e-4,
    warmup_ratio=0.1,
    lr_scheduler_type="cosine",
    bf16=True,
    logging_steps=5,
    save_strategy="epoch",
    max_seq_length=512,              # Tickets are short -- no need for the full context window
)

# ── Train ─────────────────────────────────────────────────────────────────────

trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=dataset,
    args=sft_config,
)
trainer.train()

# ── Save and merge ────────────────────────────────────────────────────────────
# Save the LoRA adapter -- small file, easy to share or version.
trainer.save_model(f"{OUTPUT_DIR}/adapter")

# Merge the adapter back into the base model weights for standalone deployment.
# The merged model loads exactly like the base model -- no PEFT dependency at inference.
merged = model.merge_and_unload()
merged.save_pretrained(f"{OUTPUT_DIR}/merged")
tokenizer.save_pretrained(f"{OUTPUT_DIR}/merged")

print(f"\nFine-tuned model saved to {OUTPUT_DIR}/merged")
print("Load it with: AutoModelForCausalLM.from_pretrained('./smollm3-ticket-router/merged')")

pip install trl>=0.9.0 peft>=0.11.0 datasets>=2.19.0
python finetune.py

trainable params: 13,631,488 || all params: 3,085,123,584 || trainable%: 0.4420
{'loss': 1.842, 'learning_rate': 2e-04, 'epoch': 0.5}
{'loss': 0.923, 'learning_rate': 1.4e-04, 'epoch': 1.0}
{'loss': 0.461, 'learning_rate': 6e-05, 'epoch': 2.0}
{'loss': 0.287, 'learning_rate': 0.0, 'epoch': 3.0}