Small Language Fashions with Hugging Face transformers Library + smolLM3

# Python 3.10 or newer required
python --version

# Create and activate a digital surroundings
python -m venv smollm-env
supply smollm-env/bin/activate       # macOS / Linux
smollm-envScriptsactivate          # Home windows

# Set up all dependencies
pip set up 
  "transformers>=4.53.0" 
  "torch>=2.3.0" 
  "speed up>=0.30.0" 
  "bitsandbytes>=0.43.0" 
  "sentencepiece" 
  "trl>=0.9.0" 
  "peft>=0.11.0" 
  "datasets>=2.19.0"

# device_check.py
# Run this earlier than anything to substantiate your setup and choose the correct dtype.

def detect_device():
    """
    Detect the very best obtainable compute gadget.
    Returns (device_str, dtype_str, load_kwargs) to be used with from_pretrained.
    """
    strive:
        import torch
    besides ImportError:
        increase RuntimeError("PyTorch not discovered. Set up with: pip set up torch")

    if torch.cuda.is_available():
        vram_gb = torch.cuda.get_device_properties(0).total_memory / 1e9
        print(f"CUDA GPU detected: {torch.cuda.get_device_name(0)} ({vram_gb:.1f} GB VRAM)")
        # bfloat16 is really useful for SmolLM3 -- it is the coaching dtype
        return "cuda", torch.bfloat16, {"device_map": "auto", "torch_dtype": torch.bfloat16}

    elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
        print("Apple Silicon MPS detected")
        # MPS helps float16 however not all bfloat16 ops -- use float16 on Apple Silicon
        return "mps", torch.float16, {"device_map": "mps", "torch_dtype": torch.float16}

    else:
        print("No GPU discovered -- operating on CPU (slower however purposeful)")
        return "cpu", torch.float32, {"device_map": "cpu", "torch_dtype": torch.float32}


if __name__ == "__main__":
    gadget, dtype, kwargs = detect_device()
    print(f"System : {gadget}")
    print(f"Dtype  : {dtype}")
    print(f"Kwargs : {kwargs}")

CUDA GPU detected: NVIDIA GeForce RTX 3060 (12.0 GB VRAM)
System : cuda
Dtype  : torch.bfloat16
Kwargs : {'device_map': 'auto', 'torch_dtype': torch.bfloat16}

# first_inference.py
# Conditions: transformers>=4.53.0, torch, speed up
# Run: python first_inference.py

import re
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL_ID = "HuggingFaceTB/SmolLM3-3B"

# ── 1. Load tokenizer and mannequin ───────────────────────────────────────────────

print(f"Loading {MODEL_ID}...")

tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

mannequin = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    torch_dtype=torch.bfloat16,    # Match the coaching dtype; use float16 on Apple Silicon
    device_map="auto",             # Spreads throughout all obtainable GPUs, or CPU if none
)
mannequin.eval()

print(f"Mannequin loaded on: {mannequin.gadget}")

# ── 2. Technology helper ──────────────────────────────────────────────────────

def generate(messages: listing[dict], max_new_tokens: int = 512) -> str:
    """
    Apply the SmolLM3 chat template, tokenize, generate, and decode.
    Strips the ... block from the output mechanically
    so callers all the time obtain the ultimate reply solely.
    """
    # apply_chat_template codecs messages utilizing SmolLM3's built-in chat template
    textual content = tokenizer.apply_chat_template(
        messages,
        tokenize=False,
        add_generation_prompt=True,
    )
    inputs = tokenizer(textual content, return_tensors="pt").to(mannequin.gadget)

    with torch.no_grad():
        output_ids = mannequin.generate(
            **inputs,
            max_new_tokens=max_new_tokens,
            temperature=0.6,   # Really helpful by the SmolLM3 group for balanced output
            top_p=0.95,        # Nucleus sampling -- retains output targeted with out being repetitive
            do_sample=True,
        )

    # Decode solely the newly generated tokens, not the enter immediate
    new_tokens = output_ids[0][inputs["input_ids"].form[-1]:]
    uncooked = tokenizer.decode(new_tokens, skip_special_tokens=True)

    # Strip the chain-of-thought block if current.
    # In suppose mode the mannequin prefixes its response with ....
    # Callers normally solely want the ultimate reply that follows.
    last = re.sub(r".*?", "", uncooked, flags=re.DOTALL).strip()
    return last


# ── 3. Examine suppose vs no_think on the identical immediate ──────────────────────────

immediate = "A buyer is charged twice for a similar order. What are three concrete steps help ought to take?"

# no_think: quick, direct reply -- good for high-throughput classification and replies
no_think_messages = [
    {"role": "system", "content": "/no_think"},
    {"role": "user",   "content": prompt},
]

# suppose: reasoning hint earlier than reply -- good for complicated selections and edge circumstances
think_messages = [
    {"role": "system", "content": "/think"},
    {"role": "user",   "content": prompt},
]

print("n── no_think mode ──")
print(generate(no_think_messages, max_new_tokens=256))

print("n── suppose mode ──")
print(generate(think_messages, max_new_tokens=512))

python first_inference.py

# ticket_router.py
# Conditions: transformers>=4.53.0, torch, speed up
# Run: python ticket_router.py

import re
import json
import torch
from dataclasses import dataclass
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL_ID      = "HuggingFaceTB/SmolLM3-3B"
ESCALATE_AT   = 0.70   # Tickets with confidence under this go to a human agent

# ── Knowledge class for a routing consequence ──────────────────────────────────────────

@dataclass
class RoutingResult:
    ticket: str
    class: str           # billing | technical | account | basic
    confidence: float       # 0.0-1.0 self-reported by the mannequin
    reply: str              # Generated in the identical language because the ticket
    escalate: bool          # True when confidence < ESCALATE_AT
    raw_output: str         # Full mannequin output for debugging


# ── System immediate ─────────────────────────────────────────────────────────────

SYSTEM_PROMPT = """You're a multilingual buyer help router for a SaaS firm.
Your job is to categorise help tickets and draft a useful, skilled reply.

Guidelines:
- Detect the language of the ticket mechanically.
- Classify into EXACTLY ONE of: billing, technical, account, basic.
- Reply within the SAME language because the ticket.
- Fee your confidence actually from 0.0 to 1.0. Low confidence means the ticket is ambiguous or outdoors your data.
- Reply ONLY with a single JSON object -- no preamble, no clarification outdoors the JSON.

Required format:
{"class": "", "confidence": <0.0-1.0>, "reply": ""}"""


# ── Router class ──────────────────────────────────────────────────────────────

class TicketRouter:
    def __init__(self, model_id: str = MODEL_ID):
        print(f"Loading {model_id}...")
        self.tokenizer = AutoTokenizer.from_pretrained(model_id)
        self.mannequin = AutoModelForCausalLM.from_pretrained(
            model_id,
            torch_dtype=torch.bfloat16,
            device_map="auto",
        )
        self.mannequin.eval()
        print(f"Prepared on {self.mannequin.gadget}")

    def _call_model(self, ticket: str) -> str:
        """
        Format the ticket right into a chat message, run inference in no_think mode
        (sooner for classification), and return the uncooked decoded output.
        """
        messages = [
            {"role": "system", "content": SYSTEM_PROMPT},
            {"role": "user",   "content": ticket},
        ]
        textual content = self.tokenizer.apply_chat_template(
            messages,
            tokenize=False,
            add_generation_prompt=True,
            enable_thinking=False,   # Quick path -- no chain-of-thought for routine classification
        )
        inputs = self.tokenizer(textual content, return_tensors="pt").to(self.mannequin.gadget)

        with torch.no_grad():
            output_ids = self.mannequin.generate(
                **inputs,
                max_new_tokens=256,
                temperature=0.3,   # Decrease temp for classification -- extra deterministic output
                top_p=0.9,
                do_sample=True,
            )

        new_tokens = output_ids[0][inputs["input_ids"].form[-1]:]
        return self.tokenizer.decode(new_tokens, skip_special_tokens=True).strip()

    def _parse_output(self, uncooked: str) -> dict:
        """
        Extract the JSON object from the mannequin's output.
        Falls again to a default 'basic' class with zero confidence if parsing fails.
        This prevents a JSON parse failure from crashing the pipeline.
        """
        # Discover any JSON object within the output, even when surrounded by stray textual content
        match = re.search(r"{.*?}", uncooked, re.DOTALL)
        if not match:
            return {"class": "basic", "confidence": 0.0, "reply": uncooked}
        strive:
            return json.hundreds(match.group())
        besides json.JSONDecodeError:
            return {"class": "basic", "confidence": 0.0, "reply": uncooked}

    def route(self, ticket: str) -> RoutingResult:
        """
        Route a single ticket. Returns a RoutingResult with classification,
        confidence, reply, and escalation flag.
        """
        uncooked = self._call_model(ticket)
        parsed = self._parse_output(uncooked)

        class   = parsed.get("class", "basic")
        confidence = float(parsed.get("confidence", 0.0))
        reply      = parsed.get("reply", "Thanks for reaching out. We are going to comply with up shortly.")

        return RoutingResult(
            ticket=ticket,
            class=class,
            confidence=confidence,
            reply=reply,
            escalate=confidence < ESCALATE_AT,
            raw_output=uncooked,
        )

    def route_batch(self, tickets: listing[str]) -> listing[RoutingResult]:
        """Route an inventory of tickets sequentially. Returns ends in enter order."""
        return [self.route(t) for t in tickets]


# ── Run it ────────────────────────────────────────────────────────────────────

if __name__ == "__main__":
    router = TicketRouter()

    test_tickets = [
        "I was charged twice for my subscription this month. Please refund the duplicate charge.",
        "L'application se bloque chaque fois que j'essaie d'exporter un fichier PDF.",   # French
        "No puedo iniciar sesión en mi cuenta desde hace dos días.",                     # Spanish
        "Die Rechnung für März fehlt in meinem Abrechnungsbereich.",                     # German
        "Il mio abbonamento non si rinnova automaticamente nonostante il pagamento.",    # Italian
    ]

    print("n" + "=" * 70)
    outcomes = router.route_batch(test_tickets)

    for r in outcomes:
        flag = "🔴 ESCALATE" if r.escalate else "🟢 AUTO"
        print(f"n{flag}")
        print(f"Ticket     : {r.ticket[:70]}...")
        print(f"Class   : {r.class}")
        print(f"Confidence : {r.confidence:.2f}")
        print(f"Reply      : {r.reply[:100]}...")

    escalated = [r for r in results if r.escalate]
    print(f"n{'─'*70}")
    print(f"Complete tickets : {len(outcomes)}")
    print(f"Auto-routed   : {len(outcomes) - len(escalated)}")
    print(f"Escalated     : {len(escalated)}")

# tool_calling.py
# Conditions: transformers>=4.53.0, torch, speed up
# Run: python tool_calling.py

import re
import json
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL_ID  = "HuggingFaceTB/SmolLM3-3B"
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
mannequin     = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, torch_dtype=torch.bfloat16, device_map="auto"
)
mannequin.eval()

# ── Instrument definition ───────────────────────────────────────────────────────────
# SmolLM3 accepts device definitions as JSON Schema objects underneath xml_tools.
# The mannequin makes use of the title and outline to determine when to name the device.
# The parameters schema tells it what arguments to incorporate within the name.

TOOLS = [
    {
        "name": "lookup_order_status",
        "description": (
            "Look up the current status, estimated delivery date, and carrier "
            "for a specific customer order. Call this when the customer mentions "
            "an order number or asks where their order is."
        ),
        "parameters": {
            "type": "object",
            "properties": {
                "order_id": {
                    "type": "string",
                    "description": "The order ID, usually in the format ORD-XXXXXX."
                }
            },
            "required": ["order_id"]
        }
    }
]

# ── Simulated order database ──────────────────────────────────────────────────

def lookup_order_status(order_id: str) -> dict:
    """
    In manufacturing, change this with an actual database or API name.
    Returns a dict the mannequin can learn and summarize for the shopper.
    """
    database = {
        "ORD-4821": {"standing": "shipped",    "eta": "June 18, 2026", "provider": "DHL"},
        "ORD-3307": {"standing": "processing", "eta": "June 20, 2026", "provider": None},
        "ORD-1190": {"standing": "delivered",  "eta": None,            "provider": "FedEx"},
    }
    return database.get(order_id, {"standing": "not_found", "eta": None, "provider": None})

# ── Instrument name parser ──────────────────────────────────────────────────────────

def parse_tool_call(output: str):
    """
    Extract a device name from the mannequin's output.
    SmolLM3 emits: {"title": "...", "arguments": {...}}
    Returns (tool_name, arguments) or (None, None) if no device name is current.
    """
    match = re.search(r"(.*?)", output, re.DOTALL)
    if not match:
        return None, None
    strive:
        payload = json.hundreds(match.group(1).strip())
        return payload.get("title"), payload.get("arguments", {})
    besides json.JSONDecodeError:
        return None, None

# ── Full tool-call spherical journey ─────────────────────────────────────────────────

def respond_with_tools(user_message: str) -> str:
    """
    Full agentic loop:
    1. Ship person message + device definitions to the mannequin.
    2. If the mannequin emits a device name, execute it and inject the consequence.
    3. Generate the ultimate customer-facing response.
    """
    # Flip 1: give the mannequin the person message and obtainable instruments
    messages = [{"role": "user", "content": user_message}]

    inputs = tokenizer.apply_chat_template(
        messages,
        xml_tools=TOOLS,           # Go device definitions right here
        enable_thinking=False,
        add_generation_prompt=True,
        tokenize=True,
        return_tensors="pt",
    ).to(mannequin.gadget)

    with torch.no_grad():
        output_ids = mannequin.generate(
            inputs, max_new_tokens=256, temperature=0.3, top_p=0.9, do_sample=True
        )
    turn1 = tokenizer.decode(
        output_ids[0][inputs.shape[-1]:], skip_special_tokens=True
    )

    # Examine if the mannequin needs to name a device
    tool_name, tool_args = parse_tool_call(turn1)

    if tool_name == "lookup_order_status":
        # Execute the true operate
        tool_result = lookup_order_status(**tool_args)
        print(f"  [Tool called] {tool_name}({tool_args}) → {tool_result}")

        # Flip 2: inject the device consequence and ask for the ultimate response
        messages += [
            {"role": "assistant", "content": turn1},
            {"role": "tool",      "content": json.dumps(tool_result), "name": tool_name},
        ]
        inputs2 = tokenizer.apply_chat_template(
            messages,
            xml_tools=TOOLS,
            enable_thinking=False,
            add_generation_prompt=True,
            tokenize=True,
            return_tensors="pt",
        ).to(mannequin.gadget)

        with torch.no_grad():
            output_ids2 = mannequin.generate(
                inputs2, max_new_tokens=256, temperature=0.3, top_p=0.9, do_sample=True
            )
        return tokenizer.decode(
            output_ids2[0][inputs2.shape[-1]:], skip_special_tokens=True
        ).strip()

    # No device name -- mannequin answered instantly
    return turn1.strip()


# ── Take a look at it ───────────────────────────────────────────────────────────────────

if __name__ == "__main__":
    queries = [
        "Where is my order ORD-4821? It's been a week.",
        "My order ORD-3307 hasn't shipped yet -- what's the status?",
        "I just want to change my email address.",  # No tool needed
    ]

    for question in queries:
        print(f"nCustomer : {question}")
        response = respond_with_tools(question)
        print(f"Agent    : {response}")

# finetune.py
# Further conditions: pip set up trl>=0.9.0 peft>=0.11.0 datasets>=2.19.0
# Run: python finetune.py
# Time: ~8-12 minutes on an RTX 3060 for 3 epochs over 50 examples

import json
import torch
from datasets import Dataset
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model
from trl import SFTTrainer, SFTConfig

MODEL_ID   = "HuggingFaceTB/SmolLM3-3B"
OUTPUT_DIR = "./smollm3-ticket-router"

# ── System immediate (similar because the inference router) ──────────────────────────────

SYSTEM_PROMPT = """You're a multilingual buyer help router for a SaaS firm.
Classify the help ticket and generate a useful reply in the identical language because the ticket.
Reply ONLY with JSON: {"class": "", "confidence": <0.0-1.0>, "reply": ""}"""

# ── Coaching information ─────────────────────────────────────────────────────────────
# In manufacturing you'd load tons of of actual labelled tickets.
# This minimal set demonstrates the format -- increase along with your actual information.

raw_examples = [
    ("I was charged twice for my subscription.", "billing",
     "We're sorry for the duplicate charge. Our billing team will review and issue a refund within 3-5 business days."),
    ("The app crashes every time I try to export a PDF.", "technical",
     "We apologize for the inconvenience. Our engineering team has been notified and will investigate."),
    ("I can't log into my account since yesterday.", "account",
     "We're sorry you're having trouble. Please try resetting your password. If the issue continues, we'll escalate to our account team."),
    ("Die App stürzt beim Exportieren von PDFs ab.", "technical",
     "Wir entschuldigen uns für die Unannehmlichkeiten. Unser Technikteam wurde benachrichtigt und untersucht das Problem."),
    ("L'application se bloque quand j'exporte un fichier.", "technical",
     "Nous nous excusons pour la gêne occasionnée. Notre équipe technique a été informée et travaille sur ce problème."),
    ("My March invoice is missing from the billing section.", "billing",
     "Thank you for flagging this. Our billing team will locate your March invoice and resend it within 24 hours."),
    ("No puedo iniciar sesión desde ayer por la noche.", "account",
     "Lamentamos el problema de acceso. Por favor, restablezca su contraseña. Si el problema persiste, escalaremos su caso."),
    ("How do I upgrade my plan to the Pro tier?", "general",
     "You can upgrade to Pro directly from Settings → Subscription. The new rate applies from your next billing cycle."),
]

def format_example(ticket: str, class: str, reply: str) -> dict:
    """
    Format a single instance into the SmolLM3 messages format.
    The assistant flip accommodates the goal JSON the mannequin ought to be taught to provide.
    """
    return {
        "messages": [
            {"role": "system",    "content": SYSTEM_PROMPT},
            {"role": "user",      "content": ticket},
            {"role": "assistant", "content": json.dumps({
                "category": category, "confidence": 0.95, "reply": reply
            })},
        ]
    }

dataset = Dataset.from_list([format_example(*ex) for ex in raw_examples])

# ── Tokenizer ─────────────────────────────────────────────────────────────────

tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
tokenizer.pad_token = tokenizer.eos_token   # SmolLM3 has no separate pad token

# ── Mannequin (4-bit quantized base for QLoRA) ────────────────────────────────────

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
)
mannequin = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    quantization_config=bnb_config,
    device_map="auto",
)

# ── LoRA config ───────────────────────────────────────────────────────────────
# We goal the eye and MLP projection layers -- these carry probably the most
# task-specific sign and provides the very best accuracy/parameter trade-off.

lora_config = LoraConfig(
    r=16,              # Rank of the LoRA replace matrices -- larger = extra expressive, extra reminiscence
    lora_alpha=32,     # Scaling issue; conventionally set to 2*r
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",   # Attention projections
        "gate_proj", "up_proj", "down_proj",        # MLP projections (SwiGLU)
    ],
)
mannequin = get_peft_model(mannequin, lora_config)
mannequin.print_trainable_parameters()
# Anticipated: trainable params: ~13M (0.4% of 3B whole)

# ── Coaching config ───────────────────────────────────────────────────────────

sft_config = SFTConfig(
    output_dir=OUTPUT_DIR,
    num_train_epochs=3,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,   # Efficient batch dimension = 8
    learning_rate=2e-4,
    warmup_ratio=0.1,
    lr_scheduler_type="cosine",
    bf16=True,
    logging_steps=5,
    save_strategy="epoch",
    max_seq_length=512,              # Tickets are quick -- no want for the complete context window
)

# ── Practice ─────────────────────────────────────────────────────────────────────

coach = SFTTrainer(
    mannequin=mannequin,
    tokenizer=tokenizer,
    train_dataset=dataset,
    args=sft_config,
)
coach.practice()

# ── Save and merge ────────────────────────────────────────────────────────────
# Save the LoRA adapter -- small file, straightforward to share or model.
coach.save_model(f"{OUTPUT_DIR}/adapter")

# Merge the adapter again into the bottom mannequin weights for standalone deployment.
# The merged mannequin hundreds precisely like the bottom mannequin -- no PEFT dependency at inference.
merged = mannequin.merge_and_unload()
merged.save_pretrained(f"{OUTPUT_DIR}/merged")
tokenizer.save_pretrained(f"{OUTPUT_DIR}/merged")

print(f"nFine-tuned mannequin saved to {OUTPUT_DIR}/merged")
print("Load it with: AutoModelForCausalLM.from_pretrained('./smollm3-ticket-router/merged')")

pip set up trl>=0.9.0 peft>=0.11.0 datasets>=2.19.0
python finetune.py

trainable params: 13,631,488 || all params: 3,085,123,584 || trainable%: 0.4420
{'loss': 1.842, 'learning_rate': 2e-04, 'epoch': 0.5}
{'loss': 0.923, 'learning_rate': 1.4e-04, 'epoch': 1.0}
{'loss': 0.461, 'learning_rate': 6e-05, 'epoch': 2.0}
{'loss': 0.287, 'learning_rate': 0.0, 'epoch': 3.0}