Buckets:

hf-doc-build
/

doc-dev

Files
xet
hf-doc-build/doc-dev / trl /pr_7115 /en /example_overview.html
HuggingFaceDocBuilder
9 days ago
download
raw
32.9 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Examples&quot;,&quot;local&quot;:&quot;examples&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Index&quot;,&quot;local&quot;:&quot;index&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Distributed Training&quot;,&quot;local&quot;:&quot;distributed-training&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
		<link href="/docs/trl/pr_7115/en/_app/immutable/entry/start.2W4yWkdf.js" rel="modulepreload">
		<link href="/docs/trl/pr_7115/en/_app/immutable/chunks/BkYbAUqr.js" rel="modulepreload">
		<link href="/docs/trl/pr_7115/en/_app/immutable/chunks/BUY8r1pV.js" rel="modulepreload">
		<link href="/docs/trl/pr_7115/en/_app/immutable/entry/app.09KZcl8P.js" rel="modulepreload">
		<link href="/docs/trl/pr_7115/en/_app/immutable/chunks/nFtNyINy.js" rel="modulepreload">
		<link href="/docs/trl/pr_7115/en/_app/immutable/chunks/DZYT5v3_.js" rel="modulepreload">
		<link href="/docs/trl/pr_7115/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
		<link href="/docs/trl/pr_7115/en/_app/immutable/chunks/VOhKbRop.js" rel="modulepreload">
		<link href="/docs/trl/pr_7115/en/_app/immutable/nodes/0.CgOXU3A0.js" rel="modulepreload">
		<link href="/docs/trl/pr_7115/en/_app/immutable/chunks/MzGjOoUa.js" rel="modulepreload">
		<link href="/docs/trl/pr_7115/en/_app/immutable/nodes/12.B2yFcjq6.js" rel="modulepreload">
		<link href="/docs/trl/pr_7115/en/_app/immutable/chunks/C6o8PJYg.js" rel="modulepreload">
		<!--rhu47t--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Examples&quot;,&quot;local&quot;:&quot;examples&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Index&quot;,&quot;local&quot;:&quot;index&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Distributed Training&quot;,&quot;local&quot;:&quot;distributed-training&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
		<link href="/docs/trl/pr_7115/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="examples" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#examples"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Examples</span></h1><!--]--><!----> <p>The <a href="https://github.com/huggingface/trl/tree/main/examples" rel="nofollow"><code>examples/</code></a> directory contains a collection of self-contained examples that demonstrate how to use the TRL library for various applications. <strong>Each example lives in its own folder</strong> named after the method and the task it demonstrates (e.g. <code>grpo_wordle</code>, <code>sft_gpt_oss</code>), and holds everything the example needs: scripts, notebooks, prompts, chat templates, and evaluation code.</p> <p>Basic single-trainer training scripts are not examples: they live in <a href="https://github.com/huggingface/trl/tree/main/trl/scripts" rel="nofollow"><code>trl/scripts</code></a> and are exposed through the <a href="clis">command line interface</a> (<code>trl sft</code>, <code>trl dpo</code>, <code>trl grpo</code>, …). Each trainer’s documentation page also contains a complete runnable snippet.</p> <p>Shared resources sit at the root of <code>examples/</code>:</p> <ul><li><a href="https://github.com/huggingface/trl/tree/main/examples/accelerate_configs" rel="nofollow"><code>examples/accelerate_configs</code></a>: 🤗 Accelerate configuration files for multi-GPU, DeepSpeed ZeRO, FSDP, and context-parallel setups, used by many examples.</li> <li><a href="https://github.com/huggingface/trl/tree/main/examples/datasets" rel="nofollow"><code>examples/datasets</code></a>: the scripts used to generate the <code>trl-lib</code> datasets used across the examples.</li></ul> <p><strong>Getting Started</strong></p> <p>Install TRL and additional dependencies as follows:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->pip install --upgrade trl[quantization]<!----></pre></div><!----> <p>Check for additional optional dependencies <a href="https://github.com/huggingface/trl/blob/main/pyproject.toml" rel="nofollow">here</a>. Notebook-based examples are self-contained and can run on <strong>free Colab</strong>; script-based examples run on single-GPU, multi-GPU, or DeepSpeed setups (see <a href="#distributed-training">Distributed Training</a> below).</p> <!--[1--><h2 class="relative group"><a id="index" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#index"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Index</span></h2><!--]--><!----> <table><thead><tr><th>Example</th><th>Description</th><th>Open in Colab</th></tr></thead><tbody><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/async_distillation_math" rel="nofollow"><code>async_distillation_math</code></a></td><td>Async on-policy distillation on GSM8K with <a href="/docs/trl/pr_7115/en/async_distillation_trainer#trl.experimental.async_distillation.AsyncDistillationTrainer">experimental.async_distillation.AsyncDistillationTrainer</a>: the teacher is served over HTTP with vLLM, including a multi-teacher (MOPD) math + code variant.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/async_grpo_harbor" rel="nofollow"><code>async_grpo_harbor</code></a></td><td>AsyncGRPO training on a <a href="https://huggingface.co/docs/openenv/environments/harbor" rel="nofollow">Harbor</a> task suite through any installed coding agent (<code>mini-swe-agent</code>, <code>opencode</code>, <code>claude-code</code>, …) served over <a href="openenv">OpenEnv</a>: the agent owns its own loop in a sandbox and TRL trains on the token ids and logprobs recorded by the capture proxy.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/async_grpo_math" rel="nofollow"><code>async_grpo_math</code></a></td><td>Asynchronous GRPO on GSM8K with <a href="/docs/trl/pr_7115/en/async_grpo_trainer#trl.experimental.async_grpo.AsyncGRPOTrainer">experimental.async_grpo.AsyncGRPOTrainer</a>, decoupling generation (vLLM server) from training, including a LoRA variant that syncs only the adapter to the vLLM server.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/async_grpo_prophet_arena" rel="nofollow"><code>async_grpo_prophet_arena</code></a></td><td>AsyncGRPO training of a binary-market forecaster on <a href="https://www.prophetarena.co" rel="nofollow">Prophet Arena</a> with a Brier-score reward, ported from thinking-machines-lab/tinker-cookbook’s <code>recipes/forecasting</code>.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/async_grpo_timesx" rel="nofollow"><code>async_grpo_timesx</code></a></td><td>AsyncGRPO training of a numeric time-series forecaster on the <a href="https://huggingface.co/papers/2607.06973" rel="nofollow">TimesX</a> evaluation benchmark, with a MASE-based reward.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/dpo_reduce_hallucinations" rel="nofollow"><code>dpo_reduce_hallucinations</code></a></td><td>DPO fine-tuning of a Vision Language Model to reduce hallucinations using the <a href="https://huggingface.co/datasets/openbmb/RLAIF-V-Dataset" rel="nofollow">openbmb/RLAIF-V-Dataset</a> dataset.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/gold_chatbot_arena" rel="nofollow"><code>gold_chatbot_arena</code></a></td><td>General Online Logit Distillation (GOLD) of a Qwen2 teacher into a Llama 3.2 student (cross-tokenizer) on chatbot_arena_completions with <a href="/docs/trl/pr_7115/en/gold_trainer#trl.experimental.gold.GOLDTrainer">experimental.gold.GOLDTrainer</a>, with full-training and LoRA variants.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/gold_qwen3_vl" rel="nofollow"><code>gold_qwen3_vl</code></a></td><td>General Online Logit Distillation (GOLD) of Qwen3-VL-8B into smaller VLM students with <a href="/docs/trl/pr_7115/en/gold_trainer#trl.experimental.gold.GOLDTrainer">experimental.gold.GOLDTrainer</a>, covering same-family (JSD loss) and cross-family (ULD loss) distillation.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_2048" rel="nofollow"><code>grpo_2048</code></a></td><td>GRPO with tool calling to teach a model to play the 2048 game.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_browsergym" rel="nofollow"><code>grpo_browsergym</code></a></td><td>GRPO with the BrowserGym <a href="openenv">OpenEnv</a> environment, with LLM and VLM variants.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_carla" rel="nofollow"><code>grpo_carla</code></a></td><td>GRPO with the CARLA autonomous-driving <a href="openenv">OpenEnv</a> environment, with LLM and VLM variants (multimodal camera-image tool responses).</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_catch" rel="nofollow"><code>grpo_catch</code></a></td><td>GRPO with the Catch (OpenSpiel) <a href="openenv">OpenEnv</a> environment.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_continuous_batching" rel="nofollow"><code>grpo_continuous_batching</code></a></td><td>GRPO with transformers’ continuous batching engine for faster generation on large batches with variable completion lengths.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_echo" rel="nofollow"><code>grpo_echo</code></a></td><td>Minimal GRPO training with the Echo <a href="openenv">OpenEnv</a> environment.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_harbor" rel="nofollow"><code>grpo_harbor</code></a></td><td>GRPO training against a Harbor task suite with a pluggable base agent (<code>bash</code> / <code>jupyter</code> / <code>terminal_notes</code> harnesses). See the <a href="harbor">Harbor Integration</a> guide.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_ministral3_vl" rel="nofollow"><code>grpo_ministral3_vl</code></a></td><td>GRPO Ministral 3 with QLoRA on free Colab.</td><td><a href="https://colab.research.google.com/github/huggingface/trl/blob/main/examples/grpo_ministral3_vl/grpo_ministral3_vl.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_multi_env" rel="nofollow"><code>grpo_multi_env</code></a></td><td>Multi-environment GRPO training: Wordle + Catch <a href="openenv">OpenEnv</a> environments in the same training run.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_qlora" rel="nofollow"><code>grpo_qlora</code></a></td><td>GRPO using QLoRA on free Colab.</td><td><a href="https://colab.research.google.com/github/huggingface/trl/blob/main/examples/grpo_qlora/grpo_qlora.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_qwen3_vl" rel="nofollow"><code>grpo_qwen3_vl</code></a></td><td>GRPO Qwen3-VL with QLoRA on free Colab.</td><td><a href="https://colab.research.google.com/github/huggingface/trl/blob/main/examples/grpo_qwen3_vl/grpo_qwen3_vl.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_rnj_1_instruct" rel="nofollow"><code>grpo_rnj_1_instruct</code></a></td><td>GRPO on rnj-1-instruct with QLoRA on Colab to add reasoning capabilities.</td><td><a href="https://colab.research.google.com/github/huggingface/trl/blob/main/examples/grpo_rnj_1_instruct/grpo_rnj_1_instruct.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_seta" rel="nofollow"><code>grpo_seta</code></a></td><td>GRPO training against the SETA ORS environment on the openreward.ai catalog. See the <a href="openreward">OpenReward Integration</a> guide.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_sql_agent" rel="nofollow"><code>grpo_sql_agent</code></a></td><td>GRPO to train an agent that answers questions by querying a SQL database (script and notebook; not runnable on free Colab due to OOM).</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_sudoku" rel="nofollow"><code>grpo_sudoku</code></a></td><td>GRPO to play Sudoku on an <a href="openenv">OpenEnv</a> environment (script and notebook).</td><td><a href="https://colab.research.google.com/github/huggingface/trl/blob/main/examples/grpo_sudoku/grpo_sudoku.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_visual_math" rel="nofollow"><code>grpo_visual_math</code></a></td><td>GRPO fine-tuning of a multimodal model for reasoning using the <a href="https://huggingface.co/datasets/lmms-lab/multimodal-open-r1-8k-verified" rel="nofollow">lmms-lab/multimodal-open-r1-8k-verified</a> dataset.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/grpo_wordle" rel="nofollow"><code>grpo_wordle</code></a></td><td>GRPO to play Wordle (TextArena) on an <a href="openenv">OpenEnv</a> environment (script and notebook).</td><td><a href="https://colab.research.google.com/github/huggingface/trl/blob/main/examples/grpo_wordle/grpo_wordle.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/gspo_math" rel="nofollow"><code>gspo_math</code></a></td><td>GSPO via the <a href="/docs/trl/pr_7115/en/grpo_trainer#trl.GRPOTrainer">GRPOTrainer</a> for math reasoning on the <a href="https://huggingface.co/datasets/AI-MO/NuminaMath-TIR" rel="nofollow">AI-MO/NuminaMath-TIR</a> dataset.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/gspo_visual_math" rel="nofollow"><code>gspo_visual_math</code></a></td><td>GSPO via the <a href="/docs/trl/pr_7115/en/grpo_trainer#trl.GRPOTrainer">GRPOTrainer</a> to fine-tune a multimodal model for reasoning using the <a href="https://huggingface.co/datasets/lmms-lab/multimodal-open-r1-8k-verified" rel="nofollow">lmms-lab/multimodal-open-r1-8k-verified</a> dataset.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/mpo_visual_preferences" rel="nofollow"><code>mpo_visual_preferences</code></a></td><td>MPO via the <a href="/docs/trl/pr_7115/en/bema_for_reference_model#trl.DPOTrainer">DPOTrainer</a> to align a multimodal model based on preferences using the <a href="https://huggingface.co/datasets/HuggingFaceH4/rlaif-v_formatted" rel="nofollow">HuggingFaceH4/rlaif-v_formatted</a> dataset and a set of loss weights.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/online_dpo_visual_math" rel="nofollow"><code>online_dpo_visual_math</code></a></td><td>Online DPO fine-tuning of a Vision Language Model with <a href="/docs/trl/pr_7115/en/online_dpo_trainer#trl.experimental.online_dpo.OnlineDPOTrainer">experimental.online_dpo.OnlineDPOTrainer</a>.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/rloo_math" rel="nofollow"><code>rloo_math</code></a></td><td>RLOO with the <a href="/docs/trl/pr_7115/en/rloo_trainer#trl.RLOOTrainer">RLOOTrainer</a> for math reasoning on the <a href="https://huggingface.co/datasets/AI-MO/NuminaMath-TIR" rel="nofollow">AI-MO/NuminaMath-TIR</a> dataset with vLLM.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/rloo_visual_math" rel="nofollow"><code>rloo_visual_math</code></a></td><td>RLOO fine-tuning of a multimodal model for reasoning using the <a href="https://huggingface.co/datasets/lmms-lab/multimodal-open-r1-8k-verified" rel="nofollow">lmms-lab/multimodal-open-r1-8k-verified</a> dataset.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/sdft_privileged_context" rel="nofollow"><code>sdft_privileged_context</code></a></td><td>Self-distillation fine-tuning with <a href="/docs/trl/pr_7115/en/sdft_trainer#trl.experimental.sdft.SDFTTrainer">experimental.sdft.SDFTTrainer</a>, distilling privileged (teacher-only) context into the model.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/sdpo_math" rel="nofollow"><code>sdpo_math</code></a></td><td>SDPO with <a href="/docs/trl/pr_7115/en/sdpo_trainer#trl.experimental.sdpo.SDPOTrainer">experimental.sdpo.SDPOTrainer</a> using verifiable math rewards and optional environment feedback on <a href="https://huggingface.co/datasets/openai/gsm8k" rel="nofollow">openai/gsm8k</a>.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/sft_diffusion_gemma" rel="nofollow"><code>sft_diffusion_gemma</code></a></td><td>SFT of the DiffusionGemma block-diffusion language model on GSM8K by extending the <a href="/docs/trl/pr_7115/en/sft_trainer#trl.SFTTrainer">SFTTrainer</a> with a block-diffusion objective.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/sft_gemma3" rel="nofollow"><code>sft_gemma3</code></a></td><td>SFT of Gemma 3 on the Codeforces COTS dataset.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/sft_gemma3_vision" rel="nofollow"><code>sft_gemma3_vision</code></a></td><td>SFT of Gemma 3 on vision to text tasks.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/sft_gpt_oss" rel="nofollow"><code>sft_gpt_oss</code></a></td><td>SFT of openai/gpt-oss-20b.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/sft_ministral3_vl" rel="nofollow"><code>sft_ministral3_vl</code></a></td><td>SFT Ministral 3 with QLoRA on free Colab.</td><td><a href="https://colab.research.google.com/github/huggingface/trl/blob/main/examples/sft_ministral3_vl/sft_ministral3_vl.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/sft_nemotron_3" rel="nofollow"><code>sft_nemotron_3</code></a></td><td>SFT of NVIDIA Nemotron 3 models (script and LoRA notebook).</td><td><a href="https://colab.research.google.com/github/huggingface/trl/blob/main/examples/sft_nemotron_3/sft_nemotron_3.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/sft_qlora" rel="nofollow"><code>sft_qlora</code></a></td><td>SFT using QLoRA on free Colab.</td><td><a href="https://colab.research.google.com/github/huggingface/trl/blob/main/examples/sft_qlora/sft_qlora.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/sft_qwen3_8b_1m_context" rel="nofollow"><code>sft_qwen3_8b_1m_context</code></a></td><td>SFT of Qwen3-8B on 1,048,576-token sequences with context parallelism, on one 8xH100 node.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/sft_qwen3_vl" rel="nofollow"><code>sft_qwen3_vl</code></a></td><td>SFT Qwen3-VL with QLoRA on free Colab.</td><td><a href="https://colab.research.google.com/github/huggingface/trl/blob/main/examples/sft_qwen3_vl/sft_qwen3_vl.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/tail_sft_gsm8k" rel="nofollow"><code>tail_sft_gsm8k</code></a></td><td>TailSFT on GSM8K by extending <a href="/docs/trl/pr_7115/en/sft_trainer#trl.SFTTrainer">SFTTrainer</a> with initial-policy loss filtering.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/sft_tool_calling" rel="nofollow"><code>sft_tool_calling</code></a></td><td>Teaching tool calling to a model without native tool-calling support using SFT with QLoRA (script, chat template, and notebook).</td><td><a href="https://colab.research.google.com/github/huggingface/trl/blob/main/examples/sft_tool_calling/sft_tool_calling.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/sft_visual_chat" rel="nofollow"><code>sft_visual_chat</code></a></td><td>SFT of a Vision Language Model in a chat setting. Only tested with <a href="https://huggingface.co/llava-hf/llava-1.5-7b-hf" rel="nofollow">LLaVA 1.5</a>, <a href="https://huggingface.co/llava-hf/llava-v1.6-mistral-7b-hf" rel="nofollow">LLaVA 1.6</a>, and <a href="https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct" rel="nofollow">Llama-3.2-11B-Vision-Instruct</a>; users may see unexpected behaviour in other model architectures.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/ssd_codegen" rel="nofollow"><code>ssd_codegen</code></a></td><td>Simple Self-Distillation for code generation with <a href="/docs/trl/pr_7115/en/ssd_trainer#trl.experimental.ssd.SSDTrainer">experimental.ssd.SSDTrainer</a>, plus evaluation on LiveCodeBench.</td><td></td></tr><tr><td><a href="https://github.com/huggingface/trl/tree/main/examples/tpo_ultrafeedback" rel="nofollow"><code>tpo_ultrafeedback</code></a></td><td>Triple Preference Optimization with <a href="/docs/trl/pr_7115/en/tpo_trainer#trl.experimental.tpo.TPOTrainer">experimental.tpo.TPOTrainer</a> using the <a href="https://huggingface.co/datasets/tpo-alignment/triple-preference-ultrafeedback-40K" rel="nofollow">tpo-alignment/triple-preference-ultrafeedback-40K</a> dataset.</td><td></td></tr></tbody></table> <!--[1--><h2 class="relative group"><a id="distributed-training" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#distributed-training"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Distributed Training</span></h2><!--]--><!----> <p>You can run the example scripts on multiple GPUs with 🤗 Accelerate:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-shell "><!---->accelerate launch --config_file=examples/accelerate_configs/multi_gpu.yaml --num_processes {NUM_GPUS} path_to_script.py --all_arguments_of_the_script<!----></pre></div><!----> <p>For DeepSpeed ZeRO-{1,2,3}:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-shell "><!---->accelerate launch --config_file=examples/accelerate_configs/deepspeed_zero{1,2,3}.yaml --num_processes {NUM_GPUS} path_to_script.py --all_arguments_of_the_script<!----></pre></div><!----> <p>Adjust <code>NUM_GPUS</code> and <code>--all_arguments_of_the_script</code> as needed.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/trl/blob/main/docs/source/example_overview.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
			
			<script>
				{
					__sveltekit_1cw9439 = {
						base: "/docs/trl/pr_7115/en",
						assets: "/docs/trl/pr_7115/en"
					};
					const element = document.currentScript.parentElement;
					Promise.all([
						import("/docs/trl/pr_7115/en/_app/immutable/entry/start.2W4yWkdf.js"),
						import("/docs/trl/pr_7115/en/_app/immutable/entry/app.09KZcl8P.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 12],
							data: [null,null],
							form: null,
							error: null
						});
					});
				}
			</script>
		
Xet Storage Details

Size:: 32.9 kB
Xet hash:: ea576eed5bd23d642550c76b985bdd9513acb0300e39830cd68de49a2520ba72
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.