Skip to content

FinQA Financial Agent

This project demonstrates training and running FinQA, a financial question-answering agent fine-tuned from Qwen3-4B-Instruct-2507 using rLLM. The agent uses specialized tools (SQL queries, table lookup, calculators) to perform multi-step reasoning over SEC 10-K financial statements, achieving 59.7% on Snorkel Finance Benchmark — outperforming Qwen3-235B (51.4%) and rivaling Gemini 2.5 Pro (60.6%).

Model Weights | Dataset | Blog Post

Overview

The FinQA project demonstrates:

  • How to use rLLM's ToolAgent and ToolEnvironment for multi-step financial reasoning
  • How to build domain-specific tools in rLLM
  • How to train agents with GRPO using LLM-as-judge rewards

Quick Start

Installation

Follow rLLM installation, then install FinQA dependencies:

uv pip install -r projects/finqa/requirements.txt

Dataset Preparation

Downloads the rLLM/finqa dataset and registers train/val/test splits:

python -m projects.finqa.prepare_finqa_data

Inference

Start a vLLM server and run the agent:

python -m vllm.entrypoints.openai.api_server \
    --model rLLM/rLLM-FinQA-4B \
    --host 0.0.0.0 \
    --port 30000 \
    --dtype bfloat16

python -m projects.finqa.run_finqa

Training

Set the required environment variables before training:

Variable Description
OPENAI_API_KEY OpenAI API key for the reward judge
PORTKEY_API_KEY Portkey gateway key for reward judge caching 
# verl backend (Qwen3-4B-Instruct-2507)
bash projects/finqa/train_finqa.sh

# tinker backend (Qwen3-30B-A3B-Instruct-2507, LoRA)
bash projects/finqa/train_finqa_tinker.sh

Code Reference

Financial Agent Runner

Main script for running financial reasoning:

projects/finqa/run_finqa.py
import asyncio
import os

from transformers import AutoTokenizer

from projects.finqa.fin_qa_agent import FinQAAgent
from projects.finqa.fin_qa_environment import FinQAEnvironment
from rllm.data.dataset import DatasetRegistry
from rllm.engine.agent_execution_engine import AgentExecutionEngine
from rllm.utils import compute_pass_at_k

if __name__ == "__main__":
    os.environ["TOKENIZERS_PARALLELISM"] = "true"

    model_name = "rLLM/rLLM-FinQA-4B"
    tokenizer = AutoTokenizer.from_pretrained(model_name)

    sampling_params = {"temperature": 0.6, "top_p": 0.95}

    engine = AgentExecutionEngine(
        agent_class=FinQAAgent,
        env_class=FinQAEnvironment,
        engine_name="openai",
        rollout_engine_args={"model": model_name, "base_url": "http://localhost:30000/v1", "api_key": "None"},
        tokenizer=tokenizer,
        sampling_params=sampling_params,
        n_parallel_agents=100,
        max_steps=20,
        max_prompt_length=4096,
    )

    test_dataset = DatasetRegistry.load_dataset("finqa", "test")
    if test_dataset is None:
        print("Dataset not found, preparing dataset...")
        from projects.finqa.prepare_finqa_data import prepare_finqa_data

        _, _, test_dataset = prepare_finqa_data()

    tasks = test_dataset.repeat(n=1)

    results = asyncio.run(engine.execute_tasks(tasks))

    compute_pass_at_k(results)

Training Script

FinQA training configuration:

projects/finqa/train_finqa.py
import hydra

from rllm.agents.agent import Episode
from rllm.data.dataset import DatasetRegistry
from rllm.engine.rollout.rollout_engine import ModelOutput
from rllm.trainer.agent_trainer import AgentTrainer
from rllm.workflows.multi_turn_workflow import MultiTurnWorkflow
from rllm.workflows.workflow import TerminationEvent, TerminationReason

from .fin_qa_agent import FinQAAgent
from .fin_qa_environment import FinQAEnvironment


class FinQAWorkflow(MultiTurnWorkflow):
    """MultiTurnWorkflow with reward logging"""

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    async def run(self, task: dict, uid: str, **kwargs) -> Episode | None:
        observation, info = await self.run_in_executor(self.reset, task=task, uid=uid)

        self.agent.update_from_env(observation, 0, False, info)

        # Calculate max context once (max_prompt_length + max_response_length)
        max_model_len = self.rollout_engine.max_prompt_length + self.rollout_engine.max_response_length
        min_response_buffer = 1000  # Minimum tokens to reserve for model response

        for _ in range(1, self.max_steps + 1):
            # Check if conversation is approaching context limit
            if hasattr(self.rollout_engine, "chat_parser"):
                # verl backend - use chat_parser
                prompt = self.rollout_engine.chat_parser.parse(
                    self.agent.chat_completions,
                    add_generation_prompt=True,
                    is_first_msg=True,
                )
                prompt_length = len(self.rollout_engine.tokenizer.encode(prompt, add_special_tokens=False))
            else:
                # Tinker backend - use tokenizer directly
                prompt_ids = self.rollout_engine.tokenizer.apply_chat_template(
                    self.agent.chat_completions,
                    add_generation_prompt=True,
                    tokenize=True,
                )
                prompt_length = len(prompt_ids)

            if prompt_length > max_model_len - min_response_buffer:
                raise TerminationEvent(TerminationReason.MAX_PROMPT_LENGTH_EXCEEDED)

            output: ModelOutput = await self.rollout_engine.get_model_response(
                self.agent.chat_completions,
                application_id=uid,
                enforce_max_prompt_length=False,
                **kwargs,
            )
            response = output.text

            action = self.agent.update_from_model(response)

            # Store model_output on step for Tinker training (verl uses chat_completions)
            if not hasattr(self.rollout_engine, "chat_parser") and self.agent.trajectory.steps:
                self.agent.trajectory.steps[-1].model_output = output

            next_obs, reward, done, info = await self.run_in_executor(self.env.step, action.action)

            self.agent.update_from_env(next_obs, reward, done, info)

            if output.finish_reason == "length":
                raise TerminationEvent(TerminationReason.MAX_RESPONSE_LENGTH_EXCEEDED)

            if done:
                raise TerminationEvent(TerminationReason.ENV_DONE)

        raise TerminationEvent(TerminationReason.MAX_TURNS_EXCEEDED)

    def assign_episode_correctness(self, episode: Episode) -> None:
        """Use is_correct from Reward, before relying on default option"""
        # Check if last step has is_correct from environment
        if episode.trajectories and episode.trajectories[0].steps:
            is_correct = episode.trajectories[0].steps[-1].info.get("is_correct")
            if is_correct is not None:
                episode.is_correct = is_correct
                return

        super().assign_episode_correctness(episode)

    def collect_metrics(self, episode: Episode) -> None:
        super().collect_metrics(episode)
        # Added metadata from last step -> for wandb logging
        if episode.trajectories and episode.trajectories[0].steps:
            metadata = episode.trajectories[0].steps[-1].info.get("metadata", {})
            episode.metrics.update(metadata)


@hydra.main(
    config_path="pkg://rllm.trainer.config",
    config_name="agent_ppo_trainer",
    version_base=None,
)
def main(config):
    train_dataset = DatasetRegistry.load_dataset("finqa", "train")
    val_dataset = DatasetRegistry.load_dataset("finqa", "val")

    config.rllm.workflow.use_workflow = True

    trainer = AgentTrainer(
        workflow_class=FinQAWorkflow,
        workflow_args={
            "agent_cls": FinQAAgent,
            "env_cls": FinQAEnvironment,
            "max_steps": 20,
        },
        config=config,
        train_dataset=train_dataset,
        val_dataset=val_dataset,
    )
    trainer.train()


if __name__ == "__main__":
    main()

For detailed setup instructions, see the README in the finqa project directory.