AE 20: Tool calling

Application exercise
R
Python
Modified

November 11, 2025

20_quiz-game-2

library(shiny)
library(bslib)
library(beepr)
library(ellmer)
library(shinychat)

# Tools ------------------------------------------------------------------------

#' Plays a sound effect.
#'
#' @param sound Which sound effect to play: `"correct"`, `"incorrect"`,
#'   `"new-round"`, or `"you-win"`.
#' @returns A confirmation that the sound was played.
play_sound <- function(
  sound = c("correct", "incorrect", "new-round", "you-win")
) {
  sound <- match.arg(sound)

  switch(
    sound,
    correct = beepr::beep("coin"),
    incorrect = beepr::beep("wilhelm"),
    "new-round" = beepr::beep("fanfare"),
    "you-win" = beepr::beep("mario")
  )

  glue::glue("The '{sound}' sound was played.")
}

# STEP 1: Create a tool definition with documentation ----
# Remember: you're teaching the LLM how and when to use this function.
tool_play_sound <- tool(
  play_sound,
  description = "____",
  arguments = list(
    ____ = type_____()
  )
)

# UI ---------------------------------------------------------------------------

ui <- page_fillable(
  chat_mod_ui("chat")
)

# Server -----------------------------------------------------------------------

server <- function(input, output, session) {
  client <- chat(
    "anthropic/claude-3-7-sonnet-20250219",
    system_prompt = interpolate_file(
      here::here("data/prompt.md")
    )
  )

  # STEP 2: Register the tool with the chat client ----
  client$____(____)

  chat <- chat_mod_server("chat", client)

  observe({
    # Note: This block starts the game when the app launches
    chat$update_user_input(
      value = "Let's play the quiz game!",
      submit = TRUE
    )
  })
}

shinyApp(ui, server)
from pathlib import Path
from typing import Literal

import chatlas
import dotenv
from playsound3 import playsound
from pyhere import here
from shiny import App, reactive, ui

dotenv.load_dotenv()

# Tools ------------------------------------------------------------------------
SoundChoice = Literal["correct", "incorrect", "new-round", "you-win"]

sound_map: dict[SoundChoice, Path] = {
    "correct": here("data/sounds/smb_coin.wav"),
    "incorrect": here("data/sounds/wilhelm.wav"),
    "new-round": here("data/sounds/victory_fanfare_mono.wav"),
    "you-win": here("data/sounds/smb_stage_clear.wav"),
}


# STEP 1: Document this function so the LLM knows how to use it ----
def play_sound(sound="correct"):
    if sound not in sound_map.keys():
        raise ValueError(
            f"sound must be one of {sorted(sound_map.keys())}; got {sound!r}"
        )

    playsound(sound_map[sound])

    return f"The '{sound}' sound was played."


# UI ---------------------------------------------------------------------------

app_ui = ui.page_fillable(
    ui.chat_ui("chat"),
)


def server(input, output, session):
    # Recall: We set up the Chat UI server logic and the chat client in the
    # server function so that each user session gets its own chat history.
    chat_ui = ui.Chat(id="chat")
    client = chatlas.ChatAnthropic(
        model="claude-3-7-sonnet-20250219",
        # Use your quiz game system prompt, or switch to _solutions to use ours
        system_prompt=here("data/prompt.md").read_text(),
    )

    # STEP 2: Register the tool with the chat client ----
    client.____(____)

    @chat_ui.on_user_submit
    async def handle_user_input(user_input: str):
        # STEP 3: Set `content="all"` when streaming from the chatlas client
        # so that the Chat UI includes tool calls
        response = await client.stream_async(user_input)
        await chat_ui.append_message_stream(response)

    @reactive.effect
    def _():
        # Note: This block starts the game when the app launches
        chat_ui.update_user_input(value="Let's play the quiz game!", submit=True)


app = App(app_ui, server)

21_quiz-game-3

library(shiny)
library(bslib)
library(beepr)
library(ellmer)
library(shinychat)

# Tools ------------------------------------------------------------------------

#' Plays a sound effect.
#'
#' @param sound Which sound effect to play: `"correct"`, `"incorrect"`,
#'   `"new-round"`, or `"you-win"`.
#' @returns A confirmation that the sound was played.
play_sound <- function(
  sound = c("correct", "incorrect", "new-round", "you-win")
) {
  sound <- match.arg(sound)

  switch(
    sound,
    correct = beepr::beep("coin"),
    incorrect = beepr::beep("wilhelm"),
    "new-round" = beepr::beep("fanfare"),
    "you-win" = beepr::beep("mario")
  )

  glue::glue("The '{sound}' sound was played.")
}

tool_play_sound <- tool(
  play_sound,
  description = "Play a sound effect",
  arguments = list(
    sound = type_enum(
      c("correct", "incorrect", "new-round", "you-win"),
      description = paste(
        "Which sound effect to play.",
        "Play 'new-round' after the user picks a theme for the round.",
        "Play 'correct' or 'incorrect' after the user answers a question.",
        "Play 'you-win' at the end of a round of questions."
      )
    )
  ),
  # STEP 1: Add nice title and icon for the tool button ----
  annotations = ____(
    ____ = "____",
    # Pick a Font Awesome icon from the "free" choices
    # https://fontawesome.com/search?q=speaker&ic=free&o=r
    ____ = fontawesome::fa_i("____")
  )
)


# UI ---------------------------------------------------------------------------

ui <- page_fillable(
  chat_mod_ui("chat")
)

# Server -----------------------------------------------------------------------

server <- function(input, output, session) {
  client <- chat(
    "anthropic/claude-3-7-sonnet-20250219",
    system_prompt = interpolate_file(
      here::here("data/prompt.md")
    )
  )

  client$register_tool(tool_play_sound)

  chat <- chat_mod_server("chat", client)

  observe({
    # Note: This block starts the game when the app launches
    chat$update_user_input(
      value = "Let's play the quiz game!",
      submit = TRUE
    )
  })
}

shinyApp(ui, server)
from pathlib import Path
from typing import Literal

import chatlas
import dotenv
import faicons
from playsound3 import playsound
from pyhere import here
from shiny import App, reactive, ui

dotenv.load_dotenv()

# Tools ------------------------------------------------------------------------
SoundChoice = Literal["correct", "incorrect", "new-round", "you-win"]

sound_map: dict[SoundChoice, Path] = {
    "correct": here("data/sounds/smb_coin.wav"),
    "incorrect": here("data/sounds/wilhelm.wav"),
    "new-round": here("data/sounds/victory_fanfare_mono.wav"),
    "you-win": here("data/sounds/smb_stage_clear.wav"),
}


def play_sound(sound: SoundChoice = "correct") -> str:
    """
    Plays a sound effect.

    Parameters
    ----------
    sound: Which sound effect to play: "correct", "incorrect", "new-round" or
           "you-win". Play the "new-round" sound after the user picks a theme
           for the round. Play the "correct" and "incorrect" sounds when the
           user answers a question correctly or incorrectly, respectively. And
           play the "you-win" sound at the end of a round of questions.

    Returns
    -------
    A confirmation that the sound was played.
    """
    if sound not in sound_map.keys():
        raise ValueError(
            f"sound must be one of {sorted(sound_map.keys())}; got {sound!r}"
        )

    playsound(sound_map[sound])

    return f"The '{sound}' sound was played."


# UI ---------------------------------------------------------------------------

app_ui = ui.page_fillable(
    ui.chat_ui("chat"),
)


def server(input, output, session):
    # Recall: We set up the Chat UI server logic and the chat client in the
    # server function so that each user session gets its own chat history.
    chat_ui = ui.Chat(id="chat")
    client = chatlas.ChatAnthropic(
        model="claude-3-7-sonnet-20250219",
        # Use your quiz game system prompt, or switch to _solutions to use ours
        system_prompt=here("data/prompt.md").read_text(),
    )

    client.register_tool(
        play_sound,
        # STEP 1: Add nice title and icon for the tool button ----
        annotations={
            "____": "____",
            "extra": {
                # Pick a Font Awesome icon from the "free" choices
                # https://fontawesome.com/search?q=speaker&ic=free&o=r
                "____": faicons.icon_svg("____"),
            },
        },
    )

    @chat_ui.on_user_submit
    async def handle_user_input(user_input: str):
        response = await client.stream_async(user_input, content="all")
        await chat_ui.append_message_stream(response)

    @reactive.effect
    def _():
        # Note: This block starts the game when the app launches
        chat_ui.update_user_input(value="Let's play the quiz game!", submit=True)


app = App(app_ui, server)

22_quiz-game-4

library(shiny)
library(bslib)
library(beepr)
library(ellmer)
library(shinychat)

# Tools ------------------------------------------------------------------------

#' Plays a sound effect.
#'
#' @param sound Which sound effect to play: `"correct"`, `"incorrect"`,
#'   `"new-round"`, or `"you-win"`.
#' @returns A confirmation that the sound was played.
play_sound <- function(
  sound = c("correct", "incorrect", "new-round", "you-win")
) {
  sound <- match.arg(sound)

  switch(
    sound,
    correct = beepr::beep("coin"),
    incorrect = beepr::beep("wilhelm"),
    "new-round" = beepr::beep("fanfare"),
    "you-win" = beepr::beep("mario")
  )

  icon <- switch(
    sound,
    # STEP 1: Pick icons for each sound/action ----
    # Search here: https://fontawesome.com/search?q=speaker&ic=free&o=r
    correct = fontawesome::fa_i(
      "____",
      class = "text-success",
      prefer_type = "solid"
    ),
    incorrect = fontawesome::fa_i(
      "____",
      class = "text-danger",
      prefer_type = "solid"
    ),
    "new-round" = fontawesome::fa_i(
      "____",
      class = "text-primary",
      prefer_type = "solid"
    ),
    "you-win" = fontawesome::fa_i("____", class = "text-warning")
  )

  # STEP 2: Give each action it's own title ----
  title <- switch(
    sound,
    correct = "____",
    incorrect = "____",
    "new-round" = "____",
    "you-win" = "____"
  )

  # STEP 3: Return tool result content, w/ the extra display data ----
  ____(
    glue::glue("The '{sound}' sound was played."),
    extra = list(
      display = list(
        title = ____,
        ____ = ____
      )
    )
  )
}

tool_play_sound <- tool(
  play_sound,
  description = "Play a sound effect",
  arguments = list(
    sound = type_enum(
      c("correct", "incorrect", "new-round", "you-win"),
      description = paste(
        "Which sound effect to play.",
        "Play 'new-round' after the user picks a theme for the round.",
        "Play 'correct' or 'incorrect' after the user answers a question.",
        "Play 'you-win' at the end of a round of questions."
      )
    )
  ),
  annotations = tool_annotations(title = "Play Sound Effect")
)


# UI ---------------------------------------------------------------------------

ui <- page_fillable(
  chat_mod_ui("chat")
)

# Server -----------------------------------------------------------------------

server <- function(input, output, session) {
  client <- chat(
    "anthropic/claude-3-7-sonnet-20250219",
    system_prompt = interpolate_file(
      here::here("data/prompt.md")
    )
  )

  client$register_tool(tool_play_sound)

  chat <- chat_mod_server("chat", client)

  observe({
    # Note: This block starts the game when the app launches
    chat$update_user_input(
      value = "Let's play the quiz game!",
      submit = TRUE
    )
  })
}

shinyApp(ui, server)
from pathlib import Path
from typing import Any, Literal

import chatlas
import dotenv
from faicons import icon_svg
from playsound3 import playsound
from pyhere import here
from shiny import App, reactive, ui

dotenv.load_dotenv()

# Tools ------------------------------------------------------------------------
SoundChoice = Literal["correct", "incorrect", "new-round", "you-win"]

sound_map: dict[SoundChoice, Path] = {
    "correct": here("data/sounds/smb_coin.wav"),
    "incorrect": here("data/sounds/wilhelm.wav"),
    "new-round": here("data/sounds/victory_fanfare_mono.wav"),
    "you-win": here("data/sounds/smb_stage_clear.wav"),
}

# STEP 1: Pick icons for each sound/action ----
# Search here: https://fontawesome.com/search?q=speaker&ic=free&o=r
icon_map: dict[SoundChoice, Any] = {
    "correct": icon_svg("____", fill="var(--bs-success)"),
    "incorrect": icon_svg("____", fill="var(--bs-danger)"),
    "new-round": icon_svg("____", fill="var(--bs-primary)"),
    "you-win": icon_svg("____", fill="var(--bs-warning)"),
}

# STEP 2: Give each action it's own title ----
title_map: dict[SoundChoice, str] = {
    "correct": "____",
    "incorrect": "____",
    "new-round": "____",
    "you-win": "____",
}


def play_sound(sound: SoundChoice = "correct") -> str:
    """
    Plays a sound effect.

    Parameters
    ----------
    sound: Which sound effect to play: "correct", "incorrect", "new-round" or
           "you-win". Play the "new-round" sound after the user picks a theme
           for the round. Play the "correct" and "incorrect" sounds when the
           user answers a question correctly or incorrectly, respectively. And
           play the "you-win" sound at the end of a round of questions.

    Returns
    -------
    A confirmation that the sound was played.
    """
    if sound not in sound_map.keys():
        raise ValueError(
            f"sound must be one of {sorted(sound_map.keys())}; got {sound!r}"
        )

    playsound(sound_map[sound])

    # STEP 3: Return tool result content, w/ the extra display data ----
    return chatlas.____(
        value=f"The '{sound}' sound was played.",
        extra={
            "display": {
                "title": ____[sound],
                "____": ____[sound],
            }
        },
    )


# UI ---------------------------------------------------------------------------

app_ui = ui.page_fillable(
    ui.chat_ui("chat"),
)


def server(input, output, session):
    # Recall: We set up the Chat UI server logic and the chat client in the
    # server function so that each user session gets its own chat history.
    chat_ui = ui.Chat(id="chat")
    client = chatlas.ChatAnthropic(
        model="claude-3-7-sonnet-20250219",
        system_prompt=here("data/prompt.md").read_text(),
    )

    client.register_tool(
        play_sound,
        annotations={"title": "Play Sound Effect"},
    )

    @chat_ui.on_user_submit
    async def handle_user_input(user_input: str):
        response = await client.stream_async(user_input, content="all")
        await chat_ui.append_message_stream(response)

    @reactive.effect
    def _():
        # Note: This block starts the game when the app launches
        chat_ui.update_user_input(value="Let's play the quiz game!", submit=True)


app = App(app_ui, server)

23_quiz-game-5

library(shiny)
library(bslib)
library(ellmer)
library(shinychat)

# UI ---------------------------------------------------------------------------

ui <- page_sidebar(
  title = "Quiz Game 5",
  sidebar = sidebar(
    position = "right",
    fillable = TRUE,
    width = 400,
    value_box(
      "Correct Answers",
      textOutput("txt_correct"),
      showcase = fontawesome::fa_i("circle-check"),
      theme = "success"
    ),
    value_box(
      "Incorrect Answers",
      textOutput("txt_incorrect"),
      showcase = fontawesome::fa_i("circle-xmark"),
      theme = "danger"
    )
  ),
  navset_card_tab(
    nav_panel("Quiz Game", chat_mod_ui("chat")),
    nav_panel("Your Answers", tableOutput("tbl_scores"))
  )
)

# Server -----------------------------------------------------------------------

server <- function(input, output, session) {
  client <- chat(
    "anthropic/claude-3-7-sonnet-20250219",
    system_prompt = interpolate_file(
      # Use your quiz game system prompt, or switch to `_solutions` to use ours
      here::here("data/prompt.md")
    ) |>
      # STEP 1: Add instructions about when to use the score-keeping tool ----
      paste(
        "\n\nAfter every question, use the 'Update Score' tool to... ____"
      )
  )

  scores <- reactiveVal(
    data.frame(
      theme = character(),
      question = character(),
      answer = character(),
      your_answer = character(),
      is_correct = logical()
    )
  )

  output$tbl_scores <- renderTable(scores())
  output$txt_correct <- renderText(sum(scores()$is_correct, na.rm = TRUE))
  output$txt_incorrect <- renderText(sum(!scores()$is_correct, na.rm = TRUE))

  # STEP 2: Implement the score-keeping tool ----
  # When the tool is called, add a new row to the `scores()` reactive data frame
  update_score <- function(theme, question, answer, your_answer, is_correct) {
    the_scores <- isolate(scores())
    the_scores <- rbind(
      the_scores,
      ____
    )
    # Now that we have new scores, update the `scores()` reactive value
    scores(the_scores)
    correct <- sum(the_scores$answer == the_scores$your_answer)
    # And return the current tally of correct and incorrect answers
    list(correct = correct, incorrect = nrow(the_scores) - correct)
  }

  client$register_tool(tool(
    update_score,
    description = paste(
      "Add a correct or incorrect answer to the score tally.",
      "Call this tool after you've graded the user's answer to a question."
    ),
    # STEP 3: Complete the tool definition ----
    arguments = list(
      theme = ____("The theme of the round."),
      question = ____("The quiz question that was asked."),
      answer = ____("The correct answer to the question."),
      your_answer = ____("The user's answer to the question."),
      is_correct = ____("Whether the user's answer was correct.")
    ) #,
    ## STEP 4: Add tool annotations ----
    # annotations = tool_annotations(
    #   title = "____",
    #   # https://fontawesome.com/search?q=plus&ic=free&o=r
    #   ____ = fontawesome::fa_i("____")
    # )
  ))

  chat <- chat_mod_server("chat", client)

  observe({
    # Note: This block starts the game when the app launches
    chat$update_user_input(
      value = "Let's play the quiz game!",
      submit = TRUE
    )
  })
}

shinyApp(ui, server)
from typing import TypedDict

import chatlas
import dotenv
import polars as pl
from faicons import icon_svg
from pyhere import here
from shiny import App, reactive, render, ui

dotenv.load_dotenv()

# UI ---------------------------------------------------------------------------

app_ui = ui.page_sidebar(
    ui.sidebar(
        ui.value_box(
            "Correct Answers",
            ui.output_text("txt_correct"),
            showcase=icon_svg("circle-check"),
            theme="success",
        ),
        ui.value_box(
            "Incorrect Answers",
            ui.output_text("txt_incorrect"),
            showcase=icon_svg("circle-xmark"),
            theme="danger",
        ),
        position="right",
        fillable=True,
        width=400,
    ),
    ui.navset_card_underline(
        ui.nav_panel(
            "Quiz Game",
            ui.chat_ui("chat"),
        ),
        ui.nav_panel(
            "Your Answers",
            ui.output_data_frame("tbl_score"),
        ),
    ),
    title="Quiz Game 5",
    fillable=True,
)


class QuestionAnswer(TypedDict):
    theme: str
    question: str
    answer: str
    your_answer: str
    is_correct: bool


def server(input, output, session):
    chat_ui = ui.Chat(id="chat")

    # Set up the chat instance
    client = chatlas.ChatAnthropic(
        model="claude-3-7-sonnet-20250219",
        # STEP 1: Add instructions about when to use the score-keeping tool ----
        system_prompt=f"""
{here("data/prompt.md").read_text()}

After every question, use the "Update Score" tool to... ____
""",
    )

    scores = reactive.value[list[QuestionAnswer]]([])

    @render.data_frame
    def tbl_score():
        df = pl.DataFrame(scores())
        return df

    @render.text
    def txt_correct() -> int:
        return len([d for d in scores() if d["is_correct"]])

    @render.text
    def txt_incorrect() -> int:
        return len([d for d in scores() if not d["is_correct"]])

    # STEP 3: Complete the tool definition by filling in the type hints ----
    def update_score(
        theme: ____,
        question: ____,
        answer: ____,
        your_answer: ____,
        is_correct: ____,
    ):
        """
        Add a correct or incorrect answer to the score. Call this tool after
        you've graded the user's answer to a question.

        Parameters
        ----------
        theme: The theme of the round.
        question: The quiz question that was asked.
        answer: The correct answer to the question.
        your_answer: The user's answer to the question.
        is_correct: Whether the user's answer was correct.
        """
        with reactive.isolate():
            val_scores = scores.get()

        # STEP 2: Implement the score-keeping tool ----
        # When the tool is called, add a new answer to the scores reactive.value
        answer = QuestionAnswer(____)

        val_scores = [*val_scores, ____]
        scores.set(val_scores)

        correct = len([d for d in val_scores if d["is_correct"]])
        incorrect = len(val_scores) - correct
        return {"correct": correct, "incorrect": incorrect}

    client.register_tool(
        update_score,
        ## STEP 4: Add tool annotations ----
        # annotations={
        #     "title": "____",
        #     # https://fontawesome.com/search?q=plus&ic=free&o=r
        #     "extra": {"____": icon_svg("____")},
        # },
    )

    @chat_ui.on_user_submit
    async def handle_user_input(user_input: str):
        response = await client.stream_async(user_input, content="all")
        await chat_ui.append_message_stream(response)

    @reactive.effect
    def _():
        chat_ui.update_user_input(value="Let's play the quiz game!", submit=True)


app = App(app_ui, server)

Acknowledgments