sdk-multimodal

Multimodal messages through the SDK — images and text as first-class content.

Run it from sema/:

sema check examples/sdk-multimodal
SEMA_STRICT=1 sema run examples/sdk-multimodal
sema assure examples/sdk-multimodal --grade silver

Source

src/main.sema

# Portable bridge fixture for the Sema SDK multimodal surface. The explicit
# `real` profile runs the development HuggingFace adapters without fallback.
from sdk_multimodal.ai import backend_profile, caption, ocr, vqa, transcribe, speak

assure silver

def exercise_fixture(out_path: str) -> list[str] !{proc.run, fs.read, fs.write, ffi.call}:
    description = caption("text.png")
    text = ocr("text.png")
    answer = vqa("text.png", "what color is the box?")
    speak("the quick brown fox jumps over the lazy dog", out_path)
    transcript = transcribe(out_path)
    return [description, text, answer, transcript]

def main() -> None !{proc.run, fs.read, fs.write, ffi.call, observe.record}:
    results = exercise_fixture("spoken.wav")
    log.info("multimodal bridge", profile=backend_profile())
    log.info("caption", text=results[0])
    log.info("ocr", text=results[1])
    log.info("vqa", answer=results[2])
    log.info("stt (round-trip)", text=results[3])

test "checked-in fixture exercises every multimodal bridge operation":
    check backend_profile() == "fixture"
    results = exercise_fixture("spoken.wav")
    check results == ["a red outlined box beside the text HELLO SEMA", "HELLO SEMA", "red", "the quick brown fox jumps over the lazy dog"]

src/ai.sema

# Sema SDK — the multimodal capability surface, written in Sema.
#
# The language does not ship blank: this module is the "standard AI library".
# Each function is a clean native Sema interface; the backend is chosen by the
# config/model registry (§5.38/§5.43). Text + embeddings run on the built-in /
# GGUF engines out of the box; vision/OCR/STT/TTS reuse small HuggingFace models
# through the Python bridge (§5.44) — we don't reinvent well-solved wheels, we
# bind them behind a Sema interface. Swap any backend in `sema.toml` (D48/D51).

import python
import tools

def backend_profile() -> str !{proc.run, ffi.call}:
    sem "Report the explicit fixture or real multimodal bridge profile"
    return python.call("multimodal_bridge", "profile", [])

# ---- text ----------------------------------------------------------------

def chat(prompt: str) -> str !{model.invoke}:
    sem "Generate a natural-language response with the configured text model"
    return tools.run(prompt, []).get("answer")

def ask(question: str, toolset: list) -> str !{model.invoke}:
    sem "Answer a question, letting the model call the given Sema functions as tools"
    return tools.run(question, toolset).get("answer")

# ---- embeddings ----------------------------------------------------------

def embed_text(text: str) -> list[f64] !{model.embed}:
    sem "Embed text into a vector with the configured embedding model"
    return embed(text)

def similarity(a: str, b: str) -> f64 !{model.embed}:
    sem "Cosine similarity of two texts' embeddings"
    return (a ~= b).score

# ---- vision (reuses a small HF caption/vision model) ---------------------

def caption(image_path: str) -> str !{proc.run, fs.read, ffi.call}:
    sem "Describe an image in natural language"
    return python.call("multimodal_bridge", "caption", [image_path])

def vqa(image_path: str, question: str) -> str !{proc.run, fs.read, ffi.call}:
    sem "Answer a question about an image"
    return python.call("multimodal_bridge", "vqa", [image_path, question])

# ---- OCR -----------------------------------------------------------------

def ocr(image_path: str) -> str !{proc.run, fs.read, ffi.call}:
    sem "Extract text from an image (OCR)"
    return python.call("multimodal_bridge", "ocr", [image_path])

# ---- speech --------------------------------------------------------------

def transcribe(audio_path: str) -> str !{proc.run, fs.read, ffi.call}:
    sem "Transcribe speech from an audio file to text (STT)"
    return python.call("multimodal_bridge", "transcribe", [audio_path])

def speak(text: str, out_path: str) -> str !{proc.run, fs.write, ffi.call}:
    sem "Synthesize speech audio from text (TTS); returns the output path"
    return python.call("multimodal_bridge", "speak", [text, out_path])

Reflected API

ai

def backend_profile

def backend_profile() -> str !{proc.run, ffi.call}

Returns str

Effects !{proc.run, ffi.call}

def chat

def chat(prompt: str) -> str !{model.invoke}

Parameters

name	type
prompt	str

Returns str

Effects !{model.invoke}

def ask

def ask(question: str, toolset: list) -> str !{model.invoke}

Parameters

name	type
question	str
toolset	list

Returns str

Effects !{model.invoke}

def embed_text

def embed_text(text: str) -> list[f64] !{model.embed}

Parameters

name	type
text	str

Returns list[f64]

Effects !{model.embed}

def similarity

def similarity(a: str, b: str) -> f64 !{model.embed}

Parameters

name	type
a	str
b	str

Returns f64

Effects !{model.embed}

def caption

def caption(image_path: str) -> str !{proc.run, fs.read, ffi.call}

Parameters

name	type
image_path	str

Returns str

Effects !{proc.run, fs.read, ffi.call}

def vqa

def vqa(image_path: str, question: str) -> str !{proc.run, fs.read, ffi.call}

Parameters

name	type
image_path	str
question	str

Returns str

Effects !{proc.run, fs.read, ffi.call}

def ocr

def ocr(image_path: str) -> str !{proc.run, fs.read, ffi.call}

Parameters

name	type
image_path	str

Returns str

Effects !{proc.run, fs.read, ffi.call}

def transcribe

def transcribe(audio_path: str) -> str !{proc.run, fs.read, ffi.call}

Parameters

name	type
audio_path	str

Returns str

Effects !{proc.run, fs.read, ffi.call}

def speak

def speak(text: str, out_path: str) -> str !{proc.run, fs.write, ffi.call}

Parameters

name	type
text	str
out_path	str

Returns str

Effects !{proc.run, fs.write, ffi.call}

main

def exercise_fixture

def exercise_fixture(out_path: str) -> list[str] !{proc.run, fs.read, fs.write, ffi.call}

Parameters

name	type
out_path	str

Returns list[str]

Effects !{proc.run, fs.read, fs.write, ffi.call}

def main

def main() -> None !{proc.run, fs.read, fs.write, ffi.call, observe.record}

Returns None

Effects !{proc.run, fs.read, fs.write, ffi.call, observe.record}