Show a computer-use or browser agent's screen

"use client"

import * as React from "react"
import type { AgentScreenConnection, AgentScreenSource } from "@/components/candor/agent-screen"

type Frame = { url: string; at: string; action: string }

/** The latest frame of a run's screen, from your server's event stream. */
export function useScreenFrames(runId: string) {
  const [frame, setFrame] = React.useState<Frame>()
  const [connection, setConnection] = React.useState<AgentScreenConnection>("connecting")

  React.useEffect(() => {
    const events = new EventSource(`/api/runs/${runId}/screen`)
    events.addEventListener("frame", (e) => {
      setFrame(JSON.parse((e as MessageEvent<string>).data))
      setConnection("live")
    })
    events.addEventListener("end", () => {
      setConnection("ended")
      events.close()
    })
    // EventSource tries again by itself. Until a frame arrives, the screen says the connection was lost.
    events.onerror = () => setConnection((c) => (c === "ended" ? c : "lost"))
    return () => events.close()
  }, [runId])

  const source: AgentScreenSource | undefined = frame && { kind: "image", src: frame.url }
  return { source, connection, frameAt: frame?.at, action: frame?.action }
}

import { canWatchRun, onScreenFrame } from "@/lib/runs" // your own

export async function GET(request: Request, { params }: { params: Promise<{ id: string }> }) {
  const { id } = await params
  // Frames show whatever the agent can see: check the viewer as you would for the run itself.
  if (!(await canWatchRun(request, id))) return new Response(null, { status: 404 })

  const encoder = new TextEncoder()
  const body = new ReadableStream({
    start(controller) {
      const send = (event: string, data: unknown) => controller.enqueue(encoder.encode(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`))
      const stop = onScreenFrame(id, {
        // `capturedAt` is when the screenshot was taken on the agent's machine, not when it reached the server.
        frame: (f) => send("frame", { url: `/api/runs/${id}/frames/${f.n}`, at: f.capturedAt, action: f.action }),
        end: () => {
          send("end", {})
          controller.close()
        },
      })
      request.signal.addEventListener("abort", stop)
    },
  })
  return new Response(body, { headers: { "Content-Type": "text/event-stream", "Cache-Control": "no-cache" } })
}

"use client"

import * as React from "react"
import type { AgentScreenConnection } from "@/components/candor/agent-screen"

const connectionOf = (state: RTCPeerConnectionState): AgentScreenConnection =>
  state === "connected" ? "live" : state === "closed" ? "ended" : state === "failed" || state === "disconnected" ? "lost" : "connecting"

/** A remote browser's screen over WebRTC: the peer connection is yours, set up with your own signalling. */
export function useScreenStream(pc: RTCPeerConnection) {
  const [stream, setStream] = React.useState<MediaStream>()
  const [connection, setConnection] = React.useState<AgentScreenConnection>(() => connectionOf(pc.connectionState))

  React.useEffect(() => {
    const onTrack = (e: RTCTrackEvent) => setStream(e.streams[0])
    const onState = () => setConnection(connectionOf(pc.connectionState))
    pc.addEventListener("track", onTrack)
    pc.addEventListener("connectionstatechange", onState)
    return () => {
      pc.removeEventListener("track", onTrack)
      pc.removeEventListener("connectionstatechange", onState)
    }
  }, [pc])

  return { source: stream && { kind: "stream" as const, stream }, connection }
}

"use client"

import { AgentScreen } from "@/components/candor/agent-screen"
import { useScreenFrames } from "./use-screen-frames"
import { useRun } from "@/lib/use-run" // your own: the run's state and the calls that change it

export function RunScreen({ runId }: { runId: string }) {
  const screen = useScreenFrames(runId)
  const run = useRun(runId)
  return (
    <AgentScreen
      agent={run.agent}
      {...screen}
      control={run.controlledBy === "agent" ? "agent" : "person"}
      person={run.controlledBy === "viewer" || run.controlledBy === "agent" ? undefined : run.controlledBy}
      onTakeOver={run.takeOver}
      onHandBack={run.controlledBy === "viewer" ? run.handBack : undefined}
      run={run.state}
      onPause={run.pause}
      onResume={run.resume}
      onStop={run.stop}
      pauseNote="It finishes the action it's on, then waits. Nothing else is clicked or typed."
      stopNote="It stops after the action it's on. Pages it opened stay as they are; nothing is undone."
      recording={run.recording}
      recordingSince={run.recordingSince}
      recordingNote="The screen is recorded, with no sound. Recordings are kept for 30 days and your team can watch them."
      onRecord={run.startRecording}
      onStopRecording={run.stopRecording}
    />
  )
}