OpenAI Realtime over Twilio SIP

export TWILIO_API=https://api.twilio.com/2010-04-01/Accounts/$TWILIO_ACCOUNT_SID

curl -X POST "$TWILIO_API/SIP/CredentialLists.json" \
  -u "$TWILIO_ACCOUNT_SID:$TWILIO_AUTH_TOKEN" \
  --data-urlencode "FriendlyName=bluejay-sip-cred"
# copy "sid" (CL...) from the response

curl -X POST "$TWILIO_API/SIP/CredentialLists/$CREDENTIAL_LIST_SID/Credentials.json" \
  -u "$TWILIO_ACCOUNT_SID:$TWILIO_AUTH_TOKEN" \
  --data-urlencode "Username=bluejay" \
  --data-urlencode "Password=YourStrongPassword123"

curl -X POST "$TWILIO_API/SIP/Domains.json" \
  -u "$TWILIO_ACCOUNT_SID:$TWILIO_AUTH_TOKEN" \
  --data-urlencode "DomainName=twilio-openai.sip.twilio.com" \
  --data-urlencode "FriendlyName=Twilio OpenAI" \
  --data-urlencode "VoiceUrl=https://<your-server>/twiml" \
  --data-urlencode "VoiceMethod=POST" \
  --data-urlencode "SipRegistration=false"
# copy "sid" (SD...) from the response

curl -X POST "$TWILIO_API/SIP/Domains/$SIP_DOMAIN_SID/Auth/Calls/CredentialListMappings.json" \
  -u "$TWILIO_ACCOUNT_SID:$TWILIO_AUTH_TOKEN" \
  -d "CredentialListSid=$CREDENTIAL_LIST_SID"

curl -X POST https://api.getbluejay.ai/v1/add-agent \
  -H "X-API-Key: $BLUEJAY_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "OpenAI Realtime via Twilio SIP",
    "system_prompt": "Front desk agent for an appliance repair company.",
    "goals": ["Greet the caller", "Collect the appliance and the day", "Book the repair and confirm the date"],
    "type": "INBOUND",
    "connection_type": "SIP",
    "mode": "VOICE",
    "sip_uri": "sip:[email protected]",
    "sip_username": "bluejay",
    "sip_password": "YourStrongPassword123"
  }'

const crypto = require('crypto');
const express = require('express');
const { WebSocket } = require('ws');
const { trace, context } = require('@opentelemetry/api');
const { BasicTracerProvider, SimpleSpanProcessor } = require('@opentelemetry/sdk-trace-base');
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
const { resourceFromAttributes } = require('@opentelemetry/resources');

const {
  PORT = 8080, OPENAI_API_KEY, OPENAI_PROJECT_ID, OPENAI_WEBHOOK_SECRET,
  CALLER_ID, BLUEJAY_API_KEY, PUBLIC_HOST, MODEL = 'gpt-realtime', VOICE = 'alloy',
} = process.env;

const INSTRUCTIONS =
  "You are Robin, the front desk agent for Bluejay Repair Services, an appliance repair company. " +
  "Open the call with: 'Thanks for calling Bluejay Repair Services, this is Robin, how can I help?' " +
  "Keep every reply to one or two short spoken sentences. To book a repair you must call the book_repair tool " +
  "with the appliance and the requested day; never claim a booking is confirmed until the tool has returned. " +
  "After it returns, read the confirmation number and the date back to the caller and say goodbye.";

const today = () => new Date().toLocaleDateString('en-US', {
  weekday: 'long', year: 'numeric', month: 'long', day: 'numeric', timeZone: 'America/Los_Angeles',
});
const instructions = () => `${INSTRUCTIONS}\n\nToday is ${today()}.`;

const TOOLS = [{
  type: 'function',
  name: 'book_repair',
  description: 'Book an appliance repair visit. Call this once the caller has given the appliance and the day.',
  parameters: {
    type: 'object',
    properties: {
      appliance: { type: 'string' },
      requested_day: { type: 'string' },
      customer_name: { type: 'string' },
    },
    required: ['appliance', 'requested_day'],
  },
}];

function runTool(name, args) {
  if (name === 'book_repair') {
    return { confirmation: 'BJ-' + crypto.randomBytes(2).toString('hex').toUpperCase(), ...args };
  }
  return { error: `unknown tool ${name}` };
}

const tracerProvider = new BasicTracerProvider({
  resource: resourceFromAttributes({ 'service.name': 'openai-twilio-bridge' }),
  spanProcessors: [new SimpleSpanProcessor(new OTLPTraceExporter({
    url: 'https://otlp.getbluejay.ai/v1/traces',
    headers: { 'X-API-KEY': BLUEJAY_API_KEY },
  }))],
});
const tracer = tracerProvider.getTracer('bridge');

async function bluejay(body) {
  const r = await fetch('https://api.getbluejay.ai/v1/update-simulation-result', {
    method: 'POST',
    headers: { 'X-API-Key': BLUEJAY_API_KEY, 'Content-Type': 'application/json' },
    body: JSON.stringify(body),
  });
  console.log('bluejay', body.simulation_result_id, r.status);
}

function runCallSession(callId, simulationResultId) {
  const startedAt = Date.now();
  const offset = () => Date.now() - startedAt;
  const toolCalls = [];
  let turns = 0;

  const rootSpan = tracer.startSpan('voice.call', { attributes: { 'call.id': callId } });
  const rootCtx = trace.setSpan(context.active(), rootSpan);
  const traceId = rootSpan.spanContext().traceId;
  if (simulationResultId) bluejay({ simulation_result_id: simulationResultId, trace_ids: [traceId] });

  const ws = new WebSocket(`wss://api.openai.com/v1/realtime?call_id=${encodeURIComponent(callId)}`, {
    headers: { Authorization: `Bearer ${OPENAI_API_KEY}` },
  });

  ws.on('message', (raw) => {
    let ev;
    try { ev = JSON.parse(raw.toString()); } catch { return; }
    if (ev.type === 'response.function_call_arguments.done') {
      const at = offset();
      let args = {};
      try { args = JSON.parse(ev.arguments || '{}'); } catch {}
      const span = tracer.startSpan(`execute_tool ${ev.name}`, { attributes: { 'tool.arguments': ev.arguments || '' } }, rootCtx);
      const output = runTool(ev.name, args);
      span.setAttribute('tool.output', JSON.stringify(output));
      span.end();
      toolCalls.push({ name: ev.name, parameters: args, output, start_offset_ms: at });
      ws.send(JSON.stringify({ type: 'conversation.item.create', item: { type: 'function_call_output', call_id: ev.call_id, output: JSON.stringify(output) } }));
      ws.send(JSON.stringify({ type: 'response.create' }));
    } else if (ev.type === 'conversation.item.input_audio_transcription.completed') {
      turns++;
      tracer.startSpan('customer.speech', { attributes: { 'speech.text': ev.transcript || '' } }, rootCtx).end();
    } else if (ev.type === 'response.output_audio_transcript.done') {
      turns++;
      tracer.startSpan('agent.speech', { attributes: { 'speech.text': ev.transcript || '' } }, rootCtx).end();
    } else if (ev.type === 'error') {
      console.log('session error', JSON.stringify(ev).slice(0, 400));
    }
  });

  ws.on('close', async () => {
    rootSpan.end();
    console.log('call ended', callId, `${toolCalls.length} tool calls, ${turns} turns`);
    if (!simulationResultId) return;
    await bluejay({
      simulation_result_id: simulationResultId,
      tool_calls: toolCalls,
      metadata: { model: MODEL, call_id: callId, duration_ms: offset(), turns },
    });
    await tracerProvider.forceFlush();
  });
  ws.on('error', (e) => console.log('session ws error', e.message));
}

function verifyWebhook(req) {
  const id = req.get('webhook-id');
  const ts = req.get('webhook-timestamp');
  const sigs = (req.get('webhook-signature') || '').split(' ');
  if (!id || !ts || !sigs.length) return false;
  if (Math.abs(Date.now() / 1000 - Number(ts)) > 300) return false;
  const key = Buffer.from(OPENAI_WEBHOOK_SECRET.replace(/^whsec_/, ''), 'base64');
  const expected = crypto.createHmac('sha256', key).update(`${id}.${ts}.`).update(req.body).digest();
  return sigs.some((s) => {
    const [version, b64] = s.split(',');
    if (version !== 'v1' || !b64) return false;
    const got = Buffer.from(b64, 'base64');
    return got.length === expected.length && crypto.timingSafeEqual(got, expected);
  });
}

function sipHeader(data, name) {
  const hit = ((data && data.sip_headers) || []).find((h) => h.name.toLowerCase() === name.toLowerCase());
  return hit ? hit.value : undefined;
}

const app = express();
app.get('/', (_req, res) => res.json({ ok: true }));

app.all('/twiml', express.urlencoded({ extended: false }), (req, res) => {
  const b = req.body || {};
  const simId = b['SipHeader_X-Simulation-Result-Id'] || '';
  const uri = `sip:${OPENAI_PROJECT_ID}@sip.api.openai.com;transport=tls` +
    (simId ? `?X-Simulation-Result-Id=${encodeURIComponent(simId)}` : '');
  console.log('twiml', b.CallSid, 'simulation_result_id', simId || '(none)');
  res.type('text/xml').send(
    `<?xml version="1.0" encoding="UTF-8"?><Response>` +
    `<Dial answerOnBridge="true" callerId="${CALLER_ID}" action="https://${PUBLIC_HOST}/dial-status" method="POST">` +
    `<Sip>${uri}</Sip></Dial></Response>`
  );
});

app.post('/dial-status', express.urlencoded({ extended: false }), (req, res) => {
  const b = req.body || {};
  console.log('dial-status', b.CallSid, b.DialCallStatus, 'sip', b.DialSipResponseCode);
  res.type('text/xml').send('<Response><Hangup/></Response>');
});

app.post('/openai-webhook', express.raw({ type: '*/*', limit: '1mb' }), async (req, res) => {
  if (!verifyWebhook(req)) return res.status(400).send('bad signature');
  let event;
  try { event = JSON.parse(req.body.toString('utf8')); } catch { return res.status(400).send('bad json'); }
  res.status(200).end();
  if (event.type !== 'realtime.call.incoming') return;

  const callId = event.data.call_id;
  const simulationResultId = sipHeader(event.data, 'X-Simulation-Result-Id');
  const r = await fetch(`https://api.openai.com/v1/realtime/calls/${callId}/accept`, {
    method: 'POST',
    headers: { Authorization: `Bearer ${OPENAI_API_KEY}`, 'Content-Type': 'application/json' },
    body: JSON.stringify({
      type: 'realtime',
      model: MODEL,
      instructions: instructions(),
      tools: TOOLS,
      tool_choice: 'auto',
      audio: { input: { transcription: { model: 'gpt-4o-mini-transcribe' } }, output: { voice: VOICE } },
    }),
  });
  console.log('accept', callId, r.status, 'simulation_result_id', simulationResultId || '(none)');
  if (r.status === 200) runCallSession(callId, simulationResultId);
});

app.listen(PORT, () => console.log('listening on', PORT));