Realtime session quickstart
A working browser client for a live emotion session, end to end. It streams camera and microphone to Imentiv over WebRTC and reads facial emotion, voice emotion, transcription and text emotion back over a WebSocket.
This is deliberately not a minimal example. The parts that make it longer than a
hello-world are the parts that make it survive contact with reality: the ordering rules
around session teardown, the frames that arrive without a session_id, the text-emotion
fields that are zero-filled while the session runs, and the one path where the server
ends your session for you.
How the pieces fit
A session is created over REST, negotiated over REST, streamed over WebRTC, and read over
a WebSocket. Four transports, one session_id tying them together.
1. POST /v2/realtime/session -> session_id + ice_servers
2. Open wss://.../v2/ws/realtime?token=...&session_id=...
3. Build RTCPeerConnection with ice_servers, attach camera/mic tracks
4. POST /v2/realtime/webrtc/offer -> SDP answer
5. POST /v2/realtime/webrtc/ice -> trickle your candidates
6. Read emotion / transcript / text_emotion frames as they stream
7. POST /v2/realtime/session/{session_id}/end -> then close the WebSocket
Four things worth getting right before you read the code:
The session_id comes first and everything else needs it. There is no separate call
for TURN credentials — the create response carries both the id and the ice_servers to
configure WebRTC with.
Open the WebSocket early. It is independent of the WebRTC handshake, but nothing is replayed, so whatever is emitted before you connect is lost.
Close the WebSocket last. The final utterance is finalised during /end, so the last
result frame arrives while that call is still in flight. Close the socket after /end
returns, not before.
Media never flows over the WebSocket. It carries results only, server to client. Your audio and video go over WebRTC, and nothing you send up the socket is interpreted.
The client
class ImentivRealtimeClient {
constructor(apiKey) {
this.apiKey = apiKey;
this.rest = 'https://devapi.imentiv.ai/v2';
this.ws = 'wss://devapi.imentiv.ai/v2/ws';
this.sessionId = null;
this.pc = null;
this.socket = null;
this.stream = null;
this.ended = false; // set when the server ends the session for us
this.transcripts = new Map(); // transcript_id -> latest text
this.segments = new Map(); // start_time_ms -> merged audio + text emotion
}
// ---------- lifecycle ----------
async start({ audioOnly = false, onUpdate = () => {} } = {}) {
this.onUpdate = onUpdate;
// 1. Create the session. Carries the id AND the ICE servers.
const session = await this._post('/realtime/session', { audio_only: audioOnly });
if (session.status !== 'active') {
throw new Error(`Session not ready to negotiate: ${session.status}`);
}
this.sessionId = session.session_id;
// 2. Open the results socket FIRST, so nothing is missed. Results are
// never replayed, so a late connect silently loses early frames.
this._openSocket();
// 3. Capture media.
this.stream = await navigator.mediaDevices.getUserMedia({
video: audioOnly ? false : { width: 640, height: 480, frameRate: 30 },
audio: { echoCancellation: true, noiseSuppression: true }
});
// 4. Negotiate, using the ICE servers from the session response verbatim.
this.pc = new RTCPeerConnection({ iceServers: session.ice_servers });
this.stream.getTracks().forEach(t => this.pc.addTrack(t, this.stream));
this.pc.onicecandidate = ({ candidate }) => {
if (!candidate) return; // null candidate = gathering complete
this._post('/realtime/webrtc/ice', {
session_id: this.sessionId,
candidate: candidate.candidate,
sdp_mid: candidate.sdpMid, // note the case change
sdp_mline_index: candidate.sdpMLineIndex
}).catch(err => console.warn('ICE post failed:', err));
};
this.pc.onconnectionstatechange = () => {
const s = this.pc.connectionState;
if (s === 'failed' || s === 'disconnected') {
// Not recoverable — there is no resume. Tear down and restart.
this.onUpdate({ kind: 'media_lost', state: s });
}
};
const offer = await this.pc.createOffer();
await this.pc.setLocalDescription(offer);
// The server's own candidates come back inside this answer.
const answer = await this._post('/realtime/webrtc/offer', {
session_id: this.sessionId, sdp: offer.sdp, type: offer.type
});
await this.pc.setRemoteDescription(new RTCSessionDescription(answer));
return { sessionId: this.sessionId, stream: this.stream };
}
async stop() {
// Stop capture first so the camera light goes out immediately. The
// peer connection and the socket stay up: the last utterance is
// finalized during /end, and its frame arrives on the socket.
if (this.stream) this.stream.getTracks().forEach(t => t.stop());
if (this.socket) this.socket.onclose = null; // teardown is expected, not a drop
let result;
// Skip /end if the server already ended it — that call would 404.
if (this.sessionId && !this.ended) {
try {
result = await this._post(`/realtime/session/${this.sessionId}/end`, null);
} catch (err) {
console.warn('End failed:', err);
}
}
// Only now: the final frame has been delivered.
if (this.socket) this.socket.close();
if (this.pc) this.pc.close();
return result;
}
// ---------- results ----------
_openSocket() {
const url = `${this.ws}/realtime?token=${encodeURIComponent(this.apiKey)}`
+ `&session_id=${encodeURIComponent(this.sessionId)}`;
this.socket = new WebSocket(url);
this.socket.onmessage = ({ data }) => {
let frame;
try { frame = JSON.parse(data); } catch { return; }
switch (frame.type) {
case 'emotion':
if (frame.data.source === 'video') this._onVideo(frame);
else if (frame.data.source === 'audio') this._onAudio(frame);
break;
case 'transcript': this._onTranscript(frame); break;
case 'text_emotion': this._onTextEmotion(frame); break;
case 'error': this._onError(frame); break;
// Unknown types are ignored on purpose — more may be added.
}
};
this.socket.onclose = () => {
// No reconnect exists. Surface it and let the caller restart.
if (!this.ended) this.onUpdate({ kind: 'disconnected' });
};
}
_onVideo({ frame_number, data }) {
this.onUpdate({
kind: 'face',
frameNumber: frame_number,
dominant: data.dominant,
scores: data.scores,
// Nullable — absent means "not computed", not zero.
valence: data.valence,
arousal: data.arousal,
expressiveness: data.expressiveness,
certainty: data.emotion_entropy === null ? null : 1 - data.emotion_entropy
});
}
_onAudio({ data }) {
// Live frames zero-fill the text-emotion fields and a separate
// text_emotion frame follows. The FINAL frame (emitted during /end)
// carries real values and gets no follow-up — so take whatever is on
// the frame, and let a later text_emotion frame overwrite it.
const hasText = data.dominant_text_emotion !== null;
const segment = {
startMs: data.start_time_ms,
endMs: data.end_time_ms,
transcript: data.transcript,
voiceDominant: data.dominant,
voiceScores: data.scores,
valence: data.valence,
arousal: data.arousal,
textDominant: hasText ? data.dominant_text_emotion : null,
textScores: hasText ? data.text_emotions : null
};
this.segments.set(data.start_time_ms, segment);
this.onUpdate({ kind: 'voice', segment });
}
_onTranscript({ data }) {
// REPLACE by transcript_id — partials resend the whole text so far.
// Appending would duplicate every word.
this.transcripts.set(data.transcript_id, data.transcript);
this.onUpdate({
kind: 'transcript',
id: data.transcript_id,
text: data.transcript,
isFinal: data.is_final, // only the final one is punctuated
language: data.language
});
}
_onTextEmotion({ data }) {
const segment = this.segments.get(data.start_time_ms);
if (segment) {
segment.textDominant = data.dominant_text_emotion;
segment.textScores = data.text_emotions;
}
// Iterate the keys rather than assuming a fixed label set.
this.onUpdate({
kind: 'text',
startMs: data.start_time_ms,
dominant: data.dominant_text_emotion,
scores: data.text_emotions,
segment
});
}
_onError(frame) {
if (frame.code === 'insufficient_credits') {
// Session is ALREADY ended server-side. Do not call /end.
this.ended = true;
this.onUpdate({ kind: 'out_of_credits', message: frame.error });
return;
}
this.onUpdate({ kind: 'error', code: frame.code, message: frame.error });
}
// ---------- transport ----------
async _post(path, body) {
const res = await fetch(`${this.rest}${path}`, {
method: 'POST',
headers: {
'X-API-Key': this.apiKey,
...(body === null ? {} : { 'Content-Type': 'application/json' })
},
body: body === null ? undefined : JSON.stringify(body)
});
if (!res.ok) {
const detail = await res.json().catch(() => ({}));
// `detail` is a string most of the time, an object when the
// condition carries a stable code. Handle both.
const code = typeof detail.detail === 'object' ? detail.detail?.code : undefined;
const message = typeof detail.detail === 'object'
? detail.detail?.message
: detail.detail ?? res.statusText;
const err = new Error(message);
err.status = res.status;
err.code = code;
throw err;
}
return res.json();
}
}
Using it
const client = new ImentivRealtimeClient('your_api_key');
try {
const { stream } = await client.start({
onUpdate: (update) => {
switch (update.kind) {
case 'face':
console.log('face:', update.dominant, 'valence', update.valence);
break;
case 'voice':
console.log('voice:', update.segment.voiceDominant,
'|', update.segment.transcript);
break;
case 'transcript':
console.log(update.isFinal ? 'final:' : 'partial:', update.text);
break;
case 'text':
console.log('text emotion:', update.dominant);
break;
case 'out_of_credits':
alert('Out of credits — the session has ended.');
break;
case 'media_lost':
case 'disconnected':
console.warn('Session lost; start a new one to continue.');
break;
}
}
});
document.getElementById('preview').srcObject = stream;
} catch (err) {
if (err.code === 'at_capacity') console.warn('Service is busy — retry shortly.');
else if (err.status === 402) console.warn('Insufficient credit to start.');
else console.error('Could not start:', err);
}
// Always end the session — including on unload, error, and cancel.
// Note: an unloading page will not wait for stop()'s awaits to finish. For
// reliable teardown on close, fire the /end call with `keepalive: true`:
// fetch(url, { method: 'POST', headers: { 'X-API-Key': key }, keepalive: true })
// (navigator.sendBeacon cannot set X-API-Key, so it will not authenticate.)
// Otherwise the session lingers until the server's duration ceiling.
window.addEventListener('pagehide', () => { client.stop(); });
What to read next
- Understanding emotion scores — what
scores,valenceand the rest actually mean, and why the video and audio label sets differ. - Errors and retries —
at_capacity, running out of credit mid-session, and why there is no reconnect. - Limits and media — the session ceiling, TURN credential lifetime, and the codecs and resolutions the pipeline is tuned for.
- Realtime API reference — every endpoint and every frame, field by field.