<title>VRM Speech Lip-Sync Demo</title>

<meta name="description" content="Demonstrates audio-driven lip-sync using Rhubarb">

<script src="js/aframe-v1.7.1.js"></script>

<script src="js/aframe-environment-component.js"></script>

<script src="js/three-vrm.js"></script>

<script src="js/three-vrm-animation.js"></script>

<script src="js/aframe-vrm-bundle.js"></script>

#info {

position: fixed;

top: 20px;

left: 20px;

color: white;

font-family: monospace;

background: rgba(0,0,0,0.8);

padding: 15px;

border-radius: 5px;

z-index: 100;

font-size: 1.2em;

}

.highlight {

color: #00ff00;

font-weight: bold;

}

#controls {

position: fixed;

bottom: 20px;

left: 20px;

background: rgba(0,0,0,0.8);

padding: 15px;

border-radius: 5px;

z-index: 100;

}

button {

background: #00ff00;

color: black;

border: none;

padding: 10px 20px;

font-size: 1em;

border-radius: 3px;

cursor: pointer;

font-family: monospace;

margin: 5px;

}

button:hover {

background: #00cc00;

}

<div id="info">

<div>Speech Lip-Sync Demo</div>

<div id="status">Loading model...</div>

<div id="current-viseme" class="highlight"></div>

<div id="current-emotion" class="highlight"></div>

<div id="audio-time"></div>

<div id="controls">

<button id="playBtn">Play Audio</button>

<button id="stopBtn">Stop</button>

<div id="start-overlay">

<div class="dialog-content">

<h1>VRM Speech Demo</h1>

<p>Audio playback requires user interaction.</p>

<button id="start-demo-btn">Click to Start</button>

</div>

#start-overlay {

position: fixed;

top: 0;

left: 0;

width: 100%;

height: 100%;

background: rgba(0, 0, 0, 0.9);

display: flex;

justify-content: center;

align-items: center;

z-index: 9999;

}

.dialog-content {

text-align: center;

color: white;

font-family: monospace;

background: #222;

padding: 40px;

border-radius: 10px;

border: 2px solid #00ff00;

}

#start-demo-btn {

font-size: 1.5em;

margin-top: 20px;

padding: 15px 30px;

}

// Handle AudioContext unlocking

document.addEventListener('DOMContentLoaded', () => {

const startBtn = document.getElementById('start-demo-btn');

const overlay = document.getElementById('start-overlay');

startBtn.addEventListener('click', () => {

// Unlock A-Frame/Three.js AudioContext if present

const scene = document.querySelector('a-scene');

if (scene && scene.audioListener && scene.audioListener.context) {

scene.audioListener.context.resume();

}

// Bless the speech audio element

const avatar = document.getElementById('avatar');

if (avatar && avatar.components && avatar.components['speech-lipsync']) {

const speechComp = avatar.components['speech-lipsync'];

if (speechComp.audio) {

// Play momentarily to unlock (volume 0)

const originalVolume = speechComp.audio.volume;

speechComp.audio.volume = 0;

speechComp.audio.play().then(() => {

speechComp.audio.pause();

speechComp.audio.currentTime = 0;

speechComp.audio.volume = originalVolume;

console.log("Audio unlocked successfully");

}).catch(e => {

console.warn("Could not unlock audio:", e);

});

}

}

// Hide overlay

overlay.style.display = 'none';

});

});

// Emotion presets — blend shape recipes using confirmed available shapes

const EMOTIONS = {

neutral: {},

happy: { mouthSmileLeft: 0.7, mouthSmileRight: 0.7, cheekPuff: 0.15 },

sad: { mouthFrownLeft: 0.6, mouthFrownRight: 0.6, browInnerUp: 0.5 },

angry: { browDownLeft: 0.8, browDownRight: 0.8,

noseSneerLeft: 0.3, noseSneerRight: 0.3,

mouthPressLeft: 0.3, mouthPressRight: 0.3 },

surprised: { eyeWideLeft: 0.8, eyeWideRight: 0.8,

browOuterUpLeft: 0.6, browOuterUpRight: 0.6,

browInnerUp: 0.4, jawOpen: 0.25 },

concerned: { browInnerUp: 0.6, browDownLeft: 0.25, browDownRight: 0.25 },

stressed: { browInnerUp: 0.4, browDownLeft: 0.5, browDownRight: 0.5,

noseSneerLeft: 0.15, noseSneerRight: 0.15 },

fear: { eyeWideLeft: 0.6, eyeWideRight: 0.6, browInnerUp: 0.7,

browOuterUpLeft: 0.4, browOuterUpRight: 0.4 },

disgust: { noseSneerLeft: 0.7, noseSneerRight: 0.7,

mouthUpperUpLeft: 0.4, mouthUpperUpRight: 0.4,

browDownLeft: 0.3, browDownRight: 0.3 }

};

// All shapes touched by any emotion — used for reset

const ALL_EMOTION_SHAPES = [...new Set(Object.values(EMOTIONS).flatMap(e => Object.keys(e)))];

// Rhubarb mouth cue to our viseme mapping

const RHUBARB_TO_VISEME = {

'X': null, // Rest position (neutral)

'A': 'A', // Open mouth (father)

'B': 'M', // Lips together (M, B, P)

'C': 'O', // Rounded mouth (O, U)

'D': 'E', // Wide mouth (E, I) - using E as primary

'E': 'E', // Relaxed mouth (schwa)

'F': 'F', // Upper teeth on lower lip (F, V)

'G': 'K', // Back of tongue (K, G)

'H': 'S' // Tongue tip up (L, R, S)

};

AFRAME.registerComponent('speech-lipsync', {

schema: {

vrm: { type: 'string', default: 'models/AIAN/AIAN_F_1_Casual_CLEANED.vrm' },

audio: { type: 'string', default: 'audio/houston.wav' },

timings: { type: 'string', default: 'audio/houston.json' },

emotion: { type: 'string', default: 'audio/houston_emotion.json' }

},

init: function () {

this.vrm = null;

this.mesh = null;

this.audio = null;

this.mouthCues = [];

this.emotionCues = [];

this.currentCueIndex = 0;

this.isPlaying = false;

this.nextBlink = 2000 + Math.random() * 2000; // first blink 2–4s in

this.blinkStart = -9999;

// Bindings

this.onModelLoaded = this.onModelLoaded.bind(this);

this.el.addEventListener('model-loaded', this.onModelLoaded);

// Set VRM src

this.el.setAttribute('vrm', 'src', this.data.vrm);

// Setup audio

this.setupAudio();

// Load timing data

this.loadTimingData();

// Load emotion data

this.loadEmotionData();

// Setup UI controls

this.setupControls();

},

onModelLoaded: function (evt) {

const vrm = evt.detail.vrm;

if (!vrm) return;

this.vrm = vrm;

console.log("VRM loaded");

document.getElementById('status').textContent = "Model Loaded. Press Play to start.";

// Find the face mesh

this.mesh = this.findFaceMesh(this.vrm.scene);

if (this.mesh) {

console.log("Face mesh found:", this.mesh.name);

} else {

console.warn("Face mesh not found");

}

},

findFaceMesh: function(root) {

let found = null;

root.traverse((node) => {

if (node.isMesh && node.name === 'H_DDS_HighRes') {

found = node;

}

});

return found;

},

loadEmotionData: function() {

if (!this.data.emotion) return;

fetch(this.data.emotion)

.then(r => r.json())

.then(data => {

this.emotionCues = data.emotions;

console.log(`Loaded ${this.emotionCues.length} emotion keyframes`);

})

.catch(() => console.warn('No emotion data loaded'));

},

applyShape: function(name, value) {

if (this.mesh && this.mesh.morphTargetDictionary && this.mesh.morphTargetInfluences) {

if (name in this.mesh.morphTargetDictionary) {

this.mesh.morphTargetInfluences[this.mesh.morphTargetDictionary[name]] = value;

}

}

},

tickEmotion: function(currentTime) {

if (!this.emotionCues || this.emotionCues.length === 0) return;

// Find the keyframe pair surrounding currentTime

let prevIdx = 0;

for (let i = 0; i < this.emotionCues.length - 1; i++) {

if (this.emotionCues[i].t <= currentTime) prevIdx = i;

}

const prev = this.emotionCues[prevIdx];

const next = this.emotionCues[Math.min(prevIdx + 1, this.emotionCues.length - 1)];

// Smoothstep alpha between keyframes

let alpha = 0;

if (next !== prev && next.t > prev.t) {

alpha = (currentTime - prev.t) / (next.t - prev.t);

alpha = Math.max(0, Math.min(1, alpha));

alpha = alpha * alpha * (3 - 2 * alpha); // smoothstep

}

const prevShapes = EMOTIONS[prev.type] || {};

const nextShapes = EMOTIONS[next.type] || {};

const prevScale = prev.intensity ?? 1;

const nextScale = next.intensity ?? 1;

// Zero all possible emotion shapes, then apply interpolated values

for (const shape of ALL_EMOTION_SHAPES) {

const from = (prevShapes[shape] || 0) * prevScale;

const to = (nextShapes[shape] || 0) * nextScale;

this.applyShape(shape, from + (to - from) * alpha);

}

// Update UI

const label = alpha < 0.5 ? prev.type : next.type;

document.getElementById('current-emotion').textContent = `emotion: ${label}`;

},

tickBlink: function(t) {

if (!this.mesh) return;

if (t > this.nextBlink) {

this.blinkStart = t;

this.nextBlink = t + 3000 + Math.random() * 4000; // next blink in 3–7s

}

const elapsed = t - this.blinkStart;

if (elapsed < 150) {

// close over first 75ms, open over next 75ms

const v = elapsed < 75 ? elapsed / 75 : 1 - (elapsed - 75) / 75;

this.applyShape('eyeBlinkLeft', v);

this.applyShape('eyeBlinkRight', v);

}

},

setupAudio: function() {

this.audio = new Audio(this.data.audio);

this.audio.addEventListener('ended', () => {

this.isPlaying = false;

this.resetVisemes();

document.getElementById('status').textContent = "Audio finished. Press Play again.";

});

},

loadTimingData: function() {

fetch(this.data.timings)

.then(response => response.json())

.then(data => {

this.mouthCues = data.mouthCues;

console.log(`Loaded ${this.mouthCues.length} mouth cues`);

console.log('Duration:', data.metadata.duration, 'seconds');

})

.catch(error => {

console.error('Error loading timing data:', error);

});

},

setupControls: function() {

document.getElementById('playBtn').addEventListener('click', () => {

if (!this.audio || !this.mouthCues.length) {

alert('Audio or timing data not loaded yet!');

return;

}

this.play();

});

document.getElementById('stopBtn').addEventListener('click', () => {

this.stop();

});

},

play: function() {

this.audio.currentTime = 0;

this.audio.play();

this.isPlaying = true;

this.currentCueIndex = 0;

document.getElementById('status').textContent = "Playing: Houston, we have a problem.";

},

stop: function() {

this.audio.pause();

this.audio.currentTime = 0;

this.isPlaying = false;

this.currentCueIndex = 0;

this.resetVisemes();

document.getElementById('status').textContent = "Stopped.";

},

resetVisemes: function() {

if (!this.mesh) return;

['A', 'I', 'U', 'E', 'O', 'F', 'M', 'S', 'CH', 'K', 'N'].forEach(viseme => {

this.applyViseme(viseme, 0);

});

for (const shape of ALL_EMOTION_SHAPES) {

this.applyShape(shape, 0);

}

document.getElementById('current-viseme').textContent = '';

document.getElementById('current-emotion').textContent = '';

},

tick: function(t, dt) {

this.tickBlink(t);

if (!this.isPlaying || !this.audio || !this.mouthCues.length) return;

const currentTime = this.audio.currentTime;

// Update time display

document.getElementById('audio-time').textContent =

`Time: ${currentTime.toFixed(2)}s`;

// Drive emotion layer

this.tickEmotion(currentTime);

// Find current mouth cue

while (this.currentCueIndex < this.mouthCues.length - 1 &&

currentTime >= this.mouthCues[this.currentCueIndex + 1].start) {

this.currentCueIndex++;

}

const currentCue = this.mouthCues[this.currentCueIndex];

if (currentCue && currentTime >= currentCue.start && currentTime < currentCue.end) {

// Calculate intensity based on position in cue

const cueProgress = (currentTime - currentCue.start) / (currentCue.end - currentCue.start);

// Use sine wave for smooth transition

// Peaks at middle of cue, fades in/out

const intensity = Math.sin(cueProgress * Math.PI);

// Get our viseme from Rhubarb cue

const rhubarbCue = currentCue.value;

const ourViseme = RHUBARB_TO_VISEME[rhubarbCue];

// Reset all visemes first

['A', 'I', 'U', 'E', 'O', 'F', 'M', 'S', 'CH', 'K', 'N'].forEach(v => {

this.applyViseme(v, 0);

});

// Apply current viseme

if (ourViseme) {

this.applyViseme(ourViseme, intensity);

// Update UI

document.getElementById('current-viseme').textContent =

`${rhubarbCue} → ${ourViseme} : ${intensity.toFixed(2)}`;

} else {

// Rest position

document.getElementById('current-viseme').textContent =

`${rhubarbCue} → rest`;

}

}

},

applyViseme: function(name, value) {

// Direct mesh access for all visemes

if (this.mesh && this.mesh.morphTargetDictionary && this.mesh.morphTargetInfluences) {

if (name in this.mesh.morphTargetDictionary) {

const idx = this.mesh.morphTargetDictionary[name];

this.mesh.morphTargetInfluences[idx] = value;

}

}

}

});

<a-sky color="#1a1a1a"></a-sky>

<!-- Lighting -->

<a-entity light="type: ambient; intensity: 0.6;"></a-entity>

<a-entity light="type: directional; intensity: 1.0; castShadow: true;" position="1 2 3"></a-entity>

<a-entity light="type: point; intensity: 0.5; color: #ffd;" position="-1 1.5 1"></a-entity>

<!-- Camera: Close up on face -->

<a-entity position="0 0 0.6">

<a-camera fov="50" wasd-controls-enabled="true" look-controls-enabled="true"></a-camera>

<!-- Headlight for face visibility -->

<a-entity light="type: point; intensity: 0.8; distance: 3; decay: 2" position="0 1.6 0"></a-entity>

</a-entity>

<!-- Avatar with Speech Lip-Sync -->

<a-entity

id="avatar"

speech-lipsync="

"

position="0 0 0"

rotation="0 180 0"

shadow="cast: true">

</a-entity>