Getting the web to speak...

Using Polly and three.js to create a realistic avatar

Paul Hodgson

www.ideasestuary.com

github.com/prh1

@hodgsonpr1

First 3D movie ?

Planning

3D library

three.js

www.threejs.org

HTML three.js
body scene
element object
style material
- light
- camera
- renderer 
					
<html>
  <head>
    <title>A very simple three.js scene</title>
    <script src="lib/three.js"></script>
  </head>
  <body>
    <div id="scene"></div>
    <script src="js/simplescene.js"></script>
  </body>
</html>					
					
				
					
							// Simple three.js scene
							var scene = new THREE.Scene();
							
							var container = document.getElementById("scene");
							var renderer = new THREE.WebGLRenderer({ antialias: true });
							renderer.setSize(window.innerWidth, window.innerHeight);
							container.appendChild(renderer.domElement);
						
					
						
							// Now we add the objects
							var pinkMaterial = new THREE.MeshPhongMaterial( { color: 0x506289, 
								emissive: 0x572534 } );
							var cube = new THREE.Mesh(new THREE.BoxGeometry( 100, 100, 100 ), 
								pinkMaterial );
							cube.rotation.x = 2;
							cube.rotation.z = 2;
							scene.add(cube);
							
							var blueMaterial = new THREE.MeshPhongMaterial( { color: 0x056289, 
								emissive: 0x072534 } );
							var sphere = new THREE.Mesh(new THREE.SphereGeometry( 80, 32, 32 ), 
								blueMaterial );
							sphere.position.x = 160;
							scene.add(sphere);
							
							var yellowMaterial = new THREE.MeshPhongMaterial( { color: 0x888420, 
								emissive: 0x888820 } );
							var torus = new THREE.Mesh(new THREE.TorusGeometry( 100, 20, 64, 64 ), 
								yellowMaterial );
							torus.position.x = -200;
							torus.rotation.x = Math.PI / 4;
							scene.add(torus);
						
					
						
							// Lights, camera, action
							var light = new THREE.PointLight( 0xffffff ); 
							light.position.y = 100;
							light.position.x = 100;
							light.position.z = 50;
							scene.add(light);
							
							var camera = new THREE.PerspectiveCamera( 70, 
								window.innerWidth / window.innerHeight, 1, 1000 );
							camera.position.z = 400;

							renderer.render(scene, camera);

Text to speech

Polly

aws.amazon.com/polly

audio 1

audio 2

audio 3 

					
						const AWS = require('aws-sdk');
						const Polly = new AWS.Polly({
							signatureVersion: 'v4',
							region: 'us-east-1',
							accessKeyId: 'xxxxxxxxxxxxxxxxxxxxx',
							secretAccessKey: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
						});
					
				
					
var text = 'This is a test';
var voice = 'Amy';

var params = {
  'Text': text,
  'OutputFormat': 'mp3',
  'VoiceId': voice
};

Polly.synthesizeSpeech(params, (err, data) => {
    if (err) {
      console.log(err.code);
    } else if (data) {
    if (data.AudioStream instanceof Buffer) {
      fs.writeFileSync(fileName + ".mp3", data.AudioStream);
    }
  }
});

Glue 

					
function getPhonemeFromViseme(v) {
  var ph;
  switch (v) {
    case "sil": ph = ""; break;
    case "k": ph = "k"; break;
    case "@": ph = "bigaah"; break;
    case "s": ph = "d.s.t"; break;
    case "t": ph = "d.s.t"; break;
    case "o": ph = "oh"; break;
    case "T": ph = "th"; break;
    case "S": ph = "d.s.t"; break;
    case "a": ph = "aah"; break;
    case "i": ph = "i"; break;
    case "u": ph = "w"; break;
    case "O": ph = "ooh.q"; break;
    case "r": ph = "r"; break;
    case "E": ph = "ee"; break;
    case "f": ph = "f.v"; break;
    case "p": ph = "b.m.p"; break;
    defaut: ph = ""; break;
  }
  return ph;
}

Demo

Questions ?

www.ideasestuary.com

github.com/prh1

@hodgsonpr1

Have fun !

www.ideasestuary.com

github.com/prh1

@hodgsonpr1