3D Interactive Cube on Screen

Idea: A cube on the screen which looks like a 3D cube.

I mean, not just a regular 3d model rotating constantly. Instead, the cube shows different angles depend on from which side you are looking at it. If you look closer, it even becomes bigger.

Therefore, I need:

  1. To detect the distance between user and screen.
    1. IR Ranger.
    2. Serial communication.
  2. To detect the position of which the user is looking.
    1. Face Tracker.
  3. Render a 3D cube in the canvas.
    1. Threejs.

1. IR Ranger

IR Ranger is simple, stick it on the computer screen, then read the number and transmit it to p5 by serial port.

#include <Arduino.h>
#include <SharpDistSensor.h>

const byte mediumFilterWindowSize = 5;

SharpDistSensor sensor(A5, mediumFilterWindowSize);

void setup() {
 Serial.begin(9600); // initialize serial communications
}

void loop() {
  int distance = sensor.getDist();
  int mappedDist = map(distance, 150, 700, 0, 100);            // read the input pin
  Serial.write(mappedDist);                             // print it out the serial port
  delay(10);                                            // slight delay to stabilize the ADC
}

2. Face Tracker

Luckily, I found a library and example by which I can track faces by p5, which is super convenient!

So from Arduino, we got depth, from p5 we had the position. Now we can get a three dimension coordinate, which can be used as the real position of our user.

//make sure HTTPS is checked
var ctracker;
var videoInput;
var positions;
var faceX = 0;
var faceY = 0;
var frameCount1;
var distances = [];
var smoothedDist = 0;
var totalDist;

var serial; // variable to hold an instance of the serialport library
var portName = '/dev/cu.usbmodem1411'; // fill in your serial port name here
var inData;
var distance = 0;


function setup() {

  serial = new p5.SerialPort(); // make a new instance of the serialport library
  serial.on('list', printList); // set a callback function for the serialport list event
  serial.on('connected', serverConnected); // callback for connecting to the server
  serial.on('open', portOpen); // callback for the port opening
  serial.on('data', serialEvent); // callback for when new data arrives
  serial.on('error', serialError); // callback for errors
  serial.on('close', portClose); // callback for the port closing
  serial.list(); // list the serial ports
  serial.open(portName); // open a serial port

  videoInput = createCapture(VIDEO);
  videoInput.size(600, 340);
  videoInput.position(0, 0);
  videoInput.hide();

  var cnv = createCanvas(windowWidth, windowHeight);
  cnv.position(0, 0);

  ctracker = new clm.tracker();
  ctracker.init(pModel);
  ctracker.start(videoInput.elt);
  noStroke();
}

function draw() {
  distance = inData;
  if (frameCount % 3 < 1) {
    distances.push(distance);
  }
  if (distances.length >= 5){
    distances.shift();
  }
  for (var i = 0; i < distances.length; i++) {
    totalDist += distances[i];
  }
  smoothedDist = totalDist/distances.length;
  totalDist = 0;

  frameCount1 = frameCount;
  // background(255);
  translate(windowWidth, 0);
  scale(-1, 1);
  // image(videoInput,0,0,320,240);
  // get array of face marker positions [x, y] format
  var positions = ctracker.getCurrentPosition();


  fill(255, 0, 0);
  if (positions.length > 0) {
    faceX = positions[33][0];
    faceX = map(faceX, 0, 600, 0, windowWidth);
    faceY = positions[33][1];
    faceY = map(faceY, 0, 340, 0, windowHeight);
    // ellipse(faceX, faceY, 5, 5);
  }
}

function serverConnected() {
  print('connected to server.');
}

function portOpen() {
  print('the serial port opened.')
}

function serialEvent() {
  inData = Number(serial.read());
}

function serialError(err) {
  print('Something went wrong with the serial port. ' + err);
}

function portClose() {
  print('The serial port closed.');
}

// get the list of ports:
function printList(portList) {
  // portList is an array of serial port names
  for (var i = 0; i < portList.length; i++) {
    // Display the list the console:
    print(i + " " + portList[i]);
  }
}

 

3. Make a 3d cube:

In the render, we change the position of the camera based on our user’s position.

var camera, scene, renderer;
var cameraX = 0,
  cameraY = 0,
  cameraZ = 0;
var mesh;
init();
animate();

function init() {
  camera1 = new THREE.PerspectiveCamera(40, window.innerWidth / window.innerHeight, 1, 1000);
  camera1.position.z = 600;
  scene = new THREE.Scene();
  var texture = new THREE.TextureLoader().load('textures/grass.png');
  var geometry = new THREE.BoxBufferGeometry(200, 200, 200);
  var material = new THREE.MeshBasicMaterial({
    map: texture
  });

  mesh = new THREE.Mesh(geometry, material);
  scene.add(mesh);
  renderer = new THREE.WebGLRenderer();
  renderer.setPixelRatio(window.devicePixelRatio);
  renderer.setSize(window.innerWidth, window.innerHeight);
  document.body.appendChild(renderer.domElement);
  //
  window.addEventListener('resize', onWindowResize, false);
}

function onWindowResize() {
  camera1.aspect = window.innerWidth / window.innerHeight;
  camera1.updateProjectionMatrix();
  renderer.setSize(window.innerWidth, window.innerHeight);
}


function animate() {
  requestAnimationFrame(animate);
  render();
  if (frameCount1 % 60 < 1) {
    debug();
  }
}

function render() {
  cameraX = (faceX - window.innerWidth / 2) / 2;
  cameraY = (faceY - window.innerHeight / 2);
  cameraZ = 400 + smoothedDist * 3;
  camera1.position.x += (-cameraX - camera1.position.x)*0.8;
  camera1.position.y += (-cameraY - camera1.position.y)*0.8;
  camera1.position.z = cameraZ;
  camera1.lookAt(scene.position);
  renderer.render(scene, camera1);
}

function debug() {
  console.log("camera");
  console.log(cameraX);
  console.log(cameraY);
  console.log(camera1.position.z);
}

Finally:

Troubles:

  1. When I was trying to get the p5 reflect the reading of an IR sensor. There are two libraries from SHARP in Arduino. Initially, I picked the wrong one.
  2. I still don’t know how to create lights in Threejs.

Libraries:

This is the project I have done with a unprecedent quantity of libraries.

<script src="./libraries/p5.js"></script>
<script src="./libraries/p5.dom.min.js"></script>
<script src="./libraries/clmtrackr.js"></script>
<script src="./libraries/model_pca_10_svm.js"></script>
<script src="./libraries/p5.serialport.js"></script>
<script src="./libraries/three.js"></script>
<script src="./libraries/three.module.js"></script>

Thank for reading.

 

Leave a Reply

Your email address will not be published. Required fields are marked *