face-controls.mdx

title	sourcecode
FaceControls	src/web/FaceControls.tsx

The camera follows your face.

Prerequisite: wrap into a FaceLandmarker provider

<FaceLandmarker>...</FaceLandmarker>

<FaceControls />

type FaceControlsProps = {
  /** The camera to be controlled, default: global state camera */
  camera?: THREE.Camera
  /** Whether to autostart the webcam, default: true */
  autostart?: boolean
  /** Enable/disable the webcam, default: true */
  webcam?: boolean
  /** A custom video URL or mediaStream, default: undefined */
  webcamVideoTextureSrc?: VideoTextureSrc
  /** Disable the rAF camera position/rotation update, default: false */
  manualUpdate?: boolean
  /** Disable the rVFC face-detection, default: false */
  manualDetect?: boolean
  /** Callback function to call on "videoFrame" event, default: undefined */
  onVideoFrame?: (e: THREE.Event) => void
  /** Reference this FaceControls instance as state's `controls` */
  makeDefault?: boolean
  /** Approximate time to reach the target. A smaller value will reach the target faster. */
  smoothTime?: number
  /** Apply position offset extracted from `facialTransformationMatrix` */
  offset?: boolean
  /** Offset sensitivity factor, less is more sensible, default: 80 */
  offsetScalar?: number
  /** Enable eye-tracking */
  eyes?: boolean
  /** Force Facemesh's `origin` to be the middle of the 2 eyes, default: true */
  eyesAsOrigin?: boolean
  /** Constant depth of the Facemesh, default: .15 */
  depth?: number
  /** Enable debug mode, default: false */
  debug?: boolean
  /** Facemesh options, default: undefined */
  facemesh?: FacemeshProps
}

type FaceControlsApi = THREE.EventDispatcher & {
  /** Detect faces from the video */
  detect: (video: HTMLVideoElement, time: number) => FaceLandmarkerResult | undefined
  /** Compute the target for the camera */
  computeTarget: () => THREE.Object3D
  /** Update camera's position/rotation to the `target` */
  update: (delta: number, target?: THREE.Object3D) => void
  /** <Facemesh> ref api */
  facemeshApiRef: RefObject<FacemeshApi>
  /** <Webcam> ref api */
  webcamApiRef: RefObject<WebcamApi>
  /** Play the video */
  play: () => void
  /** Pause the video */
  pause: () => void
}

FaceControls events

Two THREE.Events are dispatched on FaceControls ref object:

• { type: "stream", stream: MediaStream } -- when webcam's .getUserMedia() promise is resolved
• { type: "videoFrame", texture: THREE.VideoTexture, time: number } -- each time a new video frame is sent to the compositor (thanks to rVFC)

Note

rVFC

Internally, FaceControls uses requestVideoFrameCallback, you may need a polyfill (for Firefox).

FaceControls[manualDetect]

By default, detect is called on each "videoFrame". You can disable this by manualDetect and call detect yourself.

For example:

const controls = useThree((state) => state.controls)

const onVideoFrame = useCallback((event) => {
  controls.detect(event.texture.source.data, event.time)
}, [controls])

<FaceControls makeDefault
  manualDetect
  onVideoFrame={onVideoFrame}
/>

FaceControls[manualUpdate]

By default, update method is called each rAF useFrame. You can disable this by manualUpdate and call it yourself:

const controls = useThree((state) => state.controls)

useFrame((_, delta) => {
  controls.update(delta) // 60 or 120 FPS with default damping
})

<FaceControls makeDefault manualUpdate />

Or, if you want your own custom damping, use computeTarget method and update the camera pos/rot yourself with:

import * as easing from 'maath/easing'

const camera = useThree((state) => state.camera)

const [current] = useState(() => new THREE.Object3D())

useFrame((_, delta) => {
  const target = controls?.computeTarget()

  if (target) {
    //
    // A. Define your own damping
    //

    const eps = 1e-9
    easing.damp3(current.position, target.position, 0.25, delta, undefined, undefined, eps)
    easing.dampE(current.rotation, target.rotation, 0.25, delta, undefined, undefined, eps)
    camera.position.copy(current.position)
    camera.rotation.copy(current.rotation)

    //
    // B. Or maybe with no damping at all?
    //

    // camera.position.copy(target.position)
    // camera.rotation.copy(target.rotation)
  }
})