From fd335ce5cd83dfdf3c77529a23761c2bd7e95f3e Mon Sep 17 00:00:00 2001
From: Kumkwats <endercraft431@gmail.com>
Date: Fri, 28 Oct 2022 15:24:39 +0200
Subject: [PATCH] added camera view behind 3D view

---
 www/package-lock.json                         | 31 ++++++++++
 www/package.json                              |  1 +
 www/src/App.svelte                            |  7 +--
 .../3DHandler.svelte}                         | 61 +++++++++----------
 www/src/components/CameraHandler.svelte       | 45 ++++++++++++++
 www/src/components/cameraPreview.svelte       | 24 ++++++++
 www/src/helpers/mediaHelper.ts                | 10 +++
 www/src/pages/ViewAR.svelte                   | 26 ++++++++
 8 files changed, 168 insertions(+), 37 deletions(-)
 rename www/src/{pages/test3D.svelte => components/3DHandler.svelte} (84%)
 create mode 100644 www/src/components/CameraHandler.svelte
 create mode 100644 www/src/components/cameraPreview.svelte
 create mode 100644 www/src/helpers/mediaHelper.ts
 create mode 100644 www/src/pages/ViewAR.svelte

diff --git a/www/package-lock.json b/www/package-lock.json
index 1cf72c4..1065619 100644
--- a/www/package-lock.json
+++ b/www/package-lock.json
@@ -22,6 +22,7 @@
         "@rollup/plugin-typescript": "^8.0.0",
         "@tsconfig/svelte": "^2.0.0",
         "@types/three": "^0.144.0",
+        "@types/w3c-image-capture": "^1.0.7",
         "prettier": "^2.7.1",
         "prettier-plugin-svelte": "^2.8.0",
         "rollup": "^2.3.4",
@@ -353,6 +354,21 @@
         "@types/webxr": "*"
       }
     },
+    "node_modules/@types/w3c-image-capture": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/@types/w3c-image-capture/-/w3c-image-capture-1.0.7.tgz",
+      "integrity": "sha512-BNmjJT+yjwVzB6N5kf1/m9YZMF//e3JAcII+TIuhRkALn7UD9xZX0R1azVGqqyYuQKbQ6UskC+zSQctnsk9zHg==",
+      "dev": true,
+      "dependencies": {
+        "@types/webrtc": "*"
+      }
+    },
+    "node_modules/@types/webrtc": {
+      "version": "0.0.33",
+      "resolved": "https://registry.npmjs.org/@types/webrtc/-/webrtc-0.0.33.tgz",
+      "integrity": "sha512-xjN6BelzkY3lzXjIjXGqJVDS6XDleEsvp1bVIyNccXCcMoTH3wvUXFew4/qflwJdNqjmq98Zc5VcALV+XBKBvg==",
+      "dev": true
+    },
     "node_modules/@types/webxr": {
       "version": "0.5.0",
       "resolved": "https://registry.npmjs.org/@types/webxr/-/webxr-0.5.0.tgz",
@@ -1986,6 +2002,21 @@
         "@types/webxr": "*"
       }
     },
+    "@types/w3c-image-capture": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/@types/w3c-image-capture/-/w3c-image-capture-1.0.7.tgz",
+      "integrity": "sha512-BNmjJT+yjwVzB6N5kf1/m9YZMF//e3JAcII+TIuhRkALn7UD9xZX0R1azVGqqyYuQKbQ6UskC+zSQctnsk9zHg==",
+      "dev": true,
+      "requires": {
+        "@types/webrtc": "*"
+      }
+    },
+    "@types/webrtc": {
+      "version": "0.0.33",
+      "resolved": "https://registry.npmjs.org/@types/webrtc/-/webrtc-0.0.33.tgz",
+      "integrity": "sha512-xjN6BelzkY3lzXjIjXGqJVDS6XDleEsvp1bVIyNccXCcMoTH3wvUXFew4/qflwJdNqjmq98Zc5VcALV+XBKBvg==",
+      "dev": true
+    },
     "@types/webxr": {
       "version": "0.5.0",
       "resolved": "https://registry.npmjs.org/@types/webxr/-/webxr-0.5.0.tgz",
diff --git a/www/package.json b/www/package.json
index f6b40ad..6c87cf6 100644
--- a/www/package.json
+++ b/www/package.json
@@ -18,6 +18,7 @@
     "@rollup/plugin-typescript": "^8.0.0",
     "@tsconfig/svelte": "^2.0.0",
     "@types/three": "^0.144.0",
+    "@types/w3c-image-capture": "^1.0.7",
     "prettier": "^2.7.1",
     "prettier-plugin-svelte": "^2.8.0",
     "rollup": "^2.3.4",
diff --git a/www/src/App.svelte b/www/src/App.svelte
index 856b0ae..f618b66 100644
--- a/www/src/App.svelte
+++ b/www/src/App.svelte
@@ -1,11 +1,10 @@
-<script>
+<script lang="ts">
   import 'papercss/dist/paper.min.css';
   import { onMount } from 'svelte';
   import { Router, Route } from 'svelte-navigator';
   import { currentUser } from './store/user';
   import { read } from './functions/user';
-
-  import Test3D from './pages/test3D.svelte';
+  import ViewAR from './pages/ViewAR.svelte';
   import Login from './pages/Login.svelte';
   import Profile from './pages/Profile.svelte';
 
@@ -24,7 +23,7 @@
 
 <Router>
   <Route path="/">
-    <Test3D />
+    <ViewAR />
   </Route>
   <Route path="/login">
     <Login />
diff --git a/www/src/pages/test3D.svelte b/www/src/components/3DHandler.svelte
similarity index 84%
rename from www/src/pages/test3D.svelte
rename to www/src/components/3DHandler.svelte
index a617860..9c46959 100644
--- a/www/src/pages/test3D.svelte
+++ b/www/src/components/3DHandler.svelte
@@ -1,16 +1,13 @@
 <script lang="ts">
-  import NavBar from '../components/NavBar.svelte';
   import { beforeUpdate, afterUpdate, onMount, onDestroy } from 'svelte';
   import * as Three from 'three';
 
-  let container: HTMLElement;
-  let canvas: HTMLCanvasElement;
-  let width: number;
-  let height: number;
+  export let width: number;
+  export let height: number;
 
   //#region variables
-  //let innerWidth: number = 0;
-  //let innerHeight: number = 0;
+  let container: HTMLElement;
+  let canvas: HTMLCanvasElement;
   let maxInnerWidth: number = 0;
   let maxInnerHeight: number = 0;
 
@@ -18,6 +15,21 @@
   let focalLength: number = 540 / Math.atan((70 * Math.PI) / 180);
   let fov = 75;
   //let baseHorizontalFOV: number = 2.0*Math.atan(baseAspectRation*Math.tan(baseVerticalFOV*0.5));
+
+  let scene: Three.Scene = new Three.Scene();
+  let camera: Three.PerspectiveCamera = new Three.PerspectiveCamera(
+    75,
+    1,
+    0.1,
+    1000,
+  );
+
+  let renderer: Three.WebGLRenderer = new Three.WebGLRenderer();
+  let box: Three.BoxGeometry = new Three.BoxGeometry(1, 1, 1);
+  let mat: Three.MeshBasicMaterial = new Three.MeshBasicMaterial({
+    color: 0x00ff00,
+  });
+  let cube = new Three.Mesh(box, mat);
   //#endregion
 
   //#region functions
@@ -50,22 +62,6 @@
       resetMaxDimensions();
     }
   }
-  //#endregion
-
-  let scene: Three.Scene = new Three.Scene();
-  let camera: Three.PerspectiveCamera = new Three.PerspectiveCamera(
-    75,
-    1,
-    0.1,
-    1000,
-  );
-
-  let renderer: Three.WebGLRenderer = new Three.WebGLRenderer();
-  let box: Three.BoxGeometry = new Three.BoxGeometry(1, 1, 1);
-  let mat: Three.MeshBasicMaterial = new Three.MeshBasicMaterial({
-    color: 0x00ff00,
-  });
-  let cube = new Three.Mesh(box, mat);
 
   function animate() {
     requestAnimationFrame(animate);
@@ -75,6 +71,7 @@
 
     renderer.render(scene, camera);
   }
+  //#endregion
 
   onMount(() => {
     canvas = container.appendChild(renderer.domElement);
@@ -96,7 +93,7 @@
     camera.setViewOffset(width, height, 0, 0, width, height);
     renderer.setSize(width, height);
 
-    //renderer.setClearColor(0x000000, (innerHeight/(maxInnerHeight))/2 + 0.5);
+    renderer.setClearColor(0x000000, 0.1);
   });
 
   afterUpdate(() => {});
@@ -106,17 +103,15 @@
   });
 </script>
 
-<div
-  id="container"
-  bind:this={container}
-  bind:clientWidth={width}
-  bind:clientHeight={height}
-/>
-<NavBar />
+<div class="Container" bind:this={container} />
 
-<!--<svelte:window bind:innerWidth bind:innerHeight /> -->
 <style>
-  #container {
+  .Container {
+    position: absolute;
+    top: 0;
+    left: 0;
+    width: 100%;
     height: 100%;
+    overflow: hidden;
   }
 </style>
diff --git a/www/src/components/CameraHandler.svelte b/www/src/components/CameraHandler.svelte
new file mode 100644
index 0000000..49ff9a9
--- /dev/null
+++ b/www/src/components/CameraHandler.svelte
@@ -0,0 +1,45 @@
+<script lang="ts">
+  import { onMount } from 'svelte';
+  import CameraPreview from './CameraPreview.svelte';
+  import { getMediaStream, getImageCapture } from '../helpers/MediaHelper';
+
+  export let width: number;
+  export let height: number;
+
+  let container: HTMLElement;
+
+  let cameraWidth: number = 0;
+  let cameraHeight: number = 0;
+
+  let pictureUrl: string;
+  let imageCapture: ImageCapture;
+  let mediaStreamPromise: Promise<MediaStream>;
+
+  onMount(() => {
+    mediaStreamPromise = getMediaStream({ video: true }).then((mediaStream) => {
+      imageCapture = getImageCapture(mediaStream);
+      return mediaStream;
+    });
+  });
+</script>
+
+<div class="Container">
+  {#await mediaStreamPromise}
+    <h1>Waiting for camera....</h1>
+  {:then mediaStream}
+    <CameraPreview {mediaStream} {height} {width} />
+  {:catch error}
+    <h1>Error: {error.message}</h1>
+  {/await}
+</div>
+
+<style>
+  .Container {
+    position: absolute;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
+    overflow: hidden;
+  }
+</style>
diff --git a/www/src/components/cameraPreview.svelte b/www/src/components/cameraPreview.svelte
new file mode 100644
index 0000000..2b73070
--- /dev/null
+++ b/www/src/components/cameraPreview.svelte
@@ -0,0 +1,24 @@
+<script lang="ts">
+  export let mediaStream: MediaStream;
+  export let width: number;
+  export let height: number;
+
+  function srcObject(node, stream) {
+    node.srcObject = stream;
+    return {
+      update(newStream) {
+        if (node.srcObject != newStream) {
+          node.srcObject = newStream;
+        }
+      },
+    };
+  }
+</script>
+
+<video
+  use:srcObject={mediaStream}
+  width={width}
+  height={height}
+  autoplay
+  playsinline
+/>
diff --git a/www/src/helpers/mediaHelper.ts b/www/src/helpers/mediaHelper.ts
new file mode 100644
index 0000000..3dd67e3
--- /dev/null
+++ b/www/src/helpers/mediaHelper.ts
@@ -0,0 +1,10 @@
+function getMediaStream(constraints = { video: true }) {
+  return navigator.mediaDevices.getUserMedia(constraints);
+}
+
+function getImageCapture(mediaStream) {
+  const track = mediaStream.getVideoTracks()[0];
+  return new ImageCapture(track);
+}
+
+export { getMediaStream, getImageCapture };
diff --git a/www/src/pages/ViewAR.svelte b/www/src/pages/ViewAR.svelte
new file mode 100644
index 0000000..21bb396
--- /dev/null
+++ b/www/src/pages/ViewAR.svelte
@@ -0,0 +1,26 @@
+<script lang="ts">
+  import View3D from '../components/3DHandler.svelte';
+  import Camera from '../components/CameraHandler.svelte';
+  import NavBar from '../components/NavBar.svelte';
+
+  let containerHeight: number;
+  let containerWidth: number;
+</script>
+
+<div
+  class="viewAR"
+  bind:clientHeight={containerHeight}
+  bind:clientWidth={containerWidth}
+>
+  <Camera height={containerHeight} width={containerWidth} />
+  <View3D height={containerHeight} width={containerWidth} />
+</div>
+<NavBar />
+
+<style>
+  .viewAR {
+    position: absolute;
+    height: 100%;
+    width: 100%;
+  }
+</style>