From 134664e2759ae6015509d62d06422889d47832a8 Mon Sep 17 00:00:00 2001 From: Paul Elliott Date: Fri, 28 Jul 2023 10:52:05 -0400 Subject: [PATCH 1/4] feat(itk-camera): add orbit-camera dependency --- packages/element/package.json | 1 + packages/element/src/itk-camera.ts | 38 +++++++++++++++---- packages/element/src/orbit-camera.d.ts | 7 ++++ .../remote-viewport/src/remote-machine.ts | 3 +- pnpm-lock.yaml | 13 +++++++ 5 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 packages/element/src/orbit-camera.d.ts diff --git a/packages/element/package.json b/packages/element/package.json index 4d0e501a..fa838997 100644 --- a/packages/element/package.json +++ b/packages/element/package.json @@ -54,6 +54,7 @@ "@itk-viewer/viewer": "workspace:*", "gl-matrix": "^3.4.3", "lit": "^2.7.4", + "orbit-camera": "^1.0.0", "xstate-lit": "^1.3.1" } } diff --git a/packages/element/src/itk-camera.ts b/packages/element/src/itk-camera.ts index d71d85e8..addf3c75 100644 --- a/packages/element/src/itk-camera.ts +++ b/packages/element/src/itk-camera.ts @@ -1,6 +1,7 @@ -import { LitElement, PropertyValues, html } from 'lit'; +import { LitElement, PropertyValues, css, html } from 'lit'; import { customElement, property } from 'lit/decorators.js'; import { mat4 } from 'gl-matrix'; +import createOrbitCamera from 'orbit-camera'; import { Viewport } from '@itk-viewer/viewer/viewport.js'; import { createCamera } from '@itk-viewer/viewer/camera-machine.js'; @@ -14,14 +15,30 @@ export class ItkCamera extends LitElement { constructor() { super(); + const view = mat4.create(); + const cameraController = createOrbitCamera( + [0, 0, 1], + [0.5, 0.5, 0.5], + [0, 1, 0] + ); + + const prevMouse = [0, 0]; + + const shell = { width: 500, height: 400 }; + this.addEventListener('mousemove', (e: MouseEvent) => { - const pose = mat4.create(); - mat4.copy(pose, this.camera.getSnapshot().context.pose); - const dX = e.movementX / 1000; - mat4.translate(pose, pose, [dX, 0, 0]); + const [mouseX, mouseY] = [e.clientX, e.clientY]; + const [prevMouseX, prevMouseY] = prevMouse; + + cameraController.rotate( + [mouseX / shell.width - 0.5, mouseY / shell.height - 0.5], + [prevMouseX / shell.width - 0.5, prevMouseY / shell.height - 0.5] + ); + + cameraController.view(view); this.camera.send({ type: 'setPose', - pose, + pose: view, }); }); } @@ -33,8 +50,15 @@ export class ItkCamera extends LitElement { } render() { - return html` `; + return html`
`; } + + static styles = css` + .container { + min-width: 500px; + min-height: 400px; + } + `; } declare global { diff --git a/packages/element/src/orbit-camera.d.ts b/packages/element/src/orbit-camera.d.ts new file mode 100644 index 00000000..8fb1fb22 --- /dev/null +++ b/packages/element/src/orbit-camera.d.ts @@ -0,0 +1,7 @@ +declare module 'orbit-camera' { + export default function createOrbitCamera( + eye: number[], + target: number[], + up: number[] + ): any; +} diff --git a/packages/remote-viewport/src/remote-machine.ts b/packages/remote-viewport/src/remote-machine.ts index becb869f..a77f44a5 100644 --- a/packages/remote-viewport/src/remote-machine.ts +++ b/packages/remote-viewport/src/remote-machine.ts @@ -149,8 +149,7 @@ export const remoteMachine = createMachine({ idle: { always: { // Renderer props changed while rendering? Then render. - guard: ({ context }) => - Object.keys(context.queuedRendererEvents).length > 0, + guard: ({ context }) => context.queuedRendererEvents.length > 0, target: 'render', }, on: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3ea3ec74..d54bf3a7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,6 +78,9 @@ importers: lit: specifier: ^2.7.4 version: 2.7.4 + orbit-camera: + specifier: ^1.0.0 + version: 1.0.0 xstate-lit: specifier: ^1.3.1 version: 1.3.1(lit@2.7.4)(xstate@4.38.2) @@ -1786,6 +1789,10 @@ packages: assert-plus: 1.0.0 dev: true + /gl-matrix@2.8.1: + resolution: {integrity: sha512-0YCjVpE3pS5XWlN3J4X7AiAx65+nqAI54LndtVFnQZB6G/FVLkZH8y8V6R3cIoOQR4pUdfwQGd1iwyoXHJ4Qfw==} + dev: false + /gl-matrix@3.4.3: resolution: {integrity: sha512-wcCp8vu8FT22BnvKVPjXa/ICBWRq/zjFfdofZy1WSpQZpphblv12/bOQLBC1rMM7SGOFS9ltVmKOHil5+Ml7gA==} dev: false @@ -2355,6 +2362,12 @@ packages: word-wrap: 1.2.3 dev: true + /orbit-camera@1.0.0: + resolution: {integrity: sha512-RJ9ywtL3DCDeXFLoGgzDYoJN6PPi6T490r1q6niWzacLaSiY3N8WuhOQ2vEvapMOHKXQl2iCsxBnBSeOHwECyw==} + dependencies: + gl-matrix: 2.8.1 + dev: false + /ospath@1.2.2: resolution: {integrity: sha512-o6E5qJV5zkAbIDNhGSIlyOhScKXgQrSRMilfph0clDfM0nEnBOlKlH4sWDmG95BW/CvwNz0vmm7dJVtU2KlMiA==} dev: true From 167df422e2a58e95f5d260a2ac19ed4554954355 Mon Sep 17 00:00:00 2001 From: Paul Elliott Date: Mon, 31 Jul 2023 22:03:11 -0400 Subject: [PATCH 2/4] feat(itk-camera): wire up mouse events to camera controller --- packages/element/src/itk-camera.ts | 134 +++++++++++++++--- .../remote-viewport/src/remote-viewport.ts | 2 +- 2 files changed, 115 insertions(+), 21 deletions(-) diff --git a/packages/element/src/itk-camera.ts b/packages/element/src/itk-camera.ts index addf3c75..3266af51 100644 --- a/packages/element/src/itk-camera.ts +++ b/packages/element/src/itk-camera.ts @@ -1,10 +1,105 @@ import { LitElement, PropertyValues, css, html } from 'lit'; import { customElement, property } from 'lit/decorators.js'; -import { mat4 } from 'gl-matrix'; +import { ReadonlyMat4, mat4 } from 'gl-matrix'; import createOrbitCamera from 'orbit-camera'; import { Viewport } from '@itk-viewer/viewer/viewport.js'; import { createCamera } from '@itk-viewer/viewer/camera-machine.js'; +import { Ref, createRef, ref } from 'lit/directives/ref.js'; + +type OrbitCameraController = ReturnType; + +const PAN_SPEED = 1; + +const bindCamera = ( + camera: OrbitCameraController, + element: HTMLElement, + onUpdate: (view: ReadonlyMat4) => unknown +) => { + let width = element.clientWidth; + let height = element.clientHeight; + + const view = mat4.create(); + + const resizeObserver = new ResizeObserver((entries) => { + for (const entry of entries) { + width = entry.contentRect.width; + height = entry.contentRect.height; + } + }); + resizeObserver.observe(element); + + let rotate = false; + let pan = false; + let scale = false; + + const onMouseDown = (e: MouseEvent) => { + e.preventDefault(); + if (e.button === 0) { + rotate = true; + } else if (e.button === 1) { + scale = true; + } else if (e.button === 2) { + pan = true; + } + }; + element.addEventListener('mousedown', onMouseDown); + + const onMouseUp = (e: MouseEvent) => { + if (e.button === 0) { + rotate = false; + } else if (e.button === 1) { + scale = false; + } else if (e.button === 2) { + pan = false; + } + }; + element.addEventListener('mouseup', onMouseUp); + + let prevMouseX = 0; + let prevMouseY = 0; + + const onMouseMove = (e: MouseEvent) => { + const mouseX = e.offsetX; + const mouseY = e.offsetY; + + if (rotate) { + camera.rotate( + [mouseX / width - 0.5, mouseY / height - 0.5], + [prevMouseX / width - 0.5, prevMouseY / height - 0.5] + ); + } + + if (pan) { + camera.pan([ + (PAN_SPEED * (mouseX - prevMouseX)) / width, + (PAN_SPEED * (mouseY - prevMouseY)) / height, + ]); + } + + if (scale) { + const d = mouseY - prevMouseY; + if (d) camera.distance *= Math.exp(d / height); + } + + prevMouseX = mouseX; + prevMouseY = mouseY; + + if (!rotate && !pan && !scale) return; + camera.view(view); + onUpdate(view); + }; + element.addEventListener('mousemove', onMouseMove); + + const unbind = () => { + resizeObserver.disconnect(); + element.removeEventListener('mousemove', onMouseMove); + element.removeEventListener('mousedown', onMouseDown); + element.removeEventListener('mouseup', onMouseUp); + }; + + return unbind; +}; @customElement('itk-camera') export class ItkCamera extends LitElement { @@ -12,30 +107,20 @@ export class ItkCamera extends LitElement { viewport: Viewport | undefined; camera = createCamera(); + unbind: (() => unknown) | undefined; + container: Ref = createRef(); - constructor() { - super(); - const view = mat4.create(); + firstUpdated(): void { const cameraController = createOrbitCamera( - [0, 0, 1], + [-0.747528, -0.570641, 0.754992], [0.5, 0.5, 0.5], - [0, 1, 0] + [-0.505762, 0.408327, -0.759916] ); - const prevMouse = [0, 0]; - - const shell = { width: 500, height: 400 }; + const container = this.container.value; + if (!container) throw new Error('container not found'); - this.addEventListener('mousemove', (e: MouseEvent) => { - const [mouseX, mouseY] = [e.clientX, e.clientY]; - const [prevMouseX, prevMouseY] = prevMouse; - - cameraController.rotate( - [mouseX / shell.width - 0.5, mouseY / shell.height - 0.5], - [prevMouseX / shell.width - 0.5, prevMouseY / shell.height - 0.5] - ); - - cameraController.view(view); + this.unbind = bindCamera(cameraController, container, (view) => { this.camera.send({ type: 'setPose', pose: view, @@ -43,6 +128,11 @@ export class ItkCamera extends LitElement { }); } + disconnectedCallback(): void { + super.disconnectedCallback(); + this.unbind?.(); + } + willUpdate(changedProperties: PropertyValues) { if (changedProperties.has('viewport')) { this.viewport?.send({ type: 'setCamera', camera: this.camera }); @@ -50,7 +140,11 @@ export class ItkCamera extends LitElement { } render() { - return html`
`; + return html` +
+ +
+ `; } static styles = css` diff --git a/packages/remote-viewport/src/remote-viewport.ts b/packages/remote-viewport/src/remote-viewport.ts index 7b62a42c..353ad4c1 100644 --- a/packages/remote-viewport/src/remote-viewport.ts +++ b/packages/remote-viewport/src/remote-viewport.ts @@ -18,7 +18,7 @@ const createHyphaRenderer = async (server_url: string) => { server_url, }; const server = await hyphaWebsocketClient.connectToServer(config); - const renderer = await server.getService('test-agave-renderer'); + const renderer = await server.getService('test-agave-renderer-paul'); await renderer.setup(); await renderer.setImage('data/aneurism.ome.tif'); return renderer; From 0cb38147495b9e497b87f74dedef23fd4e10f8cd Mon Sep 17 00:00:00 2001 From: Paul Elliott Date: Thu, 3 Aug 2023 10:20:37 -0400 Subject: [PATCH 3/4] feat(itk-camera): send full camera pose --- packages/element/src/itk-camera.ts | 51 ++++++++++++++++--- .../remote-viewport/src/remote-machine.ts | 15 +++--- .../remote-viewport/src/remote-viewport.ts | 10 +++- 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/packages/element/src/itk-camera.ts b/packages/element/src/itk-camera.ts index 3266af51..f4c1097f 100644 --- a/packages/element/src/itk-camera.ts +++ b/packages/element/src/itk-camera.ts @@ -4,12 +4,13 @@ import { ReadonlyMat4, mat4 } from 'gl-matrix'; import createOrbitCamera from 'orbit-camera'; import { Viewport } from '@itk-viewer/viewer/viewport.js'; -import { createCamera } from '@itk-viewer/viewer/camera-machine.js'; +import { Camera, createCamera } from '@itk-viewer/viewer/camera-machine.js'; import { Ref, createRef, ref } from 'lit/directives/ref.js'; type OrbitCameraController = ReturnType; const PAN_SPEED = 1; +const ZOOM_SPEED = 0.005; const bindCamera = ( camera: OrbitCameraController, @@ -21,6 +22,12 @@ const bindCamera = ( const view = mat4.create(); + const updateView = () => { + camera.view(view); + mat4.invert(view, view); + onUpdate(view); + }; + const resizeObserver = new ResizeObserver((entries) => { for (const entry of entries) { width = entry.contentRect.width; @@ -42,10 +49,12 @@ const bindCamera = ( } else if (e.button === 2) { pan = true; } + return false; }; element.addEventListener('mousedown', onMouseDown); const onMouseUp = (e: MouseEvent) => { + e.preventDefault(); if (e.button === 0) { rotate = false; } else if (e.button === 1) { @@ -53,6 +62,7 @@ const bindCamera = ( } else if (e.button === 2) { pan = false; } + return false; }; element.addEventListener('mouseup', onMouseUp); @@ -86,16 +96,29 @@ const bindCamera = ( prevMouseY = mouseY; if (!rotate && !pan && !scale) return; - camera.view(view); - onUpdate(view); + + updateView(); }; element.addEventListener('mousemove', onMouseMove); + const onWheel = (e: WheelEvent) => { + e.preventDefault(); + camera.zoom(e.deltaY * ZOOM_SPEED); + + updateView(); + }; + element.addEventListener('wheel', onWheel, { passive: false }); + + const preventDefault = (e: Event) => e.preventDefault(); + element.addEventListener('contextmenu', preventDefault); + const unbind = () => { resizeObserver.disconnect(); element.removeEventListener('mousemove', onMouseMove); element.removeEventListener('mousedown', onMouseDown); element.removeEventListener('mouseup', onMouseUp); + element.removeEventListener('wheel', onWheel); + element.removeEventListener('contextmenu', preventDefault); }; return unbind; @@ -106,24 +129,36 @@ export class ItkCamera extends LitElement { @property({ attribute: false }) viewport: Viewport | undefined; - camera = createCamera(); + camera: Camera; + cameraController: OrbitCameraController; unbind: (() => unknown) | undefined; container: Ref = createRef(); - firstUpdated(): void { - const cameraController = createOrbitCamera( + constructor() { + super(); + this.camera = createCamera(); + + this.cameraController = createOrbitCamera( [-0.747528, -0.570641, 0.754992], [0.5, 0.5, 0.5], [-0.505762, 0.408327, -0.759916] ); + const pose = this.cameraController.view(); + this.camera.send({ + type: 'setPose', + pose, + }); + } + + firstUpdated(): void { const container = this.container.value; if (!container) throw new Error('container not found'); - this.unbind = bindCamera(cameraController, container, (view) => { + this.unbind = bindCamera(this.cameraController, container, (pose) => { this.camera.send({ type: 'setPose', - pose: view, + pose, }); }); } diff --git a/packages/remote-viewport/src/remote-machine.ts b/packages/remote-viewport/src/remote-machine.ts index a77f44a5..c0d1e5a0 100644 --- a/packages/remote-viewport/src/remote-machine.ts +++ b/packages/remote-viewport/src/remote-machine.ts @@ -63,6 +63,7 @@ export const remoteMachine = createMachine({ disconnected: { entry: ({ context, self }) => { // Update camera pose on viewport change + // FIXME: not capturing initial camera position because updateRender handler not registered before connection context.viewport.subscribe(() => { const cameraPose = context.viewport .getSnapshot() @@ -98,8 +99,8 @@ export const remoteMachine = createMachine({ actions: assign({ server: ({ event }) => event.output, // initially, send all props to renderer - queuedRendererEvents: ({ context }) => - getEntries(context.rendererProps), + // queuedRendererEvents: ({ context }) => + // getEntries(context.rendererProps), }), target: 'online', }, @@ -110,10 +111,12 @@ export const remoteMachine = createMachine({ updateRenderer: { actions: [ assign({ - rendererProps: ({ event: { props }, context }) => ({ - ...context.rendererProps, - ...props, - }), + rendererProps: ({ event: { props }, context }) => { + return { + ...context.rendererProps, + ...props, + }; + }, queuedRendererEvents: ({ event: { props }, context }) => [ ...context.queuedRendererEvents, ...(getEntries(props) as RendererEntries), diff --git a/packages/remote-viewport/src/remote-viewport.ts b/packages/remote-viewport/src/remote-viewport.ts index 353ad4c1..0f913436 100644 --- a/packages/remote-viewport/src/remote-viewport.ts +++ b/packages/remote-viewport/src/remote-viewport.ts @@ -18,7 +18,7 @@ const createHyphaRenderer = async (server_url: string) => { server_url, }; const server = await hyphaWebsocketClient.connectToServer(config); - const renderer = await server.getService('test-agave-renderer-paul'); + const renderer = await server.getService('test-agave-renderer'); await renderer.setup(); await renderer.setImage('data/aneurism.ome.tif'); return renderer; @@ -42,7 +42,13 @@ export const createHyphaActors: () => RemoteMachineActors = () => ({ if (key === 'cameraPose') { const eye = vec3.create(); mat4.getTranslation(eye, value); - return ['cameraPose', { eye }]; + + const target = vec3.fromValues(value[8], value[9], value[10]); + vec3.subtract(target, eye, target); + + const up = vec3.fromValues(value[4], value[5], value[6]); + + return ['cameraPose', { eye, up, target }]; } return [key, value]; }); From 7c1cc48f01bb0ebe5cd5cd8f3e88013ea65fe6c4 Mon Sep 17 00:00:00 2001 From: Paul Elliott Date: Fri, 4 Aug 2023 09:56:57 -0400 Subject: [PATCH 4/4] fix(itk-camera): stop moving camera on mouse up no matter cursor position --- packages/element/src/itk-camera.ts | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/packages/element/src/itk-camera.ts b/packages/element/src/itk-camera.ts index f4c1097f..45bf9e95 100644 --- a/packages/element/src/itk-camera.ts +++ b/packages/element/src/itk-camera.ts @@ -14,11 +14,11 @@ const ZOOM_SPEED = 0.005; const bindCamera = ( camera: OrbitCameraController, - element: HTMLElement, + viewport: HTMLElement, onUpdate: (view: ReadonlyMat4) => unknown ) => { - let width = element.clientWidth; - let height = element.clientHeight; + let width = viewport.clientWidth; + let height = viewport.clientHeight; const view = mat4.create(); @@ -34,7 +34,7 @@ const bindCamera = ( height = entry.contentRect.height; } }); - resizeObserver.observe(element); + resizeObserver.observe(viewport); let rotate = false; let pan = false; @@ -49,9 +49,8 @@ const bindCamera = ( } else if (e.button === 2) { pan = true; } - return false; }; - element.addEventListener('mousedown', onMouseDown); + viewport.addEventListener('mousedown', onMouseDown); const onMouseUp = (e: MouseEvent) => { e.preventDefault(); @@ -62,9 +61,8 @@ const bindCamera = ( } else if (e.button === 2) { pan = false; } - return false; }; - element.addEventListener('mouseup', onMouseUp); + window.addEventListener('mouseup', onMouseUp); let prevMouseX = 0; let prevMouseY = 0; @@ -99,7 +97,7 @@ const bindCamera = ( updateView(); }; - element.addEventListener('mousemove', onMouseMove); + viewport.addEventListener('mousemove', onMouseMove); const onWheel = (e: WheelEvent) => { e.preventDefault(); @@ -107,18 +105,18 @@ const bindCamera = ( updateView(); }; - element.addEventListener('wheel', onWheel, { passive: false }); + viewport.addEventListener('wheel', onWheel, { passive: false }); const preventDefault = (e: Event) => e.preventDefault(); - element.addEventListener('contextmenu', preventDefault); + viewport.addEventListener('contextmenu', preventDefault); const unbind = () => { resizeObserver.disconnect(); - element.removeEventListener('mousemove', onMouseMove); - element.removeEventListener('mousedown', onMouseDown); - element.removeEventListener('mouseup', onMouseUp); - element.removeEventListener('wheel', onWheel); - element.removeEventListener('contextmenu', preventDefault); + viewport.removeEventListener('mousedown', onMouseDown); + window.removeEventListener('mouseup', onMouseUp); + viewport.removeEventListener('mousemove', onMouseMove); + viewport.removeEventListener('wheel', onWheel); + viewport.removeEventListener('contextmenu', preventDefault); }; return unbind;