Renderer update and return frame loop

packages/element/src/itk-camera.ts

@@ -14,13 +14,14 @@ export class ItkCamera extends LitElement {
 
  constructor() {
  super();
- this.addEventListener('mousedown', (e: MouseEvent) => {
- console.log(e.type);
-
- const targetCameraPose = mat4.fromTranslation(mat4.create(), [1, 2, 3]);
+ this.addEventListener('mousemove', (e: MouseEvent) => {
+ const pose = mat4.create();
+ mat4.copy(pose, this.camera.getSnapshot().context.pose);
+ const dX = e.movementX / 1000;
+ mat4.translate(pose, pose, [dX, 0, 0]);
  this.camera.send({
  type: 'setPose',
- pose: targetCameraPose,
+ pose,
  });
  });
  }

packages/element/src/itk-remote-viewport.ts

@@ -41,12 +41,15 @@ export class ItkRemoteViewport extends ItkViewport {
  this.remote.send({ type: 'setAddress', address: this.address });
  }
  if (changedProperties.has('density')) {
- this.remote.send({ type: 'setDensity', density: this.density });
+ this.remote.send({
+ type: 'updateRenderer',
+ props: { density: this.density },
+ });
  }
 
  if (this.frame.value && this.frame.value !== this.lastFrameValue) {
- this.imageSrc = 'data:image/png;base64,' + this.frame.value;
  this.lastFrameValue = this.frame.value;
+ this.imageSrc = 'data:image/png;base64,' + this.frame.value;
  }
  }
 

packages/remote-viewport/src/remote-machine.ts

@@ -1,11 +1,31 @@
-import { Viewport } from '@itk-viewer/viewer/viewport.js';
+import { ReadonlyMat4, mat4 } from 'gl-matrix';
 import { ActorRef, assign, createMachine } from 'xstate';
 
+import { Viewport } from '@itk-viewer/viewer/viewport.js';
+
+type RendererProps = {
+ density: number;
+ cameraPose: ReadonlyMat4;
+};
+
+// https://stackoverflow.com/a/74823834
+type Entries<T> = {
+ [K in keyof T]-?: [K, T[K]];
+}[keyof T][];
+
+const getEntries = <T extends object>(obj: T) =>
+ Object.entries(obj) as Entries<T>;
+
+// example: [['density', 30], ['cameraPose', mat4.create()]]
+export type RendererEntries = Entries<RendererProps>;
+
 type Context = {
  address: string | undefined;
  server: any | undefined;
  frame: string | undefined;
- density: number;
+ rendererProps: RendererProps;
+ queuedRendererEvents: RendererEntries;
+ stagedRendererEvents: RendererEntries;
  viewport: Viewport;
 };
 
@@ -14,44 +34,43 @@
  address: string | undefined;
 };
 
-type SetDensityEvent = {
- type: 'setDensity';
- density: number;
+type UpdateRendererEvent = {
+ type: 'updateRenderer';
+ props: Partial<RendererProps>;
 };
 
 type RenderEvent = {
  type: 'render';
 };
 
-type CameraPoseUpdatedEvent = {
- type: 'cameraPoseUpdated';
-};
-
 export const remoteMachine = createMachine({
  types: {} as {
  context: Context;
- events:
- | SetAddressEvent
- | SetDensityEvent
- | RenderEvent
- | CameraPoseUpdatedEvent;
+ events: SetAddressEvent | UpdateRendererEvent | RenderEvent;
  },
  id: 'remote',
  initial: 'disconnected',
  context: ({ input }: { input: { viewport: Viewport } }) => ({
  address: undefined,
  server: undefined,
  frame: undefined,
- density: 30,
+ rendererProps: { density: 30, cameraPose: mat4.create() },
+ queuedRendererEvents: [],
+ stagedRendererEvents: [],
  ...input, // captures injected viewport
  }),
  states: {
  disconnected: {
  entry: ({ context, self }) => {
- console.log(context);
+ // Update camera pose on viewport change
  context.viewport.subscribe(() => {
- (self as ActorRef<CameraPoseUpdatedEvent>).send({
- type: 'cameraPoseUpdated',
+ const cameraPose = context.viewport
+ .getSnapshot()
+ .context.camera?.getSnapshot().context.pose;
+ if (!cameraPose) throw new Error('no camera pose');
+ self.send({
+ type: 'updateRenderer',
+ props: { cameraPose },
  });
  });
  },
@@ -78,19 +97,47 @@
  onDone: {
  actions: assign({
  server: ({ event }) => event.output,
+ // initially, send all props to renderer
+ queuedRendererEvents: ({ context }) =>
+ getEntries(context.rendererProps),
  }),
- target: 'connected',
+ target: 'online',
  },
  },
  },
- connected: {
+ online: {
+ on: {
+ updateRenderer: {
+ actions: [
+ assign({
+ rendererProps: ({ event: { props }, context }) => ({
+ ...context.rendererProps,
+ ...props,
+ }),
+ queuedRendererEvents: ({ event: { props }, context }) => [
+ ...context.queuedRendererEvents,
+ ...(getEntries(props) as RendererEntries),
+ ],
+ }),
+ // Trigger a render (if in idle state)
+ ({ self }) => {
+ (self as ActorRef<UpdateRendererEvent | RenderEvent>).send({
+ type: 'render',
+ });
+ },
+ ],
+ },
+ },
  initial: 'render',
  states: {
  render: {
  invoke: {
  id: 'render',
  src: 'renderer',
- input: ({ context }: { context: Context }) => context.server,
+ input: ({ context }: { context: Context }) => ({
+ server: context.server,
+ events: [...context.stagedRendererEvents],
+ }),
  onDone: {
  actions: assign({
  frame: ({ event }) => event.output,
@@ -100,36 +147,22 @@
  },
  },
  idle: {
+ always: {
+ // Renderer props changed while rendering? Then render.
+ guard: ({ context }) =>
+ Object.keys(context.queuedRendererEvents).length > 0,
+ target: 'render',
+ },
  on: {
  render: { target: 'render' },
- setDensity: {
- actions: [
- assign({
- density: ({ event }) => event.density,
- }),
- ],
- target: 'updating',
- },
- cameraPoseUpdated: {
- target: 'updating',
- },
- },
- },
- updating: {
- invoke: {
- id: 'updating',
- src: 'updater',
- input: ({ context }: { context: Context }) => ({
- renderer: context.server,
- density: context.density,
- camera: context.viewport
- ?.getSnapshot()
- .context.camera?.getSnapshot().context,
- }),
- onDone: {
- target: 'render',
- },
  },
+ exit: assign({
+ // consumes queue in prep for renderer
+ stagedRendererEvents: ({ context }) => [
+ ...context.queuedRendererEvents,
+ ],
+ queuedRendererEvents: [],
+ }),
  },
  },
  },

packages/remote-viewport/src/remote-viewport.ts

@@ -1,12 +1,13 @@
 import { fromPromise, interpret } from 'xstate';
 import { hyphaWebsocketClient } from 'imjoy-rpc';
 import { Viewport, createViewport } from '@itk-viewer/viewer/viewport.js';
-import { remoteMachine } from './remote-machine.js';
+import { RendererEntries, remoteMachine } from './remote-machine.js';
+import { mat4, vec3 } from 'gl-matrix';
 
 export type RemoteMachineActors = {
  actors: {
- connect: ReturnType<typeof fromPromise<any>>;
- renderer: ReturnType<typeof fromPromise<any>>;
+ connect: ReturnType<typeof fromPromise<unknown>>;
+ renderer: ReturnType<typeof fromPromise<string>>;
  };
 };
 
@@ -17,10 +18,9 @@ const createHyphaRenderer = async (server_url: string) => {
  server_url,
  };
  const server = await hyphaWebsocketClient.connectToServer(config);
- const renderer = await server.getService('test-agave-renderer-paul');
+ const renderer = await server.getService('test-agave-renderer');
  await renderer.setup();
  await renderer.setImage('data/aneurism.ome.tif');
-
  return renderer;
 };
 
@@ -29,13 +29,26 @@ export const createHyphaActors: () => RemoteMachineActors = () => ({
  connect: fromPromise(async ({ input }) =>
  createHyphaRenderer(input.address)
  ),
- renderer: fromPromise(async ({ input: renderer }) => renderer.render()),
- updater: fromPromise(async ({ input: { renderer, density, camera } }) => {
- renderer.setDensity(density);
-
- console.log({ density, camera });
- // renderer.updateParameters({ density, cameraPose: camera.pose });
- }),
+ renderer: fromPromise(
+ async ({
+ input: { server, events },
+ }: {
+ input: {
+ server: { updateRenderer: (events: unknown) => Promise<string> };
+ events: RendererEntries;
+ };
+ }) => {
+ const translatedEvents = events.map(([key, value]) => {
+ if (key === 'cameraPose') {
+ const eye = vec3.create();
+ mat4.getTranslation(eye, value);
+ return ['cameraPose', { eye }];
+ }
+ return [key, value];
+ });
+ return server.updateRenderer(translatedEvents);
+ }
+ ),
  },
 });