Feat: load glb terrain from .skyc files

src/desktop/launcher/media-protocol.mjs

@@ -11,19 +11,18 @@ const registerMediaProtocol = () => {
   protocol.registerFileProtocol('media', (request, callback) => {
     try {
       const parsedUrl = new URL(request.url);
-      const index =
-        parsedUrl.host === 'audio'
-          ? Number.parseInt(parsedUrl.pathname.slice(1), 10)
-          : -1;
+      const index = Number.parseInt(parsedUrl.pathname.slice(1), 10);
 
-      const audioBuffer = index >= 0 ? getAudioBuffer(index) : null;
+      let buffer = null;
+      if (Number.isFinite(index) && index >= 0) {
+        if (parsedUrl.host === 'audio') {
+          buffer = getAudioBuffer(index);
+        }
+      }
 
       /* Error -6 = file not found in net_error_list.h in Chromium */
-      callback(
-        audioBuffer && audioBuffer.path
-          ? { path: audioBuffer.path }
-          : { error: -6 }
-      );
+      callback(buffer?.path ? { path: buffer.path } : { error: -6 });
+
     } catch (error) {
       console.error('Unexpected error in media:// protocol handler');
       console.error(error);

src/features/show/actions.ts

@@ -1,6 +1,7 @@
 import {
   loadCompiledShow,
   type AudioData,
+  type TerrainModelData,
   type ShowSpecification,
 } from '@skybrush/show-format';
 
@@ -16,6 +17,14 @@ type AudioSpecWithUrl = Omit<AudioData, 'data'> & {
   url: string;
 };
 
+type TerrainModelWithUrl = Omit<TerrainModelData, 'data'> & {
+  data?: TerrainModelData['data'];
+  url: string;
+};
+
+/** Last terrain blob URL; revoked after a replacement show is committed. */
+let terrainObjectUrl: string | undefined;
+
 const loadShowFromBufferInner = async (buffer: Buffer) => {
   const { setAudioBuffer } = getElectronBridge() ?? {};
   const showSpec = await loadCompiledShow(buffer, { assets: true });
@@ -47,7 +56,36 @@ const loadShowFromBufferInner = async (buffer: Buffer) => {
     }
   }
 
-  return showSpec;
+  const terrainSpec = showSpec?.environment?.terrain;
+  const terrainModel = terrainSpec?.model;
+  const terrainData = terrainModel?.data;
+
+  // Keep the previous URL alive until the new show is committed to state.
+  const urlToRevoke = terrainObjectUrl;
+  terrainObjectUrl = undefined;
+
+  if (terrainData && terrainModel) {
+    if (terrainData instanceof Uint8Array || Buffer.isBuffer(terrainData)) {
+      const bytes =
+        terrainData instanceof Uint8Array
+          ? terrainData
+          : new Uint8Array(terrainData);
+
+      const blob = new Blob([bytes as BlobPart], {
+        type: 'model/gltf-binary',
+      });
+      const url = URL.createObjectURL(blob);
+      terrainObjectUrl = url;
+
+      const terrainModelWithUrl = terrainModel as TerrainModelWithUrl;
+      delete terrainModelWithUrl.data;
+      terrainModelWithUrl.url = url;
+    } else {
+      console.warn('Terrain model is not loaded as binary data');
+    }
+  }
+
+  return { show: showSpec, urlToRevoke };
 };
 
 export const loadShowFromBuffer =
@@ -56,8 +94,14 @@ export const loadShowFromBuffer =
     const loadAction = await dispatch(
       withProgressIndicator(() => loadShowFromBufferInner(buffer))
     );
-    const show: ShowSpecification = loadAction.payload as ShowSpecification;
+    const { show, urlToRevoke } = loadAction.payload as {
+      show: ShowSpecification;
+      urlToRevoke?: string;
+    };
     dispatch(loadShowFromRequest({ show, source: { type: 'buffer' } }));
+    if (urlToRevoke) {
+      URL.revokeObjectURL(urlToRevoke);
+    }
   };
 
 export const loadShowFromLocalFile =
@@ -81,8 +125,14 @@ export const loadShowFromLocalFile =
         return result;
       })
     );
-    const show: ShowSpecification = loadAction.payload as ShowSpecification;
+    const { show, urlToRevoke } = loadAction.payload as {
+      show: ShowSpecification;
+      urlToRevoke?: string;
+    };
     dispatch(loadShowFromRequest({ show, source: { type: 'file', filename } }));
+    if (urlToRevoke) {
+      URL.revokeObjectURL(urlToRevoke);
+    }
 
     dispatch(addRecentFile(filename));
   };

src/features/show/selectors.ts

@@ -193,9 +193,9 @@ export const getCameras = createSelector(
 const ensureHeightAboveGround = (camera: Camera, minHeight = 1): Camera =>
   Array.isArray(camera.position) && camera.position[2] < minHeight
     ? {
-        ...camera,
-        position: [camera.position[0], camera.position[1], minHeight],
-      }
+      ...camera,
+      position: [camera.position[0], camera.position[1], minHeight],
+    }
     : camera;
 
 /**
@@ -312,11 +312,11 @@ export const getPyroCues = createSelector(
       .flatMap((program, droneIndex) =>
         program
           ? program.events.map(([time, channel, payloadId]) => ({
-              time,
-              droneIndex,
-              payloadName: program.payloads[payloadId]?.name,
-              channel,
-            }))
+            time,
+            droneIndex,
+            payloadName: program.payloads[payloadId]?.name,
+            channel,
+          }))
           : []
       )
       .toSorted((a, b) => a.time - b.time);
@@ -523,6 +523,23 @@ export const getShowTitle = createSelector(
       : 'No show loaded'
 );
 
+/**
+ * Returns the blob URL of the embedded terrain model and its transform, if any
+ */
+export const getTerrainModel = (state: RootState) => {
+  const terrain = getShowSpecification(state)?.environment?.terrain;
+  const model = terrain?.model as { url?: string } | undefined;
+  if (!model?.url) return undefined;
+
+  const t = terrain?.transform;
+  return {
+    url: model.url,
+    position: t?.position ?? ([0, 0, 0] as const),
+    rotation: t?.rotation ?? ([1, 0, 0, 0] as const), // WXYZ
+    scale: t?.scale ?? ([1, 1, 1] as const),
+  };
+};
+
 /**
  * Returns whether we are currently loading a show file.
  */

src/features/show/utils.ts

@@ -2,8 +2,9 @@ import {
   skybrushRotationToQuaternion,
   type Pose,
 } from '@skybrush/aframe-components/spatial';
-import type { Camera } from '@skybrush/show-format';
+import type { Camera, QuaternionWXYZTuple } from '@skybrush/show-format';
 import type { ShowDataSource } from './types';
+import * as THREE from 'three';
 
 export const DEFAULT_CAMERA_ORIENTATION = skybrushRotationToQuaternion([
   90, 0, -90,
@@ -30,3 +31,16 @@ export function isShowDataSourceReloadable(
 ): boolean {
   return source?.type === 'file';
 }
+
+/**
+ * Converts a Skybrush (show-space) quaternion WXYZ to A-Frame Euler
+ * degrees in YXZ order, without remapping axes to Three.js world space.
+ */
+export function skybrushQuaternionToEulerDegrees(
+  wxyz: QuaternionWXYZTuple
+): [number, number, number] {
+  const quat = new THREE.Quaternion(wxyz[1], wxyz[2], wxyz[3], wxyz[0]);
+  const euler = new THREE.Euler().setFromQuaternion(quat, 'YXZ');
+  const { radToDeg } = THREE.MathUtils;
+  return [radToDeg(euler.x), radToDeg(euler.y), radToDeg(euler.z)];
+}

src/views/player/Scenery.tsx

@@ -5,6 +5,7 @@ import { objectToString } from '@skybrush/aframe-components';
 const grounds = {
   /* Minecraft-style ground texture (green) */
   default: {
+    ground: 'hills',
     groundColor: '#8eb971',
     groundColor2: '#507a32',
     groundTexture: 'walkernoise',
@@ -139,12 +140,14 @@ const Scenery = ({
     stageSize,
   };
 
-  if (showTerrainModel && type !== 'indoor') {
+  if (type === 'indoor') {
+    environment.ground = 'flat';
+  } else if (showTerrainModel) {
     environment.ground = 'none';
+  } else {
+    environment.ground = 'hills';
   }
 
-  console.log(JSON.stringify(environment));
-
   return enabled ? (
     <a-entity position='0 -0.001 0' scale={`${scale} ${scale} ${scale}`}>
       {/* Move the floor slightly down to ensure that the coordinate axes are nicely visible */}

src/views/player/ThreeDView.tsx

@@ -12,16 +12,18 @@ import { connect } from 'react-redux';
 import '~/aframe';
 
 import { objectToString } from '@skybrush/aframe-components';
-import type {
-  ThreeJsPositionTuple,
-  ThreeJsRotationTuple,
+import {
+  skybrushToThreeJsPosition,
+  type ThreeJsPositionTuple,
+  type ThreeJsRotationTuple,
 } from '@skybrush/aframe-components/spatial';
 
 import { getDroneModel } from '~/features/settings/selectors';
 import type { DroneModelType } from '~/features/settings/types';
 import {
   getLoadedShowId,
   getNumberOfDronesInShow,
+  getTerrainModel,
 } from '~/features/show/selectors';
 import {
   getEffectiveDroneRadius,
@@ -37,6 +39,8 @@ import SelectionMarkers from './SelectionMarkers';
 
 import flapperDroneModel from '~/../assets/models/flapper-drone.obj';
 import quadcopterModel from '~/../assets/models/quadcopter.obj';
+import type { QuaternionWXYZTuple } from '@skybrush/math';
+import { skybrushQuaternionToEulerDegrees } from '~/features/show/utils';
 
 type ThreeDViewProps = {
   readonly axes: boolean;
@@ -60,6 +64,12 @@ type ThreeDViewProps = {
   readonly showLabels: boolean;
   readonly showStatistics: boolean;
   readonly showYaw: boolean;
+  readonly terrainModel?: {
+    url: string;
+    position: readonly [number, number, number];
+    rotation: QuaternionWXYZTuple;
+    scale: readonly [number, number, number];
+  };
   readonly vrEnabled?: boolean;
 };
 
@@ -85,6 +95,7 @@ const ThreeDView = (props: ThreeDViewProps) => {
     showLabels,
     showStatistics,
     showYaw,
+    terrainModel,
     vrEnabled,
   } = props;
 
@@ -183,7 +194,15 @@ const ThreeDView = (props: ThreeDViewProps) => {
         {/* <VelocityArrows /> */}
       </a-entity>
 
-      <Scenery type={scenery} grid={grid} />
+      <Scenery type={scenery} grid={grid} showTerrainModel={Boolean(terrainModel)} />
+      {terrainModel ? (
+        <a-entity
+          gltf-model={terrainModel.url}
+          position={skybrushToThreeJsPosition([...terrainModel.position]).join(' ')}
+          rotation={skybrushQuaternionToEulerDegrees(terrainModel.rotation).join(' ')}
+          scale={terrainModel.scale.join(' ')}
+        />
+      ) : null}
     </a-scene>
   );
 };
@@ -199,6 +218,7 @@ export default connect(
     droneModel: getDroneModel(state),
     droneRadius: getEffectiveDroneRadius(state),
     scenery: getEffectiveScenery(state),
+    terrainModel: getTerrainModel(state),
   }),
   // mapDispatchToProps
   {},