Skip to content

fix(translation-demo): preserve audio delay after MoQ update #82

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Qizot wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(translation-demo): preserve audio delay after MoQ update #82

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions translation-demo/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@
},
"dependencies": {
"@fishjam-cloud/react-client": "0.28.3",
"@moq/publish": "0.2.15",
"@moq/signals": "0.1.9",
"@moq/watch": "0.2.17",
"@moq/publish": "0.4.4",
"@moq/signals": "0.2.2",
"@moq/watch": "0.5.0",
"@radix-ui/react-icons": "^1.3.2",
"@radix-ui/react-label": "^2.1.0",
"@radix-ui/react-popover": "^1.1.17",
Expand Down Expand Up @@ -44,5 +44,8 @@
"typescript": "^5.7.3",
"vite": "^6.0.11"
},
"resolutions": {
"@moq/net": "0.3.1"
},
"packageManager": "yarn@4.6.0"
}
14 changes: 7 additions & 7 deletions translation-demo/service/uv.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

63 changes: 42 additions & 21 deletions translation-demo/src/hooks/useMoqConnection.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,9 @@ export const useMoqConnection = () => {

const [connectionSignal] = useState(
() =>
new Signal<Publish.Lite.Connection.Established | undefined>(undefined),
new Signal<Publish.Net.Connection.Established | undefined>(undefined),
);

const reloadRef = useRef<Publish.Lite.Connection.Reload | null>(null);
const sessionCleanupRef = useRef<(() => void) | null>(null);
const streamRef = useRef<RemoteStream | null>(null);
const translationProvidersRef = useRef(
Expand All @@ -75,7 +74,9 @@ export const useMoqConnection = () => {
}

const activeLanguages = new Set(
Object.keys(provider.broadcast.catalog.peek()?.audio?.renditions ?? {}),
Object.keys(
provider.broadcast.out.catalog.peek()?.audio?.renditions ?? {},
),
);

for (const language of activeLanguages) {
Expand Down Expand Up @@ -107,7 +108,7 @@ export const useMoqConnection = () => {
}
}

const catalog = current.broadcast.catalog.peek();
const catalog = current.broadcast.out.catalog.peek();
const hasVideo =
!!catalog?.video &&
Object.keys(catalog.video.renditions ?? {}).length > 0;
Expand Down Expand Up @@ -154,15 +155,15 @@ export const useMoqConnection = () => {
const broadcast = new Watch.Broadcast({
connection: connectionSignal,
enabled: true,
name: Publish.Lite.Path.from(path),
name: Publish.Net.Path.from(path),
});

const sync = () => {
refreshStream();
};

const disposeCatalog = broadcast.catalog.subscribe(sync);
const disposeStatus = broadcast.status.subscribe(sync);
const disposeCatalog = broadcast.out.catalog.subscribe(sync);
const disposeStatus = broadcast.out.status.subscribe(sync);

streamRef.current = {
path,
Expand All @@ -187,18 +188,17 @@ export const useMoqConnection = () => {

const broadcast = new Watch.Broadcast({
connection: connectionSignal,
announced: reloadRef.current?.announced,
enabled: true,
name: Publish.Lite.Path.from(path),
name: Publish.Net.Path.from(path),
reload: true,
});

const sync = () => {
refreshStream();
};

const disposeCatalog = broadcast.catalog.subscribe(sync);
const disposeStatus = broadcast.status.subscribe(sync);
const disposeCatalog = broadcast.out.catalog.subscribe(sync);
const disposeStatus = broadcast.out.status.subscribe(sync);

translationProvidersRef.current.set(path, {
path,
Expand All @@ -220,7 +220,6 @@ export const useMoqConnection = () => {
const disconnect = useCallback(() => {
sessionCleanupRef.current?.();
sessionCleanupRef.current = null;
reloadRef.current = null;
connectionSignal.set(undefined);
setConnectionStatus("disconnected");
setHasSession(false);
Expand All @@ -234,13 +233,11 @@ export const useMoqConnection = () => {
setHasSession(true);
setConnectionStatus("connecting");

const reload = new Publish.Lite.Connection.Reload({
const reload = new Publish.Net.Connection.Reload({
enabled: true,
url: connectionUrl,
});

reloadRef.current = reload;

const disposeStatus = reload.status.subscribe((value) => {
setConnectionStatus(value);
});
Expand All @@ -249,7 +246,7 @@ export const useMoqConnection = () => {
connectionSignal.set(connection);
});

const syncStream = (announced: Set<Publish.Lite.Path.Valid>) => {
const syncStream = (announced: Iterable<Publish.Net.Path.Valid>) => {
let streamPath: string | undefined;
const providerPaths = new Map<
string,
Expand Down Expand Up @@ -295,14 +292,38 @@ export const useMoqConnection = () => {
refreshStream();
};

const disposeDiscovery = reload.announced.subscribe((announced) => {
syncStream(announced);
});
const announcements = reload.announced();
const announced = new Map<string, Publish.Net.Path.Valid>();
let discoveryCancelled = false;

void (async () => {
try {
for (;;) {
const event = await announcements.next();
if (!event || discoveryCancelled) {
break;
}

const path = event.path.toString();
if (event.active) {
announced.set(path, event.path);
} else {
announced.delete(path);
}
syncStream(announced.values());
}
} catch (error) {
if (!discoveryCancelled) {
console.error("MoQ broadcast discovery failed", error);
}
}
})();

syncStream(reload.announced.peek());
syncStream([]);

sessionCleanupRef.current = () => {
disposeDiscovery();
discoveryCancelled = true;
announcements.close();
disposeEstablished();
disposeStatus();
reload.close();
Expand Down
56 changes: 32 additions & 24 deletions translation-demo/src/hooks/usePublisher.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ export const usePublisher = () => {
);
const [connectionSignal] = useState(
() =>
new Signal<Publish.Lite.Connection.Established | undefined>(undefined),
new Signal<Publish.Net.Connection.Established | undefined>(undefined),
);

const [streamName, setStreamName] = useState<string | null>(null);
Expand All @@ -44,12 +44,13 @@ export const usePublisher = () => {
// Holds the teardown for the active session; its presence also means "already publishing".
const sessionCleanupRef = useRef<(() => void) | null>(null);

const cameraDevices = useSignalValue(camera.device.available) ?? [];
const microphoneDevices = useSignalValue(microphone.device.available) ?? [];
const selectedCameraId = useSignalValue(camera.device.active);
const selectedMicrophoneId = useSignalValue(microphone.device.active);
const cameraDevices = useSignalValue(camera.device.out.available) ?? [];
const microphoneDevices =
useSignalValue(microphone.device.out.available) ?? [];
const selectedCameraId = useSignalValue(camera.device.out.active);
const selectedMicrophoneId = useSignalValue(microphone.device.out.active);

const cameraTrack = useSignalValue(camera.source);
const cameraTrack = useSignalValue(camera.out.source);
const previewStream = useMemo(
() => (cameraTrack ? new MediaStream([cameraTrack]) : null),
[cameraTrack],
Expand Down Expand Up @@ -109,7 +110,7 @@ export const usePublisher = () => {
return;
}

const reload = new Publish.Lite.Connection.Reload({
const reload = new Publish.Net.Connection.Reload({
enabled: true,
url: connectionUrl,
});
Expand All @@ -120,35 +121,42 @@ export const usePublisher = () => {
});

// Single-segment broadcast leaf; the `translations` namespace is in the connection URL.
// The hd/sd video and audio encoders must be explicitly enabled (with a config),
// The video and audio encoders must be explicitly enabled (with a config),
// otherwise the catalog publishes with no active tracks and the relay aborts it.
const capture = new Publish.Video.Capture({ source: camera.out.source });
const broadcast = new Publish.Broadcast({
connection: connectionSignal,
enabled: true,
name: Publish.Lite.Path.from(name),
audio: {
enabled: microphone.enabled,
source: microphone.source,
name: Publish.Net.Path.from(name),
display: capture.out.display,
});
const video = new Publish.Video.Encoder("video/hd", {
broadcast,
capture,
enabled: true,
config: {
maxPixels: 1280 * 720,
maxBitrate: 1_000_000,
frameRate: 30,
},
video: {
source: camera.source,
// Single 720p rendition; the lower-quality sd encoder stays off.
hd: {
enabled: camera.enabled,
config: {
maxPixels: 1280 * 720,
maxBitrate: 1_000_000,
frameRate: 30,
},
},
sd: { enabled: false },
});
const audio = new Publish.Audio.Encoder("audio", {
broadcast,
codec: {
mime: "opus",
usedtx: false,
},
enabled: true,
source: microphone.out.source,
});

sessionCleanupRef.current = () => {
disposeEstablished();
disposeStatus();
audio.close();
video.close();
broadcast.close();
capture.close();
reload.close();
};
}, [
Expand Down
Loading