diff --git a/packages/renderer/src/components/Video.tsx b/packages/renderer/src/components/Video.tsx new file mode 100644 index 0000000..8011c07 --- /dev/null +++ b/packages/renderer/src/components/Video.tsx @@ -0,0 +1,21 @@ +import React from "react"; +import { useEffect, useState } from "react"; + +export function Video(props: React.VideoHTMLAttributes & { + srcObject?: MediaStream; +}) { + + const [ref, setRef] = useState(null); + + useEffect(() => { + if (ref === null) + return; + if (props.srcObject === undefined || props.srcObject === null) + return; + ref.srcObject = props.srcObject; + }, [props.srcObject, ref]); + + const filteredProps = Object.fromEntries(Object.entries(props).filter(([key, value]) => key !== 'srcObject')); + + return ; +} diff --git a/packages/renderer/src/components/Voice.tsx b/packages/renderer/src/components/Voice.tsx index 2c72ff0..13d46b7 100644 --- a/packages/renderer/src/components/Voice.tsx +++ b/packages/renderer/src/components/Voice.tsx @@ -1,8 +1,9 @@ -import React from "react"; +import React, { useLayoutEffect } from "react"; import { MouseEventHandler, ReactNode, useCallback, useContext, useEffect, useState } from "react" -import { MdHeadphones, MdMic, MdMicOff, MdPhoneDisabled, MdPhoneInTalk, MdScreenShare, MdSend, MdVideocam } from "react-icons/md"; +import { MdHeadphones, MdMic, MdMicOff, MdScreenShare, MdSend, MdVideoCall, MdVideocam, MdVideocamOff } from "react-icons/md"; +import { FiLogOut, FiLogIn } from 'react-icons/fi'; import { ClientsListContext } from "../contexts/EphemeralState/ClientsListState"; -import { Connection, PeerContext } from "../contexts/EphemeralState/PeerState"; +import { IParticipant, IConnection, PeerContext } from "../contexts/EphemeralState/PeerState"; import { UserMediaContext } from "../contexts/EphemeralState/UserMediaState"; import useChannel from "../hooks/useChannel"; import { useApi } from "../lib/useApi"; @@ -24,33 +25,50 @@ export default function Voice() { const { clientId } = useClientId(); const { - mute, unmute, muted, enable, mediaStream + mute, + unmute, + muted, + enable, + disable, + mediaStream, + cameraEnabled, + enableCamera, + disableCamera } = useContext(UserMediaContext); const [connectedVoiceClientIds, setConnectedVoiceClientIds] = useState([]) + const [participants, setParticipants] = useState([]); const { send } = useApi({ - 'voice:list'(data: { participants: Connection[] }) { - setConnectedVoiceClientIds(data.participants.map(c => c.clientId)); + 'voice:list'(data: { uid: string, participants: IParticipant[] }) { + if(data.uid !== channel) return; + setParticipants(data.participants); }, - 'voice:join'(data: Connection) { - setConnectedVoiceClientIds(ids => ([...ids, data.clientId])); + 'voice:join'(data: IParticipant) { + if(data.channelId !== channel) return; + setParticipants(ps => ([...ps, data])); }, - 'voice:leave'(data: Connection) { - setConnectedVoiceClientIds(ids => ids.filter(id => data.clientId !== id)); + 'voice:leave'(data: IParticipant) { + if(data.channelId !== channel) return; + setParticipants(ps => ps.filter(p => p.peerId !== data.peerId)); } - }); - - useEffect(() => { - console.log(connectedVoiceClientIds); - }, [connectedVoiceClientIds]) + }, [channel]); useEffect(() => { send('voice:list', { channelId: channel }) - }, [channel]) + }, [channel]); const joinCall = useCallback(() => { if(peerId === null || connected === false || channel === null) return; + disableCamera(); + enable(); + join(channel); + send('voice:join', { peerId, channelId: channel }); + }, [connected, peerId, channel]); + + const joinCallWithVideo = useCallback(() => { + if(peerId === null || connected === false || channel === null) return; + enableCamera(); enable(); join(channel); send('voice:join', { peerId, channelId: channel }); @@ -59,6 +77,7 @@ export default function Voice() { const leaveCall = useCallback(() => { if(peerId === null || connected === false) return; leave(); + disable(); send('voice:leave', { peerId, channelId: channel }); }, [connected, peerId, channel]); @@ -81,11 +100,18 @@ export default function Voice() { display: 'inline', }}> {(!inThisCall) ? ( - + <> + + + ) : ( <> cameraEnabled ? disableCamera() : enableCamera()} + inverted={!cameraEnabled} > @@ -121,20 +148,21 @@ export default function Voice() { width: '100%' }}> - {connectedVoiceClientIds.length === 0 ? ( + {participants.length === 0 ? ( No one is here right now ) : (
- {connectedVoiceClientIds.map(id => { - const connection = connections.find(c => c.clientId === id); - const isMe = clientId === id; - const stream = (isMe ? mediaStream : connection?.mediaStream) ?? undefined + {participants.map(participant => { + const connection = connections.find(c => c.clientId === participant.clientId); + + // if(participant.clientId !== clientId) return
; + return ( ) })} @@ -145,57 +173,67 @@ export default function Voice() { } function Participant(props: { - name: string, - stream?: MediaStream + data: IParticipant | IConnection }) { + + const [videoRoot, setVideoRoot] = useState(null); + const { videoElement } = useContext(UserMediaContext); + const { clientName } = useContext(ClientsListContext); + + const isSelf = useClientId().clientId === props.data.clientId; + const remoteVideoElement = isSelf ? ( + videoElement + ) : ( + ('videoElement' in props.data) ? ( + props.data.videoElement + ) : ( + null + ) + ); + + useLayoutEffect(() => { + if(videoRoot === null) return; + if(remoteVideoElement === null) return; + + const alreadyThere = [...videoRoot.childNodes].includes(remoteVideoElement); + + if(!alreadyThere) { + while(!!videoRoot.firstChild) { + videoRoot.firstChild.remove(); + } + videoRoot.appendChild(remoteVideoElement); + } + remoteVideoElement.play(); + }, [videoRoot, remoteVideoElement]); + return (
- - {/*
- {props.name} -
*/} +
+
+
{clientName[props.data.clientId]}
) } -function Video(props: React.VideoHTMLAttributes & { - srcObject?: MediaStream -}) { - - const [ref, setRef] = useState(null); - - useEffect(() => { - if(ref === null) return; - if(props.srcObject === undefined || props.srcObject === null) return; - console.log(ref); - ref.srcObject = props.srcObject; - }, [props.srcObject, ref]); - - const filteredProps = Object.fromEntries(Object.entries(props).filter(([key, value]) => key !== 'srcObject')); - - return -} - - function CircleButton(props: { onClick: MouseEventHandler, icon: IconType, @@ -203,6 +241,8 @@ function CircleButton(props: { inverted?: boolean, }) { + const primaryColor = props.inverted ? 'var(--neutral-9)' : (props.color ?? 'var(--neutral-4)'); + return (
{ + useEffect(() => { + console.log(prefix, v); + }, [v]); +}; diff --git a/packages/renderer/src/contexts/EphemeralState/PeerState.tsx b/packages/renderer/src/contexts/EphemeralState/PeerState.tsx index 851dc6a..0bed9ae 100644 --- a/packages/renderer/src/contexts/EphemeralState/PeerState.tsx +++ b/packages/renderer/src/contexts/EphemeralState/PeerState.tsx @@ -1,9 +1,12 @@ -import { createContext, useCallback, useContext, useEffect, useMemo, useRef, useState } from "react"; +import { createContext, ReactNode, useCallback, useContext, useEffect, useMemo, useRef, useState } from "react"; import { Peer, MediaConnection } from "peerjs"; import { UserMediaContext } from "./UserMediaState"; import { useApi } from "/@/lib/useApi"; import { Audio } from "/@/components/Audio"; import { sfx } from "/@/lib/sound"; +import { Video } from '../../components/Video'; +import React from "react"; +import { useLog } from "/@/components/useLog"; export const PeerContext = createContext<{ connected: boolean; @@ -11,7 +14,7 @@ export const PeerContext = createContext<{ peerId: string | null; join: (channelId: string) => void; leave: () => void; - connections: Connection[]; + connections: IConnection[]; connectedChannel: string | null; }>({ connected: false, @@ -33,14 +36,18 @@ function useCurrent(thing: T) { return thingRef.current; } -export interface Connection { +export interface IParticipant { peerId: string; clientId: string; channelId: string; +} + +export interface IConnection extends IParticipant { call: MediaConnection | null; isCaller: boolean; mediaStream: MediaStream | null; connected: boolean; + videoElement: HTMLVideoElement | null; } function isCaller(a: string, b:string) { @@ -59,7 +66,7 @@ export default function PeerState(props: any) { const [incomingCalls, setIncomingCalls] = useState([]); const [outgoingCalls, setOutgoingCalls] = useState([]); - const [connections, setConnections] = useState([]); + const [connections, setConnections] = useState([]); const [channel, setChannel] = useState(null); const addIncomingCall = useCurrent(useCallback((call: MediaConnection) => { @@ -71,7 +78,7 @@ export default function PeerState(props: any) { setIncomingCalls(calls => calls.filter(call => !peerIds.includes(call.peer))); } - const updateConnection = (peerId: string, data: Partial) => { + const updateConnection = (peerId: string, data: Partial) => { setConnections(connections => connections.map(connection => { if(connection.peerId !== peerId) return connection; return { @@ -86,19 +93,63 @@ export default function PeerState(props: any) { } const destroyConnection = (peerId: string) => { - setConnections(connections => connections.filter(connection => { - if(connection.peerId !== peerId) return true; - if(connection.call) { - connection.call.close(); - removeConnection(peerId); + setConnections(connections => { + const conn = connections.find(c => c.peerId === peerId) + if(conn && conn.call) { + conn.call.close(); } - })) + return connections; + }) + removeConnection(peerId); } const addStream = (id: string, stream: MediaStream) => { - updateConnection(id, { mediaStream: stream, connected: true }); + // DE BOUNCE THE INCOMING STREAMS, CAUSE WTF?! + setConnections(connections => { + const connection = connections.find(c => c.peerId === id); + if(!!connection && connection.mediaStream === null) { + return connections.map(connection => { + if(connection.peerId !== id) return connection; + if(connection.mediaStream !== null) return connection; + console.log('CREATED VIDEO ELEMENT'); + const videoElement = document.createElement('video'); + videoElement.srcObject = stream; + videoElement.autoplay = true; + videoElement.muted = true; + videoElement.style.height = '100%'; + + return { + ...connection, + connected: true, + mediaStream: stream, + videoElement + } + }) + } else { + return connections; + } + }); } + // replace mediastream in connections when mediaStream changes. + useEffect(() => { + if(mediaStream === null) return; + setConnections(connections => { + for(const conn of connections) { + if(conn.call === null) continue; + for(const sender of conn.call.peerConnection.getSenders()) { + if(sender.track === null) continue; + if(sender.track.kind === 'audio') { + sender.replaceTrack(mediaStream.getAudioTracks()[0]); + } else if(sender.track.kind === 'video') { + sender.replaceTrack(mediaStream.getVideoTracks()[0]); + } + } + } + return connections; + }) + }, [mediaStream]) + // accept / reject incoming calls useEffect(() => { if(incomingCalls.length === 0) return; @@ -130,19 +181,20 @@ export default function PeerState(props: any) { }, [outgoingCalls, mediaStream, peer]); const { send } = useApi({ - 'voice:join'(data: any) { + 'voice:join'(data: IParticipant) { if(data.channelId !== channel) return; if(data.peerId === peerId) return; if(peerId === null) return; sfx.joinCall(); - const newConn: Connection = { + const newConn: IConnection = { call: null, connected: false, clientId: data.clientId, peerId: data.peerId, channelId: data.channelId, isCaller: isCaller(peerId, data.peerId), - mediaStream: null + mediaStream: null, + videoElement: null }; if(newConn.isCaller) { setOutgoingCalls(c => [...c, data.peerId]); @@ -152,39 +204,44 @@ export default function PeerState(props: any) { newConn ])) }, - 'voice:leave'(data: any) { + 'voice:leave'(data: IParticipant) { sfx.leaveCall(); if(data.channelId !== channel) return; if(data.peerId === peerId) return; - setConnections(connections => connections.filter(connection => ( - connection.channelId !== data.channelId || - connection.clientId !== data.clientId || - connection.peerId !== data.peerId - ))); + destroyConnection(data.peerId); }, - 'voice:list'(data: any) { + 'voice:list'(data: { uid: string, participants: IParticipant[]}) { if(data.uid !== channel) return; if(peerId === null) return; + if(connections.length !== 0) return; + setConnections(connections => { + console.log(connections); return data.participants - .filter((p: any) => p.peerId !== peerId) - .map((participant: any) => { - const previousCall = null; - const caller = isCaller(peerId, participant.peerId); - if(caller) { - setOutgoingCalls(c => [...c, participant.peerId]); - } - return { - ...participant, - call: null, - isCaller: caller - } - }) + .filter((p) => p.peerId !== peerId) + .map((participant) => { + const previousCall = null; + const caller = isCaller(peerId, participant.peerId); + if(caller) { + setOutgoingCalls(c => [...c, participant.peerId]); + } + const newConnection: IConnection = { + ...participant, + call: null, + isCaller: caller, + mediaStream: null, + connected: false, + videoElement: null + } + return newConnection + }) }); } - }, [channel, peerId]); + }, [channel, peerId, connections]); + useLog(connections[0], 'connections'); + // create and maintain a peer connection useEffect(() => { if(connected) return; if(peer !== null) return; @@ -212,6 +269,7 @@ export default function PeerState(props: any) { const joinChannel = (channelId: string) => { sfx.joinCall(); setChannel(channelId); + setConnections([]); send('voice:list', { channelId }); } @@ -233,9 +291,12 @@ export default function PeerState(props: any) {
{connections.map(conn => ( (conn.mediaStream !== null) && ( -
- -
+ ) ))}
diff --git a/packages/renderer/src/contexts/EphemeralState/UserMediaState.tsx b/packages/renderer/src/contexts/EphemeralState/UserMediaState.tsx index a10bd0b..e8b8a47 100644 --- a/packages/renderer/src/contexts/EphemeralState/UserMediaState.tsx +++ b/packages/renderer/src/contexts/EphemeralState/UserMediaState.tsx @@ -1,4 +1,6 @@ -import { createContext, useCallback, useMemo, useState } from "react"; +import React from "react"; +import { createContext, ReactNode, useCallback, useEffect, useMemo, useState } from "react"; +import { Video } from "/@/components/Video"; export const UserMediaContext = createContext<{ enabled: boolean; @@ -11,6 +13,7 @@ export const UserMediaContext = createContext<{ enableCamera: () => void; disableCamera: () => void; cameraEnabled: boolean; + videoElement: HTMLVideoElement | null; }>({ enabled: false, mediaStream: null, @@ -22,6 +25,7 @@ export const UserMediaContext = createContext<{ enableCamera: () => {}, disableCamera: () => {}, cameraEnabled: false, + videoElement: null, }); export default function UserMediaState(props: any) { @@ -30,27 +34,60 @@ export default function UserMediaState(props: any) { const [enabled, setEnabled] = useState(false); const [muted, setMuted] = useState(false); const [cameraEnabled, setCameraEnabled] = useState(false); + const [videoElement, setVideoElement] = useState(null); - const enable = useCallback(async () => { - const newStream = await navigator.mediaDevices.getUserMedia({ - audio: true, - video: true, + const createBlankVideoTrack = () => { + const canvas = document.createElement('canvas'); + canvas.width = 40; + canvas.height = 30; + + return canvas.captureStream(60).getVideoTracks()[0]; + } + + const updateMediaStream = (mediaStream: MediaStream | null) => { + setMediaStream(old => { + if(old !== null) { + for(const track of old.getTracks()) { + track.stop(); + } + } + return mediaStream; }); - - setMediaStream(newStream); - setEnabled(true); - }, []); - - const disable = useCallback(async () => { - if(mediaStream === null) return; - - for(const track of mediaStream?.getTracks()) { - track.stop(); + if(mediaStream !== null) { + const videoElement = document.createElement('video'); + videoElement.muted = true; + videoElement.autoplay = true; + videoElement.srcObject = mediaStream; + videoElement.style.height = '100%'; + setVideoElement(videoElement); + } else { + setVideoElement(null); } + } - setMediaStream(null); - setEnabled(false); - }, [mediaStream]); + // maintaining the mediaStream... + useEffect(() => { + (async () => { + if(enabled) { + const newStream = await navigator.mediaDevices.getUserMedia({ + audio: true, + video: cameraEnabled, + }); + + if(!cameraEnabled) { + newStream.addTrack(createBlankVideoTrack()); + } + + if(muted) { + newStream.getAudioTracks()[0].enabled = false; + } + + updateMediaStream(newStream); + } else { + updateMediaStream(null); + } + })() + }, [enabled, cameraEnabled]); const mute = () => { if(mediaStream === null) return; @@ -67,14 +104,16 @@ export default function UserMediaState(props: any) { const value = useMemo(() => ({ enabled, mediaStream, - enable, - disable, + enable: () => setEnabled(true), + disable: () => setEnabled(false), mute, unmute, - muted - }), [enabled, mediaStream, enable, disable, muted]); - - + muted, + enableCamera: () => setCameraEnabled(true), + disableCamera: () => setCameraEnabled(false), + cameraEnabled, + videoElement + }), [enabled, mediaStream, muted]); return {props.children} diff --git a/packages/renderer/src/lib/useApi.tsx b/packages/renderer/src/lib/useApi.tsx index a1e867c..f10555e 100644 --- a/packages/renderer/src/lib/useApi.tsx +++ b/packages/renderer/src/lib/useApi.tsx @@ -1,12 +1,14 @@ -import { useContext, useEffect } from 'react'; +import { useContext, useEffect, useMemo } from 'react'; import { ServerConnectionContext } from '../components/ServerConnection'; import useSessionToken from '../hooks/useSessionToken'; import { Router, router, RouterObject } from './api'; +import { v4 } from 'uuid'; export function useApi(actions: Router | RouterObject = {}, deps: any[] = []) { const connection = useContext(ServerConnectionContext); const _router = typeof actions === 'object' ? router(actions) : actions; const { sessionToken } = useSessionToken(); + const componentId = useMemo(() => { return v4() }, []); useEffect(() => { connection.registerRouter(_router); @@ -23,7 +25,8 @@ export function useApi(actions: Router | RouterObject = {}, deps: any[] = []) { } connection.send(action, { ...(data ?? {}), - sessionToken + sessionToken, + $componentId: componentId }); } };