video is working! some features missing

packages/renderer/src/components/Video.tsx

@@ -0,0 +1,21 @@
+import React from "react";
+import { useEffect, useState } from "react";
+
+export function Video(props: React.VideoHTMLAttributes<HTMLVideoElement> & {
+  srcObject?: MediaStream;
+}) {
+
+  const [ref, setRef] = useState<HTMLVideoElement | null>(null);
+
+  useEffect(() => {
+    if (ref === null)
+      return;
+    if (props.srcObject === undefined || props.srcObject === null)
+      return;
+    ref.srcObject = props.srcObject;
+  }, [props.srcObject, ref]);
+
+  const filteredProps = Object.fromEntries(Object.entries(props).filter(([key, value]) => key !== 'srcObject'));
+
+  return <video ref={setRef} {...filteredProps}>{props.children}</video>;
+}

packages/renderer/src/components/Voice.tsx

@@ -1,8 +1,9 @@
-import React from "react";
+import React, { useLayoutEffect } from "react";
 import { MouseEventHandler, ReactNode, useCallback, useContext, useEffect, useState } from "react"
-import { MdHeadphones, MdMic, MdMicOff, MdPhoneDisabled, MdPhoneInTalk, MdScreenShare, MdSend, MdVideocam } from "react-icons/md";
+import { MdHeadphones, MdMic, MdMicOff, MdScreenShare, MdSend, MdVideoCall, MdVideocam, MdVideocamOff } from "react-icons/md";
+import { FiLogOut, FiLogIn } from 'react-icons/fi';
 import { ClientsListContext } from "../contexts/EphemeralState/ClientsListState";
-import { Connection, PeerContext } from "../contexts/EphemeralState/PeerState";
+import { IParticipant, IConnection, PeerContext } from "../contexts/EphemeralState/PeerState";
 import { UserMediaContext } from "../contexts/EphemeralState/UserMediaState";
 import useChannel from "../hooks/useChannel";
 import { useApi } from "../lib/useApi";
@@ -24,33 +25,50 @@ export default function Voice() {
   const { clientId } = useClientId();
 
   const {
-    mute, unmute, muted, enable, mediaStream
+    mute,
+    unmute,
+    muted,
+    enable,
+    disable,
+    mediaStream,
+    cameraEnabled,
+    enableCamera,
+    disableCamera
   } = useContext(UserMediaContext);
 
   const [connectedVoiceClientIds, setConnectedVoiceClientIds] = useState<string[]>([])
+  const [participants, setParticipants] = useState<IParticipant[]>([]);
 
   const { send } = useApi({
-    'voice:list'(data: { participants: Connection[] }) {
+    'voice:list'(data: { uid: string, participants: IParticipant[] }) {
-      setConnectedVoiceClientIds(data.participants.map(c => c.clientId));
+      if(data.uid !== channel) return;
+      setParticipants(data.participants);
     },
-    'voice:join'(data: Connection) {
+    'voice:join'(data: IParticipant) {
-      setConnectedVoiceClientIds(ids => ([...ids, data.clientId]));
+      if(data.channelId !== channel) return;
+      setParticipants(ps => ([...ps, data]));
     },
-    'voice:leave'(data: Connection) {
+    'voice:leave'(data: IParticipant) {
-      setConnectedVoiceClientIds(ids => ids.filter(id => data.clientId !== id));
+      if(data.channelId !== channel) return;
+      setParticipants(ps => ps.filter(p => p.peerId !== data.peerId));
     }
-  });
+  }, [channel]);
-
-  useEffect(() => {
-    console.log(connectedVoiceClientIds);
-  }, [connectedVoiceClientIds])
 
   useEffect(() => {
     send('voice:list', { channelId: channel })
-  }, [channel])
+  }, [channel]);
 
   const joinCall = useCallback(() => {
     if(peerId === null || connected === false || channel === null) return;
+    disableCamera();
+    enable();
+    join(channel);
+    send('voice:join', { peerId, channelId: channel });
+  }, [connected, peerId, channel]);
+
+  const joinCallWithVideo = useCallback(() => {
+    if(peerId === null || connected === false || channel === null) return;
+    enableCamera();
     enable();
     join(channel);
     send('voice:join', { peerId, channelId: channel });
@@ -59,6 +77,7 @@ export default function Voice() {
   const leaveCall = useCallback(() => {
     if(peerId === null || connected === false) return;
     leave();
+    disable();
     send('voice:leave', { peerId, channelId: channel });
   }, [connected, peerId, channel]);
 
@@ -81,11 +100,18 @@ export default function Voice() {
         display: 'inline',
       }}>
         {(!inThisCall) ? (
-          <CircleButton
+          <>
-            icon={MdPhoneInTalk}
+            <CircleButton
-            onClick={joinCall}
+              icon={FiLogIn}
-            color="var(--green)"
+              onClick={joinCall}
-          ></CircleButton>
+              color="var(--green)"
+            ></CircleButton>
+            <CircleButton
+              icon={MdVideoCall}
+              onClick={joinCallWithVideo}
+              color="var(--green)"
+            ></CircleButton>
+          </>
         ) : (
           <>
             <CircleButton
@@ -102,11 +128,12 @@ export default function Voice() {
               onClick={leaveCall}
             ></CircleButton>
             <CircleButton
-              icon={MdVideocam}
+              icon={cameraEnabled ? MdVideocam : MdVideocamOff}
-              onClick={leaveCall}
+              onClick={() => cameraEnabled ? disableCamera() : enableCamera()}
+              inverted={!cameraEnabled}
             ></CircleButton>
             <CircleButton
-              icon={MdPhoneDisabled}
+              icon={FiLogOut}
               onClick={leaveCall}
               color="var(--red)"
             ></CircleButton>
@@ -121,20 +148,21 @@ export default function Voice() {
       width: '100%'
     }}>
 
-      {connectedVoiceClientIds.length === 0 ? (
+      {participants.length === 0 ? (
         <span style={{ color: 'var(--neutral-6)', fontWeight: '600' }}>No one is here right now</span>
       ) : (
         <div style={{
 
         }}>
-          {connectedVoiceClientIds.map(id => {
+          {participants.map(participant => {
-            const connection = connections.find(c => c.clientId === id);
+            const connection = connections.find(c => c.clientId === participant.clientId);
-            const isMe = clientId === id;
+            
-            const stream = (isMe ? mediaStream : connection?.mediaStream) ?? undefined
+            // if(participant.clientId !== clientId) return <div key={participant.peerId}></div>;
+
             return (
               <Participant
-                name={clientName[id]}
+                key={participant.peerId}
-                stream={stream}
+                data={connection ?? participant}
               ></Participant>
             )
           })}
@@ -145,57 +173,67 @@ export default function Voice() {
 }
 
 function Participant(props: {
-  name: string,
+  data: IParticipant | IConnection
-  stream?: MediaStream
 }) {
+
+  const [videoRoot, setVideoRoot] = useState<HTMLDivElement | null>(null);
+  const { videoElement } = useContext(UserMediaContext);
+  const { clientName } = useContext(ClientsListContext);
+
+  const isSelf = useClientId().clientId === props.data.clientId;
+  const remoteVideoElement = isSelf ? (
+    videoElement
+  ) : (
+    ('videoElement' in props.data) ? (
+      props.data.videoElement
+    ) : (
+      null
+    )
+  );
+
+  useLayoutEffect(() => {
+    if(videoRoot === null) return;
+    if(remoteVideoElement === null) return;
+
+    const alreadyThere = [...videoRoot.childNodes].includes(remoteVideoElement);
+
+    if(!alreadyThere) {
+      while(!!videoRoot.firstChild) {
+        videoRoot.firstChild.remove();
+      }
+      videoRoot.appendChild(remoteVideoElement);
+    }
+    remoteVideoElement.play();
+  }, [videoRoot, remoteVideoElement]);
+
   return (
     <div style={{
-      width: '200px',
-      height: '150px',
       display: 'inline-block',
-      placeItems: 'center center',
-      borderRadius: '8px',
-      background: 'var(--neutral-4)',
-      color: 'var(--neutral-8)',
-      fontStyle: '500',
-      margin: '4px',
       verticalAlign: 'top',
-      overflow: 'hidden',
+      margin: '4px',
     }}>
-      <Video autoPlay muted style={{
+      <div style={{
-        width: '100%'
+        width: '200px',
-      }} srcObject={props.stream}></Video>
+        height: '150px',
-      {/* <div style={{
+        display: 'inline-block',
-        display: 'grid',
-        width: '100%',
-        height: '100%',
         placeItems: 'center center',
+        borderRadius: '8px',
+        background: isSelf ? 'var(--orange)' : 'var(--neutral-4)',
+        color: 'var(--neutral-8)',
+        fontStyle: '500',
+        overflow: 'hidden',
       }}>
-        {props.name}
+        <div ref={setVideoRoot} style={{
-      </div> */}
+          height: '100%'
+        }}></div>
+      </div>
+      <div style={{
+        textAlign: 'center'
+      }}>{clientName[props.data.clientId]}</div>
     </div>
   )
 }
 
-function Video(props: React.VideoHTMLAttributes<HTMLVideoElement> & {
-  srcObject?: MediaStream
-}) {
-
-  const [ref, setRef] = useState<HTMLVideoElement | null>(null);
-
-  useEffect(() => {
-    if(ref === null) return;
-    if(props.srcObject === undefined || props.srcObject === null) return;
-    console.log(ref);
-    ref.srcObject = props.srcObject;
-  }, [props.srcObject, ref]);
-
-  const filteredProps = Object.fromEntries(Object.entries(props).filter(([key, value]) => key !== 'srcObject'));
-
-  return <video ref={setRef} {...filteredProps}>{props.children}</video>
-}
-
-
 function CircleButton(props: {
   onClick: MouseEventHandler<HTMLDivElement>,
   icon: IconType,
@@ -203,6 +241,8 @@ function CircleButton(props: {
   inverted?: boolean,
 }) {
 
+  const primaryColor = props.inverted ? 'var(--neutral-9)' : (props.color ?? 'var(--neutral-4)');
+
   return (
     <div style={{
       display: 'inline-block',
@@ -213,7 +253,7 @@ function CircleButton(props: {
       boxSizing: 'border-box',
     }}>
       <div onClick={props.onClick} style={{
-        background: props.inverted ? 'var(--neutral-9)' : (props.color ?? 'var(--neutral-4)'),
+        background: primaryColor,
         width: '100%',
         height: '100%',
         borderRadius: '50%',

packages/renderer/src/components/useLog.tsx

@@ -0,0 +1,7 @@
+import { useEffect } from "react";
+
+export const useLog = (v: any, prefix = '') => {
+  useEffect(() => {
+    console.log(prefix, v);
+  }, [v]);
+};

packages/renderer/src/contexts/EphemeralState/PeerState.tsx

@@ -1,9 +1,12 @@
-import { createContext, useCallback, useContext, useEffect, useMemo, useRef, useState } from "react";
+import { createContext, ReactNode, useCallback, useContext, useEffect, useMemo, useRef, useState } from "react";
 import { Peer, MediaConnection } from "peerjs";
 import { UserMediaContext } from "./UserMediaState";
 import { useApi } from "/@/lib/useApi";
 import { Audio } from "/@/components/Audio";
 import { sfx } from "/@/lib/sound";
+import { Video } from '../../components/Video';
+import React from "react";
+import { useLog } from "/@/components/useLog";
 
 export const PeerContext = createContext<{
   connected: boolean;
@@ -11,7 +14,7 @@ export const PeerContext = createContext<{
   peerId: string | null;
   join: (channelId: string) => void;
   leave: () => void;
-  connections: Connection[];
+  connections: IConnection[];
   connectedChannel: string | null;
 }>({
   connected: false,
@@ -33,14 +36,18 @@ function useCurrent<T>(thing: T) {
   return thingRef.current;
 }
 
-export interface Connection {
+export interface IParticipant {
   peerId: string;
   clientId: string;
   channelId: string;
+}
+
+export interface IConnection extends IParticipant {
   call: MediaConnection | null;
   isCaller: boolean;
   mediaStream: MediaStream | null;
   connected: boolean;
+  videoElement: HTMLVideoElement | null;
 }
 
 function isCaller(a: string, b:string) {
@@ -59,7 +66,7 @@ export default function PeerState(props: any) {
   const [incomingCalls, setIncomingCalls] = useState<MediaConnection[]>([]);
   const [outgoingCalls, setOutgoingCalls] = useState<string[]>([]);
 
-  const [connections, setConnections] = useState<Connection[]>([]);
+  const [connections, setConnections] = useState<IConnection[]>([]);
   const [channel, setChannel] = useState<string | null>(null);
 
   const addIncomingCall = useCurrent(useCallback((call: MediaConnection) => {
@@ -71,7 +78,7 @@ export default function PeerState(props: any) {
     setIncomingCalls(calls => calls.filter(call => !peerIds.includes(call.peer)));
   }
 
-  const updateConnection = (peerId: string, data: Partial<Connection>) => {
+  const updateConnection = (peerId: string, data: Partial<IConnection>) => {
     setConnections(connections => connections.map(connection => {
       if(connection.peerId !== peerId) return connection;
       return {
@@ -86,19 +93,63 @@ export default function PeerState(props: any) {
   }
 
   const destroyConnection = (peerId: string) => {
-    setConnections(connections => connections.filter(connection => {
+    setConnections(connections => {
-      if(connection.peerId !== peerId) return true;
+      const conn = connections.find(c => c.peerId === peerId)
-      if(connection.call) {
+      if(conn && conn.call) {
-        connection.call.close();
+        conn.call.close();
-        removeConnection(peerId);
       }
-    }))
+      return connections;
+    })
+    removeConnection(peerId);
   }
 
   const addStream = (id: string, stream: MediaStream) => {
-    updateConnection(id, { mediaStream: stream, connected: true });
+    // DE BOUNCE THE INCOMING STREAMS, CAUSE WTF?!
+    setConnections(connections => {
+      const connection = connections.find(c => c.peerId === id);
+      if(!!connection && connection.mediaStream === null) {
+        return connections.map(connection => {
+          if(connection.peerId !== id) return connection;
+          if(connection.mediaStream !== null) return connection;
+          console.log('CREATED VIDEO ELEMENT');
+          const videoElement = document.createElement('video');
+          videoElement.srcObject = stream;
+          videoElement.autoplay = true;
+          videoElement.muted = true;
+          videoElement.style.height = '100%';
+          
+          return {
+            ...connection,
+            connected: true,
+            mediaStream: stream,
+            videoElement
+          }
+        })
+      } else {
+        return connections;
+      }
+    });
   }
 
+  // replace mediastream in connections when mediaStream changes.
+  useEffect(() => {
+    if(mediaStream === null) return;
+    setConnections(connections => {
+      for(const conn of connections) {
+        if(conn.call === null) continue;
+        for(const sender of conn.call.peerConnection.getSenders()) {
+          if(sender.track === null) continue;
+          if(sender.track.kind === 'audio') {
+            sender.replaceTrack(mediaStream.getAudioTracks()[0]);
+          } else if(sender.track.kind === 'video') {
+            sender.replaceTrack(mediaStream.getVideoTracks()[0]);
+          }
+        }
+      }
+      return connections;
+    })
+  }, [mediaStream])
+
   // accept / reject incoming calls
   useEffect(() => {
     if(incomingCalls.length === 0) return;
@@ -130,19 +181,20 @@ export default function PeerState(props: any) {
   }, [outgoingCalls, mediaStream, peer]);
 
   const { send } = useApi({
-    'voice:join'(data: any) {
+    'voice:join'(data: IParticipant) {
       if(data.channelId !== channel) return;
       if(data.peerId === peerId) return;
       if(peerId === null) return;
       sfx.joinCall();
-      const newConn: Connection = {
+      const newConn: IConnection = {
         call: null,
         connected: false,
         clientId: data.clientId,
         peerId: data.peerId,
         channelId: data.channelId,
         isCaller: isCaller(peerId, data.peerId),
-        mediaStream: null
+        mediaStream: null,
+        videoElement: null
       };
       if(newConn.isCaller) {
         setOutgoingCalls(c => [...c, data.peerId]);
@@ -152,39 +204,44 @@ export default function PeerState(props: any) {
         newConn
       ]))
     },
-    'voice:leave'(data: any) {
+    'voice:leave'(data: IParticipant) {
       sfx.leaveCall();
       if(data.channelId !== channel) return;
       if(data.peerId === peerId) return;
-      setConnections(connections => connections.filter(connection => (
+      destroyConnection(data.peerId);
-        connection.channelId !== data.channelId ||
-        connection.clientId !== data.clientId ||
-        connection.peerId !== data.peerId
-      )));
     },
-    'voice:list'(data: any) {
+    'voice:list'(data: { uid: string, participants: IParticipant[]}) {
       if(data.uid !== channel) return;
       if(peerId === null) return;
+      if(connections.length !== 0) return;
+
       setConnections(connections => {
+        console.log(connections);
         return data.participants
-        .filter((p: any) => p.peerId !== peerId)
+          .filter((p) => p.peerId !== peerId)
-        .map((participant: any) => {
+          .map((participant) => {
-          const previousCall = null;
+            const previousCall = null;
-          const caller = isCaller(peerId, participant.peerId);
+            const caller = isCaller(peerId, participant.peerId);
-          if(caller) {
+            if(caller) {
-            setOutgoingCalls(c => [...c, participant.peerId]);
+              setOutgoingCalls(c => [...c, participant.peerId]);
-          }
+            }
-          return {
+            const newConnection: IConnection = {
-            ...participant,
+              ...participant,
-            call: null,
+              call: null,
-            isCaller: caller
+              isCaller: caller,
-          }
+              mediaStream: null,
-        })
+              connected: false,
+              videoElement: null
+            }
+            return newConnection
+          })
       });
     }
-  }, [channel, peerId]);
+  }, [channel, peerId, connections]);
 
+  useLog(connections[0], 'connections');
 
+  // create and maintain a peer connection
   useEffect(() => {
     if(connected) return;
     if(peer !== null) return;
@@ -212,6 +269,7 @@ export default function PeerState(props: any) {
   const joinChannel = (channelId: string) => {
     sfx.joinCall();
     setChannel(channelId);
+    setConnections([]);
     send('voice:list', { channelId });
   }
 
@@ -233,9 +291,12 @@ export default function PeerState(props: any) {
     <div>
       {connections.map(conn => (
         (conn.mediaStream !== null) && (
-          <div key={conn.peerId}>
+          <Audio
-            <Audio autoPlay hidden srcObject={conn.mediaStream}></Audio>
+            key={conn.peerId}
-          </div>
+            autoPlay
+            hidden
+            srcObject={conn.mediaStream}
+          ></Audio>
         )
       ))}
     </div>

packages/renderer/src/contexts/EphemeralState/UserMediaState.tsx

@@ -1,4 +1,6 @@
-import { createContext, useCallback, useMemo, useState } from "react";
+import React from "react";
+import { createContext, ReactNode, useCallback, useEffect, useMemo, useState } from "react";
+import { Video } from "/@/components/Video";
 
 export const UserMediaContext = createContext<{
   enabled: boolean;
@@ -11,6 +13,7 @@ export const UserMediaContext = createContext<{
   enableCamera: () => void;
   disableCamera: () => void;
   cameraEnabled: boolean;
+  videoElement: HTMLVideoElement | null;
 }>({
   enabled: false,
   mediaStream: null,
@@ -22,6 +25,7 @@ export const UserMediaContext = createContext<{
   enableCamera: () => {},
   disableCamera: () => {},
   cameraEnabled: false,
+  videoElement: null,
 });
 
 export default function UserMediaState(props: any) {
@@ -30,27 +34,60 @@ export default function UserMediaState(props: any) {
   const [enabled, setEnabled] = useState(false);
   const [muted, setMuted] = useState(false);
   const [cameraEnabled, setCameraEnabled] = useState(false);
+  const [videoElement, setVideoElement] = useState<HTMLVideoElement | null>(null);
 
-  const enable = useCallback(async () => {
+  const createBlankVideoTrack = () => {
-    const newStream = await navigator.mediaDevices.getUserMedia({
+    const canvas = document.createElement('canvas');
-      audio: true,
+    canvas.width = 40;
-      video: true,
+    canvas.height = 30;
+
+    return canvas.captureStream(60).getVideoTracks()[0];
+  }
+
+  const updateMediaStream = (mediaStream: MediaStream | null) => {
+    setMediaStream(old => {
+      if(old !== null) {
+        for(const track of old.getTracks()) {
+          track.stop();
+        }
+      }
+      return mediaStream;
     });
-
+    if(mediaStream !== null) {
-    setMediaStream(newStream);
+      const videoElement = document.createElement('video');
-    setEnabled(true);
+      videoElement.muted = true;
-  }, []);
+      videoElement.autoplay = true;
-
+      videoElement.srcObject = mediaStream;
-  const disable = useCallback(async () => {
+      videoElement.style.height = '100%';
-    if(mediaStream === null) return;
+      setVideoElement(videoElement);
-
+    } else {
-    for(const track of mediaStream?.getTracks()) {
+      setVideoElement(null);
-      track.stop();
     }
+  }
 
-    setMediaStream(null);
+  // maintaining the mediaStream...
-    setEnabled(false);
+  useEffect(() => {
-  }, [mediaStream]);
+    (async () => {
+      if(enabled) {
+        const newStream = await navigator.mediaDevices.getUserMedia({
+          audio: true,
+          video: cameraEnabled,
+        });
+
+        if(!cameraEnabled) {
+          newStream.addTrack(createBlankVideoTrack());
+        }
+
+        if(muted) {
+          newStream.getAudioTracks()[0].enabled = false;
+        }
+    
+        updateMediaStream(newStream);
+      } else {
+        updateMediaStream(null);
+      }
+    })()
+  }, [enabled, cameraEnabled]);
 
   const mute = () => {
     if(mediaStream === null) return;
@@ -67,14 +104,16 @@ export default function UserMediaState(props: any) {
   const value = useMemo(() => ({
     enabled,
     mediaStream,
-    enable,
+    enable: () => setEnabled(true),
-    disable,
+    disable: () => setEnabled(false),
     mute,
     unmute,
-    muted
+    muted,
-  }), [enabled, mediaStream, enable, disable, muted]);
+    enableCamera: () => setCameraEnabled(true),
-
+    disableCamera: () => setCameraEnabled(false),
-
+    cameraEnabled,
+    videoElement
+  }), [enabled, mediaStream, muted]);
 
   return <UserMediaContext.Provider value={value}>
     {props.children}

packages/renderer/src/lib/useApi.tsx

@@ -1,12 +1,14 @@
-import { useContext, useEffect } from 'react';
+import { useContext, useEffect, useMemo } from 'react';
 import { ServerConnectionContext } from '../components/ServerConnection';
 import useSessionToken from '../hooks/useSessionToken';
 import { Router, router, RouterObject } from './api';
+import { v4 } from 'uuid';
 
 export function useApi(actions: Router | RouterObject = {}, deps: any[] = []) {
   const connection = useContext(ServerConnectionContext);
   const _router = typeof actions === 'object' ? router(actions) : actions;
   const { sessionToken } = useSessionToken();
+  const componentId = useMemo(() => { return v4() }, []);
 
   useEffect(() => {
     connection.registerRouter(_router);
@@ -23,7 +25,8 @@ export function useApi(actions: Router | RouterObject = {}, deps: any[] = []) {
       }
       connection.send(action, {
         ...(data ?? {}),
-        sessionToken
+        sessionToken,
+        $componentId: componentId
       });
     }
   };