deps

Dockerfile

@@ -51,11 +51,11 @@ RUN cd frontend && npm install
 
 # Copy frontend app and build
 COPY --chown=user frontend/ frontend/
-#RUN cd frontend && npm run build
+RUN cd frontend && npm run build
 
 # Copy everything else
 COPY --chown=user app/ app/
 COPY --chown=user server.py server.py
 
-#ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
+ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
 CMD ["python3", "server.py"]

app/bot.py

@@ -30,13 +30,14 @@ class DailyVision(EventHandler):
         self.__pipeline = Pipeline
         self.__camera = None
         self.__time = time.time()
-        self.__queue = queue.Queue()
+        #self.__queue = queue.Queue()
         self.__app_quit = False
         self.__image_buffer = None
         self.__bot_name = bot_name
         self.__room_url = room_url
         self.__room_name = room_name
         self.__expiration = expiration
+        self.__params = Pipeline.InputParams()
         self.__idle = idle
 
         # Create the pipeline (this might take a moment)
@@ -101,7 +102,7 @@ class DailyVision(EventHandler):
             })
 
     def process_frames(self):
-        params = Pipeline.InputParams()
+        params = self.__params
 
         while not self.__app_quit:
             # Is anyone watching?
@@ -114,6 +115,7 @@ class DailyVision(EventHandler):
                 self.logger.info(f"Expiration timer exceeded. Exiting...")
                 self.__app_quit = True
                 break
+            """
             try:
                 #video_frame = self.__queue.get(timeout=5)
                 video_frame = self.__image_buffer
@@ -124,7 +126,8 @@ class DailyVision(EventHandler):
                     self.__camera.write_frame(result_image.tobytes())
             except queue.Empty:
                 pass
-
+            """
+            
     def on_video_frame(self, participant_id, video_frame):
         # Process ~15 frames per second (considering incoming frames at 30fps).
         if time.time() - self.__time > float(os.getenv("FPS_CAP", 0.0333)):
@@ -132,16 +135,18 @@ class DailyVision(EventHandler):
             self.__image_buffer = video_frame
             #self.__queue.put(video_frame)
 
-    def wave(self, emoji="👋"):
+    def on_app_message(self, message, sender):
+        # Update pipeline settings based on message data
+        print(message)
+        self.__params = self.__pipeline.InputParams(**message)
+        print(self.__params)
+        #print(self.__pipeline.Info())
+        return
+
+    def wave(self):
         self.__client.send_app_message(
             {
-                "event": "sync-emoji-reaction",
-                "reaction": {
-                    "emoji": emoji,
-                    "room": "main-room",
-                    "sessionId": "bot",
-                    "id": time.time(),
-                },
+                "prompt": self.__params.prompt,
             }
         )
 

frontend/app/components/Call.js

@@ -1,17 +1,18 @@
 "use client";
 
-import { useCallback, useEffect, useState } from "react";
-import Card from "../components/Card";
-import { VideoCameraIcon, PaintBrushIcon } from "@heroicons/react/24/outline";
-import Avatar from "../components/Avatar";
-import CreateRoom from "../components/CreateRoom";
-import { apiUrl } from "../utils";
-import Join from "../components/Joining";
 import {
   DailyVideo,
+  useAppMessage,
   useLocalSessionId,
   useParticipantIds,
 } from "@daily-co/daily-react";
+import { PaintBrushIcon, VideoCameraIcon } from "@heroicons/react/24/outline";
+import { useCallback, useEffect, useState } from "react";
+import Avatar from "../components/Avatar";
+import Card from "../components/Card";
+import CreateRoom from "../components/CreateRoom";
+import Join from "../components/Joining";
+import { apiUrl } from "../utils";
 
 const STATE_IDLE = "idle";
 const STATE_JOINING = "joining";
@@ -24,8 +25,13 @@ export default function Call() {
   const [callState, setCallState] = useState(STATE_IDLE);
   const [roomUrl, setRoomUrl] = useState();
   const [botState, setBotState] = useState(BOT_STATE_STARTING);
+  const [params, setParams] = useState({});
+
   const localSessionId = useLocalSessionId();
   const participantIds = useParticipantIds({ filter: "remote" });
+  const sendAppMessage = useAppMessage({
+    onAppMessage: useCallback((ev) => setParams(ev.data), []),
+  });
 
   const start = useCallback(async () => {
     const resp = await fetch(`${apiUrl}/start`, {
@@ -62,11 +68,11 @@ export default function Call() {
     return <Join roomUrl={roomUrl} onJoin={() => setCallState(STATE_JOINED)} />;
   }
 
+  console.log(params);
   // Main call loop
   return (
-    <main className="container py-24">
-      {roomUrl}
-      <div className="grid grid-cols-2 grid-flow-col gap-4">
+    <main className="container py-12">
+      <div className="grid grid-cols-2 grid-flow-col gap-12">
         <div>
           <Card headerText="Local Webcam" HeaderIcon={VideoCameraIcon}>
             <div className="overflow-hidden bg-gray-50 sm:rounded-lg">
@@ -75,7 +81,31 @@ export default function Call() {
               </div>
             </div>
           </Card>
-          <div className="relative">Config - Resolution, Mbps, FPS</div>
+          <div className="relative">
+            <div>
+              <label
+                htmlFor="comment"
+                className="block text-sm font-medium leading-6 text-gray-900"
+              >
+                Add your comment
+              </label>
+              <div className="mt-2">
+                <textarea
+                  rows={4}
+                  name="comment"
+                  id="comment"
+                  className="block w-full rounded-md border-0 py-1.5 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
+                  defaultValue={params.prompt || ""}
+                />
+              </div>
+            </div>
+            Config - Resolution, Mbps, FPS
+            <button
+              onClick={() => sendAppMessage({ prompt: "Big ol' car" }, "*")}
+            >
+              Send test app message
+            </button>
+          </div>
         </div>
         <div>
           <Card headerText="Inference" HeaderIcon={PaintBrushIcon}>

frontend/app/components/CreateRoom.js

@@ -1,9 +1,11 @@
-import { ArrowRightIcon } from "@heroicons/react/20/solid";
+import { ArrowRightIcon, ChevronRightIcon } from "@heroicons/react/20/solid";
 import { useState } from "react";
 import { apiUrl } from "../utils";
 
 export default function CreateRoom({ onCreateRoom }) {
   const [fetching, setFetching] = useState(false);
+  const [roomUrl, setRoomUrl] = useState();
+  const [showManual, setShowManual] = useState(false);
 
   async function create() {
     setFetching(true);
@@ -48,6 +50,45 @@ export default function CreateRoom({ onCreateRoom }) {
             <ArrowRightIcon className="-h-5 w-5" aria-hidden="true" />
           </button>
         </div>
+
+        <hr className="my-6" />
+
+        <div
+          className="inline-flex text-sm text-gray-500 cursor-pointer"
+          onClick={() => setShowManual(!showManual)}
+        >
+          Join an existing room
+          <ChevronRightIcon
+            className={`w-5 h-5 text-gray-300 transition-transform ${
+              showManual && "rotate-90"
+            }`}
+          />
+        </div>
+
+        {showManual && (
+          <div className="mt-5 sm:flex sm:items-center">
+            <div className="w-full">
+              <label htmlFor="room" className="sr-only">
+                Daily Room URL
+              </label>
+              <input
+                type="text"
+                name="room"
+                id="room"
+                className="block w-full rounded-md border-0 py-1.5 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
+                placeholder="https://..."
+                onChange={(e) => setRoomUrl(e.target.value)}
+              />
+            </div>
+            <button
+              type="submit"
+              onClick={() => onCreateRoom(roomUrl)}
+              className="mt-3 inline-flex w-full items-center justify-center rounded-md bg-indigo-600 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-indigo-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 sm:ml-3 sm:mt-0 sm:w-auto"
+            >
+              Join
+            </button>
+          </div>
+        )}
       </div>
     </div>
   );

frontend/package.json

@@ -14,11 +14,13 @@
     "@headlessui/react": "^1.7.17",
     "@heroicons/react": "^2.0.18",
     "next": "14.0.3",
+    "nextjs": "^0.0.3",
     "react": "^18",
     "react-dom": "^18",
     "recoil": "^0.7.7"
   },
   "devDependencies": {
+    "@tailwindcss/forms": "^0.5.7",
     "autoprefixer": "^10.0.1",
     "eslint": "^8",
     "eslint-config-next": "14.0.3",

frontend/tailwind.config.js

@@ -24,5 +24,5 @@ module.exports = {
       },
     },
   },
-  plugins: [],
+  plugins: [require("@tailwindcss/forms")],
 };

frontend/yarn.lock

@@ -285,6 +285,13 @@
   dependencies:
     tslib "^2.4.0"
 
+"@tailwindcss/forms@^0.5.7":
+  version "0.5.7"
+  resolved "https://registry.yarnpkg.com/@tailwindcss/forms/-/forms-0.5.7.tgz#db5421f062a757b5f828bc9286ba626c6685e821"
+  integrity sha512-QE7X69iQI+ZXwldE+rzasvbJiyV/ju1FGHH0Qn2W3FKbuYtqp8LKcy6iSw79fVUT5/Vvf+0XgLCeYVG+UV6hOw==
+  dependencies:
+    mini-svg-data-uri "^1.2.3"
+
 "@types/json5@^0.0.29":
   version "0.0.29"
   resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
@@ -1721,6 +1728,11 @@ micromatch@^4.0.4, micromatch@^4.0.5:
     braces "^3.0.2"
     picomatch "^2.3.1"
 
+mini-svg-data-uri@^1.2.3:
+  version "1.4.4"
+  resolved "https://registry.yarnpkg.com/mini-svg-data-uri/-/mini-svg-data-uri-1.4.4.tgz#8ab0aabcdf8c29ad5693ca595af19dd2ead09939"
+  integrity sha512-r9deDe9p5FJUPZAk3A59wGH7Ii9YrjjWw0jmw/liSbHl2CHiyXj6FcDXDu2K3TjVAXqiJdaw3xxwlZZr9E6nHg==
+
 minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
@@ -1785,6 +1797,11 @@ [email protected]:
     "@next/swc-win32-ia32-msvc" "14.0.3"
     "@next/swc-win32-x64-msvc" "14.0.3"
 
+nextjs@^0.0.3:
+  version "0.0.3"
+  resolved "https://registry.yarnpkg.com/nextjs/-/nextjs-0.0.3.tgz#4f4d1d6a257be920d9b9649d4d9522c724a4e543"
+  integrity sha512-mYbDUo4/sRAZ8TqK63PCpYnFiLg7BICG/ot9+guOrUKd4/Fo71ZmEQ41IZbH6nqbQvG7SXTBuofJXAIWfNho0w==
+
 node-releases@^2.0.14:
   version "2.0.14"
   resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.14.tgz#2ffb053bceb8b2be8495ece1ab6ce600c4461b0b"