init-my-chrome-backend-frontend

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda",
+    "python-envs.pythonProjects": []
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Andrew Yang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1 @@
+chrome-plugin

backend/.gitignore

@@ -0,0 +1 @@
+.idea

backend/Procfile

@@ -0,0 +1 @@
+web: gunicorn --worker-class eventlet -w 1 app:app

backend/app.py

@@ -0,0 +1,120 @@
+from flask import Flask, jsonify, request
+from flask_socketio import SocketIO, emit
+from flask_cors import CORS
+from flask_sqlalchemy import SQLAlchemy
+from datetime import datetime, timedelta
+from threading import Thread
+import time
+import random
+
+app = Flask(__name__)
+CORS(app)
+socketio = SocketIO(app, cors_allowed_origins="*")
+app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///pairing_codes.db'
+db = SQLAlchemy(app)
+
+
+class PairingCode(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    code = db.Column(db.String(80), unique=True, nullable=False)
+    session_id = db.Column(db.String(120), unique=True, nullable=True)
+    timestamp = db.Column(db.DateTime, default=db.func.current_timestamp())
+    last_heartbeat = db.Column(db.DateTime, default=db.func.current_timestamp())
+
+
+with app.app_context():
+    db.create_all()
+
+
+def cleanup_expired_codes():
+    while True:
+        with app.app_context():
+            expired_time = datetime.utcnow() - timedelta(minutes=15)
+            expired_codes = PairingCode.query.filter(PairingCode.last_heartbeat < expired_time).all()
+
+            for code in expired_codes:
+                db.session.delete(code)
+            db.session.commit()
+
+        time.sleep(600)
+
+
+cleanup_thread = Thread(target=cleanup_expired_codes)
+cleanup_thread.start()
+
+
+@app.route('/generate_code', methods=['GET'])
+def generate_code():
+    while True:
+        code = str(random.randint(1000, 9999))
+        existing_code = PairingCode.query.filter_by(code=code).first()
+        if not existing_code:
+            break
+
+    new_code = PairingCode(code=code)
+    db.session.add(new_code)
+    db.session.commit()
+
+    return jsonify({"code": code})
+
+
+@socketio.on('pair')
+def handle_pairing(json):
+    code = json.get('code')
+    pairing_code = PairingCode.query.filter_by(code=code).first()
+
+    if pairing_code:
+        pairing_code.session_id = request.sid
+
+        PairingCode.query.filter(PairingCode.session_id == request.sid, PairingCode.id != pairing_code.id).delete()
+
+        db.session.commit()
+        emit('paired', {'status': 'success'}, room=request.sid)
+    else:
+        emit('error', {'status': 'invalid_code'}, room=request.sid)
+
+
+@socketio.on('heartbeat')
+def handle_heartbeat(json):
+    code = json.get('code')
+    pairing_code = PairingCode.query.filter_by(code=code).first()
+
+    if pairing_code:
+        now_without_ms = datetime.utcnow().replace(microsecond=0)
+        pairing_code.last_heartbeat = now_without_ms
+        db.session.commit()
+
+
+@app.route('/validate_code', methods=['POST'])
+def validate_code():
+    data = request.json
+    code = data.get('code')
+
+    pairing_code = PairingCode.query.filter_by(code=code).first()
+
+    if pairing_code and pairing_code.last_heartbeat:
+        time_since_last_heartbeat = datetime.utcnow() - pairing_code.last_heartbeat
+        if time_since_last_heartbeat < timedelta(minutes=15):
+            return jsonify({'status': 'valid'})
+        else:
+            return jsonify({'status': 'expired'}), 400
+    else:
+        return jsonify({'status': 'invalid'}), 400
+
+@app.route('/send_gesture', methods=['POST'])
+def handle_gesture():
+    data = request.json
+    code = data.get('code')
+    gesture_value = data.get('gesture')
+
+    pairing_code = PairingCode.query.filter_by(code=code).first()
+
+    if pairing_code and pairing_code.session_id:
+        socketio.emit('gesture_event', {'gesture': gesture_value}, room=pairing_code.session_id)
+        return jsonify({'status': 'success'})
+    else:
+        return jsonify({'status': 'code_not_paired'}), 400
+
+
+if __name__ == '__main__':
+    socketio.run(app, debug=True)

backend/requirements.txt

@@ -0,0 +1,24 @@
+bidict==0.22.1
+blinker==1.7.0
+click==8.1.7
+dnspython==2.4.2
+eventlet==0.34.2
+Flask==3.0.0
+Flask-Cors==4.0.0
+Flask-SocketIO==5.3.6
+Flask-SQLAlchemy==3.1.1
+greenlet==3.0.3
+gunicorn==21.2.0
+h11==0.14.0
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.3
+packaging==23.2
+python-engineio==4.8.0
+python-socketio==5.10.0
+simple-websocket==1.0.0
+six==1.16.0
+SQLAlchemy==2.0.41
+typing_extensions==4.13.1
+Werkzeug==3.0.1
+wsproto==1.2.0

backend/runtime.txt

@@ -0,0 +1 @@
+python-3.10.13

frontend/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "gesture-presenter-frontend",
+  "version": "0.1.0",
+  "private": true,
+  "dependencies": {
+    "@fortawesome/fontawesome-svg-core": "^6.5.1",
+    "@fortawesome/free-brands-svg-icons": "^6.5.1",
+    "@fortawesome/free-solid-svg-icons": "^6.5.1",
+    "@fortawesome/react-fontawesome": "^0.2.0",
+    "@mediapipe/drawing_utils": "^0.3.1675466124",
+    "@mediapipe/hands": "^0.4.1675469240",
+    "@testing-library/jest-dom": "^5.17.0",
+    "@testing-library/react": "^13.4.0",
+    "@testing-library/user-event": "^13.5.0",
+    "@types/jest": "^27.5.2",
+    "@types/node": "^16.18.68",
+    "@types/react": "^18.2.45",
+    "@types/react-dom": "^18.2.18",
+    "axios": "^1.6.2",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-scripts": "5.0.1",
+    "socket.io-client": "^4.7.2",
+    "typescript": "^4.9.5",
+    "web-vitals": "^2.1.4"
+  },
+  "scripts": {
+    "start": "react-scripts start",
+    "build": "react-scripts build",
+    "test": "react-scripts test",
+    "eject": "react-scripts eject",
+    "predeploy": "npm run build",
+    "deploy": "gh-pages -d build"
+  },
+  "eslintConfig": {
+    "extends": [
+      "react-app",
+      "react-app/jest"
+    ]
+  },
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  },
+  "devDependencies": {
+    "gh-pages": "^6.1.1"
+  },
+  "homepage": "https://gesturepresenter.com/"
+}

frontend/public/index.html

@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
+    <meta name="theme-color" content="#000000" />
+    <meta
+      name="description"
+      content="Web site created using create-react-app"
+    />
+    <link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
+    <!--
+      manifest.json provides metadata used when your web app is installed on a
+      user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
+    -->
+    <link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
+    <!--
+      Notice the use of %PUBLIC_URL% in the tags above.
+      It will be replaced with the URL of the `public` folder during the build.
+      Only files inside the `public` folder can be referenced from the HTML.
+
+      Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+      work correctly both with client-side routing and a non-root public URL.
+      Learn how to configure a non-root public URL by running `npm run build`.
+    -->
+    <title>Gesture Presenter</title>
+  </head>
+  <body>
+    <noscript>You need to enable JavaScript to run this app.</noscript>
+    <div id="root"></div>
+    <!--
+      This HTML file is a template.
+      If you open it directly in the browser, you will see an empty page.
+
+      You can add webfonts, meta tags, or analytics to this file.
+      The build step will place the bundled scripts into the <body> tag.
+
+      To begin the development, run `npm start` or `yarn start`.
+      To create a production bundle, use `npm run build` or `yarn build`.
+    -->
+  </body>
+</html>

frontend/public/manifest.json

@@ -0,0 +1,25 @@
+{
+  "short_name": "Gesture Presenter",
+  "name": "Control Google Slides With Just Hand Gestures!",
+  "icons": [
+    {
+      "src": "favicon.ico",
+      "sizes": "64x64 32x32 24x24 16x16",
+      "type": "image/x-icon"
+    },
+    {
+      "src": "logo192.png",
+      "type": "image/png",
+      "sizes": "192x192"
+    },
+    {
+      "src": "logo512.png",
+      "type": "image/png",
+      "sizes": "512x512"
+    }
+  ],
+  "start_url": ".",
+  "display": "standalone",
+  "theme_color": "#000000",
+  "background_color": "#ffffff"
+}

frontend/public/robots.txt

@@ -0,0 +1,3 @@
+# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Disallow:

frontend/src/App.css

@@ -0,0 +1,280 @@
+* {
+  margin: 0;
+  padding: 0;
+  box-sizing: border-box;
+}
+
+html {
+  box-sizing: border-box;
+}
+
+*, *::before, *::after {
+  box-sizing: inherit;
+}
+
+body {
+  font-family: 'Open Sans', sans-serif;
+  font-size: 16px;
+  line-height: 1.6;
+  color: #333;
+  background-color: #fff;
+}
+
+.modal-overlay {
+  position: fixed;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  background-color: rgba(0, 0, 0, 0.5);
+  z-index: 15;
+}
+
+.settings-modal {
+  position: fixed;
+  top: 50%;
+  left: 50%;
+  transform: translate(-50%, -50%);
+  z-index: 20;
+  background-color: rgba(0, 0, 0, 0.5);
+  color: #fff;
+  padding: 10px;
+  border-radius: 10px;
+  box-shadow: 0 5px 15px rgba(0, 0, 0, 0.3);
+  width: 80%;
+  max-width: 450px;
+  max-height: 80dvh;
+  overflow-y: auto;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: flex-start;
+}
+
+.title-container {
+  display: flex;
+  flex-direction: row !important;
+  align-items: center;
+  justify-content: center;
+}
+
+.title-container h2 {
+  color: #fff;
+  text-align: center;
+  font-size: 1.5rem;
+  margin-right: 10px;
+}
+
+.logo {
+  width: 30px;
+  margin-bottom: 5px;
+  height: auto;
+}
+
+.instructions {
+  color: #fff;
+  text-align: center;
+  margin-bottom: 10px;
+}
+
+.error-message {
+  color: #ff8080;
+  background-color: rgba(255, 107, 107, 0.1);
+  border: 1px solid #ff8080;
+  padding: 8px;
+  margin-top: 10px;
+  margin-bottom: 20px;
+  border-radius: 4px;
+  text-align: center;
+  font-size: 0.9rem;
+  width: calc(100% - 20px) !important;
+}
+
+.safari-note {
+  color: #fff;
+  font-size: 0.9rem;
+  font-style: italic;
+  text-align: center;
+  margin: 10px 0;
+}
+
+.settings-modal div {
+  width: 100%;
+  margin: 5px 0;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+.settings-modal label {
+  margin-bottom: 0.5rem;
+  color: #fff;
+  font-weight: bold;
+}
+
+.settings-modal input[type="tel"] {
+  width: calc(100% - 20px);
+  padding: 10px;
+  border: 1px solid #ddd;
+  border-radius: 5px;
+  background-color: rgba(51, 51, 51, 0.5);
+  color: #fff;
+  font-size: 1rem;
+  text-align: center;
+  margin-bottom: 10px;
+}
+
+.swap-container {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  border: 1px solid #ddd;
+  background-color: rgba(51, 51, 51, 0.5);
+  border-radius: 5px;
+  cursor: pointer;
+  flex-direction: row !important;
+  margin: 0 0 10px 0!important;
+  width: calc(100% - 20px) !important;
+}
+
+.swap-label {
+  font-weight: bold;
+  text-align: center;
+  width: 64px;
+}
+
+.instruction-container {
+  display: flex;
+  flex-direction: row !important;
+  justify-content: space-evenly;
+  align-items: center;
+  width: 100%;
+}
+
+.instruction-img {
+  width: 40px;
+  height: auto;
+  filter: invert();
+  padding-right: 5px;
+}
+
+.img-flip {
+  -webkit-transform: scaleX(-1);
+  transform: scaleX(-1);
+  padding-right: 0;
+  padding-left: 5px;
+}
+
+.settings-modal input[type="checkbox"] {
+  width: 1.75rem;
+  height: 1.75rem;
+  cursor: pointer;
+}
+
+.settings-modal .submit-button {
+  background-color: #007bff;
+  color: #fff;
+  border: none;
+  padding: 10px 15px;
+  border-radius: 5px;
+  cursor: pointer;
+  transition: background-color 0.3s ease;
+  font-size: 1rem;
+  width: 50%;
+  margin: 10px 0;
+  opacity: 1;
+}
+
+.settings-modal .submit-button.disabled {
+  background-color: #007bff;
+  opacity: 0.7;
+  cursor: not-allowed;
+}
+
+.additional-links {
+  position: fixed;
+  bottom: 10px;
+  right: 10px;
+  border-radius: 10px;
+  background-color: rgba(0, 0, 0, 0.5);
+  display: flex;
+  flex-direction: row;
+  justify-content: space-evenly;
+  width: 110px;
+  box-shadow: 0 5px 15px rgba(0, 0, 0, 0.3);
+}
+
+.additional-links a {
+  color: white;
+  font-size: 35px;
+  text-decoration: none;
+  margin: 0 5px;
+}
+
+.camera-feed,
+.canvas-overlay {
+  position: absolute;
+  top: 0;
+  left: 0;
+}
+
+.canvas-overlay {
+  z-index: 10;
+}
+
+.camera-container {
+  position: relative;
+  width: 100%;
+  height: 100dvh;
+  overflow: hidden;
+}
+
+.camera-feed, .canvas-overlay {
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+}
+
+.camera-feed.user-facing {
+  transform: scaleX(-1);
+}
+
+.camera-button {
+  position: absolute;
+  font-size: 2.5rem;
+  z-index: 10;
+  background-color: rgba(0, 0, 0, 0.5);
+  color: #fff;
+  border: none;
+  border-radius: 50%;
+  padding: 15px;
+  width: 4.5rem;
+  cursor: pointer;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.show-settings {
+  bottom: 20px;
+  left: 20px;
+}
+
+.switch-camera {
+  bottom: 20px;
+  right: 20px;
+}
+
+@media screen and (orientation: landscape) {
+  .switch-camera {
+    top: 20px;
+    right: 20px;
+    bottom: auto;
+  }
+
+  .show-settings {
+    bottom: 20px;
+    right: 20px;
+    left: auto;
+  }
+}

frontend/src/App.test.tsx

@@ -0,0 +1,9 @@
+import React from 'react';
+import { render, screen } from '@testing-library/react';
+import App from './App';
+
+test('renders learn react link', () => {
+  render(<App />);
+  const linkElement = screen.getByText(/learn react/i);
+  expect(linkElement).toBeInTheDocument();
+});

frontend/src/App.tsx

@@ -0,0 +1,59 @@
+import React, { useState } from 'react';
+import Camera from './Camera';
+import SettingsModal from "./SettingsModal";
+import {FontAwesomeIcon} from "@fortawesome/react-fontawesome";
+import {faChrome, faGithub} from "@fortawesome/free-brands-svg-icons";
+
+const App: React.FC = () => {
+  const [flipped, setFlipped] = useState(false);
+  const [isModalVisible, setIsModalVisible] = useState(true);
+  const [pairingCode, setPairingCode] = useState('');
+
+  const showModal = () => {
+    setIsModalVisible(true);
+  };
+
+  const hideModal = () => {
+    setIsModalVisible(false);
+  };
+
+  const handlePairingCodeSubmit = (code: string) => {
+    console.log("Pairing code submitted:", code);
+    setPairingCode(code);
+    setIsModalVisible(false);
+  };
+
+  const handleToggleGestureSwap = () => {
+    setFlipped(!flipped);
+  };
+
+  return (
+    <div>
+      <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
+      {isModalVisible && <div className="modal-overlay">
+        <div className="additional-links">
+          <a href="https://chromewebstore.google.com/detail/twiblocker-video-adblocke/mdohdkncgoaamplcaokhmlppgafhlima" target="_blank" rel="noopener noreferrer" title="Download Chrome Extension">
+            <FontAwesomeIcon icon={faChrome} />
+          </a>
+          <a href="https://github.com/AnonymousAAArdvark/GesturePresenter" target="_blank" rel="noopener noreferrer" title="GitHub Repository">
+            <FontAwesomeIcon icon={faGithub} />
+          </a>
+        </div>
+      </div>}
+      {isModalVisible &&  <SettingsModal
+        onPairingCodeSubmit={handlePairingCodeSubmit}
+        onToggleGestureSwap={handleToggleGestureSwap}
+        flipped={flipped}
+        pairingCode={pairingCode}
+      />}
+      <Camera
+        pairingCode={pairingCode}
+        flipped={flipped}
+        onSettingsClick={showModal}
+        modalVisible={isModalVisible}
+      />
+    </div>
+  );
+};
+
+export default App;

frontend/src/Camera.tsx

@@ -0,0 +1,251 @@
+import React, { useState, useEffect, useRef } from 'react';
+import axios from "axios";
+import { Hands, HAND_CONNECTIONS, Results, VERSION } from '@mediapipe/hands';
+import { drawConnectors, drawLandmarks } from '@mediapipe/drawing_utils';
+import { getGesture } from './utils/GestureDetection'
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
+import { faArrowsRotate, faGear } from '@fortawesome/free-solid-svg-icons';
+import './App.css';
+
+
+interface CameraProps {
+  pairingCode: string;
+  flipped: boolean;
+  onSettingsClick: () => void;
+  modalVisible: boolean;
+}
+
+const Camera: React.FC<CameraProps> = ({ pairingCode, flipped, onSettingsClick, modalVisible }) => {
+  const [isUserFacing, setIsUserFacing] = useState(true);
+  const videoRef = useRef<HTMLVideoElement>(null);
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const handsRef = useRef<Hands | null>(null);
+  const isSendingRef = useRef(false);
+  const intervalIdRef = useRef<number | undefined>(undefined);
+  const [gestureResult, setGestureResult] = useState(0);
+
+  useEffect(() => {
+    if (intervalIdRef.current !== undefined) {
+      clearInterval(intervalIdRef.current);
+      intervalIdRef.current = undefined;
+    }
+
+    const handleGestureStable = () => {
+      if (gestureResult !== 0 && pairingCode !== '' && !modalVisible) {
+        let gestureValue = gestureResult === 1 ? 'Right' : 'Left';
+        const url = 'https://gesture-presenter-bc9d819e6d43.herokuapp.com/send_gesture';
+        axios.post(url, {
+          code: pairingCode,
+          gesture: gestureValue
+        })
+          .then(response => {
+            console.log('Gesture sent successfully:', response.data);
+          })
+          .catch(error => {
+            console.error('Error sending gesture:', error);
+            onSettingsClick();
+          });
+      }
+    };
+
+    let secondTimeoutId: NodeJS.Timeout;
+
+    if (gestureResult !== 0) {
+      const timeoutId = setTimeout(() => {
+        handleGestureStable();
+
+        secondTimeoutId = setTimeout(() => {
+          intervalIdRef.current = window.setInterval(handleGestureStable, 500);
+        }, 250);
+      }, 150);
+
+      return () => {
+        clearTimeout(timeoutId);
+        if (secondTimeoutId !== undefined) {
+          clearTimeout(secondTimeoutId);
+        }
+        if (intervalIdRef.current !== undefined) {
+          clearInterval(intervalIdRef.current);
+          intervalIdRef.current = undefined;
+        }
+      };
+    }
+  }, [gestureResult]);
+
+  useEffect(() => {
+    const getUserMedia = async () => {
+      try {
+        const stream = await navigator.mediaDevices.getUserMedia({
+          video: {
+            facingMode: isUserFacing ? "user" : "environment",
+            width: { ideal: 500 },
+            height: { ideal: 500 }
+          }
+        });
+        const video = videoRef.current;
+        if (video) {
+          video.srcObject = stream;
+          video.onloadedmetadata = () => {
+            video.play().then(() => {
+              if (canvasRef.current) {
+                canvasRef.current.width = video.videoWidth;
+                canvasRef.current.height = video.videoHeight;
+                initializeMediaPipe();
+              }
+            });
+          };
+        }
+      } catch (err) {
+        console.error("Error accessing media devices:", err);
+      }
+    };
+
+    const handleResize = () => {
+      setTimeout(() => {
+        const video = videoRef.current;
+        if (video && canvasRef.current) {
+          canvasRef.current.width = video.videoWidth;
+          canvasRef.current.height = video.videoHeight;
+        }
+      }, 1000); // might not be very reliable
+    };
+
+    getUserMedia();
+    window.addEventListener('resize', handleResize);
+
+    return () => {
+      window.addEventListener('resize', handleResize);
+
+      const stream = videoRef.current?.srcObject as MediaStream;
+      stream?.getTracks().forEach(track => track.stop());
+      if (handsRef.current) {
+        handsRef.current.close();
+        handsRef.current = null;
+      }
+    };
+  }, [isUserFacing, flipped]);
+
+  const initializeMediaPipe = () => {
+    if (handsRef.current) {
+      handsRef.current.close();
+    }
+    const hands = new Hands({
+      locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/hands@${VERSION}/${file}`,
+    });
+
+    hands.setOptions({
+      modelComplexity: 1,
+      minDetectionConfidence: 0.5,
+      minTrackingConfidence: 0.5,
+      maxNumHands: 6,
+    });
+
+    hands.onResults(onResults);
+    handsRef.current = hands;
+    sendToMediaPipe();
+  };
+
+  const sendToMediaPipe = async () => {
+    if (isSendingRef.current) return;
+    isSendingRef.current = true;
+
+    if (videoRef.current && handsRef.current) {
+      try {
+        await handsRef.current.send({ image: videoRef.current });
+      } catch (error) {
+        console.error('Error in sendToMediaPipe:', error);
+        setTimeout(sendToMediaPipe, 500);
+      } finally {
+        isSendingRef.current = false;
+      }
+    } else {
+      isSendingRef.current = false;
+    }
+  };
+
+  const onResults = (results: Results) => {
+    const canvas = canvasRef.current;
+    const ctx = canvas?.getContext('2d');
+    const video = videoRef.current;
+    if (!canvas || !ctx || !video) return;
+
+    const videoWidth = video.videoWidth;
+    const videoHeight = video.videoHeight;
+
+    ctx.save();
+    ctx.clearRect(0, 0, canvas.width, canvas.height);
+    if (isUserFacing) {
+      ctx.translate(canvas.width, 0);
+      ctx.scale(-1, 1);
+    }
+    ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
+
+    let vote = 0;
+    results.multiHandLandmarks?.forEach(landmarks => {
+      let scaledLandmarks = landmarks.map(landmark => ({
+        x: landmark.x * videoWidth,
+        y: landmark.y * videoHeight,
+        z: landmark.z
+      }));
+
+      let gestureResult = getGesture(scaledLandmarks);
+
+      if ((flipped || isUserFacing) && flipped !== isUserFacing) {
+        gestureResult = gestureResult === "Left" ? "Right" : gestureResult === "Right" ? "Left" : gestureResult;
+      }
+
+      vote += (gestureResult === "Right") ? 1 : (gestureResult === "Left") ? -1 : 0;
+
+
+      const handStyles: { [key: string]: { connectorStyle: any, landmarkStyle: any } } = {
+        'Right': {
+          connectorStyle: { color: '#00FF00', lineWidth: 1 },
+          landmarkStyle: { color: '#00FF00', radius: 1 }
+        },
+        'Left': {
+          connectorStyle: { color: '#FF0000', lineWidth: 1 },
+          landmarkStyle: { color: '#FF0000', radius: 1 }
+        },
+        'None': {
+          connectorStyle: { color: '#00BFFF', lineWidth: .5 },
+          landmarkStyle: { color: '#00BFFF', radius: .5 }
+        }
+      };
+
+      const currentStyle = handStyles[gestureResult] || handStyles['None'];
+
+      drawConnectors(ctx, landmarks, HAND_CONNECTIONS, currentStyle.connectorStyle);
+      drawLandmarks(ctx, landmarks, currentStyle.landmarkStyle);
+
+    });
+    setGestureResult(vote)
+
+    ctx.restore();
+    requestAnimationFrame(sendToMediaPipe);
+  };
+
+  const flipCamera = () => {
+    setIsUserFacing(!isUserFacing);
+  }
+
+  return (
+    <div className="camera-container">
+      <canvas ref={canvasRef} className="canvas-overlay" />
+      <video ref={videoRef} className={`camera-feed ${isUserFacing ? 'user-facing' : ''}`} autoPlay playsInline />
+
+      {!modalVisible && (
+        <button onClick={onSettingsClick} className="camera-button show-settings">
+          <FontAwesomeIcon icon={faGear} />
+        </button>
+      )}
+
+      {!modalVisible && (
+        <button onClick={flipCamera} className="camera-button switch-camera">
+          <FontAwesomeIcon icon={faArrowsRotate} />
+        </button>
+      )}
+    </div>
+  );
+};
+
+export default Camera;

frontend/src/SettingsModal.tsx

@@ -0,0 +1,146 @@
+import React, { useState, useEffect } from 'react';
+import axios from "axios";
+import logo from './assets/logo.png';
+import instruction from './assets/instruction.png';
+
+interface SettingsModalProps {
+  onPairingCodeSubmit: (code: string) => void;
+  onToggleGestureSwap: () => void;
+  flipped: boolean;
+  pairingCode: string;
+}
+
+interface PairingResponse {
+  status: string;
+  message?: string;
+}
+
+const SettingsModal: React.FC<SettingsModalProps> = ({
+   onPairingCodeSubmit,
+   onToggleGestureSwap,
+   flipped,
+   pairingCode,
+ }) => {
+  const [pairingCodeInput, setPairingCodeInput] = useState<string>(pairingCode);
+  const [errorMessage, setErrorMessage] = useState<string>('');
+
+  useEffect(() => {
+    if (pairingCode !== '') {
+      validatePairingCode(pairingCodeInput)
+        .then(isValid => {
+          if (isValid) {
+            console.log('Pairing code is valid');
+            setErrorMessage('');
+          } else {
+            console.log('Pairing code no longer available');
+            setErrorMessage('Pairing code no longer available. Please try another code.');
+            setPairingCodeInput('');
+          }
+        })
+        .catch(error => {
+          console.error('Error validating code:', error);
+          setErrorMessage('Error validating pairing code. Please try again.');
+        });
+    }
+  }, [pairingCode]);
+
+  const handlePairingCodeInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const input = e.target.value;
+    if (input.match(/^\d{0,4}$/)) {
+      setPairingCodeInput(input);
+      setErrorMessage('');
+    }
+  };
+
+  const validatePairingCode = async (code: string): Promise<boolean> => {
+    try {
+      const response = await axios.post('https://gesture-presenter-bc9d819e6d43.herokuapp.com/validate_code', { code });
+      return response.data.status === 'valid';
+    } catch (error) {
+      console.error('Error validating code:', error);
+      return false;
+    }
+  };
+
+  const handleKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
+    if (e.key === 'Enter' && pairingCodeInput.length === 4) {
+      handleSubmit();
+    }
+  };
+
+  const handleSubmit = async () => {
+    const isValid = await validatePairingCode(pairingCodeInput);
+    if (isValid) {
+      console.log('Pairing code is valid');
+      setErrorMessage('');
+      onPairingCodeSubmit(pairingCodeInput);
+    } else {
+      console.log('Invalid pairing code');
+      setErrorMessage('Invalid pairing code. Please try again.');
+      setPairingCodeInput('');
+    }
+  };
+
+  const isMobileSafari = () => {
+    return /iP(ad|od|hone)/i.test(navigator.platform) && /Safari/i.test(navigator.userAgent) && !/CriOS/i.test(navigator.userAgent);
+  };
+
+  return (
+    <div className="settings-modal">
+      <div className="title-container">
+        <h2>GesturePresenter</h2>
+        <img src={logo} alt="Logo" className="logo" />
+      </div>
+      <p className="instructions">Enter your pairing code and adjust gesture settings. </p>
+      {errorMessage && <div className="error-message">{errorMessage}</div>}
+      <div>
+        <label htmlFor="pairingCode">Pairing Code:</label>
+        <input
+          id="pairingCode"
+          type="tel"
+          pattern="[0-9]*"
+          inputMode="numeric"
+          value={pairingCodeInput}
+          onChange={handlePairingCodeInputChange}
+          onKeyDown={handleKeyDown}
+          autoComplete="off"
+        />
+      </div>
+      <label htmlFor="gestureSwap">Swap Gesture Directions:</label>
+      <div className="swap-container" onClick={onToggleGestureSwap}>
+        <div className="instruction-container">
+          <img src={instruction} alt="Instruction Left" className="instruction-img img-flip" />
+          <span className="swap-label" style={{ color: flipped ? '#0ab20a' : '#ff0000'}}>
+            {flipped ? 'Next' : 'Back'}
+          </span>
+        </div>
+        <input
+          id="gestureSwap"
+          type="checkbox"
+          checked={flipped}
+          onChange={onToggleGestureSwap}
+        />
+        <div className="instruction-container">
+          <span className="swap-label" style={{ color: flipped ? '#ff0000' : '#0ab20a'}}>
+            {flipped ? 'Back' : 'Next'}
+          </span>
+          <img src={instruction} alt="Instruction Right" className="instruction-img" />
+        </div>
+      </div>
+      {isMobileSafari() && (
+        <p className="safari-note">iOS Safari users: hide the toolbar by tapping the
+          "<span style={{ fontSize: '.6rem' }}>A</span>A" icon on the bar,
+          and then selecting the "Hide Toolbar" option.</p>
+      )}
+      <button
+        onClick={() => handleSubmit()}
+        disabled={pairingCodeInput.length !== 4}
+        className={`submit-button ${pairingCodeInput.length !== 4 ? 'disabled' : ''}`}
+      >
+        Submit
+      </button>
+    </div>
+  );
+};
+
+export default SettingsModal;

frontend/src/index.css

@@ -0,0 +1,13 @@
+body {
+  margin: 0;
+  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
+    'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
+    sans-serif;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+code {
+  font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
+    monospace;
+}

frontend/src/index.tsx

@@ -0,0 +1,19 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import './index.css';
+import App from './App';
+import reportWebVitals from './reportWebVitals';
+
+const root = ReactDOM.createRoot(
+  document.getElementById('root') as HTMLElement
+);
+root.render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+);
+
+// If you want to start measuring performance in your app, pass a function
+// to log results (for example: reportWebVitals(console.log))
+// or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals
+reportWebVitals();

frontend/src/logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 841.9 595.3"><g fill="#61DAFB"><path d="M666.3 296.5c0-32.5-40.7-63.3-103.1-82.4 14.4-63.6 8-114.2-20.2-130.4-6.5-3.8-14.1-5.6-22.4-5.6v22.3c4.6 0 8.3.9 11.4 2.6 13.6 7.8 19.5 37.5 14.9 75.7-1.1 9.4-2.9 19.3-5.1 29.4-19.6-4.8-41-8.5-63.5-10.9-13.5-18.5-27.5-35.3-41.6-50 32.6-30.3 63.2-46.9 84-46.9V78c-27.5 0-63.5 19.6-99.9 53.6-36.4-33.8-72.4-53.2-99.9-53.2v22.3c20.7 0 51.4 16.5 84 46.6-14 14.7-28 31.4-41.3 49.9-22.6 2.4-44 6.1-63.6 11-2.3-10-4-19.7-5.2-29-4.7-38.2 1.1-67.9 14.6-75.8 3-1.8 6.9-2.6 11.5-2.6V78.5c-8.4 0-16 1.8-22.6 5.6-28.1 16.2-34.4 66.7-19.9 130.1-62.2 19.2-102.7 49.9-102.7 82.3 0 32.5 40.7 63.3 103.1 82.4-14.4 63.6-8 114.2 20.2 130.4 6.5 3.8 14.1 5.6 22.5 5.6 27.5 0 63.5-19.6 99.9-53.6 36.4 33.8 72.4 53.2 99.9 53.2 8.4 0 16-1.8 22.6-5.6 28.1-16.2 34.4-66.7 19.9-130.1 62-19.1 102.5-49.9 102.5-82.3zm-130.2-66.7c-3.7 12.9-8.3 26.2-13.5 39.5-4.1-8-8.4-16-13.1-24-4.6-8-9.5-15.8-14.4-23.4 14.2 2.1 27.9 4.7 41 7.9zm-45.8 106.5c-7.8 13.5-15.8 26.3-24.1 38.2-14.9 1.3-30 2-45.2 2-15.1 0-30.2-.7-45-1.9-8.3-11.9-16.4-24.6-24.2-38-7.6-13.1-14.5-26.4-20.8-39.8 6.2-13.4 13.2-26.8 20.7-39.9 7.8-13.5 15.8-26.3 24.1-38.2 14.9-1.3 30-2 45.2-2 15.1 0 30.2.7 45 1.9 8.3 11.9 16.4 24.6 24.2 38 7.6 13.1 14.5 26.4 20.8 39.8-6.3 13.4-13.2 26.8-20.7 39.9zm32.3-13c5.4 13.4 10 26.8 13.8 39.8-13.1 3.2-26.9 5.9-41.2 8 4.9-7.7 9.8-15.6 14.4-23.7 4.6-8 8.9-16.1 13-24.1zM421.2 430c-9.3-9.6-18.6-20.3-27.8-32 9 .4 18.2.7 27.5.7 9.4 0 18.7-.2 27.8-.7-9 11.7-18.3 22.4-27.5 32zm-74.4-58.9c-14.2-2.1-27.9-4.7-41-7.9 3.7-12.9 8.3-26.2 13.5-39.5 4.1 8 8.4 16 13.1 24 4.7 8 9.5 15.8 14.4 23.4zM420.7 163c9.3 9.6 18.6 20.3 27.8 32-9-.4-18.2-.7-27.5-.7-9.4 0-18.7.2-27.8.7 9-11.7 18.3-22.4 27.5-32zm-74 58.9c-4.9 7.7-9.8 15.6-14.4 23.7-4.6 8-8.9 16-13 24-5.4-13.4-10-26.8-13.8-39.8 13.1-3.1 26.9-5.8 41.2-7.9zm-90.5 125.2c-35.4-15.1-58.3-34.9-58.3-50.6 0-15.7 22.9-35.6 58.3-50.6 8.6-3.7 18-7 27.7-10.1 5.7 19.6 13.2 40 22.5 60.9-9.2 20.8-16.6 41.1-22.2 60.6-9.9-3.1-19.3-6.5-28-10.2zM310 490c-13.6-7.8-19.5-37.5-14.9-75.7 1.1-9.4 2.9-19.3 5.1-29.4 19.6 4.8 41 8.5 63.5 10.9 13.5 18.5 27.5 35.3 41.6 50-32.6 30.3-63.2 46.9-84 46.9-4.5-.1-8.3-1-11.3-2.7zm237.2-76.2c4.7 38.2-1.1 67.9-14.6 75.8-3 1.8-6.9 2.6-11.5 2.6-20.7 0-51.4-16.5-84-46.6 14-14.7 28-31.4 41.3-49.9 22.6-2.4 44-6.1 63.6-11 2.3 10.1 4.1 19.8 5.2 29.1zm38.5-66.7c-8.6 3.7-18 7-27.7 10.1-5.7-19.6-13.2-40-22.5-60.9 9.2-20.8 16.6-41.1 22.2-60.6 9.9 3.1 19.3 6.5 28.1 10.2 35.4 15.1 58.3 34.9 58.3 50.6-.1 15.7-23 35.6-58.4 50.6zM320.8 78.4z"/><circle cx="420.9" cy="296.5" r="45.7"/><path d="M520.5 78.1z"/></g></svg>

frontend/src/react-app-env.d.ts

@@ -0,0 +1 @@
+/// <reference types="react-scripts" />

frontend/src/reportWebVitals.ts

@@ -0,0 +1,15 @@
+import { ReportHandler } from 'web-vitals';
+
+const reportWebVitals = (onPerfEntry?: ReportHandler) => {
+  if (onPerfEntry && onPerfEntry instanceof Function) {
+    import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => {
+      getCLS(onPerfEntry);
+      getFID(onPerfEntry);
+      getFCP(onPerfEntry);
+      getLCP(onPerfEntry);
+      getTTFB(onPerfEntry);
+    });
+  }
+};
+
+export default reportWebVitals;

frontend/src/setupTests.ts

@@ -0,0 +1,5 @@
+// jest-dom adds custom jest matchers for asserting on DOM nodes.
+// allows you to do things like:
+// expect(element).toHaveTextContent(/react/i)
+// learn more: https://github.com/testing-library/jest-dom
+import '@testing-library/jest-dom';

frontend/src/utils/GestureDetection.ts

@@ -0,0 +1,71 @@
+import { NormalizedLandmark, NormalizedLandmarkList } from '@mediapipe/hands';
+
+function getCoordinates(landmark: NormalizedLandmark): number[] {
+  return [landmark.x, landmark.y, landmark.z];
+}
+
+function vectorSubtract(vec1: number[], vec2: number[]): number[] {
+  return vec1.map((v, i) => v - vec2[i]);
+}
+
+function magnitude(vec: number[]): number {
+  return Math.sqrt(vec.reduce((acc, val) => acc + val * val, 0));
+}
+
+function dotProduct(vec1: number[], vec2: number[]): number {
+  return vec1.reduce((acc, v, i) => acc + v * vec2[i], 0);
+}
+
+function cosineSimilarity(vec1: number[], vec2: number[]): number {
+  return dotProduct(vec1, vec2) / (magnitude(vec1) * magnitude(vec2));
+}
+
+function euclideanDistance(coord1: number[], coord2: number[]): number {
+  return Math.sqrt(coord1.reduce((acc, v, i) => acc + Math.pow(v - coord2[i], 2), 0));
+}
+
+function determineHandOrientation(landmark0: NormalizedLandmark, landmark9: NormalizedLandmark): string {
+  const [x0, y0] = getCoordinates(landmark0);
+  const [x9, y9] = getCoordinates(landmark9);
+
+  const xd = x9 - x0;
+  const yd = y9 - y0;
+
+  if (xd > 0 && -2 <= yd / xd && yd / xd <= -0.05) return "Right";
+  if (xd < 0 && 0.05 <= yd / xd && yd / xd <= 2) return "Left";
+  return "None";
+}
+
+function isFingerClosed(base: NormalizedLandmark, knuckle: NormalizedLandmark, joint: NormalizedLandmark, tip: NormalizedLandmark): boolean {
+  const baseCoords = getCoordinates(base);
+  return euclideanDistance(getCoordinates(tip), baseCoords) <
+    1.2 * euclideanDistance(getCoordinates(knuckle), baseCoords) &&
+    euclideanDistance(getCoordinates(tip), baseCoords) <
+    euclideanDistance(getCoordinates(joint), baseCoords);
+}
+function isThumbPointerExtended(thumb: NormalizedLandmark[], pointer: NormalizedLandmark[]): boolean {
+  const vecThumb = vectorSubtract(getCoordinates(thumb[0]), getCoordinates(thumb[1]));
+  if (vecThumb[1] / Math.abs(vecThumb[0]) < .75) return false;
+
+  const vecPointer = pointer.map((p, i, arr) =>
+    (i < arr.length - 1 ? vectorSubtract(getCoordinates(p), getCoordinates(arr[i + 1])) : [0, 0, 0]));
+  const pointerStraight = cosineSimilarity(vecPointer[0], vecPointer[1]) > 0.95 &&
+    cosineSimilarity(vecPointer[1], vecPointer[2]) > 0.95;
+  const thumbPointerOrthogonal = cosineSimilarity(vecThumb, vecPointer[0]) < 0.85;
+
+  return pointerStraight && thumbPointerOrthogonal;
+}
+
+function getGesture(lmList: NormalizedLandmarkList): string {
+  if (!lmList || lmList.length === 0) return "None";
+
+  const thumbExtended = isThumbPointerExtended([lmList[2], lmList[3]], [lmList[5], lmList[6], lmList[7], lmList[8]]);
+  if (!thumbExtended) return "None";
+
+  const closedFingers = [9, 13, 17].every(i => isFingerClosed(lmList[0], lmList[i], lmList[i + 2], lmList[i + 3]));
+  if (!closedFingers) return "None";
+
+  return determineHandOrientation(lmList[0], lmList[9]);
+}
+
+export { getGesture };

frontend/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": [
+      "dom",
+      "dom.iterable",
+      "esnext"
+    ],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noFallthroughCasesInSwitch": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "react-jsx"
+  },
+  "include": [
+    "src"
+  ]
+}