(core) integrate semantic search pod

apps/backend/src/modules/course/controller.ts

@@ -14,6 +14,10 @@ export const getCourse = async (subject: string, number: string) => {
   return formatCourse(course as ICourseItem);
 };
 
+// TODO: Query courses by embedded search
+//       Get topK subject and number back from Weaviate
+//       Query and return courses from MongoDB
+
 export const getClassesByCourse = async (courseId: string) => {
   const classes = await ClassModel.find({
     courseId,

apps/backend/src/modules/course/resolver.ts

@@ -47,6 +47,8 @@ const resolvers: CourseModule.Resolvers = {
       return course as unknown as CourseModule.Course;
     },
 
+    // TODO: Add semantic search query resolver
+
     courses: async (_, _arguments, _context, _info) => {
       const courses = getCourses();
 

apps/backend/src/modules/course/typedefs/course.ts

@@ -7,6 +7,8 @@ export default gql`
   type Query {
     course(subject: String!, number: CourseNumber!): Course
     courses: [Course!]!
+
+    # TODO: Add semantic search query
   }
 
   type Course {

apps/datapuller/src/pullers/courses.ts

@@ -43,6 +43,9 @@ const updateCourses = async ({
   const allCourses = await getCourses(log, COURSE_APP_ID, COURSE_APP_KEY);
   const courses = getLatestUniqueCourses(allCourses);
 
+  // TODO: Add embeddings for courses to Weaviate alongside any metadata necessary to query courses
+  //       Use the best model from https://github.com/asuc-octo/semantic-search-fall-2024
+
   log.info(`Fetched ${courses.length.toLocaleString()} courses.`);
   if (!courses) {
     log.warn("No courses found, skipping update.");

docker-compose.yml

@@ -27,6 +27,28 @@ services:
       - ./apps/frontend/src:/frontend/apps/frontend/src
       - ./packages/theme/src:/frontend/packages/theme/src
       - ./apps/frontend/public:/frontend/apps/frontend/public
+  weaviate:
+    command:
+    - --host
+    - 0.0.0.0
+    - --port
+    - '8080'
+    - --scheme
+    - http
+    image: cr.weaviate.io/semitechnologies/weaviate:1.27.2
+    ports:
+    - 8081:8080
+    - 50051:50051
+    volumes:
+    - weaviate_data:/var/lib/weaviate
+    restart: on-failure:0
+    environment:
+      QUERY_DEFAULTS_LIMIT: 25
+      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
+      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
+      DEFAULT_VECTORIZER_MODULE: 'none'
+      ENABLE_API_BASED_MODULES: 'true'
+      CLUSTER_HOSTNAME: 'node1'
   mongodb:
     image: mongo:7.0.5
     command: ["--replSet", "rs0", "--bind_ip_all", "--port", "27017"]
@@ -63,3 +85,5 @@ services:
     networks:
       - bt
     restart: always
+volumes:
+  weaviate_data: