Improve the unit test flow and conflict name in the document (zilliztech#397)

Signed-off-by: SimFG <[email protected]>

Author: SimFG

.github/workflows/unit_test_main.yaml

@@ -68,21 +68,42 @@ jobs:
         run: |
           python3 -m spacy download en_core_web_sm
 
-      - name: Unit Tests
+      - name: Remove coverage xml
+        run: |
+          rm -rf ./coverage.xml
+
+      - name: Normal Unit Tests
         timeout-minutes: 30
         shell: bash
-        working-directory: tests
         run: |
           export IS_CI=true
           export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
-          python3 -m pytest ./
+          python3 -m pytest -k "not embedding and not processor" --cov=gptcache --cov-report xml:coverage.xml --cov-append ./tests/
 
-      - name: Generate coverage report
+      - name: Embedding Unit Tests
+        timeout-minutes: 30
+        shell: bash
         run: |
-          rm -rf ./coverage.xml
-          coverage erase
-          coverage run --source=gptcache  -m pytest ./tests
-          coverage xml
+          export IS_CI=true
+          export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+          python3 -m pytest --cov=gptcache --cov-report xml:coverage.xml --cov-append ./tests/unit_tests/embedding/
+
+      - name: Processor Unit Tests
+        timeout-minutes: 30
+        shell: bash
+        run: |
+          export IS_CI=true
+          export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+          python3 -m pytest --cov=gptcache --cov-append --cov-report xml:coverage.xml ./tests/unit_tests/processor/
+
+#      - name: Generate coverage report
+#        run: |
+#          rm -rf ./coverage.xml
+#          coverage erase
+#          coverage run --source=gptcache  -m pytest -k "not embedding and not processor" ./tests
+#          coverage run --source=gptcache  -m pytest ./tests/unit_tests/embedding/
+#          coverage run --source=gptcache  -m pytest ./tests/unit_tests/processor/
+#          coverage xml
 
       - name: Upload coverage to Codecov
         uses: codecov/[email protected]

docs/_exts/docgen2.py

@@ -5,10 +5,19 @@
 
 _default_skip_file_list = ["__init__.py"]
 _default_skip_dir_list = ["__pycache__"]
+_conflict_name_dict = {
+            "manager": ["eviction", "object_data", "scalar_data", "vector_data"]
+        }
 
 
 class DocGen:
-    def __init__(self, lib_name="gptcache", source_dir="../gptcache", output_dir="references", skip_list=[]):
+    def __init__(
+        self,
+        lib_name="gptcache",
+        source_dir="../gptcache",
+        output_dir="references",
+        skip_list=[],
+    ):
         self.lib_name = lib_name
         self.output_dir = os.path.abspath(output_dir)
         self.source_dir = os.path.abspath(source_dir)
@@ -26,8 +35,13 @@ def section_bar(input_str):
     def get_filename(input_str):
         if input_str == "gptcache":
             return input_str
-        input_str = os.path.splitext(input_str)[1][1:]
-        return input_str
+        suffix = os.path.splitext(input_str)[1][1:]
+        for conflict_dir, conflict_names in _conflict_name_dict.items():
+            for conflict_name in conflict_names:
+                if f"{conflict_dir}.{conflict_name}" in input_str:
+                    return f"{conflict_name}.{suffix}"
+
+        return suffix
 
     @staticmethod
     def cap(input_str):
@@ -37,17 +51,25 @@ def cap(input_str):
         return str.join(" ", [i.capitalize() for i in input_str.split("_")])
 
     def model_name(self, input_str: str):
-        return self.lib_name + input_str[len(self.source_dir):].replace("/", ".")
+        return self.lib_name + input_str[len(self.source_dir) :].replace("/", ".")
 
     def get_module_and_libs(self, module_dir, is_root):
         module = self.model_name(module_dir)
         libs = []
         for file in os.listdir(module_dir):
-            if os.path.isfile(os.path.join(module_dir, file)) and file not in _default_skip_file_list:
+            if (
+                os.path.isfile(os.path.join(module_dir, file))
+                and file not in _default_skip_file_list
+            ):
                 libs.append(module + "." + os.path.splitext(file)[0])
             if not is_root:
-                if os.path.isdir(os.path.join(module_dir, file)) and file not in _default_skip_dir_list:
-                    _, child_libs = self.get_module_and_libs(os.path.join(module_dir, file), False)
+                if (
+                    os.path.isdir(os.path.join(module_dir, file))
+                    and file not in _default_skip_dir_list
+                ):
+                    _, child_libs = self.get_module_and_libs(
+                        os.path.join(module_dir, file), False
+                    )
                     libs.extend(child_libs)
         if len(libs) > 0:
             sorted(libs)
@@ -57,7 +79,8 @@ def get_module_and_libs(self, module_dir, is_root):
     def generate(self):
         # Set the output directory
         env = Environment(
-            loader=FileSystemLoader(os.path.join(self.output_dir, "../_templates")), autoescape=select_autoescape()
+            loader=FileSystemLoader(os.path.join(self.output_dir, "../_templates")),
+            autoescape=select_autoescape(),
         )
 
         # Add custom filters
@@ -89,18 +112,29 @@ def generate(self):
         index_temp = env.get_template("index.rst")
 
         with open(os.path.join(self.output_dir, "index.rst"), "w") as f:
-            t = index_temp.render({"modules": [DocGen.get_filename(module) for module in modules]})
+            t = index_temp.render(
+                {"modules": [DocGen.get_filename(module) for module in modules]}
+            )
             f.write(t)
 
         # Render the function templates and write rendered output to files
         func_temp = env.get_template("function.rst")
 
         for index, module in enumerate(modules):
-            with open(os.path.join(self.output_dir, f"{DocGen.get_filename(module)}.rst"), "w") as f:
-                t = func_temp.render({"module_name": module, "funcs": [(DocGen.get_filename(lib), lib) for lib in libs[index]]})
+            with open(
+                os.path.join(self.output_dir, f"{DocGen.get_filename(module)}.rst"), "w"
+            ) as f:
+                t = func_temp.render(
+                    {
+                        "module_name": module,
+                        "funcs": [
+                            (DocGen.get_filename(lib), lib) for lib in libs[index]
+                        ],
+                    }
+                )
                 f.write(t)
 
 
 # if __name__ == "__main__":
 #     gen = DocGen(source_dir="/Users/derek/fubang/gptcache/gptcache", output_dir="/Users/derek/fubang/gptcache/docs/references")
-#     gen.generate()
+#     gen.generate()

tests/requirements.txt

@@ -1,6 +1,6 @@
 --extra-index-url https://test.pypi.org/simple/
 loguru==0.5.3
-pytest-cov==2.8.1
+pytest-cov==4.1.0
 pytest==7.2.0
 coverage==7.2.3
 pytest-assume==2.4.3
@@ -21,4 +21,6 @@ pexpect
 spacy
 safetensors
 typing_extensions<4.6.0
+stability-sdk
+grpcio==1.53.0
 protobuf==3.20.0