chore: GenAI SDK client(evals) - add RubricMetric as an alias to PrebuiltMetric in evals module, merge customizability of general quality and remove separate metric names

PiperOrigin-RevId: 799670236

Author: jsondai

README.rst

@@ -95,7 +95,7 @@ Then run evaluation by providing the inference results and specifying the metric
         metrics=[
             types.Metric(name='exact_match'),
             types.Metric(name='rouge_l_sum'),
-            types.PrebuiltMetric.TEXT_QUALITY,
+            types.RubricMetric.TEXT_QUALITY,
         ]
     )
 

tests/unit/vertexai/genai/replays/test_batch_evaluate.py

@@ -30,7 +30,7 @@ def test_batch_eval(client):
     batch_eval_operation = client.evals.batch_evaluate(
         dataset=eval_dataset,
         metrics=[
-            types.PrebuiltMetric.TEXT_QUALITY,
+            types.RubricMetric.TEXT_QUALITY,
         ],
         dest="gs://genai-eval-sdk-replay-test/test_data/batch_eval_output",
     )
@@ -60,7 +60,7 @@ async def test_batch_eval_async(client):
     response = await client.aio.evals.batch_evaluate(
         dataset=eval_dataset,
         metrics=[
-            types.PrebuiltMetric.TEXT_QUALITY,
+            types.RubricMetric.TEXT_QUALITY,
         ],
         dest="gs://genai-eval-sdk-replay-test/test_data/batch_eval_output",
     )

tests/unit/vertexai/genai/replays/test_evaluate.py

@@ -29,7 +29,7 @@ def test_evaluation_result(client):
     )
 
     metrics_to_run = [
-        types.PrebuiltMetric.TEXT_QUALITY,
+        types.RubricMetric.TEXT_QUALITY,
     ]
 
     evaluation_result = client.evals.evaluate(

tests/unit/vertexai/genai/test_evals.py

@@ -3436,7 +3436,7 @@ def test_execute_evaluation_prebuilt_metric_via_loader(
             eval_dataset_df=dataset_df
         )
 
-        prebuilt_metric = vertexai_genai_types.PrebuiltMetric.FLUENCY
+        prebuilt_metric = vertexai_genai_types.RubricMetric.FLUENCY
 
         result = _evals_common._execute_evaluation(
             api_client=mock_api_client_fixture,

vertexai/_genai/_evals_common.py

@@ -746,7 +746,7 @@ def _resolve_dataset_inputs(
 def _resolve_metrics(
     metrics: list[types.Metric], api_client: Any
 ) -> list[types.Metric]:
-    """Resolves a list of metric instances, loading prebuilt metrics if necessary."""
+    """Resolves a list of metric instances, loading RubricMetric if necessary."""
     resolved_metrics_list = []
     for metric_instance in metrics:
         if isinstance(metric_instance, _evals_utils.LazyLoadedPrebuiltMetric):
@@ -756,7 +756,7 @@ def _resolve_metrics(
                 )
             except Exception as e:
                 logger.error(
-                    "Failed to resolve prebuilt metric %s@%s: %s",
+                    "Failed to resolve RubricMetric %s@%s: %s",
                     metric_instance.name,
                     metric_instance.version,
                     e,
@@ -768,7 +768,7 @@ def _resolve_metrics(
             try:
                 metric_name_str = str(metric_instance)
                 lazy_metric_instance = getattr(
-                    _evals_utils.PrebuiltMetric, metric_name_str.upper()
+                    _evals_utils.RubricMetric, metric_name_str.upper()
                 )
                 if isinstance(
                     lazy_metric_instance, _evals_utils.LazyLoadedPrebuiltMetric
@@ -778,12 +778,11 @@ def _resolve_metrics(
                     )
                 else:
                     raise TypeError(
-                        f"PrebuiltMetric.{metric_name_str.upper()} did not return a"
-                        " LazyLoadedPrebuiltMetric proxy."
+                        f"RubricMetric.{metric_name_str.upper()} cannot be resolved."
                     )
             except AttributeError as exc:
                 raise TypeError(
-                    "Unsupported metric type or invalid prebuilt metric name:"
+                    "Unsupported metric type or invalid RubricMetric name:"
                     f" {metric_instance}"
                 ) from exc
     return resolved_metrics_list

vertexai/_genai/_evals_constant.py

@@ -25,7 +25,5 @@
         "multi_turn_text_quality_v1",
         "final_response_match_v2",
         "final_response_reference_free_v1",
-        "partially_customizable_general_quality_v1",
-        "fully_customizable_general_quality_v1",
     }
 )

vertexai/_genai/_evals_utils.py

@@ -521,8 +521,8 @@ class PrebuiltMetricLoader:
     when they are first accessed.
 
     Example:
-      metric = PrebuiltMetric.TEXT_QUALITY
-      metric = PrebuiltMetric.TEXT_QUALITY(version="v1")
+      from vertexai import types
+      text_quality_metric = types.RubricMetric.TEXT_QUALITY
     """
 
     def __getattr__(
@@ -554,14 +554,6 @@ def MULTI_TURN_GENERAL_QUALITY(self) -> LazyLoadedPrebuiltMetric:
     def MULTI_TURN_TEXT_QUALITY(self) -> LazyLoadedPrebuiltMetric:
         return self.__getattr__("MULTI_TURN_TEXT_QUALITY")
 
-    @property
-    def PARTIALLY_CUSTOMIZABLE_GENERAL_QUALITY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("PARTIALLY_CUSTOMIZABLE_GENERAL_QUALITY")
-
-    @property
-    def FULLY_CUSTOMIZABLE_GENERAL_QUALITY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("FULLY_CUSTOMIZABLE_GENERAL_QUALITY")
-
     @property
     def FINAL_RESPONSE_MATCH(self) -> LazyLoadedPrebuiltMetric:
         return self.__getattr__("FINAL_RESPONSE_MATCH", version="v2")
@@ -600,6 +592,7 @@ def MULTI_TURN_SAFETY(self) -> LazyLoadedPrebuiltMetric:
 
 
 PrebuiltMetric = PrebuiltMetricLoader()
+RubricMetric = PrebuiltMetric
 
 
 class BatchEvaluateRequestPreparer:

vertexai/_genai/evals.py

@@ -1115,7 +1115,7 @@ def evaluate(
         dataset: Union[
             types.EvaluationDatasetOrDict, list[types.EvaluationDatasetOrDict]
         ],
-        metrics: list[types.MetricOrDict],
+        metrics: list[types.MetricOrDict] = None,
         config: Optional[types.EvaluateMethodConfigOrDict] = None,
     ) -> types.EvaluationResult:
         """Evaluates candidate responses in the provided dataset(s) using the specified metrics.
@@ -1147,6 +1147,8 @@ def evaluate(
         else:
             if isinstance(dataset, dict):
                 dataset = types.EvaluationDataset.model_validate(dataset)
+        if metrics is None:
+            metrics = [types.Metric(name="general_quality_v1")]
 
         return _evals_common._execute_evaluation(
             api_client=self._api_client,
@@ -1279,7 +1281,7 @@ def generate_rubrics(
             metric_spec_parameters: Optional. Parameters for the Predefined
               Metric, used to customize rubric generation. Only used if
               `predefined_spec_name` is set.
-                Example: {"requirements": ["The response must be in Japanese."]}
+                Example: {"guidelines": ["The response must be in Japanese."]}
             config: Optional. Configuration for the rubric generation process.
 
         Returns:

vertexai/_genai/types.py

@@ -50,11 +50,11 @@
 
 logger = logging.getLogger("vertexai_genai.types")
 
-__all__ = ["PrebuiltMetric"]  # noqa: F822
+__all__ = ["PrebuiltMetric", "RubricMetric"]  # noqa: F822
 
 
 def __getattr__(name: str) -> typing.Any:
-    if name == "PrebuiltMetric":
+    if name == "PrebuiltMetric" or name == "RubricMetric":
         module = importlib.import_module("._evals_utils", __package__)
         prebuilt_metric_obj = getattr(module, name)
         globals()[name] = prebuilt_metric_obj