Releases v0.12.5 (#302)

Author: wjsi

README.rst

@@ -32,16 +32,16 @@ Or from source code (not recommended for production use):
 Dependencies
 ------------
 
--  Python (>=2.7), including Python 3+, pypy, Python 3.7 recommended
--  setuptools (>=3.0)
+- Python (>=2.7), including Python 3+, pypy, Python 3.7 recommended
+- setuptools (>=3.0)
 
 Run Tests
 ---------
 
--  install pytest
--  copy conf/test.conf.template to odps/tests/test.conf, and fill it
-   with your account
--  run ``pytest odps``
+- install pytest
+- copy conf/test.conf.template to odps/tests/test.conf, and fill it with
+  your account
+- run ``pytest odps``
 
 Usage
 -----

odps/_version.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version_info = (0, 12, 5, "b1")
+version_info = (0, 12, 5)
 _num_index = max(idx if isinstance(v, int) else 0 for idx, v in enumerate(version_info))
 __version__ = ".".join(map(str, version_info[: _num_index + 1])) + "".join(
     version_info[_num_index + 1 :]

odps/apis/storage_api/conftest.py

@@ -1,4 +1,4 @@
-# Copyright 1999-2024 Alibaba Group Holding Ltd.
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import sys
 
 import pytest
@@ -42,8 +43,9 @@ def storage_api_client(odps):
         ("a BIGINT, b BIGINT, c BIGINT, d BIGINT", "pt string"),
         if_not_exists=True,
     )
+    api_endpoint = os.getenv("ODPS_STORAGE_API_ENDPOINT")
     try:
-        yield StorageApiArrowClient(odps, table)
+        yield StorageApiArrowClient(odps, table, rest_endpoint=api_endpoint)
     finally:
         table.drop(async_=True)
         options.enable_schema = False

odps/apis/storage_api/storage_api.py

@@ -197,7 +197,7 @@ def __init__(self, **kwargs):
 
 
 class TableBatchScanResponse(serializers.JSONSerializableModel):
-    __slots__ = ["status", "request_id"]
+    __slots__ = ("status", "request_id")
 
     session_id = serializers.JSONNodeField("SessionId")
     session_type = serializers.JSONNodeField("SessionType")
@@ -655,8 +655,9 @@ def tunnel_rest(self):
         self._tunnel_rest = tunnel.tunnel_rest
         return self._tunnel_rest
 
-    def _get_resource(self, *args) -> str:
-        endpoint = self.tunnel_rest.endpoint + URL_PREFIX
+    def _get_resource(self, *args, url_prefix=None) -> str:
+        url_prefix = url_prefix or URL_PREFIX
+        endpoint = self.tunnel_rest.endpoint + url_prefix
         url = self._table.table_resource(endpoint=endpoint, force_schema=True)
         return "/".join([url] + list(args))
 
@@ -917,6 +918,16 @@ def commit_write_session(
         return response
 
 
+try:
+    from ...internal.apis.storage_api.storage_api import InternalStorageAPIClientMixin
+
+    StorageApiClient = type(
+        "StorageApiClient", (InternalStorageAPIClientMixin, StorageApiClient), {}
+    )
+except ImportError:
+    pass
+
+
 class StorageApiArrowClient(StorageApiClient):
     """Arrow batch client to bundle configuration needed for API requests."""
 
@@ -951,3 +962,14 @@ def write_rows_arrow(self, request: WriteRowsRequest) -> ArrowWriter:
             )
 
         return ArrowWriter(self.write_rows_stream(request), request.compression)
+
+
+try:
+    from ...internal.apis.storage_api.storage_api import (  # noqa: F401
+        CloseBlobWriterRequest,
+        CreateBlobWriterRequest,
+        ReadBlobRequest,
+        WriteBlobRequest,
+    )
+except ImportError:
+    pass

odps/config.py

@@ -502,7 +502,7 @@ def emit(self, record):
 default_options.register_option("enable_v4_sign", True, validator=is_bool)
 default_options.register_option("align_supported_python_tag", True, validator=is_bool)
 
-# c or python mode, use for UT, in other cases, please do not modify the value
+# c or python mode for UT only. In other cases, please do not modify the value
 default_options.register_option("force_c", False, validator=is_integer)
 default_options.register_option("force_py", False, validator=is_integer)
 
@@ -612,6 +612,14 @@ def emit(self, record):
     "sql.use_odps2_extension", None, validator=any_validator(is_null, is_bool)
 )
 
+# Catalog API
+default_options.register_option(
+    "catalog.endpoint", None, validator=any_validator(is_null, is_string)
+)
+default_options.register_option(
+    "catalog.url_prefix", "/api/catalog/v1alpha", validator=is_string
+)
+
 # sqlalchemy
 default_options.register_option(
     "sqlalchemy.project_as_schema", None, validator=any_validator(is_null, is_bool)

odps/core.py

@@ -29,6 +29,7 @@
 DEFAULT_ENDPOINT = "http://service.odps.aliyun.com/api"
 DEFAULT_REGION_NAME = "cn"
 LOGVIEW_HOST_DEFAULT = "http://logview.aliyun.com"
+CATALOG_API_ENDPOINT_DEFAULT = "https://catalogapi.{region}.maxcompute.aliyun.com"
 
 _ALTER_TABLE_REGEX = re.compile(
     r"^\s*(drop|alter)\s+table\s*(|if\s+exists)\s+(?P<table_name>[^\s;]+)", re.I
@@ -104,6 +105,7 @@ def __init__(
         region_name=None,
         quota_name=None,
         namespace=None,
+        catalog_endpoint=None,
         **kw
     ):
         # avoid polluted copy sources :(
@@ -114,6 +116,7 @@ def __init__(
         schema = utils.strip_if_str(schema)
         logview_host = utils.strip_if_str(logview_host)
         tunnel_endpoint = utils.strip_if_str(tunnel_endpoint)
+        catalog_endpoint = utils.strip_if_str(catalog_endpoint)
         region_name = utils.strip_if_str(region_name)
         quota_name = utils.strip_if_str(quota_name)
         namespace = utils.strip_if_str(namespace)
@@ -133,6 +136,7 @@ def __init__(
             app_account=app_account,
             logview_host=logview_host,
             tunnel_endpoint=tunnel_endpoint,
+            catalog_endpoint=catalog_endpoint,
             region_name=region_name,
             quota_name=quota_name,
             namespace=namespace,
@@ -178,7 +182,11 @@ def _init(
         self.project = (
             project or options.default_project or os.getenv("ODPS_PROJECT_NAME")
         )
-        self.region_name = region_name or self._get_region_from_endpoint(self.endpoint)
+        self.region_name = (
+            region_name
+            or os.getenv("ODPS_REGION_NAME")
+            or self._get_region_from_endpoint(self.endpoint)
+        )
         self.namespace = (
             namespace or options.default_namespace or os.getenv("ODPS_NAMESPACE")
         )
@@ -206,6 +214,32 @@ def _init(
             or os.getenv("ODPS_TUNNEL_ENDPOINT")
         )
 
+        self._catalog_endpoint = (
+            kw.pop("catalog_endpoint", None)
+            or options.catalog.endpoint
+            or os.getenv("ODPS_CATALOG_ENDPOINT")
+            or (
+                CATALOG_API_ENDPOINT_DEFAULT.format(region=self.region_name)
+                if self.region_name and CATALOG_API_ENDPOINT_DEFAULT
+                else None
+            )
+        )
+        if self._catalog_endpoint is None:
+            self.catalog_rest = None
+        else:
+            self.catalog_rest = rest_client_cls(
+                self.account,
+                self._catalog_endpoint.rstrip("/"),
+                project,
+                schema,
+                app_account=self.app_account,
+                proxy=options.api_proxy,
+                region_name=self.region_name,
+                namespace=self.namespace,
+                tag="Catalog",
+                **rest_client_kwargs
+            )
+
         self._logview_host = (
             kw.pop("logview_host", None)
             or options.logview_host
@@ -506,6 +540,7 @@ def get_project(self, name=None, default_schema=None):
         # use _schema to avoid requesting for tenant options
         proj._default_schema = default_schema or self._schema
         proj._quota_name = self._quota_name
+        proj._catalog_endpoint = self._catalog_endpoint
 
         proj_ref = weakref.ref(proj)
 
@@ -2137,6 +2172,45 @@ def delete_offline_model(self, name, project=None, if_exists=False):
             if not if_exists:
                 raise
 
+    def list_models(self, project=None, schema=None):
+        """
+        List models of project by optional filter conditions including prefix and owner.
+
+        :param project: project name, if not provided, will be the default project
+        :param str schema: schema name, if not provided, will be the default schema
+        :return: models
+        :rtype: list
+        """
+        parent = self._get_project_or_schema(project, schema)
+        return parent.models.iterate()
+
+    def get_model(self, name, project=None, schema=None):
+        """
+        Get model by given name
+
+        :param name: model name
+        :param project: project name, if not provided, will be the default project
+        :param str schema: schema name, if not provided, will be the default schema
+        :return: model
+        :rtype: :class:`odps.models.ml.Model`
+        :raise: :class:`odps.errors.NoSuchObject` if not exists
+        """
+        parent = self._get_project_or_schema(project, schema)
+        return parent.models[name]
+
+    def exist_model(self, name, project=None, schema=None):
+        """
+        If the model with given name exists or not.
+
+        :param name: model's name
+        :param project: project name, if not provided, will be the default project
+        :param str schema: schema name, if not provided, will be the default schema
+        :return: True if model exists else False
+        :rtype: bool
+        """
+        parent = self._get_project_or_schema(project, schema)
+        return name in parent.models
+
     def get_logview_host(self):
         """
         Get logview host address.
@@ -2514,7 +2588,8 @@ def _get_odps_from_model(self):
     return cur.odps if cur else None
 
 
-models.RestModel.odps = property(fget=_get_odps_from_model)
+models.JSONRestModel.odps = property(fget=_get_odps_from_model)
+models.XMLRestModel.odps = property(fget=_get_odps_from_model)
 del _get_odps_from_model
 
 try:

odps/errors.py

@@ -55,11 +55,17 @@ def parse_response(resp, endpoint=None, tag=None):
             request_id = resp.headers.get("x-odps-request-id", None)
             if len(resp.content) > 0:
                 obj = json.loads(resp.text)
-                msg = obj["Message"]
-                code = obj.get("Code")
-                host_id = obj.get("HostId")
-                if request_id is None:
-                    request_id = obj.get("RequestId")
+                if tag == "Catalog":
+                    msg = obj["message"]
+                    reason = obj.get("reason")
+                    code = _CATALOG_ERROR_MAPPING.get(reason, reason)
+                    host_id = None
+                else:
+                    msg = obj["Message"]
+                    code = obj.get("Code")
+                    host_id = obj.get("HostId")
+                    if request_id is None:
+                        request_id = obj.get("RequestId")
             else:
                 raise
         clz = globals().get(code, ODPSError)
@@ -126,6 +132,10 @@ def throw_if_parsable(resp, endpoint=None, tag=None):
     "ODPS-186": "SQAQueryTimedout",
 }
 
+_CATALOG_ERROR_MAPPING = {
+    "NotFound": "NoSuchObject",
+}
+
 _nginx_bad_gateway_message = "the page you are looking for is currently unavailable"
 
 

odps/models/__init__.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright 1999-2024 Alibaba Group Holding Ltd.
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 import sys
 import warnings
 
-from .core import RestModel
+from .core import JSONRestModel, XMLRestModel
 from .function import Function
 from .functions import Functions
 from .instance import Instance

odps/models/cache.py

@@ -101,7 +101,7 @@ def cache_container(self, func, cls, **kwargs):
         return obj
 
     def del_item_cache(self, obj, item):
-        from .core import LazyLoad
+        from .core import XMLLazyLoad
 
         try:
             item = obj._get_parent_typed(item)
@@ -118,38 +118,42 @@ def del_item_cache(self, obj, item):
             if cache_key in self._caches:
                 # make sure original object will be reloaded
                 obj = self._caches[cache_key]
-                if isinstance(obj, LazyLoad):
+                if isinstance(obj, XMLLazyLoad):
                     obj.reset()
 
                 del self._caches[cache_key]
 
 
 _object_cache = ObjectCache()
+_cls_bases_cache = dict()
 
 
 def cache(func):
+    @six.wraps(func)
     def inner(cls, **kwargs):
-        bases = [base.__name__ for base in inspect.getmro(cls)]
-        if "LazyLoad" in bases:
+        try:
+            bases_set = _cls_bases_cache[cls]
+        except KeyError:
+            bases_set = set(base.__name__ for base in inspect.getmro(cls))
+            _cls_bases_cache[cls] = bases_set
+
+        if "XMLLazyLoad" in bases_set:
             return _object_cache.cache_lazyload(func, cls, **kwargs)
-        elif "Container" in bases:
+        elif "XMLContainer" in bases_set:
             return _object_cache.cache_container(func, cls, **kwargs)
 
         return func(cls, **kwargs)
 
-    inner.__name__ = func.__name__
-    inner.__doc__ = func.__doc__
     return inner
 
 
 def del_cache(func):
+    @six.wraps(func)
     def inner(obj, item):
         if func.__name__ == "__delitem__":
             _object_cache.del_item_cache(obj, item)
         return func(obj, item)
 
-    inner.__name__ = func.__name__
-    inner.__doc__ = func.__doc__
     inner._cache_maker = True
     return inner