Hotfix Release: use system-provided cyrus-sasl/libsasl2 at runtime (#9)

* Use system-provided cyrus-sasl/libsasl2 at runtime

Instead of having multiple builds with and without `libsasl2` on Linux,
one of which cannot be redistributed within Python wheels, we use
`dlopen` to load `libsasl2` at runtime on Unix.
The SASL GSSAPI mechanism availability is thus checked at runtime.
Because of differences in soname ABI bumps between Debian-based and
RPM-based distros, the previous SASL builds did not work on Debian-based
systems. This is also solved in this change, by probing the various
known names for `libsasl2`.

* Compute version from tag or from vcpkg.json

* Run style checks on tag too

* Re-enable tests that were disabled in our CI

* Allow build metadata in nuget version

Author: jgiannuzzi

.github/workflows/build.yml

@@ -55,7 +55,6 @@ jobs:
 
   style-check:
     runs-on: ubuntu-24.04
-    if: "!startsWith(github.ref, 'refs/tags/v')"
     steps:
       - uses: actions/checkout@v4
       - name: Install dependencies
@@ -117,28 +116,18 @@ jobs:
     strategy:
       matrix:
         include:
-          - name: "centos8 glibc +gssapi"
-            artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-std__extra-gssapi
-            image: quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1
-            extra_args: ""
           - name: "centos8 glibc"
             artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-all
             image: quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1
-            extra_args: "--disable-gssapi"
-          - name: "alpine musl +gssapi"
-            artifact_key: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-std__extra-gssapi
-            image: alpine:3.16.9
-            extra_args: ""
           - name: "alpine musl"
             artifact_key: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-all
             image: alpine:3.16.9
-            extra_args: "--disable-gssapi"
     steps:
       - uses: actions/checkout@v4
       - name: Build
         run: |
           mkdir artifacts
-          packaging/tools/build-release-artifacts.sh ${{ matrix.extra_args }} ${{ matrix.image }} artifacts/librdkafka.tgz
+          packaging/tools/build-release-artifacts.sh ${{ matrix.image }} artifacts/librdkafka.tgz
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
@@ -150,28 +139,18 @@ jobs:
     strategy:
       matrix:
         include:
-          - name: "centos8 glibc +gssapi"
-            artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi
-            image: quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1
-            extra_args: ""
           - name: "centos8 glibc"
             artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-all
             image: quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1
-            extra_args: "--disable-gssapi"
-          - name: "alpine musl +gssapi"
-            artifact_key: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all__extra-gssapi
-            image: alpine:3.16.9
-            extra_args: ""
           - name: "alpine musl"
             artifact_key: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all
             image: alpine:3.16.9
-            extra_args: "--disable-gssapi"
     steps:
       - uses: actions/checkout@v4
       - name: Build
         run: |
           mkdir artifacts
-          packaging/tools/build-release-artifacts.sh ${{ matrix.extra_args }} ${{ matrix.image }} artifacts/librdkafka.tgz
+          packaging/tools/build-release-artifacts.sh ${{ matrix.image }} artifacts/librdkafka.tgz
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
@@ -259,11 +238,10 @@ jobs:
         shell: pwsh
         run: |
           # Different packaging for tagged vs untagged builds
-          $vstring = "2.11.0-"
-          if ($env:GITHUB_REF -match '^refs/tags/') {
-            $vstring += "gr"
+          if ($env:GITHUB_REF -match '^refs/tags/v') {
+            $vstring = $env:GITHUB_REF -replace '^refs/tags/v', ''
           } else {
-            $vstring += "ci-$env:GITHUB_RUN_ID"
+            $vstring = "$((Get-Content .\vcpkg.json | ConvertFrom-Json).version)-ci-$($env:GITHUB_RUN_ID)"
           }
 
           mkdir packages

.semaphore/semaphore.yml

@@ -185,33 +185,19 @@ blocks:
         commands:
           - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
       jobs:
-        - name: 'Build: centos8 glibc +gssapi'
-          env_vars:
-            - name: ARTIFACT_KEY
-              value: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-std__extra-gssapi
-          commands:
-            - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 artifacts/librdkafka.tgz
-
         - name: 'Build: centos8 glibc'
           env_vars:
             - name: ARTIFACT_KEY
               value: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-all
           commands:
-            - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 artifacts/librdkafka.tgz
-
-        - name: 'Build: alpine musl +gssapi'
-          env_vars:
-            - name: ARTIFACT_KEY
-              value: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-std__extra-gssapi
-          commands:
-            - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz
+            - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 artifacts/librdkafka.tgz
 
         - name: 'Build: alpine musl'
           env_vars:
             - name: ARTIFACT_KEY
               value: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-all
           commands:
-            - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz
+            - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz
 
 
   - name: 'Linux arm64: release artifact docker builds'
@@ -227,33 +213,19 @@ blocks:
         commands:
           - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
       jobs:
-        - name: 'Build: centos8 glibc +gssapi'
-          env_vars:
-            - name: ARTIFACT_KEY
-              value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi
-          commands:
-            - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz
-
         - name: 'Build: centos8 glibc'
           env_vars:
             - name: ARTIFACT_KEY
               value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-all
           commands:
-            - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz
-
-        - name: 'Build: alpine musl +gssapi'
-          env_vars:
-            - name: ARTIFACT_KEY
-              value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all__extra-gssapi
-          commands:
-            - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz
+            - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz
 
         - name: 'Build: alpine musl'
           env_vars:
             - name: ARTIFACT_KEY
               value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all
           commands:
-            - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz
+            - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz
 
 
   - name: 'Windows x64: MinGW-w64'

CHANGELOG.md

@@ -1,7 +1,8 @@
 # librdkafka v2.11.0 + gr
  - https://github.com/confluentinc/librdkafka/pull/4972 (Avoid unnecessary producer epoch bumps)
  - https://github.com/confluentinc/librdkafka/pull/4989 (Fully utilize the max.in.flight.requests.per.connection parameter on the idempotent producer)
- 
+ - https://github.com/confluentinc/librdkafka/pull/5168 (Use system-provided cyrus-sasl/libsasl2 at runtime)
+
 # librdkafka v2.11.0
 
 librdkafka v2.11.0 is a feature release:

CMakeLists.txt

@@ -123,38 +123,19 @@ endif()
 # SASL {
 if(WIN32)
   set(with_sasl_default ON)
+elseif(WITH_LIBDL)
+  set(with_sasl_default ON)
 else()
-  if(PkgConfig_FOUND)
-    pkg_check_modules(SASL libsasl2)
-    if(SASL_FOUND)
-      set(with_sasl_default ON)
-    else()
-      try_compile(
-          WITH_SASL_CYRUS_BOOL
-          "${CMAKE_CURRENT_BINARY_DIR}/try_compile"
-          "${TRYCOMPILE_SRC_DIR}/libsasl2_test.c"
-          LINK_LIBRARIES "-lsasl2"
-      )
-      if(WITH_SASL_CYRUS_BOOL)
-        set(with_sasl_default ON)
-        set(SASL_LIBRARIES "-lsasl2")
-      else()
-        set(with_sasl_default OFF)
-      endif()
-    endif()
-  endif()
+  set(with_sasl_default OFF)
 endif()
 option(WITH_SASL "With SASL" ${with_sasl_default})
 if(WITH_SASL)
-  if(SASL_FOUND)
-    link_directories(${SASL_LIBRARY_DIRS})
-  endif()
   if(WITH_SSL)
     set(WITH_SASL_SCRAM ON)
     set(WITH_SASL_OAUTHBEARER ON)
     list(APPEND BUILT_WITH "SASL_SCRAM SASL_OAUTHBEARER")
   endif()
-  if(NOT WIN32)
+  if(NOT WIN32 AND WITH_LIBDL)
     set(WITH_SASL_CYRUS ON)
     list(APPEND BUILT_WITH "SASL_CYRUS")
   endif()

README.md

@@ -107,7 +107,6 @@ If the version is out of date, please [create an issue or pull request](https://
    	pthreads
 	zlib-dev (optional, for gzip compression support)
 	libssl-dev (optional, for SSL and SASL SCRAM support)
-	libsasl2-dev (optional, for SASL GSSAPI support)
 	libzstd-dev (optional, for ZStd compression support)
 	libcurl-dev (optional, for SASL OAUTHBEARER OIDC support)
 

configure.self

@@ -19,7 +19,6 @@ mkl_require socket
 mkl_require zlib
 mkl_require libzstd
 mkl_require libssl
-mkl_require libsasl2
 mkl_require libcurl
 
 # Generate version variables from rdkafka.h hex version define
@@ -99,12 +98,12 @@ void foo (void) {
 
     if [[ $WITH_LIBDL == "y" ]]; then
         mkl_allvar_set WITH_PLUGINS WITH_PLUGINS y
+        mkl_allvar_set WITH_SASL_CYRUS WITH_SASL_CYRUS y
     fi
 
     # optional libs
     mkl_check "zlib"
     mkl_check "libssl"
-    mkl_check "libsasl2"
     mkl_check "libzstd"
     mkl_check "libcurl"
 

debian/control

@@ -2,7 +2,7 @@ Source: librdkafka
 Priority: optional
 Maintainer: Faidon Liambotis <[email protected]>
 Uploaders: Christos Trochalakis <[email protected]>
-Build-Depends: debhelper (>= 9), zlib1g-dev, libssl-dev, libsasl2-dev, liblz4-dev, python3
+Build-Depends: debhelper (>= 9), zlib1g-dev, libssl-dev, liblz4-dev, python3
 Standards-Version: 3.9.7
 Section: libs
 Homepage: https://github.com/confluentinc/librdkafka

mklove/modules/configure.libsasl2

@@ -262,11 +262,6 @@ you will end up with:
    (`librdkafka.so.1` or `librdkafka.1.dylib` on OSX).
 
 
-**NOTE**: Due to libsasl2/cyrus-sasl's dynamically loaded plugins, it is
-not possible for us to provide a self-contained static library with
-GSSAPI/Kerberos support.
-
-
 
 ### The artifact pipeline
 

packaging/RELEASE.md

@@ -14,7 +14,7 @@ if [ "$1" = "--in-docker" ]; then
     git clone /v /librdkafka
 
     cd /librdkafka
-    ./configure --install-deps --disable-gssapi --disable-lz4-ext --enable-static $*
+    ./configure --install-deps --disable-lz4-ext --enable-static $*
     make -j
     examples/rdkafka_example -X builtin.features
     CI=true make -C tests run_local_quick