Skip to content

Commit e62ec18

Browse files
jgiannuzzijgiannuzzi
committed
Hotfix Release: v2.11.1.1+gr - collective changes
- G-Research#3 (CI/CD script) - confluentinc#4972 (Avoid unnecessary producer epoch bumps) - confluentinc#4989 (Fully utilize the max.in.flight.requests.per.connection parameter on the idempotent producer) - confluentinc#5168 (Use system-provided cyrus-sasl/libsasl2 at runtime)
1 parent 69b1865 commit e62ec18

29 files changed

+597
-228
lines changed

‎.github/workflows/build.yml

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
name: librdkafka build and release artifact pipeline
2+
3+
on:
4+
push:
5+
pull_request:
6+
7+
jobs:
8+
osx-arm64:
9+
runs-on: macos-14
10+
env:
11+
ARTIFACT_KEY: p-librdkafka__plat-osx__arch-arm64__lnk-all
12+
steps:
13+
- uses: actions/checkout@v4
14+
- name: Build
15+
run: |
16+
mkdir artifacts dest
17+
./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip
18+
make -j all examples check
19+
examples/rdkafka_example -X builtin.features
20+
otool -L src/librdkafka.dylib
21+
otool -L src-cpp/librdkafka++.dylib
22+
make -j -C tests build
23+
make -C tests run_local_quick
24+
DESTDIR="$PWD/dest" make install
25+
(cd dest && tar cvzf ../artifacts/librdkafka.tgz .)
26+
- name: Upload artifacts
27+
uses: actions/upload-artifact@v4
28+
with:
29+
name: ${{ env.ARTIFACT_KEY }}
30+
path: artifacts/
31+
32+
osx-x64:
33+
runs-on: macos-13
34+
env:
35+
ARTIFACT_KEY: p-librdkafka__plat-osx__arch-x64__lnk-all
36+
steps:
37+
- uses: actions/checkout@v4
38+
- name: Build
39+
run: |
40+
mkdir artifacts dest
41+
./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip
42+
make -j all examples check
43+
examples/rdkafka_example -X builtin.features
44+
otool -L src/librdkafka.dylib
45+
otool -L src-cpp/librdkafka++.dylib
46+
make -j -C tests build
47+
make -C tests run_local_quick
48+
DESTDIR="$PWD/dest" make install
49+
(cd dest && tar cvzf ../artifacts/librdkafka.tgz .)
50+
- name: Upload artifacts
51+
uses: actions/upload-artifact@v4
52+
with:
53+
name: ${{ env.ARTIFACT_KEY }}
54+
path: artifacts/
55+
56+
style-check:
57+
runs-on: ubuntu-24.04
58+
steps:
59+
- uses: actions/checkout@v4
60+
- name: Install dependencies
61+
run: |
62+
sudo apt update
63+
sudo apt install -y clang-format-18 python3 python3-pip python3-setuptools
64+
python3 -m pip install -r packaging/tools/requirements.txt
65+
- name: Style check
66+
run: make style-check
67+
68+
documentation:
69+
runs-on: ubuntu-22.04
70+
steps:
71+
- uses: actions/checkout@v4
72+
- name: Install dependencies
73+
run: sudo apt install -y doxygen graphviz
74+
- name: Generate documentation
75+
run: |
76+
mkdir artifacts
77+
make docs
78+
(cd staging-docs && tar cvzf ../artifacts/librdkafka-docs.tgz .)
79+
- name: Upload documentation
80+
uses: actions/upload-artifact@v4
81+
with:
82+
name: librdkafka-docs
83+
path: artifacts/librdkafka-docs.tgz
84+
85+
linux-ubuntu-source:
86+
runs-on: ubuntu-22.04
87+
env:
88+
CFLAGS: -std=gnu90
89+
steps:
90+
- uses: actions/checkout@v4
91+
- name: Build configuration checks
92+
run: |
93+
sudo apt install -y rapidjson-dev
94+
python3 -m pip install -U pip
95+
./packaging/tools/build-configurations-checks.sh
96+
- name: Build and test
97+
run: |
98+
python3 -m pip -V
99+
(cd tests && python3 -m pip install -r requirements.txt)
100+
./configure --install-deps
101+
./packaging/tools/rdutcoverage.sh
102+
make copyright-check
103+
make -j all examples check
104+
echo "Verifying that CONFIGURATION.md does not have manual changes"
105+
git diff --exit-code CONFIGURATION.md
106+
examples/rdkafka_example -X builtin.features
107+
ldd src/librdkafka.so.1
108+
ldd src-cpp/librdkafka++.so.1
109+
make -j -C tests build
110+
make -C tests run_local_quick
111+
DESTDIR="$PWD/dest" make install
112+
(cd tests && python3 -m trivup.clusters.KafkaCluster --version 3.4.0 --cmd "PATH=\"$PATH\" make quick")
113+
114+
linux-x64-release:
115+
runs-on: ubuntu-22.04
116+
strategy:
117+
matrix:
118+
include:
119+
- name: "centos8 glibc"
120+
artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-all
121+
image: quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1
122+
- name: "alpine musl"
123+
artifact_key: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-all
124+
image: alpine:3.16.9
125+
steps:
126+
- uses: actions/checkout@v4
127+
- name: Build
128+
run: |
129+
mkdir artifacts
130+
packaging/tools/build-release-artifacts.sh ${{ matrix.image }} artifacts/librdkafka.tgz
131+
- name: Upload artifacts
132+
uses: actions/upload-artifact@v4
133+
with:
134+
name: ${{ matrix.artifact_key }}
135+
path: artifacts/
136+
137+
linux-arm64-release:
138+
runs-on: ubuntu-22.04-arm
139+
strategy:
140+
matrix:
141+
include:
142+
- name: "centos8 glibc"
143+
artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-all
144+
image: quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1
145+
- name: "alpine musl"
146+
artifact_key: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all
147+
image: alpine:3.16.9
148+
steps:
149+
- uses: actions/checkout@v4
150+
- name: Build
151+
run: |
152+
mkdir artifacts
153+
packaging/tools/build-release-artifacts.sh ${{ matrix.image }} artifacts/librdkafka.tgz
154+
- name: Upload artifacts
155+
uses: actions/upload-artifact@v4
156+
with:
157+
name: ${{ matrix.artifact_key }}
158+
path: artifacts/
159+
160+
windows-mingw:
161+
runs-on: windows-latest
162+
strategy:
163+
matrix:
164+
include:
165+
- name: "MinGW-w64 Dynamic"
166+
artifact_key: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-std
167+
extra_args: ""
168+
- name: "MinGW-w64 Static"
169+
artifact_key: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-static
170+
extra_args: "--static"
171+
env:
172+
CHERE_INVOKING: yes
173+
MSYSTEM: UCRT64
174+
steps:
175+
- uses: actions/checkout@v4
176+
- name: Setup MSYS2
177+
uses: msys2/setup-msys2@v2
178+
with:
179+
msystem: UCRT64
180+
update: true
181+
- name: Build
182+
shell: msys2 {0}
183+
run: |
184+
mkdir artifacts
185+
./packaging/mingw-w64/semaphoreci-build.sh ${{ matrix.extra_args }} ./artifacts/librdkafka.tgz
186+
- name: Upload artifacts
187+
uses: actions/upload-artifact@v4
188+
with:
189+
name: ${{ matrix.artifact_key }}
190+
path: artifacts/
191+
192+
windows-msvc:
193+
runs-on: windows-latest
194+
strategy:
195+
matrix:
196+
include:
197+
- platform: x64
198+
triplet: x64-windows
199+
artifact_key: p-librdkafka__plat-windows__dist-msvc__arch-x64__lnk-std
200+
- platform: Win32
201+
triplet: x86-windows
202+
artifact_key: p-librdkafka__plat-windows__dist-msvc__arch-x86__lnk-std
203+
env:
204+
VCPKG_DISABLE_METRICS: yes
205+
steps:
206+
- uses: actions/checkout@v4
207+
- name: Setup vcpkg
208+
run: |
209+
cd ..
210+
& .\librdkafka\win32\setup-vcpkg.ps1
211+
cd librdkafka
212+
..\vcpkg\vcpkg integrate install
213+
..\vcpkg\vcpkg --feature-flags=versions install --triplet ${{ matrix.triplet }}
214+
- name: Build
215+
run: |
216+
& .\win32\msbuild.ps1 -platform ${{ matrix.platform }}
217+
& .\win32\package-zip.ps1 -platform ${{ matrix.platform }}
218+
- name: List artifacts
219+
run: |
220+
Get-ChildItem . -include *.dll -recurse
221+
Get-ChildItem . -include *.lib -recurse
222+
- name: Upload artifacts
223+
uses: actions/upload-artifact@v4
224+
with:
225+
name: ${{ matrix.artifact_key }}
226+
path: artifacts/
227+
228+
packaging:
229+
needs: [documentation, osx-arm64, osx-x64, linux-x64-release, linux-arm64-release, windows-mingw, windows-msvc]
230+
runs-on: ubuntu-22.04
231+
steps:
232+
- uses: actions/checkout@v4
233+
- name: Download all artifacts
234+
uses: actions/download-artifact@v4
235+
with:
236+
path: artifacts
237+
- name: Build packages
238+
shell: pwsh
239+
run: |
240+
# Different packaging for tagged vs untagged builds
241+
if ($env:GITHUB_REF -match '^refs/tags/v') {
242+
$vstring = $env:GITHUB_REF -replace '^refs/tags/v', ''
243+
} else {
244+
$vstring = "$((Get-Content .\vcpkg.json | ConvertFrom-Json).version)-ci-$($env:GITHUB_RUN_ID)"
245+
}
246+
247+
mkdir packages
248+
cd packaging/nuget
249+
python3 -m pip install -U -r requirements.txt
250+
./release.py --directory ../../artifacts --ignore-tag --class NugetPackage $vstring --nuget-version $vstring
251+
cp -v librdkafka.redist.*.nupkg ../../packages
252+
./release.py --directory ../../artifacts --ignore-tag --class StaticPackage $vstring
253+
cp -v librdkafka-static-bundle*.tgz ../../packages
254+
cd ../../
255+
cp -v artifacts/librdkafka-docs/librdkafka-docs.tgz packages/
256+
cd packages
257+
ls -la
258+
sha256sum *
259+
- name: Upload packages
260+
uses: actions/upload-artifact@v4
261+
with:
262+
name: librdkafka-artifacts
263+
path: packages/
264+
265+
# Publish NuGet packages when a tag is pushed.
266+
# Tests need to succeed for all components and on all platforms first,
267+
# including having a tag name that matches the version number.
268+
publish-release:
269+
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
270+
needs: packaging
271+
runs-on: ubuntu-latest
272+
steps:
273+
- name: Download NuGet package artifacts
274+
uses: actions/download-artifact@v4
275+
with:
276+
name: librdkafka-artifacts
277+
path: dist
278+
- name: Publish to NuGet
279+
run: |
280+
ls -al dist
281+
dotnet nuget push "dist/librdkafka*.nupkg" --source https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json --api-key ${GITHUB_TOKEN}
282+
env:
283+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎.semaphore/semaphore.yml

Lines changed: 4 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -185,33 +185,19 @@ blocks:
185185
commands:
186186
- '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
187187
jobs:
188-
- name: 'Build: centos8 glibc +gssapi'
189-
env_vars:
190-
- name: ARTIFACT_KEY
191-
value: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-std__extra-gssapi
192-
commands:
193-
- packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 artifacts/librdkafka.tgz
194-
195188
- name: 'Build: centos8 glibc'
196189
env_vars:
197190
- name: ARTIFACT_KEY
198191
value: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-all
199192
commands:
200-
- packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 artifacts/librdkafka.tgz
201-
202-
- name: 'Build: alpine musl +gssapi'
203-
env_vars:
204-
- name: ARTIFACT_KEY
205-
value: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-std__extra-gssapi
206-
commands:
207-
- packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz
193+
- packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 artifacts/librdkafka.tgz
208194

209195
- name: 'Build: alpine musl'
210196
env_vars:
211197
- name: ARTIFACT_KEY
212198
value: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-all
213199
commands:
214-
- packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz
200+
- packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz
215201

216202

217203
- name: 'Linux arm64: release artifact docker builds'
@@ -227,33 +213,19 @@ blocks:
227213
commands:
228214
- '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
229215
jobs:
230-
- name: 'Build: centos8 glibc +gssapi'
231-
env_vars:
232-
- name: ARTIFACT_KEY
233-
value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi
234-
commands:
235-
- packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz
236-
237216
- name: 'Build: centos8 glibc'
238217
env_vars:
239218
- name: ARTIFACT_KEY
240219
value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-all
241220
commands:
242-
- packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz
243-
244-
- name: 'Build: alpine musl +gssapi'
245-
env_vars:
246-
- name: ARTIFACT_KEY
247-
value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all__extra-gssapi
248-
commands:
249-
- packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz
221+
- packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz
250222

251223
- name: 'Build: alpine musl'
252224
env_vars:
253225
- name: ARTIFACT_KEY
254226
value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all
255227
commands:
256-
- packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz
228+
- packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz
257229

258230

259231
- name: 'Windows x64: MinGW-w64'

‎CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
# librdkafka v2.11.1.1+gr
2+
- https://github.com/confluentinc/librdkafka/pull/4972 (Avoid unnecessary producer epoch bumps)
3+
- https://github.com/confluentinc/librdkafka/pull/4989 (Fully utilize the max.in.flight.requests.per.connection parameter on the idempotent producer)
4+
- https://github.com/confluentinc/librdkafka/pull/5168 (Use system-provided cyrus-sasl/libsasl2 at runtime)
5+
16
# librdkafka v2.11.1
27

38
librdkafka v2.11.1 is a maintenance release:

0 commit comments

Comments
 (0)