Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[upgrade](thirdparty) upgrade librdkafka from 1.8.2 to 2.0.2 #28210

Merged
merged 2 commits into from
Dec 12, 2023
Merged

[upgrade](thirdparty) upgrade librdkafka from 1.8.2 to 2.0.2 #28210

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
de86ce1
upgrade librdkafka from 1.8.2 to 2.0.2
sollhui Dec 10, 2023
c6c3a9d
update
sollhui Dec 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions thirdparty/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}`

## v20231210
- Modified: librdkafka 1.8.2 -> 2.0.2

## v20231020
- Added: avx2neon -> 1.0.0

Expand Down
4 changes: 2 additions & 2 deletions thirdparty/download-thirdparty.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,10 +327,10 @@ fi
echo "Finished patching ${ARROW_SOURCE}"

# patch librdkafka to avoid crash
if [[ "${LIBRDKAFKA_SOURCE}" == "librdkafka-1.8.2" ]]; then
if [[ "${LIBRDKAFKA_SOURCE}" == "librdkafka-2.0.2" ]]; then
cd "${TP_SOURCE_DIR}/${LIBRDKAFKA_SOURCE}"
if [[ ! -f "${PATCHED_MARK}" ]]; then
patch -p0 <"${TP_PATCH_DIR}/librdkafka-1.8.2.patch"
patch -p0 <"${TP_PATCH_DIR}/librdkafka-2.0.2.patch"
touch "${PATCHED_MARK}"
fi
cd -
Expand Down
55 changes: 27 additions & 28 deletions thirdparty/patches/librdkafka-1.8.2.patch → thirdparty/patches/librdkafka-2.0.2.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,17 @@
--- src/rdkafka_broker.c 2021-10-12 04:15:44.000000000 +0800
+++ src/rdkafka_broker.c 2022-03-10 10:31:45.141882467 +0800
@@ -5464,7 +5464,9 @@
*/
void rd_kafka_broker_destroy_final (rd_kafka_broker_t *rkb) {
--- lds-gen.py
+++ lds-gen.py
@@ -58,7 +58,7 @@ if __name__ == '__main__':

- rd_assert(thrd_is_current(rkb->rkb_thread));
+ // To avoid the error describe in https://github.com/edenhill/librdkafka/issues/3608
+ // comment this line to fix it temporarily.
+ // rd_assert(thrd_is_current(rkb->rkb_thread));
rd_assert(TAILQ_EMPTY(&rkb->rkb_monitors));
rd_assert(TAILQ_EMPTY(&rkb->rkb_outbufs.rkbq_bufs));
rd_assert(TAILQ_EMPTY(&rkb->rkb_waitresps.rkbq_bufs));

--- mklove/modules/configure.base 2022-11-21 03:47:14.072094876 +0800
+++ mklove/modules/configure.base 2022-11-21 03:47:01.360075164 +0800
@@ -1716,7 +1716,7 @@
# Special symbols not covered by above matches or not exposed in
# the public header files.
- funcs.append('rd_ut_coverage_check')
+ # funcs.append('rd_ut_coverage_check')

print('# Automatically generated by lds-gen.py - DO NOT EDIT')
print('{\n global:')
--- mklove/modules/configure.base
+++ mklove/modules/configure.base
@@ -1741,7 +1741,7 @@ function mkl_pkg_config_check {
mkl_check_begin "$cname" "$2" "no-cache" "$1 (by pkg-config)" && return $?

local cflags=
Expand All @@ -23,7 +20,7 @@
mkl_dbg "pkg-config check $libname for CFLAGS ($2): $cmd"

cflags=$($cmd 2>&1)
@@ -1738,11 +1738,11 @@
@@ -1764,11 +1764,11 @@ $cflags"
fi

local libs=
Expand All @@ -37,15 +34,17 @@
# Clear define name ($2): caller may have additional checks
mkl_check_failed "$cname" "" "$3" "pkg-config --libs failed"
return 1

--- lds-gen.py 2023-11-19 15:47:14.083929584 +0800
+++ lds-gen.py 2023-11-19 15:47:51.759274031 +0800
@@ -53,7 +53,7 @@

# Special symbols not covered by above matches or not exposed in
# the public header files.
- funcs.append('rd_ut_coverage_check')
+ # funcs.append('rd_ut_coverage_check')
--- src/rdkafka_broker.c
+++ src/rdkafka_broker.c
@@ -4613,7 +4613,9 @@ static int rd_kafka_broker_thread_main(void *arg) {
*/
void rd_kafka_broker_destroy_final(rd_kafka_broker_t *rkb) {

print('# Automatically generated by lds-gen.py - DO NOT EDIT')
print('{\n global:')
- rd_assert(thrd_is_current(rkb->rkb_thread));
+ // To avoid the error describe in https://github.com/edenhill/librdkafka/issues/3608
+ // comment this line to fix it temporarily.
+ // rd_assert(thrd_is_current(rkb->rkb_thread));
rd_assert(TAILQ_EMPTY(&rkb->rkb_monitors));
rd_assert(TAILQ_EMPTY(&rkb->rkb_outbufs.rkbq_bufs));
rd_assert(TAILQ_EMPTY(&rkb->rkb_waitresps.rkbq_bufs));

10 changes: 5 additions & 5 deletions thirdparty/vars.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,11 +215,11 @@ CYRUS_SASL_NAME=cyrus-sasl-2.1.27.tar.gz
CYRUS_SASL_SOURCE=cyrus-sasl-2.1.27
CYRUS_SASL_MD5SUM="a33820c66e0622222c5aefafa1581083"

# librdkafka-1.8.2
LIBRDKAFKA_DOWNLOAD="https://github.com/edenhill/librdkafka/archive/refs/tags/v1.8.2.tar.gz"
LIBRDKAFKA_NAME=librdkafka-1.8.2.tar.gz
LIBRDKAFKA_SOURCE=librdkafka-1.8.2
LIBRDKAFKA_MD5SUM="0abec0888d10c9553cdcbcbf9172d558"
# librdkafka-2.0.2
LIBRDKAFKA_DOWNLOAD="https://github.com/confluentinc/librdkafka/archive/refs/tags/v2.0.2.tar.gz"
LIBRDKAFKA_NAME=librdkafka-2.0.2.tar.gz
LIBRDKAFKA_SOURCE=librdkafka-2.0.2
LIBRDKAFKA_MD5SUM="c0120dc32acc129bfb4656fe17568da1"

# zstd
ZSTD_DOWNLOAD="https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz"
Expand Down