Skip to content

Commit

Permalink
Merge branch 'apache:trunk' into trunk
Browse files Browse the repository at this point in the history
  • Loading branch information
cwebbtw authored Mar 1, 2024
2 parents d089550 + c8843f0 commit 3ed11f5
Show file tree
Hide file tree
Showing 1,516 changed files with 107,381 additions and 30,893 deletions.
66 changes: 66 additions & 0 deletions .github/workflows/docker_build_and_test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Docker Build Test

on:
workflow_dispatch:
inputs:
image_type:
type: choice
description: Docker image type to build and test
options:
- "jvm"
kafka_url:
description: Kafka url to be used to build the docker image
required: true

jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: "3.10"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r docker/requirements.txt
- name: Build image and run tests
working-directory: ./docker
run: |
python docker_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -u=${{ github.event.inputs.kafka_url }}
- name: Run CVE scan
uses: aquasecurity/trivy-action@master
with:
image-ref: 'kafka/test:test'
format: 'table'
severity: 'CRITICAL,HIGH'
output: scan_report_${{ github.event.inputs.image_type }}.txt
exit-code: '1'
- name: Upload test report
if: always()
uses: actions/upload-artifact@v3
with:
name: report_${{ github.event.inputs.image_type }}.html
path: docker/test/report_${{ github.event.inputs.image_type }}.html
- name: Upload CVE scan report
if: always()
uses: actions/upload-artifact@v3
with:
name: scan_report_${{ github.event.inputs.image_type }}.txt
path: scan_report_${{ github.event.inputs.image_type }}.txt
44 changes: 44 additions & 0 deletions .github/workflows/docker_promote.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Promote Release Candidate Docker Image

on:
workflow_dispatch:
inputs:
rc_docker_image:
description: RC docker image that needs to be promoted (Example:- apache/kafka:3.6.0-rc0)
required: true
promoted_docker_image:
description: Docker image name of the promoted image (Example:- apache/kafka:3.6.0)
required: true

jobs:
promote:
if: github.repository == 'apache/kafka'
runs-on: ubuntu-latest
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Copy RC Image to promoted image
run: |
docker buildx imagetools create --tag ${{ github.event.inputs.promoted_docker_image }} ${{ github.event.inputs.rc_docker_image }}
59 changes: 59 additions & 0 deletions .github/workflows/docker_rc_release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Build and Push Release Candidate Docker Image

on:
workflow_dispatch:
inputs:
image_type:
type: choice
description: Docker image type to be built and pushed
options:
- "jvm"
rc_docker_image:
description: RC docker image that needs to be built and pushed to Dockerhub (Example:- apache/kafka:3.6.0-rc0)
required: true
kafka_url:
description: Kafka url to be used to build the docker image
required: true

jobs:
release:
if: github.repository == 'apache/kafka'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: "3.10"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r docker/requirements.txt
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Release the RC docker image
run: |
python docker/docker_release.py ${{ github.event.inputs.rc_docker_image }} --kafka-url ${{ github.event.inputs.kafka_url }} --image-type ${{ github.event.inputs.image_type }}
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -59,3 +59,6 @@ jmh-benchmarks/src/main/generated
**/src/generated-test

storage/kafka-tiered-storage/

docker/test/report_*.html
__pycache__
64 changes: 38 additions & 26 deletions LICENSE-binary
Original file line number Diff line number Diff line change
Expand Up @@ -205,40 +205,43 @@
This project bundles some components that are also licensed under the Apache
License Version 2.0:

audience-annotations-0.13.0
audience-annotations-0.12.0
caffeine-2.9.3
commons-beanutils-1.9.4
commons-cli-1.4
commons-collections-3.2.2
commons-digester-2.1
commons-lang3-3.8.1
commons-io-2.11.0
commons-lang3-3.12.0
commons-logging-1.2
commons-validator-1.7
jackson-annotations-2.13.5
jackson-core-2.13.5
jackson-databind-2.13.5
jackson-dataformat-csv-2.13.5
jackson-datatype-jdk8-2.13.5
jackson-jaxrs-base-2.13.5
jackson-jaxrs-json-provider-2.13.5
jackson-module-jaxb-annotations-2.13.5
jackson-module-scala_2.13-2.13.5
jackson-module-scala_2.12-2.13.5
error_prone_annotations-2.10.0
jackson-annotations-2.16.1
jackson-core-2.16.1
jackson-databind-2.16.1
jackson-dataformat-csv-2.16.1
jackson-datatype-jdk8-2.16.1
jackson-jaxrs-base-2.16.1
jackson-jaxrs-json-provider-2.16.1
jackson-module-afterburner-2.16.1
jackson-module-jaxb-annotations-2.16.1
jackson-module-scala_2.13-2.16.1
jackson-module-scala_2.12-2.16.1
jakarta.validation-api-2.0.2
javassist-3.29.2-GA
jetty-client-9.4.52.v20230823
jetty-continuation-9.4.52.v20230823
jetty-http-9.4.52.v20230823
jetty-io-9.4.52.v20230823
jetty-security-9.4.52.v20230823
jetty-server-9.4.52.v20230823
jetty-servlet-9.4.52.v20230823
jetty-servlets-9.4.52.v20230823
jetty-util-9.4.52.v20230823
jetty-util-ajax-9.4.52.v20230823
jose4j-0.9.3
jetty-client-9.4.53.v20231009
jetty-continuation-9.4.53.v20231009
jetty-http-9.4.53.v20231009
jetty-io-9.4.53.v20231009
jetty-security-9.4.53.v20231009
jetty-server-9.4.53.v20231009
jetty-servlet-9.4.53.v20231009
jetty-servlets-9.4.53.v20231009
jetty-util-9.4.53.v20231009
jetty-util-ajax-9.4.53.v20231009
jose4j-0.9.4
lz4-java-1.8.0
maven-artifact-3.8.8
maven-artifact-3.9.6
metrics-core-4.1.12.1
metrics-core-2.2.0
netty-buffer-4.1.100.Final
Expand All @@ -250,14 +253,20 @@ netty-transport-4.1.100.Final
netty-transport-classes-epoll-4.1.100.Final
netty-transport-native-epoll-4.1.100.Final
netty-transport-native-unix-common-4.1.100.Final
plexus-utils-3.3.0
opentelemetry-proto-1.0.0-alpha
plexus-utils-3.5.1
reflections-0.10.2
reload4j-1.2.25
rocksdbjni-7.9.2
scala-collection-compat_2.12-2.10.0
scala-collection-compat_2.13-2.10.0
scala-library-2.12.18
scala-library-2.13.12
scala-logging_2.12-3.9.4
scala-logging_2.13-3.9.4
scala-reflect-2.12.18
scala-reflect-2.13.12
scala-java8-compat_2.12-1.0.2
scala-java8-compat_2.13-1.0.2
snappy-java-1.1.10.5
swagger-annotations-2.2.8
Expand Down Expand Up @@ -310,6 +319,7 @@ activation-1.1.1
MIT License

argparse4j-0.7.0, see: licenses/argparse-MIT
checker-qual-3.19.0, see: licenses/checker-qual-MIT
jopt-simple-5.0.4, see: licenses/jopt-simple-MIT
slf4j-api-1.7.36, see: licenses/slf4j-MIT
slf4j-reload4j-1.7.36, see: licenses/slf4j-MIT
Expand All @@ -318,13 +328,15 @@ pcollections-4.0.1, see: licenses/pcollections-MIT
---------------------------------------
BSD 2-Clause

zstd-jni-1.5.5-6 see: licenses/zstd-jni-BSD-2-clause
zstd-jni-1.5.5-11 see: licenses/zstd-jni-BSD-2-clause

---------------------------------------
BSD 3-Clause

jline-3.22.0, see: licenses/jline-BSD-3-clause
jsr305-3.0.2, see: licenses/jsr305-BSD-3-clause
paranamer-2.8, see: licenses/paranamer-BSD-3-clause
protobuf-java-3.23.4, see: licenses/protobuf-java-BSD-3-clause

---------------------------------------
Do What The F*ck You Want To Public License
Expand Down
4 changes: 2 additions & 2 deletions NOTICE
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
Apache Kafka
Copyright 2023 The Apache Software Foundation.
Copyright 2024 The Apache Software Foundation.

This product includes software developed at
The Apache Software Foundation (https://www.apache.org/).
Expand All @@ -20,4 +20,4 @@ clients/src/main/java/org/apache/kafka/common/utils/PureJavaCrc32C.java
Some portions of this file Copyright (c) 2004-2006 Intel Corporation and licensed under the BSD license.

This project contains the following code copied from Apache Hive:
streams/src/main/java/org/apache/kafka/streams/state/internals/Murmur3.java
streams/src/main/java/org/apache/kafka/streams/state/internals/Murmur3.java
11 changes: 9 additions & 2 deletions NOTICE-binary
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,13 @@ This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).


Apache Commons IO
Copyright 2002-2021 The Apache Software Foundation

This product includes software developed at
The Apache Software Foundation (https://www.apache.org/).


Apache Commons Lang
Copyright 2001-2018 The Apache Software Foundation

Expand Down Expand Up @@ -553,7 +560,7 @@ The Apache Software Foundation (http://www.apache.org/).


Maven Artifact
Copyright 2001-2019 The Apache Software Foundation
Copyright 2001-2024 The Apache Software Foundation

This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
Expand Down Expand Up @@ -853,4 +860,4 @@ This private header is also used by Apple's open source
* LICENSE:
* license/LICENSE.dnsinfo.txt (Apple Public Source License 2.0)
* HOMEPAGE:
* https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h
* https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h
6 changes: 4 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ You need to have [Java](http://www.oracle.com/technetwork/java/javase/downloads/

We build and test Apache Kafka with Java 8, 11, 17 and 21. We set the `release` parameter in javac and scalac
to `8` to ensure the generated binaries are compatible with Java 8 or higher (independently of the Java version
used for compilation). Java 8 support has been deprecated since Apache Kafka 3.0 and will be removed in Apache
Kafka 4.0 (see [KIP-750](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=181308223) for more details).
used for compilation). Java 8 support project-wide has been deprecated since Apache Kafka 3.0, Java 11 support for
the broker and tools has been deprecated since Apache Kafka 3.7 and removal of both is planned for Apache Kafka 4.0 (
see [KIP-750](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=181308223) and
[KIP-1013](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=284789510) for more details).

Scala 2.12 and 2.13 are supported and 2.13 is used by default. Scala 2.12 support has been deprecated since
Apache Kafka 3.0 and will be removed in Apache Kafka 4.0 (see [KIP-751](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=181308218)
Expand Down
2 changes: 1 addition & 1 deletion bin/connect-standalone.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

if [ $# -lt 1 ];
then
echo "USAGE: $0 [-daemon] connect-standalone.properties"
echo "USAGE: $0 [-daemon] connect-standalone.properties [connector1.properties connector2.json ...]"
exit 1
fi

Expand Down
17 changes: 17 additions & 0 deletions bin/kafka-client-metrics.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.tools.ClientMetricsCommand "$@"
2 changes: 1 addition & 1 deletion bin/kafka-console-consumer.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@ if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
export KAFKA_HEAP_OPTS="-Xmx512M"
fi

exec $(dirname $0)/kafka-run-class.sh kafka.tools.ConsoleConsumer "$@"
exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.tools.consumer.ConsoleConsumer "$@"
Loading

0 comments on commit 3ed11f5

Please sign in to comment.