Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[build][sql] Make pulsar-trino-disto self-contained #17062

Merged
merged 27 commits into from
Sep 9, 2022
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
21c1164
[build][sql] Make trino disto self-contained
tisonkun Aug 11, 2022
60d8579
checkstyle LineLength only for Java files
tisonkun Aug 11, 2022
a3454a6
update related refs
tisonkun Aug 11, 2022
fa875a7
fix relpath
tisonkun Aug 11, 2022
b15ebbe
Merge branch 'master' into trino-disto-self-contain
tisonkun Aug 20, 2022
e250496
Merge branch 'master' into trino-disto-self-contain
tisonkun Aug 25, 2022
7ae043f
lib/presto to trino
tisonkun Aug 25, 2022
9c59029
more lib/presto to trino
tisonkun Aug 25, 2022
c2f605e
avoid generate dir to reduce target space occupation
tisonkun Aug 25, 2022
cb16fc4
Merge branch 'master' into trino-disto-self-contain
tisonkun Aug 29, 2022
0425513
Merge branch 'master' into trino-disto-self-contain
tisonkun Aug 29, 2022
530b857
Merge branch 'master' into trino-disto-self-contain
tisonkun Aug 29, 2022
093c840
Merge branch 'master' into trino-disto-self-contain
tisonkun Aug 31, 2022
43c8c7d
fallback to previous directory
tisonkun Aug 31, 2022
362207d
fix and fallback to previous variable name
tisonkun Aug 31, 2022
54fb61e
revert fallback internal folder
tisonkun Aug 31, 2022
f059c6a
Merge branch 'master' into trino-disto-self-contain
tisonkun Sep 2, 2022
a05e737
Merge branch 'master' into trino-disto-self-contain
tisonkun Sep 2, 2022
8e2a497
Merge branch 'master' into trino-disto-self-contain
tisonkun Sep 2, 2022
2709a9d
Merge branch 'master' into trino-disto-self-contain
tisonkun Sep 3, 2022
8522975
Merge branch 'master' into trino-disto-self-contain
tisonkun Sep 4, 2022
4fd524d
Merge branch 'master' into trino-disto-self-contain
tisonkun Sep 5, 2022
f5b575d
Merge branch 'master' into trino-disto-self-contain
tisonkun Sep 5, 2022
c32eea9
Merge branch 'master' into trino-disto-self-contain
tisonkun Sep 7, 2022
52c486a
Merge branch 'master' into trino-disto-self-contain
tisonkun Sep 7, 2022
0e80e3a
Merge branch 'master' into trino-disto-self-contain
tisonkun Sep 9, 2022
10f7209
Merge branch 'master' into trino-disto-self-contain
tisonkun Sep 9, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions bin/pulsar
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ DEFAULT_PROXY_CONF=$PULSAR_HOME/conf/proxy.conf
DEFAULT_STANDALONE_CONF=$PULSAR_HOME/conf/standalone.conf
DEFAULT_WEBSOCKET_CONF=$PULSAR_HOME/conf/websocket.conf
DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml
DEFAULT_PULSAR_PRESTO_CONF=${PULSAR_HOME}/conf/presto
DEFAULT_FUNCTIONS_LOG_CONF=$PULSAR_HOME/conf/functions_log4j2.xml

# functions related variables
Expand All @@ -47,6 +46,7 @@ DEFAULT_FUNCTIONS_EXTRA_DEPS_DIR=$PULSAR_HOME/instances/deps
FUNCTIONS_EXTRA_DEPS_DIR=${PULSAR_FUNCTIONS_EXTRA_DEPS_DIR:-"${DEFAULT_FUNCTIONS_EXTRA_DEPS_DIR}"}
SQL_HOME=$PULSAR_HOME/pulsar-sql
PRESTO_HOME=${PULSAR_HOME}/lib/presto
DEFAULT_PULSAR_PRESTO_CONF=${PRESTO_HOME}/conf

pulsar_help() {
cat <<EOF
Expand Down Expand Up @@ -90,7 +90,7 @@ Environment variables:
PULSAR_PROXY_CONF Configuration file for Pulsar proxy (default: $DEFAULT_PROXY_CONF)
PULSAR_WORKER_CONF Configuration file for functions worker (default: $DEFAULT_WORKER_CONF)
PULSAR_STANDALONE_CONF Configuration file for standalone (default: $DEFAULT_STANDALONE_CONF)
PULSAR_PRESTO_CONF Configuration directory for Pulsar Presto (default: $DEFAULT_PULSAR_PRESTO_CONF)
PULSAR_PRESTO_CONF Configuration directory for Pulsar SQL (default: $DEFAULT_PULSAR_PRESTO_CONF)
PULSAR_EXTRA_OPTS Extra options to be passed to the jvm
PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath
PULSAR_PID_DIR Folder where the pulsar server PID file should be stored
Expand Down Expand Up @@ -180,14 +180,14 @@ fi
# find pulsar sql presto distribution location
check_presto_libraries() {
if [ ! -d "${PRESTO_HOME}" ]; then

BUILT_PRESTO_HOME="${SQL_HOME}/presto-distribution/target/pulsar-presto-distribution"
if [ ! -d "${BUILT_PRESTO_HOME}" ]; then
echo "\nCouldn't find presto distribution.";
echo "Make sure you've run 'mvn package'\n";
exit 1;
fi
PRESTO_HOME=${BUILT_PRESTO_HOME}
PULSAR_PRESTO_CONF=${BUILT_PRESTO_HOME}/conf
fi
}

Expand Down
3 changes: 3 additions & 0 deletions buildtools/src/main/resources/pulsar/checkstyle.xml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ page at http://checkstyle.sourceforge.net/config.html -->
<property name="max" value="120"/>
<property name="severity" value="error"/>

<!-- Checks only for Java files. -->
<property name="fileExtensions" value="java"/>

<!--
The default ignore pattern exempts the following elements:
- import statements
Expand Down
1 change: 1 addition & 0 deletions distribution/server/src/assemble/bin.xml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0 http://maven.apache.org/xsd/assembly-2.1.0.xsd">
<id>bin</id>
<formats>
<format>dir</format>
<format>tar.gz</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
Expand Down
2 changes: 1 addition & 1 deletion docker-compose/kitchen-sink/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,7 @@ services:
image: apachepulsar/pulsar-all:latest
restart: on-failure
command: >
bash -c "bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ conf/presto/catalog/pulsar.properties && \
bash -c "bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ lib/presto/conf/catalog/pulsar.properties && \
bin/apply-config-from-env.py conf/pulsar_env.sh && \
bin/watch-znode.py -z $$zookeeperServers -p /initialized-$$clusterName -w && \
exec bin/pulsar sql-worker run"
Expand Down
4 changes: 4 additions & 0 deletions pulsar-sql/presto-distribution/src/assembly/assembly.xml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@
<directory>${basedir}/../presto-pulsar-plugin/target/pulsar-presto-connector/</directory>
<outputDirectory>plugin/</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/resources/conf/</directory>
<outputDirectory>conf/</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
Expand Down
File renamed without changes.
2 changes: 1 addition & 1 deletion tests/docker-images/latest-version-image/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ COPY scripts/init-cluster.sh scripts/run-global-zk.sh scripts/run-local-zk.sh \
scripts/run-standalone.sh \
/pulsar/bin/

COPY conf/presto/jvm.config /pulsar/conf/presto/
COPY conf/presto/jvm.config /pulsar/lib/presto/conf

# copy python test examples
RUN mkdir -p /pulsar/instances/deps
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
# under the License.
#

bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ conf/presto/catalog/pulsar.properties && \
bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ lib/presto/conf/catalog/pulsar.properties && \
bin/apply-config-from-env.py conf/pulsar_env.sh

if [ -z "$NO_AUTOSTART" ]; then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -435,7 +435,7 @@ private PrestoWorkerContainer buildPrestoWorkerContainer(String hostName, boolea
.withEnv("pulsar.web-service-url", "http://pulsar-broker-0:8080")
.withEnv("SQL_PREFIX_pulsar.max-message-size", "" + spec.maxMessageSize)
.withClasspathResourceMapping(
resourcePath, "/pulsar/conf/presto/config.properties", BindMode.READ_WRITE);
resourcePath, "/pulsar/lib/presto/conf/config.properties", BindMode.READ_WRITE);
if (spec.queryLastMessage) {
container.withEnv("pulsar.bookkeeper-use-v2-protocol", "false")
.withEnv("pulsar.bookkeeper-explicit-interval", "10");
Expand Down