Skip to content

Commit

Permalink
Merge 9b14d17 into 4372e74
Browse files Browse the repository at this point in the history
  • Loading branch information
azeey authored Jul 17, 2023
2 parents 4372e74 + 9b14d17 commit 80679bd
Show file tree
Hide file tree
Showing 8 changed files with 169 additions and 32 deletions.
2 changes: 2 additions & 0 deletions .github/ci/after_make.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# It's necessary to install the python modules for the test.
make install
2 changes: 2 additions & 0 deletions .github/ci/packages.apt
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,5 @@ libprotoc-dev
libtinyxml2-dev
protobuf-compiler
ruby
python3-pytest
python3-protobuf
16 changes: 16 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,15 @@ set(
"gz_msgs_gen executable used in the gz_msgs_protoc CMake function.")
mark_as_advanced(GZ_MSGS_GEN_EXECUTABLE)

# Python interfaces vars
option(USE_SYSTEM_PATHS_FOR_PYTHON_INSTALLATION
"Install python modules in standard system paths in the system"
OFF)

option(USE_DIST_PACKAGES_FOR_PYTHON
"Use dist-packages instead of site-package to install python modules"
OFF)

#============================================================================
# Search for project-specific dependencies
#============================================================================
Expand Down Expand Up @@ -87,6 +96,10 @@ set(GZ_TOOLS_VER 1)
# Find Tinyxml2
gz_find_package(TINYXML2 REQUIRED PRIVATE PRETTY tinyxml2)

#--------------------------------------
# Find Python
find_package(Python3 REQUIRED COMPONENTS Interpreter)

#============================================================================
# Configure the build
#============================================================================
Expand All @@ -111,6 +124,9 @@ add_subdirectory(tools)
# projects.
add_subdirectory(proto)

# Generate python
add_subdirectory(python)

#============================================================================
# Create package information
#============================================================================
Expand Down
36 changes: 36 additions & 0 deletions python/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Append `_configured` to the file name so it doesn't interfere with tests.
# This happens because pytest will load the `gz.msgs` package from the build directory
# (because there's an __init__.py file there) instead of being redirected to
# `gz.msgs10` in the install directory, which is the intent of this `__init__.py` file.
set(python_init_file ${PROJECT_BINARY_DIR}/python/gz/${GS_DESIGNATION}/__init__.py_configured)
configure_file(${PROJECT_SOURCE_DIR}/python/src/__init__.py.in ${python_init_file})

install(FILES ${python_init_file} DESTINATION ${CMAKE_INSTALL_PREFIX}/${GZ_LIB_INSTALL_DIR}/python/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR} RENAME __init__.py)

if (BUILD_TESTING AND NOT WIN32)
set(python_tests
basic_TEST
)
execute_process(COMMAND "${Python3_EXECUTABLE}" -m pytest --version
OUTPUT_VARIABLE PYTEST_output
ERROR_VARIABLE PYTEST_error
RESULT_VARIABLE PYTEST_result)
if(${PYTEST_result} EQUAL 0)
set(pytest_FOUND TRUE)
else()
message(WARNING "Pytest package not available: ${PYTEST_error}")
message(WARNING "Output: ${PYTEST_output}")
endif()

foreach (test ${python_tests})
if (pytest_FOUND)
add_test(NAME ${test}.py COMMAND
"${Python3_EXECUTABLE}" -m pytest "${CMAKE_SOURCE_DIR}/python/test/${test}.py" --junitxml "${CMAKE_BINARY_DIR}/test_results/${test}.xml")
else()
add_test(NAME ${test}.py COMMAND
"${Python3_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/python/test/${test}.py")
endif()
set(_env_vars "PYTHONPATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/python/")
set_tests_properties(${test}.py PROPERTIES ENVIRONMENT "${_env_vars}")
endforeach()
endif()
33 changes: 33 additions & 0 deletions python/src/__init__.py.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Copyright (C) 2023 Open Source Robotics Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License")
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file is a workaround for a limitation in out protobuf python generation
# where a message that depends on another message will try to import the
# corresponding python module using `gz.msgs` as the package name. However,
# we're installing the python modules in a directory that contains the gz-msgs
# major version number, so the import fails. This hack here overwrites the
# entry for the unversioned module name in `sys.modules` to point to the
# versioned module the first time a message module is loaded. Subsequent
# imports with or without the major version number will work properly.

import sys

unversioned_module = "gz.msgs"
versioned_module = "gz.msgs@PROJECT_VERSION_MAJOR@"
if unversioned_module in sys.modules:
print("Looks like you are combining different versions of {}. Found {} and"
"{} This is not supported".format(sys.modules[unversioned_module],
sys.modules[versioned_module]))
else:
sys.modules[unversioned_module] = sys.modules[versioned_module]
38 changes: 38 additions & 0 deletions python/test/basic_TEST.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# Copyright (C) 2023 Open Source Robotics Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License")
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from gz.msgs10.vector3d_pb2 import Vector3d

import unittest


class BasicTest(unittest.TestCase):

def test_serialization(self):
msg = Vector3d()
msg.x = 1
msg.y = 2
msg.z = 3

serialized_msg = msg.SerializeToString()
self.assertGreater(len(serialized_msg), 0)

msg_from_serialized = Vector3d()
self.assertNotEqual(msg_from_serialized, msg)
msg_from_serialized.ParseFromString(serialized_msg)
self.assertEqual(msg_from_serialized, msg)


if __name__ == '__main__':
unittest.main()
61 changes: 35 additions & 26 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,40 +19,38 @@ if(INSTALL_GZ_MSGS_GEN_EXECUTABLE)
install(FILES $<TARGET_FILE:gz_msgs_gen> DESTINATION ${GZ_BIN_INSTALL_DIR} RENAME ign_msgs_gen PERMISSIONS OWNER_EXECUTE)
endif()

find_package(Python3 REQUIRED COMPONENTS Interpreter)

##################################################
# A function that calls protoc on a protobuf file
# Options:
# GENERATE_RUBY - generates ruby code for the message if specified
# GENERATE_PYTHON - generates python code for the message if specified
# GENERATE_CPP - generates c++ code for the message if specified
# One value arguments:
# PROTO_PACKAGE - Protobuf package the file belongs to (e.g. ".gz.msgs")
# PROTOC_EXEC - Path to protoc
# INPUT_PROTO - Path to the input .proto file
# OUTPUT_CPP_DIR - Path where C++ files are saved
# OUTPUT_RUBY_DIR - Path where Ruby files are saved
# OUTPUT_PYTHON_DIR - Path where Python files are saved
# OUTPUT_INCLUDES - A CMake variable name containing a list that the C++ header path should be appended to
# OUTPUT_CPP_HH_VAR - A CMake variable name containing a list that the C++ header path should be appended to
# OUTPUT_GZ_CPP_HH_VAR - A CMake variable name containing a list that the C++ header path should be appended to
# OUTPUT_CPP_CC_VAR - A Cmake variable name containing a list that the C++ source path should be appended to
# OUTPUT_RUBY_VAR - A Cmake variable name containing a list that the ruby file should be apenned to
# OUTPUT_PYTHON_VAR - A Cmake variable name containing a list that the python file should be appended to
# Multi value arguments
# PROTO_PATH - Passed to protoc --proto_path
function(gz_msgs_protoc)
set(options GENERATE_RUBY GENERATE_CPP)
set(options GENERATE_PYTHON GENERATE_CPP)
set(oneValueArgs
PROTO_PACKAGE
PROTOC_EXEC
INPUT_PROTO
OUTPUT_CPP_DIR
OUTPUT_RUBY_DIR
OUTPUT_PYTHON_DIR
OUTPUT_INCLUDES
OUTPUT_CPP_HH_VAR
OUTPUT_GZ_CPP_HH_VAR
OUTPUT_DETAIL_CPP_HH_VAR
OUTPUT_CPP_CC_VAR
OUTPUT_RUBY_VAR)
OUTPUT_PYTHON_VAR)
set(multiValueArgs PROTO_PATH)

cmake_parse_arguments(gz_msgs_protoc "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
Expand Down Expand Up @@ -97,13 +95,13 @@ function(gz_msgs_protoc)
set(${gz_msgs_protoc_OUTPUT_CPP_CC_VAR} ${${gz_msgs_protoc_OUTPUT_CPP_CC_VAR}} PARENT_SCOPE)
endif()

if(gz_msgs_protoc_GENERATE_RUBY)
file(MAKE_DIRECTORY ${gz_msgs_protoc_OUTPUT_RUBY_DIR})
set(output_ruby "${gz_msgs_protoc_OUTPUT_RUBY_DIR}${proto_package_dir}/${FIL_WE}_pb.rb")
list(APPEND ${gz_msgs_protoc_OUTPUT_RUBY_VAR} ${output_ruby})
list(APPEND output_files ${output_ruby})
list(APPEND protoc_args "--ruby_out=${gz_msgs_protoc_OUTPUT_RUBY_DIR}")
set(${gz_msgs_protoc_OUTPUT_RUBY_VAR} ${${gz_msgs_protoc_OUTPUT_RUBY_VAR}} PARENT_SCOPE)
if(gz_msgs_protoc_GENERATE_PYTHON)
file(MAKE_DIRECTORY ${gz_msgs_protoc_OUTPUT_PYTHON_DIR})
# Note: Both proto2 and proto3 use the _pb2.py suffix (https://protobuf.dev/reference/python/python-generated/#invocation)
set(output_python "${gz_msgs_protoc_OUTPUT_PYTHON_DIR}${proto_package_dir}/${FIL_WE}_pb2.py")
list(APPEND ${gz_msgs_protoc_OUTPUT_PYTHON_VAR} ${output_python})
list(APPEND output_files ${output_python})
set(${gz_msgs_protoc_OUTPUT_PYTHON_VAR} ${${gz_msgs_protoc_OUTPUT_PYTHON_VAR}} PARENT_SCOPE)
endif()


Expand All @@ -120,10 +118,10 @@ function(gz_msgs_protoc)
--output-cpp-path "${gz_msgs_protoc_OUTPUT_CPP_DIR}")
endif()

if(${gz_msgs_protoc_GENERATE_RUBY})
if(${gz_msgs_protoc_GENERATE_PYTHON})
list(APPEND GENERATE_ARGS
--generate-ruby
--output-ruby-path "${gz_msgs_protoc_OUTPUT_RUBY_DIR}")
--generate-python
--output-python-path "${gz_msgs_protoc_OUTPUT_PYTHON_DIR}")
endif()

add_custom_command(
Expand Down Expand Up @@ -151,15 +149,15 @@ foreach(proto_file ${proto_files})
PROTO_PACKAGE
.gz.msgs
GENERATE_CPP
GENERATE_RUBY
GENERATE_PYTHON
INPUT_PROTO
${proto_file}
PROTOC_EXEC
protobuf::protoc
OUTPUT_CPP_DIR
"${PROJECT_BINARY_DIR}/include"
OUTPUT_RUBY_DIR
"${PROJECT_BINARY_DIR}/ruby"
OUTPUT_PYTHON_DIR
"${PROJECT_BINARY_DIR}/python"
OUTPUT_INCLUDES
gen_includes
OUTPUT_CPP_HH_VAR
Expand All @@ -170,8 +168,8 @@ foreach(proto_file ${proto_files})
gen_ign_headers
OUTPUT_CPP_CC_VAR
gen_sources
OUTPUT_RUBY_VAR
gen_ruby_scripts
OUTPUT_PYTHON_VAR
gen_python_scripts
PROTO_PATH
"${PROJECT_SOURCE_DIR}/proto")
endforeach()
Expand All @@ -193,11 +191,22 @@ if(MSVC)
add_definitions(/bigobj)
endif()

set_source_files_properties(${gen_headers} ${gen_ign_headers} ${gen_detail_headers} ${gen_sources} ${gen_ruby_scripts}
set_source_files_properties(${gen_headers} ${gen_ign_headers} ${gen_detail_headers} ${gen_sources} ${gen_python_scripts}
PROPERTIES GENERATED TRUE)

message(STATUS "Installing Ruby messages to ${CMAKE_INSTALL_PREFIX}/${GZ_LIB_INSTALL_DIR}/ruby/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR}")
install(FILES ${gen_ruby_scripts} DESTINATION ${CMAKE_INSTALL_PREFIX}/${GZ_LIB_INSTALL_DIR}/ruby/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR})
if(USE_SYSTEM_PATHS_FOR_PYTHON_INSTALLATION)
if(USE_DIST_PACKAGES_FOR_PYTHON)
string(REPLACE "site-packages" "dist-packages" GZ_PYTHON_INSTALL_PATH ${Python3_SITELIB})
else()
# Python3_SITELIB might use dist-packages in some platforms
string(REPLACE "dist-packages" "site-packages" GZ_PYTHON_INSTALL_PATH ${Python3_SITELIB})
endif()
else()
# If not a system installation, respect local paths
set(GZ_PYTHON_INSTALL_PATH ${GZ_LIB_INSTALL_DIR}/python)
endif()
message(STATUS "Installing Python messages to ${GZ_PYTHON_INSTALL_PATH}/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR}")
install(FILES ${gen_python_scripts} DESTINATION ${GZ_PYTHON_INSTALL_PATH}/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR})

# Install gz/msgs
gz_install_includes(
Expand Down
13 changes: 7 additions & 6 deletions tools/gz_msgs_generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,14 @@ def main(argv=sys.argv[1:]):
help='Flag to indicate if C++ bindings should be generated',
action='store_true')
parser.add_argument(
'--generate-ruby',
help='Flag to indicate if Ruby bindings should be generated',
'--generate-python',
help='Flag to indicate if Python bindings should be generated',
action='store_true')
parser.add_argument(
'--output-cpp-path',
help='The basepath of the generated C++ files')
parser.add_argument(
'--output-ruby-path',
'--output-python-path',
help='The basepath of the generated C++ files')
parser.add_argument(
'--proto-path',
Expand All @@ -57,7 +57,7 @@ def main(argv=sys.argv[1:]):
args = parser.parse_args(argv)

for input_file in args.input_path:
# First generate the base cpp and ruby files
# First generate the base cpp and python files
cmd = [args.protoc_exec]

for pp in args.proto_path:
Expand All @@ -67,11 +67,12 @@ def main(argv=sys.argv[1:]):
cmd += [f'--plugin=protoc-gen-ignmsgs={args.gz_generator_bin}']
cmd += [f'--cpp_out=dllexport_decl=GZ_MSGS_VISIBLE:{args.output_cpp_path}']
cmd += [f'--ignmsgs_out={args.output_cpp_path}']
if args.generate_ruby:
cmd += [f'--ruby_out=dllexport_decl=GZ_MSGS_VISIBLE:{args.output_ruby_path}']
if args.generate_python:
cmd += [f'--python_out={args.output_python_path}']
cmd += [input_file]

try:
print("cmd:", cmd)
subprocess.check_call(cmd)
except subprocess.CalledProcessError as e:
print(f'Failed to execute protoc compiler: {e}')
Expand Down

0 comments on commit 80679bd

Please sign in to comment.