Plasma Python extension packaging: It compiles!

cpp/src/plasma/CMakeLists.txt

@@ -24,14 +24,13 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../python/cmake_
 find_package(PythonLibsNew REQUIRED)
 find_package(Threads)
 
-option(PLASMA_PYTHON
-  "Build the Plasma Python extensions"
-  OFF)
-
 if(APPLE)
   SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
 endif(APPLE)
 
+set(PLASMA_SO_VERSION "0")
+set(PLASMA_ABI_VERSION "${PLASMA_SO_VERSION}.0.0")
+
 include_directories(SYSTEM ${PYTHON_INCLUDE_DIRS})
 include_directories("${FLATBUFFERS_INCLUDE_DIR}" "${CMAKE_CURRENT_LIST_DIR}/" "${CMAKE_CURRENT_LIST_DIR}/thirdparty/" "${CMAKE_CURRENT_LIST_DIR}/../")
 
@@ -42,7 +41,7 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-conversion")
 # Compile flatbuffers
 
 set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs" "${CMAKE_CURRENT_LIST_DIR}/format/common.fbs")
-set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
+set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/)
 
 set(PLASMA_FBS_OUTPUT_FILES
   "${OUTPUT_DIR}/common_generated.h"
@@ -99,28 +98,26 @@ set_source_files_properties(malloc.cc PROPERTIES COMPILE_FLAGS "-Wno-error -O3")
 add_executable(plasma_store store.cc)
 target_link_libraries(plasma_store plasma_static)
 
+# Headers: top level
+install(FILES
+  common.h
+  client.h
+  plasma.h
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/plasma")
+
+# pkg-config support
+configure_file(plasma.pc.in
+  "${CMAKE_CURRENT_BINARY_DIR}/plasma.pc"
+  @ONLY)
+install(
+  FILES "${CMAKE_CURRENT_BINARY_DIR}/plasma.pc"
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
+
+#######################################
+# Unit tests
+#######################################
+
 ADD_ARROW_TEST(test/serialization_tests)
 ARROW_TEST_LINK_LIBRARIES(test/serialization_tests plasma_static)
 ADD_ARROW_TEST(test/client_tests)
 ARROW_TEST_LINK_LIBRARIES(test/client_tests plasma_static)
-
-if (PLASMA_PYTHON)
-  include(UseCython)
-  find_package(NumPy REQUIRED)
-
-  include_directories(SYSTEM
-    ${NUMPY_INCLUDE_DIRS}
-    ${PYTHON_INCLUDE_DIRS})
-
-  set_source_files_properties(plasma/plasma.pyx PROPERTIES CYTHON_IS_CXX 1)
-
-  # for Cython
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable")
-
-  cython_add_module(plasma
-      plasma_pyx
-      plasma_output
-      plasma/plasma.pyx)
-
-  target_link_libraries(plasma arrow_static plasma_static)
-endif()

cpp/src/plasma/client.cc

@@ -58,8 +58,24 @@ constexpr int64_t kThreadPoolSize = 8;
 constexpr int64_t kBytesInMB = 1 << 20;
 static std::vector<std::thread> threadpool_(kThreadPoolSize);
 
+struct ObjectInUseEntry {
+  /// A count of the number of times this client has called PlasmaClient::Create
+  /// or
+  /// PlasmaClient::Get on this object ID minus the number of calls to
+  /// PlasmaClient::Release.
+  /// When this count reaches zero, we remove the entry from the ObjectsInUse
+  /// and decrement a count in the relevant ClientMmapTableEntry.
+  int count;
+  /// Cached information to read the object.
+  PlasmaObject object;
+  /// A flag representing whether the object has been sealed.
+  bool is_sealed;
+};
+
 PlasmaClient::PlasmaClient() {}
 
+PlasmaClient::~PlasmaClient() {}
+
 // If the file descriptor fd has been mmapped in this client process before,
 // return the pointer that was returned by mmap, otherwise mmap it and store the
 // pointer in a hash table.

cpp/src/plasma/client.h

@@ -22,10 +22,13 @@
 #include <time.h>
 
 #include <deque>
+#include <memory>
 #include <string>
+#include <unordered_map>
 
+#include "arrow/status.h"
 #include "arrow/util/visibility.h"
-#include "plasma/plasma.h"
+#include "plasma/common.h"
 
 using arrow::Status;
 
@@ -66,24 +69,16 @@ struct ClientMmapTableEntry {
   int count;
 };
 
-struct ObjectInUseEntry {
-  /// A count of the number of times this client has called PlasmaClient::Create
-  /// or
-  /// PlasmaClient::Get on this object ID minus the number of calls to
-  /// PlasmaClient::Release.
-  /// When this count reaches zero, we remove the entry from the ObjectsInUse
-  /// and decrement a count in the relevant ClientMmapTableEntry.
-  int count;
-  /// Cached information to read the object.
-  PlasmaObject object;
-  /// A flag representing whether the object has been sealed.
-  bool is_sealed;
-};
+struct ObjectInUseEntry;
+class ObjectRequest;
+class PlasmaObject;
 
 class ARROW_EXPORT PlasmaClient {
  public:
   PlasmaClient();
 
+  ~PlasmaClient();
+
   /// Connect to the local plasma store and plasma manager. Return
   /// the resulting connection.
   ///

cpp/src/plasma/common.cc

@@ -19,7 +19,7 @@
 
 #include <random>
 
-#include "format/plasma_generated.h"
+#include "plasma/plasma_generated.h"
 
 namespace plasma {
 

cpp/src/plasma/common.h

@@ -62,6 +62,9 @@ typedef UniqueID ObjectID;
 
 arrow::Status plasma_error_status(int plasma_error);
 
+/// Size of object hash digests.
+constexpr int64_t kDigestSize = sizeof(uint64_t);
+
 } // namespace plasma
 
 #endif  // PLASMA_COMMON_H

cpp/src/plasma/plasma.h

@@ -32,8 +32,8 @@
 
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
-#include "format/common_generated.h"
 #include "plasma/common.h"
+#include "plasma/common_generated.h"
 
 namespace plasma {
 
@@ -62,14 +62,11 @@ namespace plasma {
 /// Allocation granularity used in plasma for object allocation.
 #define BLOCK_SIZE 64
 
-/// Size of object hash digests.
-constexpr int64_t kDigestSize = sizeof(uint64_t);
-
 struct Client;
 
 /// Object request data structure. Used in the plasma_wait_for_objects()
 /// argument.
-typedef struct {
+struct ObjectRequest {
   /// The ID of the requested object. If ID_NIL request any object.
   ObjectID object_id;
   /// Request associated to the object. It can take one of the following values:
@@ -86,23 +83,23 @@ typedef struct {
   ///  - PLASMA_CLIENT_IN_TRANSFER, if the object is currently being scheduled
   ///    for being transferred or it is transferring.
   int status;
-} ObjectRequest;
+};
 
 /// Mapping from object IDs to type and status of the request.
 typedef std::unordered_map<ObjectID, ObjectRequest, UniqueIDHasher> ObjectRequestMap;
 
 /// Handle to access memory mapped file and map it into client address space.
-typedef struct {
+struct object_handle {
   /// The file descriptor of the memory mapped file in the store. It is used as
   /// a unique identifier of the file in the client to look up the corresponding
   /// file descriptor on the client's side.
   int store_fd;
   /// The size in bytes of the memory mapped file.
   int64_t mmap_size;
-} object_handle;
+};
 
 // TODO(pcm): Replace this by the flatbuffers message PlasmaObjectSpec.
-typedef struct {
+struct PlasmaObject {
   /// Handle for memory mapped file the object is stored in.
   object_handle handle;
   /// The offset in bytes in the memory mapped file of the data.
@@ -113,28 +110,28 @@ typedef struct {
   int64_t data_size;
   /// The size in bytes of the metadata.
   int64_t metadata_size;
-} PlasmaObject;
+};
 
-typedef enum {
+enum object_state {
   /// Object was created but not sealed in the local Plasma Store.
   PLASMA_CREATED = 1,
   /// Object is sealed and stored in the local Plasma Store.
   PLASMA_SEALED
-} object_state;
+};
 
-typedef enum {
+enum object_status {
   /// The object was not found.
   OBJECT_NOT_FOUND = 0,
   /// The object was found.
   OBJECT_FOUND = 1
-} object_status;
+};
 
-typedef enum {
+enum object_request_type {
   /// Query for object in the local plasma store.
   PLASMA_QUERY_LOCAL = 1,
   /// Query for object in the local plasma store or in a remote plasma store.
   PLASMA_QUERY_ANYWHERE
-} object_request_type;
+};
 
 /// This type is used by the Plasma store. It is here because it is exposed to
 /// the eviction policy.

cpp/src/plasma/plasma.pc.in

@@ -15,4 +15,15 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from .plasma import *
+prefix=@CMAKE_INSTALL_PREFIX@
+libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
+includedir=${prefix}/include
+
+so_version=@PLASMA_SO_VERSION@
+abi_version=@PLASMA_ABI_VERSION@
+
+Name: Plasma
+Description: Plasma is an in-memory object store and cache for big data.
+Version: @PLASMA_VERSION@
+Libs: -L${libdir} -lplasma
+Cflags: -I${includedir}

cpp/src/plasma/protocol.cc

@@ -18,7 +18,7 @@
 #include "plasma/protocol.h"
 
 #include "flatbuffers/flatbuffers.h"
-#include "format/plasma_generated.h"
+#include "plasma/plasma_generated.h"
 
 #include "plasma/common.h"
 #include "plasma/io.h"

cpp/src/plasma/protocol.h

@@ -21,7 +21,7 @@
 #include <vector>
 
 #include "arrow/status.h"
-#include "format/plasma_generated.h"
+#include "plasma/plasma_generated.h"
 #include "plasma/plasma.h"
 
 namespace plasma {

cpp/src/plasma/setup.py

@@ -1,133 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-import shutil
-import sys
-
-from setuptools import setup, find_packages, Distribution
-import setuptools.command.build_ext as _build_ext
-from distutils import sysconfig
-
-class build_ext(_build_ext.build_ext):
-    def run(self):
-        self._run_cmake()
-
-    def _run_cmake(self):
-        self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower()
-
-        # The directory containing this setup.py
-        source = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../")
-
-        # The staging directory for the module being built
-        build_temp = os.path.join(os.getcwd(), self.build_temp)
-        build_lib = os.path.join(os.getcwd(), self.build_lib)
-
-        # Change to the build directory
-        saved_cwd = os.getcwd()
-        if not os.path.isdir(self.build_temp):
-            self.mkpath(self.build_temp)
-        os.chdir(self.build_temp)
-
-        static_lib_option = ''
-
-        cmake_options = [
-            '-DPYTHON_EXECUTABLE=%s' % sys.executable,
-            '-DARROW_PLASMA=on',
-            '-DPLASMA_PYTHON=on',
-            '-DARROW_BUILD_TESTS=off',
-            static_lib_option,
-        ]
-
-        cmake_command = (['cmake'] + cmake_options + [source])
-
-        print("-- Runnning cmake for plasma")
-        self.spawn(cmake_command)
-        print("-- Finished cmake for plasma")
-
-        args = ['make']
-        print("-- Running make for plasma")
-        self.spawn(args)
-        print("-- Finished make for plasma")
-
-        self._found_names = []
-        for name in ["plasma"]:
-            built_path = self.get_ext_built(name)
-            if not os.path.exists(built_path):
-                raise RuntimeError('plasma C-extension failed to build:',
-                                   os.path.abspath(built_path))
-
-            ext_path = os.path.join(build_lib, self._get_cmake_ext_path(name))
-            if os.path.exists(ext_path):
-                os.remove(ext_path)
-            self.mkpath(os.path.dirname(ext_path))
-            print('Moving built C-extension', built_path,
-                  'to build path', ext_path)
-            shutil.move(self.get_ext_built(name), ext_path)
-            self._found_names.append(name)
-
-        # Move the plasma store
-        build_py = self.get_finalized_command('build_py')
-        source = os.path.join(self.build_type, "plasma_store")
-        target = os.path.join(build_lib, build_py.get_package_dir('plasma'), "plasma_store")
-        shutil.move(source, target)
-
-        os.chdir(saved_cwd)
-
-    def _get_cmake_ext_path(self, name):
-        # Get the package directory from build_py
-        build_py = self.get_finalized_command('build_py')
-        package_dir = build_py.get_package_dir('plasma')
-        # This is the name of the arrow C-extension
-        suffix = sysconfig.get_config_var('EXT_SUFFIX')
-        if suffix is None:
-            suffix = sysconfig.get_config_var('SO')
-        filename = name + suffix
-        return os.path.join(package_dir, filename)
-
-    def get_ext_built(self, name):
-        if sys.platform == 'win32':
-            head, tail = os.path.split(name)
-            suffix = sysconfig.get_config_var('SO')
-            return os.path.join(head, self.build_type, tail + suffix)
-        else:
-            suffix = sysconfig.get_config_var('SO')
-            return os.path.join(self.build_type, name + suffix)
-
-    def get_names(self):
-        return self._found_names
-
-    def get_outputs(self):
-        # Just the C extensions
-        # regular_exts = _build_ext.get_outputs(self)
-        return [self._get_cmake_ext_path(name)
-                for name in self.get_names()]
-
-class BinaryDistribution(Distribution):
-  def has_ext_modules(self):
-    return True
-
-setup(name="plasma",
-      version="0.0.1",
-      packages=["plasma"],
-      cmdclass={"build_ext": build_ext},
-      # The BinaryDistribution argument triggers build_ext.
-      distclass=BinaryDistribution,
-      install_requires=["numpy"],
-      include_package_data=True,
-      zip_safe=False,
-      license="Apache 2.0")

cpp/src/plasma/store.cc

@@ -49,7 +49,7 @@
 #include <unordered_set>
 #include <vector>
 
-#include "format/common_generated.h"
+#include "plasma/common_generated.h"
 #include "plasma/common.h"
 #include "plasma/fling.h"
 #include "plasma/io.h"

python/CMakeLists.txt

@@ -220,6 +220,10 @@ include_directories(SYSTEM
 find_package(Arrow REQUIRED)
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 
+## Plasma
+find_package(Plasma)
+include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
+
 function(bundle_arrow_lib library_path)
   get_filename_component(LIBRARY_DIR ${${library_path}} DIRECTORY)
   get_filename_component(LIBRARY_NAME ${${library_path}} NAME_WE)
@@ -276,6 +280,7 @@ endif()
 
 set(CYTHON_EXTENSIONS
   lib
+  plasma
 )
 
 set(LINK_LIBS

python/cmake_modules/FindPlasma.cmake

@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# - Find PLASMA (plasma/client.h, libplasma.a, libplasma.so)
+# This module defines
+#  PLASMA_INCLUDE_DIR, directory containing headers
+#  PLASMA_LIBS, directory containing plasma libraries
+#  PLASMA_STATIC_LIB, path to libplasma.a
+#  PLASMA_SHARED_LIB, path to libplasma's shared library
+#  PLASMA_SHARED_IMP_LIB, path to libplasma's import library (MSVC only)
+#  PLASMA_FOUND, whether plasma has been found
+
+include(FindPkgConfig)
+
+if ("$ENV{PLASMA_HOME}" STREQUAL "")
+  pkg_check_modules(PLASMA plasma)
+  if (PLASMA_FOUND)
+    pkg_get_variable(PLASMA_ABI_VERSION plasma abi_version)
+    message(STATUS "Plasma ABI version: ${PLASMA_ABI_VERSION}")
+    pkg_get_variable(PLASMA_SO_VERSION plasma so_version)
+    message(STATUS "Plasma SO version: ${PLASMA_SO_VERSION}")
+    set(PLASMA_INCLUDE_DIR ${PLASMA_INCLUDE_DIRS})
+    set(PLASMA_LIBS ${PLASMA_LIBRARY_DIRS})
+    set(PLASMA_SEARCH_LIB_PATH ${PLASMA_LIBRARY_DIRS})
+  endif()
+else()
+  set(PLASMA_HOME "$ENV{PLASMA_HOME}")
+
+  set(PLASMA_SEARCH_HEADER_PATHS
+    ${PLASMA_HOME}/include
+    )
+
+  set(PLASMA_SEARCH_LIB_PATH
+    ${PLASMA_HOME}/lib
+    )
+
+  find_path(PLASMA_INCLUDE_DIR plasma/client.h PATHS
+    ${PLASMA_SEARCH_HEADER_PATHS}
+    # make sure we don't accidentally pick up a different version
+    NO_DEFAULT_PATH
+    )
+endif()
+
+find_library(PLASMA_LIB_PATH NAMES plasma
+  PATHS
+  ${PLASMA_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
+get_filename_component(PLASMA_LIBS ${PLASMA_LIB_PATH} DIRECTORY)
+
+if (PLASMA_INCLUDE_DIR AND PLASMA_LIBS)
+  set(PLASMA_FOUND TRUE)
+  set(PLASMA_LIB_NAME plasma)
+
+  set(PLASMA_STATIC_LIB ${PLASMA_PYTHON_LIB_PATH}/lib${PLASMA_LIB_NAME}.a)
+
+  set(PLASMA_SHARED_LIB ${PLASMA_LIBS}/lib${PLASMA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+endif()
+
+if (PLASMA_FOUND)
+  if (NOT Plasma_FIND_QUIETLY)
+    message(STATUS "Found the Plasma core library: ${PLASMA_LIB_PATH}")
+  endif ()
+else ()
+  if (NOT Plasma_FIND_QUIETLY)
+    set(PLASMA_ERR_MSG "Could not find the Plasma library. Looked for headers")
+    set(PLASMA_ERR_MSG "${PLASMA_ERR_MSG} in ${PLASMA_SEARCH_HEADER_PATHS}, and for libs")
+    set(PLASMA_ERR_MSG "${PLASMA_ERR_MSG} in ${PLASMA_SEARCH_LIB_PATH}")
+    if (Plasma_FIND_REQUIRED)
+      message(FATAL_ERROR "${PLASMA_ERR_MSG}")
+    else (Plasma_FIND_REQUIRED)
+      message(STATUS "${PLASMA_ERR_MSG}")
+    endif (Plasma_FIND_REQUIRED)
+  endif ()
+  set(PLASMA_FOUND FALSE)
+endif ()
+
+mark_as_advanced(
+  PLASMA_INCLUDE_DIR
+  PLASMA_STATIC_LIB
+  PLASMA_SHARED_LIB
+  PLASMA_PYTHON_STATIC_LIB
+  PLASMA_PYTHON_SHARED_LIB
+)

python/pyarrow/plasma.pyx

@@ -53,7 +53,7 @@ cdef extern from "plasma/common.h" nogil:
         c_string binary() const
 
 
-cdef extern from "plasma/plasma.h":
+cdef extern from "plasma/common.h":
     cdef int64_t kDigestSize" plasma::kDigestSize"