From 9147d00f50fafd45d6df6c4dbb7ec0dcd6b3dfe0 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 29 Jun 2018 12:30:19 +0100 Subject: [PATCH] incorporate https://github.com/tensorflow/tensorflow/pull/20284 up to c858523 --- tensorflow/tools/pip_package/BUILD | 8 +- tensorflow/tools/pip_package/build_pip_package.sh | 2 +- tensorflow/workspace.bzl | 14 ++ third_party/repo.bzl | 43 ++++-- third_party/systemlibs/BUILD | 1 + third_party/systemlibs/BUILD.tpl | 1 + third_party/systemlibs/astor.BUILD | 12 ++ third_party/systemlibs/build_defs.bzl.tpl | 32 +++++ third_party/systemlibs/curl.BUILD | 13 ++ third_party/systemlibs/cython.BUILD | 20 +++ third_party/systemlibs/flatbuffers.BUILD | 40 ++++++ third_party/systemlibs/gif.BUILD | 13 ++ third_party/systemlibs/grpc.BUILD | 66 +++++++++ third_party/systemlibs/jemalloc.BUILD | 31 +++++ third_party/systemlibs/jpeg.BUILD | 13 ++ third_party/systemlibs/jsoncpp.BUILD | 38 +++++ third_party/systemlibs/lmdb.BUILD | 13 ++ third_party/systemlibs/nasm.BUILD | 12 ++ third_party/systemlibs/pcre.BUILD | 13 ++ third_party/systemlibs/png.BUILD | 13 ++ third_party/systemlibs/re2.BUILD | 13 ++ third_party/systemlibs/six.BUILD | 11 ++ third_party/systemlibs/snappy.BUILD | 13 ++ third_party/systemlibs/sqlite.BUILD | 16 +++ third_party/systemlibs/swig.BUILD | 23 ++++ third_party/systemlibs/syslibs_configure.bzl | 160 ++++++++++++++++++++++ third_party/systemlibs/termcolor.BUILD | 12 ++ third_party/systemlibs/zlib.BUILD | 13 ++ 28 files changed, 644 insertions(+), 15 deletions(-) mode change 100755 => 100644 tensorflow/tools/pip_package/build_pip_package.sh create mode 100644 third_party/systemlibs/BUILD create mode 100644 third_party/systemlibs/BUILD.tpl create mode 100644 third_party/systemlibs/astor.BUILD create mode 100644 third_party/systemlibs/build_defs.bzl.tpl create mode 100644 third_party/systemlibs/curl.BUILD create mode 100644 third_party/systemlibs/cython.BUILD create mode 100644 third_party/systemlibs/flatbuffers.BUILD create mode 100644 third_party/systemlibs/gif.BUILD create mode 100644 third_party/systemlibs/grpc.BUILD create mode 100644 third_party/systemlibs/jemalloc.BUILD create mode 100644 third_party/systemlibs/jpeg.BUILD create mode 100644 third_party/systemlibs/jsoncpp.BUILD create mode 100644 third_party/systemlibs/lmdb.BUILD create mode 100644 third_party/systemlibs/nasm.BUILD create mode 100644 third_party/systemlibs/pcre.BUILD create mode 100644 third_party/systemlibs/png.BUILD create mode 100644 third_party/systemlibs/re2.BUILD create mode 100644 third_party/systemlibs/six.BUILD create mode 100644 third_party/systemlibs/snappy.BUILD create mode 100644 third_party/systemlibs/sqlite.BUILD create mode 100644 third_party/systemlibs/swig.BUILD create mode 100644 third_party/systemlibs/syslibs_configure.bzl create mode 100644 third_party/systemlibs/termcolor.BUILD create mode 100644 third_party/systemlibs/zlib.BUILD diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index 9d4148c..aa359ef 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -12,6 +12,7 @@ load( load("//third_party/mkl:build_defs.bzl", "if_mkl") load("//tensorflow:tensorflow.bzl", "if_cuda") load("@local_config_tensorrt//:build_defs.bzl", "if_tensorrt") +load("@local_config_syslibs//:build_defs.bzl", "if_not_system_lib") load("//tensorflow/core:platform/default/build_config_root.bzl", "tf_additional_license_deps") # This returns a list of headers of all public header libraries (e.g., @@ -138,7 +139,6 @@ filegroup( "@gast_archive//:PKG-INFO", "@gemmlowp//:LICENSE", "@gif_archive//:COPYING", - "@grpc//:LICENSE", "@highwayhash//:LICENSE", "@jemalloc//:COPYING", "@jpeg//:LICENSE.md", @@ -147,8 +147,6 @@ filegroup( "@lmdb//:LICENSE", "@local_config_nccl//:LICENSE", "@local_config_sycl//sycl:LICENSE.text", - "@grpc//third_party/nanopb:LICENSE.txt", - "@grpc//third_party/address_sorting:LICENSE", "@nasm//:LICENSE", "@nsync//:LICENSE", "@pcre//:LICENCE", @@ -162,6 +160,10 @@ filegroup( "@org_python_pypi_backports_weakref//:LICENSE", ] + if_mkl([ "//third_party/mkl:LICENSE", + ]) + if_not_system_lib("grpc", [ + "@grpc//:LICENSE", + "@grpc//third_party/nanopb:LICENSE.txt", + "@grpc//third_party/address_sorting:LICENSE", ]) + tf_additional_license_deps(), ) diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh old mode 100755 new mode 100644 index a7af3f3..5be5265 --- a/tensorflow/tools/pip_package/build_pip_package.sh +++ b/tensorflow/tools/pip_package/build_pip_package.sh @@ -24,7 +24,7 @@ function real_path() { function cp_external() { local src_dir=$1 local dest_dir=$2 - for f in `find "$src_dir" -maxdepth 1 -mindepth 1 ! -name '*local_config_cuda*' ! -name '*local_config_tensorrt*' ! -name '*org_tensorflow*'`; do + for f in `find "$src_dir" -maxdepth 1 -mindepth 1 ! -name '*local_config_cuda*' ! -name '*local_config_tensorrt*' ! -name '*local_config_syslibs*' ! -name '*org_tensorflow*'`; do cp -R "$f" "$dest_dir" done mkdir -p "${dest_dir}/local_config_cuda/cuda/cuda/" diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 50a6959..e803ab4 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -8,6 +8,7 @@ load("//third_party/git:git_configure.bzl", "git_configure") load("//third_party/py:python_configure.bzl", "python_configure") load("//third_party/sycl:sycl_configure.bzl", "sycl_configure") +load("//third_party/systemlibs:syslibs_configure.bzl", "syslibs_configure") load("//third_party/toolchains/clang6:repo.bzl", "clang6_configure") load("//third_party/toolchains/cpus/arm:arm_compiler_configure.bzl", "arm_compiler_configure") load("//third_party:repo.bzl", "tf_http_archive") @@ -35,6 +36,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): nccl_configure(name="local_config_nccl") git_configure(name="local_config_git") sycl_configure(name="local_config_sycl") + syslibs_configure(name="local_config_syslibs") python_configure(name="local_config_python") # For windows bazel build @@ -205,6 +207,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "00b0891c678c065446ca59bcee64719d0096d54d6886e6e472aeee2e170ae324", strip_prefix = "nasm-2.12.02", build_file = clean_dep("//third_party:nasm.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:nasm.BUILD"), ) tf_http_archive( @@ -216,6 +219,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "1a17020f859cb12711175a67eab5c71fc1904e04b587046218e36106e07eabde", strip_prefix = "libjpeg-turbo-1.5.3", build_file = clean_dep("//third_party/jpeg:jpeg.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:jpeg.BUILD"), ) tf_http_archive( @@ -228,6 +232,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): strip_prefix = "libpng-1.6.34", build_file = clean_dep("//third_party:png.BUILD"), patch_file = clean_dep("//third_party:png_fix_rpi.patch"), + system_build_file = clean_dep("//third_party/systemlibs:png.BUILD"), ) tf_http_archive( @@ -250,6 +255,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "34a7377ba834397db019e8eb122e551a49c98f49df75ec3fcc92b9a794a4f6d1", strip_prefix = "giflib-5.1.4", build_file = clean_dep("//third_party:gif.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:gif.BUILD"), ) tf_http_archive( @@ -261,6 +267,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a", strip_prefix = "six-1.10.0", build_file = clean_dep("//third_party:six.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:six.BUILD"), ) tf_http_archive( @@ -272,6 +279,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "ff6d2e2962d834acb125cc4dcc80c54a8c17c253f4cc9d9c43b5102a560bb75d", strip_prefix = "astor-0.6.2", build_file = clean_dep("//third_party:astor.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:astor.BUILD"), ) tf_http_archive( @@ -294,6 +302,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b", strip_prefix = "termcolor-1.1.0", build_file = clean_dep("//third_party:termcolor.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:termcolor.BUILD"), ) tf_http_archive( @@ -412,6 +421,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): ], strip_prefix = "swig-3.0.8", build_file = clean_dep("//third_party:swig.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:swig.BUILD"), ) tf_http_archive( @@ -480,6 +490,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "07d34db40593d257324ec5fb9debc4dc33f29f8fb44e33a2eeb35503e61d0fe2", strip_prefix = "jsoncpp-11086dd6a7eba04289944367ca82cea71299ed70", build_file = clean_dep("//third_party:jsoncpp.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:jsoncpp.BUILD"), ) tf_http_archive( @@ -501,6 +512,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1", strip_prefix = "zlib-1.2.11", build_file = clean_dep("//third_party:zlib.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:zlib.BUILD"), ) tf_http_archive( @@ -522,6 +534,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "3dfa02e873ff51a11ee02b9ca391807f0c8ea0529a4924afa645fbf97163f9d4", strip_prefix = "snappy-1.1.7", build_file = clean_dep("//third_party:snappy.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:snappy.BUILD"), ) tf_http_archive( @@ -592,6 +605,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "3c8f25c02e806c3ce0ab5fb7da1817f89fc9732709024e2a81b6b82f7cc792a8", strip_prefix = "jemalloc-4.4.0", build_file = clean_dep("//third_party:jemalloc.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:jemalloc.BUILD"), ) java_import_external( diff --git a/third_party/repo.bzl b/third_party/repo.bzl index cb67d3e..88e7197 100644 --- a/third_party/repo.bzl +++ b/third_party/repo.bzl @@ -36,6 +36,15 @@ def _get_env_var(ctx, name): else: return None +# Checks if we should use the system lib instead of the bundled one +def _use_system_lib(ctx, name): + syslibenv = _get_env_var(ctx, "TF_SYSTEM_LIBS") + if syslibenv: + syslibs = list(syslibenv.strip().split(",")) + if name in syslibs: + return True + return False + # Executes specified command with arguments and calls 'fail' if it exited with # non-zero code def _execute_and_check_ret_code(repo_ctx, cmd_and_args): @@ -76,17 +85,28 @@ def _tf_http_archive(ctx): "Even if you don't have permission to mirror the file, please " + "put the correctly formatted mirror URL there anyway, because " + "someone will come along shortly thereafter and mirror the file.") - ctx.download_and_extract( - ctx.attr.urls, - "", - ctx.attr.sha256, - ctx.attr.type, - ctx.attr.strip_prefix) - if ctx.attr.delete: - _apply_delete(ctx, ctx.attr.delete) - if ctx.attr.patch_file != None: - _apply_patch(ctx, ctx.attr.patch_file) - if ctx.attr.build_file != None: + + use_syslib = _use_system_lib(ctx, ctx.attr.name) + if not use_syslib: + ctx.download_and_extract( + ctx.attr.urls, + "", + ctx.attr.sha256, + ctx.attr.type, + ctx.attr.strip_prefix) + if ctx.attr.delete: + _apply_delete(ctx, ctx.attr.delete) + if ctx.attr.patch_file != None: + _apply_patch(ctx, ctx.attr.patch_file) + + if use_syslib and ctx.attr.system_build_file != None: + # Use BUILD.bazel to avoid conflict with third party projects with + # BUILD or build (directory) underneath. + ctx.template("BUILD.bazel", ctx.attr.system_build_file, { + "%prefix%": ".." if _repos_are_siblings() else "external", + }, False) + + elif ctx.attr.build_file != None: # Use BUILD.bazel to avoid conflict with third party projects with # BUILD or build (directory) underneath. ctx.template("BUILD.bazel", ctx.attr.build_file, { @@ -103,6 +123,7 @@ tf_http_archive = repository_rule( "delete": attr.string_list(), "patch_file": attr.label(), "build_file": attr.label(), + "system_build_file": attr.label(), }) """Downloads and creates Bazel repos for dependencies. diff --git a/third_party/systemlibs/BUILD b/third_party/systemlibs/BUILD new file mode 100644 index 0000000..575710f --- /dev/null +++ b/third_party/systemlibs/BUILD @@ -0,0 +1 @@ +# dummy comment to ensure that file is created when applying patch diff --git a/third_party/systemlibs/BUILD.tpl b/third_party/systemlibs/BUILD.tpl new file mode 100644 index 0000000..575710f --- /dev/null +++ b/third_party/systemlibs/BUILD.tpl @@ -0,0 +1 @@ +# dummy comment to ensure that file is created when applying patch diff --git a/third_party/systemlibs/astor.BUILD b/third_party/systemlibs/astor.BUILD new file mode 100644 index 0000000..497ec4b --- /dev/null +++ b/third_party/systemlibs/astor.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # New BSD + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +py_library( + name = "astor", + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/build_defs.bzl.tpl b/third_party/systemlibs/build_defs.bzl.tpl new file mode 100644 index 0000000..3faa46c --- /dev/null +++ b/third_party/systemlibs/build_defs.bzl.tpl @@ -0,0 +1,32 @@ +# -*- Python -*- +"""Skylark macros for system libraries. +""" + +SYSTEM_LIBS_ENABLED = %{syslibs_enabled} + +SYSTEM_LIBS_LIST = [ +%{syslibs_list} +] + + +def if_any_system_libs(a, b=[]): + """Conditional which evaluates to 'a' if any system libraries are configured.""" + if SYSTEM_LIBS_ENABLED: + return a + else: + return b + + +def if_system_lib(lib, a, b=[]): + """Conditional which evaluates to 'a' if we're using the system version of lib""" + + if SYSTEM_LIBS_ENABLED and lib in SYSTEM_LIBS_LIST: + return a + else: + return b + + +def if_not_system_lib(lib, a, b=[]): + """Conditional which evaluates to 'a' if we're using the system version of lib""" + + return if_system_lib(lib, b, a) diff --git a/third_party/systemlibs/curl.BUILD b/third_party/systemlibs/curl.BUILD new file mode 100644 index 0000000..851ac0c --- /dev/null +++ b/third_party/systemlibs/curl.BUILD @@ -0,0 +1,13 @@ +licenses(["notice"]) # MIT/X derivative license + +filegroup( + name = "COPYING", + visibility = ["//visibility:public"], +) + +cc_library( + name = "curl", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lcurl", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/cython.BUILD b/third_party/systemlibs/cython.BUILD new file mode 100644 index 0000000..60d9827 --- /dev/null +++ b/third_party/systemlibs/cython.BUILD @@ -0,0 +1,20 @@ +licenses(["notice"]) # Apache-2.0 + +# find cython.py in python site-packages. $PYTHON_LIB_PATH is not defined here. +genrule( + name = "lncython", + outs = ["cython.py"], + cmd = """ + for i in $(PREFIX)/lib*/python*/site-packages/cython.py; do + if [[ -f $$i ]]; then + ln -s $$i $@ + break + fi + done""", +) + +sh_binary( + name = "cython_binary", + srcs = ["cython.py"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/flatbuffers.BUILD b/third_party/systemlibs/flatbuffers.BUILD new file mode 100644 index 0000000..de413b9 --- /dev/null +++ b/third_party/systemlibs/flatbuffers.BUILD @@ -0,0 +1,40 @@ +licenses(["notice"]) # Apache 2.0 + +filegroup( + name = "LICENSE.txt", + visibility = ["//visibility:public"], +) + +# Public flatc library to compile flatbuffer files at runtime. +cc_library( + name = "flatbuffers", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lflatbuffers", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) + +# Public flatc compiler library. +cc_library( + name = "flatc_library", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lflatbuffers", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) + +genrule( + name = "lnflatc", + outs = ["flatc.bin"], + cmd = "ln -s $$(which flatc) $@", +) + +# Public flatc compiler. +sh_binary( + name = "flatc", + srcs = ["flatc.bin"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "runtime_cc", + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/gif.BUILD b/third_party/systemlibs/gif.BUILD new file mode 100644 index 0000000..05f5429 --- /dev/null +++ b/third_party/systemlibs/gif.BUILD @@ -0,0 +1,13 @@ +licenses(["notice"]) # MIT + +filegroup( + name = "COPYING", + visibility = ["//visibility:public"], +) + +cc_library( + name = "gif", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lgif", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/grpc.BUILD b/third_party/systemlibs/grpc.BUILD new file mode 100644 index 0000000..7c9b504 --- /dev/null +++ b/third_party/systemlibs/grpc.BUILD @@ -0,0 +1,66 @@ +licenses(["notice"]) # Apache v2 + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +cc_library( + name = "grpc", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lgrpc", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "grpc++", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lgrpc++", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "grpc_unsecure", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lgrpc_unsecure", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "grpc++_unsecure", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lgrpc++_unsecure", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) + +genrule( + name = "cp_grpc_cpp_plugin", + outs = [ + "grpc_cpp_plugin.bin", + ], + cmd = "ln -s $(which grpc_cpp_plugin) $@", +) + +sh_binary( + name = "grpc_cpp_plugin", + srcs = [ + "grpc_cpp_plugin.bin", + ], + visibility = ["//visibility:public"], +) + +genrule( + name = "cp_grpc_python_plugin", + outs = [ + "grpc_python_plugin.bin", + ], + cmd = "ln -s $(which grpc_python_plugin) $@", +) + +sh_binary( + name = "grpc_python_plugin", + srcs = [ + "grpc_python_plugin.bin", + ], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/jemalloc.BUILD b/third_party/systemlibs/jemalloc.BUILD new file mode 100644 index 0000000..402b35f --- /dev/null +++ b/third_party/systemlibs/jemalloc.BUILD @@ -0,0 +1,31 @@ +licenses(["notice"]) # BSD + +filegroup( + name = "COPYING", + visibility = ["//visibility:public"], +) + +cc_library( + name = "jemalloc_headers", + defines = [ + "jemalloc_posix_memalign=posix_memalign", + "jemalloc_malloc=malloc", + "jemalloc_realloc=realloc", + "jemalloc_free=free", + ], + visibility = ["//visibility:public"], +) + +cc_library( + name = "jemalloc_impl", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-ljemalloc", "-L$(PREFIX)/lib"], + defines = [ + "jemalloc_posix_memalign=posix_memalign", + "jemalloc_malloc=malloc", + "jemalloc_realloc=realloc", + "jemalloc_free=free", + ], + visibility = ["//visibility:public"], + deps = [":jemalloc_headers"], +) diff --git a/third_party/systemlibs/jpeg.BUILD b/third_party/systemlibs/jpeg.BUILD new file mode 100644 index 0000000..9eb9b97 --- /dev/null +++ b/third_party/systemlibs/jpeg.BUILD @@ -0,0 +1,13 @@ +licenses(["notice"]) # custom notice-style license, see LICENSE.md + +filegroup( + name = "LICENSE.md", + visibility = ["//visibility:public"], +) + +cc_library( + name = "jpeg", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-ljpeg", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/jsoncpp.BUILD b/third_party/systemlibs/jsoncpp.BUILD new file mode 100644 index 0000000..43c6c68 --- /dev/null +++ b/third_party/systemlibs/jsoncpp.BUILD @@ -0,0 +1,38 @@ +licenses(["unencumbered"]) # Public Domain or MIT + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +HEADERS = [ + "include/json/autolink.h", + "include/json/config.h", + "include/json/features.h", + "include/json/forwards.h", + "include/json/json.h", + "include/json/reader.h", + "include/json/value.h", + "include/json/version.h", + "include/json/writer.h", +] + +genrule( + name = "link_headers", + outs = HEADERS, + cmd = """ + for i in $(OUTS); do + i=$${i##*/} + ln -vsf /usr/include/jsoncpp/json/$$i $(@D)/include/json/$$i + done + """, +) + +cc_library( + name = "jsoncpp", + hdrs = HEADERS, + includes = ["."], + copts = ["-I$(PREFIX)/include"], + linkopts = ["-ljsoncpp", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/lmdb.BUILD b/third_party/systemlibs/lmdb.BUILD new file mode 100644 index 0000000..5c8f966 --- /dev/null +++ b/third_party/systemlibs/lmdb.BUILD @@ -0,0 +1,13 @@ +licenses(["notice"]) # OpenLDAP Public License + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +cc_library( + name = "lmdb", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-llmdb", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/nasm.BUILD b/third_party/systemlibs/nasm.BUILD new file mode 100644 index 0000000..deed74d --- /dev/null +++ b/third_party/systemlibs/nasm.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # BSD 2-clause + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +sh_binary( + name = "nasm", + srcs = ["$$(which nasm)"], + visibility = ["@jpeg//:__pkg__"], +) diff --git a/third_party/systemlibs/pcre.BUILD b/third_party/systemlibs/pcre.BUILD new file mode 100644 index 0000000..ba57c88 --- /dev/null +++ b/third_party/systemlibs/pcre.BUILD @@ -0,0 +1,13 @@ +licenses(["notice"]) # BSD + +filegroup( + name = "LICENCE", + visibility = ["//visibility:public"], +) + +cc_library( + name = "pcre", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lpcre", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/png.BUILD b/third_party/systemlibs/png.BUILD new file mode 100644 index 0000000..96e56ee --- /dev/null +++ b/third_party/systemlibs/png.BUILD @@ -0,0 +1,13 @@ +licenses(["notice"]) # BSD/MIT-like license + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +cc_library( + name = "png", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lpng", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/re2.BUILD b/third_party/systemlibs/re2.BUILD new file mode 100644 index 0000000..abc6aee --- /dev/null +++ b/third_party/systemlibs/re2.BUILD @@ -0,0 +1,13 @@ +licenses(["notice"]) # BSD/MIT-like license + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +cc_library( + name = "re2", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lre2", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/six.BUILD b/third_party/systemlibs/six.BUILD new file mode 100644 index 0000000..ff9b1a5 --- /dev/null +++ b/third_party/systemlibs/six.BUILD @@ -0,0 +1,11 @@ +licenses(["notice"]) # MIT + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +py_library( + name = "six", + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/snappy.BUILD b/third_party/systemlibs/snappy.BUILD new file mode 100644 index 0000000..8798e80 --- /dev/null +++ b/third_party/systemlibs/snappy.BUILD @@ -0,0 +1,13 @@ +licenses(["notice"]) # BSD 3-Clause + +filegroup( + name = "COPYING", + visibility = ["//visibility:public"], +) + +cc_library( + name = "snappy", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lsnappy", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/sqlite.BUILD b/third_party/systemlibs/sqlite.BUILD new file mode 100644 index 0000000..3d97623 --- /dev/null +++ b/third_party/systemlibs/sqlite.BUILD @@ -0,0 +1,16 @@ +licenses(["unencumbered"]) # Public Domain + +# Production build of SQLite library that's baked into TensorFlow. +cc_library( + name = "org_sqlite", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lsqlite3", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) + +# This is a Copybara sync helper for Google. +py_library( + name = "python", + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/swig.BUILD b/third_party/systemlibs/swig.BUILD new file mode 100644 index 0000000..2529840 --- /dev/null +++ b/third_party/systemlibs/swig.BUILD @@ -0,0 +1,23 @@ +licenses(["restricted"]) # GPLv3 + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +filegroup( + name = "templates", + visibility = ["//visibility:public"], +) + +genrule( + name = "cpswiglink", + outs = ["swiglink"], + cmd = "cp /usr/bin/swig $@", +) + +sh_binary( + name = "swig", + srcs = ["swiglink"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/syslibs_configure.bzl b/third_party/systemlibs/syslibs_configure.bzl new file mode 100644 index 0000000..2188ffc --- /dev/null +++ b/third_party/systemlibs/syslibs_configure.bzl @@ -0,0 +1,160 @@ +# -*- Python -*- +"""Repository rule for system library autoconfiguration. + +`syslibs_configure` depends on the following environment variables: + + * `TF_SYSTEM_LIBS`: list of third party dependencies that should use + the system version instead +""" + +_TF_SYSTEM_LIBS="TF_SYSTEM_LIBS" + +VALID_LIBS=[ + "astor_archive", + "com_googlesource_code_re2", + "curl", + "cython", + "flatbuffers", + "gif_archive", + "grpc", + "jemalloc", + "jpeg", + "jsoncpp_git", + "lmdb", + "nasm", + "org_sqlite", + "pcre", + "png_archive", + "six_archive", + "snappy", + "swig", + "termcolor_archive", + "zlib_archive", +] + + +def auto_configure_fail(msg): + """Output failure message when syslibs configuration fails.""" + red = "\033[0;31m" + no_color = "\033[0m" + fail("\n%sSystem Library Configuration Error:%s %s\n" % (red, no_color, msg)) + + +def _is_windows(repository_ctx): + """Returns true if the host operating system is windows.""" + os_name = repository_ctx.os.name.lower() + if os_name.find("windows") != -1: + return True + return False + + +def _enable_syslibs(repository_ctx): + s = repository_ctx.os.environ.get(_TF_SYSTEM_LIBS, '').strip() + if not _is_windows(repository_ctx) and s != None and s != '': + return True + return False + + +def _get_system_lib_list(repository_ctx): + """Gets the list of deps that should use the system lib. + + Args: + repository_ctx: The repository context. + + Returns: + A string version of a python list + """ + if _TF_SYSTEM_LIBS not in repository_ctx.os.environ: + return [] + + libenv = repository_ctx.os.environ[_TF_SYSTEM_LIBS].strip() + libs = [] + + for l in list(libenv.split(',')): + l = l.strip() + if l == "": + continue + if l not in VALID_LIBS: + auto_configure_fail("Invalid system lib set: %s" % l) + return [] + libs.append(l) + + return libs + + +def _format_system_lib_list(repository_ctx): + """Formats the list of deps that should use the system lib. + + Args: + repository_ctx: The repository context. + + Returns: + A list of the names of deps that should use the system lib. + """ + libs = _get_system_lib_list(repository_ctx) + ret = '' + for l in libs: + ret += "'%s',\n" % l + + return ret + + +def _tpl(repository_ctx, tpl, substitutions={}, out=None): + if not out: + out = tpl.replace(":", "") + repository_ctx.template( + out, + Label("//third_party/systemlibs%s.tpl" % tpl), + substitutions, + False) + + +def _create_dummy_repository(repository_ctx): + """Creates the dummy repository to build with all bundled libraries.""" + + _tpl(repository_ctx, ":BUILD") + _tpl(repository_ctx, ":build_defs.bzl", + { + "%{syslibs_enabled}": 'False', + "%{syslibs_list}": '', + }) + + +def _create_local_repository(repository_ctx): + """Creates the repository to build with system libraries.""" + + _tpl(repository_ctx, ":BUILD") + _tpl(repository_ctx, ":build_defs.bzl", + { + "%{syslibs_enabled}": 'True', + "%{syslibs_list}": _format_system_lib_list(repository_ctx), + }) + + +def _syslibs_autoconf_impl(repository_ctx): + """Implementation of the syslibs_configure repository rule.""" + if not _enable_syslibs(repository_ctx): + _create_dummy_repository(repository_ctx) + else: + _create_local_repository(repository_ctx) + + +syslibs_configure = repository_rule( + implementation = _syslibs_autoconf_impl, + environ = [ + _TF_SYSTEM_LIBS, + ], +) + +"""Configures the build to link to system libraries +instead of using bundled versions. + +Add the following to your WORKSPACE FILE: + +```python +syslibs_configure(name = "local_config_syslibs") +``` + +Args: + name: A unique name for this workspace rule. +""" diff --git a/third_party/systemlibs/termcolor.BUILD b/third_party/systemlibs/termcolor.BUILD new file mode 100644 index 0000000..915eb62 --- /dev/null +++ b/third_party/systemlibs/termcolor.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # MIT + +filegroup( + name = "COPYING.txt", + visibility = ["//visibility:public"], +) + +py_library( + name = "termcolor", + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/zlib.BUILD b/third_party/systemlibs/zlib.BUILD new file mode 100644 index 0000000..75f2899 --- /dev/null +++ b/third_party/systemlibs/zlib.BUILD @@ -0,0 +1,13 @@ +licenses(["notice"]) # BSD/MIT-like license (for zlib) + +filegroup( + name = "zlib.h", + visibility = ["//visibility:public"], +) + +cc_library( + name = "zlib", + copts = ["-I$(PREFIX)/include"], + linkopts = ["-lz", "-L$(PREFIX)/lib"], + visibility = ["//visibility:public"], +) -- 2.11.1