Upgrade Struct2Tensor to TensorFlow 2.21.0

.bazelrc

@@ -35,15 +35,21 @@ build:linux --copt=-w
 build:macos --copt=-w
 
 # By default, build TF in C++ 17 mode.
-build:linux --cxxopt=-std=c++17
+build --cxxopt=-std=c++17
+build --host_cxxopt=-std=c++17
 build:linux --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=1
-build:linux --host_cxxopt=-std=c++17
-build:macos --cxxopt=-std=c++17
 build:macos --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=1
-build:macos --host_cxxopt=-std=c++17
 
 
 # Suppress all warning messages.
+
 build:short_logs --output_filter=DONT_MATCH_ANYTHING
 
+# Config to disable sframe in assembler to avoid linker errors with gcc 15
+build:sframe_fix --copt=-Wa,--gsframe=no
+
 build:macos --macos_minimum_os=10.12
+
+common --noenable_bzlmod
+
+test --noincompatible_check_sharding_support

.bazelversion

@@ -0,0 +1 @@
+7.7.0

.github/workflows/build.yml

@@ -8,11 +8,12 @@ on:
 
 jobs:
   build:
-    name: Build (${{ matrix.os }})
+    name: Build (${{ matrix.os }}, Python ${{ matrix.python-version }})
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
     - uses: actions/checkout@v4
@@ -23,6 +24,7 @@ jobs:
         auto-activate-base: false
         activate-environment: s2t-env
         environment-file: environment.yml
+        python-version: ${{ matrix.python-version }}
 
     - name: Install Bazel
       shell: bash -l {0}

.github/workflows/pre-commit.yml

@@ -0,0 +1,21 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+     branches: [master]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.7
+      with:
+        # Ensure the full history is fetched
+        # This is required to run pre-commit on a specific set of commits
+        # TODO: Remove this when all the pre-commit issues are fixed
+        fetch-depth: 0
+    - uses: actions/setup-python@v5.1.1
+      with:
+        python-version: 3.13
+    - uses: pre-commit/action@v3.0.1

.pre-commit-config.yaml

@@ -0,0 +1,8 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files

LICENSE

@@ -198,4 +198,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.

RELEASE.md

@@ -4,12 +4,28 @@
 
 ## Major Features and Improvements
 
+*   N/A
+
 ## Bug Fixes and Other Changes
 
+*   Depends on `tensorflow 2.21.0`.
+*   Depends on `protobuf==6.31.1`.
+*   Depends on `pyarrow==23.0.1`.
+*   Enforced C++17 in `.bazelrc`.
+*   Disabled Bzlmod in `.bazelrc` to resolve protobuf conflicts.
+*   Added dummy repositories in `WORKSPACE` to bypass circular dependencies with TensorFlow.
+*   Fixed missing `#include <cstdint>` in various files to support compilation with `gcc 15`.
+*   Added support for Python 3.12.
+*   Dropped support for Python 3.9.
+
 ## Breaking Changes
 
+*   N/A
+
 ## Deprecations
 
+*   N/A
+
 # Version 0.48.1
 
 ## Major Features and Improvements

WORKSPACE

@@ -16,6 +16,58 @@
 
 workspace(name = "struct2tensor")
 
+local_repository(
+    name = "python_version_repo",
+    path = "third_party/python_version_repo",
+)
+
+local_repository(
+    name = "python_3_11_host",
+    path = "third_party/python_3_11_host",
+)
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+
+local_repository(
+    name = "rules_java",
+    path = "third_party/rules_java",
+)
+
+local_repository(
+    name = "local_config_cuda",
+    path = "third_party/local_config_cuda",
+)
+
+local_repository(
+    name = "local_config_tensorrt",
+    path = "third_party/local_config_tensorrt",
+)
+
+local_repository(
+    name = "local_config_rocm",
+    path = "third_party/local_config_rocm",
+)
+
+local_repository(
+    name = "local_config_sycl",
+    path = "third_party/local_config_sycl",
+)
+
+local_repository(
+    name = "tf_wheel_version_suffix",
+    path = "third_party/tf_wheel_version_suffix",
+)
+
+maybe(
+    http_archive,
+    name = "platforms",
+    urls = [
+        "https://github.com/bazelbuild/platforms/releases/download/0.0.11/platforms-0.0.11.tar.gz",
+    ],
+    sha256 = "29742e87275809b5e598dc2f04d86960cc7a55b3067d97221c9abbc9926bff0f",
+)
+
 load("//tf:tf_configure.bzl", "tf_configure")
 
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
@@ -26,23 +78,53 @@ tf_configure(name = "local_config_tf")
 
 #####################################################################################
 
+
+
+# ===== Abseil dependency =====
 http_archive(
-    name = "zlib",
-    build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
-    sha256 = "17e88863f3600672ab49182f217281b6fc4d3c762bde361935e436a95214d05c",
-    strip_prefix = "zlib-1.3.1",
-    urls = ["https://github.com/madler/zlib/archive/v1.3.1.tar.gz"],
+    name = "com_google_absl",
+    sha256 = "d1abe9da2003e6cbbd7619b0ced3e52047422f4f4ac6c66a9bef5d2e99fea837",
+    strip_prefix = "abseil-cpp-d38452e1ee03523a208362186fd42248ff2609f6",
+    urls = [
+        "https://github.com/abseil/abseil-cpp/archive/d38452e1ee03523a208362186fd42248ff2609f6.tar.gz",
+    ],
+    patches = [
+        "//third_party:abseil_visibility.patch",
+    ],
+    patch_args = ["-p0"],
+)
+
+http_archive(
+    name = "abseil-cpp",
+    sha256 = "d1abe9da2003e6cbbd7619b0ced3e52047422f4f4ac6c66a9bef5d2e99fea837",
+    strip_prefix = "abseil-cpp-d38452e1ee03523a208362186fd42248ff2609f6",
+    urls = [
+        "https://github.com/abseil/abseil-cpp/archive/d38452e1ee03523a208362186fd42248ff2609f6.tar.gz",
+    ],
+    patches = [
+        "//third_party:abseil_visibility.patch",
+    ],
+    patch_args = ["-p0"],
 )
 
-# ===== Protobuf 4.25.6 dependency =====
+
+
+
+# ===== Protobuf 6.31.1 dependency =====
 # Must be declared BEFORE TensorFlow's workspaces to override the version they pull
 http_archive(
     name = "com_google_protobuf",
-    sha256 = "4e6727bc5d23177edefa3ad86fd2f5a92cd324151636212fd1f7f13aef3fd2b7",
-    strip_prefix = "protobuf-4.25.6",
+    sha256 = "6e09bbc950ba60c3a7b30280210cd285af8d7d8ed5e0a6ed101c72aff22e8d88",
+    strip_prefix = "protobuf-6.31.1",
     urls = [
-        "https://github.com/protocolbuffers/protobuf/archive/v4.25.6.tar.gz",
+        "https://github.com/protocolbuffers/protobuf/archive/refs/tags/v6.31.1.zip",
     ],
+    patches = ["//third_party:protobuf_tensorflow.patch"],
+    patch_args = ["-p1"],
+)
+
+register_toolchains(
+    "@com_google_protobuf//bazel/private/toolchains:cc_source_toolchain_bazel7",
 )
 
 # ===== TensorFlow dependency =====
@@ -60,23 +142,55 @@ http_archive(
 # 3. Request the new archive to be mirrored on mirror.bazel.build for more
 #    reliable downloads.
 
-_TENSORFLOW_GIT_COMMIT = "3c92ac03cab816044f7b18a86eb86aa01a294d95"  # tf 2.17.1
-_TENSORFLOW_ARCHIVE_SHA256 = "317dd95c4830a408b14f3e802698eb68d70d81c7c7cfcd3d28b0ba023fe84a68"
+_TENSORFLOW_GIT_COMMIT = "2.21.0"  # tf 2.21.0
+_TENSORFLOW_ARCHIVE_SHA256 = "ef3568bb4865d6c1b2564fb5689c19b6b9a5311572cd1f2ff9198636a8520921"
 
 http_archive(
     name = "org_tensorflow",
     sha256 = _TENSORFLOW_ARCHIVE_SHA256,
     urls = [
-        "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
+        "https://github.com/tensorflow/tensorflow/archive/v%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
     ],
     strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
     patches = ["//third_party:tensorflow.patch"],
     patch_args = ["-p1"],
+    repo_mapping = {
+        "@abseil-cpp": "@com_google_absl",
+    },
+)
+
+http_archive(
+    name = "llvm-raw",
+    sha256 = "3f986184ee126677dbd77edb16d6b82c057ec869fefd7a9871979941e52e837a",
+    strip_prefix = "llvm-project-909041e4802c4b9a2223ca04099f35bf1dbbd460",
+    urls = [
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/909041e4802c4b9a2223ca04099f35bf1dbbd460.tar.gz",
+        "https://github.com/llvm/llvm-project/archive/909041e4802c4b9a2223ca04099f35bf1dbbd460.tar.gz",
+    ],
+    build_file = "@xla//third_party/llvm:llvm.BUILD",
+    patches = [
+        "@xla//third_party/llvm:generated.patch",
+        "@xla//third_party/llvm:build.patch",
+        "@xla//third_party/llvm:mathextras.patch",
+        "@xla//third_party/llvm:toolchains.patch",
+        "@xla//third_party/llvm:zstd.patch",
+        "@xla//third_party/llvm:lit_test.patch",
+        "//third_party:llvm_configure.patch",
+    ],
+    patch_args = ["-p1"],
+)
+
+http_archive(
+    name = "zlib",
+    build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
+    sha256 = "17e88863f3600672ab49182f217281b6fc4d3c762bde361935e436a95214d05c",
+    strip_prefix = "zlib-1.3.1",
+    urls = ["https://github.com/madler/zlib/archive/v1.3.1.tar.gz"],
 )
 
-load("//third_party:python_configure.bzl", "local_python_configure")
-local_python_configure(name = "local_config_python")
-local_python_configure(name = "local_execution_config_python")
+# load("//third_party:python_configure.bzl", "local_python_configure")
+# local_python_configure(name = "local_config_python")
+# local_python_configure(name = "local_execution_config_python")
 
 
 # Please add all new struct2tensor dependencies in workspace.bzl.

configure.sh

@@ -53,12 +53,7 @@ else
   fi
 fi
 
-# Set Bazel version based on OS
-if [[ "$(uname)" == "Darwin" ]]; then
-  echo "7.4.1" > .bazelversion
-else
-  echo "6.5.0" > .bazelversion
-fi
+# Bazel version is controlled by .bazelversion file.
 
 ensure_tensorflow
 TF_CFLAGS=( $(${PYTHON_BIN_PATH} -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') )

environment.yml

@@ -1,10 +1,12 @@
-name: s2t-env-py311
+name: s2t-env-py312
 channels:
   - conda-forge
 dependencies:
-  - python=3.11
+  - python>=3.10,<3.14
+
   - pip
   - pip:
+    - --index-url https://pypi.org/simple
     - absl-py==2.4.0
     - altgraph==0.17.5
     - astunparse==1.6.3
@@ -16,30 +18,31 @@ dependencies:
     - google-pasta==0.2.0
     - googleapis-common-protos==1.72.0
     - grpcio==1.78.0
-    - h5py==3.16.0
+    - h5py==3.14.0
     - idna==3.11
-    - keras==3.13.2
+    - keras>=3.0.0
+
     - libclang==18.1.1
     - macholib==1.16.4
     - markdown==3.10.2
     - markdown-it-py==4.0.0
     - markupsafe==3.0.3
     - mdurl==0.1.2
-    - ml-dtypes==0.4.1
+    - ml-dtypes==0.5.1
     - namex==0.1.0
-    - numpy==1.26.4
+    - numpy>=1.26.4
     - opt-einsum==3.4.0
     - optree==0.19.0
-    - protobuf==4.25.8
-    - pyarrow==10.0.1
-    - pygments==2.19.2
-    - requests==2.32.5
+    - protobuf==6.31.1
+    - pyarrow==23.0.1
+    - pygments==2.20
+    - requests==2.33
+
     - rich==14.3.3
     - six==1.17.0
     - tensorboard==2.17.1
     - tensorboard-data-server==0.7.2
-    - tensorflow==2.17.1
-    - tensorflow-io-gcs-filesystem==0.37.1
+    - tensorflow>=2.21.0
     - tensorflow-metadata==1.17.3
     - termcolor==3.3.0
     - typing-extensions==4.15.0

g3doc/api_docs/python/expression_impl.md

@@ -67,4 +67,3 @@ s2t.expression_impl.apply_schema
 [`size`](./expression_impl/size.md) module: Functions for creating new size or has expression.
 
 [`slice_expression`](./expression_impl/slice_expression.md) module: Implementation of slice.
-

g3doc/api_docs/python/expression_impl/all_symbols.md

@@ -64,4 +64,4 @@
 *  <a href="../expression_impl/size/size_anonymous.md"><code>expression_impl.size.size_anonymous</code></a>
 *  <a href="../expression_impl/slice_expression.md"><code>expression_impl.slice_expression</code></a>
 *  <a href="../expression_impl/slice_expression/IndexValue.md"><code>expression_impl.slice_expression.IndexValue</code></a>
-*  <a href="../expression_impl/slice_expression/slice_expression.md"><code>expression_impl.slice_expression.slice_expression</code></a>
+*  <a href="../expression_impl/slice_expression/slice_expression.md"><code>expression_impl.slice_expression.slice_expression</code></a>

g3doc/api_docs/python/expression_impl/apply_schema.md

@@ -57,4 +57,3 @@ my_new_schema has semantically identical information on the fields as my_schema.
 ## Functions
 
 [`apply_schema(...)`](../expression_impl/apply_schema/apply_schema.md)
-

g3doc/api_docs/python/expression_impl/broadcast.md

@@ -95,4 +95,3 @@ session: {
 [`broadcast(...)`](../expression_impl/broadcast/broadcast.md)
 
 [`broadcast_anonymous(...)`](../expression_impl/broadcast/broadcast_anonymous.md)
-

g3doc/api_docs/python/expression_impl/depth_limit.md

@@ -49,4 +49,3 @@ You get:
 ## Functions
 
 [`limit_depth(...)`](../expression_impl/depth_limit/limit_depth.md): Limit the depth to nodes k steps from expr.
-