From 230c684efb64e58bfa12d1ef3e71ce5d3f2bdb58 Mon Sep 17 00:00:00 2001 From: Aditya Karnam Date: Wed, 15 Apr 2026 22:58:17 -0500 Subject: [PATCH 1/2] feat: add encode_json and loads helpers for better JSON null support --- src/toon_format/__init__.py | 10 ++++++++- src/toon_format/utils.py | 39 +++++++++++++++++++++++++++++++++- tests/test_json_integration.py | 39 ++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 tests/test_json_integration.py diff --git a/src/toon_format/__init__.py b/src/toon_format/__init__.py index f664ec0..2058c81 100644 --- a/src/toon_format/__init__.py +++ b/src/toon_format/__init__.py @@ -23,12 +23,20 @@ from .decoder import ToonDecodeError, decode from .encoder import encode from .types import DecodeOptions, Delimiter, DelimiterKey, EncodeOptions -from .utils import compare_formats, count_tokens, estimate_savings +from .utils import ( + compare_formats, + count_tokens, + encode_json, + estimate_savings, + loads, +) __version__ = "0.9.0-beta.1" __all__ = [ "encode", "decode", + "encode_json", + "loads", "ToonDecodeError", "Delimiter", "DelimiterKey", diff --git a/src/toon_format/utils.py b/src/toon_format/utils.py index 935a074..48449f5 100644 --- a/src/toon_format/utils.py +++ b/src/toon_format/utils.py @@ -31,7 +31,7 @@ # __init__.py defines encode() before importing utils, so this is safe from . import encode -__all__ = ["count_tokens", "estimate_savings", "compare_formats"] +__all__ = ["count_tokens", "estimate_savings", "compare_formats", "encode_json", "loads"] _TIKTOKEN_MISSING_MSG = ( @@ -40,6 +40,43 @@ ) +def loads(json_string: str) -> Any: + """Parse JSON string into Python objects. + + This is an alias for `json.loads()` provided for convenience and to ensure + a TOON-friendly integration flow where JSON 'null' is correctly converted + to Python 'None'. + + Args: + json_string: The JSON string to parse. + + Returns: + Any: Parsed Python data structure. + """ + return json.loads(json_string) + + +def encode_json(json_string: str) -> str: + """Encode a JSON string directly into TOON format. + + Parses the JSON string (converting 'null' to 'None' automatically) + and then encodes the resulting Python object into TOON. + + Args: + json_string: The JSON string to encode. + + Returns: + str: TOON-formatted string. + + Example: + >>> import toon_format + >>> toon_format.encode_json('{"abc": null}') + 'abc: null' + """ + data = loads(json_string) + return encode(data) + + def _require_tiktoken(): try: import tiktoken # type: ignore[import-not-found] diff --git a/tests/test_json_integration.py b/tests/test_json_integration.py new file mode 100644 index 0000000..3643faf --- /dev/null +++ b/tests/test_json_integration.py @@ -0,0 +1,39 @@ +import json +from toon_format import encode_json, loads, encode + +def test_loads_null_to_none(): + json_str = '{"abc": null, "xyz": 123}' + data = loads(json_str) + assert data["abc"] is None + assert data["xyz"] == 123 + print("test_loads_null_to_none passed") + +def test_encode_json_integration(): + json_str = '{"abc": null, "xyz": null}' + # This should automatically handle null -> None -> TOON null + toon_output = encode_json(json_str) + expected = "abc: null\nxyz: null" + assert toon_output.strip() == expected + print("test_encode_json_integration passed") + +def test_complex_json_integration(): + json_str = ''' + { + "status": "success", + "data": { + "user": null, + "items": [1, null, 3] + } + } + ''' + toon_output = encode_json(json_str) + assert "user: null" in toon_output + # Check for null in items array (can be inline "1,null,3" or list "- null") + assert "null" in toon_output + print("test_complex_json_integration passed") + +if __name__ == "__main__": + test_loads_null_to_none() + test_encode_json_integration() + test_complex_json_integration() + print("All JSON integration tests passed!") From e95a1ff07d2e41359bb8fc0a2e48b952906591d2 Mon Sep 17 00:00:00 2001 From: Aditya Karnam Date: Thu, 16 Apr 2026 16:58:24 -0500 Subject: [PATCH 2/2] fix: address review comments on PR #57 - Update utils.py module docstring to include JSON helpers - Fix circular import of encode in utils.py - Remove unused imports and print statements in tests - Fix formatting in tests using ruff --- src/toon_format/utils.py | 23 ++++++++--------------- tests/test_json_integration.py | 19 ++++++------------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/src/toon_format/utils.py b/src/toon_format/utils.py index 48449f5..91f9a10 100644 --- a/src/toon_format/utils.py +++ b/src/toon_format/utils.py @@ -1,35 +1,28 @@ # Copyright (c) 2025 TOON Format Organization # SPDX-License-Identifier: MIT -"""Token analysis utilities for TOON format. +"""Utilities for TOON format. -This module provides utilities for counting tokens and comparing -token efficiency between JSON and TOON formats. Useful for: -- Estimating API costs (tokens are the primary cost driver) -- Optimizing prompt sizes for LLM context windows -- Benchmarking TOON's token efficiency +This module provides utilities for: +- Token analysis and efficiency comparison between JSON and TOON formats +- JSON integration and null value handling +- Estimating API costs and optimizing prompt sizes Functions: count_tokens: Count tokens in a text string estimate_savings: Compare JSON vs TOON token counts compare_formats: Generate formatted comparison table + loads: Parse JSON string into Python objects (alias for json.loads) + encode_json: Encode a JSON string directly into TOON format Requirements: tiktoken: Install with `uv add tiktoken` or `uv add toon_format[benchmark]` - -Example: - >>> import toon_format - >>> data = {"name": "Alice", "age": 30} - >>> result = toon_format.estimate_savings(data) - >>> print(f"TOON saves {result['savings_percent']:.1f}% tokens") """ import functools import json from typing import Any, Dict -# Import encode from parent package (defined in __init__.py before this module is imported) -# __init__.py defines encode() before importing utils, so this is safe -from . import encode +from .encoder import encode __all__ = ["count_tokens", "estimate_savings", "compare_formats", "encode_json", "loads"] diff --git a/tests/test_json_integration.py b/tests/test_json_integration.py index 3643faf..cab3c2c 100644 --- a/tests/test_json_integration.py +++ b/tests/test_json_integration.py @@ -1,12 +1,12 @@ -import json -from toon_format import encode_json, loads, encode +from toon_format import encode_json, loads + def test_loads_null_to_none(): json_str = '{"abc": null, "xyz": 123}' data = loads(json_str) assert data["abc"] is None assert data["xyz"] == 123 - print("test_loads_null_to_none passed") + def test_encode_json_integration(): json_str = '{"abc": null, "xyz": null}' @@ -14,10 +14,10 @@ def test_encode_json_integration(): toon_output = encode_json(json_str) expected = "abc: null\nxyz: null" assert toon_output.strip() == expected - print("test_encode_json_integration passed") + def test_complex_json_integration(): - json_str = ''' + json_str = """ { "status": "success", "data": { @@ -25,15 +25,8 @@ def test_complex_json_integration(): "items": [1, null, 3] } } - ''' + """ toon_output = encode_json(json_str) assert "user: null" in toon_output # Check for null in items array (can be inline "1,null,3" or list "- null") assert "null" in toon_output - print("test_complex_json_integration passed") - -if __name__ == "__main__": - test_loads_null_to_none() - test_encode_json_integration() - test_complex_json_integration() - print("All JSON integration tests passed!")