Address big integers bug (#892)

* coerce all integer types to int64_t

- to remove integer ambiguity around 32/64bit integers, we coerce all
  ints to int64_t.

* handle int64_t constraint/ ImageSequenceReference

* plumb uint64_t support all the way through the SDK

- When a uint64_t is serialized, it is bitwise & with INT64_MAX first to
  avoid overflowing the integer.  In order to do that comparison at
  serialization time, the uint64_t needs to be stored as a uint64_t all
  the way through the python bindings into the any mapping.
- the next step is to move the & operation up into the python bindings
  and prevent people from storing invalid numbers in OTIO files.

* add negative numbers to test and compare values
 Please enter the commit message for your changes. Lines starting

* add in C++ sdk tests

* add unicode unit test as well

* clean up big integers test

* add serialization support for nan, inf, etc

- this is to allow parity with the values in python files

* [de]serialize a variety of numeric types correctly

- add support in the serializer/deserializer for NaN, Inf, -Infinity etc
  from rapidjson
- block pybind11 from automatically casting numbers that don't fit into
  double or int64_t into bool
- add a unit test and test data to test the serialization and behavior

* Clean up invalid type/value exceptions in Any

- Adds specific messages that test if something is an out of range
  integer, or an invalid type
- includes a list of supported types

* cleanup

* Add note to docs about supported numeric types

* Lint fixes

* Fix ImageSequenceReference cloning for int64_t

- The 'Writer' part of SerializableObject needed the same tightening as
  the RapidJSON dispatcher
- this is used for things that serialize and compare JSON (like cloning
  and the `is_equivalent_to` method.
- Because the ImageSequenceReference schema has `int` in it (which
  should at some point be changed to int64_t), it was tripping this
  check on certain platforms

* Some more minor tightening. ready to ship!

* use the big_int_type for python2 support

Co-authored-by: ssteinbach <[email protected]>

Author: ssteinbach

docs/tutorials/otio-file-format-specification.md

@@ -11,7 +11,20 @@ It is strongly recommended that everyone use the OpenTimelineIO library to read
 OpenTimelineIO files should have a `.otio` path extension. Please do not use `.json` to name OTIO files.
 
 ## Contents
-OpenTimelineIO files are serialized as JSON (http://www.json.org)
+OpenTimelineIO files are serialized as JSON (http://www.json.org).
+
+### Number Types
+
+Supported number types:
+
+- integers: `int64_t` (signed 64 bit integer)
+- floating point numbers: `double` (IEEE754 64 bit signed floating point number)
+
+In addition to the basic JSON spec, OTIO allows the following values for doubles:
+
+- `NaN` (not a number)
+- `Inf`, `Infinity` (positive infinity)
+- `-Inf, -Infinity (negative infinity)
 
 ## Structure
 An OTIO file is a tree structure of nested OTIO objects. Each OTIO object is stored as a JSON dictionary with member fields,

src/opentimelineio/deserialization.cpp

@@ -35,12 +35,19 @@ class JSONDecoder : public OTIO_rapidjson::BaseReaderHandler<OTIO_rapidjson::UTF
         }
     }
 
-    bool Null() {  return store(any()); }
+    bool Null() { return store(any()); }
     bool Bool(bool b) { return store(any(b)); }
-    bool Int(int i) {  return store(any(i)); }
-    bool Uint(unsigned u) {  return store(any(int(u))); }
-    bool Int64(int64_t i) { return store(any(i)); }
-    bool Uint64(uint64_t u) { return store(any(int64_t(u))); }
+ 
+    // coerce all integer types to int64_t...
+    bool Int(int i) { return store(any(static_cast<int64_t>(i))); }
+    bool Int64(int64_t i) { return store(any(static_cast<int64_t>(i))); }
+    bool Uint(unsigned u) { return store(any(static_cast<int64_t>(u))); }
+    bool Uint64(uint64_t u) {
+        /// prevent an overflow
+        return store(any(static_cast<int64_t>(u & 0x7FFFFFFFFFFFFFFF)));
+    }
+
+    // ...and all floating point types to double
     bool Double(double d) { return store(any(d)); }
 
     bool String(const char* str, OTIO_rapidjson::SizeType length, bool /* copy */) {
@@ -571,7 +578,7 @@ bool deserialize_json_from_string(std::string const& input, any* destination, Er
     OTIO_rapidjson::CursorStreamWrapper<decltype(ss)> csw(ss);
     JSONDecoder handler(std::bind(&decltype(csw)::GetLine, &csw));
 
-    bool status = reader.Parse(csw, handler);
+    bool status = reader.Parse<OTIO_rapidjson::kParseNanAndInfFlag>(csw, handler);
     handler.finalize();    
 
     if (handler.has_errored(error_status)) {
@@ -613,7 +620,7 @@ bool deserialize_json_from_file(std::string const& file_name, any* destination,
     OTIO_rapidjson::CursorStreamWrapper<decltype(fs)> csw(fs);
     JSONDecoder handler(std::bind(&decltype(csw)::GetLine, &csw));
 
-    bool status = reader.Parse(csw, handler);
+    bool status = reader.Parse<OTIO_rapidjson::kParseNanAndInfFlag>(csw, handler);
     fclose(fp);
 
     handler.finalize();

src/opentimelineio/imageSequenceReference.cpp

@@ -121,13 +121,22 @@ ImageSequenceReference::ImageSequenceReference(std::string const& target_url_bas
     }
 
     bool ImageSequenceReference::read_from(Reader& reader) {
+
+        int64_t start_frame_value = 0;
+        int64_t frame_step_value = 0;
+        int64_t frame_zero_padding_value = 0;
+
         auto result = reader.read("target_url_base", &_target_url_base) &&
             reader.read("name_prefix", &_name_prefix) &&
             reader.read("name_suffix", &_name_suffix) &&
-            reader.read("start_frame", &_start_frame) &&
-            reader.read("frame_step", &_frame_step) &&
+            reader.read("start_frame", &start_frame_value) &&
+            reader.read("frame_step", &frame_step_value) &&
             reader.read("rate", &_rate) &&
-            reader.read("frame_zero_padding", &_frame_zero_padding);
+            reader.read("frame_zero_padding", &frame_zero_padding_value);
+
+        _start_frame = static_cast<int>(start_frame_value);
+        _frame_step = static_cast<int>(frame_step_value);
+        _frame_zero_padding = static_cast<int>(frame_zero_padding_value);
 
         std::string missing_frame_policy_value;
         result && reader.read("missing_frame_policy", &missing_frame_policy_value);
@@ -156,14 +165,18 @@ ImageSequenceReference::ImageSequenceReference(std::string const& target_url_bas
     }
 
     void ImageSequenceReference::write_to(Writer& writer) const {
+        int64_t start_frame_value = static_cast<int64_t>(_start_frame);
+        int64_t frame_step_value = static_cast<int64_t>(_frame_step);
+        int64_t frame_zero_padding_value = static_cast<int64_t>(_frame_zero_padding);
+
         Parent::write_to(writer);
         writer.write("target_url_base", _target_url_base);
         writer.write("name_prefix", _name_prefix);
         writer.write("name_suffix", _name_suffix);
-        writer.write("start_frame", _start_frame);
-        writer.write("frame_step", _frame_step);
+        writer.write("start_frame", start_frame_value);
+        writer.write("frame_step", frame_step_value);
         writer.write("rate", _rate);
-        writer.write("frame_zero_padding", _frame_zero_padding);
+        writer.write("frame_zero_padding", frame_zero_padding_value);
 
         std::string missing_frame_policy_value;
         switch (_missing_frame_policy)

src/opentimelineio/safely_typed_any.cpp

@@ -14,6 +14,10 @@ any create_safely_typed_any(int64_t&& value) {
     return any(value);
 }
 
+any create_safely_typed_any(uint64_t&& value) {
+    return any(value);
+}
+
 any create_safely_typed_any(double&& value) {
     return any(value);
 }    
@@ -59,6 +63,10 @@ int64_t safely_cast_int64_any(any const& a) {
     return any_cast<int64_t>(a);
 }
 
+uint64_t safely_cast_uint64_any(any const& a) {
+    return any_cast<uint64_t>(a);
+}
+
 double safely_cast_double_any(any const& a) {
     return any_cast<double>(a);
 }

src/opentimelineio/safely_typed_any.h

@@ -27,6 +27,7 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION  {
 any create_safely_typed_any(bool&&);
 any create_safely_typed_any(int&&);
 any create_safely_typed_any(int64_t&&);
+any create_safely_typed_any(uint64_t&&);
 any create_safely_typed_any(double&&);
 any create_safely_typed_any(std::string&&);
 any create_safely_typed_any(RationalTime&&);
@@ -39,6 +40,7 @@ any create_safely_typed_any(SerializableObject*);
 bool safely_cast_bool_any(any const& a);
 int safely_cast_int_any(any const& a);
 int64_t safely_cast_int64_any(any const& a);
+uint64_t safely_cast_uint64_any(any const& a);
 double safely_cast_double_any(any const& a);
 std::string safely_cast_string_any(any const& a);
 RationalTime safely_cast_rational_time_any(any const& a);

src/opentimelineio/serializableObject.h

@@ -319,7 +319,7 @@ class SerializableObject {
         static bool write_root(any const& value, class Encoder& encoder, ErrorStatus* error_status);
 
         void write(std::string const& key, bool value);
-        void write(std::string const& key, int value);
+        void write(std::string const& key, int64_t value);
         void write(std::string const& key, double value);
         void write(std::string const& key, std::string const& value);
         void write(std::string const& key, RationalTime value);

src/opentimelineio/serialization.cpp

@@ -47,6 +47,7 @@ class Encoder {
     virtual void write_value(bool value) = 0;
     virtual void write_value(int value) = 0;
     virtual void write_value(int64_t value) = 0;
+    virtual void write_value(uint64_t value) = 0;
     virtual void write_value(double value) = 0;
     virtual void write_value(std::string const& value) = 0;
     virtual void write_value(class RationalTime const& value) = 0;
@@ -128,6 +129,10 @@ class CloningEncoder : public Encoder {
         _store(any(value));
     }
 
+    void write_value(uint64_t value) {
+        _store(any(value));
+    }
+
     void write_value(std::string const& value) {
         _store(any(value));
     }
@@ -282,6 +287,10 @@ class JSONEncoder : public Encoder {
         _writer.Int64(value);
     }
 
+    void write_value(uint64_t value) {
+        _writer.Uint64(value);
+    }
+
     void write_value(std::string const& value) {
         _writer.String(value.c_str());
     }
@@ -392,7 +401,6 @@ void SerializableObject::Writer::_build_dispatch_tables() {
     auto& wt = _write_dispatch_table;
     wt[&typeid(void)] = [this](any const&) { _encoder.write_null_value(); };
     wt[&typeid(bool)] = [this](any const& value) { _encoder.write_value(any_cast<bool>(value)); };
-    wt[&typeid(int)] = [this](any const& value) { _encoder.write_value(any_cast<int>(value)); };
     wt[&typeid(int64_t)] = [this](any const& value) { _encoder.write_value(any_cast<int64_t>(value)); };
     wt[&typeid(double)] = [this](any const& value) { _encoder.write_value(any_cast<double>(value)); };
     wt[&typeid(std::string)] = [this](any const& value) { _encoder.write_value(any_cast<std::string const&>(value)); };
@@ -425,7 +433,6 @@ void SerializableObject::Writer::_build_dispatch_tables() {
     auto& et = _equality_dispatch_table;
     et[&typeid(void)] = &_simple_any_comparison<void>;
     et[&typeid(bool)] = &_simple_any_comparison<bool>;
-    et[&typeid(int)] = &_simple_any_comparison<int>;
     et[&typeid(int64_t)] = &_simple_any_comparison<int64_t>;
     et[&typeid(double)] = &_simple_any_comparison<double>;
     et[&typeid(std::string)] = &_simple_any_comparison<std::string>;
@@ -509,7 +516,7 @@ void SerializableObject::Writer::write(std::string const& key, bool value) {
     _encoder.write_value(value);
 }
 
-void SerializableObject::Writer::write(std::string const& key, int value) {
+void SerializableObject::Writer::write(std::string const& key, int64_t value) {
     _encoder_write_key(key);
     _encoder.write_value(value);
 }
@@ -724,15 +731,28 @@ std::string serialize_json_to_string(any const& value, ErrorStatus* error_status
     OTIO_rapidjson::StringBuffer s;    
 
     if (indent < 0) {
-        OTIO_rapidjson::Writer<decltype(s)> json_writer(s);
+        OTIO_rapidjson::Writer<
+            decltype(s), 
+            OTIO_rapidjson::UTF8<>,
+            OTIO_rapidjson::UTF8<>,
+            OTIO_rapidjson::CrtAllocator,
+            OTIO_rapidjson::kWriteNanAndInfFlag
+            > json_writer(s);
         JSONEncoder<decltype(json_writer)> json_encoder(json_writer);
 
         if (!SerializableObject::Writer::write_root(value, json_encoder, error_status)) {
             return std::string();
         }
     }
     else {
-        OTIO_rapidjson::PrettyWriter<decltype(s)> json_writer(s);
+        OTIO_rapidjson::PrettyWriter<
+            decltype(s), 
+            OTIO_rapidjson::UTF8<>,
+            OTIO_rapidjson::UTF8<>,
+            OTIO_rapidjson::CrtAllocator,
+            OTIO_rapidjson::kWriteNanAndInfFlag
+            > json_writer(s);
+
         JSONEncoder<decltype(json_writer)> json_encoder(json_writer);
 
         json_writer.SetIndent(' ', indent);
@@ -756,12 +776,24 @@ bool serialize_json_to_file(any const& value, std::string const& file_name,
     bool status;
 
     if (indent < 0) {
-        OTIO_rapidjson::Writer<decltype(osw)> json_writer(osw);
+        OTIO_rapidjson::Writer<
+            decltype(osw),
+            OTIO_rapidjson::UTF8<>,
+            OTIO_rapidjson::UTF8<>,
+            OTIO_rapidjson::CrtAllocator,
+            OTIO_rapidjson::kWriteNanAndInfFlag
+            > json_writer(osw);
         JSONEncoder<decltype(json_writer)> json_encoder(json_writer);
         status = SerializableObject::Writer::write_root(value, json_encoder, error_status);
     }
     else {
-        OTIO_rapidjson::PrettyWriter<decltype(osw)> json_writer(osw);
+        OTIO_rapidjson::PrettyWriter<
+            decltype(osw),
+            OTIO_rapidjson::UTF8<>,
+            OTIO_rapidjson::UTF8<>,
+            OTIO_rapidjson::CrtAllocator,
+            OTIO_rapidjson::kWriteNanAndInfFlag
+        > json_writer(osw);
         JSONEncoder<decltype(json_writer)> json_encoder(json_writer);
 
         json_writer.SetIndent(' ', indent);

src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp

@@ -94,18 +94,29 @@ PYBIND11_MODULE(_otio, m) {
           }, "filename"_a);
 
     py::class_<PyAny>(m, "PyAny")
-        .def(py::init([](bool b) { return new PyAny(b); }))
-        .def(py::init([](int i) { return new PyAny(i); }))
-        .def(py::init([](int64_t i) { return new PyAny(i); }))
-        .def(py::init([](double d) { return new PyAny(d); }))
+        // explicitly map python bool, int and double classes so that they
+        // do NOT accidentally cast in valid values
+        .def(py::init([](py::bool_ b) { 
+                    bool result = b.cast<bool>();
+                    return new PyAny(result); 
+                    }))
+        .def(py::init([](py::int_ i) { 
+                    int64_t result = i.cast<int64_t>();
+                    return new PyAny(result); 
+                    }))
+        .def(py::init([](py::float_ d) { 
+                    double result = d.cast<double>();
+                    return new PyAny(result); 
+                    }))
         .def(py::init([](std::string s) { return new PyAny(s); }))
         .def(py::init([](py::none) { return new PyAny(); }))
         .def(py::init([](SerializableObject* s) { return new PyAny(s); }))
         .def(py::init([](RationalTime rt) { return new PyAny(rt); }))
         .def(py::init([](TimeRange tr) { return new PyAny(tr); }))
         .def(py::init([](TimeTransform tt) { return new PyAny(tt); }))
         .def(py::init([](AnyVectorProxy* p) { return new PyAny(p->fetch_any_vector()); }))
-        .def(py::init([](AnyDictionaryProxy* p) { return new PyAny(p->fetch_any_dictionary()); }));
+        .def(py::init([](AnyDictionaryProxy* p) { return new PyAny(p->fetch_any_dictionary()); }))
+        ;
 
     m.def("register_serializable_object_type", &register_python_type,
           "class_object"_a, "schema_name"_a, "schema_version"_a);

src/py-opentimelineio/opentimelineio-bindings/otio_tests.cpp

@@ -4,6 +4,7 @@
 #include "opentimelineio/serializableObject.h"
 #include "opentimelineio/serializableObjectWithMetadata.h"
 #include "opentimelineio/serializableCollection.h"
+#include "opentimelineio/timeline.h"
 #include "otio_utils.h"
 
 namespace py = pybind11;
@@ -130,6 +131,26 @@ void otio_xyzzy(std::string msg) {
     /* used as a debugger breakpoint */
 }
 
+/// test the behavior of big integers in OTIO
+bool test_big_uint() {
+    int64_t some_int = 4;
+    uint64_t number_base = INT64_MAX;
+    uint64_t giant_number = number_base + some_int;
+
+    SerializableObjectWithMetadata* so = new SerializableObjectWithMetadata();
+
+    so->metadata()["giant_number"] = giant_number;
+
+    bool result = true;
+
+    if (any_cast<uint64_t>(so->metadata()["giant_number"]) != giant_number) {
+        return false;
+    }
+
+    so->possibly_delete();
+    return true;
+}
+
 void otio_tests_bindings(py::module m) {
     TypeRegistry& r = TypeRegistry::instance();
     r.register_type<TestObject>();
@@ -146,4 +167,5 @@ void otio_tests_bindings(py::module m) {
     test.def("bash_retainers2", &test_bash_retainers2);
     test.def("gil_scoping", &test_gil_scoping);
     test.def("xyzzy", &otio_xyzzy);
+    test.def("test_big_uint", &test_big_uint);
 }

src/py-opentimelineio/opentimelineio-bindings/otio_utils.cpp

@@ -39,13 +39,19 @@ py::object plain_int(int64_t i) {
     return py::reinterpret_steal<py::object>(p);
 }
 
+py::object plain_uint(uint64_t i) {
+    PyObject *p = PyLong_FromUnsignedLongLong(i);
+    return py::reinterpret_steal<py::object>(p);
+}
+
 void _build_any_to_py_dispatch_table() {
     auto& t = _py_cast_dispatch_table;
 
     t[&typeid(void)] = [](any const& /* a */, bool) { return py::none(); };
     t[&typeid(bool)] = [](any const& a, bool) { return py::cast(safely_cast_bool_any(a)); };
     t[&typeid(int)] = [](any const& a, bool) {  return plain_int(safely_cast_int_any(a)); };
     t[&typeid(int64_t)] = [](any const& a, bool) {  return plain_int(safely_cast_int64_any(a)); };
+    t[&typeid(uint64_t)] = [](any const& a, bool) {  return plain_uint(safely_cast_uint64_any(a)); };
     t[&typeid(double)] = [](any const& a, bool) { return py::cast(safely_cast_double_any(a)); };
     t[&typeid(std::string)] = [](any const& a, bool) { return py::cast(safely_cast_string_any(a)); };
     t[&typeid(RationalTime)] = [](any const& a, bool) { return py::cast(safely_cast_rational_time_any(a)); };