include/ylt/standalone/iguana/pb_writer.hpp (479 lines of code) (raw):

#pragma once #include "detail/string_resize.hpp" #include "pb_util.hpp" namespace iguana { namespace detail { template <uint32_t key, typename V, typename Writer> IGUANA_INLINE void encode_varint_field(V val, Writer& writer) { static_assert(std::is_integral_v<V>, "must be integral"); if constexpr (key != 0) { serialize_varint_u32<key>(writer); } serialize_varint(val, writer); } template <uint32_t key, typename V, typename Writer> IGUANA_INLINE void encode_fixed_field(V val, Writer& writer) { if constexpr (key != 0) { serialize_varint_u32<key>(writer); } constexpr size_t size = sizeof(V); // TODO: check Stream continuous writer.write((const char*)&val, size); } template <uint32_t key, bool omit_default_val = true, typename Type, typename Writer> IGUANA_INLINE void to_pb_impl(Type&& t, uint32_t*& sz_ptr, Writer& writer); template <uint32_t key, typename V, typename Writer> IGUANA_INLINE void encode_pair_value(V&& val, size_t size, uint32_t*& sz_ptr, Writer& writer) { if (size == 0) IGUANA_UNLIKELY { // map keys can't be omitted even if values are empty // TODO: repeated ? serialize_varint_u32<key>(writer); serialize_varint(0, writer); } else { to_pb_impl<key, false>(val, sz_ptr, writer); } } template <uint32_t key, bool omit_default_val, typename T, typename Writer> IGUANA_INLINE void encode_numeric_field(T t, Writer& writer) { if constexpr (omit_default_val) { if constexpr (is_fixed_v<T> || is_signed_varint_v<T>) { if (t.val == 0) { return; } } else { if (t == static_cast<T>(0)) IGUANA_UNLIKELY { return; } } } if constexpr (std::is_integral_v<T>) { detail::encode_varint_field<key>(t, writer); } else if constexpr (detail::is_signed_varint_v<T>) { detail::encode_varint_field<key>(encode_zigzag(t.val), writer); } else if constexpr (detail::is_fixed_v<T>) { detail::encode_fixed_field<key>(t.val, writer); } else if constexpr (std::is_same_v<T, double> || std::is_same_v<T, float>) { detail::encode_fixed_field<key>(t, writer); } else if constexpr (std::is_enum_v<T>) { using U = std::underlying_type_t<T>; detail::encode_varint_field<key>(static_cast<U>(t), writer); } else { static_assert(!sizeof(T), "unsupported type"); } } template <uint32_t field_no, typename Type, typename Writer> IGUANA_INLINE void to_pb_oneof(Type&& t, uint32_t*& sz_ptr, Writer& writer) { using T = std::decay_t<Type>; std::visit( [&sz_ptr, &writer](auto&& value) IGUANA__INLINE_LAMBDA { using raw_value_type = decltype(value); using value_type = std::remove_const_t<std::remove_reference_t<decltype(value)>>; constexpr auto offset = get_variant_index<T, value_type, std::variant_size_v<T> - 1>(); constexpr uint32_t key = ((field_no + offset) << 3) | static_cast<uint32_t>(get_wire_type<value_type>()); to_pb_impl<key, false>(std::forward<raw_value_type>(value), sz_ptr, writer); }, std::forward<Type>(t)); } // omit_default_val = true indicates to omit the default value in searlization template <uint32_t key, bool omit_default_val, typename Type, typename Writer> IGUANA_INLINE void to_pb_impl(Type&& t, uint32_t*& sz_ptr, Writer& writer) { using T = std::remove_const_t<std::remove_reference_t<Type>>; if constexpr (ylt_refletable_v<T> || is_custom_reflection_v<T>) { // can't be omitted even if values are empty if constexpr (key != 0) { auto len = pb_value_size(t, sz_ptr); serialize_varint_u32<key>(writer); serialize_varint(len, writer); if (len == 0) IGUANA_UNLIKELY { return; } } static auto tuple = get_pb_members_tuple(std::forward<Type>(t)); constexpr size_t SIZE = std::tuple_size_v<std::decay_t<decltype(tuple)>>; for_each_n( [&t, &sz_ptr, &writer](auto i) IGUANA__INLINE_LAMBDA { using field_type = std::tuple_element_t<decltype(i)::value, std::decay_t<decltype(tuple)>>; auto value = std::get<decltype(i)::value>(tuple); auto& val = value.value(t); using U = typename field_type::value_type; using sub_type = typename field_type::sub_type; if constexpr (variant_v<U>) { constexpr auto offset = get_variant_index<U, sub_type, std::variant_size_v<U> - 1>(); if constexpr (offset == 0) { to_pb_oneof<value.field_no>(val, sz_ptr, writer); } } else { constexpr uint32_t sub_key = (value.field_no << 3) | static_cast<uint32_t>(get_wire_type<U>()); to_pb_impl<sub_key>(val, sz_ptr, writer); } }, std::make_index_sequence<SIZE>{}); } else if constexpr (is_sequence_container<T>::value) { // TODO support std::array // repeated values can't be omitted even if values are empty using item_type = typename T::value_type; if constexpr (is_lenprefix_v<item_type>) { // non-packed for (auto& item : t) { to_pb_impl<key, false>(item, sz_ptr, writer); } } else { if (t.empty()) IGUANA_UNLIKELY { return; } serialize_varint_u32<key>(writer); serialize_varint(pb_value_size(t, sz_ptr), writer); for (auto& item : t) { encode_numeric_field<false, 0>(item, writer); } } } else if constexpr (is_map_container<T>::value) { using first_type = typename T::key_type; using second_type = typename T::mapped_type; constexpr uint32_t key1 = (1 << 3) | static_cast<uint32_t>(get_wire_type<first_type>()); constexpr auto key1_size = variant_uint32_size_constexpr(key1); constexpr uint32_t key2 = (2 << 3) | static_cast<uint32_t>(get_wire_type<second_type>()); constexpr auto key2_size = variant_uint32_size_constexpr(key2); for (auto& [k, v] : t) { serialize_varint_u32<key>(writer); // k must be string or numeric auto k_val_len = str_numeric_size<0, false>(k); auto v_val_len = pb_value_size<false>(v, sz_ptr); auto pair_len = key1_size + key2_size + k_val_len + v_val_len; if constexpr (is_lenprefix_v<first_type>) { pair_len += variant_uint32_size(k_val_len); } if constexpr (is_lenprefix_v<second_type>) { pair_len += variant_uint32_size(v_val_len); } serialize_varint(pair_len, writer); // map k and v can't be omitted even if values are empty encode_pair_value<key1>(k, k_val_len, sz_ptr, writer); encode_pair_value<key2>(v, v_val_len, sz_ptr, writer); } } else if constexpr (optional_v<T>) { if (!t.has_value()) { return; } to_pb_impl<key, omit_default_val>(*t, sz_ptr, writer); } else if constexpr (std::is_same_v<T, std::string> || std::is_same_v<T, std::string_view>) { if constexpr (omit_default_val) { if (t.size() == 0) IGUANA_UNLIKELY { return; } } serialize_varint_u32<key>(writer); serialize_varint(t.size(), writer); writer.write(t.data(), t.size()); } else { encode_numeric_field<key, omit_default_val>(t, writer); } } #if defined(__clang__) || defined(_MSC_VER) || \ (defined(__GNUC__) && __GNUC__ > 8) template <typename T> IGUANA_INLINE constexpr std::string_view get_type_string() { if constexpr (std::is_integral_v<T>) { if constexpr (std::is_same_v<T, bool>) { return "bool"; } else if constexpr (sizeof(T) <= 4) { if constexpr (std::is_unsigned_v<T>) { return "uint32"; } else { return "int32"; } } else { if constexpr (std::is_unsigned_v<T>) { return "uint64"; } else { return "int64"; } } } else if constexpr (std::is_same_v<T, iguana::sint32_t>) { return "sint32"; } else if constexpr (std::is_same_v<T, iguana::sint64_t>) { return "sint64"; } else if constexpr (std::is_same_v<T, iguana::fixed32_t>) { return "fixed32"; } else if constexpr (std::is_same_v<T, iguana::fixed64_t>) { return "fixed64"; } else if constexpr (std::is_same_v<T, iguana::sfixed32_t>) { return "sfixed32"; } else if constexpr (std::is_same_v<T, iguana::sfixed64_t>) { return "sfixed64"; } else if constexpr (std::is_same_v<T, std::string> || std::is_same_v<T, std::string_view>) { return "string"; } else if constexpr (std::is_floating_point_v<T>) { return type_string<T>(); } else { constexpr auto str_type_name = type_string<T>(); constexpr size_t pos = str_type_name.rfind("::"); if constexpr (pos != std::string_view::npos) { constexpr size_t pos = str_type_name.rfind("::") + 2; if constexpr (detail::is_signed_varint_v<T> || detail::is_fixed_v<T>) { return str_type_name.substr(pos, str_type_name.size() - pos - 2); } else { return str_type_name.substr(pos); } } else { return str_type_name; } } } template <typename T, typename Stream> IGUANA_INLINE void numeric_to_proto(Stream& out, std::string_view field_name, uint32_t field_no) { constexpr auto name = get_type_string<T>(); out.append(name).append(" "); out.append(field_name) .append(" = ") .append(std::to_string(field_no)) .append(";\n"); } template <size_t space_count = 2, typename Stream> IGUANA_INLINE void build_proto_field(Stream& out, std::string_view str_type, std::string_view field_name, uint32_t field_no) { for (size_t i = 0; i < space_count; i++) { out.append(" "); } if (!str_type.empty()) { out.append(str_type); } out.append(" ") .append(field_name) .append(" = ") .append(std::to_string(field_no)) .append(";\n"); } template <typename T, typename Map> IGUANA_INLINE void build_sub_proto(Map& map, std::string_view str_type, std::string& sub_str); template <typename Type, typename Stream> IGUANA_INLINE void to_proto_impl( Stream& out, std::unordered_map<std::string_view, std::string>& map, std::string_view field_name = "", uint32_t field_no = 0) { std::string sub_str; using T = std::remove_const_t<std::remove_reference_t<Type>>; if constexpr (ylt_refletable_v<T> || is_custom_reflection_v<T>) { constexpr auto name = type_string<T>(); out.append("message ").append(name).append(" {\n"); static T t; static auto tuple = get_pb_members_tuple(t); constexpr size_t SIZE = std::tuple_size_v<std::decay_t<decltype(tuple)>>; for_each_n( [&out, &sub_str, &map](auto i) mutable { using field_type = std::tuple_element_t<decltype(i)::value, std::decay_t<decltype(tuple)>>; auto value = std::get<decltype(i)::value>(tuple); using U = typename field_type::value_type; using sub_type = typename field_type::sub_type; if constexpr (ylt_refletable_v<U>) { constexpr auto str_type = get_type_string<U>(); build_proto_field( out, str_type, {value.field_name.data(), value.field_name.size()}, value.field_no); build_sub_proto<U>(map, str_type, sub_str); } else if constexpr (variant_v<U>) { constexpr size_t var_size = std::variant_size_v<U>; constexpr auto offset = get_variant_index<U, sub_type, var_size - 1>(); if (offset == 0) { out.append(" oneof "); out.append(value.field_name.data(), value.field_name.size()) .append(" {\n"); } constexpr auto str_type = get_type_string<sub_type>(); std::string field_name = " one_of_"; field_name.append(str_type); out.append(" "); build_proto_field(out, str_type, field_name, value.field_no); if constexpr (ylt_refletable_v<sub_type>) { build_sub_proto<sub_type>(map, str_type, sub_str); } if (offset == var_size - 1) { out.append(" }\n"); } } else { to_proto_impl<U>(out, map, {value.field_name.data(), value.field_name.size()}, value.field_no); } }, std::make_index_sequence<SIZE>{}); out.append("}\r\n\r\n"); } else if constexpr (is_sequence_container<T>::value) { out.append(" repeated"); using item_type = typename T::value_type; if constexpr (is_lenprefix_v<item_type>) { // non-packed if constexpr (ylt_refletable_v<item_type>) { constexpr auto str_type = get_type_string<item_type>(); build_proto_field(out, str_type, field_name, field_no); build_sub_proto<item_type>(map, str_type, sub_str); } else { to_proto_impl<item_type>(out, map, field_name, field_no); } } else { out.append(" "); numeric_to_proto<item_type>(out, field_name, field_no); } } else if constexpr (is_map_container<T>::value) { out.append(" map<"); using first_type = typename T::key_type; using second_type = typename T::mapped_type; constexpr auto str_first = get_type_string<first_type>(); constexpr auto str_second = get_type_string<second_type>(); out.append(str_first).append(", ").append(str_second).append(">"); build_proto_field<1>(out, "", field_name, field_no); if constexpr (ylt_refletable_v<second_type>) { constexpr auto str_type = get_type_string<second_type>(); build_sub_proto<second_type>(map, str_type, sub_str); } } else if constexpr (optional_v<T>) { to_proto_impl<typename T::value_type>( out, map, {field_name.data(), field_name.size()}, field_no); } else if constexpr (std::is_same_v<T, std::string> || std::is_same_v<T, std::string_view>) { build_proto_field(out, "string ", field_name, field_no); } else if constexpr (enum_v<T>) { constexpr auto str_type = get_type_string<T>(); static constexpr auto enum_to_str = get_enum_map<false, std::decay_t<T>>(); if constexpr (bool_v<decltype(enum_to_str)>) { build_proto_field(out, "int32", field_name, field_no); } else { static_assert(enum_to_str.size() > 0, "empty enum not allowed"); static_assert((int)(enum_to_str.begin()->first) == 0, "the first enum value must be zero in proto3"); build_proto_field(out, str_type, field_name, field_no); if (map.find(str_type) == map.end()) { sub_str.append("enum ").append(str_type).append(" {\n"); for (auto& [k, field_name] : enum_to_str) { std::string_view name{field_name.data(), field_name.size()}; size_t pos = name.rfind("::"); if (pos != std::string_view::npos) { name = name.substr(pos + 2); } sub_str.append(" ") .append(name) .append(" = ") .append(std::to_string(static_cast<std::underlying_type_t<T>>(k))) .append(";\n"); } sub_str.append("}\r\n\r\n"); map.emplace(str_type, std::move(sub_str)); } } } else { out.append(" "); numeric_to_proto<Type>(out, field_name, field_no); } } template <typename T, typename Map> IGUANA_INLINE void build_sub_proto(Map& map, std::string_view str_type, std::string& sub_str) { if (map.find(str_type) == map.end()) { to_proto_impl<T>(sub_str, map); map.emplace(str_type, std::move(sub_str)); } } #endif } // namespace detail template < typename T, typename Stream, std::enable_if_t<ylt_refletable_v<T> || detail::is_custom_reflection_v<T>, int> = 0> IGUANA_INLINE void to_pb(T const& t, Stream& out) { std::vector<uint32_t> size_arr; auto byte_len = detail::pb_key_value_size<0>(t, size_arr); auto sz_ptr = size_arr.empty() ? nullptr : &size_arr[0]; if constexpr (is_resizable_char_container_v<Stream>) { detail::resize(out, byte_len); memory_writer writer{out.data()}; detail::to_pb_impl<0>(t, sz_ptr, writer); } else if constexpr (char_writer<Stream>) { detail::to_pb_impl<0>(t, sz_ptr, out); } else { static_assert(!sizeof(Stream), "Invalid stream type"); } } #if defined(__clang__) || defined(_MSC_VER) || \ (defined(__GNUC__) && __GNUC__ > 8) template <typename T, bool gen_header = true, typename Stream> IGUANA_INLINE void to_proto(Stream& out, std::string_view ns = "") { if (gen_header) { constexpr std::string_view crlf = "\r\n\r\n"; out.append(R"(syntax = "proto3";)").append(crlf); if (!ns.empty()) { out.append("package ").append(ns).append(";").append(crlf); } out.append(R"(option optimize_for = SPEED;)").append(crlf); out.append(R"(option cc_enable_arenas = true;)").append(crlf); } std::unordered_map<std::string_view, std::string> map; detail::to_proto_impl<T>(out, map); for (auto& [k, s] : map) { out.append(s); } } template <typename T, bool gen_header = true, typename Stream> IGUANA_INLINE void to_proto_file(Stream& stream, std::string_view ns = "") { if (!stream.is_open()) { return; } std::string out; to_proto<T, gen_header>(out, ns); stream.write(out.data(), out.size()); } #endif template <typename T, typename Stream> IGUANA_INLINE void to_pb_adl(iguana_adl_t* p, T const& t, Stream& out) { to_pb(t, out); } } // namespace iguana