namespace io = boost::iostreams;
void saveCompressedData(const std::string& file_path, const std::vector<double>& bid, const std::vector<double>& ask, const std::vector<double>& close_prices, const std::vector<int>& tick_numbers) {
try {
// Create an output file stream and filtering stream
std::ofstream file(file_path, std::ios_base::out | std::ios_base::binary);
if (!file) {
std::cerr << "Error opening compressed file for writing: " << file_path << std::endl;
return;
}
io::filtering_ostream out;
out.push(io::gzip_compressor());
out.push(file);
// Write the data to the compressed file
uint64_t size = bid.size();
out.write((const char* ) (&size), sizeof(size));
out.write(reinterpret_cast<const char*>(bid.data()), size * sizeof(double));
out.write(reinterpret_cast<const char*>(ask.data()), size * sizeof(double));
size_t close_size = close_prices.size();
out.write(reinterpret_cast<const char*>(&close_size), sizeof(close_size));
out.write(reinterpret_cast<const char*>(close_prices.data()), close_size * sizeof(double));
size_t tick_size = tick_numbers.size();
out.write(reinterpret_cast<const char*>(&tick_size), sizeof(tick_size));
out.write(reinterpret_cast<const char*>(tick_numbers.data()), tick_size * sizeof(int));
out.pop();
}
catch (const std::exception& e) {
std::cerr << "Exception while writing compressed data: " << e.what() << std::endl;
}
}
void loadCompressedData(const std::string& file_path, std::vector<double>& bid, std::vector<double>& ask, std::vector<double>& close_prices, std::vector<int>& tick_numbers) {
try {
// Create an input file stream and filtering stream
std::ifstream file(file_path, std::ios_base::in | std::ios_base::binary);
if (!file) {
std::cerr << "Error opening compressed file for reading: " << file_path << std::endl;
return;
}
io::filtering_istream in;
in.push(io::gzip_decompressor());
in.push(file);
// Read the data from the compressed file
uint64_t size;
in.read((char*)(&size), sizeof(size));
bid.resize(size);
ask.resize(size);
in.read(reinterpret_cast<char*>(bid.data()), size * sizeof(double));
in.read(reinterpret_cast<char*>(ask.data()), size * sizeof(double));
size_t close_size;
in.read(reinterpret_cast<char*>(&close_size), sizeof(close_size));
close_prices.resize(close_size);
in.read(reinterpret_cast<char*>(close_prices.data()), close_size * sizeof(double));
size_t tick_size;
in.read(reinterpret_cast<char*>(&tick_size), sizeof(tick_size));
tick_numbers.resize(tick_size);
in.read(reinterpret_cast<char*>(tick_numbers.data()), tick_size * sizeof(int));
}
catch (const std::exception& e) {
std::cerr << "Exception while reading compressed data: " << e.what() << std::endl;
}
}
我测试了这个，但是 uint64_t size = bid.size();读取坏字节..可能是什么原因？
我无法重现这一点。我建议概括代码，以便您可以更直接地编写它：
using Amount = double;
using Amounts = std::vector<Amount>;
using Tick = int;
using Ticks = std::vector<Tick>;
struct Data {
Amounts bid, ask, close_prices;
Ticks tick_numbers;
auto operator<=>(Data const&) const = default;
void save(std::ostream& os) const {
assert(ask.size() == bid.size());
WriteRaw( //
os, bid.size(), bid, ask, //
close_prices.size(), close_prices, //
tick_numbers.size(), tick_numbers //
); //
}
void load(std::istream& is) {
auto n = ReadVector(is, bid);
ReadVector(is, ask, n);
ReadVector(is, close_prices);
ReadVector(is, tick_numbers);
}
};
void saveCompressedData(std::string const& file_path, Data const& data) try {
std::ofstream file(file_path, std::ios::binary);
io::filtering_ostream out;
out.push(io::gzip_compressor{});
out.push(file);
data.save(out);
} catch (std::exception const& e) {
std::cerr << "Exception while writing compressed data: " << e.what() << std::endl;
}
void loadCompressedData(std::string const& file_path, Data& data) try {
std::ifstream file(file_path, std::ios::binary);
io::filtering_istream in;
in.push(io::gzip_decompressor{});
in.push(file);
data.load(in);
} catch (std::exception const& e) {
std::cerr << "Exception while reading compressed data: " << e.what() << std::endl;
}
读/写助手使用字节跨度来避免编程错误。
namespace /* filestatic */ {
void WriteRaw(std::ostream& os, std::span<std::byte const> v);
void ReadRaw(std::istream& is, std::span<std::byte> v);
// write helpers
template <typename T> void WriteRaw(std::ostream& os, std::vector<T> const& v);
template <typename T> void WriteRaw(std::ostream& os, T const& v);
template <typename... T> void WriteRaw(std::ostream& os, T const&... vs);
// read helpers
template <typename T> void ReadRaw(std::istream& is, T& v);
template <typename... T> void ReadRaw(std::istream& is, T&... vs);
template <typename T> void ReadVector(std::istream& is, std::vector<T>& v, size_t size);
template <typename T> size_t ReadVector(std::istream& is, std::vector<T>& v);
} // namespace
实现还应确保元素值是可按位复制的。
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/iostreams/filtering_stream.hpp>
#include <fstream>
#include <iostream>
#include <span>
namespace io = boost::iostreams;
namespace /* filestatic */ {
void WriteRaw(std::ostream& os, std::span<std::byte const> v) {
os.write(reinterpret_cast<char const*>(v.data()), v.size());
}
void ReadRaw(std::istream& is, std::span<std::byte> v) {
is.read(reinterpret_cast<char*>(v.data()), v.size());
}
// write helpers
template <typename T> void WriteRaw(std::ostream& os, std::vector<T> const& v) {
static_assert(std::is_trivially_copyable_v<T>);
WriteRaw(os, std::as_bytes(std::span(v)));
}
template <typename T> void WriteRaw(std::ostream& os, T const& v) {
static_assert(std::is_trivially_copyable_v<T>);
WriteRaw(os, std::as_bytes(std::span(&v, 1)));
}
template <typename... T> void WriteRaw(std::ostream& os, T const&... vs) {
(WriteRaw(os, vs), ...);
}
// read helpers
template <typename T> void ReadRaw(std::istream& is, T& v) {
static_assert(std::is_trivially_copyable_v<T>);
ReadRaw(is, std::as_writable_bytes(std::span(&v, 1)));
}
template <typename... T> void ReadRaw(std::istream& is, T&... vs) {
(ReadRaw(is, vs), ...);
}
template <typename T> void ReadVector(std::istream& is, std::vector<T>& v, size_t size) {
static_assert(std::is_trivially_copyable_v<T>);
v.resize(size);
ReadRaw(is, std::as_writable_bytes(std::span(v)));
}
template <typename T> size_t ReadVector(std::istream& is, std::vector<T>& v) {
static_assert(std::is_trivially_copyable_v<T>);
size_t tmpsize;
ReadRaw(is, tmpsize);
ReadVector(is, v, tmpsize);
return tmpsize;
}
}
using Amount = double;
using Amounts = std::vector<Amount>;
using Tick = int;
using Ticks = std::vector<Tick>;
struct Data {
Amounts bid, ask, close_prices;
Ticks tick_numbers;
auto operator<=>(Data const&) const = default;
void save(std::ostream& os) const {
assert(ask.size() == bid.size());
WriteRaw( //
os, bid.size(), bid, ask, //
close_prices.size(), close_prices, //
tick_numbers.size(), tick_numbers //
); //
}
void load(std::istream& is) {
auto n = ReadVector(is, bid);
ReadVector(is, ask, n);
ReadVector(is, close_prices);
ReadVector(is, tick_numbers);
}
};
void saveCompressedData(std::string const& file_path, Data const& data) try {
std::ofstream file(file_path, std::ios::binary);
io::filtering_ostream out;
out.push(io::gzip_compressor{});
out.push(file);
data.save(out);
} catch (std::exception const& e) {
std::cerr << "Exception while writing compressed data: " << e.what() << std::endl;
}
void loadCompressedData(std::string const& file_path, Data& data) try {
std::ifstream file(file_path, std::ios::binary);
io::filtering_istream in;
in.push(io::gzip_decompressor{});
in.push(file);
data.load(in);
} catch (std::exception const& e) {
std::cerr << "Exception while reading compressed data: " << e.what() << std::endl;
}
#include <fmt/ranges.h>
int main() {
using L = std::numeric_limits<Amount>;
for (Data const& data : {
Data{{1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}, {7.0, 8.0, 9.0}, {10, 11, 12}},
Data{{1.1111e11, 2.2222e11, 3.3333e11, 4.4444e11},
{5.5555e11, 6.6666e11, 7.7777e11, 8.8888e11},
{7.7777e11},
{10, 11, 12, 13, 14}},
Data{{}, {}, {0.10, L::infinity(), -L::infinity(), 10.2, 10.3, 10.4}, {}},
Data{},
// nan is always "not equal" to itself, but also roundtrips
Data{{}, {}, {L::quiet_NaN()}, {}},
}) {
saveCompressedData("test.gz", data);
Data roundtrip;
loadCompressedData("test.gz", roundtrip);
auto& [bid, ask, close_prices, tick_numbers] = roundtrip;
fmt::print ("bid: {}\nask: {}\nclose_prices: {}\ntick_numbers: {}\n", bid, ask, close_prices, tick_numbers);
std::cout << "Roundtrip: " << (data == roundtrip ? "OK" : "FAIL") << "\n---------------\n" << std::endl;
}
}
打印
bid: [1, 2, 3]
ask: [4, 5, 6]
close_prices: [7, 8, 9]
tick_numbers: [10, 11, 12]
Roundtrip: OK
---------------
bid: [111110000000, 222220000000, 333330000000, 444440000000]
ask: [555550000000, 666660000000, 777770000000, 888880000000]
close_prices: [777770000000]
tick_numbers: [10, 11, 12, 13, 14]
Roundtrip: OK
---------------
bid: []
ask: []
close_prices: [0.1, inf, -inf, 10.2, 10.3, 10.4]
tick_numbers: []
Roundtrip: OK
---------------
bid: []
ask: []
close_prices: []
tick_numbers: []
Roundtrip: OK
---------------
bid: []
ask: []
close_prices: [nan]
tick_numbers: []
Roundtrip: FAIL
---------------