'Save Struct with Armadillo Vec to hdf5 file
I am trying to save a struct that contains a few arma::vec and I'm not sure on how to write those to the hdf5 file.
#define ARMA_DONT_USE_WRAPPER
#include <armadillo>
#include <string>
#include <iostream>
#include "H5Cpp.h"
#define ARMA_ALLOW_FAKE_GCC //allows the use of GCC compiled armadillo with ICC compiler
using namespace std;
using namespace arma;
struct info_struct{
double frequency_MHz;
vec beam_centres_deg;
vec beam_crossover_deg;
double task_centre_azimuth_deg;
double WRF_Hz;
double bandwidth_kHz;
double beam_spacing_dB;
int direct_wave_range_bin;
double fs;
vec rx_rng_twoway_vec_km;
string date;
};
int main(void)
{
info_struct header;
header.frequency_MHz=7;
header.beam_centres_deg=8*ones<vec>(10);
header.beam_crossover_deg=9*ones<vec>(10);
header.task_centre_azimuth_deg=10;
header.WRF_Hz = 11;
header.bandwidth_kHz=12;
header.beam_spacing_dB=13;
header.direct_wave_range_bin=14;
header.fs=15e3;
header.rx_rng_twoway_vec_km=16*ones<vec>(10);
header.date="14-Oct-2020 17:02:30";
// the array of each length of multidimentional data.
hsize_t dim_header[1];
dim_header[0] = sizeof(header) / sizeof(info_struct);
// the length of dim
int rank_header = sizeof(dim_header) / sizeof(hsize_t);
// defining the datatype to pass HDF55
H5::CompType h5_header(sizeof(info_struct));
h5_header.insertMember("frequency_MHz", HOFFSET(info_struct, frequency_MHz), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("beam_centres_deg", HOFFSET(info_struct, beam_centres_deg), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("beam_crossover_deg", HOFFSET(info_struct, beam_crossover_deg), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("task_centre_azimuth_deg", HOFFSET(info_struct, task_centre_azimuth_deg), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("WRF_Hz", HOFFSET(info_struct, WRF_Hz), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("bandwidth_kHz", HOFFSET(info_struct, bandwidth_kHz), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("beam_spacing_dB", HOFFSET(info_struct, beam_spacing_dB), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("direct_wave_range_bin", HOFFSET(info_struct, direct_wave_range_bin), H5::PredType::NATIVE_INT);
h5_header.insertMember("fs", HOFFSET(info_struct, fs), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("rx_rng_twoway_vec_km", HOFFSET(info_struct, rx_rng_twoway_vec_km), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("date", HOFFSET(info_struct, date), H5::StrType(H5::PredType::C_S1, H5T_VARIABLE));
// preparation of a dataset and a file.
H5::DataSpace space_header(rank_header, dim_header);
H5::H5File *file = new H5::H5File("data_test.h5", H5F_ACC_TRUNC);
H5::DataSet *dataset_header = new H5::DataSet(file->createDataSet("MOTHR_header", h5_header, space_header));
// Write
dataset_header->write(&header, h5_header);
delete dataset_header;
delete file;
return 0;
}
if I comment out everything to do with the vectors, it writes to the file just fine but when I leave them uncommented I get this warning when compiling:
In file included from /usr/include/unistd.h:226,
from /usr/include/armadillo:62,
from save_data.cpp:2:
save_data.cpp: In function ‘int main()’:
save_data.cpp:65:53: warning: offsetof within non-standard-layout type ‘info_struct’ is conditionally-supported [-Winvalid-offsetof]
h5_header.insertMember("frequency_MHz", HOFFSET(info_struct, frequency_MHz), H5::PredType::NATIVE_DOUBLE);
save_data.cpp:65:45: note: in expansion of macro ‘HOFFSET’
h5_header.insertMember("frequency_MHz", HOFFSET(info_struct, frequency_MHz), H5::PredType::NATIVE_DOUBLE);
^~~~~~~
save_data.cpp:67:56: warning: offsetof within non-standard-layout type ‘info_struct’ is conditionally-supported [-Winvalid-offsetof]
h5_header.insertMember("beam_centres_deg", HOFFSET(info_struct, beam_centres_deg), H5::PredType::NATIVE_DOUBLE);
save_data.cpp:67:48: note: in expansion of macro ‘HOFFSET’
h5_header.insertMember("beam_centres_deg", HOFFSET(info_struct, beam_centres_deg), H5::PredType::NATIVE_DOUBLE);
^~~~~~~
save_data.cpp:68:58: warning: offsetof within non-standard-layout type ‘info_struct’ is conditionally-supported [-Winvalid-offsetof]
h5_header.insertMember("beam_crossover_deg", HOFFSET(info_struct, beam_crossover_deg), H5::PredType::NATIVE_DOUBLE);
save_data.cpp:68:50: note: in expansion of macro ‘HOFFSET’
h5_header.insertMember("beam_crossover_deg", HOFFSET(info_struct, beam_crossover_deg), H5::PredType::NATIVE_DOUBLE);
^~~~~~~
save_data.cpp:70:63: warning: offsetof within non-standard-layout type ‘info_struct’ is conditionally-supported [-Winvalid-offsetof]
h5_header.insertMember("task_centre_azimuth_deg", HOFFSET(info_struct, task_centre_azimuth_deg), H5::PredType::NATIVE_DOUBLE);
save_data.cpp:70:55: note: in expansion of macro ‘HOFFSET’
h5_header.insertMember("task_centre_azimuth_deg", HOFFSET(info_struct, task_centre_azimuth_deg), H5::PredType::NATIVE_DOUBLE);
^~~~~~~
save_data.cpp:71:46: warning: offsetof within non-standard-layout type ‘info_struct’ is conditionally-supported [-Winvalid-offsetof]
h5_header.insertMember("WRF_Hz", HOFFSET(info_struct, WRF_Hz), H5::PredType::NATIVE_DOUBLE);
save_data.cpp:71:38: note: in expansion of macro ‘HOFFSET’
h5_header.insertMember("WRF_Hz", HOFFSET(info_struct, WRF_Hz), H5::PredType::NATIVE_DOUBLE);
^~~~~~~
save_data.cpp:72:53: warning: offsetof within non-standard-layout type ‘info_struct’ is conditionally-supported [-Winvalid-offsetof]
h5_header.insertMember("bandwidth_kHz", HOFFSET(info_struct, bandwidth_kHz), H5::PredType::NATIVE_DOUBLE);
save_data.cpp:72:45: note: in expansion of macro ‘HOFFSET’
h5_header.insertMember("bandwidth_kHz", HOFFSET(info_struct, bandwidth_kHz), H5::PredType::NATIVE_DOUBLE);
^~~~~~~
save_data.cpp:73:55: warning: offsetof within non-standard-layout type ‘info_struct’ is conditionally-supported [-Winvalid-offsetof]
h5_header.insertMember("beam_spacing_dB", HOFFSET(info_struct, beam_spacing_dB), H5::PredType::NATIVE_DOUBLE);
save_data.cpp:73:47: note: in expansion of macro ‘HOFFSET’
h5_header.insertMember("beam_spacing_dB", HOFFSET(info_struct, beam_spacing_dB), H5::PredType::NATIVE_DOUBLE);
^~~~~~~
save_data.cpp:74:61: warning: offsetof within non-standard-layout type ‘info_struct’ is conditionally-supported [-Winvalid-offsetof]
h5_header.insertMember("direct_wave_range_bin", HOFFSET(info_struct, direct_wave_range_bin), H5::PredType::NATIVE_INT);
save_data.cpp:74:53: note: in expansion of macro ‘HOFFSET’
h5_header.insertMember("direct_wave_range_bin", HOFFSET(info_struct, direct_wave_range_bin), H5::PredType::NATIVE_INT);
^~~~~~~
save_data.cpp:75:42: warning: offsetof within non-standard-layout type ‘info_struct’ is conditionally-supported [-Winvalid-offsetof]
h5_header.insertMember("fs", HOFFSET(info_struct, fs), H5::PredType::NATIVE_DOUBLE);
save_data.cpp:75:34: note: in expansion of macro ‘HOFFSET’
h5_header.insertMember("fs", HOFFSET(info_struct, fs), H5::PredType::NATIVE_DOUBLE);
^~~~~~~
save_data.cpp:79:44: warning: offsetof within non-standard-layout type ‘info_struct’ is conditionally-supported [-Winvalid-offsetof]
h5_header.insertMember("date", HOFFSET(info_struct, date), H5::StrType(H5::PredType::C_S1, H5T_VARIABLE));
save_data.cpp:79:36: note: in expansion of macro ‘HOFFSET’
h5_header.insertMember("date", HOFFSET(info_struct, date), H5::StrType(H5::PredType::C_S1, H5T_VARIABLE));
And the function seems to run fine but when i try h5dump on the generated file I get:
HDF5 "data_test.h5" {
GROUP "/" {
DATASET "MOTHR_header" {
DATATYPE H5T_COMPOUND {
H5T_IEEE_F64LE "frequency_MHz";
H5T_IEEE_F64LE "beam_centres_deg";
H5T_IEEE_F64LE "beam_crossover_deg";
H5T_IEEE_F64LE "task_centre_azimuth_deg";
H5T_IEEE_F64LE "WRF_Hz";
H5T_IEEE_F64LE "bandwidth_kHz";
H5T_IEEE_F64LE "beam_spacing_dB";
H5T_STD_I32LE "direct_wave_range_bin";
H5T_IEEE_F64LE "fs";
H5T_IEEE_F64LE "rx_rng_twoway_vec_km";
H5T_STRING {
STRSIZE H5T_VARIABLE;
STRPAD H5T_STR_NULLTERM;
CSET H5T_CSET_ASCII;
CTYPE H5T_C_S1;
} "date";
}
DATASPACE SIMPLE { ( 1 ) / ( 1 ) }
DATA {
(0): {
7,
4.94066e-323,
4.94066e-323,
10,
11,
12,
13,
14,
15000,
4.94066e-323,
"14-Oct-2020 17:02:30"
}
}
}
}
}
I'm very new to hdf5 so I am completely lost on how to save these vectors and any help would be greatly appreciated. I also have another struct with arma:cx_mat. I know that when reading them I had to read them separately then combine them, so is it safe to assume I'll have to do the same for writing them?
UPDATE:
After reading some more about compound datatypes I made modifications to my code:
#define ARMA_DONT_USE_WRAPPER
#include <armadillo>
#include <string>
#include <iostream>
#include "H5Cpp.h"
#define ARMA_ALLOW_FAKE_GCC //allows the use of GCC compiled armadillo with ICC compiler
using namespace std;
using namespace arma;
typedef std::vector<double> stdvec;
struct info_struct{
double frequency_MHz;
vec beam_centres_deg;
stdvec beam_centres_deg_c;
vec beam_crossover_deg;
stdvec beam_crossover_deg_c;
double task_centre_azimuth_deg;
double WRF_Hz;
double bandwidth_kHz;
double beam_spacing_dB;
int direct_wave_range_bin;
double fs;
vec rx_rng_twoway_vec_km;
stdvec rx_rng_twoway_vec_km_c;
string date;
};
struct info_struct_hdf5{
double frequency_MHz;
double* beam_centres_deg;
hvl_t beam_centres_deg_handle;
double* beam_crossover_deg;
hvl_t beam_crossover_deg_handle;
double task_centre_azimuth_deg;
double WRF_Hz;
double bandwidth_kHz;
double beam_spacing_dB;
int direct_wave_range_bin;
double fs;
double* rx_rng_twoway_vec_km;
hvl_t rx_rng_twoway_vec_km_handle;
string date;
};
int main(void)
{
info_struct header;
info_struct_hdf5 header_hdf5;
header.frequency_MHz=7;
header.beam_centres_deg=8*ones<vec>(10);
header.beam_crossover_deg=9*ones<vec>(10);
header.task_centre_azimuth_deg=10;
header.WRF_Hz = 11;
header.bandwidth_kHz=12;
header.beam_spacing_dB=13;
header.direct_wave_range_bin=14;
header.fs=15e3;
header.rx_rng_twoway_vec_km=16*ones<vec>(10);
header.date="14-Oct-2020 17:02:30";
header_hdf5.frequency_MHz=header.frequency_MHz;
header_hdf5.beam_centres_deg=header.beam_centres_deg.memptr();
//header_hdf5.beam_crossover_deg=eader.beam_crossover_deg;
header_hdf5.task_centre_azimuth_deg=header.task_centre_azimuth_deg;
header_hdf5.WRF_Hz = header.WRF_Hz;
header_hdf5.bandwidth_kHz=header.bandwidth_kHz;
header_hdf5.beam_spacing_dB=header.beam_spacing_dB;
header_hdf5.direct_wave_range_bin=header.direct_wave_range_bin;
header_hdf5.fs=header.fs;
//header_hdf5.rx_rng_twoway_vec_km=header.rx_rng_twoway_vec_km;
header_hdf5.date=header.date;
// the array of each length of multidimentional data.
hsize_t dim_header[1];
dim_header[0] = sizeof(header_hdf5) / sizeof(info_struct_hdf5);
// the length of dim
int rank_header = sizeof(dim_header) / sizeof(hsize_t);
// defining the datatype to pass HDF55
H5::CompType h5_header(sizeof(info_struct_hdf5));
h5_header.insertMember("frequency_MHz", HOFFSET(info_struct_hdf5, frequency_MHz), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("beam_centres_deg", HOFFSET(info_struct_hdf5, beam_centres_deg_handle), H5::VarLenType(H5::PredType::NATIVE_DOUBLE));
h5_header.insertMember("beam_crossover_deg", HOFFSET(info_struct_hdf5, beam_crossover_deg_handle), H5::VarLenType(H5::PredType::NATIVE_DOUBLE));
h5_header.insertMember("task_centre_azimuth_deg", HOFFSET(info_struct_hdf5, task_centre_azimuth_deg), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("WRF_Hz", HOFFSET(info_struct_hdf5, WRF_Hz), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("bandwidth_kHz", HOFFSET(info_struct_hdf5, bandwidth_kHz), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("beam_spacing_dB", HOFFSET(info_struct_hdf5, beam_spacing_dB), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("direct_wave_range_bin", HOFFSET(info_struct_hdf5, direct_wave_range_bin), H5::PredType::NATIVE_INT);
h5_header.insertMember("fs", HOFFSET(info_struct_hdf5, fs), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("rx_rng_twoway_vec_km", HOFFSET(info_struct_hdf5, rx_rng_twoway_vec_km_handle), H5::VarLenType(H5::PredType::NATIVE_DOUBLE));
h5_header.insertMember("date", HOFFSET(info_struct_hdf5, date), H5::StrType(H5::PredType::C_S1, H5T_VARIABLE));
// preparation of a dataset and a file.
H5::DataSpace space_header(rank_header, dim_header);
H5::H5File *file = new H5::H5File("data_test.h5", H5F_ACC_TRUNC);
H5::DataSet *dataset_header = new H5::DataSet(file->createDataSet("MOTHR_header", h5_header, space_header));
// Write
dataset_header->write(&header_hdf5, h5_header);
delete dataset_header;
delete file;
return 0;
}
This compiles with now warnings or error so I thought it was at least a step in the right direction but when I run the code I get:
HDF5-DIAG: Error detected in HDF5 (1.10.4) thread 140528006936384:
#000: ../../../src/H5Dio.c line 336 in H5Dwrite(): can't write data
major: Dataset
minor: Write failed
#001: ../../../src/H5Dio.c line 828 in H5D__write(): can't write data
major: Dataset
minor: Write failed
#002: ../../../src/H5Dcontig.c line 633 in H5D__contig_write(): contiguous write failed
major: Dataset
minor: Write failed
#003: ../../../src/H5Dscatgath.c line 704 in H5D__scatgath_write(): datatype conversion failed
major: Dataset
minor: Can't convert datatypes
#004: ../../../src/H5T.c line 5025 in H5T_convert(): datatype conversion failed
major: Datatype
minor: Can't convert datatypes
#005: ../../../src/H5Tconv.c line 2512 in H5T__conv_struct_opt(): unable to convert compound datatype member
major: Datatype
minor: Unable to initialize object
#006: ../../../src/H5T.c line 5025 in H5T_convert(): datatype conversion failed
major: Datatype
minor: Can't convert datatypes
#007: ../../../src/H5Tconv.c line 3272 in H5T__conv_vlen(): can't write VL data
major: Datatype
minor: Write failed
#008: ../../../src/H5Tvlen.c line 913 in H5T_vlen_disk_write(): Unable to write VL information
major: Datatype
minor: Write failed
#009: ../../../src/H5HG.c line 543 in H5HG_insert(): unable to allocate a global heap collection
major: Heap
minor: Unable to initialize object
#010: ../../../src/H5HG.c line 163 in H5HG__create(): memory allocation failed
major: Resource unavailable
minor: No space available for allocation
#011: ../../../src/H5FL.c line 922 in H5FL_blk_malloc(): memory allocation failed for chunk
major: Resource unavailable
minor: No space available for allocation
#012: ../../../src/H5FL.c line 250 in H5FL_malloc(): memory allocation failed for chunk
major: Resource unavailable
minor: No space available for allocation
terminate called after throwing an instance of 'H5::DataSetIException'
Aborted (core dumped)
Solution 1:[1]
I finally figured it out!
First, 2 structs that are very similar:
struct info_struct{
double frequency_MHz;
vec beam_centres_deg;
vec beam_crossover_deg;
double task_centre_azimuth_deg;
double WRF_Hz;
double bandwidth_kHz;
double beam_spacing_dB;
int direct_wave_range_bin;
double fs;
vec rx_rng_twoway_vec_km;
string date;
};
struct info_struct_hdf5{
double frequency_MHz;
hvl_t beam_centres_deg_handle;
hvl_t beam_crossover_deg_handle;
double task_centre_azimuth_deg;
double WRF_Hz;
double bandwidth_kHz;
double beam_spacing_dB;
int direct_wave_range_bin;
double fs;
hvl_t rx_rng_twoway_vec_km_handle;
string date;
};
Next transfer the data from the original struct to this new buffer struct:
header_hdf5.frequency_MHz=header.frequency_MHz;
header_hdf5.beam_centres_deg_handle.p=header.beam_centres_deg.memptr();
header_hdf5.beam_centres_deg_handle.len = header.beam_centres_deg.n_elem;
header_hdf5.beam_crossover_deg_handle.p=header.beam_crossover_deg.memptr();
header_hdf5.beam_crossover_deg_handle.len = header.beam_crossover_deg.n_elem;
header_hdf5.task_centre_azimuth_deg=header.task_centre_azimuth_deg;
header_hdf5.WRF_Hz = header.WRF_Hz;
header_hdf5.bandwidth_kHz=header.bandwidth_kHz;
header_hdf5.beam_spacing_dB=header.beam_spacing_dB;
header_hdf5.direct_wave_range_bin=header.direct_wave_range_bin;
header_hdf5.fs=header.fs;
header_hdf5.rx_rng_twoway_vec_km_handle.p=header.rx_rng_twoway_vec_km.memptr();
header_hdf5.rx_rng_twoway_vec_km_handle.len = header.rx_rng_twoway_vec_km.n_elem;
header_hdf5.date=header.date;
Create a new type:
hid_t vlen_tid = H5Tvlen_create(H5T_NATIVE_DOUBLE);
Then insert the members using the buffer:
h5_header.insertMember("frequency_MHz", HOFFSET(info_struct_hdf5, frequency_MHz), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("beam_centres_deg", HOFFSET(info_struct_hdf5, beam_centres_deg_handle), vlen_tid);
h5_header.insertMember("beam_crossover_deg", HOFFSET(info_struct_hdf5, beam_crossover_deg_handle), vlen_tid);
h5_header.insertMember("task_centre_azimuth_deg", HOFFSET(info_struct_hdf5, task_centre_azimuth_deg), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("WRF_Hz", HOFFSET(info_struct_hdf5, WRF_Hz), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("bandwidth_kHz", HOFFSET(info_struct_hdf5, bandwidth_kHz), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("beam_spacing_dB", HOFFSET(info_struct_hdf5, beam_spacing_dB), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("direct_wave_range_bin", HOFFSET(info_struct_hdf5, direct_wave_range_bin), H5::PredType::NATIVE_INT);
h5_header.insertMember("fs", HOFFSET(info_struct_hdf5, fs), H5::PredType::NATIVE_DOUBLE);
h5_header.insertMember("rx_rng_twoway_vec_km", HOFFSET(info_struct_hdf5, rx_rng_twoway_vec_km_handle), vlen_tid);
h5_header.insertMember("date", HOFFSET(info_struct_hdf5, date), H5::StrType(H5::PredType::C_S1, H5T_VARIABLE));
And it works beautifully for me!
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|---|
| Solution 1 | Tyler |