Skip to content

Commit

Permalink
Read compressed MAT file in blocks
Browse files Browse the repository at this point in the history 
As reported by #65
  • Loading branch information
@tbeu
tbeu committed Jul 25, 2019
1 parent 64f7936 commit db29fe3
Show file tree
Hide file tree
Showing 6 changed files with 799 additions and 1,262 deletions.
5 changes: 3 additions & 2 deletions src/Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
VPATH = @srcdir@

LT_CURRENT=10
LT_REVISION=2
LT_REVISION=2
LT_AGE=1

AM_CFLAGS = -I. $(HDF5_CFLAGS) $(ZLIB_CFLAGS) $(LT_CFLAGS)
Expand All @@ -40,7 +40,8 @@ endif
nodist_noinst_HEADERS = matioConfig.h
nodist_include_HEADERS = matio_pubconf.h
include_HEADERS = matio.h
noinst_HEADERS = exact-int.h matio_private.h mat4.h mat5.h mat73.h safe-math.h
noinst_HEADERS = exact-int.h matio_private.h mat4.h mat5.h mat73.h \
read_data_impl.h safe-math.h
lib_LTLIBRARIES = libmatio.la
libmatio_la_SOURCES = snprintf.c endian.c io.c $(ZLIB_SRC) read_data.c \
mat5.c mat4.c mat.c matvar_cell.c matvar_struct.c
Expand Down
38 changes: 19 additions & 19 deletions src/inflate.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,14 +47,14 @@
size_t
InflateSkip(mat_t *mat, z_streamp z, int nbytes)
{
mat_uint8_t comp_buf[512],uncomp_buf[512];
mat_uint8_t comp_buf[READ_BLOCK_SIZE], uncomp_buf[READ_BLOCK_SIZE];
int n, err, cnt = 0;
size_t bytesread = 0;

if ( nbytes < 1 )
return 0;

n = (nbytes<512) ? nbytes : 512;
n = nbytes < READ_BLOCK_SIZE ? nbytes : READ_BLOCK_SIZE;
if ( !z->avail_in ) {
z->next_in = comp_buf;
z->avail_in += fread(comp_buf,1,n,(FILE*)mat->fp);
Expand All @@ -71,7 +71,7 @@ InflateSkip(mat_t *mat, z_streamp z, int nbytes)
}
if ( !z->avail_out ) {
cnt += n;
n = ((nbytes-cnt)<512) ? nbytes-cnt : 512;
n = nbytes - cnt < READ_BLOCK_SIZE ? nbytes - cnt : READ_BLOCK_SIZE;
z->avail_out = n;
z->next_out = uncomp_buf;
}
Expand All @@ -89,8 +89,8 @@ InflateSkip(mat_t *mat, z_streamp z, int nbytes)
break;
}
if ( !z->avail_out ) {
cnt += n;
n = ((nbytes-cnt)<512) ? nbytes-cnt : 512;
cnt += n;
n = nbytes - cnt < READ_BLOCK_SIZE ? nbytes - cnt : READ_BLOCK_SIZE;
z->avail_out = n;
z->next_out = uncomp_buf;
}
Expand Down Expand Up @@ -252,7 +252,7 @@ InflateVarTag(mat_t *mat, matvar_t *matvar, void *buf)
bytesread += fread(comp_buf,1,1,(FILE*)mat->fp);
}
matvar->internal->z->avail_out = 8;
matvar->internal->z->next_out = (Bytef*)buf;
matvar->internal->z->next_out = ZLIB_BYTE_PTR(buf);
err = inflate(matvar->internal->z,Z_NO_FLUSH);
if ( err != Z_OK ) {
Mat_Critical("InflateVarTag: inflate returned %s",zError(err == Z_NEED_DICT ? Z_DATA_ERROR : err));
Expand Down Expand Up @@ -304,7 +304,7 @@ InflateArrayFlags(mat_t *mat, matvar_t *matvar, void *buf)
bytesread += fread(comp_buf,1,1,(FILE*)mat->fp);
}
matvar->internal->z->avail_out = 16;
matvar->internal->z->next_out = (Bytef*)buf;
matvar->internal->z->next_out = ZLIB_BYTE_PTR(buf);
err = inflate(matvar->internal->z,Z_NO_FLUSH);
if ( err != Z_OK ) {
Mat_Critical("InflateArrayFlags: inflate returned %s",zError(err == Z_NEED_DICT ? Z_DATA_ERROR : err));
Expand Down Expand Up @@ -361,7 +361,7 @@ InflateRankDims(mat_t *mat, matvar_t *matvar, void *buf, size_t nbytes, mat_uint
bytesread += fread(comp_buf,1,1,(FILE*)mat->fp);
}
matvar->internal->z->avail_out = 8;
matvar->internal->z->next_out = (Bytef*)buf;
matvar->internal->z->next_out = ZLIB_BYTE_PTR(buf);
err = inflate(matvar->internal->z,Z_NO_FLUSH);
if ( err != Z_OK ) {
Mat_Critical("InflateRankDims: inflate returned %s",zError(err == Z_NEED_DICT ? Z_DATA_ERROR : err));
Expand Down Expand Up @@ -403,12 +403,12 @@ InflateRankDims(mat_t *mat, matvar_t *matvar, void *buf, size_t nbytes, mat_uint

matvar->internal->z->avail_out = rank;
if ( sizeof(mat_uint32_t)*(rank + 2) <= nbytes ) {
matvar->internal->z->next_out = (Bytef*)((mat_int32_t *)buf+2);
matvar->internal->z->next_out = ZLIB_BYTE_PTR((mat_int32_t *)buf+2);
} else {
/* Cannot use too small buf, but can allocate output buffer dims */
*dims = (mat_uint32_t*)calloc(rank, sizeof(mat_uint32_t));
if ( NULL != *dims ) {
matvar->internal->z->next_out = (Bytef*)*dims;
matvar->internal->z->next_out = ZLIB_BYTE_PTR(*dims);
} else {
*((mat_int32_t *)buf+1) = 0;
Mat_Critical("Error allocating memory for dims");
Expand Down Expand Up @@ -467,7 +467,7 @@ InflateVarName(mat_t *mat, matvar_t *matvar, void *buf, int N)
bytesread += fread(comp_buf,1,1,(FILE*)mat->fp);
}
matvar->internal->z->avail_out = N;
matvar->internal->z->next_out = (Bytef*)buf;
matvar->internal->z->next_out = ZLIB_BYTE_PTR(buf);
err = inflate(matvar->internal->z,Z_NO_FLUSH);
if ( err != Z_OK ) {
Mat_Critical("InflateVarName: inflate returned %s",zError(err == Z_NEED_DICT ? Z_DATA_ERROR : err));
Expand Down Expand Up @@ -519,7 +519,7 @@ InflateDataTag(mat_t *mat, matvar_t *matvar, void *buf)
bytesread += fread(comp_buf,1,1,(FILE*)mat->fp);
}
matvar->internal->z->avail_out = 8;
matvar->internal->z->next_out = (Bytef*)buf;
matvar->internal->z->next_out = ZLIB_BYTE_PTR(buf);
err = inflate(matvar->internal->z,Z_NO_FLUSH);
if ( err == Z_STREAM_END ) {
return bytesread;
Expand Down Expand Up @@ -575,7 +575,7 @@ InflateDataType(mat_t *mat, z_streamp z, void *buf)
bytesread += fread(comp_buf,1,1,(FILE*)mat->fp);
}
z->avail_out = 4;
z->next_out = (Bytef*)buf;
z->next_out = ZLIB_BYTE_PTR(buf);
err = inflate(z,Z_NO_FLUSH);
if ( err != Z_OK ) {
Mat_Critical("InflateDataType: inflate returned %s",zError(err == Z_NEED_DICT ? Z_DATA_ERROR : err));
Expand Down Expand Up @@ -615,7 +615,7 @@ InflateDataType(mat_t *mat, z_streamp z, void *buf)
size_t
InflateData(mat_t *mat, z_streamp z, void *buf, unsigned int nBytes)
{
mat_uint8_t comp_buf[1024];
mat_uint8_t comp_buf[READ_BLOCK_SIZE];
int err;
unsigned int bytesread = 0;

Expand All @@ -626,16 +626,16 @@ InflateData(mat_t *mat, z_streamp z, void *buf, unsigned int nBytes)
}

if ( !z->avail_in ) {
if ( nBytes > 1024 ) {
z->avail_in = fread(comp_buf,1,1024,(FILE*)mat->fp);
if ( nBytes > READ_BLOCK_SIZE ) {
z->avail_in = fread(comp_buf,1,READ_BLOCK_SIZE,(FILE*)mat->fp);
} else {
z->avail_in = fread(comp_buf,1,nBytes,(FILE*)mat->fp);
}
bytesread += z->avail_in;
z->next_in = comp_buf;
}
z->avail_out = nBytes;
z->next_out = (Bytef*)buf;
z->next_out = ZLIB_BYTE_PTR(buf);
err = inflate(z,Z_FULL_FLUSH);
if ( err == Z_STREAM_END ) {
return bytesread;
Expand All @@ -644,8 +644,8 @@ InflateData(mat_t *mat, z_streamp z, void *buf, unsigned int nBytes)
return bytesread;
}
while ( z->avail_out && !z->avail_in ) {
if ( nBytes > 1024 + bytesread ) {
z->avail_in = fread(comp_buf,1,1024,(FILE*)mat->fp);
if ( nBytes > READ_BLOCK_SIZE + bytesread ) {
z->avail_in = fread(comp_buf,1,READ_BLOCK_SIZE,(FILE*)mat->fp);
} else if ( nBytes < 1 + bytesread ) { /* Read a byte at a time */
z->avail_in = fread(comp_buf,1,1,(FILE*)mat->fp);
} else {
Expand Down
31 changes: 19 additions & 12 deletions src/matio_private.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,13 @@
# define ZLIB_BYTE_PTR(a) ((Bytef *)(a))
#endif

#if !defined(READ_BLOCK_SIZE)
#define READ_BLOCK_SIZE (8192)
#endif

#define _CAT(X, Y) X ## Y
#define CAT(X, Y) _CAT(X, Y)

/** @if mat_devman
* @brief Matlab MAT File information
*
Expand Down Expand Up @@ -133,30 +140,30 @@ EXTERN mat_int16_t Mat_int16Swap(mat_int16_t *a);
EXTERN mat_uint16_t Mat_uint16Swap(mat_uint16_t *a);

/* read_data.c */
EXTERN int ReadDoubleData(mat_t *mat,double *data,enum matio_types data_type,
int len);
EXTERN int ReadSingleData(mat_t *mat,float *data,enum matio_types data_type,
int len);
EXTERN int ReadDoubleData(mat_t *mat,double *data,enum matio_types data_type,
size_t len);
EXTERN int ReadSingleData(mat_t *mat,float *data,enum matio_types data_type,
size_t len);
#ifdef HAVE_MAT_INT64_T
EXTERN int ReadInt64Data (mat_t *mat,mat_int64_t *data,
enum matio_types data_type,int len);
enum matio_types data_type,size_t len);
#endif /* HAVE_MAT_INT64_T */
#ifdef HAVE_MAT_UINT64_T
EXTERN int ReadUInt64Data(mat_t *mat,mat_uint64_t *data,
enum matio_types data_type,int len);
enum matio_types data_type,size_t len);
#endif /* HAVE_MAT_UINT64_T */
EXTERN int ReadInt32Data (mat_t *mat,mat_int32_t *data,
enum matio_types data_type,int len);
enum matio_types data_type,size_t len);
EXTERN int ReadUInt32Data(mat_t *mat,mat_uint32_t *data,
enum matio_types data_type,int len);
enum matio_types data_type,size_t len);
EXTERN int ReadInt16Data (mat_t *mat,mat_int16_t *data,
enum matio_types data_type,int len);
enum matio_types data_type,size_t len);
EXTERN int ReadUInt16Data(mat_t *mat,mat_uint16_t *data,
enum matio_types data_type,int len);
enum matio_types data_type,size_t len);
EXTERN int ReadInt8Data (mat_t *mat,mat_int8_t *data,
enum matio_types data_type,int len);
enum matio_types data_type,size_t len);
EXTERN int ReadUInt8Data (mat_t *mat,mat_uint8_t *data,
enum matio_types data_type,int len);
enum matio_types data_type,size_t len);
EXTERN int ReadCharData (mat_t *mat,char *data,enum matio_types data_type,
int len);
EXTERN int ReadDataSlab1(mat_t *mat,void *data,enum matio_classes class_type,
Expand Down
Loading

0 comments on commit db29fe3

Please sign in to comment.