我需要打开一个巨大的内存映射。该文件为 1 TB。
然而,我得到了一个 errno: ENOMEM 12 Cannot allocate memory
。我不明白是什么阻碍了我。请求RLIMIT_AS
值中的结果:18446744073709551615。这就足够了。我的系统也是64位的所以并不是我的虚拟内存太小。ulimit -v
是ulimited
我使用 python 创建了数据,np.lib.format.open_memmap
因此它在物理上是可能的。我正在尝试用 C 语言阅读它。Python 阅读没问题,numpy.load('terabytearray.npy', mmap_mode='r')
有效。
这是一个最小的例子。
像这样创建一个 numpy 数组:
import numpy as np
shape = (75000, 5000000)
filename = 'datafile.obj'
if __name__ == '__main__':
arr = np.lib.format.open_memmap(filename, mode='w+', dtype=np.float32, shape=shape)
这样读:
#include <stdbool.h>
#include <assert.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdint.h>
#include <unistd.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <stdio.h>
#include <errno.h>
typedef enum {
CNPY_LE, /* little endian (least significant byte to most significant byte) */
CNPY_BE, /* big endian (most significant byte to least significant byte) */
CNPY_NE, /* no / neutral endianness (each element is a single byte) */
/* Host endianness is not supported because it is an incredibly bad idea to
use it for storage. */
} cnpy_byte_order;
typedef enum {
CNPY_B = 0, /* We want to use the values as index to the following arrays. */
CNPY_I1,
CNPY_I2,
CNPY_I4,
CNPY_I8,
CNPY_U1,
CNPY_U2,
CNPY_U4,
CNPY_U8,
CNPY_F4,
CNPY_F8,
CNPY_C8,
CNPY_C16,
} cnpy_dtype;
typedef enum {
CNPY_C_ORDER, /* C order (row major) */
CNPY_FORTRAN_ORDER, /* Fortran order (column major) */
} cnpy_flat_order;
typedef enum {
CNPY_SUCCESS, /* success */
CNPY_ERROR_FILE, /* some error regarding handling of a file */
CNPY_ERROR_MMAP, /* some error regarding mmaping a file */
CNPY_ERROR_FORMAT, /* file format error while reading some file */
} cnpy_status;
#define CNPY_MAX_DIM 4
typedef struct {
cnpy_byte_order byte_order;
cnpy_dtype dtype;
cnpy_flat_order order;
size_t n_dim;
size_t dims[CNPY_MAX_DIM];
char *raw_data;
size_t data_begin;
size_t raw_data_size;
} cnpy_array;
cnpy_status cnpy_open(const char * const fn, bool writable, cnpy_array *arr) {
assert(arr != NULL);
cnpy_array tmp_arr;
/* open, mmap, and close the file */
int fd = open(fn, writable? O_RDWR : O_RDONLY);
if (fd == -1) {
return CNPY_ERROR_FILE;
}
size_t raw_data_size = (size_t) lseek(fd, 0, SEEK_END);
lseek(fd, 0, SEEK_SET);
printf("%lu\n", raw_data_size);
if (raw_data_size == 0) {
close(fd); /* no point in checking for errors */
return CNPY_ERROR_FORMAT;
}
if (raw_data_size == SIZE_MAX) {
/* This is just because the author is too lazy to check for overflow on every pos+1 calculation. */
close(fd);
return CNPY_ERROR_FORMAT;
}
void *raw_data = mmap(
NULL,
raw_data_size,
PROT_READ | PROT_WRITE,
writable? MAP_SHARED : MAP_PRIVATE,
fd,
0
);
if (raw_data == MAP_FAILED) {
close(fd);
return CNPY_ERROR_MMAP;
}
if (close(fd) != 0) {
munmap(raw_data, raw_data_size);
return CNPY_ERROR_FILE;
}
/* parse the file */
// cnpy_status status = cnpy_parse(raw_data, raw_data_size, &tmp_arr); // library call ignore
// if (status != CNPY_SUCCESS) {
// munmap(raw_data, raw_data_size);
// return status;
// }
// *arr = tmp_arr;
return CNPY_SUCCESS;
}
int main(){
cnpy_array arr = {};
cnpy_status status = cnpy_open("datafile.obj", false, &arr);
printf("status %i\n",(int) status);
if(status != CNPY_SUCCESS){
printf("failure\n");
printf("errno %i\n", errno);
}
struct rlimit lim;
printf("getrlimit RLIMIT_AS %s\n", (getrlimit(RLIMIT_AS, &lim) == 0 ? "success" : "failure") );
printf("lim.rlim_cur %lu\n", lim.rlim_cur );
printf("lim.rlim_max %lu\n", lim.rlim_max );
printf("RLIM_INFINITY; %lu\n", RLIM_INFINITY );
return 0;
}
编译
gcc -std=c11 -o mmap_testing main.c
我正在使用~quf/cnpy库,我包含了相关部分,以使其与 numpy 的东西一起工作。