当内存出现问题时如何在numpy / python中处理大数组

问题描述 投票:1回答:1

我有一个尺寸为(19494500,376)的数组all,我需要按照我生成的特定顺序排列这376列,

l

array([  0,  94, 188, 282,   1,  95, 189, 283,   2,  96, 190, 284,   3,
        97, 191, 285,   4,  98, 192, 286,   5,  99, 193, 287,   6, 100,
       194, 288,   7, 101, 195, 289,   8, 102, 196, 290,   9, 103, 197,
       291,  10, 104, 198, 292,  11, 105, 199, 293,  12, 106, 200, 294,
        13, 107, 201, 295,  14, 108, 202, 296,  15, 109, 203, 297,  16,
       110, 204, 298,  17, 111, 205, 299,  18, 112, 206, 300,  19, 113,
       207, 301,  20, 114, 208, 302,  21, 115, 209, 303,  22, 116, 210,
       304,  23, 117, 211, 305,  24, 118, 212, 306,  25, 119, 213, 307,
        26, 120, 214, 308,  27, 121, 215, 309,  28, 122, 216, 310,  29,
       123, 217, 311,  30, 124, 218, 312,  31, 125, 219, 313,  32, 126,
       220, 314,  33, 127, 221, 315,  34, 128, 222, 316,  35, 129, 223,
       317,  36, 130, 224, 318,  37, 131, 225, 319,  38, 132, 226, 320,
        39, 133, 227, 321,  40, 134, 228, 322,  41, 135, 229, 323,  42,
       136, 230, 324,  43, 137, 231, 325,  44, 138, 232, 326,  45, 139,
       233, 327,  46, 140, 234, 328,  47, 141, 235, 329,  48, 142, 236,
       330,  49, 143, 237, 331,  50, 144, 238, 332,  51, 145, 239, 333,
        52, 146, 240, 334,  53, 147, 241, 335,  54, 148, 242, 336,  55,
       149, 243, 337,  56, 150, 244, 338,  57, 151, 245, 339,  58, 152,
       246, 340,  59, 153, 247, 341,  60, 154, 248, 342,  61, 155, 249,
       343,  62, 156, 250, 344,  63, 157, 251, 345,  64, 158, 252, 346,
        65, 159, 253, 347,  66, 160, 254, 348,  67, 161, 255, 349,  68,
       162, 256, 350,  69, 163, 257, 351,  70, 164, 258, 352,  71, 165,
       259, 353,  72, 166, 260, 354,  73, 167, 261, 355,  74, 168, 262,
       356,  75, 169, 263, 357,  76, 170, 264, 358,  77, 171, 265, 359,
        78, 172, 266, 360,  79, 173, 267, 361,  80, 174, 268, 362,  81,
       175, 269, 363,  82, 176, 270, 364,  83, 177, 271, 365,  84, 178,
       272, 366,  85, 179, 273, 367,  86, 180, 274, 368,  87, 181, 275,
       369,  88, 182, 276, 370,  89, 183, 277, 371,  90, 184, 278, 372,
        91, 185, 279, 373,  92, 186, 280, 374,  93, 187, 281, 375]) 

所以我正在关注

all_c = all[:,l]

但是我得到了

“内存错误”

您能建议什么是最节省内存的方法吗?

python numpy out-of-memory
1个回答
0
投票

而不是立即置换整个数组,而是可以逐行就地进行处理。尝试

for r in range(all.shape[0]):
    all[r] = all[r, l]
© www.soinside.com 2019 - 2024. All rights reserved.