如何在不超过堆内存限制的情况下迭代和处理客户端应用程序中超过 35k 的 json 对象？

Question

我有 13 个大 JSON 文件块（每个 3k 个对象的 JSON 对象数组），它们本地存在于 nuxt3 应用程序的数据文件夹中。我需要处理所有 13 个 JSON 块并将结果存储在地图中。 最初，我在主线程中进行繁重的处理，之后，我决定使用 Webworkers 将繁重的计算过程隔离到一个单独的线程中。在本地运行应用程序时，它工作正常但是当我尝试构建它时，它抛出 javascript

FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory

workerVite.js

function fillMaps(resourceGroups, resourceGroupUnderService, skuCounts, allSkus) {
    for (const skuService of allSkus) {
      const serviceName = skuService.category.service_display_name;
      const resourceGroup = skuService.category.resource_group;
      const skuData = {
        name: skuService.description,
        serviceName: skuService.category.service_display_name,
        region: skuService.service_regions,
        usageType: skuService.category.usage_type,
        skuCode: skuService.sku_id,
      };
  
      if (resourceGroups.has(resourceGroup)) {
        resourceGroups.get(resourceGroup).add(skuData);
      } else resourceGroups.set(resourceGroup, new Set([skuData]));
  
      if (resourceGroupUnderService.has(serviceNa`enter code here`me)) {
        resourceGroupUnderService.get(serviceName).add(resourceGroup);
      } else resourceGroupUnderService.set(serviceName, new Set([resourceGroup]));
  
      if (skuCounts.has(serviceName)) {
        skuCounts.set(serviceName, skuCounts.get(serviceName) + 1);
      } else skuCounts.set(serviceName, 1);
    }
  
    return {resourceGroups, resourceGroupUnderService, skuCounts}
  }


onmessage = (event) => {
    const args = event.data.callback.args;
    const result = fillMaps.apply(this, [...args]);
    postMessage({resourceGroups: result.resourceGroups, resourceGroupUnderService: result.resourceGroupUnderService, skuCounts: result.skuCounts});
}

compute.js

import MyWorker from "../assets/workers/workerVite?worker";

const resourceGroups = new Map();
const resourceGroupUnderService = new Map();
const skuCounts = new Map();

const workerVite = (resourceGroupsMap, resourceGroupUnderServiceMap, skuCountsMap, skuList) =>
  new Promise((resolve, reject) => {
    const worker = new MyWorker();
    worker.postMessage({
      callback: {
        args: [resourceGroupsMap, resourceGroupUnderServiceMap, skuCountsMap, skuList],
      },
    });

    worker.onmessage = (responseMessage) => {
      const result = responseMessage.data;
      if (result && result.error) {
        reject(result.error);
      } else {
        resolve({ result: result, worker: worker });
      }
    };

    worker.onerror = (err) => {
      reject(err);
      worker.terminate();
    };
  });

async function computeGcpSkuData() {
  const chunkSize = 500;
  let it = 0;
  // Split allSkuServices into chunks of 100
  for (let i = 0; i < 4; i++) {
    let chunk = await import(`../data/gcp_sku_list/sku_chunk_${i}.json`);
    for (let j = 0; j < chunk.default.length; j += chunkSize) {
      const rgCopy = new Map(resourceGroups);
      const rgusCopy = new Map(resourceGroupUnderService);
      const skuCountCopy = new Map(skuCounts);
      try {
        const { result, worker } = await workerVite(
          rgCopy,
          rgusCopy,
          skuCountCopy,
          chunk.default.slice(j, j + chunkSize)
        );
        console.log(it++);

        result.resourceGroups.forEach((val, key) => {
          resourceGroups.set(key, val);
        });

        result.resourceGroupUnderService.forEach((val, key) => {
          resourceGroupUnderService.set(key, val);
        });

        result.skuCounts.forEach((val, key) => {
          skuCounts.set(key, val);
        });

        worker.terminate();
      } catch (error) {
        console.error(error);
        continue;
      } finally {
        rgCopy.clear();
        rgusCopy.clear();
        skuCountCopy.clear();
        console.log("deletes map & terminates worker");
      }
    }
    chunk = null;
  }

  return { resourceGroups, resourceGroupUnderService, skuCounts };
}

我也尝试用

NODE_OPTIONS=--max_old_space_size=8192 nuxi build

构建 nuxt 应用程序它也没有帮助。

观察

我可以注意到，创建多个具有 Set 值的 Maps 会消耗大量堆内存，但对于每次迭代，我都会清除 Map。这是否允许垃圾收集器释放内存？

另外，我可以看到堆内存使用量逐渐增加并达到最大限制。我该如何处理这种情况？有人可以吗