我正在使用aws-sdk-go-v2运行lambda，但下载较大文件时遇到内存问题。我尝试了partSize和并发性的各种组合，但是我将并发性设置为较小值时遇到了超时问题，或者遇到了内存不足的问题。
有人知道如何解决这个问题，或者使用go从S3下载文件的更好或其他方式？
package main
import (
"context"
"fmt"
awsevents "github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/external"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/s3manager"
)
func handleDemoEvent(_ context.Context, s3Event awsevents.S3Event) error {
for _, record := range s3Event.Records {
return handleDemoRecord(record)
}
return nil
}
func handleDemoRecord(record awsevents.S3EventRecord) error {
fmt.Println(record.ResponseElements)
fmt.Printf("[%s - %s] Bucket = %s, Key = %s \n", record.EventSource, record.EventTime, record.S3.Bucket.Name, record.S3.Object.Key)
bucket := record.S3.Bucket.Name
item := record.S3.Object.Key
config, err := external.LoadDefaultAWSConfig()
if err != nil {
return err
}
headObject := getHeadObject(bucket, item, config)
fmt.Printf("%+v", headObject)
downloader := s3manager.NewDownloader(config, func(d *s3manager.Downloader) {
d.PartSize = 2 * 1024 * 1024
d.Concurrency = 70
})
var buf []byte
buffer := aws.NewWriteAtBuffer(buf)
fmt.Println("Starting download of file")
numBytesDownloaded, err := downloader.Download(buffer, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(item),
})
if err != nil {
return err
}
fmt.Println("Downloaded ", numBytesDownloaded, " bytes")
return nil
}
func getHeadObject(bucket string, key string, config aws.Config) *s3.HeadObjectResponse {
client := s3.New(config)
input := &s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}
request := client.HeadObjectRequest(input)
headObjectResponse, err := request.Send(context.TODO())
if err != nil {
panic(err)
}
fmt.Printf("Downloaded HeadObject: %v\n", request)
return headObjectResponse
}
func main() {
lambda.Start(handleDemoEvent)
}
这是日志流中的错误，因为它很长，所以没有包含所有内容。
START RequestId: 8f5bc332-b4ad-4749-97ac-ee7c65a4404a Version: $LATEST
map[x-amz-request-id:AA3E4F41C6FCA9B9 x-amz-id-2:HEoJcflBdxLbSLkOOUD19kI7TboN8hDfkBf8vqQlFy1FQni2wUUi09cRwliZeDn6nFiNx73V2v9V5jreSlgOr1bpPDl9Lywe]
[aws:s3 - 2020-01-24 08:25:11.271 +0000 UTC] Bucket = 5kstats-demos, Key = to-be-parsed/vitality-vs-mousesports-m1-inferno.dem
Downloaded HeadObject: {0xc000105b80 {
Bucket: "5kstats-demos",
Key: "to-be-parsed/vitality-vs-mousesports-m1-inferno.dem",
RequestPayer:
} 0x908e80}
{
AcceptRanges: "bytes",
ContentLength: 491103546,
ContentType: "application/octet-stream",
ETag: "\"f3870e5b519ea52dbd249f69f6072f45-30\"",
LastModified: 2020-01-24 08:25:12 +0000 UTC,
Metadata: {
Event-Id: "4979",
Event-Name: "EPICENTER 2019",
Event-End-Sec: "1577012400000",
Match-Id: "2338373",
Event-Start-Sec: "1576580400000"
},
ObjectLockLegalHoldStatus: ,
ObjectLockMode: ,
ReplicationStatus: ,
RequestCharged: ,
ServerSideEncryption: ,
StorageClass:
}Starting download of file
fatal error: runtime: out of memory
runtime stack:
runtime.throw(0xac163b, 0x16)
/usr/local/go/src/runtime/panic.go:608 +0x72
runtime.sysMap(0xc0b8000000, 0x1c000000, 0x1112d18)
/usr/local/go/src/runtime/mem_linux.go:156 +0xc7
runtime.(*mheap).sysAlloc(0x10f96c0, 0x1c000000, 0x80, 0x0)
/usr/local/go/src/runtime/malloc.go:619 +0x1c7
runtime.(*mheap).grow(0x10f96c0, 0xd202, 0x0)
/usr/local/go/src/runtime/mheap.go:920 +0x42
runtime.(*mheap).allocSpanLocked(0x10f96c0, 0xd202, 0x1112d28, 0x432c55)
/usr/local/go/src/runtime/mheap.go:848 +0x337
runtime.(*mheap).alloc_m(0x10f96c0, 0xd202, 0xffffffffffff0101, 0x7ffe2e1d0780)
/usr/local/go/src/runtime/mheap.go:692 +0x119
runtime.(*mheap).alloc.func1()
/usr/local/go/src/runtime/mheap.go:759 +0x4c
runtime.(*mheap).alloc(0x10f96c0, 0xd202, 0x7ffe2e010101, 0x7ffe2e1d0820)
/usr/local/go/src/runtime/mheap.go:758 +0x8a
runtime.largeAlloc(0x1a403fe8, 0x101, 0xc00ba5bbb0)
/usr/local/go/src/runtime/malloc.go:1019 +0x97
runtime.mallocgc.func1()
/usr/local/go/src/runtime/malloc.go:914 +0x46
runtime.systemstack(0x456509)
/usr/local/go/src/runtime/asm_amd64.s:351 +0x66
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1229
goroutine 10 [running]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_amd64.s:311 fp=0xc00ba5bb40 sp=0xc00ba5bb38 pc=0x456600
runtime.mallocgc(0x1a403fe8, 0x9c9780, 0x41829684901, 0xc00ba5bc18)
/usr/local/go/src/runtime/malloc.go:913 +0x896 fp=0xc00ba5bbe0 sp=0xc00ba5bb40 pc=0x40d646
runtime.makeslice(0x9c9780, 0x1a403fe8, 0x1a403fe8, 0x8000, 0x3fe8, 0x0)
/usr/local/go/src/runtime/slice.go:70 +0x77 fp=0xc00ba5bc10 sp=0xc00ba5bbe0 pc=0x441877
5kparser/vendor/github.com/aws/aws-sdk-go-v2/aws.(*WriteAtBuffer).WriteAt(0xc0002c8210, 0xc001ade000, 0x3fe8, 0x8000, 0x1a400000, 0x0, 0x0, 0x0)
/home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/aws/types.go:190 +0xf9 fp=0xc00ba5bc78 sp=0xc00ba5bc10 pc=0x76bb69
5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*dlchunk).Write(0xc000068740, 0xc001ade000, 0x3fe8, 0x8000, 0x3fe8, 0x0, 0x0)
/home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:590 +0x75 fp=0xc00ba5bcc8 sp=0xc00ba5bc78 pc=0x970c65
io.copyBuffer(0xc20f20, 0xc000068740, 0xc215e0, 0xc0007da880, 0xc001ade000, 0x8000, 0x8000, 0xa162e0, 0xc000293600, 0xc215e0)
/usr/local/go/src/io/io.go:404 +0x201 fp=0xc00ba5bd38 sp=0xc00ba5bcc8 pc=0x46f701
io.Copy(0xc20f20, 0xc000068740, 0xc215e0, 0xc0007da880, 0xc0007da880, 0xc000419970, 0x0)
/usr/local/go/src/io/io.go:364 +0x5a fp=0xc00ba5bd98 sp=0xc00ba5bd38 pc=0x46f4ba
5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*downloader).tryDownloadChunk(0xc00008c750, 0xc0005cd0e0, 0xc20f20, 0xc000068740, 0x0, 0x0, 0x0)
/home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:470 +0x1e0 fp=0xc00ba5be28 sp=0xc00ba5bd98 pc=0x970430
5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*downloader).downloadChunk(0xc00008c750, 0xc20280, 0xc0002c8210, 0x1a400000, 0x200000, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:429 +0x2a5 fp=0xc00ba5bf00 sp=0xc00ba5be28 pc=0x9700d5
5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*downloader).downloadPart(0xc00008c750, 0xc00006c2a0)
/home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:378 +0x142 fp=0xc00ba5bfd0 sp=0xc00ba5bf00 pc=0x96fb42
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00ba5bfd8 sp=0xc00ba5bfd0 pc=0x4586e1
created by 5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*downloader).download
/home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:322 +0x1ed
goroutine 1 [IO wait]:
internal/poll.runtime_pollWait(0x7f018740cf00, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:173 +0x66
internal/poll.(*pollDesc).wait(0xc000120218, 0x72, 0xc00009a000, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9a
internal/poll.(*pollDesc).waitRead(0xc000120218, 0xffffffffffffff00, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Accept(0xc000120200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1a0
net.(*netFD).accept(0xc000120200, 0x7f0187415200, 0x0, 0x0)
/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0000965a8, 0xc000171e30, 0xc000171e38, 0x18)
/usr/local/go/src/net/tcpsock_posix.go:139 +0x2e
net.(*TCPListener).Accept(0xc0000965a8, 0xada0f8, 0xc0000ba320, 0x7f0187415200, 0xc0000965f8)
/usr/local/go/src/net/tcpsock.go:260 +0x47
net/rpc.(*Server).Accept(0xc0000ba320, 0xc26c00, 0xc0000965a8)
/usr/local/go/src/net/rpc/server.go:632 +0x69
net/rpc.Accept(0xc26c00, 0xc0000965a8)
/usr/local/go/src/net/rpc/server.go:692 +0x41
5kparser/vendor/github.com/aws/aws-lambda-go/lambda.StartHandler(0xc21a20, 0xc00009bb80)
/home/softarn/go/src/5kparser/vendor/github.com/aws/aws-lambda-go/lambda/entry.go:60 +0x174
5kparser/vendor/github.com/aws/aws-lambda-go/lambda.Start(0x9e9140, 0xad9dd8)
/home/softarn/go/src/5kparser/vendor/github.com/aws/aws-lambda-go/lambda/entry.go:41 +0x4d
main.main()
/home/softarn/go/src/5kparser/aws-lambda-runner.go:209 +0x39
https://docs.aws.amazon.com/lambda/latest/dg/limits.html
您可以写至/tmp目录存储中的最大512mb可用距离
或增加可用于您的功能的内存
$ aws lambda update-function-configuration \
--function-name my-function --memory-size 512