来自 Power Query 中 API 的分页结果

Question

一位友好的开发人员帮助我为他为我构建的系统创建了一个 API，这样我就可以将数据导出到 Power BI 中。

我已成功导入它，但遇到了一个问题，即较大的数据集会导致 JsonResponse 应用步骤中的一行以某些代码结尾：?top=200&skip=200。

我了解到这与分页有关，并且有效检索数据很重要。但我的表都限制为 200 行。

如何获取其余数据？

这是我当前使用的高级编辑器中的代码（这是在 ChatGPT 的帮助下生成的，我对复制和粘贴按钮非常熟练，但不太知道我在看什么） :

    let
url = "https://finder.bloodsandbeyond.co.uk/myurl",
headers = [
#"x-api-token-name" = "x-api-token", // Add this line if it is required
#"x-api-token" = "????????????????????????????" // Replace with your actual API key],
source = Web.Contents(url, [Headers=headers]),
jsonResponse = Json.Document(source),
value = jsonResponse[value],
#"Converted to Table" = Table.FromList(value, Splitter.SplitByNothing(), null, null, ExtraValues.Error),
#"Expanded Column1" = Table.ExpandRecordColumn(#"Converted to Table", "Column1", {"id", "lab_id", "assigned_user_id", "lab_invoice_id", "user_invoice_id", "appointment_at", "matched_at", "sample_posted_at", "is_under_18", "is_urgent", "patient_fee", "patient_cost", "collections_fee", "collections_cost", "reference", "status", "status_reason", "name", "address_line_1", "address_line_2", "city", "post_code", "lat", "lng", "phone", "email", "patients", "preferred_datetimes", "notes", "admin_notes", "deleted_at", "created_at", "updated_at", "confirmed_at", "is_issue", "invoice_notes", "issue_type"}, {"id", "lab_id", "assigned_user_id", "lab_invoice_id", "user_invoice_id", "appointment_at", "matched_at", "sample_posted_at", "is_under_18", "is_urgent", "patient_fee", "patient_cost", "collections_fee", "collections_cost", "reference", "status", "status_reason", "name", "address_line_1", "address_line_2", "city", "post_code", "lat", "lng", "phone", "email", "patients", "preferred_datetimes", "notes", "admin_notes", "deleted_at", "created_at", "updated_at", "confirmed_at", "is_issue", "invoice_notes", "issue_type"})
in
#"Expanded Column1"

Chat GPT 还提出了一些其他想法，但充其量最终也只是再次产生 200 行：

let
// Initial URL
initialUrl = "https://finder.bloodsandbeyond.co.uk/myURL",

// Function to fetch each page of data
GetAppointments = (url as text) as table =>
let
    // Fetch the data from the API
    source = Json.Document(Web.Contents(url, [Headers = [#"x-api-token" = "?????????????????????????"]])),
    // Extract the value part of the response
    value = source[value],
    // Check if there's a next link for pagination
    nextLink = try source[#"@odata.nextLink"] otherwise null,
    // Convert the result to a table
    result = Table.FromList(value, Splitter.SplitByNothing(), null, null, ExtraValues.Error),
    // Expand the columns in the result
    expanded = Table.ExpandRecordColumn(result, "Column1", {"id", "lab_id", "assigned_user_id", "lab_invoice_id", "user_invoice_id", "appointment_at", "matched_at", "sample_posted_at", "is_under_18", "is_urgent", "patient_fee", "patient_cost", "collections_fee", "collections_cost", "reference", "status", "status_reason", "name", "address_line_1", "address_line_2", "city", "post_code", "lat", "lng", "phone", "email", "patients", "preferred_datetimes", "notes", "admin_notes", "deleted_at", "created_at", "updated_at", "confirmed_at", "is_issue", "invoice_notes", "issue_type"}),
    // If there's more data, recursively fetch the next page
    finalResult = if nextLink <> null then
        Table.Combine({expanded, @GetAppointments(nextLink)})
    else
        expanded
in
    finalResult,

// Call the function to get all the data automatically
allData = GetAppointments(initialUrl)

在 所有数据

任何帮助将不胜感激。

Answer 1

我最终到达了那里，经过一些调试和错误处理后，聊天 gpt 最终确实对我有用。

对于任何有同样想法的人，希望这段代码可以有所帮助。

let
// Define the Table.GenerateByPage function for pagination
Table.GenerateByPage = (getNextPage as function) as table =>
let        
    listOfPages = List.Generate(
        () => getNextPage(0),            // Start with the first page (skip=0)
        (lastPage) => lastPage <> null,  // Stop when null is returned (no more data)
        (lastPage) => getNextPage(Table.RowCount(lastPage)) // Pass the row count as the next skip value
    ),
    // Concatenate the pages together
    tableOfPages = Table.FromList(listOfPages, Splitter.SplitByNothing(), {"Column1"}),
    firstRow = tableOfPages{0}?
in
    // If no pages are returned, return an empty table
    if (firstRow = null) then
        Table.FromRows({})
    else if (Table.IsEmpty(firstRow[Column1])) then
        firstRow[Column1]
    else
        Value.ReplaceType(
            Table.ExpandTableColumn(tableOfPages, "Column1", Table.ColumnNames(firstRow[Column1])),
            Value.Type(firstRow[Column1])
        ),

// Define the getNextPage function with enhanced error handling
getNextPage = (skipCount as number) =>
let
    // Construct the API URL with pagination logic
    url = "https://finder.bloodsandbeyond.co.uk/myURL?$skip=" & Number.ToText(skipCount),
    headers = [
        #"x-api-token" = "?????????????????????????"
    ],
    source = try Web.Contents(url, [Headers=headers]) otherwise null,
    // Convert the response to text and parse it as JSON
    rawResponse = if source = null then error "No data from API!" else Text.FromBinary(source),
    jsonResponse = try Json.Document(rawResponse) otherwise null,
    // Check if the "value" field exists and is valid
    value = if jsonResponse = null or not Record.HasFields(jsonResponse, "value") then null else jsonResponse[value],
    // If value is null or empty, stop fetching further pages
    nextPage = if value = null or List.IsEmpty(value) then null else value
in
    // Return the result as a table or null
    if nextPage = null then null else Table.FromList(nextPage, Splitter.SplitByNothing(), null, null, ExtraValues.Error),

// Use Table.GenerateByPage to get all data
allData = Table.GenerateByPage(getNextPage)

在 所有数据