|
|
|
@ -1,28 +1,20 @@ |
|
|
|
import { |
|
|
|
AutomationLogPage, |
|
|
|
AutomationResults, |
|
|
|
AutomationStatus, |
|
|
|
} from "../../definitions/automation" |
|
|
|
import { getAppId, getProdAppDB } from "@budibase/backend-core/context" |
|
|
|
import { |
|
|
|
DocumentTypes, |
|
|
|
generateAutomationLogID, |
|
|
|
getAutomationLogParams, |
|
|
|
getQueryIndex, |
|
|
|
ViewNames, |
|
|
|
SEPARATOR, |
|
|
|
isProdAppID, |
|
|
|
} from "../../db/utils" |
|
|
|
import { createLogByAutomationView } from "../../db/views/staticViews" |
|
|
|
import { Automation, MetadataErrors } from "../../definitions/common" |
|
|
|
import { app } from "@budibase/backend-core/cache" |
|
|
|
import { backOff } from "../../utilities" |
|
|
|
import * as env from "../../environment" |
|
|
|
|
|
|
|
const PAGE_SIZE = 9 |
|
|
|
const EARLIEST_DATE = new Date(0).toISOString() |
|
|
|
const FREE_EXPIRY_SEC = 86400 |
|
|
|
const PRO_EXPIRY_SEC = FREE_EXPIRY_SEC * 30 |
|
|
|
import { logs } from "@budibase/pro" |
|
|
|
import { |
|
|
|
AutomationResults, |
|
|
|
AutomationStatus, |
|
|
|
} from "../../definitions/automation" |
|
|
|
const { logAlert } = require("@budibase/backend-core/logging") |
|
|
|
|
|
|
|
function getStatus(results: AutomationResults) { |
|
|
|
let status = AutomationStatus.SUCCESS |
|
|
|
@ -40,108 +32,29 @@ function getStatus(results: AutomationResults) { |
|
|
|
return status |
|
|
|
} |
|
|
|
|
|
|
|
export function oneMonthAgo() { |
|
|
|
return new Date(new Date().getTime() - PRO_EXPIRY_SEC * 1000).toISOString() |
|
|
|
} |
|
|
|
|
|
|
|
export function oneDayAgo() { |
|
|
|
return new Date(new Date().getTime() - FREE_EXPIRY_SEC * 1000).toISOString() |
|
|
|
} |
|
|
|
|
|
|
|
async function clearOldHistory() { |
|
|
|
const db = getProdAppDB() |
|
|
|
// TODO: handle license lookup for deletion
|
|
|
|
const expiredEnd = oneDayAgo() |
|
|
|
const results = await getAllLogs(EARLIEST_DATE, expiredEnd, { |
|
|
|
docs: false, |
|
|
|
paginate: false, |
|
|
|
}) |
|
|
|
const toDelete = results.data.map((doc: any) => ({ |
|
|
|
_id: doc.id, |
|
|
|
_rev: doc.value.rev, |
|
|
|
_deleted: true, |
|
|
|
})) |
|
|
|
const errorLogIds = results.data |
|
|
|
.filter((doc: any) => { |
|
|
|
const parts = doc.id.split(SEPARATOR) |
|
|
|
const status = parts[parts.length - 1] |
|
|
|
return status === AutomationStatus.ERROR |
|
|
|
}) |
|
|
|
.map((doc: any) => doc.id) |
|
|
|
await db.bulkDocs(toDelete) |
|
|
|
if (errorLogIds.length) { |
|
|
|
await updateAppMetadataWithErrors(errorLogIds, { clearing: true }) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
function pagination( |
|
|
|
response: any, |
|
|
|
paginate: boolean = true |
|
|
|
): AutomationLogPage { |
|
|
|
const data = response.rows.map((row: any) => { |
|
|
|
return row.doc ? row.doc : row |
|
|
|
}) |
|
|
|
if (!paginate) { |
|
|
|
return { data, hasNextPage: false } |
|
|
|
} |
|
|
|
const hasNextPage = data.length > PAGE_SIZE |
|
|
|
return { |
|
|
|
data: data.slice(0, PAGE_SIZE), |
|
|
|
hasNextPage, |
|
|
|
nextPage: hasNextPage ? data[PAGE_SIZE]?._id : undefined, |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
async function getAllLogs( |
|
|
|
startDate: string, |
|
|
|
endDate: string, |
|
|
|
opts: { |
|
|
|
docs: boolean |
|
|
|
status?: string |
|
|
|
paginate?: boolean |
|
|
|
page?: string |
|
|
|
} = { docs: true } |
|
|
|
): Promise<AutomationLogPage> { |
|
|
|
const db = getProdAppDB() |
|
|
|
let optional: any = { status: opts.status } |
|
|
|
const params = getAutomationLogParams(startDate, endDate, optional, { |
|
|
|
include_docs: opts.docs, |
|
|
|
limit: opts?.paginate ? PAGE_SIZE + 1 : undefined, |
|
|
|
}) |
|
|
|
if (opts?.page) { |
|
|
|
params.startkey = opts.page |
|
|
|
} |
|
|
|
let response = await db.allDocs(params) |
|
|
|
return pagination(response, opts?.paginate) |
|
|
|
} |
|
|
|
|
|
|
|
async function getLogsByView( |
|
|
|
startDate: string, |
|
|
|
endDate: string, |
|
|
|
viewParams: { automationId?: string; status?: string; page?: string } = {} |
|
|
|
): Promise<AutomationLogPage> { |
|
|
|
const db = getProdAppDB() |
|
|
|
let response |
|
|
|
try { |
|
|
|
let optional = { |
|
|
|
automationId: viewParams?.automationId, |
|
|
|
status: viewParams?.status, |
|
|
|
} |
|
|
|
const params = getAutomationLogParams(startDate, endDate, optional, { |
|
|
|
include_docs: true, |
|
|
|
limit: PAGE_SIZE, |
|
|
|
}) |
|
|
|
if (viewParams?.page) { |
|
|
|
params.startkey = viewParams.page |
|
|
|
} |
|
|
|
response = await db.query(getQueryIndex(ViewNames.AUTO_LOGS), params) |
|
|
|
} catch (err: any) { |
|
|
|
if (err != null && err.name === "not_found") { |
|
|
|
await createLogByAutomationView() |
|
|
|
return getLogsByView(startDate, endDate, viewParams) |
|
|
|
const expired = await logs.automations.getExpiredLogs() |
|
|
|
const toDelete = expired.data.map((doc: any) => ({ |
|
|
|
_id: doc.id, |
|
|
|
_rev: doc.value.rev, |
|
|
|
_deleted: true, |
|
|
|
})) |
|
|
|
const errorLogIds = expired.data |
|
|
|
.filter((doc: any) => { |
|
|
|
const parts = doc.id.split(SEPARATOR) |
|
|
|
const status = parts[parts.length - 1] |
|
|
|
return status === AutomationStatus.ERROR |
|
|
|
}) |
|
|
|
.map((doc: any) => doc.id) |
|
|
|
await db.bulkDocs(toDelete) |
|
|
|
if (errorLogIds.length) { |
|
|
|
await updateAppMetadataWithErrors(errorLogIds, { clearing: true }) |
|
|
|
} |
|
|
|
} catch (err) { |
|
|
|
logAlert(`Failed to cleanup automation log history - Database "${db.name}"`) |
|
|
|
} |
|
|
|
return pagination(response) |
|
|
|
} |
|
|
|
|
|
|
|
async function updateAppMetadataWithErrors( |
|
|
|
@ -214,28 +127,3 @@ export async function storeLog( |
|
|
|
// clear up old logging for app
|
|
|
|
await clearOldHistory() |
|
|
|
} |
|
|
|
|
|
|
|
export async function getLogs( |
|
|
|
startDate: string, |
|
|
|
status?: string, |
|
|
|
automationId?: string, |
|
|
|
page?: string |
|
|
|
): Promise<AutomationLogPage> { |
|
|
|
let response: AutomationLogPage |
|
|
|
let endDate = new Date().toISOString() |
|
|
|
if (automationId || status) { |
|
|
|
response = await getLogsByView(startDate, endDate, { |
|
|
|
automationId, |
|
|
|
status, |
|
|
|
page, |
|
|
|
}) |
|
|
|
} else { |
|
|
|
response = await getAllLogs(startDate, endDate, { |
|
|
|
status, |
|
|
|
page, |
|
|
|
docs: true, |
|
|
|
paginate: true, |
|
|
|
}) |
|
|
|
} |
|
|
|
return response |
|
|
|
} |
|
|
|
|
mike12345567
4 years ago