Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

testing proof of concept #1116

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 101 additions & 0 deletions assets/data/auctions.csv

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions src/client/views/cube-view/cube-view.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ export class CubeView extends React.Component<CubeViewProps, CubeViewState> {
}

refreshMaxTime = () => {
alert('refreshMaxTime')
const { essence, timekeeper } = this.state;
const { dataCube: { name, executor, timeAttribute, refreshRule } } = essence;
this.setState({ updatingMaxTime: true });
Expand Down
1 change: 1 addition & 0 deletions src/common/models/data-cube/queryable-data-cube.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ export interface QueryableDataCube extends DataCube {
}

export function addAttributes(dataCube: DataCube, newAttributes: Attributes): DataCube {
console.log("addAttributes", dataCube, newAttributes);
const { attributes, introspection } = dataCube;
let { dimensions, measures } = dataCube;
if (introspection === "none") return dataCube;
Expand Down
7 changes: 7 additions & 0 deletions src/server/routes/query/routes/visualization.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import makeQuery from "../../../../common/utils/query/visualization-query";
import { executeQuery } from "../../../utils/query/execute-query";
import { logQueryInfo } from "../../../utils/query/log-query";
import { QueryRouterRequest } from "../query";
import { data } from "cypress/types/jquery";

function getQuery(essence: Essence, timekeeper: Timekeeper): Expression {
return essence.visualization.name === "grid" ? makeGridQuery(essence, timekeeper) : makeQuery(essence, timekeeper);
Expand All @@ -31,8 +32,14 @@ function getQuery(essence: Essence, timekeeper: Timekeeper): Expression {
export default async function visualizationRoute({ turniloMetadata, context }: QueryRouterRequest, res: Response) {
const { dataCube, essence, decorator, timekeeper, logger } = context;
const query = getQuery(essence, timekeeper);
// console.log(query);
// console.log(JSON.stringify(essence));
// console.log(JSON.stringify(dataCube));
// console.log(decorator)
const queryTimeStart = Date.now();
const result = await executeQuery(dataCube, query, essence.timezone, decorator);
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think we can just stub this portion out with whatever BQ jobs we need to get the data, or implement a real data cube API if we wanted

logQueryInfo(essence, timekeeper, logger.setLoggerId("turnilo-visualization-query"), Date.now() - queryTimeStart, turniloMetadata.loggerContext);
console.log('result')
console.log(JSON.stringify(result));
res.json({ result });
}
286 changes: 286 additions & 0 deletions src/server/utils/query/bigquery.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,286 @@
import { parse } from "path";
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this file is mostly AI generated version that yield super broken SQL but demonstrates something hat may look kind of close to the end goal

import { cons } from "../../../common/utils/functional/functional";

interface Expression {
op: string;
operand?: Expression;
expression?: Expression;
name?: string;
value?: any;
type?: string;
setType?: string;
elements?: any[];
attributes?: any[];
data?: any[];
direction?: string;
dataName?: string;
}

export function translateExpressionToBigQuery(expr: Expression): string {
if (!expr || !expr.op) {
throw new Error("Invalid expression");
}

const fragments = {
select: [] as string[],
from: "`your_table_name`", // Replace with actual table name
where: [] as string[],
groupBy: [] as string[],
orderBy: [] as string[],
limit: ""
};

(function parseExpression(e: Expression) {
switch (e.op) {
case "apply":
parseExpression(e.expression!);
break;
case "filter":
fragments.where.push(handleFilter(e));
break;
case "overlap":
fragments.where.push(handleOverlap(e));
break;
case "literal":
fragments.select.push(handleLiteral(e));
break;
case "ref":
fragments.select.push(`${handleRef(e)}`);
if (e.name) fragments.groupBy.push(e.name);
break;
case "count":
fragments.select.push(`COUNT(${handleRef(e.operand!)}) AS ${e.name || 'count'}`);
break;
case "sort":
parseExpression(e.operand!);
fragments.orderBy.push(`${handleRef(e.expression!)} ${(e.direction || "").toUpperCase()}`);
break;
case "limit":
fragments.limit = `LIMIT ${e.value}`;
break;
case "split":
fragments.select.push(`${handleRef(e.expression!)} AS ${e.name}`);
fragments.groupBy.push(e.name || "");
parseExpression(e.operand!); // in case there are nested operations
break;
default:
throw new Error(`Unsupported operation: ${e.op}`);
}

if (e.operand) parseExpression(e.operand);
if (e.expression) parseExpression(e.expression);
})(expr);

const query = [
"SELECT",
fragments.select.join(", "),
`FROM ${fragments.from}`,
fragments.where.length ? `WHERE ${fragments.where.join(" AND ")}` : "",
fragments.groupBy.length ? `GROUP BY ${fragments.groupBy.join(", ")}` : "",
fragments.orderBy.length ? `ORDER BY ${fragments.orderBy.join(", ")}` : "",
fragments.limit
].filter(Boolean).join(" ");

return query;
}

function handleLiteral(expr: Expression): string {
if (expr.type === "SET" && expr.setType === "TIME_RANGE") {
const [timeRange] = expr.value.elements;
return `TIMESTAMP('${timeRange.start}') AND TIMESTAMP('${timeRange.end}')`;
} else {
return expr.value.toString();
}
}

function handleRef(expr: Expression): string {
return `\`${expr.name}\``;
}

function handleFilter(expr: Expression): string {
const condition = translateExpressionToBigQuery(expr.expression!);
return condition;
}

function handleOverlap(expr: Expression): string {
const operand = handleRef(expr.operand!);
const overlapExpression = handleLiteral(expr.expression!);
return `${operand} BETWEEN ${overlapExpression}`;
}

// Example usage with the provided JSON structure
const expression: Expression = {
"op": "apply",
"operand": {
"op": "apply",
"operand": {
"op": "apply",
"operand": {
"op": "apply",
"operand": {
"op": "literal",
"value": {
"attributes": [],
"data": [
{}
]
},
"type": "DATASET"
},
"expression": {
"op": "filter",
"operand": {
"op": "ref",
"name": "main"
},
"expression": {
"op": "overlap",
"operand": {
"op": "ref",
"name": "__time"
},
"expression": {
"op": "literal",
"value": {
"setType": "TIME_RANGE",
"elements": [
{
"start": "2015-12-01T00:01:00.000Z",
"end": "2016-01-01T00:01:00.000Z"
}
]
},
"type": "SET"
}
}
},
"name": "main"
},
"expression": {
"op": "literal",
"value": 2592000000
},
"name": "MillisecondsInInterval"
},
"expression": {
"op": "count",
"operand": {
"op": "ref",
"name": "main"
}
},
"name": "count"
},
"expression": {
"op": "limit",
"operand": {
"op": "sort",
"operand": {
"op": "apply",
"operand": {
"op": "split",
"operand": {
"op": "ref",
"name": "main"
},
"name": "sex",
"expression": {
"op": "ref",
"name": "sex"
},
"dataName": "main"
},
"expression": {
"op": "count",
"operand": {
"op": "ref",
"name": "main"
}
},
"name": "count"
},
"expression": {
"op": "ref",
"name": "count"
},
"direction": "descending"
},
"value": 100
},
"name": "SPLIT"
};

let n = 0;
let limit = -1;
let start;
let end;
let timeBucketSize = 0;
let groupBys: string[] = [];
const parseExpression = (exp: Expression) => {
if (!exp) {
return
}
n++;

parseExpression(exp.operand)
parseExpression(exp.expression)

switch (exp.op) {
case "apply":
switch (exp.name) {
case "MillisecondsInInterval":
timeBucketSize = exp.expression.value
break;
default:
break
// throw new Error(`Unsupported operation: ${exp.name}`);
}
break;
case "filter":
// TODO
break;
case "overlap":
switch (exp.expression.value.setType) {
case "TIME_RANGE":
start = exp.expression.value.elements[0].start
end = exp.expression.value.elements[0].end
break;
default:
throw new Error(`Unsupported operation: ${exp.expression.value.setType}`);
break;
}
// console.log(JSON.stringify(exp))
break;
case "literal":
break;
case "ref":
break;
case "split":
// console.log(exp)
groupBys.push(exp.name)
break;
case "count":
break;
case "sort":
break;
case "limit":
// console.log(exp)
limit = exp.value
break;
default:
throw new Error(`Unsupported operation: ${exp.op}`);
}
}

// parseExpression(expression)
// console.log(`groupBys: ${groupBys}`)
// console.log(`timeBucketSize: ${timeBucketSize}`)
// console.log(`start: ${start}`)
// console.log(`end: ${end}`)
// console.log(`limit: ${limit}`)

// console.log(`select ${groupBys.map(g => `count(${g})`).join(', ')} from TABLE_NAME group by ${groupBys.join(', ')} order by ${groupBys.join(', ')} desc limit ${limit}`)

// console.log(n)
// const sqlQuery = translateExpressionToBigQuery(expression);
// console.log(sqlQuery);
4 changes: 4 additions & 0 deletions src/server/utils/query/execute-query.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,14 @@ import { Timezone } from "chronoshift";
import { Dataset, DatasetJS, Expression } from "plywood";
import { QueryableDataCube } from "../../../common/models/data-cube/queryable-data-cube";
import { AppliedQueryDecorator } from "../query-decorator-loader/get-query-decorator";
import {translateExpressionToBigQuery} from "./bigquery";

export async function executeQuery(dataCube: QueryableDataCube, query: Expression, timezone: Timezone | null, decorator: AppliedQueryDecorator): Promise<DatasetJS> {
const maxQueries = dataCube.maxQueries;
const expression = decorator(query);
console.log('query')
console.log(JSON.stringify(expression));
// console.log(translateExpressionToBigQuery(JSON.parse(JSON.stringify(expression))));
const data = await dataCube.executor(expression, { maxQueries, timezone });
return Dataset.isDataset(data) ? data.toJS() : data as DatasetJS;
}