Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Evals] Update dataset schema types to match ActionSchema #1201

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+37 −34
Open

[Evals] Update dataset schema types to match ActionSchema #1201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions genkit-tools/common/src/eval/localFileDatasetStore.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ export class LocalFileDatasetStore implements DatasetStore {
req: CreateDatasetRequest
): Promise<DatasetMetadata> {
const { data, datasetId, schema, targetAction } = req;

console.log("Schema set to", schema);
const id = await this.generateDatasetId(datasetId);
const filePath = path.resolve(this.storeRoot, this.generateFileName(id));

Expand All @@ -96,6 +98,7 @@ export class LocalFileDatasetStore implements DatasetStore {
updateTime: now,
};

console.log("Dataset metadata set to", schema);
let metadataMap = await this.getMetadataMap();
metadataMap[id] = metadata;

Expand Down
15 changes: 5 additions & 10 deletions genkit-tools/common/src/types/eval.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/

import { z } from 'zod';
import { CustomAnySchema, JSONSchema7Schema } from './action';
import {
CreateDatasetRequest,
ListEvalKeysRequest,
Expand Down Expand Up @@ -160,16 +161,10 @@ export interface EvalStore {
list(query?: ListEvalKeysRequest): Promise<ListEvalKeysResponse>;
}

export const DatasetSchemaSchema = z.object({
inputSchema: z
.record(z.any())
.describe('Valid JSON Schema for the `input` field of dataset entry.')
.optional(),
referenceSchema: z
.record(z.any())
.describe('Valid JSON Schema for the `reference` field of dataset entry.')
.optional(),
});
export const DatasetSchemaSchema = z
.record(z.string(), CustomAnySchema)
.describe('Schema objects related to the dataset.')
.nullish();

/**
* Metadata for Dataset objects containing version, create and update time, etc.
Expand Down
53 changes: 29 additions & 24 deletions genkit-tools/common/tests/eval/localFileDatasetStore_test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,17 +89,30 @@ const CREATE_DATASET_REQUEST = CreateDatasetRequestSchema.parse({
data: { samples: SAMPLE_DATASET_1_V1 },
});

const INPUT_SCHEMA = {
"type": "string",
"$schema": "http://json-schema.org/draft-07/schema#"
};

const REFERENCE_SCHEMA = {
"type": "object",
"properties": {
"output": {
"type": "string"
},
},
"required": [
"output",
],
"additionalProperties": true,
"$schema": "http://json-schema.org/draft-07/schema#"
};

const CREATE_DATASET_REQUEST_WITH_SCHEMA = CreateDatasetRequestSchema.parse({
data: { samples: SAMPLE_DATASET_1_V1 },
schema: {
inputSchema: {
type: 'string',
$schema: 'http://json-schema.org/draft-07/schema#',
},
referenceSchema: {
type: 'number',
$schema: 'http://json-schema.org/draft-07/schema#',
},
"inputSchema": INPUT_SCHEMA,
"referenceSchema": REFERENCE_SCHEMA,
},
targetAction: '/flow/my-flow',
});
Expand Down Expand Up @@ -235,20 +248,16 @@ describe('localFileDatasetStore', () => {
...s,
}));

console.log("CREATE_DATASET_REQUEST_WITH_SCHEMA schema", CREATE_DATASET_REQUEST_WITH_SCHEMA.schema);

const datasetMetadata = await DatasetStore.createDataset({
...CREATE_DATASET_REQUEST_WITH_SCHEMA,
datasetId: SAMPLE_DATASET_ID_1,
});

expect(datasetMetadata.schema).toMatchObject({
inputSchema: {
type: 'string',
$schema: 'http://json-schema.org/draft-07/schema#',
},
referenceSchema: {
type: 'number',
$schema: 'http://json-schema.org/draft-07/schema#',
},
inputSchema: INPUT_SCHEMA,
referenceSchema: REFERENCE_SCHEMA,
});
expect(datasetMetadata.targetAction).toEqual('/flow/my-flow');
});
Expand Down Expand Up @@ -415,19 +424,15 @@ describe('localFileDatasetStore', () => {
const datasetMetadata = await DatasetStore.updateDataset({
datasetId: SAMPLE_DATASET_ID_1,
schema: {
inputSchema: {
type: 'string',
$schema: 'http://json-schema.org/draft-07/schema#',
},
inputSchema: INPUT_SCHEMA,
referenceSchema: REFERENCE_SCHEMA,
},
targetAction: '/flow/my-flow-2',
});

expect(datasetMetadata.schema).toMatchObject({
inputSchema: {
type: 'string',
$schema: 'http://json-schema.org/draft-07/schema#',
},
inputSchema: INPUT_SCHEMA,
referenceSchema: REFERENCE_SCHEMA,
});
expect(datasetMetadata.targetAction).toEqual('/flow/my-flow-2');
});
Expand Down
Loading