diff --git a/.github/workflows/bump-js-plugins-version.yml b/.github/workflows/bump-js-plugins-version.yml index 386e4bfc3..5d7076ce9 100644 --- a/.github/workflows/bump-js-plugins-version.yml +++ b/.github/workflows/bump-js-plugins-version.yml @@ -171,3 +171,14 @@ jobs: preid: ${{ inputs.preid }} commit-message: 'chore: bump @genkit-ai/vertexai version to {{version}}' tag-prefix: '@genkit-ai/vertexai@' + - name: 'js/plugins/checks version bump' + uses: 'phips28/gh-action-bump-version@master' + env: + GITHUB_TOKEN: ${{ secrets.GENKIT_RELEASER_GITHUB_TOKEN }} + PACKAGEJSON_DIR: js/plugins/checks + with: + default: ${{ inputs.releaseType }} + version-type: ${{ inputs.releaseType }} + preid: ${{ inputs.preid }} + commit-message: 'chore: bump @genkit-ai/checks version to {{version}}' + tag-prefix: '@genkit-ai/checks@' diff --git a/js/plugins/checks/.npmignore b/js/plugins/checks/.npmignore new file mode 100644 index 000000000..b512c09d4 --- /dev/null +++ b/js/plugins/checks/.npmignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file diff --git a/js/plugins/checks/LICENSE b/js/plugins/checks/LICENSE new file mode 100644 index 000000000..26a870243 --- /dev/null +++ b/js/plugins/checks/LICENSE @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + \ No newline at end of file diff --git a/js/plugins/checks/README.md b/js/plugins/checks/README.md new file mode 100644 index 000000000..5cf804438 --- /dev/null +++ b/js/plugins/checks/README.md @@ -0,0 +1,103 @@ +# Checks + +Checks is an AI safety platform built by Google: [checks.google.com/ai-safety](https://checks.google.com/ai-safety). + +This plugin provides evaluators for each Checks AI safety policy. Text is cassified by calling the [Checks Guardrails API](https://console.cloud.google.com/marketplace/product/google/checks.googleapis.com). + +> Note: The Guardrails is currently in private preview and you will need to request quota. To request quota fill out this [Google form](https://docs.google.com/forms/d/e/1FAIpQLSdcLZkOJMiqodS8KSG1bg0-jAgtE9W-AludMbArCKqgz99OCA/viewform?usp=sf_link) + +Curently that list includes: + +```text +DANGEROUS_CONTENT +PII_SOLICITING_RECITING +HARASSMENT +SEXUALLY_EXPLICIT +HATE_SPEECH +MEDICAL_INFO +VIOLENCE_AND_GORE +OBSCENITY_AND_PROFANITY +``` + +## How to use + +### Configure the plugin + +Add the `checks` plugin to your Genkit entrypoint and configured the evaluators you want to use: + +```ts +import { checks, ChecksEvaluationMetricType } from '@genkit-ai/checks'; + +export const ai = genkit({ + plugins: [ + checks({ + // Project to charge quota to. + // Note: If your credentials have a quota project associated with them. + // That value will take precedence over this. + projectId: 'your-project-id', + evaluation: { + metrics: [ + // Policies configured with the default threshold(0.5). + ChecksEvaluationMetricType.DANGEROUS_CONTENT, + ChecksEvaluationMetricType.HARASSMENT, + ChecksEvaluationMetricType.HATE_SPEECH, + ChecksEvaluationMetricType.MEDICAL_INFO, + ChecksEvaluationMetricType.OBSCENITY_AND_PROFANITY, + // Policies configured with non-default threshold. + { + type: ChecksEvaluationMetricType.PII_SOLICITING_RECITING, + threshold: 0.6, + }, + { + type: ChecksEvaluationMetricType.SEXUALLY_EXPLICIT, + threshold: 0.3, + }, + { + type: ChecksEvaluationMetricType.VIOLENCE_AND_GORE, + threshold: 0.55, + }, + ], + }, + }), + ], +}); +``` + +### Create a test dataset + +Create a JSON file with the data you want to test. Add as many test cases as you want. `output` is the text that will be classified. + +```JSON +[ + { + "testCaseId": "test_case_id_1", + "input": "The input to your model.", + "output": "Example model output which. This is what will be evaluated." + } +] + +``` + +### Run the evaluators + +```bash +# Run just the DANGEROUS_CONTENT classifier. +genkit eval:run test-dataset.json --evaluators=checks/dangerous_content +``` + +```bash +# Run all classifiers. +genkit eval:run test-dataset.json --evaluators=checks/dangerous_content,checks/pii_soliciting_reciting,checks/harassment,checks/sexually_explicit,checks/hate_speech,checks/medical_info,checks/violence_and_gore,checks/obscenity_and_profanity +``` + +### View the results + +Run `genkit start -- tsx --watch src/index.ts` and open the genkit ui. Usually at `localhost:4000` and select the Evaluate tab. + +# Genkit + +The sources for this package are in the main [Genkit](https://github.com/firebase/genkit) repo. Please file issues and pull requests against that repo. + +Usage information and reference details can be found in [Genkit documentation](https://firebase.google.com/docs/genkit). + +License: Apache 2.0 diff --git a/js/plugins/checks/package.json b/js/plugins/checks/package.json new file mode 100644 index 000000000..018a34e19 --- /dev/null +++ b/js/plugins/checks/package.json @@ -0,0 +1,55 @@ +{ + "name": "@genkit-ai/checks", + "description": "Google Checks AI Safety plugins for classifying the safety of text against Checks AI safety policies.", + "keywords": [ + "genkit", + "genkit-plugin", + "google cloud", + "google ai", + "ai", + "genai", + "generative-ai", + "checks", + "google checks", + "guardrails" + ], + "version": "0.9.0-rc.3", + "type": "commonjs", + "scripts": { + "check": "tsc", + "compile": "tsup-node", + "build:clean": "rimraf ./lib", + "build": "npm-run-all build:clean check compile", + "build:watch": "tsup-node --watch" + }, + "repository": { + "type": "git", + "url": "https://github.com/firebase/genkit.git", + "directory": "js/plugins/checks" + }, + "author": "genkit", + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^9.6.3" + }, + "peerDependencies": { + "genkit": "workspace:*" + }, + "devDependencies": { + "@types/node": "^20.11.16", + "npm-run-all": "^4.1.5", + "rimraf": "^6.0.1", + "tsup": "^8.0.2", + "tsx": "^4.7.0", + "typescript": "^4.9.0" + }, + "types": "./lib/index.d.ts", + "exports": { + ".": { + "require": "./lib/index.js", + "import": "./lib/index.mjs", + "types": "./lib/index.d.ts", + "default": "./lib/index.js" + } + } +} \ No newline at end of file diff --git a/js/plugins/checks/src/evaluation.ts b/js/plugins/checks/src/evaluation.ts new file mode 100644 index 000000000..58ad2b115 --- /dev/null +++ b/js/plugins/checks/src/evaluation.ts @@ -0,0 +1,190 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EvaluatorAction, Genkit, z } from 'genkit'; +import { BaseEvalDataPoint } from 'genkit/evaluator'; +import { runInNewSpan } from 'genkit/tracing'; +import { GoogleAuth } from 'google-auth-library'; + +/** + * Currently supported Checks AI Safety policies. + */ +export enum ChecksEvaluationMetricType { + // The model facilitates, promotes or enables access to harmful goods, + // services, and activities. + DANGEROUS_CONTENT = 'DANGEROUS_CONTENT', + // The model reveals an individual’s personal information and data. + PII_SOLICITING_RECITING = 'PII_SOLICITING_RECITING', + // The model generates content that is malicious, intimidating, bullying, or + // abusive towards another individual. + HARASSMENT = 'HARASSMENT', + // The model generates content that is sexually explicit in nature. + SEXUALLY_EXPLICIT = 'SEXUALLY_EXPLICIT', + // The model promotes violence, hatred, discrimination on the basis of race, + // religion, etc. + HATE_SPEECH = 'HATE_SPEECH', + // The model facilitates harm by providing health advice or guidance. + MEDICAL_INFO = 'MEDICAL_INFO', + // The model generates content that contains gratuitous, realistic + // descriptions of violence or gore. + VIOLENCE_AND_GORE = 'VIOLENCE_AND_GORE', + // The model generates content that contains vulgar, profane, or offensive + // language. + OBSCENITY_AND_PROFANITY = 'OBSCENITY_AND_PROFANITY', +} + +/** + * Checks evaluation metric config. Use `threshold` to override the default violation threshold. + * The value of `metricSpec` will be included in the request to the API. See the API documentation + */ +export type ChecksEvaluationMetricConfig = { + type: ChecksEvaluationMetricType; + threshold?: number; +}; + +export type ChecksEvaluationMetric = + | ChecksEvaluationMetricType + | ChecksEvaluationMetricConfig; + +export function checksEvaluators( + ai: Genkit, + auth: GoogleAuth, + metrics: ChecksEvaluationMetric[], + projectId: string +): EvaluatorAction[] { + const policy_configs: ChecksEvaluationMetricConfig[] = metrics.map( + (metric) => { + const metricType = isConfig(metric) ? metric.type : metric; + const threshold = isConfig(metric) ? metric.threshold : undefined; + + return { + type: metricType, + threshold, + }; + } + ); + + const evaluators = policy_configs.map((policy_config) => { + return createPolicyEvaluator(projectId, auth, ai, policy_config); + }); + + return evaluators; +} + +function isConfig( + config: ChecksEvaluationMetric +): config is ChecksEvaluationMetricConfig { + return (config as ChecksEvaluationMetricConfig).type !== undefined; +} + +const ResponseSchema = z.object({ + policyResults: z.array( + z.object({ + policyType: z.string(), + score: z.number(), + violationResult: z.string(), + }) + ), +}); + +function createPolicyEvaluator( + projectId: string, + auth: GoogleAuth, + ai: Genkit, + policy_config: ChecksEvaluationMetricConfig +): EvaluatorAction { + const policyType = policy_config.type as string; + + return ai.defineEvaluator( + { + name: `checks/${policyType.toLowerCase()}`, + displayName: policyType, + definition: `Evaluates text against the Checks ${policyType} policy.`, + }, + async (datapoint: BaseEvalDataPoint) => { + const partialRequest = { + input: { + text_input: { + content: datapoint.output as string, + }, + }, + policies: { + policy_type: policy_config.type, + threshold: policy_config.threshold, + }, + }; + + const response = await checksEvalInstance( + projectId, + auth, + partialRequest, + ResponseSchema + ); + + return { + evaluation: { + score: response.policyResults[0].score, + details: { + reasoning: response.policyResults[0].violationResult, + }, + }, + testCaseId: datapoint.testCaseId, + }; + } + ); +} + +async function checksEvalInstance( + projectId: string, + auth: GoogleAuth, + partialRequest: any, + responseSchema: ResponseType +): Promise> { + return await runInNewSpan( + { + metadata: { + name: 'EvaluationService#evaluateInstances', + }, + }, + async (metadata, _otSpan) => { + const request = { + ...partialRequest, + }; + + metadata.input = request; + const client = await auth.getClient(); + const url = + 'https://checks.googleapis.com/v1alpha/aisafety:classifyContent'; + + const response = await client.request({ + url, + method: 'POST', + body: JSON.stringify(request), + headers: { + 'x-goog-user-project': projectId, + 'Content-Type': 'application/json', + }, + }); + metadata.output = response.data; + + try { + return responseSchema.parse(response.data); + } catch (e) { + throw new Error(`Error parsing ${url} API response: ${e}`); + } + } + ); +} diff --git a/js/plugins/checks/src/index.ts b/js/plugins/checks/src/index.ts new file mode 100644 index 000000000..103830a92 --- /dev/null +++ b/js/plugins/checks/src/index.ts @@ -0,0 +1,94 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Genkit } from 'genkit'; +import { logger } from 'genkit/logging'; +import { GenkitPlugin, genkitPlugin } from 'genkit/plugin'; +import { GoogleAuth, GoogleAuthOptions } from 'google-auth-library'; +import { + ChecksEvaluationMetric, + ChecksEvaluationMetricType, + checksEvaluators, +} from './evaluation.js'; +export { ChecksEvaluationMetricType as ChecksEvaluationMetricType }; + +export interface PluginOptions { + /** The Google Cloud project id to call. Must have quota for the Checks API. */ + projectId?: string; + /** Provide custom authentication configuration for connecting to Checks API. */ + googleAuth?: GoogleAuthOptions; + /** Configure Checks evaluators. */ + evaluation?: { + metrics: ChecksEvaluationMetric[]; + }; +} + +const CLOUD_PLATFROM_OAUTH_SCOPE = + 'https://www.googleapis.com/auth/cloud-platform'; + +const CHECKS_OAUTH_SCOPE = 'https://www.googleapis.com/auth/checks'; + +/** + * Add Google Checks evaluators. + */ +export function checks(options?: PluginOptions): GenkitPlugin { + return genkitPlugin('checks', async (ai: Genkit) => { + let authClient: GoogleAuth; + let authOptions = options?.googleAuth; + + // Allow customers to pass in cloud credentials from environment variables + // following: https://github.com/googleapis/google-auth-library-nodejs?tab=readme-ov-file#loading-credentials-from-environment-variables + if (process.env.GCLOUD_SERVICE_ACCOUNT_CREDS) { + const serviceAccountCreds = JSON.parse( + process.env.GCLOUD_SERVICE_ACCOUNT_CREDS + ); + authOptions = { + credentials: serviceAccountCreds, + scopes: [CLOUD_PLATFROM_OAUTH_SCOPE, CHECKS_OAUTH_SCOPE], + }; + authClient = new GoogleAuth(authOptions); + } else { + authClient = new GoogleAuth( + authOptions ?? { + scopes: [CLOUD_PLATFROM_OAUTH_SCOPE, CHECKS_OAUTH_SCOPE], + } + ); + } + + const client = await authClient.getClient(); + if (client.quotaProjectId) { + logger.warn( + `Checks Evaluator: Your Google cloud authentication has a default quota project(${client.quotaProjectId}) associated with it which will overrid the projectId in your Checks plugin config(${options?.projectId}).` + ); + } + + const projectId = options?.projectId || (await authClient.getProjectId()); + + if (!projectId) { + throw new Error( + `Checks Plugin is missing the 'projectId' configuration. Please set the 'GCLOUD_PROJECT' environment variable or explicitly pass 'projectId' into genkit config.` + ); + } + + const metrics = + options?.evaluation && options.evaluation.metrics.length > 0 + ? options.evaluation.metrics + : []; + checksEvaluators(ai, authClient, metrics, projectId); + }); +} + +export default checks; diff --git a/js/plugins/checks/tsconfig.json b/js/plugins/checks/tsconfig.json new file mode 100644 index 000000000..596e2cf72 --- /dev/null +++ b/js/plugins/checks/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src"] +} diff --git a/js/plugins/checks/tsup.config.ts b/js/plugins/checks/tsup.config.ts new file mode 100644 index 000000000..01dce0a6b --- /dev/null +++ b/js/plugins/checks/tsup.config.ts @@ -0,0 +1,22 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defineConfig, Options } from 'tsup'; +import { defaultOptions } from '../../tsup.common'; + +export default defineConfig({ + ...(defaultOptions as Options), +}); diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index 171e058a7..624899cc6 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -208,6 +208,34 @@ importers: specifier: ^4.9.0 version: 4.9.5 + plugins/checks: + dependencies: + genkit: + specifier: workspace:* + version: link:../../genkit + google-auth-library: + specifier: ^9.6.3 + version: 9.14.2(encoding@0.1.13) + devDependencies: + '@types/node': + specifier: ^20.11.16 + version: 20.16.9 + npm-run-all: + specifier: ^4.1.5 + version: 4.1.5 + rimraf: + specifier: ^6.0.1 + version: 6.0.1 + tsup: + specifier: ^8.0.2 + version: 8.3.5(postcss@8.4.47)(tsx@4.19.2)(typescript@4.9.5) + tsx: + specifier: ^4.7.0 + version: 4.19.2 + typescript: + specifier: ^4.9.0 + version: 4.9.5 + plugins/chroma: dependencies: chromadb: @@ -815,6 +843,31 @@ importers: specifier: ^5.3.3 version: 5.4.5 + testapps/checks-evaluation: + dependencies: + '@genkit-ai/checks': + specifier: workspace:* + version: link:../../plugins/checks + genkit: + specifier: workspace:* + version: link:../../genkit + path: + specifier: ^0.12.7 + version: 0.12.7 + devDependencies: + cross-env: + specifier: ^7.0.3 + version: 7.0.3 + rimraf: + specifier: ^6.0.1 + version: 6.0.1 + tsx: + specifier: ^4.7.0 + version: 4.19.2 + typescript: + specifier: ^5.3.3 + version: 5.6.2 + testapps/dev-ui-gallery: dependencies: '@genkit-ai/dev-local-vectorstore': @@ -7282,7 +7335,7 @@ snapshots: '@google-cloud/logging-winston@6.0.0(encoding@0.1.13)(winston@3.13.0)': dependencies: '@google-cloud/logging': 11.0.0(encoding@0.1.13) - google-auth-library: 9.11.0(encoding@0.1.13) + google-auth-library: 9.14.2(encoding@0.1.13) lodash.mapvalues: 4.6.0 winston: 3.13.0 winston-transport: 4.7.0 @@ -7301,7 +7354,7 @@ snapshots: eventid: 2.0.1 extend: 3.0.2 gcp-metadata: 6.1.0(encoding@0.1.13) - google-auth-library: 9.11.0(encoding@0.1.13) + google-auth-library: 9.14.2(encoding@0.1.13) google-gax: 4.3.2(encoding@0.1.13) on-finished: 2.4.1 pumpify: 2.0.1 @@ -7319,7 +7372,7 @@ snapshots: '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-metrics': 1.25.1(@opentelemetry/api@1.9.0) - google-auth-library: 9.11.0(encoding@0.1.13) + google-auth-library: 9.14.2(encoding@0.1.13) googleapis: 137.1.0(encoding@0.1.13) transitivePeerDependencies: - encoding @@ -7334,7 +7387,7 @@ snapshots: '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 1.25.1(@opentelemetry/api@1.9.0) - google-auth-library: 9.11.0(encoding@0.1.13) + google-auth-library: 9.14.2(encoding@0.1.13) transitivePeerDependencies: - encoding - supports-color @@ -7506,7 +7559,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@sinonjs/fake-timers': 10.3.0 - '@types/node': 20.11.30 + '@types/node': 20.16.9 jest-message-util: 29.7.0 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -7598,7 +7651,7 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 20.11.30 + '@types/node': 20.16.9 '@types/yargs': 17.0.33 chalk: 4.1.2 @@ -10037,7 +10090,7 @@ snapshots: '@types/long': 4.0.2 abort-controller: 3.0.0 duplexify: 4.1.3 - google-auth-library: 9.11.0(encoding@0.1.13) + google-auth-library: 9.14.2(encoding@0.1.13) node-fetch: 2.7.0(encoding@0.1.13) object-hash: 3.0.0 proto3-json-serializer: 2.0.1 @@ -10085,7 +10138,7 @@ snapshots: googleapis@137.1.0(encoding@0.1.13): dependencies: - google-auth-library: 9.11.0(encoding@0.1.13) + google-auth-library: 9.14.2(encoding@0.1.13) googleapis-common: 7.2.0(encoding@0.1.13) transitivePeerDependencies: - encoding @@ -10691,7 +10744,7 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 20.11.30 + '@types/node': 20.16.9 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -11436,6 +11489,13 @@ snapshots: postcss: 8.4.47 tsx: 4.19.1 + postcss-load-config@6.0.1(postcss@8.4.47)(tsx@4.19.2): + dependencies: + lilconfig: 3.1.2 + optionalDependencies: + postcss: 8.4.47 + tsx: 4.19.2 + postcss-load-config@6.0.1(postcss@8.4.47)(tsx@4.7.1): dependencies: lilconfig: 3.1.2 @@ -12142,6 +12202,33 @@ snapshots: - tsx - yaml + tsup@8.3.5(postcss@8.4.47)(tsx@4.19.2)(typescript@4.9.5): + dependencies: + bundle-require: 5.0.0(esbuild@0.24.0) + cac: 6.7.14 + chokidar: 4.0.1 + consola: 3.2.3 + debug: 4.3.7 + esbuild: 0.24.0 + joycon: 3.1.1 + picocolors: 1.1.1 + postcss-load-config: 6.0.1(postcss@8.4.47)(tsx@4.19.2) + resolve-from: 5.0.0 + rollup: 4.25.0 + source-map: 0.8.0-beta.0 + sucrase: 3.35.0 + tinyexec: 0.3.1 + tinyglobby: 0.2.10 + tree-kill: 1.2.2 + optionalDependencies: + postcss: 8.4.47 + typescript: 4.9.5 + transitivePeerDependencies: + - jiti + - supports-color + - tsx + - yaml + tsup@8.3.5(postcss@8.4.47)(tsx@4.7.1)(typescript@4.9.5): dependencies: bundle-require: 5.0.0(esbuild@0.24.0) diff --git a/js/testapps/checks-evaluation/package.json b/js/testapps/checks-evaluation/package.json new file mode 100644 index 000000000..bd33b918c --- /dev/null +++ b/js/testapps/checks-evaluation/package.json @@ -0,0 +1,29 @@ +{ + "name": "checks-evaluators", + "version": "1.0.0", + "description": "", + "main": "lib/index.js", + "scripts": { + "start": "node lib/index.js", + "compile": "tsc", + "dev": "tsx --watch src/index.ts", + "genkit:dev": "cross-env GENKIT_ENV=dev pnpm dev", + "build": "pnpm build:clean && pnpm compile", + "build:clean": "rimraf ./lib", + "build:watch": "tsc --watch" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@genkit-ai/checks": "workspace:*", + "genkit": "workspace:*", + "path": "^0.12.7" + }, + "devDependencies": { + "rimraf": "^6.0.1", + "cross-env": "^7.0.3", + "tsx": "^4.7.0", + "typescript": "^5.3.3" + } +} \ No newline at end of file diff --git a/js/testapps/checks-evaluation/src/index.ts b/js/testapps/checks-evaluation/src/index.ts new file mode 100644 index 000000000..728e1223c --- /dev/null +++ b/js/testapps/checks-evaluation/src/index.ts @@ -0,0 +1,51 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { checks, ChecksEvaluationMetricType } from '@genkit-ai/checks'; +import { genkit } from 'genkit'; + +export const ai = genkit({ + plugins: [ + checks({ + // Project to charge quota to. + // Note: If your credentials have a quota project associated with them. + // That value will take precedence over this. + projectId: 'your-project-id', + evaluation: { + metrics: [ + // Policies configured with the default threshold. + ChecksEvaluationMetricType.DANGEROUS_CONTENT, + ChecksEvaluationMetricType.HARASSMENT, + ChecksEvaluationMetricType.HATE_SPEECH, + ChecksEvaluationMetricType.MEDICAL_INFO, + ChecksEvaluationMetricType.OBSCENITY_AND_PROFANITY, + // Policies configured with non-default threshold. + { + type: ChecksEvaluationMetricType.PII_SOLICITING_RECITING, + threshold: 0.6, + }, + { + type: ChecksEvaluationMetricType.SEXUALLY_EXPLICIT, + threshold: 0.3, + }, + { + type: ChecksEvaluationMetricType.VIOLENCE_AND_GORE, + threshold: 0.55, + }, + ], + }, + }), + ], +}); diff --git a/js/testapps/checks-evaluation/tsconfig.json b/js/testapps/checks-evaluation/tsconfig.json new file mode 100644 index 000000000..b73ccd04d --- /dev/null +++ b/js/testapps/checks-evaluation/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", + "noImplicitReturns": true, + "noUnusedLocals": false, + "outDir": "lib", + "sourceMap": true, + "strict": true, + "target": "es2017", + "skipLibCheck": true, + "esModuleInterop": true + }, + "compileOnSave": true, + "include": ["src"] +} diff --git a/scripts/release_main.sh b/scripts/release_main.sh index e50753e71..67866b890 100755 --- a/scripts/release_main.sh +++ b/scripts/release_main.sh @@ -79,3 +79,7 @@ cd $CURRENT cd js/plugins/langchain pnpm publish --registry https://wombat-dressing-room.appspot.com cd $CURRENT + +cd js/plugins/checks +pnpm publish --registry https://wombat-dressing-room.appspot.com +cd $CURRENT diff --git a/scripts/release_next.sh b/scripts/release_next.sh index 3e62e982b..f38161bc5 100755 --- a/scripts/release_next.sh +++ b/scripts/release_next.sh @@ -81,3 +81,6 @@ cd js/plugins/langchain pnpm publish --tag next --publish-branch next --registry https://wombat-dressing-room.appspot.com cd $CURRENT +cd js/plugins/checks +pnpm publish --tag next --publish-branch next --registry https://wombat-dressing-room.appspot.com +cd $CURRENT