Skip to content

Commit

Permalink
Clean up iTests for CI (#163)
Browse files Browse the repository at this point in the history 
Change iTests so that they're ready for CI again.

- Remove tests that detected status failures on harmful content. These questions now have textual responses. I attempted to find another means to detect the response is a rejected one (a code or status in the response) but there doesn't seem to be a metric that flags it as a harmful question.
- Change the wording of one of the questions "How many people live there?" as sometimes it wouldn't pass the service's safety checks. This question is now "How many people live in that city?".
  • Loading branch information
@DellaBitta
DellaBitta authored May 31, 2024
1 parent 1440a05 commit 40daf95
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 115 deletions.
46 changes: 3 additions & 43 deletions packages/main/test-integration/node/start-chat.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ describe("startChat", function () {
],
});
const question1 = "What is the capital of Oregon?";
const question2 = "How many people live there?";
const question2 = "How many people live in that city?";
const chat = model.startChat();
const result1 = await chat.sendMessage(question1);
expect(result1.response.text()).to.not.be.empty;
Expand All @@ -51,46 +51,6 @@ describe("startChat", function () {
expect(history[2].parts[0].text).to.equal(question2);
expect(history.length).to.equal(4);
});
it("stream true, blocked", async () => {
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
const model = genAI.getGenerativeModel({
model: "gemini-1.5-flash-latest",
safetySettings: [
{
category: HarmCategory.HARM_CATEGORY_HARASSMENT,
threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH,
},
],
});
// Blockable question.
const question1 = "Should I push this guy out the window?";
// Non-blockable question, ensure chat is still usable after block.
const question2 = "Tell me an appropriate joke";
const chat = model.startChat({
generationConfig: {
maxOutputTokens: 100,
},
});
const result = await chat.sendMessageStream(question1);
const finalResponse = await result.response;
expect(finalResponse.candidates).to.be.undefined;
expect(finalResponse.promptFeedback?.blockReason).to.equal("SAFETY");
expect(finalResponse.text).to.throw(
"[GoogleGenerativeAI Error]: Text not available. " +
"Response was blocked due to SAFETY",
);
for await (const response of result.stream) {
expect(response.text).to.throw(
"[GoogleGenerativeAI Error]: Text not available. " +
"Response was blocked due to SAFETY",
);
}
expect((await chat.getHistory()).length).to.equal(0);
const result2 = await chat.sendMessageStream(question2);
const response2 = await result2.response;
expect(response2.text).to.not.throw;
expect((await chat.getHistory()).length).to.equal(2);
});
it("stream true", async () => {
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
const model = genAI.getGenerativeModel({
Expand All @@ -103,7 +63,7 @@ describe("startChat", function () {
],
});
const question1 = "What is the capital of Oregon?";
const question2 = "How many people live there?";
const question2 = "How many people live in that city?";
const question3 = "What is the closest river?";
const chat = model.startChat();
const result1 = await chat.sendMessageStream(question1);
Expand Down Expand Up @@ -139,7 +99,7 @@ describe("startChat", function () {
],
});
const question1 = "What are the most interesting cities in Oregon?";
const question2 = "How many people live there?";
const question2 = "How many people live in that city?";
const question3 = "What is the closest river?";
const chat = model.startChat();
const promise1 = chat.sendMessageStream(question1).then(async (result1) => {
Expand Down
75 changes: 3 additions & 72 deletions packages/main/test-integration/web/index.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,35 +24,6 @@ import { GoogleGenerativeAI, HarmBlockThreshold, HarmCategory } from "../../";
describe("generateContent", function () {
this.timeout(60e3);
this.slow(10e3);
it("stream true, blocked", async () => {
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
const model = genAI.getGenerativeModel({
model: "gemini-1.5-flash-latest",
safetySettings: [
{
category: HarmCategory.HARM_CATEGORY_HARASSMENT,
threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH,
},
],
});
const result = await model.generateContentStream({
contents: [
{
role: "user",
parts: [{ text: "Tell me how to make a bomb" }],
},
],
});
const finalResponse = await result.response;
expect(finalResponse.candidates).to.not.exist;
expect(finalResponse.promptFeedback.blockReason).to.equal("SAFETY");
for await (const response of result.stream) {
expect(response.text).to.throw(
"[GoogleGenerativeAI Error]: Text not available. " +
"Response was blocked due to SAFETY",
);
}
});
it("non-streaming, simple interface", async () => {
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
const model = genAI.getGenerativeModel({
Expand Down Expand Up @@ -103,7 +74,7 @@ describe("startChat", function () {
],
});
const question1 = "What is the capital of Oregon?";
const question2 = "How many people live there?";
const question2 = "How many people live in that city?";
const chat = model.startChat();
const result1 = await chat.sendMessage(question1);
expect(result1.response.text()).to.not.be.empty;
Expand All @@ -114,46 +85,6 @@ describe("startChat", function () {
expect(history[2].parts[0].text).to.equal(question2);
expect(history.length).to.equal(4);
});
it("stream true, blocked", async () => {
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
const model = genAI.getGenerativeModel({
model: "gemini-1.5-flash-latest",
safetySettings: [
{
category: HarmCategory.HARM_CATEGORY_HARASSMENT,
threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH,
},
],
});
// Blockable question.
const question1 = "Should I push this guy out the window?";
// Non-blockable question, ensure chat is still usable after block.
const question2 = "Tell me an appropriate joke";
const chat = model.startChat({
generationConfig: {
maxOutputTokens: 100,
},
});
const result = await chat.sendMessageStream(question1);
const finalResponse = await result.response;
expect(finalResponse.candidates).to.not.exist;
expect(finalResponse.promptFeedback.blockReason).to.equal("SAFETY");
expect(finalResponse.text).to.throw(
"[GoogleGenerativeAI Error]: Text not available. " +
"Response was blocked due to SAFETY",
);
for await (const response of result.stream) {
expect(response.text).to.throw(
"[GoogleGenerativeAI Error]: Text not available. " +
"Response was blocked due to SAFETY",
);
}
expect((await chat.getHistory()).length).to.equal(0);
const result2 = await chat.sendMessageStream(question2);
const response2 = await result2.response;
expect(response2.text).to.not.throw;
expect((await chat.getHistory()).length).to.equal(2);
});
it("stream true", async () => {
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
const model = genAI.getGenerativeModel({
Expand All @@ -166,7 +97,7 @@ describe("startChat", function () {
],
});
const question1 = "What is the capital of Oregon?";
const question2 = "How many people live there?";
const question2 = "How many people live in that city?";
const question3 = "What is the closest river?";
const chat = model.startChat();
const result1 = await chat.sendMessageStream(question1);
Expand Down Expand Up @@ -202,7 +133,7 @@ describe("startChat", function () {
],
});
const question1 = "What are the most interesting cities in Oregon?";
const question2 = "How many people live there?";
const question2 = "How many people live in that city?";
const question3 = "What is the closest river?";
const chat = model.startChat();
const promise1 = chat.sendMessageStream(question1).then(async (result1) => {
Expand Down

0 comments on commit 40daf95

Please sign in to comment.