Skip to content

Commit 33fc200

Browse files
committed
Check final block validation eval result
1 parent 7fd5cfe commit 33fc200

1 file changed

Lines changed: 5 additions & 7 deletions

File tree

‎eval/promptfoo.config.yaml‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -147,17 +147,15 @@ tests:
147147
if (validationResults.length === 0) {
148148
return { pass: false, score: 0, reason: 'validate_blocks was not called after building the page' };
149149
}
150-
const failures = validationResults.filter(r => {
151-
const text = r.text || '';
152-
return r.isError || /Invalid blocks:/i.test(text) || !/Validation:\s+\d+\/\d+\s+blocks valid/i.test(text);
153-
});
154-
if (failures.length > 0) {
150+
const last = validationResults[validationResults.length - 1];
151+
const text = last.text || '';
152+
const failed = last.isError || /Invalid blocks:/i.test(text) || !/Validation:\s+\d+\/\d+\s+blocks valid/i.test(text);
153+
if (failed) {
155154
return {
156155
pass: false,
157156
score: 0,
158-
reason: `validate_blocks reported invalid or inconclusive output: ${(failures[0].text || JSON.stringify(failures[0])).slice(0, 500)}`,
157+
reason: `final validate_blocks result was invalid or inconclusive: ${(text || JSON.stringify(last)).slice(0, 500)}`,
159158
};
160159
}
161-
const last = validationResults[validationResults.length - 1];
162160
return { pass: true, score: 1, reason: `validate_blocks passed: ${(last.text || '').split(/\r?\n/)[0]}` };
163161
});

0 commit comments

Comments
 (0)