From 51ecf67edda58a62663deeccc1d0d9862a1132ac Mon Sep 17 00:00:00 2001 From: Lindar90 Date: Tue, 31 Mar 2026 14:30:19 +0200 Subject: [PATCH 1/2] feat(metadata): add confidence score and reference to autofill extraction --- src/api/Intelligence.js | 5 +- src/api/__tests__/Intelligence.test.js | 36 +- src/api/schemas/AiExtractResponse.js | 35 +- src/api/schemas/AiExtractStructured.js | 8 + .../useSidebarMetadataFetcher.test.tsx | 381 +++++++++++++++--- .../hooks/useSidebarMetadataFetcher.ts | 48 ++- .../MetadataSidebarRedesignedMocks.ts | 66 ++- ...MetadataSidebarRedesign-visual.stories.tsx | 160 ++++++++ 8 files changed, 638 insertions(+), 101 deletions(-) diff --git a/src/api/Intelligence.js b/src/api/Intelligence.js index 4c5bde865e..c0f089ae1d 100644 --- a/src/api/Intelligence.js +++ b/src/api/Intelligence.js @@ -77,6 +77,7 @@ class Intelligence extends Base { throw new Error('Invalid item!'); } + // See: https://developer.box.com/reference/post-ai-extract-structured const url = `${this.getBaseApiUrl()}/ai/extract_structured`; const suggestionsResponse = await this.xhr.post({ @@ -85,9 +86,7 @@ class Intelligence extends Base { id: `file_${item.id}`, }); - return !!suggestionsResponse?.data?.answer && typeof suggestionsResponse.data.answer === 'object' - ? suggestionsResponse.data.answer - : suggestionsResponse.data; + return suggestionsResponse.data; } } diff --git a/src/api/__tests__/Intelligence.test.js b/src/api/__tests__/Intelligence.test.js index abb4fe2115..8415c6811f 100644 --- a/src/api/__tests__/Intelligence.test.js +++ b/src/api/__tests__/Intelligence.test.js @@ -124,26 +124,24 @@ describe('api/Intelligence', () => { } }); - test.each` - suggestionsFromServer | responseData - ${{ stringFieldKey: 'fieldVal1', floatFieldKey: 124.0, enumFieldKey: 'EnumOptionKey', multiSelectFieldKey: ['multiSelectOption1', 'multiSelectOption5'] }} | ${{ data: { stringFieldKey: 'fieldVal1', floatFieldKey: 124.0, enumFieldKey: 'EnumOptionKey', multiSelectFieldKey: ['multiSelectOption1', 'multiSelectOption5'] } }} - ${{ stringFieldKey: 'fieldVal1', floatFieldKey: 124.0, enumFieldKey: 'EnumOptionKey', multiSelectFieldKey: ['multiSelectOption1', 'multiSelectOption5'] }} | ${{ data: { answer: { stringFieldKey: 'fieldVal1', floatFieldKey: 124.0, enumFieldKey: 'EnumOptionKey', multiSelectFieldKey: ['multiSelectOption1', 'multiSelectOption5'] }, create_at: '2025-01-14T00:00:00-00:00' } }} - ${{}} | ${{ data: {} }} - ${{}} | ${{ data: { answer: {}, create_at: '2025-01-14T00:00:00-00:00' } }} - `( - 'should return a successful response including the answer from the LLM', - async ({ suggestionsFromServer, responseData }) => { - intelligence.xhr.post = jest.fn().mockReturnValueOnce(responseData); + test('should return data as-is', async () => { + const responseData = { + data: { + answer: { field1: 'value1' }, + created_at: '2026-03-27T08:10:14.106-07:00', + completion_reason: 'done', + }, + }; + intelligence.xhr.post = jest.fn().mockReturnValueOnce(responseData); - const suggestions = await intelligence.extractStructured(request); - expect(suggestions).toEqual(suggestionsFromServer); - expect(intelligence.xhr.post).toHaveBeenCalledWith({ - url: `${intelligence.getBaseApiUrl()}/ai/extract_structured`, - id: 'file_123', - data: request, - }); - }, - ); + const result = await intelligence.extractStructured(request); + expect(result).toEqual(responseData.data); + expect(intelligence.xhr.post).toHaveBeenCalledWith({ + url: `${intelligence.getBaseApiUrl()}/ai/extract_structured`, + id: 'file_123', + data: request, + }); + }); test('should not return any suggestions when error is 400', async () => { const error = new Error(); diff --git a/src/api/schemas/AiExtractResponse.js b/src/api/schemas/AiExtractResponse.js index 2ee036ad78..cce5554a41 100644 --- a/src/api/schemas/AiExtractResponse.js +++ b/src/api/schemas/AiExtractResponse.js @@ -3,4 +3,37 @@ * @author Box */ -export interface AiExtractResponse {} +export interface AiExtractConfidenceScore { + level: string; + score: number; +} + +export interface AiExtractReference { + itemId: string; + page: number; + text: string; + boundingBox?: { + left: number, + top: number, + right: number, + bottom: number, + }; +} + +export interface AiAgentInfo { + models?: Array<{ + name?: string, + provider?: string, + supported_purpose?: string, + }>; + processor?: string; +} + +export interface AiExtractResponse { + answer: { [key: string]: any }; + created_at: string; + completion_reason?: string; + confidence_score?: { [key: string]: AiExtractConfidenceScore }; + reference?: { [key: string]: Array }; + ai_agent_info?: AiAgentInfo; +} diff --git a/src/api/schemas/AiExtractStructured.js b/src/api/schemas/AiExtractStructured.js index d57706e4bd..d0ad491e24 100644 --- a/src/api/schemas/AiExtractStructured.js +++ b/src/api/schemas/AiExtractStructured.js @@ -87,4 +87,12 @@ export interface AiExtractStructured { * – `AiAgentReference` : reference a custom AI-Agent by ID */ +ai_agent?: AiAgentExtractStructured | AiAgentReference; + /** + * When `true`, the response includes confidence scores for each extracted field. + */ + +include_confidence_score?: boolean; + /** + * When `true`, the response includes reference locations (bounding boxes) for each extracted field. + */ + +include_reference?: boolean; } diff --git a/src/elements/content-sidebar/__tests__/useSidebarMetadataFetcher.test.tsx b/src/elements/content-sidebar/__tests__/useSidebarMetadataFetcher.test.tsx index f7566ec0a5..20647bf2ca 100644 --- a/src/elements/content-sidebar/__tests__/useSidebarMetadataFetcher.test.tsx +++ b/src/elements/content-sidebar/__tests__/useSidebarMetadataFetcher.test.tsx @@ -89,46 +89,62 @@ const newTemplateInstance = { hidden: false, }; +const defaultGetFileImplementation = (_id, successCallback, errorCallback) => { + try { + successCallback(mockFile); + } catch (error) { + errorCallback(error); + } +}; + +const defaultGetMetadataImplementation = (_file, successCallback, errorCallback) => { + try { + successCallback({ + editors: [], + templates: mockTemplates, + templateInstances: mockTemplateInstances, + }); + } catch (error) { + errorCallback(error); + } +}; + +const defaultDeleteMetadataImplementation = (_file, template, successCallback, errorCallback) => { + try { + successCallback(template); + } catch (error) { + errorCallback(error); + } +}; + +const defaultCreateMetadataRedesignImplementation = (_file, _template, successCallback, errorCallback) => { + try { + successCallback(); + } catch (error) { + errorCallback(error); + } +}; + +const defaultUpdateMetadataRedesignImplementation = ( + _file, + _metadataInstance, + _JSONPatch, + successCallback, + errorCallback, +) => { + try { + successCallback(); + } catch (error) { + errorCallback(error); + } +}; + const mockAPI = { - getFile: jest.fn((id, successCallback, errorCallback) => { - try { - successCallback(mockFile); - } catch (error) { - errorCallback(error); - } - }), - getMetadata: jest.fn((_file, successCallback, errorCallback) => { - try { - successCallback({ - editors: [], - templates: mockTemplates, - templateInstances: mockTemplateInstances, - }); - } catch (error) { - errorCallback(error); - } - }), - deleteMetadata: jest.fn((_file, template, successCallback, errorCallback) => { - try { - successCallback(template); - } catch (error) { - errorCallback(error); - } - }), - createMetadataRedesign: jest.fn((_file, template, successCallback, errorCallback) => { - try { - successCallback(); - } catch (error) { - errorCallback(error); - } - }), - updateMetadataRedesign: jest.fn((_file, _metadataInstance, _JSONPatch, successCallback, errorCallback) => { - try { - successCallback(); - } catch (error) { - errorCallback(error); - } - }), + getFile: jest.fn(), + getMetadata: jest.fn(), + deleteMetadata: jest.fn(), + createMetadataRedesign: jest.fn(), + updateMetadataRedesign: jest.fn(), extractStructured: jest.fn(), }; const api = { @@ -137,6 +153,14 @@ const api = { getIntelligenceAPI: jest.fn().mockReturnValue(mockAPI), }; +const setupDefaultMockImplementations = () => { + mockAPI.getFile.mockImplementation(defaultGetFileImplementation); + mockAPI.getMetadata.mockImplementation(defaultGetMetadataImplementation); + mockAPI.deleteMetadata.mockImplementation(defaultDeleteMetadataImplementation); + mockAPI.createMetadataRedesign.mockImplementation(defaultCreateMetadataRedesignImplementation); + mockAPI.updateMetadataRedesign.mockImplementation(defaultUpdateMetadataRedesignImplementation); +}; + describe('useSidebarMetadataFetcher', () => { const onErrorMock = jest.fn(); const onSuccessMock = jest.fn(); @@ -157,11 +181,14 @@ describe('useSidebarMetadataFetcher', () => { beforeEach(() => { onErrorMock.mockClear(); onSuccessMock.mockClear(); - mockAPI.getFile.mockClear(); - mockAPI.getMetadata.mockClear(); - mockAPI.deleteMetadata.mockClear(); - mockAPI.updateMetadataRedesign.mockClear(); - mockAPI.extractStructured.mockClear(); + // Reset call history and per-test overrides, then restore deterministic defaults. + mockAPI.getFile.mockReset(); + mockAPI.getMetadata.mockReset(); + mockAPI.deleteMetadata.mockReset(); + mockAPI.createMetadataRedesign.mockReset(); + mockAPI.updateMetadataRedesign.mockReset(); + mockAPI.extractStructured.mockReset(); + setupDefaultMockImplementations(); }); test('should fetch the file and metadata successfully', async () => { @@ -394,11 +421,10 @@ describe('useSidebarMetadataFetcher', () => { describe('extractSuggestions', () => { test('should extract suggestions successfully', async () => { - const mockSuggestions = { - field1: 'value1', - field2: 'value2', - }; - mockAPI.extractStructured.mockResolvedValue(mockSuggestions); + mockAPI.extractStructured.mockResolvedValue({ + answer: { field1: 'value1', field2: 'value2' }, + created_at: '2026-03-27T08:10:14.106-07:00', + }); const { result } = setupHook(); @@ -427,13 +453,6 @@ describe('useSidebarMetadataFetcher', () => { templateKey: 'taxonomyTemplateKey', }; - const mockTaxonomySuggestions = { - taxonomyField: [ - { id: 'taxonomy-id-1', displayName: 'Taxonomy Item 1' }, - { id: 'taxonomy-id-2', displayName: 'Taxonomy Item 2' }, - ], - }; - mockAPI.getMetadata.mockImplementation((file, successCallback) => { successCallback({ editors: [], @@ -441,7 +460,15 @@ describe('useSidebarMetadataFetcher', () => { templateInstances: [], }); }); - mockAPI.extractStructured.mockResolvedValue(mockTaxonomySuggestions); + mockAPI.extractStructured.mockResolvedValue({ + answer: { + taxonomyField: [ + { id: 'taxonomy-id-1', displayName: 'Taxonomy Item 1' }, + { id: 'taxonomy-id-2', displayName: 'Taxonomy Item 2' }, + ], + }, + created_at: '2026-03-27T08:10:14.106-07:00', + }); const { result } = setupHook(); @@ -498,8 +525,13 @@ describe('useSidebarMetadataFetcher', () => { await waitFor(() => expect(result.current.extractErrorCode).toBeNull()); }); - test('should handle empty suggestions', async () => { - mockAPI.extractStructured.mockResolvedValue([]); + test.each` + description | response + ${'empty answer'} | ${{ answer: {}, created_at: '2026-03-27T08:10:14.106-07:00' }} + ${'null response'} | ${null} + ${'undefined response'} | ${undefined} + `('should handle $description from extractStructured', async ({ response }) => { + mockAPI.extractStructured.mockResolvedValue(response); const { result } = setupHook(); const suggestions = await result.current.extractSuggestions('templateKey', 'global'); @@ -571,5 +603,236 @@ describe('useSidebarMetadataFetcher', () => { }), ); }); + + test('should include include_confidence_score and include_reference when isConfidenceScoreEnabled is true', async () => { + mockAPI.extractStructured.mockResolvedValue({ + answer: { field1: 'value1' }, + created_at: '2026-03-27T08:10:14.106-07:00', + completion_reason: 'done', + }); + + const { result } = setupHook('123', true); + + await result.current.extractSuggestions('templateKey', 'global'); + + expect(mockAPI.extractStructured).toHaveBeenCalledWith({ + items: [{ id: mockFile.id, type: mockFile.type }], + metadata_template: { template_key: 'templateKey', scope: 'global', type: 'metadata_template' }, + include_confidence_score: true, + include_reference: true, + }); + }); + + test('should not include include_confidence_score and include_reference when isConfidenceScoreEnabled is false', async () => { + mockAPI.extractStructured.mockResolvedValue({ + answer: { field1: 'value1' }, + created_at: '2026-03-27T08:10:14.106-07:00', + }); + + const { result } = setupHook('123', false); + + await result.current.extractSuggestions('templateKey', 'global'); + + expect(mockAPI.extractStructured).toHaveBeenCalledWith( + expect.not.objectContaining({ + include_confidence_score: expect.anything(), + include_reference: expect.anything(), + }), + ); + }); + + test('should parse response with confidence scores', async () => { + mockAPI.extractStructured.mockResolvedValue({ + answer: { field1: 'value1', field2: 'value2' }, + created_at: '2026-03-27T08:10:14.106-07:00', + confidence_score: { + field1: { level: 'HIGH', score: 0.95 }, + field2: { level: 'LOW', score: 0.3 }, + }, + completion_reason: 'done', + }); + + const { result } = setupHook('123', true); + await waitFor(() => expect(result.current.status).toBe(STATUS.SUCCESS)); + + const suggestions = await result.current.extractSuggestions('templateKey', 'global'); + + expect(suggestions).toEqual([ + { + ...mockTemplates[0].fields[0], + aiSuggestion: 'value1', + aiSuggestionConfidenceScore: { value: 0.95, level: 'HIGH', isAccepted: false }, + }, + { + ...mockTemplates[0].fields[1], + aiSuggestion: 'value2', + aiSuggestionConfidenceScore: { value: 0.3, level: 'LOW', isAccepted: false }, + }, + ]); + }); + + test('should parse response with references', async () => { + mockAPI.extractStructured.mockResolvedValue({ + answer: { field1: 'value1' }, + created_at: '2026-03-27T08:10:14.106-07:00', + confidence_score: { + field1: { level: 'MEDIUM', score: 0.85 }, + }, + reference: { + field1: [ + { + itemId: 'file_123', + page: 0, + text: 'extracted text', + boundingBox: { left: 0.1, top: 0.2, right: 0.3, bottom: 0.4 }, + }, + ], + }, + completion_reason: 'done', + }); + + const { result } = setupHook('123', true); + await waitFor(() => expect(result.current.status).toBe(STATUS.SUCCESS)); + + const suggestions = await result.current.extractSuggestions('templateKey', 'global'); + + expect(suggestions).toEqual([ + { + ...mockTemplates[0].fields[0], + aiSuggestion: 'value1', + aiSuggestionConfidenceScore: { value: 0.85, level: 'MEDIUM', isAccepted: false }, + targetLocation: [ + { + itemId: 'file_123', + page: 0, + text: 'extracted text', + boundingBox: { left: 0.1, top: 0.2, right: 0.3, bottom: 0.4 }, + }, + ], + }, + mockTemplates[0].fields[1], + ]); + }); + + test('should parse taxonomy response with confidence scores', async () => { + const taxonomyTemplate = { + canEdit: true, + id: 'metadata_template_taxonomy', + fields: [ + { + key: 'taxonomyField', + type: 'taxonomy' as MetadataTemplateFieldType, + hidden: false, + }, + ], + scope: 'global', + templateKey: 'taxonomyTemplateKey', + }; + + mockAPI.getMetadata.mockImplementation((file, successCallback) => { + successCallback({ + editors: [], + templates: [taxonomyTemplate], + templateInstances: [], + }); + }); + mockAPI.extractStructured.mockResolvedValue({ + answer: { + taxonomyField: [ + { id: 'taxonomy-id-1', displayName: 'Taxonomy Item 1' }, + { id: 'taxonomy-id-2', displayName: 'Taxonomy Item 2' }, + ], + }, + confidence_score: { + taxonomyField: { level: 'MEDIUM', score: 0.72 }, + }, + completion_reason: 'done', + }); + + const { result } = setupHook('123', true); + + await waitFor(() => expect(result.current.templates).toEqual([taxonomyTemplate])); + + const suggestions = await result.current.extractSuggestions('taxonomyTemplateKey', 'global'); + + expect(suggestions).toEqual([ + { + ...taxonomyTemplate.fields[0], + aiSuggestion: [ + { value: 'taxonomy-id-1', displayValue: 'Taxonomy Item 1' }, + { value: 'taxonomy-id-2', displayValue: 'Taxonomy Item 2' }, + ], + aiSuggestionConfidenceScore: { value: 0.72, level: 'MEDIUM', isAccepted: false }, + }, + ]); + }); + + test('should handle response with empty confidence_score when FF is enabled', async () => { + mockAPI.extractStructured.mockResolvedValue({ + answer: { field1: 'value1' }, + confidence_score: {}, + created_at: '2026-03-27T08:10:14.106-07:00', + }); + + const { result } = setupHook('123', true); + await waitFor(() => expect(result.current.status).toBe(STATUS.SUCCESS)); + + const suggestions = await result.current.extractSuggestions('templateKey', 'global'); + + expect(suggestions).toEqual([ + { + ...mockTemplates[0].fields[0], + aiSuggestion: 'value1', + }, + mockTemplates[0].fields[1], + ]); + }); + + test('should handle response where only some fields have confidence scores', async () => { + mockAPI.extractStructured.mockResolvedValue({ + answer: { field1: 'value1', field2: 'value2' }, + confidence_score: { + field1: { level: 'HIGH', score: 0.9 }, + }, + completion_reason: 'done', + }); + + const { result } = setupHook('123', true); + await waitFor(() => expect(result.current.status).toBe(STATUS.SUCCESS)); + + const suggestions = await result.current.extractSuggestions('templateKey', 'global'); + + expect(suggestions).toEqual([ + { + ...mockTemplates[0].fields[0], + aiSuggestion: 'value1', + aiSuggestionConfidenceScore: { value: 0.9, level: 'HIGH', isAccepted: false }, + }, + { + ...mockTemplates[0].fields[1], + aiSuggestion: 'value2', + }, + ]); + }); + + test('should handle reference entries without bounding box', async () => { + mockAPI.extractStructured.mockResolvedValue({ + answer: { field1: 'value1' }, + confidence_score: { + field1: { level: 'HIGH', score: 0.8 }, + }, + reference: { + field1: [{ itemId: 'file_123', page: 1, text: 'some text' }], + }, + completion_reason: 'done', + }); + + const { result } = setupHook('123', true); + await waitFor(() => expect(result.current.status).toBe(STATUS.SUCCESS)); + + const suggestions = await result.current.extractSuggestions('templateKey', 'global'); + + expect(suggestions[0].targetLocation).toEqual([{ itemId: 'file_123', page: 1, text: 'some text' }]); + }); }); }); diff --git a/src/elements/content-sidebar/hooks/useSidebarMetadataFetcher.ts b/src/elements/content-sidebar/hooks/useSidebarMetadataFetcher.ts index 5940180560..030a5e2dcc 100644 --- a/src/elements/content-sidebar/hooks/useSidebarMetadataFetcher.ts +++ b/src/elements/content-sidebar/hooks/useSidebarMetadataFetcher.ts @@ -6,10 +6,10 @@ import { type MetadataTemplate, type MetadataTemplateInstance, type MetadataTemplateField, - type MetadataFieldValue, } from '@box/metadata-editor'; import isEmpty from 'lodash/isEmpty'; import API from '../../../api'; +import { formatMetadataFieldValue } from '../../../api/utils'; import { type ElementsXhrError } from '../../../common/types/api'; import { isUserCorrectableError } from '../../../utils/error'; import { @@ -222,16 +222,20 @@ function useSidebarMetadataFetcher( async (templateKey: string, scope: string, agentId?: string): Promise => { const aiAPI = api.getIntelligenceAPI(); setExtractErrorCode(null); - let answer = null; + let response = null; const customAiAgent = agentId ? { ai_agent: { type: 'ai_agent_id', id: agentId } } : {}; + const confidenceScoreParams = isConfidenceScoreEnabled + ? { include_confidence_score: true, include_reference: true } + : {}; const requestBody: AiExtractStructured = { items: [{ id: file.id, type: file.type }], metadata_template: { template_key: templateKey, scope, type: 'metadata_template' }, ...customAiAgent, + ...confidenceScoreParams, }; try { - answer = (await aiAPI.extractStructured(requestBody)) as Record; + response = await aiAPI.extractStructured(requestBody); } catch (error) { // Axios makes the status code nested under the response object if (error.response?.status === 408) { @@ -255,6 +259,10 @@ function useSidebarMetadataFetcher( return []; } + const answer = response?.answer; + const confidenceScores = response?.confidence_score; + const references = response?.reference; + if (isEmpty(answer)) { const error = new Error('No suggestions found.'); onError(error, ERROR_CODE_EMPTY_METADATA_SUGGESTIONS, { showNotification: true }); @@ -265,27 +273,33 @@ function useSidebarMetadataFetcher( const fields = templateInstance?.fields || []; return fields.map(field => { - const value = answer[field.key]; + const rawValue = answer[field.key]; - if (!value) { + if (rawValue == null || rawValue === '') { return field; } - if (field.type === 'taxonomy') { - return { - ...field, - aiSuggestion: value.map(item => ({ - value: item.id, - displayValue: item.displayName, - })), + + const aiSuggestion = formatMetadataFieldValue(field, rawValue); + const result = { ...field, aiSuggestion }; + + const fieldConfidence = confidenceScores?.[field.key]; + if (fieldConfidence) { + result.aiSuggestionConfidenceScore = { + value: fieldConfidence.score, + level: fieldConfidence.level, + isAccepted: false, }; } - return { - ...field, - aiSuggestion: value, - }; + + const ref = references?.[field.key]; + if (ref) { + result.targetLocation = ref; + } + + return result; }); }, - [api, file, onError, templates], + [api, file, isConfidenceScoreEnabled, onError, templates], ); React.useEffect(() => { diff --git a/src/elements/content-sidebar/stories/__mocks__/MetadataSidebarRedesignedMocks.ts b/src/elements/content-sidebar/stories/__mocks__/MetadataSidebarRedesignedMocks.ts index 27ea3f0da9..739428e667 100644 --- a/src/elements/content-sidebar/stories/__mocks__/MetadataSidebarRedesignedMocks.ts +++ b/src/elements/content-sidebar/stories/__mocks__/MetadataSidebarRedesignedMocks.ts @@ -368,14 +368,76 @@ export const mockUpdateCustomMetadataRequest = { export const aiSuggestionsForMyAttribute = { url: `${apiV2Path}/ai/extract_structured`, response: { - myAttribute: 'it works fine', + answer: { + myAttribute: 'it works fine', + }, + created_at: '2024-04-01T00:00:00.000+00:00', + completion_reason: 'done', }, }; export const aiSuggestionForDateField = { url: `${apiV2Path}/ai/extract_structured`, response: { - dateField: '2024-04-01T00:00:00Z', + answer: { + dateField: '2024-04-01T00:00:00Z', + }, + created_at: '2024-04-01T00:00:00.000+00:00', + completion_reason: 'done', + }, +}; + +export const aiSuggestionsMultiFieldLowConfidence = { + url: `${apiV2Path}/ai/extract_structured`, + response: { + answer: { + someAttribute1: 'Suggestion Value 1', + someAttribute2: 'Suggestion Value 2', + someAttribute3: 'Suggestion Value 3', + }, + created_at: '2024-04-01T00:00:00.000+00:00', + completion_reason: 'done', + confidence_score: { + someAttribute1: { + level: 'LOW', + score: 0.2, + }, + someAttribute2: { + level: 'LOW', + score: 0.15, + }, + someAttribute3: { + level: 'LOW', + score: 0.25, + }, + }, + }, +}; + +export const aiSuggestionsMultiFieldHighConfidence = { + url: `${apiV2Path}/ai/extract_structured`, + response: { + answer: { + someAttribute1: 'Suggestion Value 1', + someAttribute2: 'Suggestion Value 2', + someAttribute3: 'Suggestion Value 3', + }, + created_at: '2024-04-01T00:00:00.000+00:00', + completion_reason: 'done', + confidence_score: { + someAttribute1: { + level: 'HIGH', + score: 0.95, + }, + someAttribute2: { + level: 'HIGH', + score: 0.92, + }, + someAttribute3: { + level: 'HIGH', + score: 0.97, + }, + }, }, }; diff --git a/src/elements/content-sidebar/stories/tests/MetadataSidebarRedesign-visual.stories.tsx b/src/elements/content-sidebar/stories/tests/MetadataSidebarRedesign-visual.stories.tsx index dd0789a76c..a5968e4e4d 100644 --- a/src/elements/content-sidebar/stories/tests/MetadataSidebarRedesign-visual.stories.tsx +++ b/src/elements/content-sidebar/stories/tests/MetadataSidebarRedesign-visual.stories.tsx @@ -8,6 +8,8 @@ import MetadataSidebarRedesign from '../../MetadataSidebarRedesign'; import { aiSuggestionForDateField, aiSuggestionsForMyAttribute, + aiSuggestionsMultiFieldHighConfidence, + aiSuggestionsMultiFieldLowConfidence, fileIdWithMetadata, fileIdWithoutMetadata, mockEmptyMetadataInstances, @@ -41,6 +43,11 @@ const defaultMetadataSidebarProps: ComponentProps { // eslint-disable-next-line no-console @@ -577,6 +584,159 @@ export const SuggestionForNewlyCreatedTemplateInstance: StoryObj = { + args: { features: confidenceFeatures }, + parameters: { + msw: { + handlers: [ + ...defaultMockHandlers, + http.post(aiSuggestionsMultiFieldLowConfidence.url, () => + HttpResponse.json(aiSuggestionsMultiFieldLowConfidence.response), + ), + ], + }, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + + const autofillButton = await canvas.findByRole('button', { + name: 'Autofill Select Dropdowns with Box AI', + }); + await userEvent.click(autofillButton); + + await waitFor( + () => { + const lowBadges = canvas.getAllByText('LOW'); + expect(lowBadges.length).toBeGreaterThanOrEqual(1); + expect(canvas.getAllByRole('button', { name: 'Accept extracted value' }).length).toBeGreaterThanOrEqual( + 1, + ); + expect(canvas.getAllByRole('button', { name: 'Clear extracted value' }).length).toBeGreaterThanOrEqual( + 1, + ); + }, + { timeout: 5000 }, + ); + }, +}; + +export const AutofillWithHighConfidenceOnEmptyFields: StoryObj = { + args: { features: confidenceFeatures }, + parameters: { + msw: { + handlers: [ + ...defaultMockHandlers, + http.post(aiSuggestionsMultiFieldHighConfidence.url, () => + HttpResponse.json(aiSuggestionsMultiFieldHighConfidence.response), + ), + ], + }, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + + const autofillButton = await canvas.findByRole('button', { + name: 'Autofill Select Dropdowns with Box AI', + }); + await userEvent.click(autofillButton); + + await waitFor( + () => { + const aiBadges = canvas.getAllByTestId('ai-confidence-badge'); + expect(aiBadges.length).toBeGreaterThanOrEqual(1); + expect(canvas.queryByText('LOW')).not.toBeInTheDocument(); + expect(canvas.queryByRole('button', { name: 'Accept extracted value' })).not.toBeInTheDocument(); + }, + { timeout: 5000 }, + ); + }, +}; + +// ────────────────────────────────────────────────────────── +// Confidence score stories: Multi-field review notice +// ────────────────────────────────────────────────────────── + +export const MultiFieldAutofillShowsReviewNotice: StoryObj = { + args: { features: confidenceFeatures }, + parameters: { + msw: { + handlers: [ + ...defaultMockHandlers, + http.post(aiSuggestionsMultiFieldLowConfidence.url, () => + HttpResponse.json(aiSuggestionsMultiFieldLowConfidence.response), + ), + ], + }, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + + const autofillButton = await canvas.findByRole('button', { + name: 'Autofill Select Dropdowns with Box AI', + }); + await userEvent.click(autofillButton); + + await waitFor( + () => { + expect(canvas.getByText(/3 fields need review/i)).toBeVisible(); + expect(canvas.getByRole('button', { name: 'View' })).toBeVisible(); + }, + { timeout: 5000 }, + ); + }, +}; + +export const AcceptAllDismissesReviewNotice: StoryObj = { + args: { features: confidenceFeatures }, + parameters: { + msw: { + handlers: [ + ...defaultMockHandlers, + http.post(aiSuggestionsMultiFieldLowConfidence.url, () => + HttpResponse.json(aiSuggestionsMultiFieldLowConfidence.response), + ), + ], + }, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + + const autofillButton = await canvas.findByRole('button', { + name: 'Autofill Select Dropdowns with Box AI', + }); + await userEvent.click(autofillButton); + + await waitFor( + () => { + expect(canvas.getByText(/3 fields need review/i)).toBeVisible(); + }, + { timeout: 5000 }, + ); + + let acceptButtons = canvas.getAllByRole('button', { name: 'Accept extracted value' }); + await userEvent.click(acceptButtons[0]); + await waitFor(() => { + expect(canvas.getByText(/2 fields need review/i)).toBeVisible(); + }); + + acceptButtons = canvas.getAllByRole('button', { name: 'Accept extracted value' }); + await userEvent.click(acceptButtons[0]); + await waitFor(() => { + expect(canvas.getByText(/1 field needs review/i)).toBeVisible(); + }); + + acceptButtons = canvas.getAllByRole('button', { name: 'Accept extracted value' }); + await userEvent.click(acceptButtons[0]); + await waitFor(() => { + expect(canvas.queryByText(/fields? needs? review/i)).not.toBeInTheDocument(); + }); + }, +}; + export const ShowErrorOnDelete: StoryObj = { parameters: { msw: { From ce2cebd2862ee18b1d3360c77af4798a2191b63c Mon Sep 17 00:00:00 2001 From: Lindar90 Date: Tue, 31 Mar 2026 17:40:57 +0200 Subject: [PATCH 2/2] feat(metadata): add confidence score and reference to autofill extraction - bump metadata-editor with support of aiSuggestionConfidenceScore field --- package.json | 4 ++-- yarn.lock | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 104bb73a99..a9d40260ec 100644 --- a/package.json +++ b/package.json @@ -135,7 +135,7 @@ "@box/frontend": "^11.0.1", "@box/item-icon": "^2.30.42", "@box/languages": "^1.0.0", - "@box/metadata-editor": "^1.55.5", + "@box/metadata-editor": "^1.58.0", "@box/metadata-filter": "^1.41.3", "@box/metadata-view": "^1.10.0", "@box/react-virtualized": "^9.22.3-rc-box.10", @@ -297,7 +297,7 @@ "@box/combobox-with-api": "^1.41.42", "@box/copy-input": "^1.39.41", "@box/item-icon": "^2.30.42", - "@box/metadata-editor": "^1.55.5", + "@box/metadata-editor": "^1.58.0", "@box/metadata-filter": "^1.41.3", "@box/metadata-view": "^1.10.0", "@box/react-virtualized": "^9.22.3-rc-box.10", diff --git a/yarn.lock b/yarn.lock index 07d61efcea..775aa317b0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1512,10 +1512,10 @@ resolved "https://registry.yarnpkg.com/@box/languages/-/languages-1.1.2.tgz#cd4266b3da62da18560d881e10b429653186be29" integrity sha512-d64TGosx+KRmrLZj4CIyLp42LUiEbgBJ8n8cviMQwTJmfU0g+UwZqLjmQZR1j+Q9D64yV4xHzY9K1t5nInWWeQ== -"@box/metadata-editor@^1.55.5": - version "1.55.5" - resolved "https://registry.yarnpkg.com/@box/metadata-editor/-/metadata-editor-1.55.5.tgz#75fcb0b248c9ac3ab0d005495996230a8aad6794" - integrity sha512-4cYWOWSA2MOWKLsCrPB0THkSNx1lAAeauI8YLfryn6Na1UGWSFNl8OihrdKkXvIai5dH94oea+M3ZOplhX/imA== +"@box/metadata-editor@^1.58.0": + version "1.58.0" + resolved "https://registry.yarnpkg.com/@box/metadata-editor/-/metadata-editor-1.58.0.tgz#6a7cb54e52768f84daf9ca4d559b45ac9f739222" + integrity sha512-FUJehqFO5v2y9X4BajmrZMGGSd7nKY83eoc7jMVyau5GB9y2P79fkWGP675HvgLy7YIAc16iWgnHs1wUQYV9nw== "@box/metadata-filter@^1.41.3": version "1.41.3"