From 260ac6980d855862390601d476839b51583b05ba Mon Sep 17 00:00:00 2001 From: Sophia Xu Date: Sat, 16 Apr 2022 16:46:40 -0400 Subject: [PATCH 1/3] feat(search): add title matching for search capabilities --- .../executables/initializeSearchCache.js | 52 +++++++++++++ .../unigraph.search/package.json | 18 +++++ packages/unigraph-dev-backend/src/caches.ts | 31 +++++++- .../src/executableManager.ts | 5 ++ .../src/localUnigraphApi.ts | 5 +- packages/unigraph-dev-backend/src/server.ts | 78 +++---------------- .../src/templates/defaultDb.ts | 2 + .../unigraph-dev-common/src/api/unigraph.ts | 4 +- .../unigraph-dev-common/src/types/unigraph.ts | 11 ++- .../UnigraphCore/InlineSearchPopup.tsx | 14 +++- 10 files changed, 145 insertions(+), 75 deletions(-) create mode 100644 packages/default-packages/unigraph.search/executables/initializeSearchCache.js create mode 100644 packages/default-packages/unigraph.search/package.json diff --git a/packages/default-packages/unigraph.search/executables/initializeSearchCache.js b/packages/default-packages/unigraph.search/executables/initializeSearchCache.js new file mode 100644 index 00000000..dd4fa6f1 --- /dev/null +++ b/packages/default-packages/unigraph.search/executables/initializeSearchCache.js @@ -0,0 +1,52 @@ +const newTitles = ( + await unigraph.getQueries([ + `(func: uid(u1, u2, u3, u4, u5, u6)) @filter(NOT eq(_hide, true) AND type(Entity)) @normalize { + uid + _updatedAt: _updatedAt + type { type: } + { + name { + name: <_value.%> + _value { _value { name: <_value.%> } } + } + } + } + var(func: eq(, "$/schema/note_block")) { + <~type> { + u1 as uid + } + } + + var(func: eq(, "$/schema/calendar_event")) { + <~type> { + u2 as uid + } + } + + var(func: eq(, "$/schema/email_message")) { + <~type> { + u3 as uid + } + } + + var(func: eq(, "$/schema/todo")) { + <~type> { + u4 as uid + } + } + + var(func: eq(, "$/schema/contact")) { + <~type> { + u5 as uid + } + } + + var(func: eq(, "$/schema/tag")) { + <~type> { + u6 as uid + } + }`, + ]) +)[0]; + +unigraph.updateClientCache('searchTitles', newTitles); diff --git a/packages/default-packages/unigraph.search/package.json b/packages/default-packages/unigraph.search/package.json new file mode 100644 index 00000000..5031dee5 --- /dev/null +++ b/packages/default-packages/unigraph.search/package.json @@ -0,0 +1,18 @@ +{ + "displayName": "Unigraph search", + "version": "0.2.9", + "description": "Search commons for Unigraph", + "name": "unigraph.search", + "unigraph": { + "executables": [ + { + "id": "initialize-search-cache", + "env": "backend-startup/js", + "src": "executables/initializeSearchCache.js", + "periodic": "* * * * *", + "editable": true, + "name": "Initializes search cache for backend" + } + ] + } +} diff --git a/packages/unigraph-dev-backend/src/caches.ts b/packages/unigraph-dev-backend/src/caches.ts index cd697cf7..fa7d9c58 100644 --- a/packages/unigraph-dev-backend/src/caches.ts +++ b/packages/unigraph-dev-backend/src/caches.ts @@ -1,5 +1,7 @@ // Abstract definition for caches +import stringify from 'json-stable-stringify'; +import { getCircularReplacer } from 'unigraph-dev-common/lib/utils/utils'; import DgraphClient from './dgraphClient'; export type Cache = { @@ -9,4 +11,31 @@ export type Cache = { cacheType: 'subscription' | 'manual'; /* eslint-disable */ // TODO: Temporarily appease the linter, remember to fix it later subscribe(listener: Function): any -} \ No newline at end of file +} + +export function updateClientCache (states: any, key: string, newValue: any) { + states.clientCaches[key] = newValue; + Object.values(states.connections).forEach((conn: any) => { + conn.send(stringify( + { + type: 'cache_updated', + name: key, + result: newValue, + }, + { replacer: getCircularReplacer() }, + ),) + }) +} + +export function initCaches (states: any, conn: any) { + Object.entries(states.clientCaches).forEach(([key, value]: any) => { + conn.send(stringify( + { + type: 'cache_updated', + name: key, + result: value, + }, + { replacer: getCircularReplacer() }, + )); + }); +} diff --git a/packages/unigraph-dev-backend/src/executableManager.ts b/packages/unigraph-dev-backend/src/executableManager.ts index 5e07751e..08c2fda9 100644 --- a/packages/unigraph-dev-backend/src/executableManager.ts +++ b/packages/unigraph-dev-backend/src/executableManager.ts @@ -116,6 +116,10 @@ export function initExecutables( buildExecutable(el, { ...context, definition: el, params }, unigraph, states)(), ); } + if (key.startsWith('0x') && el.env === 'backend-startup/js' && !states.finishedStartups[key]) { + states.finishedStartups[key] = true; + buildExecutable(el, { ...context, definition: el, params: {} }, unigraph, states)(); + } }); states.hooks = _.mergeWith({}, states.defaultHooks, newHooks, mergeWithConcatArray); } @@ -187,6 +191,7 @@ const returnSrcFromEnvClientJs: ExecRunner = (src, context, unigraph) => src; export const environmentRunners = { 'routine/js': runEnvRoutineJs, + 'backend-startup/js': runEnvRoutineJs, 'lambda/js': runEnvLambdaJs, 'component/react-jsx': runEnvReactJSX, 'client/js': returnSrcFromEnvClientJs, // TODO: should we just forbid this in backend? diff --git a/packages/unigraph-dev-backend/src/localUnigraphApi.ts b/packages/unigraph-dev-backend/src/localUnigraphApi.ts index 04599d9e..d1c2d1a7 100644 --- a/packages/unigraph-dev-backend/src/localUnigraphApi.ts +++ b/packages/unigraph-dev-backend/src/localUnigraphApi.ts @@ -27,7 +27,7 @@ import { buildExecutable } from './executableManager'; import { callHooks } from './hooks'; import { addNotification } from './notifications'; import { createSubscriptionLocal, createSubscriptionWS, getFragment, resolveSubscriptionUpdate } from './subscriptions'; -import { Cache } from './caches'; +import { Cache, updateClientCache } from './caches'; import { getQueryString } from './search'; // eslint-disable-next-line import/prefer-default-export @@ -740,6 +740,9 @@ export function getLocalUnigraphAPI( const deleter = (api.deleteItemFromArray as any)(resKeyUid, uids, resUid, [], false); return deleter; }, + updateClientCache(key: string, newValue: any) { + return updateClientCache(states, key, newValue); + }, }; return api; diff --git a/packages/unigraph-dev-backend/src/server.ts b/packages/unigraph-dev-backend/src/server.ts index 3e070b0f..255e9db1 100644 --- a/packages/unigraph-dev-backend/src/server.ts +++ b/packages/unigraph-dev-backend/src/server.ts @@ -63,7 +63,7 @@ import { UnigraphUpsert, } from './custom.d'; import { checkOrCreateDefaultDataModel, createSchemaCache } from './datamodelManager'; -import { Cache } from './caches'; +import { Cache, initCaches, updateClientCache } from './caches'; import { createSubscriptionLocal, MsgCallbackFn, @@ -207,18 +207,7 @@ export default async function startServer(client: DgraphClient) { serverStates.executableSchedule, serverStates, ); - Object.values(connections).forEach((el) => - el.send( - stringify( - { - type: 'cache_updated', - name: 'schemaMap', - result: serverStates.caches.schemas.data, - }, - { replacer: getCircularReplacer() }, - ), - ), - ); + updateClientCache(serverStates, 'schemaMap', serverStates.caches.schemas.data); }, ], after_object_changed: [ @@ -285,9 +274,12 @@ export default async function startServer(client: DgraphClient) { let debounceId: NodeJS.Timeout; Object.assign(serverStates, { + connections, + clientCaches: {}, caches, subscriptions: _subscriptions, dgraphClient, + finishedStartups: {}, hooks, defaultHooks: hooks, namespaceMap, @@ -302,36 +294,14 @@ export default async function startServer(client: DgraphClient) { serverStates.runningExecutables.push(defn); clearTimeout(debounceId); debounceId = setTimeout(() => { - Object.values(connections).forEach((el) => { - el.send( - stringify( - { - type: 'cache_updated', - name: 'runningExecutables', - result: serverStates.runningExecutables, - }, - { replacer: getCircularReplacer() }, - ), - ); - }); + updateClientCache(serverStates, 'runningExecutables', serverStates.runningExecutables); }, 250); }, removeRunningExecutable: (id: any) => { serverStates.runningExecutables = serverStates.runningExecutables.filter((el: any) => el.id !== id); clearTimeout(debounceId); debounceId = setTimeout(() => { - Object.values(connections).forEach((el) => { - el.send( - stringify( - { - type: 'cache_updated', - name: 'runningExecutables', - result: serverStates.runningExecutables, - }, - { replacer: getCircularReplacer() }, - ), - ); - }); + updateClientCache(serverStates, 'runningExecutables', serverStates.runningExecutables); }, 250); }, entityHeadByType: {}, @@ -346,18 +316,7 @@ export default async function startServer(client: DgraphClient) { (data) => { namespaceMap = data[0]; serverStates.namespaceMap = data[0]; - Object.values(connections).forEach((el) => { - el.send( - stringify( - { - type: 'cache_updated', - name: 'namespaceMap', - result: data[0], - }, - { replacer: getCircularReplacer() }, - ), - ); - }); + updateClientCache(serverStates, 'namespaceMap', data[0]); }, { type: 'query', @@ -1046,26 +1005,7 @@ export default async function startServer(client: DgraphClient) { serverStates.leasedUids.push(...(serverStates.clientLeasedUids[connId] || [])); delete serverStates.clientLeasedUids[connId]; }); - ws.send( - stringify( - { - type: 'cache_updated', - name: 'namespaceMap', - result: serverStates.namespaceMap, - }, - { replacer: getCircularReplacer() }, - ), - ); - ws.send( - stringify( - { - type: 'cache_updated', - name: 'schemaMap', - result: serverStates.caches['schemas'].data, - }, - { replacer: getCircularReplacer() }, - ), - ); + initCaches(serverStates, ws); leaseToClient(connId); ws.send( stringify( diff --git a/packages/unigraph-dev-backend/src/templates/defaultDb.ts b/packages/unigraph-dev-backend/src/templates/defaultDb.ts index 3fa10c95..0491b3ba 100644 --- a/packages/unigraph-dev-backend/src/templates/defaultDb.ts +++ b/packages/unigraph-dev-backend/src/templates/defaultDb.ts @@ -13,6 +13,7 @@ import { pkg as calendar } from 'unigraph-dev-common/lib/data/unigraph.calendar. import { pkg as notes } from 'unigraph-dev-common/lib/data/unigraph.notes.pkg'; import { pkg as contacts } from 'unigraph-dev-common/lib/data/unigraph.contacts.pkg'; import { pkg as home } from 'unigraph-dev-common/lib/data/unigraph.home.pkg'; +import { pkg as search } from 'unigraph-dev-common/lib/data/unigraph.search.pkg'; // Userspace packages import { pkg as onboarding } from 'unigraph-dev-common/lib/data/unigraph.onboarding.pkg'; @@ -350,6 +351,7 @@ export const defaultPackages = [ execexample, coreuser, home, + search, calendar, notes, contacts, diff --git a/packages/unigraph-dev-common/src/api/unigraph.ts b/packages/unigraph-dev-common/src/api/unigraph.ts index 063c0708..85465fd2 100644 --- a/packages/unigraph-dev-common/src/api/unigraph.ts +++ b/packages/unigraph-dev-common/src/api/unigraph.ts @@ -397,11 +397,13 @@ export default function unigraph(url: string, browserId: string): Unigraph { + onCacheUpdated: (cache, callback, currentCache) => { if (Array.isArray(cacheCallbacks[cache])) { cacheCallbacks[cache].push(callback); } else cacheCallbacks[cache] = [callback]; + if (currentCache) callback(caches[cache]); }, + getCache: (cache) => caches[cache], createSchema: (schema) => new Promise((resolve, reject) => { const id = getRandomInt(); diff --git a/packages/unigraph-dev-common/src/types/unigraph.ts b/packages/unigraph-dev-common/src/types/unigraph.ts index 353829d1..92391d22 100644 --- a/packages/unigraph-dev-common/src/types/unigraph.ts +++ b/packages/unigraph-dev-common/src/types/unigraph.ts @@ -117,7 +117,8 @@ export interface Unigraph { getStatus(): Promise; /** The specified callback will be invoked once initial Unigraph connecton is established. */ onReady?(callback: () => void): void; - onCacheUpdated?(cache: string, callback: (newEl: any) => void): void; + onCacheUpdated?(cache: string, callback: (newEl: any) => void, currentCache?: boolean): void; + getCache?: (cache: string) => any; /** * Create a new schema using the json-ts format and add it to cache. * @@ -554,5 +555,13 @@ export interface Unigraph { * @param uids The list of uids that have been synced successfully. */ acknowledgeSync(resource: string, key: string, uids: any[]): any; + /** + * Updates client cache with given key and value. + * Clients will be immediately notified of this cache update through websocket. + * + * @param key A string key to identify the cache + * @param newValue Tne new value to set + */ + updateClientCache?(key: string, newValue: any): any; } /** End of unigraph interface */ // Don't remove this line - needed for Monaco to work diff --git a/packages/unigraph-dev-explorer/src/components/UnigraphCore/InlineSearchPopup.tsx b/packages/unigraph-dev-explorer/src/components/UnigraphCore/InlineSearchPopup.tsx index ffe26a15..c49f2a41 100644 --- a/packages/unigraph-dev-explorer/src/components/UnigraphCore/InlineSearchPopup.tsx +++ b/packages/unigraph-dev-explorer/src/components/UnigraphCore/InlineSearchPopup.tsx @@ -45,6 +45,7 @@ const ResultDisplay = ({ el }: any) => { }; export function InlineSearch() { + const [isFulltext, setIsFulltext] = React.useState(false); const [ctxMenuState, setCtxMenuState] = React.useState>>( window.unigraph.getState('global/searchPopup'), ); @@ -60,6 +61,15 @@ export function InlineSearch() { const [currentAction, setCurrentAction] = React.useState(0); + const titleSearch = (key: string) => { + const names = (window.unigraph as any).getCache('searchTitles'); + const results = (names || []).filter((el: any) => el?.name?.toLowerCase().includes(key?.toLowerCase().trim())); + setSearchResults( + results + .sort((a: any, b: any) => new Date(b._updatedAt || 0).getTime() - new Date(a._updatedAt || 0).getTime()) + .slice(0, 100), + ); + }; const search = React.useCallback( _.debounce((key: string) => { if (key !== undefined && key.length > 1) { @@ -103,8 +113,8 @@ export function InlineSearch() { setSearchResults([]); setTopResults([]); } else setCurrentAction(0); - search(state.search as string); - }, [state]); + (isFulltext ? search : titleSearch)(state.search as string); + }, [state.show, state.search, isFulltext]); const [actionItems, setActionItems] = React.useState([]); React.useEffect(() => { From 6666686bae8aed416ccdf7c8b7ed433519cbfd82 Mon Sep 17 00:00:00 2001 From: Sophia Xu Date: Sat, 16 Apr 2022 17:27:27 -0400 Subject: [PATCH 2/3] feat(search): enhanced search flow --- .../executables/initializeSearchCache.js | 1 + .../UnigraphCore/InlineSearchPopup.tsx | 97 +++++++++++++------ packages/unigraph-dev-explorer/src/index.css | 13 +++ .../src/react-app-env.d.ts | 1 + 4 files changed, 80 insertions(+), 32 deletions(-) diff --git a/packages/default-packages/unigraph.search/executables/initializeSearchCache.js b/packages/default-packages/unigraph.search/executables/initializeSearchCache.js index dd4fa6f1..63d76389 100644 --- a/packages/default-packages/unigraph.search/executables/initializeSearchCache.js +++ b/packages/default-packages/unigraph.search/executables/initializeSearchCache.js @@ -3,6 +3,7 @@ const newTitles = ( `(func: uid(u1, u2, u3, u4, u5, u6)) @filter(NOT eq(_hide, true) AND type(Entity)) @normalize { uid _updatedAt: _updatedAt + incoming: count(unigraph.origin) type { type: } { name { diff --git a/packages/unigraph-dev-explorer/src/components/UnigraphCore/InlineSearchPopup.tsx b/packages/unigraph-dev-explorer/src/components/UnigraphCore/InlineSearchPopup.tsx index c49f2a41..0a305427 100644 --- a/packages/unigraph-dev-explorer/src/components/UnigraphCore/InlineSearchPopup.tsx +++ b/packages/unigraph-dev-explorer/src/components/UnigraphCore/InlineSearchPopup.tsx @@ -3,6 +3,7 @@ import React from 'react'; import { AppState } from 'unigraph-dev-common/lib/types/unigraph'; import { UnigraphObject } from 'unigraph-dev-common/lib/api/unigraph'; import _ from 'lodash'; +import Levenshtein from 'levenshtein'; import { parseQuery } from './UnigraphSearch'; import { setSearchPopup } from '../../examples/notes/searchPopup'; import { SearchPopupState } from '../../global.d'; @@ -64,6 +65,14 @@ export function InlineSearch() { const titleSearch = (key: string) => { const names = (window.unigraph as any).getCache('searchTitles'); const results = (names || []).filter((el: any) => el?.name?.toLowerCase().includes(key?.toLowerCase().trim())); + if (key?.length) + setTopResults( + results + .filter((it: any) => it.incoming >= 5) + .sort((a: any, b: any) => b.incoming - a.incoming) + .slice(0, 10), + ); + setSearchResults( results .sort((a: any, b: any) => new Date(b._updatedAt || 0).getTime() - new Date(a._updatedAt || 0).getTime()) @@ -112,6 +121,7 @@ export function InlineSearch() { if (!state.show) { setSearchResults([]); setTopResults([]); + setIsFulltext(false); } else setCurrentAction(0); (isFulltext ? search : titleSearch)(state.search as string); }, [state.show, state.search, isFulltext]); @@ -176,6 +186,10 @@ export function InlineSearch() { ev.preventDefault(); ev.stopPropagation(); ctxMenuState.setValue({ show: false }); + } else if (ev.key === 'f' && (ev.ctrlKey || ev.metaKey)) { + ev.preventDefault(); + ev.stopPropagation(); + setIsFulltext((ft: any) => !ft); } }; @@ -208,42 +222,61 @@ export function InlineSearch() { elevation: 4, style: { maxHeight: '320px', - padding: '10px', + maxWidth: '600px', + padding: '0px', borderRadius: '6px', + display: 'flex', + flexDirection: 'column', }, }} > - {Object.entries(_.groupBy(actionItems, (el: any) => el[2])).map(([key, value]) => { - return ( - <> - {key === 'default' ? ( - [] - ) : ( - - {key === 'top' ? 'Top linked' : 'Recently updated'} - - )} - {value.map((el: any, index: number) => ( -
- {el[0]} -
- ))} - - ); - })} +
+ {Object.entries(_.groupBy(actionItems, (el: any) => el[2])).map(([key, value]) => { + return ( + <> + {key === 'default' ? ( + [] + ) : ( + + {key === 'top' + ? `${isFulltext ? 'Top linked' : 'Relevant'}` + : 'Recently updated'} + + )} + {value.map((el: any, index: number) => ( +
+ {el[0]} +
+ ))} + + ); + })} +
+
+ + ⌘+F {!isFulltext ? 'Fulltext' : 'Title-only'} search + ↑/↓ Navigate + Enter Link text + +
); diff --git a/packages/unigraph-dev-explorer/src/index.css b/packages/unigraph-dev-explorer/src/index.css index f7b28db0..a82323f5 100644 --- a/packages/unigraph-dev-explorer/src/index.css +++ b/packages/unigraph-dev-explorer/src/index.css @@ -12,6 +12,19 @@ body { background-color: var(--app-drawer-background-color); } +kbd { + background-color: #eee; + border-radius: 3px; + border: 1px solid #b4b4b4; + box-shadow: 0 1px 1px rgba(0, 0, 0, .2), 0 2px 0 0 rgba(255, 255, 255, .7) inset; + display: inline-block; + font-size: .85em; + font-weight: 700; + line-height: 1; + padding: 2px 4px; + white-space: nowrap; +} + code { font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', monospace; diff --git a/packages/unigraph-dev-explorer/src/react-app-env.d.ts b/packages/unigraph-dev-explorer/src/react-app-env.d.ts index 592c9800..b2eca750 100644 --- a/packages/unigraph-dev-explorer/src/react-app-env.d.ts +++ b/packages/unigraph-dev-explorer/src/react-app-env.d.ts @@ -80,3 +80,4 @@ declare module '*.pkg' { } declare module 'remark-wiki-link'; +declare module 'levenshtein'; From 4479a065d1f5200c9ef98aa49cc03b2ba3f72b60 Mon Sep 17 00:00:00 2001 From: Sophia Xu Date: Sat, 16 Apr 2022 17:38:39 -0400 Subject: [PATCH 3/3] feat(search): use client-side update for search --- .../src/examples/notes/NoteEditor.tsx | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/unigraph-dev-explorer/src/examples/notes/NoteEditor.tsx b/packages/unigraph-dev-explorer/src/examples/notes/NoteEditor.tsx index cccc9063..d726a73c 100644 --- a/packages/unigraph-dev-explorer/src/examples/notes/NoteEditor.tsx +++ b/packages/unigraph-dev-explorer/src/examples/notes/NoteEditor.tsx @@ -228,7 +228,7 @@ export const useNoteEditor: (...args: any) => [any, (text: string) => void, () = getCurrentText(), textInputRef, caret, - async (match: any, newName: string, newUid: string) => { + async (match: any, newName: string, newUid: string, newType: string) => { const parents = getParentsAndReferences( dataRef.current['~_value'], dataRef.current['unigraph.origin'] || [], @@ -244,11 +244,13 @@ export const useNoteEditor: (...args: any) => [any, (text: string) => void, () = edited.current = true; // resetEdited(); setCaret(document, textInputRef.current, match.index + newName.length + 4); - await window.unigraph.updateObject( + window.unigraph.updateObject( locateInlineChildren(dataRef.current).uid, { _value: { + uid: locateInlineChildren(dataRef.current)._value.uid, children: { + uid: locateInlineChildren(dataRef.current)._value.children?.uid, '_value[': [ { _key: `[[${newName}]]`, @@ -256,17 +258,18 @@ export const useNoteEditor: (...args: any) => [any, (text: string) => void, () = 'dgraph.type': ['Interface'], type: { 'unigraph.id': '$/schema/interface/semantic' }, _hide: true, - _value: { uid: newUid }, + _value: { uid: newUid, type: { 'unigraph.id': newType } }, }, }, ], }, }, }, - true, + false, false, callbacks.subsId, parents, + true, ); touchParents(data); window.unigraph.getState('global/searchPopup').setValue({ show: false });