diff --git a/keyserver/src/fetchers/message-fetchers.js b/keyserver/src/fetchers/message-fetchers.js index 5d0ca242c..afb679565 100644 --- a/keyserver/src/fetchers/message-fetchers.js +++ b/keyserver/src/fetchers/message-fetchers.js @@ -1,1061 +1,1048 @@ // @flow import invariant from 'invariant'; import { sortMessageInfoList, shimUnsupportedRawMessageInfos, isInvalidSidebarSource, isUnableToBeRenderedIndependently, isInvalidPinSource, } from 'lib/shared/message-utils.js'; import { messageSpecs } from 'lib/shared/messages/message-specs.js'; import { getNotifCollapseKey } from 'lib/shared/notif-utils.js'; -import { hasMinCodeVersion } from 'lib/shared/version-utils.js'; import { messageTypes, type MessageType, assertMessageType, } from 'lib/types/message-types-enum.js'; import { type RawMessageInfo, type RawComposableMessageInfo, type RawRobotextMessageInfo, type EditMessageContent, type MessageSelectionCriteria, type MessageTruncationStatus, messageTruncationStatus, type FetchMessageInfosResult, defaultMaxMessageAge, type FetchPinnedMessagesRequest, type FetchPinnedMessagesResult, type SearchMessagesResponse, type MessageTruncationStatuses, } from 'lib/types/message-types.js'; import { defaultNumberPerThread } from 'lib/types/message-types.js'; import { threadPermissions } from 'lib/types/thread-permission-types.js'; import { ServerError } from 'lib/utils/errors.js'; import { constructMediaFromMediaMessageContentsAndUploadRows, imagesFromRow, } from './upload-fetchers.js'; import { dbQuery, SQL, mergeOrConditions, mergeAndConditions, } from '../database/database.js'; import { processQueryForSearch } from '../database/search-utils.js'; import type { SQLStatementType } from '../database/types.js'; import type { PushInfo } from '../push/send.js'; import type { Viewer } from '../session/viewer.js'; import { creationString, localIDFromCreationString, } from '../utils/idempotent.js'; export type CollapsableNotifInfo = { collapseKey: ?string, existingMessageInfos: RawMessageInfo[], newMessageInfos: RawMessageInfo[], }; export type FetchCollapsableNotifsResult = { [userID: string]: CollapsableNotifInfo[], }; const visibleExtractString = `$.${threadPermissions.VISIBLE}.value`; // This function doesn't filter RawMessageInfos based on what messageTypes the // client supports, since each user can have multiple clients. The caller must // handle this filtering. async function fetchCollapsableNotifs( pushInfo: PushInfo, ): Promise { // First, we need to fetch any notifications that should be collapsed const usersToCollapseKeysToInfo: { [string]: { [string]: CollapsableNotifInfo }, } = {}; const usersToCollapsableNotifInfo: { [string]: Array } = {}; for (const userID in pushInfo) { usersToCollapseKeysToInfo[userID] = {}; usersToCollapsableNotifInfo[userID] = []; for (let i = 0; i < pushInfo[userID].messageInfos.length; i++) { const rawMessageInfo = pushInfo[userID].messageInfos[i]; const messageData = pushInfo[userID].messageDatas[i]; const collapseKey = getNotifCollapseKey(rawMessageInfo, messageData); if (!collapseKey) { const collapsableNotifInfo: CollapsableNotifInfo = { collapseKey, existingMessageInfos: [], newMessageInfos: [rawMessageInfo], }; usersToCollapsableNotifInfo[userID].push(collapsableNotifInfo); continue; } if (!usersToCollapseKeysToInfo[userID][collapseKey]) { usersToCollapseKeysToInfo[userID][collapseKey] = ({ collapseKey, existingMessageInfos: [], newMessageInfos: [], }: CollapsableNotifInfo); } usersToCollapseKeysToInfo[userID][collapseKey].newMessageInfos.push( rawMessageInfo, ); } } const sqlTuples = []; for (const userID in usersToCollapseKeysToInfo) { const collapseKeysToInfo = usersToCollapseKeysToInfo[userID]; for (const collapseKey in collapseKeysToInfo) { sqlTuples.push( SQL`(n.user = ${userID} AND n.collapse_key = ${collapseKey})`, ); } } if (sqlTuples.length === 0) { return usersToCollapsableNotifInfo; } const collapseQuery = SQL` SELECT m.id, m.thread AS threadID, m.content, m.time, m.type, m.user AS creatorID, m.target_message as targetMessageID, stm.permissions AS subthread_permissions, n.user, n.collapse_key, up.id AS uploadID, up.type AS uploadType, up.secret AS uploadSecret, up.extra AS uploadExtra FROM notifications n LEFT JOIN messages m ON m.id = n.message LEFT JOIN uploads up ON up.container = m.id LEFT JOIN memberships mm ON mm.thread = m.thread AND mm.user = n.user LEFT JOIN memberships stm ON m.type = ${messageTypes.CREATE_SUB_THREAD} AND stm.thread = m.content AND stm.user = n.user WHERE n.rescinded = 0 AND JSON_EXTRACT(mm.permissions, ${visibleExtractString}) IS TRUE AND `; collapseQuery.append(mergeOrConditions(sqlTuples)); collapseQuery.append(SQL`ORDER BY m.time DESC, m.id DESC`); const [collapseResult] = await dbQuery(collapseQuery); const rowsByUser = new Map>(); for (const row of collapseResult) { const user = row.user.toString(); const currentRowsForUser = rowsByUser.get(user); if (currentRowsForUser) { currentRowsForUser.push(row); } else { rowsByUser.set(user, [row]); } } const derivedMessages = await fetchDerivedMessages(collapseResult); for (const userRows of rowsByUser.values()) { const messages = parseMessageSQLResult(userRows, derivedMessages); for (const message of messages) { const { rawMessageInfo, rows } = message; const [row] = rows; const userID = row.user.toString(); const collapseKey = row.collapse_key; invariant( collapseKey !== null && collapseKey !== undefined, 'We expect all collapseQuery results to match on a collapseKey', ); const info = usersToCollapseKeysToInfo[userID][collapseKey]; info.existingMessageInfos.push(rawMessageInfo); } } for (const userID in usersToCollapseKeysToInfo) { const collapseKeysToInfo = usersToCollapseKeysToInfo[userID]; for (const collapseKey in collapseKeysToInfo) { const info = collapseKeysToInfo[collapseKey]; usersToCollapsableNotifInfo[userID].push({ collapseKey: info.collapseKey, existingMessageInfos: sortMessageInfoList(info.existingMessageInfos), newMessageInfos: sortMessageInfoList(info.newMessageInfos), }); } } return usersToCollapsableNotifInfo; } type MessageSQLResultRow = { +id: ?number, +threadID: ?number, +content: ?string, +time: ?number, +type: ?number, +creatorID: ?number, +targetMessageID: ?number, +subthread_permissions: ?string, +user: number, +collapse_key: ?string, +uploadID: ?number, +uploadType: ?number, +uploadSecret: ?string, +uploadExtra: ?string, }; type MessageSQLResult = $ReadOnlyArray<{ +rawMessageInfo: RawMessageInfo, +rows: $ReadOnlyArray, }>; function parseMessageSQLResult( rows: $ReadOnlyArray, derivedMessages: $ReadOnlyMap< string, RawComposableMessageInfo | RawRobotextMessageInfo, >, viewer?: Viewer, ): MessageSQLResult { const rowsByID = new Map>(); for (const row of rows) { if (!row.id) { continue; } const id = row.id.toString(); const currentRowsForID = rowsByID.get(id); if (currentRowsForID) { currentRowsForID.push(row); } else { rowsByID.set(id, [row]); } } const messages = []; for (const messageRows of rowsByID.values()) { const rawMessageInfo = rawMessageInfoFromRows( messageRows, viewer, derivedMessages, ); if (rawMessageInfo) { messages.push({ rawMessageInfo, rows: messageRows }); } } return messages; } function assertSingleRow(rows: $ReadOnlyArray): Object { if (rows.length === 0) { throw new Error('expected single row, but none present!'); } else if (rows.length !== 1) { const messageIDs = rows.map(row => row.id.toString()); console.warn( `expected single row, but there are multiple! ${messageIDs.join(', ')}`, ); } return rows[0]; } function mostRecentRowType(rows: $ReadOnlyArray): MessageType { if (rows.length === 0) { throw new Error('expected row, but none present!'); } return assertMessageType(rows[0].type); } function rawMessageInfoFromRows( rawRows: $ReadOnlyArray, viewer?: Viewer, derivedMessages: $ReadOnlyMap< string, RawComposableMessageInfo | RawRobotextMessageInfo, >, ): ?RawMessageInfo { const rows = rawRows.map(row => ({ ...row, subthread_permissions: JSON.parse(row.subthread_permissions), })); const type = mostRecentRowType(rows); const messageSpec = messageSpecs[type]; const requiresDerivedMessages = messageSpec.parseDerivedMessages !== undefined; if (type === messageTypes.IMAGES || type === messageTypes.MULTIMEDIA) { let media; if (type === messageTypes.MULTIMEDIA) { const mediaMessageContents = JSON.parse(rows[0].content); media = constructMediaFromMediaMessageContentsAndUploadRows( mediaMessageContents, rows, ); } else { media = rows.filter(row => row.uploadID).map(imagesFromRow); } const [row] = rows; const localID = localIDFromCreationString(viewer, row.creation); let rawMessageInfoFromServerDBRowParams = { localID, media }; if (requiresDerivedMessages) { rawMessageInfoFromServerDBRowParams = { ...rawMessageInfoFromServerDBRowParams, derivedMessages, }; } invariant( messageSpec.rawMessageInfoFromServerDBRow, `multimedia message spec should have rawMessageInfoFromServerDBRow`, ); return messageSpec.rawMessageInfoFromServerDBRow( row, rawMessageInfoFromServerDBRowParams, ); } const row = assertSingleRow(rows); const localID = localIDFromCreationString(viewer, row.creation); let rawMessageInfoFromServerDBRowParams = { localID }; if (requiresDerivedMessages) { rawMessageInfoFromServerDBRowParams = { ...rawMessageInfoFromServerDBRowParams, derivedMessages, }; } invariant( messageSpec.rawMessageInfoFromServerDBRow, `message spec ${type} should have rawMessageInfoFromServerDBRow`, ); return messageSpec.rawMessageInfoFromServerDBRow( row, rawMessageInfoFromServerDBRowParams, ); } async function fetchMessageInfos( viewer: Viewer, criteria: MessageSelectionCriteria, numberPerThread: number, ): Promise { const { sqlClause: selectionClause, timeFilterData } = parseMessageSelectionCriteria(viewer, criteria); const truncationStatuses: MessageTruncationStatuses = {}; const viewerID = viewer.id; const query = SQL` WITH thread_window AS ( SELECT m.id, m.thread AS threadID, m.user AS creatorID, m.target_message as targetMessageID, m.content, m.time, m.type, m.creation, stm.permissions AS subthread_permissions, ROW_NUMBER() OVER ( PARTITION BY threadID ORDER BY m.time DESC, m.id DESC ) n FROM messages m LEFT JOIN memberships mm ON mm.thread = m.thread AND mm.user = ${viewerID} LEFT JOIN memberships stm ON m.type = ${messageTypes.CREATE_SUB_THREAD} AND stm.thread = m.content AND stm.user = ${viewerID} WHERE JSON_EXTRACT(mm.permissions, ${visibleExtractString}) IS TRUE AND `; query.append(selectionClause); query.append(SQL` ) SELECT tw.*, up.id AS uploadID, up.type AS uploadType, up.secret AS uploadSecret, up.extra AS uploadExtra FROM thread_window tw LEFT JOIN uploads up ON up.container = tw.id WHERE tw.n <= ${numberPerThread} ORDER BY tw.threadID, tw.time DESC, tw.id DESC `); const [result] = await dbQuery(query); const derivedMessages = await fetchDerivedMessages(result, viewer); const messages = parseMessageSQLResult(result, derivedMessages, viewer); const rawMessageInfos = []; const threadToMessageCount = new Map(); for (const message of messages) { const { rawMessageInfo } = message; rawMessageInfos.push(rawMessageInfo); const { threadID } = rawMessageInfo; const currentCountValue = threadToMessageCount.get(threadID); const currentCount = currentCountValue ? currentCountValue : 0; threadToMessageCount.set(threadID, currentCount + 1); } for (const [threadID, messageCount] of threadToMessageCount) { // If we matched the exact amount we limited to, we're probably truncating // our result set. By setting TRUNCATED here, we tell the client that the // result set might not be continguous with what's already in their // MessageStore. More details about TRUNCATED can be found in // lib/types/message-types.js if (messageCount >= numberPerThread) { // We won't set TRUNCATED if a cursor was specified for a given thread, // since then the result is guaranteed to be contiguous with what the // client has if (criteria.threadCursors && criteria.threadCursors[threadID]) { truncationStatuses[threadID] = messageTruncationStatus.UNCHANGED; } else { truncationStatuses[threadID] = messageTruncationStatus.TRUNCATED; } continue; } const hasTimeFilter = hasTimeFilterForThread(timeFilterData, threadID); if (!hasTimeFilter) { // If there is no time filter for a given thread, and there are fewer // messages returned than the max we queried for a given thread, we can // conclude that our result set includes all messages for that thread truncationStatuses[threadID] = messageTruncationStatus.EXHAUSTIVE; } } for (const rawMessageInfo of rawMessageInfos) { if (messageSpecs[rawMessageInfo.type].startsThread) { truncationStatuses[rawMessageInfo.threadID] = messageTruncationStatus.EXHAUSTIVE; } } for (const threadID in criteria.threadCursors) { const truncationStatus = truncationStatuses[threadID]; if (truncationStatus !== null && truncationStatus !== undefined) { continue; } const hasTimeFilter = hasTimeFilterForThread(timeFilterData, threadID); if (!hasTimeFilter) { // If there is no time filter for a given thread, and zero messages were // returned, we can conclude that this thread has zero messages. This is // a case of database corruption that should not be possible, but likely // we have some threads like this on prod (either due to some transient // issues or due to old buggy code) truncationStatuses[threadID] = messageTruncationStatus.EXHAUSTIVE; } else { // If this thread was explicitly queried for, and we got no results, but // we can't conclude that it's EXHAUSTIVE, then we'll set to UNCHANGED. truncationStatuses[threadID] = messageTruncationStatus.UNCHANGED; } } const shimmedRawMessageInfos = shimUnsupportedRawMessageInfos( rawMessageInfos, viewer.platformDetails, ); return { rawMessageInfos: shimmedRawMessageInfos, truncationStatuses, }; } // ESLint doesn't recognize that invariant always throws // eslint-disable-next-line consistent-return function hasTimeFilterForThread( timeFilterData: TimeFilterData, threadID: string, ) { if (timeFilterData.timeFilter === 'ALL') { return true; - } else if (timeFilterData.timeFilter === 'NONE') { - return false; } else if (timeFilterData.timeFilter === 'ALL_EXCEPT_EXCLUDED') { return !timeFilterData.excludedFromTimeFilter.has(threadID); } else { invariant( false, `unrecognized timeFilter type ${timeFilterData.timeFilter}`, ); } } type TimeFilterData = - | { +timeFilter: 'ALL' | 'NONE' } + | { +timeFilter: 'ALL' } | { +timeFilter: 'ALL_EXCEPT_EXCLUDED', +excludedFromTimeFilter: $ReadOnlySet, }; type ParsedMessageSelectionCriteria = { +sqlClause: SQLStatementType, +timeFilterData: TimeFilterData, }; function parseMessageSelectionCriteria( viewer: Viewer, criteria: MessageSelectionCriteria, ): ParsedMessageSelectionCriteria { const minMessageTime = Date.now() - defaultMaxMessageAge; - const shouldApplyTimeFilter = hasMinCodeVersion(viewer.platformDetails, { - native: 130, - }); - let globalTimeFilter; if (criteria.newerThan) { globalTimeFilter = SQL`m.time > ${criteria.newerThan}`; - } else if (!criteria.threadCursors && shouldApplyTimeFilter) { + } else if (!criteria.threadCursors) { globalTimeFilter = SQL` (m.time > ${minMessageTime} OR m.id = mm.last_message) `; } const threadConditions = []; - if ( - criteria.joinedThreads === true && - shouldApplyTimeFilter && - !globalTimeFilter - ) { + if (criteria.joinedThreads === true && !globalTimeFilter) { threadConditions.push(SQL` (mm.role > 0 AND (m.time > ${minMessageTime} OR m.id = mm.last_message)) `); } else if (criteria.joinedThreads === true) { threadConditions.push(SQL`mm.role > 0`); } if (criteria.threadCursors) { for (const threadID in criteria.threadCursors) { const cursor = criteria.threadCursors[threadID]; if (cursor) { threadConditions.push( SQL`(m.thread = ${threadID} AND m.id < ${cursor})`, ); } else { threadConditions.push(SQL`m.thread = ${threadID}`); } } } if (threadConditions.length === 0) { throw new ServerError('internal_error'); } const threadClause = mergeOrConditions(threadConditions); let timeFilterData; if (globalTimeFilter) { timeFilterData = { timeFilter: 'ALL' }; - } else if (!shouldApplyTimeFilter) { - timeFilterData = { timeFilter: 'NONE' }; } else { invariant( criteria.threadCursors, 'ALL_EXCEPT_EXCLUDED should correspond to threadCursors being set', ); const excludedFromTimeFilter = new Set(Object.keys(criteria.threadCursors)); timeFilterData = { timeFilter: 'ALL_EXCEPT_EXCLUDED', excludedFromTimeFilter, }; } const conditions = [globalTimeFilter, threadClause].filter(Boolean); const sqlClause = mergeAndConditions(conditions); return { sqlClause, timeFilterData }; } function messageSelectionCriteriaToInitialTruncationStatuses( criteria: MessageSelectionCriteria, defaultTruncationStatus: MessageTruncationStatus, ) { const truncationStatuses: MessageTruncationStatuses = {}; if (criteria.threadCursors) { for (const threadID in criteria.threadCursors) { truncationStatuses[threadID] = defaultTruncationStatus; } } return truncationStatuses; } async function fetchMessageInfosSince( viewer: Viewer, criteria: MessageSelectionCriteria, maxNumberPerThread: number, ): Promise { const { sqlClause: selectionClause } = parseMessageSelectionCriteria( viewer, criteria, ); const truncationStatuses = messageSelectionCriteriaToInitialTruncationStatuses( criteria, messageTruncationStatus.UNCHANGED, ); const viewerID = viewer.id; const query = SQL` SELECT m.id, m.thread AS threadID, m.content, m.time, m.type, m.creation, m.user AS creatorID, m.target_message as targetMessageID, stm.permissions AS subthread_permissions, up.id AS uploadID, up.type AS uploadType, up.secret AS uploadSecret, up.extra AS uploadExtra FROM messages m LEFT JOIN uploads up ON up.container = m.id LEFT JOIN memberships mm ON mm.thread = m.thread AND mm.user = ${viewerID} LEFT JOIN memberships stm ON m.type = ${messageTypes.CREATE_SUB_THREAD} AND stm.thread = m.content AND stm.user = ${viewerID} WHERE JSON_EXTRACT(mm.permissions, ${visibleExtractString}) IS TRUE AND `; query.append(selectionClause); query.append(SQL` ORDER BY m.thread, m.time DESC, m.id DESC `); const [result] = await dbQuery(query); const derivedMessages = await fetchDerivedMessages(result, viewer); const messages = parseMessageSQLResult(result, derivedMessages, viewer); const rawMessageInfos = []; let currentThreadID = null; let numMessagesForCurrentThreadID = 0; for (const message of messages) { const { rawMessageInfo } = message; const { threadID } = rawMessageInfo; if (threadID !== currentThreadID) { currentThreadID = threadID; numMessagesForCurrentThreadID = 1; truncationStatuses[threadID] = messageTruncationStatus.UNCHANGED; } else { numMessagesForCurrentThreadID++; } if (numMessagesForCurrentThreadID <= maxNumberPerThread) { if (messageSpecs[rawMessageInfo.type].startsThread) { truncationStatuses[threadID] = messageTruncationStatus.EXHAUSTIVE; } rawMessageInfos.push(rawMessageInfo); } else if (numMessagesForCurrentThreadID === maxNumberPerThread + 1) { truncationStatuses[threadID] = messageTruncationStatus.TRUNCATED; } } const shimmedRawMessageInfos = shimUnsupportedRawMessageInfos( rawMessageInfos, viewer.platformDetails, ); return { rawMessageInfos: shimmedRawMessageInfos, truncationStatuses, }; } function getMessageFetchResultFromRedisMessages( viewer: Viewer, rawMessageInfos: $ReadOnlyArray, ): FetchMessageInfosResult { const truncationStatuses: MessageTruncationStatuses = {}; for (const rawMessageInfo of rawMessageInfos) { truncationStatuses[rawMessageInfo.threadID] = messageTruncationStatus.UNCHANGED; } const shimmedRawMessageInfos = shimUnsupportedRawMessageInfos( rawMessageInfos, viewer.platformDetails, ); return { rawMessageInfos: shimmedRawMessageInfos, truncationStatuses, }; } async function fetchMessageInfoForLocalID( viewer: Viewer, localID: ?string, ): Promise { if (!localID || !viewer.hasSessionInfo) { return null; } const creation = creationString(viewer, localID); const viewerID = viewer.id; const query = SQL` SELECT m.id, m.thread AS threadID, m.content, m.time, m.type, m.creation, m.user AS creatorID, m.target_message as targetMessageID, stm.permissions AS subthread_permissions, up.id AS uploadID, up.type AS uploadType, up.secret AS uploadSecret, up.extra AS uploadExtra FROM messages m LEFT JOIN uploads up ON up.container = m.id LEFT JOIN memberships mm ON mm.thread = m.thread AND mm.user = ${viewerID} LEFT JOIN memberships stm ON m.type = ${messageTypes.CREATE_SUB_THREAD} AND stm.thread = m.content AND stm.user = ${viewerID} WHERE m.user = ${viewerID} AND m.creation = ${creation} AND JSON_EXTRACT(mm.permissions, ${visibleExtractString}) IS TRUE `; const [result] = await dbQuery(query); if (result.length === 0) { return null; } const derivedMessages = await fetchDerivedMessages(result, viewer); return rawMessageInfoFromRows(result, viewer, derivedMessages); } const entryIDExtractString = '$.entryID'; async function fetchMessageInfoForEntryAction( viewer: Viewer, messageType: MessageType, entryID: string, threadID: string, ): Promise { const viewerID = viewer.id; const query = SQL` SELECT m.id, m.thread AS threadID, m.content, m.time, m.type, m.creation, m.user AS creatorID, m.target_message AS targetMessageID, NULL AS subthread_permissions, up.id AS uploadID, up.type AS uploadType, up.secret AS uploadSecret, up.extra AS uploadExtra FROM messages m LEFT JOIN uploads up ON up.container = m.id LEFT JOIN memberships mm ON mm.thread = m.thread AND mm.user = ${viewerID} WHERE m.user = ${viewerID} AND m.thread = ${threadID} AND m.type = ${messageType} AND JSON_EXTRACT(m.content, ${entryIDExtractString}) = ${entryID} AND JSON_EXTRACT(mm.permissions, ${visibleExtractString}) IS TRUE `; const [result] = await dbQuery(query); if (result.length === 0) { return null; } const derivedMessages = await fetchDerivedMessages(result, viewer); return rawMessageInfoFromRows(result, viewer, derivedMessages); } async function fetchMessageRowsByIDs(messageIDs: $ReadOnlyArray) { const query = SQL` SELECT m.id, m.thread AS threadID, m.content, m.time, m.type, m.creation, m.user AS creatorID, m.target_message as targetMessageID, stm.permissions AS subthread_permissions, up.id AS uploadID, up.type AS uploadType, up.secret AS uploadSecret, up.extra AS uploadExtra FROM messages m LEFT JOIN uploads up ON up.container = m.id LEFT JOIN memberships stm ON m.type = ${messageTypes.CREATE_SUB_THREAD} AND stm.thread = m.content AND stm.user = m.user WHERE m.id IN (${messageIDs}) `; const [result] = await dbQuery(query); return result; } async function fetchPinnedMessageInfos( viewer: Viewer, request: FetchPinnedMessagesRequest, ): Promise { // The only message types that can be pinned are 0, 14, and 15 // (text, images, and multimedia), so we don't need to worry about // an admin pinning a message about creating a secret subchannel. This is // why we don't check subthread permissions (as opposed to other queries). const messageRowsQuery = SQL` SELECT m.id, m.thread AS threadID, m.content, m.time, m.type, m.creation, m.user AS creatorID, m.target_message as targetMessageID, NULL AS subthread_permissions, u.id AS uploadID, u.type AS uploadType, u.secret AS uploadSecret, u.extra AS uploadExtra FROM messages m LEFT JOIN uploads u ON u.container = m.id LEFT JOIN memberships mm ON mm.thread = m.thread AND mm.user = ${viewer.id} WHERE m.thread = ${request.threadID} AND m.pinned = 1 AND JSON_EXTRACT(mm.permissions, ${visibleExtractString}) IS TRUE ORDER BY m.pin_time DESC `; const [messageRows] = await dbQuery(messageRowsQuery); if (messageRows.length === 0) { return { pinnedMessages: [] }; } const pinnedAndRelatedMessages = await rawMessageInfoForRowsAndRelatedMessages(messageRows, viewer); const shimmedPinnedRawMessageInfos = shimUnsupportedRawMessageInfos( pinnedAndRelatedMessages, viewer.platformDetails, ); return { pinnedMessages: shimmedPinnedRawMessageInfos, }; } async function fetchDerivedMessages( rows: $ReadOnlyArray, viewer?: Viewer, ): Promise< $ReadOnlyMap, > { const requiredIDs = new Set(); for (const row of rows) { // parseDerivedMessages should be defined for SIDEBAR_SOURCE and TOGGLE_PIN const { parseDerivedMessages } = messageSpecs[row.type]; parseDerivedMessages?.(row, requiredIDs); } const messagesByID = new Map< string, RawComposableMessageInfo | RawRobotextMessageInfo, >(); if (requiredIDs.size === 0) { return messagesByID; } const [result, edits] = await Promise.all([ fetchMessageRowsByIDs([...requiredIDs]), fetchLatestEditMessageContentByIDs([...requiredIDs]), ]); const messages = parseMessageSQLResult(result, new Map(), viewer); for (const message of messages) { let { rawMessageInfo } = message; if (rawMessageInfo.type === messageTypes.SIDEBAR_SOURCE) { invariant( !isInvalidSidebarSource(rawMessageInfo), 'SIDEBAR_SOURCE should not point to a ' + 'SIDEBAR_SOURCE, REACTION, EDIT_MESSAGE or TOGGLE_PIN', ); } if (rawMessageInfo.type === messageTypes.TOGGLE_PIN) { invariant( !isInvalidPinSource(rawMessageInfo), 'TOGGLE_PIN should not point to a non-composable message type', ); } if (rawMessageInfo.id) { const editedContent = edits.get(rawMessageInfo.id); if (editedContent && rawMessageInfo.type === messageTypes.TEXT) { rawMessageInfo = { ...rawMessageInfo, text: editedContent.text, }; } invariant(rawMessageInfo.id, 'rawMessageInfo.id should not be null'); // Flow doesn't refine the types if we don't explicitly invariant on // or check against all of the unexpected message types, and that list // can both get long and generally defeats the purpose of moving the // logic into message specs to have one 'single source of truth'. // $FlowFixMe messagesByID.set(rawMessageInfo.id, rawMessageInfo); } } return messagesByID; } async function fetchMessageInfoByID( viewer?: Viewer, messageID: string, ): Promise { const result = await fetchMessageRowsByIDs([messageID]); if (result.length === 0) { return null; } const derivedMessages = await fetchDerivedMessages(result, viewer); return rawMessageInfoFromRows(result, viewer, derivedMessages); } async function fetchThreadMessagesCount(threadID: string): Promise { const query = SQL` SELECT COUNT(*) AS count FROM messages WHERE thread = ${threadID} `; const [result] = await dbQuery(query); return result[0].count; } async function fetchLatestEditMessageContentByIDs( messageIDs: $ReadOnlyArray, ): Promise<$ReadOnlyMap> { const latestEditedMessageQuery = SQL` SELECT m.id, ( SELECT m2.content FROM messages m2 WHERE m.id = m2.target_message AND m.thread = m2.thread AND m2.type = ${messageTypes.EDIT_MESSAGE} ORDER BY time DESC, id DESC LIMIT 1 ) content FROM messages m WHERE m.id IN(${messageIDs}) `; const [result] = await dbQuery(latestEditedMessageQuery); const latestContentByID = new Map(); for (const row of result) { if (!row.content) { continue; } const content = JSON.parse(row.content); latestContentByID.set(row.id.toString(), content); } return latestContentByID; } async function fetchLatestEditMessageContentByID( messageID: string, ): Promise { const result = await fetchLatestEditMessageContentByIDs([messageID]); const content = result.get(messageID); return content; } async function fetchRelatedMessages( viewer?: Viewer, messages: $ReadOnlyMap< string, RawComposableMessageInfo | RawRobotextMessageInfo, >, ): Promise<$ReadOnlyArray> { if (messages.size === 0) { return []; } const originalMessageIDs = [...messages.keys()]; const query = SQL` SELECT m.id, m.thread AS threadID, m.content, m.time, m.type, m.creation, m.user AS creatorID, m.target_message as targetMessageID, stm.permissions AS subthread_permissions, up.id AS uploadID, up.type AS uploadType, up.secret AS uploadSecret, up.extra AS uploadExtra FROM messages m LEFT JOIN uploads up ON up.container = m.id LEFT JOIN memberships stm ON m.type = ${messageTypes.CREATE_SUB_THREAD} AND stm.thread = m.content AND stm.user = m.user WHERE m.target_message IN (${originalMessageIDs}) AND ( m.type = ${messageTypes.SIDEBAR_SOURCE} OR m.type = ${messageTypes.REACTION} OR m.type = ${messageTypes.TOGGLE_PIN} ) UNION SELECT m.id, m.thread AS threadID, m.content, m.time, m.type, m.creation, m.user AS creatorID, m.target_message as targetMessageID, stm.permissions AS subthread_permissions, up.id AS uploadID, up.type AS uploadType, up.secret AS uploadSecret, up.extra AS uploadExtra FROM messages m2 INNER JOIN messages m ON m.id = ( SELECT m3.id FROM messages m3 WHERE m3.target_message = m2.id AND m3.thread = m2.thread AND m3.type = ${messageTypes.EDIT_MESSAGE} ORDER BY time DESC, id DESC LIMIT 1 ) LEFT JOIN uploads up ON up.container = m2.id LEFT JOIN memberships stm ON m2.type = ${messageTypes.CREATE_SUB_THREAD} AND stm.thread = m2.content AND stm.user = m2.user WHERE m2.id IN (${originalMessageIDs}) `; const [resultRows] = await dbQuery(query); if (resultRows.length === 0) { return []; } const SQLResult = parseMessageSQLResult(resultRows, messages, viewer); return SQLResult.map(item => item.rawMessageInfo); } async function rawMessageInfoForRowsAndRelatedMessages( rows: $ReadOnlyArray, viewer?: Viewer, ): Promise<$ReadOnlyArray> { const parsedResults = parseMessageSQLResult(rows, new Map(), viewer); const rawMessageInfoMap = new Map< string, RawComposableMessageInfo | RawRobotextMessageInfo, >(); for (const message of parsedResults) { const { rawMessageInfo } = message; if (isUnableToBeRenderedIndependently(rawMessageInfo)) { continue; } invariant(rawMessageInfo.id, 'rawMessageInfo.id should not be null'); // Flow fails to refine types correctly since // isUnableToBeRenderedIndependently introspects into a message spec // instead of directly checking message types. We use "continue" to avoid // invalid messages, but Flow doesn't recognize this. The // alternative is to check against every message type, but that defeats // the purpose of a 'single source of truth.' // $FlowFixMe rawMessageInfoMap.set(rawMessageInfo.id, rawMessageInfo); } const rawMessageInfos = parsedResults.map(item => item.rawMessageInfo); const rawRelatedMessageInfos = await fetchRelatedMessages( viewer, rawMessageInfoMap, ); return [...rawMessageInfos, ...rawRelatedMessageInfos]; } const searchMessagesPageSize = defaultNumberPerThread + 1; async function searchMessagesInSingleChat( inputQuery: string, threadID: string, viewer?: Viewer, cursor?: ?string, ): Promise { if (inputQuery === '') { console.warn('received empty search query'); return { messages: [], endReached: true }; } const pattern = processQueryForSearch(inputQuery); if (pattern === '') { return { endReached: true, messages: [], }; } const query = SQL` SELECT m.id, m.thread AS threadID, m.content, m.time, m.type, m.creation, m.user AS creatorID, m.target_message as targetMessageID, stm.permissions AS subthread_permissions, up.id AS uploadID, up.type AS uploadType, up.secret AS uploadSecret, up.extra AS uploadExtra FROM message_search s LEFT JOIN messages m ON m.id = s.original_message_id LEFT JOIN memberships stm ON m.type = ${messageTypes.CREATE_SUB_THREAD} AND stm.thread = m.content AND stm.user = m.user LEFT JOIN uploads up ON up.container = m.id LEFT JOIN messages m2 ON m2.target_message = m.id AND m2.type = ${messageTypes.SIDEBAR_SOURCE} AND m2.thread = ${threadID} WHERE MATCH(s.processed_content) AGAINST(${pattern} IN BOOLEAN MODE) AND (m.thread = ${threadID} OR m2.id IS NOT NULL) `; if (cursor) { query.append(SQL`AND m.id < ${cursor} `); } query.append(SQL` ORDER BY m.time DESC, m.id DESC LIMIT ${searchMessagesPageSize} `); const [results] = await dbQuery(query); if (results.length === 0) { return { messages: [], endReached: true }; } const endReached = results.length < searchMessagesPageSize; const resultsPage = endReached ? results : results.slice(0, -1); const rawMessageInfos = await rawMessageInfoForRowsAndRelatedMessages( resultsPage, viewer, ); return { messages: shimUnsupportedRawMessageInfos( rawMessageInfos, viewer?.platformDetails, ), endReached: endReached, }; } export { fetchCollapsableNotifs, fetchMessageInfos, fetchMessageInfosSince, getMessageFetchResultFromRedisMessages, fetchMessageInfoForLocalID, fetchMessageInfoForEntryAction, fetchMessageInfoByID, fetchThreadMessagesCount, fetchLatestEditMessageContentByID, fetchPinnedMessageInfos, searchMessagesInSingleChat, }; diff --git a/keyserver/src/push/rescind.js b/keyserver/src/push/rescind.js index 8f9778e0a..99d0ac5ee 100644 --- a/keyserver/src/push/rescind.js +++ b/keyserver/src/push/rescind.js @@ -1,391 +1,384 @@ // @flow import apn from '@parse/node-apn'; import type { ResponseFailure } from '@parse/node-apn'; import type { FirebaseError } from 'firebase-admin'; import invariant from 'invariant'; import { createAndroidNotificationRescind } from 'lib/push/android-notif-creators.js'; import type { PlatformDetails } from 'lib/types/device-types.js'; import type { NotificationTargetDevice, TargetedAndroidNotification, SenderDeviceDescriptor, EncryptedNotifUtilsAPI, } from 'lib/types/notif-types.js'; import { threadSubscriptions } from 'lib/types/subscription-types.js'; import { threadPermissions } from 'lib/types/thread-permission-types.js'; import { promiseAll } from 'lib/utils/promises.js'; import { tID } from 'lib/utils/validation-utils.js'; import { prepareEncryptedIOSNotificationRescind } from './crypto.js'; import encryptedNotifUtilsAPI from './encrypted-notif-utils-api.js'; import { getAPNsNotificationTopic } from './providers.js'; import type { TargetedAPNsNotification } from './types.js'; import { apnPush, fcmPush, type APNPushResult, type FCMPushResult, } from './utils.js'; import createIDs from '../creators/id-creator.js'; import { dbQuery, SQL } from '../database/database.js'; import type { SQLStatementType } from '../database/types.js'; import { thisKeyserverID } from '../user/identity.js'; import { validateOutput } from '../utils/validation-utils.js'; type ParsedDelivery = { +platform: 'ios' | 'macos' | 'android', +codeVersion: ?number, +stateVersion: ?number, +notificationID: string, +deviceTokens: $ReadOnlyArray, }; type RescindDelivery = { source: 'rescind', rescindedID: string, errors?: | $ReadOnlyArray | $ReadOnlyArray, }; async function rescindPushNotifs( notifCondition: SQLStatementType, inputCountCondition?: SQLStatementType, ) { const notificationExtractString = `$.${threadSubscriptions.home}`; const visPermissionExtractString = `$.${threadPermissions.VISIBLE}.value`; const fetchQuery = SQL` SELECT n.id, n.user, n.thread, n.message, n.delivery, n.collapse_key, COUNT( `; fetchQuery.append(inputCountCondition ? inputCountCondition : SQL`m.thread`); fetchQuery.append(SQL` ) AS unread_count FROM notifications n LEFT JOIN memberships m ON m.user = n.user AND m.last_message > m.last_read_message AND m.role > 0 AND JSON_EXTRACT(subscription, ${notificationExtractString}) AND JSON_EXTRACT(permissions, ${visPermissionExtractString}) WHERE n.rescinded = 0 AND `); fetchQuery.append(notifCondition); fetchQuery.append(SQL` GROUP BY n.id, m.user`); const [[fetchResult], keyserverID] = await Promise.all([ dbQuery(fetchQuery), thisKeyserverID(), ]); const allDeviceTokens = new Set(); const parsedDeliveries: { [string]: $ReadOnlyArray } = {}; for (const row of fetchResult) { const rawDelivery = JSON.parse(row.delivery); const deliveries = Array.isArray(rawDelivery) ? rawDelivery : [rawDelivery]; const id = row.id.toString(); const rowParsedDeliveries = []; for (const delivery of deliveries) { if ( delivery.iosID || delivery.deviceType === 'ios' || delivery.deviceType === 'macos' ) { const deviceTokens = delivery.iosDeviceTokens ?? delivery.deviceTokens; rowParsedDeliveries.push({ notificationID: delivery.iosID, codeVersion: delivery.codeVersion, stateVersion: delivery.stateVersion, platform: delivery.deviceType ?? 'ios', deviceTokens, }); deviceTokens.forEach(deviceToken => allDeviceTokens.add(deviceToken)); } else if (delivery.androidID || delivery.deviceType === 'android') { const deviceTokens = delivery.androidDeviceTokens ?? delivery.deviceTokens; rowParsedDeliveries.push({ notificationID: row.collapse_key ? row.collapse_key : id, codeVersion: delivery.codeVersion, stateVersion: delivery.stateVersion, platform: 'android', deviceTokens, }); deviceTokens.forEach(deviceToken => allDeviceTokens.add(deviceToken)); } } parsedDeliveries[id] = rowParsedDeliveries; } const deviceTokenToCookieID = await getDeviceTokenToCookieID(allDeviceTokens); const deliveryPromises: { [string]: Promise | Promise, } = {}; const notifInfo = {}; const rescindedIDs = []; for (const row of fetchResult) { const id = row.id.toString(); const threadID = row.thread.toString(); notifInfo[id] = { userID: row.user.toString(), threadID, messageID: row.message.toString(), }; for (const delivery of parsedDeliveries[id]) { let platformDetails: PlatformDetails = { platform: delivery.platform }; if (delivery.codeVersion) { platformDetails = { ...platformDetails, codeVersion: delivery.codeVersion, }; } if (delivery.stateVersion) { platformDetails = { ...platformDetails, stateVersion: delivery.stateVersion, }; } if (delivery.platform === 'ios') { const devices = delivery.deviceTokens.map(deviceToken => ({ deliveryID: deviceToken, cryptoID: deviceTokenToCookieID[deviceToken], })); const deliveryPromise = (async () => { const targetedNotifications = await prepareIOSNotification( keyserverID, delivery.notificationID, row.unread_count, threadID, platformDetails, devices, ); return await apnPush({ targetedNotifications, platformDetails: { platform: 'ios', codeVersion: delivery.codeVersion, }, }); })(); deliveryPromises[id] = deliveryPromise; } else if (delivery.platform === 'android') { const devices = delivery.deviceTokens.map(deviceToken => ({ deliveryID: deviceToken, cryptoID: deviceTokenToCookieID[deviceToken], })); const deliveryPromise = (async () => { const targetedNotifications = await prepareAndroidNotification( keyserverID, delivery.notificationID, row.unread_count, threadID, platformDetails, devices, ); return await fcmPush({ targetedNotifications, codeVersion: delivery.codeVersion, }); })(); deliveryPromises[id] = deliveryPromise; } } rescindedIDs.push(id); } const numRescinds = Object.keys(deliveryPromises).length; const dbIDsPromise: Promise> = (async () => { if (numRescinds === 0) { return undefined; } return await createIDs('notifications', numRescinds); })(); const rescindPromise: Promise = (async () => { if (rescindedIDs.length === 0) { return undefined; } const rescindQuery = SQL` UPDATE notifications SET rescinded = 1 WHERE id IN (${rescindedIDs}) `; return await dbQuery(rescindQuery); })(); const [deliveryResults, dbIDs] = await Promise.all([ promiseAll(deliveryPromises), dbIDsPromise, rescindPromise, ]); const newNotifRows = []; if (numRescinds > 0) { invariant(dbIDs, 'dbIDs should be set'); for (const rescindedID in deliveryResults) { const delivery: RescindDelivery = { source: 'rescind', rescindedID, }; const { errors } = deliveryResults[rescindedID]; if (errors) { delivery.errors = errors; } const dbID = dbIDs.shift(); const { userID, threadID, messageID } = notifInfo[rescindedID]; newNotifRows.push([ dbID, userID, threadID, messageID, null, JSON.stringify([delivery]), 1, ]); } } if (newNotifRows.length > 0) { const insertQuery = SQL` INSERT INTO notifications (id, user, thread, message, collapse_key, delivery, rescinded) VALUES ${newNotifRows} `; await dbQuery(insertQuery); } } async function getDeviceTokenToCookieID( deviceTokens: Set, ): Promise<{ +[string]: string }> { if (deviceTokens.size === 0) { return {}; } const deviceTokenToCookieID = {}; const fetchCookiesQuery = SQL` SELECT id, device_token FROM cookies WHERE device_token IN (${[...deviceTokens]}) `; const [fetchResult] = await dbQuery(fetchCookiesQuery); for (const row of fetchResult) { deviceTokenToCookieID[row.device_token.toString()] = row.id.toString(); } return deviceTokenToCookieID; } async function conditionallyEncryptNotification( encryptedNotifUtilsAPIInstance: EncryptedNotifUtilsAPI, senderDeviceDescriptor: SenderDeviceDescriptor, notification: T, codeVersion: ?number, devices: $ReadOnlyArray, encryptCallback: ( encryptedNotifUtilsAPI: EncryptedNotifUtilsAPI, senderDeviceDescriptor: SenderDeviceDescriptor, devices: $ReadOnlyArray, notification: T, codeVersion?: ?number, ) => Promise< $ReadOnlyArray<{ +notification: T, +cryptoID: string, +deliveryID: string, +encryptionOrder?: number, }>, >, ): Promise<$ReadOnlyArray<{ +deliveryID: string, +notification: T }>> { const shouldBeEncrypted = codeVersion && codeVersion >= 233; if (!shouldBeEncrypted) { return devices.map(({ deliveryID }) => ({ notification, deliveryID, })); } const notifications = await encryptCallback( encryptedNotifUtilsAPI, senderDeviceDescriptor, devices, notification, codeVersion, ); return notifications.map(({ deliveryID, notification: notif }) => ({ deliveryID, notification: notif, })); } async function prepareIOSNotification( keyserverID: string, iosID: string, unreadCount: number, threadID: string, platformDetails: PlatformDetails, devices: $ReadOnlyArray, ): Promise<$ReadOnlyArray> { threadID = await validateOutput(platformDetails, tID, threadID); const { codeVersion } = platformDetails; const notification = new apn.Notification(); notification.topic = getAPNsNotificationTopic({ platform: 'ios', codeVersion, }); if (codeVersion && codeVersion > 198) { notification.mutableContent = true; notification.pushType = 'alert'; notification.badge = unreadCount; } else { notification.priority = 5; notification.contentAvailable = true; notification.pushType = 'background'; } - notification.payload = - codeVersion && codeVersion > 135 - ? { - backgroundNotifType: 'CLEAR', - notificationId: iosID, - setUnreadStatus: true, - threadID, - keyserverID, - } - : { - managedAps: { - action: 'CLEAR', - notificationId: iosID, - }, - }; + notification.payload = { + backgroundNotifType: 'CLEAR', + notificationId: iosID, + setUnreadStatus: true, + threadID, + keyserverID, + }; + return await conditionallyEncryptNotification( encryptedNotifUtilsAPI, { keyserverID }, notification, codeVersion, devices, prepareEncryptedIOSNotificationRescind, ); } async function prepareAndroidNotification( keyserverID: string, notifID: string, unreadCount: number, threadID: string, platformDetails: PlatformDetails, devices: $ReadOnlyArray, ): Promise<$ReadOnlyArray> { threadID = await validateOutput(platformDetails, tID, threadID); return await createAndroidNotificationRescind( encryptedNotifUtilsAPI, { senderDeviceDescriptor: { keyserverID }, badge: unreadCount.toString(), platformDetails, rescindID: notifID, threadID, }, devices, ); } export { rescindPushNotifs }; diff --git a/keyserver/src/session/version.js b/keyserver/src/session/version.js index 2074c160d..d57ba771c 100644 --- a/keyserver/src/session/version.js +++ b/keyserver/src/session/version.js @@ -1,21 +1,21 @@ // @flow import { hasMinCodeVersion } from 'lib/shared/version-utils.js'; import type { PlatformDetails } from 'lib/types/device-types.js'; import { ServerError } from 'lib/utils/errors.js'; import type { Viewer } from './viewer.js'; async function verifyClientSupported( viewer: Viewer, platformDetails: ?PlatformDetails, ) { - if (hasMinCodeVersion(platformDetails, { native: 128 })) { + if (hasMinCodeVersion(platformDetails, { native: 149 })) { return; } const error = new ServerError('client_version_unsupported'); error.platformDetails = platformDetails; throw error; } export { verifyClientSupported };