diff --git a/keyserver/src/cron/cron.js b/keyserver/src/cron/cron.js index 4c6243816..8142a4175 100644 --- a/keyserver/src/cron/cron.js +++ b/keyserver/src/cron/cron.js @@ -1,76 +1,93 @@ // @flow import cluster from 'cluster'; import schedule from 'node-schedule'; import { deleteOrphanedActivity } from '../deleters/activity-deleters'; import { deleteExpiredCookies } from '../deleters/cookie-deleters'; import { deleteOrphanedDays } from '../deleters/day-deleters'; import { deleteOrphanedEntries } from '../deleters/entry-deleters'; import { deleteOrphanedMemberships } from '../deleters/membership-deleters'; import { deleteOrphanedMessages } from '../deleters/message-deleters'; import { deleteOrphanedNotifs } from '../deleters/notif-deleters'; import { deleteOrphanedRevisions } from '../deleters/revision-deleters'; import { deleteOrphanedRoles } from '../deleters/role-deleters'; import { deleteOrphanedSessions, deleteOldWebSessions, } from '../deleters/session-deleters'; import { deleteStaleSIWENonceEntries } from '../deleters/siwe-nonce-deleters.js'; import { deleteInaccessibleThreads } from '../deleters/thread-deleters'; import { deleteExpiredUpdates } from '../deleters/update-deleters'; import { deleteUnassignedUploads } from '../deleters/upload-deleters'; import { backupDB } from './backups'; +import { createDailyUpdatesThread } from './daily-updates'; import { updateAndReloadGeoipDB } from './update-geoip-db'; if (cluster.isMaster) { schedule.scheduleJob( '30 3 * * *', // every day at 3:30 AM Pacific Time async () => { try { // Do everything one at a time to reduce load since we're in no hurry, // and since some queries depend on previous ones. await deleteExpiredCookies(); await deleteInaccessibleThreads(); await deleteOrphanedMemberships(); await deleteOrphanedDays(); await deleteOrphanedEntries(); await deleteOrphanedRevisions(); await deleteOrphanedRoles(); await deleteOrphanedMessages(); await deleteOrphanedActivity(); await deleteOrphanedNotifs(); await deleteOrphanedSessions(); await deleteOldWebSessions(); await deleteExpiredUpdates(); await deleteUnassignedUploads(); await deleteStaleSIWENonceEntries(); } catch (e) { console.warn('encountered error while trying to clean database', e); } }, ); schedule.scheduleJob( '0 */4 * * *', // every four hours async () => { try { await backupDB(); } catch (e) { console.warn('encountered error while trying to backup database', e); } }, ); schedule.scheduleJob( - '0 3 ? * 0', // every Sunday at 3:00 AM Pacific Time + '0 3 ? * 0', // every Sunday at 3:00 AM GMT async () => { try { await updateAndReloadGeoipDB(); } catch (e) { console.warn( 'encountered error while trying to update GeoIP database', e, ); } }, ); + schedule.scheduleJob( + '0 0 * * *', // every day at midnight GMT + async () => { + try { + if (process.env.RUN_COMM_TEAM_DEV_SCRIPTS) { + // This is a job that the Comm internal team uses + await createDailyUpdatesThread(); + } + } catch (e) { + console.warn( + 'encountered error while trying to create daily updates thread', + e, + ); + } + }, + ); } diff --git a/keyserver/src/cron/daily-updates.js b/keyserver/src/cron/daily-updates.js new file mode 100644 index 000000000..9ed0cf094 --- /dev/null +++ b/keyserver/src/cron/daily-updates.js @@ -0,0 +1,101 @@ +// @flow + +import invariant from 'invariant'; + +import ashoat from 'lib/facts/ashoat'; +import { messageTypes } from 'lib/types/message-types'; +import { threadTypes } from 'lib/types/thread-types'; +import { + getDate, + dateString, + prettyDateWithoutYear, + prettyDateWithoutDay, +} from 'lib/utils/date-utils'; + +import createMessages from '../creators/message-creator'; +import { createThread } from '../creators/thread-creator'; +import { fetchEntryInfosForThreadThisWeek } from '../fetchers/entry-fetchers'; +import { createScriptViewer } from '../session/scripts'; + +const devUpdateThread = '1358777'; +const weeklyDevSyncScheduleThread = '4138372'; + +const dailyUpdateMessage = (dateWithoutYear: string, dateWithoutDay: string) => + `### ${dateWithoutDay} update + +Share your updates for ${dateWithoutYear} here please!`; + +const dateIsWeekend = (date: Date) => + date.getDay() === 0 || date.getDay() === 6; + +// This function will do something four days a week. It skips Saturday and +// Sunday. The hard part is the third skipped day, which is the day of the +// weekly dev sync. By default this is Monday, but if the dev sync is on a +// different day, then an admin will put a calendar entry in the +// weeklyDevSyncScheduleThread indicating which day to skip. +async function createDailyUpdatesThread() { + if (!process.env.RUN_COMM_TEAM_DEV_SCRIPTS) { + // This is a job that the Comm internal team uses + return; + } + + const viewer = createScriptViewer(ashoat.id); + const now = new Date(); + if (dateIsWeekend(now)) { + // nothing happens on Saturday or Sunday + return; + } + + // Figure out which day the dev sync is on + let devSyncDay = 1; // default to Monday + const entryInfosInDevSyncScheduleThreadThisWeek = await fetchEntryInfosForThreadThisWeek( + viewer, + weeklyDevSyncScheduleThread, + ); + for (const entryInfo of entryInfosInDevSyncScheduleThreadThisWeek) { + const entryInfoDate = getDate( + entryInfo.year, + entryInfo.month, + entryInfo.day, + ); + if (dateIsWeekend(entryInfoDate)) { + // Ignore calendar entries on weekend + continue; + } + devSyncDay = entryInfoDate.getDay(); + // Use the newest entryInfo. fetchEntryInfos sorts by creation time + break; + } + + if (devSyncDay === now.getDay()) { + // Skip the dev sync day + return; + } + + const dayString = dateString(now); + const dateWithoutYear = prettyDateWithoutYear(dayString); + const dateWithoutDay = prettyDateWithoutDay(dayString); + + const [{ id: messageID }] = await createMessages(viewer, [ + { + type: messageTypes.TEXT, + threadID: devUpdateThread, + creatorID: ashoat.id, + time: Date.now(), + text: dailyUpdateMessage(dateWithoutYear, dateWithoutDay), + }, + ]); + invariant( + messageID, + 'message returned from createMessages always has ID set', + ); + + await createThread(viewer, { + type: threadTypes.SIDEBAR, + parentThreadID: devUpdateThread, + name: `${dateWithoutDay} update`, + sourceMessageID: messageID, + }); +} + +export { createDailyUpdatesThread }; diff --git a/keyserver/src/fetchers/entry-fetchers.js b/keyserver/src/fetchers/entry-fetchers.js index 26c93916f..9d55a0423 100644 --- a/keyserver/src/fetchers/entry-fetchers.js +++ b/keyserver/src/fetchers/entry-fetchers.js @@ -1,328 +1,354 @@ // @flow import invariant from 'invariant'; import { permissionLookup } from 'lib/permissions/thread-permissions'; import { filteredThreadIDs, filterExists, nonExcludeDeletedCalendarFilters, } from 'lib/selectors/calendar-filter-selectors'; import { rawEntryInfoWithinCalendarQuery } from 'lib/shared/entry-utils'; import type { CalendarQuery, FetchEntryInfosBase, DeltaEntryInfosResponse, RawEntryInfo, } from 'lib/types/entry-types'; import { calendarThreadFilterTypes } from 'lib/types/filter-types'; import type { HistoryRevisionInfo } from 'lib/types/history-types'; import { threadPermissions, type ThreadPermission, } from 'lib/types/thread-types'; +import { dateString } from 'lib/utils/date-utils'; import { ServerError } from 'lib/utils/errors'; import { dbQuery, SQL, mergeAndConditions, mergeOrConditions, } from '../database/database'; import type { SQLStatementType } from '../database/types'; import type { Viewer } from '../session/viewer'; import { creationString } from '../utils/idempotent'; import { checkIfThreadIsBlocked } from './thread-permission-fetchers'; async function fetchEntryInfo( viewer: Viewer, entryID: string, ): Promise { const results = await fetchEntryInfosByID(viewer, [entryID]); if (results.length === 0) { return null; } return results[0]; } function rawEntryInfoFromRow(row: Object): RawEntryInfo { return { id: row.id.toString(), threadID: row.threadID.toString(), text: row.text, year: row.year, month: row.month, day: row.day, creationTime: row.creationTime, creatorID: row.creatorID.toString(), deleted: !!row.deleted, }; } const visPermissionExtractString = `$.${threadPermissions.VISIBLE}.value`; async function fetchEntryInfosByID( viewer: Viewer, entryIDs: $ReadOnlyArray, ): Promise { if (entryIDs.length === 0) { return []; } const viewerID = viewer.id; const query = SQL` SELECT DAY(d.date) AS day, MONTH(d.date) AS month, YEAR(d.date) AS year, e.id, e.text, e.creation_time AS creationTime, d.thread AS threadID, e.deleted, e.creator AS creatorID FROM entries e LEFT JOIN days d ON d.id = e.day LEFT JOIN memberships m ON m.thread = d.thread AND m.user = ${viewerID} WHERE e.id IN (${entryIDs}) AND JSON_EXTRACT(m.permissions, ${visPermissionExtractString}) IS TRUE `; const [result] = await dbQuery(query); return result.map(rawEntryInfoFromRow); } function sqlConditionForCalendarQuery( calendarQuery: CalendarQuery, ): ?SQLStatementType { const { filters, startDate, endDate } = calendarQuery; const conditions = []; conditions.push(SQL`d.date BETWEEN ${startDate} AND ${endDate}`); const filterToThreadIDs = filteredThreadIDs(filters); if (filterToThreadIDs && filterToThreadIDs.size > 0) { conditions.push(SQL`d.thread IN (${[...filterToThreadIDs]})`); } else if (filterToThreadIDs) { // Filter to empty set means the result is empty return null; } else { conditions.push(SQL`m.role > 0`); } if (filterExists(filters, calendarThreadFilterTypes.NOT_DELETED)) { conditions.push(SQL`e.deleted = 0`); } return mergeAndConditions(conditions); } async function fetchEntryInfos( viewer: Viewer, calendarQueries: $ReadOnlyArray, ): Promise { const queryConditions = calendarQueries .map(sqlConditionForCalendarQuery) .filter(Boolean); if (queryConditions.length === 0) { return { rawEntryInfos: [] }; } const queryCondition = mergeOrConditions(queryConditions); const viewerID = viewer.id; const query = SQL` SELECT DAY(d.date) AS day, MONTH(d.date) AS month, YEAR(d.date) AS year, e.id, e.text, e.creation_time AS creationTime, d.thread AS threadID, e.deleted, e.creator AS creatorID FROM entries e LEFT JOIN days d ON d.id = e.day LEFT JOIN memberships m ON m.thread = d.thread AND m.user = ${viewerID} WHERE JSON_EXTRACT(m.permissions, ${visPermissionExtractString}) IS TRUE AND `; query.append(queryCondition); query.append(SQL`ORDER BY e.creation_time DESC`); const [result] = await dbQuery(query); const rawEntryInfos = []; for (const row of result) { rawEntryInfos.push(rawEntryInfoFromRow(row)); } return { rawEntryInfos }; } async function checkThreadPermissionForEntry( viewer: Viewer, entryID: string, permission: ThreadPermission, ): Promise { const viewerID = viewer.id; const query = SQL` SELECT m.permissions, t.id FROM entries e LEFT JOIN days d ON d.id = e.day LEFT JOIN threads t ON t.id = d.thread LEFT JOIN memberships m ON m.thread = t.id AND m.user = ${viewerID} WHERE e.id = ${entryID} `; const [result] = await dbQuery(query); if (result.length === 0) { return false; } const row = result[0]; if (row.id === null) { return false; } const threadIsBlocked = await checkIfThreadIsBlocked( viewer, row.id.toString(), permission, ); if (threadIsBlocked) { return false; } const permissions = JSON.parse(row.permissions); return permissionLookup(permissions, permission); } async function fetchEntryRevisionInfo( viewer: Viewer, entryID: string, ): Promise<$ReadOnlyArray> { const hasPermission = await checkThreadPermissionForEntry( viewer, entryID, threadPermissions.VISIBLE, ); if (!hasPermission) { throw new ServerError('invalid_credentials'); } const query = SQL` SELECT r.id, u.username AS author, r.text, r.last_update AS lastUpdate, r.deleted, d.thread AS threadID, r.entry AS entryID FROM revisions r LEFT JOIN users u ON u.id = r.author LEFT JOIN entries e ON e.id = r.entry LEFT JOIN days d ON d.id = e.day WHERE r.entry = ${entryID} ORDER BY r.last_update DESC `; const [result] = await dbQuery(query); const revisions = []; for (const row of result) { revisions.push({ id: row.id.toString(), author: row.author, text: row.text, lastUpdate: row.lastUpdate, deleted: !!row.deleted, threadID: row.threadID.toString(), entryID: row.entryID.toString(), }); } return revisions; } // calendarQueries are the "difference" queries we get from subtracting the old // CalendarQuery from the new one. See calendarQueryDifference. // oldCalendarQuery is the old CalendarQuery. We make sure none of the returned // RawEntryInfos match the old CalendarQuery, so that only the difference is // returned. async function fetchEntriesForSession( viewer: Viewer, calendarQueries: $ReadOnlyArray, oldCalendarQuery: CalendarQuery, ): Promise { // If we're not including deleted entries, we will try and set deletedEntryIDs // so that the client can catch possibly stale deleted entryInfos let filterDeleted = null; for (const calendarQuery of calendarQueries) { const notDeletedFilterExists = filterExists( calendarQuery.filters, calendarThreadFilterTypes.NOT_DELETED, ); if (filterDeleted === null) { filterDeleted = notDeletedFilterExists; } else { invariant( filterDeleted === notDeletedFilterExists, 'one of the CalendarQueries returned by calendarQueryDifference has ' + 'a NOT_DELETED filter but another does not: ' + JSON.stringify(calendarQueries), ); } } let calendarQueriesForFetch = calendarQueries; if (filterDeleted) { // Because in the filterDeleted case we still need the deleted RawEntryInfos // in order to construct deletedEntryIDs, we get rid of the NOT_DELETED // filters before passing the CalendarQueries to fetchEntryInfos. We will // filter out the deleted RawEntryInfos in a later step. calendarQueriesForFetch = calendarQueriesForFetch.map(calendarQuery => ({ ...calendarQuery, filters: nonExcludeDeletedCalendarFilters(calendarQuery.filters), })); } const { rawEntryInfos } = await fetchEntryInfos( viewer, calendarQueriesForFetch, ); const entryInfosNotInOldQuery = rawEntryInfos.filter( rawEntryInfo => !rawEntryInfoWithinCalendarQuery(rawEntryInfo, oldCalendarQuery), ); let filteredRawEntryInfos = entryInfosNotInOldQuery; let deletedEntryIDs = []; if (filterDeleted) { filteredRawEntryInfos = entryInfosNotInOldQuery.filter( rawEntryInfo => !rawEntryInfo.deleted, ); deletedEntryIDs = entryInfosNotInOldQuery .filter(rawEntryInfo => rawEntryInfo.deleted) .map(rawEntryInfo => { const { id } = rawEntryInfo; invariant( id !== null && id !== undefined, 'serverID should be set in fetchEntryInfos result', ); return id; }); } return { rawEntryInfos: filteredRawEntryInfos, deletedEntryIDs, }; } async function fetchEntryInfoForLocalID( viewer: Viewer, localID: ?string, ): Promise { if (!localID || !viewer.hasSessionInfo) { return null; } const creation = creationString(viewer, localID); const viewerID = viewer.id; const query = SQL` SELECT DAY(d.date) AS day, MONTH(d.date) AS month, YEAR(d.date) AS year, e.id, e.text, e.creation_time AS creationTime, d.thread AS threadID, e.deleted, e.creator AS creatorID FROM entries e LEFT JOIN days d ON d.id = e.day LEFT JOIN memberships m ON m.thread = d.thread AND m.user = ${viewerID} WHERE e.creator = ${viewerID} AND e.creation = ${creation} AND JSON_EXTRACT(m.permissions, ${visPermissionExtractString}) IS TRUE `; const [result] = await dbQuery(query); if (result.length === 0) { return null; } return rawEntryInfoFromRow(result[0]); } +function getSunday(weeksFromLastSunday: number) { + const date = new Date(); + const today = date.getDate(); + const currentDay = date.getDay(); + const newDate = date.setDate(today - currentDay + 7 * weeksFromLastSunday); + return new Date(newDate); +} + +async function fetchEntryInfosForThreadThisWeek( + viewer: Viewer, + threadID: string, +): Promise<$ReadOnlyArray> { + const startDate = dateString(getSunday(0)); + const endDate = dateString(getSunday(1)); + const filters = [ + { type: 'not_deleted' }, + { type: 'threads', threadIDs: [threadID] }, + ]; + const { rawEntryInfos } = await fetchEntryInfos(viewer, [ + { startDate, endDate, filters }, + ]); + return rawEntryInfos; +} + export { fetchEntryInfo, fetchEntryInfosByID, fetchEntryInfos, checkThreadPermissionForEntry, fetchEntryRevisionInfo, fetchEntriesForSession, fetchEntryInfoForLocalID, + fetchEntryInfosForThreadThisWeek, }; diff --git a/lib/types/filter-types.js b/lib/types/filter-types.js index 2c79dbef3..3af90fe64 100644 --- a/lib/types/filter-types.js +++ b/lib/types/filter-types.js @@ -1,34 +1,34 @@ // @flow import { type ThreadInfo } from './thread-types'; export const calendarThreadFilterTypes = Object.freeze({ THREAD_LIST: 'threads', NOT_DELETED: 'not_deleted', }); export type CalendarThreadFilterType = $Values< typeof calendarThreadFilterTypes, >; export type CalendarThreadFilter = { +type: 'threads', +threadIDs: $ReadOnlyArray, }; -export type CalendarFilter = { type: 'not_deleted' } | CalendarThreadFilter; +export type CalendarFilter = { +type: 'not_deleted' } | CalendarThreadFilter; export const defaultCalendarFilters: $ReadOnlyArray = [ { type: calendarThreadFilterTypes.NOT_DELETED }, ]; export const updateCalendarThreadFilter = 'UPDATE_CALENDAR_THREAD_FILTER'; export const clearCalendarThreadFilter = 'CLEAR_CALENDAR_THREAD_FILTER'; export const setCalendarDeletedFilter = 'SET_CALENDAR_DELETED_FILTER'; export type SetCalendarDeletedFilterPayload = { +includeDeleted: boolean, }; export type FilterThreadInfo = { threadInfo: ThreadInfo, numVisibleEntries: number, }; diff --git a/lib/utils/date-utils.js b/lib/utils/date-utils.js index 3ee185720..66fd479eb 100644 --- a/lib/utils/date-utils.js +++ b/lib/utils/date-utils.js @@ -1,169 +1,174 @@ // @flow import dateFormat from 'dateformat'; import invariant from 'invariant'; // Javascript uses 0-indexed months which is weird?? function getDate( yearInput: number, monthInput: number, dayOfMonth: number, ): Date { return new Date(yearInput, monthInput - 1, dayOfMonth); } function padMonthOrDay(n: number): string | number { return n < 10 ? '0' + n : n; } function daysInMonth(year: number, month: number) { switch (month) { case 2: return (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0 ? 29 : 28; case 4: case 6: case 9: case 11: return 30; default: return 31; } } function dateString( first: Date | number, month?: number, day?: number, ): string { if (arguments.length === 1) { return dateFormat(first, 'yyyy-mm-dd'); } else if (arguments.length === 3) { invariant(month && day, 'month/day should be set in call to dateString'); invariant(typeof first === 'number', 'first param should be a number'); return `${first}-${padMonthOrDay(month)}-${padMonthOrDay(day)}`; } invariant(false, 'incorrect number of params passed to dateString'); } function startDateForYearAndMonth(year: number, month: number): string { return dateString(year, month, 1); } function endDateForYearAndMonth(year: number, month: number): string { return dateString(year, month, daysInMonth(year, month)); } function fifteenDaysEarlier(timeZone?: ?string): string { const earlier = currentDateInTimeZone(timeZone); earlier.setDate(earlier.getDate() - 15); return dateString(earlier); } function fifteenDaysLater(timeZone?: ?string): string { const later = currentDateInTimeZone(timeZone); later.setDate(later.getDate() + 15); return dateString(later); } function prettyDate(dayString: string): string { return dateFormat(dateFromString(dayString), 'dddd, mmmm dS, yyyy'); } function prettyDateWithoutDay(dayString: string): string { return dateFormat(dateFromString(dayString), 'mmmm dS, yyyy'); } +function prettyDateWithoutYear(dayString: string): string { + return dateFormat(dateFromString(dayString), 'dddd, mmmm dS'); +} + function dateFromString(dayString: string): Date { const matches = dayString.match(/^([0-9]+)-([0-1][0-9])-([0-3][0-9])$/); invariant(matches && matches.length === 4, `invalid dayString ${dayString}`); return getDate( parseInt(matches[1], 10), parseInt(matches[2], 10), parseInt(matches[3], 10), ); } const millisecondsInDay = 24 * 60 * 60 * 1000; const millisecondsInWeek = millisecondsInDay * 7; const millisecondsInYear = millisecondsInDay * 365; // Takes a millisecond timestamp and displays the time in the local timezone function shortAbsoluteDate(timestamp: number): string { const now = Date.now(); const msSince = now - timestamp; const date = new Date(timestamp); if (msSince < millisecondsInDay) { return dateFormat(date, 'h:MM TT'); } else if (msSince < millisecondsInWeek) { return dateFormat(date, 'ddd'); } else if (msSince < millisecondsInYear) { return dateFormat(date, 'mmm d'); } else { return dateFormat(date, 'mmm d yyyy'); } } // Same as above, but longer function longAbsoluteDate(timestamp: number): string { const now = Date.now(); const msSince = now - timestamp; const date = new Date(timestamp); if (msSince < millisecondsInDay) { return dateFormat(date, 'h:MM TT'); } else if (msSince < millisecondsInWeek) { return dateFormat(date, 'ddd h:MM TT'); } else if (msSince < millisecondsInYear) { return dateFormat(date, 'mmmm d, h:MM TT'); } else { return dateFormat(date, 'mmmm d yyyy, h:MM TT'); } } function thisMonthDates( timeZone?: ?string, ): { startDate: string, endDate: string } { const now = currentDateInTimeZone(timeZone); const year = now.getFullYear(); const month = now.getMonth() + 1; return { startDate: startDateForYearAndMonth(year, month), endDate: endDateForYearAndMonth(year, month), }; } // The Date object doesn't support time zones, and is hardcoded to the server's // time zone. Thus, the best way to convert Date between time zones is to offset // the Date by the difference between the time zones function changeTimeZone(date: Date, timeZone: ?string): Date { if (!timeZone) { return date; } const localeString = date.toLocaleString('en-US', { timeZone }); const localeDate = new Date(localeString); const diff = localeDate.getTime() - date.getTime(); return new Date(date.getTime() + diff); } function currentDateInTimeZone(timeZone: ?string): Date { return changeTimeZone(new Date(), timeZone); } const threeDays = millisecondsInDay * 3; export { getDate, padMonthOrDay, dateString, startDateForYearAndMonth, endDateForYearAndMonth, fifteenDaysEarlier, fifteenDaysLater, prettyDate, prettyDateWithoutDay, + prettyDateWithoutYear, dateFromString, shortAbsoluteDate, longAbsoluteDate, thisMonthDates, currentDateInTimeZone, threeDays, };