diff --git a/keyserver/src/scripts/biggest-farcaster-channels.js b/keyserver/src/scripts/biggest-farcaster-channels.js new file mode 100644 index 000000000..6d5abd984 --- /dev/null +++ b/keyserver/src/scripts/biggest-farcaster-channels.js @@ -0,0 +1,25 @@ +// @flow + +import invariant from 'invariant'; + +import type { NeynarChannel } from 'lib/types/farcaster-types.js'; + +import { main } from './utils.js'; +import { neynarClient } from '../utils/fc-cache.js'; + +async function fetchAllFarcasterChannelsAndPrintSortedByFollowerCount() { + invariant(neynarClient, 'neynarClient should be defined'); + const allChannels = await neynarClient.getAllChannels(); + allChannels.sort( + (channelA: NeynarChannel, channelB: NeynarChannel) => + channelB.follower_count - channelA.follower_count, + ); + const simplifiedChannelInfo = allChannels.map(({ id, follower_count }) => ({ + id, + follower_count, + })); + simplifiedChannelInfo.splice(1000); + console.log(JSON.stringify(simplifiedChannelInfo, undefined, ' ')); +} + +main([fetchAllFarcasterChannelsAndPrintSortedByFollowerCount]); diff --git a/keyserver/src/scripts/utils.js b/keyserver/src/scripts/utils.js index 1c6430129..1edbecb04 100644 --- a/keyserver/src/scripts/utils.js +++ b/keyserver/src/scripts/utils.js @@ -1,30 +1,32 @@ // @flow import { endPool } from '../database/database.js'; import { endFirebase, endAPNs } from '../push/providers.js'; import { publisher } from '../socket/redis.js'; +import { initFCCache } from '../utils/fc-cache.js'; import { prefetchAllURLFacts } from '../utils/urls.js'; function endScript() { endPool(); publisher.end(); endFirebase(); endAPNs(); } function main(functions: $ReadOnlyArray<() => Promise>) { void (async () => { await prefetchAllURLFacts(); + await initFCCache(); try { for (const f of functions) { await f(); } } catch (e) { console.warn(e); } finally { endScript(); } })(); } export { endScript, main }; diff --git a/keyserver/src/utils/fc-cache.js b/keyserver/src/utils/fc-cache.js index 0ac65b88d..3476ffba2 100644 --- a/keyserver/src/utils/fc-cache.js +++ b/keyserver/src/utils/fc-cache.js @@ -1,30 +1,31 @@ // @flow import { getCommConfig } from 'lib/utils/comm-config.js'; import { getFCNames as baseGetFCNames, type GetFCNames, type BaseFCInfo, } from 'lib/utils/farcaster-helpers.js'; import { FCCache } from 'lib/utils/fc-cache.js'; import { NeynarClient } from 'lib/utils/neynar-client.js'; type NeynarConfig = { +key: string }; let getFCNames: ?GetFCNames; +let neynarClient: ?NeynarClient; async function initFCCache() { const neynarSecret = await getCommConfig({ folder: 'secrets', name: 'neynar', }); const neynarKey = neynarSecret?.key; if (!neynarKey) { return; } - const neynarClient = new NeynarClient(neynarKey); + neynarClient = new NeynarClient(neynarKey); const fcCache = new FCCache(neynarClient); getFCNames = (users: $ReadOnlyArray): Promise => baseGetFCNames(fcCache, users); } -export { initFCCache, getFCNames }; +export { initFCCache, getFCNames, neynarClient }; diff --git a/lib/types/farcaster-types.js b/lib/types/farcaster-types.js index 082285eda..e1fd94908 100644 --- a/lib/types/farcaster-types.js +++ b/lib/types/farcaster-types.js @@ -1,35 +1,36 @@ // @flow // This is a message that the rendered webpage // (landing/connect-farcaster.react.js) uses to communicate back // to the React Native WebView that is rendering it // (native/components/farcaster-web-view.react.js) export type FarcasterWebViewMessage = | { +type: 'farcaster_url', +url: string, } | { +type: 'farcaster_data', +fid: string, }; export type NeynarUser = { +fid: number, +username: string, ... }; export type NeynarUserWithViewerContext = $ReadOnly<{ ...NeynarUser, +viewerContext: { +following: boolean, }, ... }>; export type NeynarChannel = { +id: string, +name: string, + +follower_count: number, ... }; diff --git a/lib/utils/neynar-client.js b/lib/utils/neynar-client.js index 9b6eb5ce6..a8f1d7dee 100644 --- a/lib/utils/neynar-client.js +++ b/lib/utils/neynar-client.js @@ -1,234 +1,276 @@ // @flow import invariant from 'invariant'; import { getMessageForException } from './errors.js'; import type { NeynarChannel, NeynarUser, NeynarUserWithViewerContext, } from '../types/farcaster-types.js'; type FetchFollowersResponse = { +result: { +users: $ReadOnlyArray, +next: { +cursor: ?string, }, }, }; -type FetchFollowedFarcasterChannelsResponse = { +type FetchFarcasterChannelsResponse = { +channels: $ReadOnlyArray, +next: { +cursor: ?string, }, }; type FetchFarcasterChannelByNameResponse = { +channels: $ReadOnlyArray, }; type FetchUsersResponse = { +users: $ReadOnlyArray, }; const neynarBaseURL = 'https://api.neynar.com/'; const neynarURLs = { '1': `${neynarBaseURL}v1/farcaster/`, '2': `${neynarBaseURL}v2/farcaster/`, }; function getNeynarURL( apiVersion: string, apiCall: string, params: { [string]: string }, ): string { const neynarURL = neynarURLs[apiVersion]; invariant( neynarURL, `could not find Neynar URL for apiVersion ${apiVersion}`, ); return `${neynarURL}${apiCall}?${new URLSearchParams(params).toString()}`; } const fetchFollowerLimit = 150; const fetchFollowedChannelsLimit = 100; +const fetchChannelsLimit = 50; class NeynarClient { apiKey: string; constructor(apiKey: string) { this.apiKey = apiKey; } // We're using the term "friend" for a bidirectional follow async fetchFriendFIDs(fid: string): Promise { const fids = []; let paginationCursor = null; do { const params: { [string]: string } = { fid, viewerFid: fid, limit: fetchFollowerLimit.toString(), ...(paginationCursor ? { cursor: paginationCursor } : null), }; const url = getNeynarURL('1', 'followers', params); try { const response = await fetch(url, { method: 'GET', headers: { Accept: 'application/json', api_key: this.apiKey, }, }); const json: FetchFollowersResponse = await response.json(); const { users } = json.result; for (const user of users) { if (user.viewerContext.following) { fids.push(user.fid.toString()); } } paginationCursor = json.result.next.cursor; } catch (error) { console.log( 'Failed to fetch friend FIDs:', getMessageForException(error) ?? 'unknown', ); throw error; } } while (paginationCursor); return fids; } async fetchFollowedFarcasterChannels(fid: string): Promise { const farcasterChannels = []; let paginationCursor = null; do { const params: { [string]: string } = { fid, limit: fetchFollowedChannelsLimit.toString(), ...(paginationCursor ? { cursor: paginationCursor } : null), }; const url = getNeynarURL('2', 'user/channels', params); try { const response = await fetch(url, { method: 'GET', headers: { Accept: 'application/json', api_key: this.apiKey, }, }); - const json: FetchFollowedFarcasterChannelsResponse = - await response.json(); + const json: FetchFarcasterChannelsResponse = await response.json(); const { channels, next } = json; channels.forEach(channel => { farcasterChannels.push(channel); }); paginationCursor = next.cursor; } catch (error) { console.log( 'Failed to fetch followed Farcaster channels:', getMessageForException(error) ?? 'unknown', ); throw error; } } while (paginationCursor); return farcasterChannels; } async fetchFarcasterChannelByName( channelName: string, ): Promise { const params: { [string]: string } = { q: channelName, }; const url = getNeynarURL('2', 'channel/search', params); try { const response = await fetch(url, { method: 'GET', headers: { Accept: 'application/json', api_key: this.apiKey, }, }); const json: FetchFarcasterChannelByNameResponse = await response.json(); const { channels } = json; for (const channel of channels) { if (channel.name.toLowerCase() === channelName.toLowerCase()) { return channel; } } return null; } catch (error) { console.log( 'Failed to search Farcaster channel by name:', getMessageForException(error) ?? 'unknown', ); throw error; } } async getFarcasterUsernames( fids: $ReadOnlyArray, ): Promise> { const fidsLeft = [...fids]; const results: Array = []; do { // Neynar API allows querying 100 at a time const batch = fidsLeft.splice(0, 100); const url = getNeynarURL('2', 'user/bulk', { fids: batch.join(',') }); try { const response = await fetch(url, { method: 'GET', headers: { Accept: 'application/json', api_key: this.apiKey, }, }); const json: FetchUsersResponse = await response.json(); const { users } = json; const neynarUserMap = new Map(); for (const neynarUser of users) { neynarUserMap.set(neynarUser.fid, neynarUser); } for (const fid of batch) { const neynarUser = neynarUserMap.get(parseInt(fid)); results.push(neynarUser ? neynarUser.username : null); } } catch (error) { console.log( 'Failed to fetch Farcaster usernames:', getMessageForException(error) ?? 'unknown', ); throw error; } } while (fidsLeft.length > 0); return results; } + + async getAllChannels(): Promise> { + const farcasterChannels = []; + let paginationCursor = null; + + do { + const params: { [string]: string } = { + limit: fetchChannelsLimit.toString(), + ...(paginationCursor ? { cursor: paginationCursor } : null), + }; + + const url = getNeynarURL('2', 'channel/list', params); + + try { + const response = await fetch(url, { + method: 'GET', + headers: { + Accept: 'application/json', + api_key: this.apiKey, + }, + }); + + const json: FetchFarcasterChannelsResponse = await response.json(); + + const { channels, next } = json; + + channels.forEach(channel => { + farcasterChannels.push(channel); + }); + + paginationCursor = next.cursor; + } catch (error) { + console.log( + 'Failed to fetch all Farcaster channels:', + getMessageForException(error) ?? 'unknown', + ); + throw error; + } + } while (paginationCursor); + + return farcasterChannels; + } } export { NeynarClient };