Remove matrix-bot-sdk usage in tests

package.json

@@ -31,8 +31,6 @@
     "express": "^4.17.2",
     "hydrogen-view-sdk": "^0.0.4",
     "linkedom": "^0.14.1",
-    "matrix-bot-sdk": "^0.5.19",
-    "matrix-js-sdk": "^15.5.2",
     "matrix-public-archive-shared": "file:./shared/",
     "nconf": "^0.11.3",
     "node-fetch": "^2.6.7",

server/fetch-events-in-range.js

@@ -16,7 +16,7 @@ assert(matrixServerUrl);
 // Consider this scenario: dayStart(fromTs) <---- msg1 <- msg2 <-- msg3 <---- dayEnd(toTs)
 //  - ❌ If we start from dayStart and look backwards, we will find nothing.
 //  - ❌ If we start from dayStart and look forwards, we will find msg1, but federated backfill won't be able to paginate forwards
-//  - ✅ If we start from dayEnd and look backwards, we will msg3
+//  - ✅ If we start from dayEnd and look backwards, we will find msg3
 //  - ❌ If we start from dayEnd and look forwards, we will find nothing
 //
 // Returns events in reverse-chronological order.

server/lib/config.js

@@ -20,7 +20,7 @@ function configureNodeEnv() {
 }
 
 const nodeEnv = configureNodeEnv();
-console.log('nodeEnv', nodeEnv);
+console.log(`Config is using nodeEnv=${nodeEnv}`);
 const configDir = path.join(__dirname, '../../config');
 
 nconf.argv().env('__');

server/lib/fetch-endpoint.js

@@ -33,6 +33,7 @@ async function fetchEndpoint(endpoint, options = {}) {
   const res = await fetch(endpoint, {
     method,
     headers,
+    body: options.body,
   });
   await checkResponseStatus(res);
 
@@ -49,11 +50,16 @@ async function fetchEndpointAsJson(endpoint, options) {
   const opts = {
     ...options,
     headers: {
+      Accept: 'application/json',
       'Content-Type': 'application/json',
       ...(options.headers || {}),
     },
   };
 
+  if (options.body) {
+    opts.body = JSON.stringify(options.body);
+  }
+
   const res = await fetchEndpoint(endpoint, opts);
   const data = await res.json();
   return data;

shared/hydrogen-vm-render-script.js

@@ -174,7 +174,7 @@ async function mountHydrogen() {
     const memberEvent = workingStateEventMap[event.user_id];
     return makeEventEntryFromEventJson(event, memberEvent);
   });
-  console.log('eventEntries', eventEntries.length);
+  //console.log('eventEntries', eventEntries.length);
 
   // Map of `event_id` to `EventEntry`
   const eventEntriesByEventId = eventEntries.reduce((currentMap, eventEntry) => {

test/README.md

@@ -1,17 +1,17 @@
-```
+```sh
 $ docker pull matrixdotorg/synapse:latest
 $ docker build -t matrix-public-archive-test-homeserver -f test/dockerfiles/Synapse.Dockerfile test/dockerfiles/
 ```
 
-```
-docker-compose -f test/docker-compose.yml up -d --no-recreate
+```sh
+$ docker-compose --project-name matrix_public_archive_test -f test/docker-compose.yml up -d --no-recreate
 ```
 
-```
+```sh
 $ docker ps --all | grep test_hs
-$ docker logs test_hs1_1
-$ docker logs test_hs2_1
+$ docker logs -f --tail 10 matrix_public_archive_test_hs1_1
+$ docker logs -f --tail 10 matrix_public_archive_test_hs2_1
 
-$ docker stop test_hs1_1 test_hs2_1
-$ docker rm test_hs1_1 test_hs2_1
+$ docker stop matrix_public_archive_test_hs1_1 matrix_public_archive_test_hs2_1
+$ docker rm matrix_public_archive_test_hs1_1 matrix_public_archive_test_hs2_1
 ```

test/client-utils.js

@@ -0,0 +1,211 @@
+'use strict';
+
+const assert = require('assert');
+const { URLSearchParams } = require('url');
+const urlJoin = require('url-join');
+const { fetchEndpointAsJson, fetchEndpoint } = require('../server/lib/fetch-endpoint');
+
+const config = require('../server/lib/config');
+const matrixAccessToken = config.get('matrixAccessToken');
+assert(matrixAccessToken);
+
+let txnCount = 0;
+function getTxnId() {
+  txnCount++;
+  return `${new Date().getTime()}--${txnCount}`;
+}
+
+// Get client to act with for all of the client methods. This will use the
+// application service access token and client methods will append `?user_id`
+// for the specific user to act upon so we can use the `?ts` message timestamp
+// massaging when sending.
+async function getTestClientForHs(testMatrixServerUrl) {
+  // Register the virtual user
+  const username = `user-t${new Date().getTime()}-r${Math.floor(Math.random() * 1000000000)}`;
+  const registerResponse = await fetchEndpointAsJson(
+    urlJoin(testMatrixServerUrl, '/_matrix/client/v3/register'),
+    {
+      method: 'POST',
+      body: {
+        type: 'm.login.application_service',
+        username,
+      },
+      accessToken: matrixAccessToken,
+    }
+  );
+
+  const userId = registerResponse['user_id'];
+  assert(userId);
+
+  return {
+    homeserverUrl: testMatrixServerUrl,
+    // We use the application service AS token because we need to be able to use
+    // the `?ts` timestamp massaging when sending events
+    accessToken: matrixAccessToken,
+    userId: userId,
+  };
+}
+
+// Create a public room to test in
+async function createTestRoom(client) {
+  let qs = new URLSearchParams();
+  if (client.userId) {
+    qs.append('user_id', client.userId);
+  }
+
+  const createRoomResponse = await fetchEndpointAsJson(
+    urlJoin(client.homeserverUrl, `/_matrix/client/v3/createRoom?${qs.toString()}`),
+    {
+      method: 'POST',
+      body: {
+        preset: 'public_chat',
+        name: 'the hangout spot',
+        initial_state: [
+          {
+            type: 'm.room.history_visibility',
+            state_key: '',
+            content: {
+              history_visibility: 'world_readable',
+            },
+          },
+        ],
+      },
+      accessToken: client.accessToken,
+    }
+  );
+
+  const roomId = createRoomResponse['room_id'];
+  assert(roomId);
+  return roomId;
+}
+
+async function joinRoom({ client, roomId, viaServers }) {
+  let qs = new URLSearchParams();
+  if (viaServers) {
+    [].concat(viaServers).forEach((viaServer) => {
+      qs.append('server_name', viaServer);
+    });
+  }
+
+  if (client.userId) {
+    qs.append('user_id', client.userId);
+  }
+
+  const joinRoomResponse = await fetchEndpointAsJson(
+    urlJoin(client.homeserverUrl, `/_matrix/client/v3/join/${roomId}?${qs.toString()}`),
+    {
+      method: 'POST',
+      accessToken: client.accessToken,
+    }
+  );
+
+  const joinedRoomId = joinRoomResponse['room_id'];
+  assert(joinedRoomId);
+  return joinedRoomId;
+}
+
+async function sendEvent({ client, roomId, eventType, content, timestamp }) {
+  assert(client);
+  assert(roomId);
+  assert(content);
+
+  let qs = new URLSearchParams();
+  if (timestamp) {
+    assert(
+      timestamp && client.userId,
+      'We can only do `?ts` massaging from an application service access token. ' +
+        'Expected `client.userId` to be defined so we can act on behalf of that user'
+    );
+
+    qs.append('ts', timestamp);
+  }
+
+  if (client.userId) {
+    qs.append('user_id', client.userId);
+  }
+
+  const sendResponse = await fetchEndpointAsJson(
+    urlJoin(
+      client.homeserverUrl,
+      `/_matrix/client/v3/rooms/${roomId}/send/${eventType}/${getTxnId()}?${qs.toString()}`
+    ),
+    {
+      method: 'PUT',
+      body: content,
+      accessToken: client.accessToken,
+    }
+  );
+
+  const eventId = sendResponse['event_id'];
+  assert(eventId);
+  return eventId;
+}
+
+async function sendMessage({ client, roomId, content, timestamp }) {
+  return sendEvent({ client, roomId, eventType: 'm.room.message', content, timestamp });
+}
+
+// Create a number of messages in the given room
+async function createMessagesInRoom({ client, roomId, numMessages, prefix, timestamp }) {
+  let eventIds = [];
+  for (let i = 0; i < numMessages; i++) {
+    const eventId = await sendMessage({
+      client,
+      roomId,
+      content: {
+        msgtype: 'm.text',
+        body: `${prefix} - message${i}`,
+      },
+      timestamp,
+    });
+    eventIds.push(eventId);
+  }
+
+  return eventIds;
+}
+
+// Uploads the given data Buffer and returns the MXC URI of the uploaded content
+async function uploadContent({ client, roomId, data, fileName, contentType }) {
+  assert(client);
+  assert(roomId);
+  assert(data);
+
+  let qs = new URLSearchParams();
+  if (client.userId) {
+    qs.append('user_id', client.userId);
+  }
+
+  if (fileName) {
+    qs.append('filename', fileName);
+  }
+
+  // We don't want to use `fetchEndpointAsJson` here because it will
+  // `JSON.stringify(...)` the body data
+  const uploadResponse = await fetchEndpoint(
+    urlJoin(client.homeserverUrl, `/_matrix/media/v3/upload`),
+    {
+      method: 'POST',
+      body: data,
+      headers: {
+        'Content-Type': contentType || 'application/octet-stream',
+      },
+      accessToken: client.accessToken,
+    }
+  );
+
+  const uploadResponseData = await uploadResponse.json();
+
+  const mxcUri = uploadResponseData['content_uri'];
+  assert(mxcUri);
+  return mxcUri;
+}
+
+module.exports = {
+  getTestClientForHs,
+  createTestRoom,
+  joinRoom,
+  sendEvent,
+  sendMessage,
+  createMessagesInRoom,
+  uploadContent,
+};

test/dockerfiles/Synapse.Dockerfile

@@ -24,7 +24,7 @@ COPY keys/* /ca/
 RUN openssl genrsa -out /conf/server.tls.key 2048
 
 # generate a signing key
-RUN generate_signing_key.py -o /conf/server.signing.key
+RUN generate_signing_key -o /conf/server.signing.key
 
 WORKDIR /data
 

test/dockerfiles/synapse/homeserver.yaml

@@ -6,6 +6,7 @@ report_stats: False
 signing_key_path: /conf/server.signing.key
 trusted_key_servers: []
 enable_registration: true
+enable_registration_without_verification: true
 
 ## Listeners ##
 
@@ -94,6 +95,13 @@ rc_joins:
 
 federation_rr_transactions_per_room_per_second: 9999
 
+## Media Store ##
+
+# Whether to generate new thumbnails on the fly. This lets the image thumbnails
+# load in the tests.
+#
+dynamic_thumbnails: true
+
 ## API Configuration ##
 
 # A list of application service config files to use