mirror of https://github.com/apache/superset.git
feat(SIP-39): Websocket sidecar app (#11498)
* WIP node.js websocket app * Load testing * Multi-stream publish with blocking reads * Use JWT for auth and channel ID * Update ws jwt cookie name * Typescript * Frontend WebSocket transport support * ws server ping/pong and GC logic * ws server unit tests * GC interval config, debug logging * Cleanup JWT cookie logic * Refactor asyncEvents.ts to support non-Redux use cases * Update tests for refactored asyncEvents * Add eslint, write READMEs, reorg files * CI workflow * Moar Apache license headers * pylint found something * adjust GH actions workflow * Improve documentation & comments * Prettier * Add configurable logging via Winston * Add SSL support for Redis connections * Fix incompatible logger statements * Apply suggestions from code review Co-authored-by: David Aaron Suddjian <1858430+suddjian@users.noreply.github.com> * rename streamPrefix config Co-authored-by: David Aaron Suddjian <1858430+suddjian@users.noreply.github.com>
This commit is contained in:
Rob DiCiuccio
GitHub
parent
6a81a7961c
commit
806fb73d25
|
|
@ -0,0 +1,33 @@
|
|||
name: WebSocket server
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "superset-websocket/**"
|
||||
pull_request:
|
||||
paths:
|
||||
- "superset-websocket/**"
|
||||
|
||||
jobs:
|
||||
app-checks:
|
||||
if: github.event.pull_request.draft == false
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install dependencies
|
||||
working-directory: ./superset-websocket
|
||||
run: npm install
|
||||
- name: lint
|
||||
working-directory: ./superset-websocket
|
||||
run: npm run lint
|
||||
- name: prettier
|
||||
working-directory: ./superset-websocket
|
||||
run: npm run prettier-check
|
||||
- name: unit tests
|
||||
working-directory: ./superset-websocket
|
||||
run: npm run test
|
||||
- name: build
|
||||
working-directory: ./superset-websocket
|
||||
run: npm run build
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
*.min.js
|
||||
node_modules
|
||||
dist
|
||||
coverage
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
module.exports = {
|
||||
root: true,
|
||||
parser: '@typescript-eslint/parser',
|
||||
env: {
|
||||
node: true,
|
||||
browser: true,
|
||||
},
|
||||
plugins: [
|
||||
'@typescript-eslint',
|
||||
],
|
||||
extends: [
|
||||
'eslint:recommended',
|
||||
'plugin:@typescript-eslint/recommended',
|
||||
'prettier',
|
||||
],
|
||||
rules: {
|
||||
"@typescript-eslint/explicit-module-boundary-types": 0,
|
||||
"@typescript-eslint/no-var-requires": 0,
|
||||
},
|
||||
};
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
config.json
|
||||
dist
|
||||
node_modules
|
||||
*.log
|
||||
|
|
@ -0,0 +1 @@
|
|||
v14.15.5
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
*.min.js
|
||||
node_modules
|
||||
dist
|
||||
coverage
|
||||
.eslintrc.js
|
||||
.prettierrc.json
|
||||
*.md
|
||||
*.json
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"trailingComma": "all",
|
||||
"singleQuote": true,
|
||||
"arrowParens": "avoid"
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
# Superset WebSocket Server
|
||||
|
||||
A Node.js WebSocket server for sending async event data to the Superset web application frontend.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Node.js 12+ (not tested with older versions)
|
||||
- Redis 5+
|
||||
|
||||
To use this feature, Superset needs to be configured to enable global async queries and to use WebSockets as the transport (see below).
|
||||
|
||||
## Architecture
|
||||
|
||||
This implementation is based on the architecture defined in [SIP-39](https://github.com/apache/superset/issues/9190).
|
||||
|
||||
### Streams
|
||||
|
||||
Async events are pushed to [Redis Streams](https://redis.io/topics/streams-intro) from the [Superset Flask app](https://github.com/preset-io/superset/blob/master/superset/utils/async_query_manager.py). An event for a particular user is published to two streams: 1) the global event stream that includes events for all users, and 2) a channel/session-specific stream only for the user. This approach provides a good balance of performance (reading off of a single global stream) and fault tolerance (dropped connections can "catch up" by reading from the channel-specific stream).
|
||||
|
||||
Note that Redis Stream [consumer groups](https://redis.io/topics/streams-intro#consumer-groups) are not used here due to the fact that each group receives a subset of the data for a stream, and WebSocket clients have a persistent connection to each app instance, requiring access to all data in a stream. Horizontal scaling of the WebSocket app requires having multiple WebSocket servers, each with full access to the Redis Stream data.
|
||||
|
||||
### Connection
|
||||
|
||||
When a user's browser initially connects to the WebSocket server, it does so over HTTP, which includes the JWT authentication cookie, set by the Flask app, in the request. _Note that due to the cookie-based authentication method, the WebSocket server must be run on the same host as the web application._ The server validates the JWT token by using the shared secret (config: `jwtSecret`), and if valid, proceeds to upgrade the connection to a WebSocket. The user's session-based "channel" ID is contained in the JWT, and serves as the basis for sending received events to the user's connected socket(s).
|
||||
|
||||
A user may have multiple WebSocket connections under a single channel (session) ID. This would be the case if the user has multiple browser tabs open, for example. In this scenario, **all events received for a specific channel are sent to all connected sockets**, leaving it to the consumer to decide which events are relevant to the current application context.
|
||||
|
||||
### Reconnection
|
||||
|
||||
It is expected that a user's WebSocket connection may be dropped or interrupted due to fluctuating network conditions. The Superset frontend code keeps track of the last received async event ID, and attempts to reconnect to the WebSocket server with a `last_id` query parameter in the initial HTTP request. If a connection includes a valid `last_id` value, events that may have already been received and sent unsuccessfully are read from the channel-based Redis Stream and re-sent to the new WebSocket connection. The global event stream flow then assumes responsibility for sending subsequent events to the connected socket(s).
|
||||
|
||||
### Connection Management
|
||||
|
||||
The server utilizes the standard WebSocket [ping/pong functionality](https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API/Writing_WebSocket_servers#pings_and_pongs_the_heartbeat_of_websockets) to determine if active WebSocket connections are still alive. Active sockets are sent a _ping_ regularly (config: `pingSocketsIntervalMs`), and the internal _sockets_ registry is updated with a timestamp when a _pong_ response is received. If a _pong_ response has not been received before the timeout period (config: `socketResponseTimeoutMs`), the socket is terminated and removed from the internal registry.
|
||||
|
||||
In addition to periodic socket connection cleanup, the internal _channels_ registry is regularly "cleaned" (config: `gcChannelsIntervalMs`) to remove stale references and prevent excessive memory consumption over time.
|
||||
|
||||
## Install
|
||||
|
||||
Install dependencies:
|
||||
```
|
||||
npm install
|
||||
```
|
||||
|
||||
## WebSocket Server Configuration
|
||||
|
||||
Copy `config.example.json` to `config.json` and adjust the values for your environment.
|
||||
|
||||
## Superset Configuration
|
||||
|
||||
Configure the Superset Flask app to enable global async queries (in `superset_config.py`):
|
||||
|
||||
Enable the `GLOBAL_ASYNC_QUERIES` feature flag:
|
||||
```
|
||||
"GLOBAL_ASYNC_QUERIES": True
|
||||
```
|
||||
|
||||
Configure the following Superset values:
|
||||
```
|
||||
GLOBAL_ASYNC_QUERIES_TRANSPORT = "ws"
|
||||
GLOBAL_ASYNC_QUERIES_WEBSOCKET_URL = "ws://<host>:<port>/"
|
||||
```
|
||||
|
||||
Note that the WebSocket server must be run on the same hostname (different port) for cookies to be shared between the Flask app and the WebSocket server.
|
||||
|
||||
The following config values must contain the same values in both the Flask app config and `config.json`:
|
||||
```
|
||||
GLOBAL_ASYNC_QUERIES_REDIS_CONFIG
|
||||
GLOBAL_ASYNC_QUERIES_REDIS_STREAM_PREFIX
|
||||
GLOBAL_ASYNC_QUERIES_JWT_COOKIE_NAME
|
||||
GLOBAL_ASYNC_QUERIES_JWT_SECRET
|
||||
```
|
||||
|
||||
More info on Superset configuration values for async queries: https://github.com/apache/superset/blob/master/CONTRIBUTING.md#async-chart-queries
|
||||
|
||||
## Running
|
||||
|
||||
Running locally via dev server:
|
||||
```
|
||||
npm run dev-server
|
||||
```
|
||||
|
||||
Running in production:
|
||||
```
|
||||
npm run build && npm start
|
||||
```
|
||||
|
||||
*TODO: containerization*
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"port": 8080,
|
||||
"logLevel": "info",
|
||||
"logToFile": false,
|
||||
"logFilename": "app.log",
|
||||
"redis": {
|
||||
"port": 6379,
|
||||
"host": "127.0.0.1",
|
||||
"password": "",
|
||||
"db": 0,
|
||||
"ssl": false
|
||||
},
|
||||
"redisStreamPrefix": "async-events-",
|
||||
"jwtSecret": "CHANGE-ME",
|
||||
"jwtCookieName": "async-token"
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"redis": {
|
||||
"port": 6379,
|
||||
"host": "127.0.0.1",
|
||||
"password": "",
|
||||
"db": 10,
|
||||
"ssl": false
|
||||
},
|
||||
"redisStreamPrefix": "test-async-events-",
|
||||
"jwtSecret": "test123-test123-test123-test123-test123-test123-test123",
|
||||
"jwtCookieName": "test-async-token"
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,41 @@
|
|||
{
|
||||
"name": "superset-websocket",
|
||||
"version": "0.0.1",
|
||||
"description": "Websocket sidecar application for Superset",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"start": "node dist/index.js start",
|
||||
"test": "NODE_ENV=test jest -i spec",
|
||||
"type": "tsc --noEmit",
|
||||
"lint": "eslint . --ext .js,.jsx,.ts,.tsx && npm run type",
|
||||
"dev-server": "ts-node src/index.ts start",
|
||||
"build": "tsc",
|
||||
"prettier-check": "prettier --check .",
|
||||
"format": "prettier --write ."
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"cookie": "^0.4.1",
|
||||
"ioredis": "^4.16.1",
|
||||
"jsonwebtoken": "^8.5.1",
|
||||
"uuid": "^8.3.2",
|
||||
"winston": "^3.3.3",
|
||||
"ws": "^7.4.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/ioredis": "^4.22.0",
|
||||
"@types/jest": "^26.0.20",
|
||||
"@types/node": "^14.14.22",
|
||||
"@types/uuid": "^8.3.0",
|
||||
"@types/ws": "^7.4.0",
|
||||
"@typescript-eslint/eslint-plugin": "^4.19.0",
|
||||
"@typescript-eslint/parser": "^4.19.0",
|
||||
"eslint": "^7.22.0",
|
||||
"eslint-config-prettier": "^8.1.0",
|
||||
"jest": "^26.6.3",
|
||||
"prettier": "2.2.1",
|
||||
"ts-jest": "^26.5.3",
|
||||
"ts-node": "^9.1.1",
|
||||
"typescript": "^4.2.3"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,486 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
const jwt = require('jsonwebtoken');
|
||||
const config = require('../config.test.json');
|
||||
|
||||
import { describe, expect, test, beforeEach, afterEach } from '@jest/globals';
|
||||
import * as http from 'http';
|
||||
import * as net from 'net';
|
||||
import WebSocket from 'ws';
|
||||
|
||||
// NOTE: these mock variables needs to start with "mock" due to
|
||||
// calls to `jest.mock` being hoisted to the top of the file.
|
||||
// https://jestjs.io/docs/es6-class-mocks#calling-jestmock-with-the-module-factory-parameter
|
||||
const mockRedisXrange = jest.fn();
|
||||
|
||||
jest.mock('ws');
|
||||
jest.mock('ioredis', () => {
|
||||
return jest.fn().mockImplementation(() => {
|
||||
return { xrange: mockRedisXrange };
|
||||
});
|
||||
});
|
||||
|
||||
const wsMock = WebSocket as jest.Mocked<typeof WebSocket>;
|
||||
const channelId = 'bc9e040c-7b4a-4817-99b9-292832d97ec7';
|
||||
const streamReturnValue: server.StreamResult[] = [
|
||||
[
|
||||
'1615426152415-0',
|
||||
[
|
||||
'data',
|
||||
`{"channel_id": "${channelId}", "job_id": "c9b99965-8f1e-4ce5-aa43-d6fc94d6a510", "user_id": "1", "status": "done", "errors": [], "result_url": "/superset/explore_json/data/ejr-37281682b1282cdb8f25e0de0339b386"}`,
|
||||
],
|
||||
],
|
||||
[
|
||||
'1615426152516-0',
|
||||
[
|
||||
'data',
|
||||
`{"channel_id": "${channelId}", "job_id": "f1e5bb1f-f2f1-4f21-9b2f-c9b91dcc9b59", "user_id": "1", "status": "done", "errors": [], "result_url": "/api/v1/chart/data/qc-64e8452dc9907dd77746cb75a19202de"}`,
|
||||
],
|
||||
],
|
||||
];
|
||||
|
||||
import * as server from '../src/index';
|
||||
|
||||
describe('server', () => {
|
||||
beforeEach(() => {
|
||||
mockRedisXrange.mockClear();
|
||||
server.resetState();
|
||||
});
|
||||
|
||||
describe('incrementId', () => {
|
||||
test('it increments a valid Redis stream ID', () => {
|
||||
expect(server.incrementId('1607477697866-0')).toEqual('1607477697866-1');
|
||||
});
|
||||
|
||||
test('it handles an invalid Redis stream ID', () => {
|
||||
expect(server.incrementId('foo')).toEqual('foo');
|
||||
});
|
||||
});
|
||||
|
||||
describe('redisUrlFromConfig', () => {
|
||||
test('it builds a valid Redis URL from defaults', () => {
|
||||
expect(
|
||||
server.redisUrlFromConfig({
|
||||
port: 6379,
|
||||
host: '127.0.0.1',
|
||||
password: '',
|
||||
db: 0,
|
||||
ssl: false,
|
||||
}),
|
||||
).toEqual('redis://127.0.0.1:6379/0');
|
||||
});
|
||||
test('it builds a valid Redis URL with a password', () => {
|
||||
expect(
|
||||
server.redisUrlFromConfig({
|
||||
port: 6380,
|
||||
host: 'redis.local',
|
||||
password: 'foo',
|
||||
db: 1,
|
||||
ssl: false,
|
||||
}),
|
||||
).toEqual('redis://:foo@redis.local:6380/1');
|
||||
});
|
||||
test('it builds a valid Redis URL with SSL', () => {
|
||||
expect(
|
||||
server.redisUrlFromConfig({
|
||||
port: 6379,
|
||||
host: '127.0.0.1',
|
||||
password: '',
|
||||
db: 0,
|
||||
ssl: true,
|
||||
}),
|
||||
).toEqual('rediss://127.0.0.1:6379/0');
|
||||
});
|
||||
});
|
||||
|
||||
describe('processStreamResults', () => {
|
||||
test('sends data to channel', async () => {
|
||||
const ws = new wsMock('localhost');
|
||||
const sendMock = jest.spyOn(ws, 'send');
|
||||
const socketInstance = { ws: ws, channel: channelId, pongTs: Date.now() };
|
||||
server.trackClient(channelId, socketInstance);
|
||||
|
||||
server.processStreamResults(streamReturnValue);
|
||||
|
||||
const message1 = `{"id":"1615426152415-0","channel_id":"${channelId}","job_id":"c9b99965-8f1e-4ce5-aa43-d6fc94d6a510","user_id":"1","status":"done","errors":[],"result_url":"/superset/explore_json/data/ejr-37281682b1282cdb8f25e0de0339b386"}`;
|
||||
const message2 = `{"id":"1615426152516-0","channel_id":"${channelId}","job_id":"f1e5bb1f-f2f1-4f21-9b2f-c9b91dcc9b59","user_id":"1","status":"done","errors":[],"result_url":"/api/v1/chart/data/qc-64e8452dc9907dd77746cb75a19202de"}`;
|
||||
expect(sendMock).toHaveBeenCalledWith(message1);
|
||||
expect(sendMock).toHaveBeenCalledWith(message2);
|
||||
});
|
||||
|
||||
test('channel not present', async () => {
|
||||
const ws = new wsMock('localhost');
|
||||
const sendMock = jest.spyOn(ws, 'send');
|
||||
|
||||
server.processStreamResults(streamReturnValue);
|
||||
|
||||
expect(sendMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('error sending data to client', async () => {
|
||||
const ws = new wsMock('localhost');
|
||||
const sendMock = jest.spyOn(ws, 'send').mockImplementation(() => {
|
||||
throw new Error();
|
||||
});
|
||||
const cleanChannelMock = jest.spyOn(server, 'cleanChannel');
|
||||
const socketInstance = { ws: ws, channel: channelId, pongTs: Date.now() };
|
||||
server.trackClient(channelId, socketInstance);
|
||||
|
||||
server.processStreamResults(streamReturnValue);
|
||||
|
||||
expect(sendMock).toHaveBeenCalled();
|
||||
expect(cleanChannelMock).toHaveBeenCalledWith(channelId);
|
||||
|
||||
cleanChannelMock.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('fetchRangeFromStream', () => {
|
||||
beforeEach(() => {
|
||||
mockRedisXrange.mockClear();
|
||||
});
|
||||
|
||||
test('success with results', async () => {
|
||||
mockRedisXrange.mockResolvedValueOnce(streamReturnValue);
|
||||
const cb = jest.fn();
|
||||
await server.fetchRangeFromStream({
|
||||
sessionId: '123',
|
||||
startId: '-',
|
||||
endId: '+',
|
||||
listener: cb,
|
||||
});
|
||||
|
||||
expect(mockRedisXrange).toHaveBeenCalledWith(
|
||||
'test-async-events-123',
|
||||
'-',
|
||||
'+',
|
||||
);
|
||||
expect(cb).toHaveBeenCalledWith(streamReturnValue);
|
||||
});
|
||||
|
||||
test('success no results', async () => {
|
||||
const cb = jest.fn();
|
||||
await server.fetchRangeFromStream({
|
||||
sessionId: '123',
|
||||
startId: '-',
|
||||
endId: '+',
|
||||
listener: cb,
|
||||
});
|
||||
|
||||
expect(mockRedisXrange).toHaveBeenCalledWith(
|
||||
'test-async-events-123',
|
||||
'-',
|
||||
'+',
|
||||
);
|
||||
expect(cb).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('error', async () => {
|
||||
const cb = jest.fn();
|
||||
mockRedisXrange.mockRejectedValueOnce(new Error());
|
||||
await server.fetchRangeFromStream({
|
||||
sessionId: '123',
|
||||
startId: '-',
|
||||
endId: '+',
|
||||
listener: cb,
|
||||
});
|
||||
|
||||
expect(mockRedisXrange).toHaveBeenCalledWith(
|
||||
'test-async-events-123',
|
||||
'-',
|
||||
'+',
|
||||
);
|
||||
expect(cb).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('wsConnection', () => {
|
||||
let ws: WebSocket;
|
||||
let wsEventMock: jest.SpyInstance;
|
||||
let trackClientSpy: jest.SpyInstance;
|
||||
let fetchRangeFromStreamSpy: jest.SpyInstance;
|
||||
let dateNowSpy: jest.SpyInstance;
|
||||
let socketInstanceExpected: server.SocketInstance;
|
||||
|
||||
const getRequest = (token: string, url: string): http.IncomingMessage => {
|
||||
const request = new http.IncomingMessage(new net.Socket());
|
||||
request.method = 'GET';
|
||||
request.headers = { cookie: `${config.jwtCookieName}=${token}` };
|
||||
request.url = url;
|
||||
return request;
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
ws = new wsMock('localhost');
|
||||
wsEventMock = jest.spyOn(ws, 'on');
|
||||
trackClientSpy = jest.spyOn(server, 'trackClient');
|
||||
fetchRangeFromStreamSpy = jest.spyOn(server, 'fetchRangeFromStream');
|
||||
dateNowSpy = jest
|
||||
.spyOn(global.Date, 'now')
|
||||
.mockImplementation(() =>
|
||||
new Date('2021-03-10T11:01:58.135Z').valueOf(),
|
||||
);
|
||||
socketInstanceExpected = {
|
||||
ws,
|
||||
channel: channelId,
|
||||
pongTs: 1615374118135,
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
wsEventMock.mockRestore();
|
||||
trackClientSpy.mockRestore();
|
||||
fetchRangeFromStreamSpy.mockRestore();
|
||||
dateNowSpy.mockRestore();
|
||||
});
|
||||
|
||||
test('invalid JWT', async () => {
|
||||
const invalidToken = jwt.sign({ channel: channelId }, 'invalid secret');
|
||||
const request = getRequest(invalidToken, 'http://localhost');
|
||||
|
||||
expect(() => {
|
||||
server.wsConnection(ws, request);
|
||||
}).toThrow();
|
||||
});
|
||||
|
||||
test('valid JWT, no lastId', async () => {
|
||||
const validToken = jwt.sign({ channel: channelId }, config.jwtSecret);
|
||||
const request = getRequest(validToken, 'http://localhost');
|
||||
|
||||
server.wsConnection(ws, request);
|
||||
|
||||
expect(trackClientSpy).toHaveBeenCalledWith(
|
||||
channelId,
|
||||
socketInstanceExpected,
|
||||
);
|
||||
expect(fetchRangeFromStreamSpy).not.toHaveBeenCalled();
|
||||
expect(wsEventMock).toHaveBeenCalledWith('pong', expect.any(Function));
|
||||
});
|
||||
|
||||
test('valid JWT, with lastId', async () => {
|
||||
const validToken = jwt.sign({ channel: channelId }, config.jwtSecret);
|
||||
const lastId = '1615426152415-0';
|
||||
const request = getRequest(
|
||||
validToken,
|
||||
`http://localhost?last_id=${lastId}`,
|
||||
);
|
||||
|
||||
server.wsConnection(ws, request);
|
||||
|
||||
expect(trackClientSpy).toHaveBeenCalledWith(
|
||||
channelId,
|
||||
socketInstanceExpected,
|
||||
);
|
||||
expect(fetchRangeFromStreamSpy).toHaveBeenCalledWith({
|
||||
sessionId: channelId,
|
||||
startId: '1615426152415-1',
|
||||
endId: '+',
|
||||
listener: server.processStreamResults,
|
||||
});
|
||||
expect(wsEventMock).toHaveBeenCalledWith('pong', expect.any(Function));
|
||||
});
|
||||
|
||||
test('valid JWT, with lastId and lastFirehoseId', async () => {
|
||||
const validToken = jwt.sign({ channel: channelId }, config.jwtSecret);
|
||||
const lastId = '1615426152415-0';
|
||||
const lastFirehoseId = '1715426152415-0';
|
||||
const request = getRequest(
|
||||
validToken,
|
||||
`http://localhost?last_id=${lastId}`,
|
||||
);
|
||||
|
||||
server.setLastFirehoseId(lastFirehoseId);
|
||||
server.wsConnection(ws, request);
|
||||
|
||||
expect(trackClientSpy).toHaveBeenCalledWith(
|
||||
channelId,
|
||||
socketInstanceExpected,
|
||||
);
|
||||
expect(fetchRangeFromStreamSpy).toHaveBeenCalledWith({
|
||||
sessionId: channelId,
|
||||
startId: '1615426152415-1',
|
||||
endId: lastFirehoseId,
|
||||
listener: server.processStreamResults,
|
||||
});
|
||||
expect(wsEventMock).toHaveBeenCalledWith('pong', expect.any(Function));
|
||||
});
|
||||
});
|
||||
|
||||
describe('httpUpgrade', () => {
|
||||
let socket: net.Socket;
|
||||
let socketDestroySpy: jest.SpyInstance;
|
||||
let wssUpgradeSpy: jest.SpyInstance;
|
||||
|
||||
const getRequest = (token: string, url: string): http.IncomingMessage => {
|
||||
const request = new http.IncomingMessage(new net.Socket());
|
||||
request.method = 'GET';
|
||||
request.headers = { cookie: `${config.jwtCookieName}=${token}` };
|
||||
request.url = url;
|
||||
return request;
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
socket = new net.Socket();
|
||||
socketDestroySpy = jest.spyOn(socket, 'destroy');
|
||||
wssUpgradeSpy = jest.spyOn(server.wss, 'handleUpgrade');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
wssUpgradeSpy.mockRestore();
|
||||
});
|
||||
|
||||
test('invalid JWT', async () => {
|
||||
const invalidToken = jwt.sign({ channel: channelId }, 'invalid secret');
|
||||
const request = getRequest(invalidToken, 'http://localhost');
|
||||
|
||||
server.httpUpgrade(request, socket, Buffer.alloc(5));
|
||||
expect(socketDestroySpy).toHaveBeenCalled();
|
||||
expect(wssUpgradeSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('valid JWT, no channel', async () => {
|
||||
const validToken = jwt.sign({ foo: 'bar' }, config.jwtSecret);
|
||||
const request = getRequest(validToken, 'http://localhost');
|
||||
|
||||
server.httpUpgrade(request, socket, Buffer.alloc(5));
|
||||
|
||||
expect(socketDestroySpy).toHaveBeenCalled();
|
||||
expect(wssUpgradeSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('valid upgrade', async () => {
|
||||
const validToken = jwt.sign({ channel: channelId }, config.jwtSecret);
|
||||
const request = getRequest(validToken, 'http://localhost');
|
||||
|
||||
server.httpUpgrade(request, socket, Buffer.alloc(5));
|
||||
|
||||
expect(socketDestroySpy).not.toHaveBeenCalled();
|
||||
expect(wssUpgradeSpy).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkSockets', () => {
|
||||
let ws: WebSocket;
|
||||
let pingSpy: jest.SpyInstance;
|
||||
let terminateSpy: jest.SpyInstance;
|
||||
let socketInstance: server.SocketInstance;
|
||||
|
||||
beforeEach(() => {
|
||||
ws = new wsMock('localhost');
|
||||
pingSpy = jest.spyOn(ws, 'ping');
|
||||
terminateSpy = jest.spyOn(ws, 'terminate');
|
||||
socketInstance = { ws: ws, channel: channelId, pongTs: Date.now() };
|
||||
});
|
||||
|
||||
test('active sockets', () => {
|
||||
ws.readyState = WebSocket.OPEN;
|
||||
server.trackClient(channelId, socketInstance);
|
||||
|
||||
server.checkSockets();
|
||||
|
||||
expect(pingSpy).toHaveBeenCalled();
|
||||
expect(terminateSpy).not.toHaveBeenCalled();
|
||||
expect(Object.keys(server.sockets).length).toBe(1);
|
||||
});
|
||||
|
||||
test('stale sockets', () => {
|
||||
ws.readyState = WebSocket.OPEN;
|
||||
socketInstance.pongTs = Date.now() - 60000;
|
||||
server.trackClient(channelId, socketInstance);
|
||||
|
||||
server.checkSockets();
|
||||
|
||||
expect(pingSpy).not.toHaveBeenCalled();
|
||||
expect(terminateSpy).toHaveBeenCalled();
|
||||
expect(Object.keys(server.sockets).length).toBe(0);
|
||||
});
|
||||
|
||||
test('closed sockets', () => {
|
||||
ws.readyState = WebSocket.CLOSED;
|
||||
server.trackClient(channelId, socketInstance);
|
||||
|
||||
server.checkSockets();
|
||||
|
||||
expect(pingSpy).not.toHaveBeenCalled();
|
||||
expect(terminateSpy).not.toHaveBeenCalled();
|
||||
expect(Object.keys(server.sockets).length).toBe(0);
|
||||
});
|
||||
|
||||
test('no sockets', () => {
|
||||
// don't error
|
||||
server.checkSockets();
|
||||
});
|
||||
});
|
||||
|
||||
describe('cleanChannel', () => {
|
||||
let ws: WebSocket;
|
||||
let socketInstance: server.SocketInstance;
|
||||
|
||||
beforeEach(() => {
|
||||
ws = new wsMock('localhost');
|
||||
socketInstance = { ws: ws, channel: channelId, pongTs: Date.now() };
|
||||
});
|
||||
|
||||
test('active sockets', () => {
|
||||
ws.readyState = WebSocket.OPEN;
|
||||
server.trackClient(channelId, socketInstance);
|
||||
|
||||
server.cleanChannel(channelId);
|
||||
|
||||
expect(server.channels[channelId].sockets.length).toBe(1);
|
||||
});
|
||||
|
||||
test('closing sockets', () => {
|
||||
ws.readyState = WebSocket.CLOSING;
|
||||
server.trackClient(channelId, socketInstance);
|
||||
|
||||
server.cleanChannel(channelId);
|
||||
|
||||
expect(server.channels[channelId]).toBeUndefined();
|
||||
});
|
||||
|
||||
test('multiple sockets', () => {
|
||||
ws.readyState = WebSocket.OPEN;
|
||||
server.trackClient(channelId, socketInstance);
|
||||
|
||||
const ws2 = new wsMock('localhost');
|
||||
ws2.readyState = WebSocket.OPEN;
|
||||
const socketInstance2 = {
|
||||
ws: ws2,
|
||||
channel: channelId,
|
||||
pongTs: Date.now(),
|
||||
};
|
||||
server.trackClient(channelId, socketInstance2);
|
||||
|
||||
server.cleanChannel(channelId);
|
||||
|
||||
expect(server.channels[channelId].sockets.length).toBe(2);
|
||||
|
||||
ws2.readyState = WebSocket.CLOSED;
|
||||
server.cleanChannel(channelId);
|
||||
|
||||
expect(server.channels[channelId].sockets.length).toBe(1);
|
||||
});
|
||||
|
||||
test('invalid channel', () => {
|
||||
// don't error
|
||||
server.cleanChannel(channelId);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,465 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
import * as http from 'http';
|
||||
import * as net from 'net';
|
||||
import WebSocket from 'ws';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
|
||||
const winston = require('winston');
|
||||
const jwt = require('jsonwebtoken');
|
||||
const cookie = require('cookie');
|
||||
const Redis = require('ioredis');
|
||||
|
||||
export type StreamResult = [
|
||||
recordId: string,
|
||||
record: [label: 'data', data: string],
|
||||
];
|
||||
|
||||
// sync with superset-frontend/src/components/ErrorMessage/types
|
||||
export type ErrorLevel = 'info' | 'warning' | 'error';
|
||||
export type SupersetError<ExtraType = Record<string, any> | null> = {
|
||||
error_type: string;
|
||||
extra: ExtraType;
|
||||
level: ErrorLevel;
|
||||
message: string;
|
||||
};
|
||||
|
||||
type ListenerFunction = (results: StreamResult[]) => void;
|
||||
interface EventValue {
|
||||
id: string;
|
||||
channel_id: string;
|
||||
job_id: string;
|
||||
user_id?: string;
|
||||
status: string;
|
||||
errors?: SupersetError[];
|
||||
result_url?: string;
|
||||
}
|
||||
interface JwtPayload {
|
||||
channel: string;
|
||||
}
|
||||
interface FetchRangeFromStreamParams {
|
||||
sessionId: string;
|
||||
startId: string;
|
||||
endId: string;
|
||||
listener: ListenerFunction;
|
||||
}
|
||||
export interface SocketInstance {
|
||||
ws: WebSocket;
|
||||
channel: string;
|
||||
pongTs: number;
|
||||
}
|
||||
interface RedisConfig {
|
||||
port: number;
|
||||
host: string;
|
||||
password?: string | null;
|
||||
db: number;
|
||||
ssl: boolean;
|
||||
}
|
||||
|
||||
interface ChannelValue {
|
||||
sockets: Array<string>;
|
||||
}
|
||||
|
||||
const environment = process.env.NODE_ENV;
|
||||
|
||||
// default options
|
||||
export const opts = {
|
||||
port: 8080,
|
||||
logLevel: 'info',
|
||||
logToFile: false,
|
||||
logFilename: 'app.log',
|
||||
redis: {
|
||||
port: 6379,
|
||||
host: '127.0.0.1',
|
||||
password: '',
|
||||
db: 0,
|
||||
ssl: false,
|
||||
},
|
||||
redisStreamPrefix: 'async-events-',
|
||||
redisStreamReadCount: 100,
|
||||
redisStreamReadBlockMs: 5000,
|
||||
jwtSecret: '',
|
||||
jwtCookieName: 'async-token',
|
||||
socketResponseTimeoutMs: 60 * 1000,
|
||||
pingSocketsIntervalMs: 20 * 1000,
|
||||
gcChannelsIntervalMs: 120 * 1000,
|
||||
};
|
||||
|
||||
const startServer = process.argv[2] === 'start';
|
||||
const configFile =
|
||||
environment === 'test' ? '../config.test.json' : '../config.json';
|
||||
let config = {};
|
||||
try {
|
||||
config = require(configFile);
|
||||
} catch (err) {
|
||||
console.error('config.json not found, using defaults');
|
||||
}
|
||||
// apply config overrides
|
||||
Object.assign(opts, config);
|
||||
|
||||
// init logger
|
||||
const logTransports = [
|
||||
new winston.transports.Console({ handleExceptions: true }),
|
||||
];
|
||||
if (opts.logToFile && opts.logFilename) {
|
||||
logTransports.push(
|
||||
new winston.transports.File({
|
||||
filename: opts.logFilename,
|
||||
handleExceptions: true,
|
||||
}),
|
||||
);
|
||||
}
|
||||
const logger = winston.createLogger({
|
||||
level: opts.logLevel,
|
||||
transports: logTransports,
|
||||
});
|
||||
|
||||
// enforce JWT secret length
|
||||
if (startServer && opts.jwtSecret.length < 32)
|
||||
throw new Error('Please provide a JWT secret at least 32 bytes long');
|
||||
|
||||
export const redisUrlFromConfig = (redisConfig: RedisConfig): string => {
|
||||
let url = redisConfig.ssl ? 'rediss://' : 'redis://';
|
||||
if (redisConfig.password) url += `:${redisConfig.password}@`;
|
||||
url += `${redisConfig.host}:${redisConfig.port}/${redisConfig.db}`;
|
||||
return url;
|
||||
};
|
||||
|
||||
// initialize servers
|
||||
const redis = new Redis(redisUrlFromConfig(opts.redis));
|
||||
const httpServer = http.createServer();
|
||||
export const wss = new WebSocket.Server({
|
||||
noServer: true,
|
||||
clientTracking: false,
|
||||
});
|
||||
|
||||
const SOCKET_ACTIVE_STATES = [WebSocket.OPEN, WebSocket.CONNECTING];
|
||||
const GLOBAL_EVENT_STREAM_NAME = `${opts.redisStreamPrefix}full`;
|
||||
const DEFAULT_STREAM_LAST_ID = '$';
|
||||
|
||||
// initialize internal registries
|
||||
export let channels: Record<string, ChannelValue> = {};
|
||||
export let sockets: Record<string, SocketInstance> = {};
|
||||
let lastFirehoseId: string = DEFAULT_STREAM_LAST_ID;
|
||||
|
||||
export const setLastFirehoseId = (id: string): void => {
|
||||
lastFirehoseId = id;
|
||||
};
|
||||
|
||||
/**
|
||||
* Adds the passed channel and socket instance to the internal registries.
|
||||
*/
|
||||
export const trackClient = (
|
||||
channel: string,
|
||||
socketInstance: SocketInstance,
|
||||
): string => {
|
||||
const socketId = uuidv4();
|
||||
sockets[socketId] = socketInstance;
|
||||
|
||||
if (channel in channels) {
|
||||
channels[channel].sockets.push(socketId);
|
||||
} else {
|
||||
channels[channel] = { sockets: [socketId] };
|
||||
}
|
||||
|
||||
return socketId;
|
||||
};
|
||||
|
||||
/**
|
||||
* Sends a single async event payload to a single channel.
|
||||
* A channel may have multiple connected sockets, this emits
|
||||
* the event to all connected sockets within a channel.
|
||||
*/
|
||||
export const sendToChannel = (channel: string, value: EventValue): void => {
|
||||
const strData = JSON.stringify(value);
|
||||
if (!channels[channel]) {
|
||||
logger.debug(`channel ${channel} is unknown, skipping`);
|
||||
return;
|
||||
}
|
||||
channels[channel].sockets.forEach(socketId => {
|
||||
const socketInstance: SocketInstance = sockets[socketId];
|
||||
if (!socketInstance) return cleanChannel(channel);
|
||||
try {
|
||||
socketInstance.ws.send(strData);
|
||||
} catch (err) {
|
||||
logger.debug(`Error sending to socket: ${err}`);
|
||||
// check that the connection is still active
|
||||
cleanChannel(channel);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Reads a range of events from a channel-specific Redis event stream.
|
||||
* Invoked in the client re-connection flow.
|
||||
*/
|
||||
export const fetchRangeFromStream = async ({
|
||||
sessionId,
|
||||
startId,
|
||||
endId,
|
||||
listener,
|
||||
}: FetchRangeFromStreamParams) => {
|
||||
const streamName = `${opts.redisStreamPrefix}${sessionId}`;
|
||||
try {
|
||||
const reply = await redis.xrange(streamName, startId, endId);
|
||||
if (!reply || !reply.length) return;
|
||||
listener(reply);
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Reads from the global Redis event stream continuously.
|
||||
* Utilizes a blocking connection to Redis to wait for data to
|
||||
* be returned from the stream.
|
||||
*/
|
||||
export const subscribeToGlobalStream = async (
|
||||
stream: string,
|
||||
listener: ListenerFunction,
|
||||
) => {
|
||||
/*eslint no-constant-condition: ["error", { "checkLoops": false }]*/
|
||||
while (true) {
|
||||
try {
|
||||
const reply = await redis.xread(
|
||||
'BLOCK',
|
||||
opts.redisStreamReadBlockMs,
|
||||
'COUNT',
|
||||
opts.redisStreamReadCount,
|
||||
'STREAMS',
|
||||
stream,
|
||||
lastFirehoseId,
|
||||
);
|
||||
if (!reply) {
|
||||
continue;
|
||||
}
|
||||
const results = reply[0][1];
|
||||
const { length } = results;
|
||||
if (!results.length) {
|
||||
continue;
|
||||
}
|
||||
listener(results);
|
||||
setLastFirehoseId(results[length - 1][0]);
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Callback function to process events received from a Redis Stream
|
||||
*/
|
||||
export const processStreamResults = (results: StreamResult[]): void => {
|
||||
logger.debug(`events received: ${results}`);
|
||||
results.forEach(item => {
|
||||
try {
|
||||
const id = item[0];
|
||||
const data = JSON.parse(item[1][1]);
|
||||
sendToChannel(data.channel_id, { id, ...data });
|
||||
} catch (err) {
|
||||
logger.error(err);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Verify and parse a JWT cookie from an HTTP request.
|
||||
* Returns the JWT payload or throws an error on invalid token.
|
||||
*/
|
||||
const getJwtPayload = (request: http.IncomingMessage): JwtPayload => {
|
||||
const cookies = cookie.parse(request.headers.cookie);
|
||||
const token = cookies[opts.jwtCookieName];
|
||||
|
||||
if (!token) throw new Error('JWT not present');
|
||||
return jwt.verify(token, opts.jwtSecret);
|
||||
};
|
||||
|
||||
/**
|
||||
* Extracts the `last_id` query param value from an HTTP request
|
||||
*/
|
||||
const getLastId = (request: http.IncomingMessage): string | null => {
|
||||
const url = new URL(String(request.url), 'http://0.0.0.0');
|
||||
const queryParams = url.searchParams;
|
||||
return queryParams.get('last_id');
|
||||
};
|
||||
|
||||
/**
|
||||
* Increments a Redis Stream ID
|
||||
*/
|
||||
export const incrementId = (id: string): string => {
|
||||
// redis stream IDs are in this format: '1607477697866-0'
|
||||
const parts = id.split('-');
|
||||
if (parts.length < 2) return id;
|
||||
return parts[0] + '-' + (Number(parts[1]) + 1);
|
||||
};
|
||||
|
||||
/**
|
||||
* WebSocket `connection` event handler, called via wss
|
||||
*/
|
||||
export const wsConnection = (ws: WebSocket, request: http.IncomingMessage) => {
|
||||
const jwtPayload: JwtPayload = getJwtPayload(request);
|
||||
const channel: string = jwtPayload.channel;
|
||||
const socketInstance: SocketInstance = { ws, channel, pongTs: Date.now() };
|
||||
|
||||
// add this ws instance to the internal registry
|
||||
const socketId = trackClient(channel, socketInstance);
|
||||
logger.debug(`socket ${socketId} connected on channel ${channel}`);
|
||||
|
||||
// reconnection logic
|
||||
const lastId = getLastId(request);
|
||||
if (lastId) {
|
||||
// fetch range of events from lastId to most recent event received on
|
||||
// via global event stream
|
||||
const endId =
|
||||
lastFirehoseId === DEFAULT_STREAM_LAST_ID ? '+' : lastFirehoseId;
|
||||
fetchRangeFromStream({
|
||||
sessionId: channel,
|
||||
startId: incrementId(lastId), // inclusive
|
||||
endId, // inclusive
|
||||
listener: processStreamResults,
|
||||
});
|
||||
}
|
||||
|
||||
// init event handler for `pong` events (connection management)
|
||||
ws.on('pong', function pong(data: Buffer) {
|
||||
const socketId = data.toString();
|
||||
const socketInstance = sockets[socketId];
|
||||
if (!socketInstance) {
|
||||
logger.warn(`pong received for nonexistent socket ${socketId}`);
|
||||
} else {
|
||||
socketInstance.pongTs = Date.now();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* HTTP `upgrade` event handler, called via httpServer
|
||||
*/
|
||||
export const httpUpgrade = (
|
||||
request: http.IncomingMessage,
|
||||
socket: net.Socket,
|
||||
head: Buffer,
|
||||
) => {
|
||||
try {
|
||||
const jwtPayload: JwtPayload = getJwtPayload(request);
|
||||
if (!jwtPayload.channel) throw new Error('Channel ID not present');
|
||||
} catch (err) {
|
||||
// JWT invalid, do not establish a WebSocket connection
|
||||
logger.error(err);
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
|
||||
// upgrade the HTTP request into a WebSocket connection
|
||||
wss.handleUpgrade(
|
||||
request,
|
||||
socket,
|
||||
head,
|
||||
function cb(ws: WebSocket, request: http.IncomingMessage) {
|
||||
wss.emit('connection', ws, request);
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
// Connection cleanup and garbage collection
|
||||
|
||||
/**
|
||||
* Iterate over all tracked sockets, terminating and removing references to
|
||||
* connections that have not responded with a _pong_ within the timeout window.
|
||||
* Sends a _ping_ to all active connections.
|
||||
*/
|
||||
export const checkSockets = () => {
|
||||
logger.debug(`channel count: ${Object.keys(channels).length}`);
|
||||
logger.debug(`socket count: ${Object.keys(sockets).length}`);
|
||||
for (const socketId in sockets) {
|
||||
const socketInstance = sockets[socketId];
|
||||
const timeout = Date.now() - socketInstance.pongTs;
|
||||
let isActive = true;
|
||||
|
||||
if (timeout >= opts.socketResponseTimeoutMs) {
|
||||
logger.debug(
|
||||
`terminating unresponsive socket: ${socketId}, channel: ${socketInstance.channel}`,
|
||||
);
|
||||
socketInstance.ws.terminate();
|
||||
isActive = false;
|
||||
} else if (!SOCKET_ACTIVE_STATES.includes(socketInstance.ws.readyState)) {
|
||||
isActive = false;
|
||||
}
|
||||
|
||||
if (isActive) {
|
||||
socketInstance.ws.ping(socketId);
|
||||
} else {
|
||||
delete sockets[socketId];
|
||||
logger.debug(`forgetting socket ${socketId}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Iterate over all sockets within a channel, removing references to
|
||||
* inactive connections, ultimately removing the channel from the
|
||||
* _channels_ registry if no active connections remain.
|
||||
*/
|
||||
export const cleanChannel = (channel: string) => {
|
||||
const activeSockets: string[] =
|
||||
channels[channel]?.sockets.filter(socketId => {
|
||||
const socketInstance = sockets[socketId];
|
||||
if (!socketInstance) return false;
|
||||
if (SOCKET_ACTIVE_STATES.includes(socketInstance.ws.readyState))
|
||||
return true;
|
||||
return false;
|
||||
}) || [];
|
||||
|
||||
if (activeSockets.length === 0) {
|
||||
delete channels[channel];
|
||||
} else {
|
||||
channels[channel].sockets = activeSockets;
|
||||
}
|
||||
};
|
||||
|
||||
// server startup
|
||||
|
||||
if (startServer) {
|
||||
// init server event listeners
|
||||
wss.on('connection', wsConnection);
|
||||
httpServer.on('upgrade', httpUpgrade);
|
||||
httpServer.listen(opts.port);
|
||||
logger.info(`Server started on port ${opts.port}`);
|
||||
|
||||
// start reading from event stream
|
||||
subscribeToGlobalStream(GLOBAL_EVENT_STREAM_NAME, processStreamResults);
|
||||
|
||||
// init garbage collection routines
|
||||
setInterval(checkSockets, opts.pingSocketsIntervalMs);
|
||||
setInterval(function gc() {
|
||||
// clean all channels
|
||||
for (const channel in channels) {
|
||||
cleanChannel(channel);
|
||||
}
|
||||
}, opts.gcChannelsIntervalMs);
|
||||
}
|
||||
|
||||
// test utilities
|
||||
|
||||
export const resetState = () => {
|
||||
channels = {};
|
||||
sockets = {};
|
||||
lastFirehoseId = DEFAULT_STREAM_LAST_ID;
|
||||
};
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"outDir": "dist",
|
||||
"target": "es2019",
|
||||
"module": "commonjs",
|
||||
"esModuleInterop": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
# Test & development utilities
|
||||
The files provided here are for testing and development only, and are not required to run the WebSocket server application.
|
||||
|
||||
## Test client application
|
||||
The Express web application in `client-ws-app` is provided for testing the WebSocket server. See `client-ws-app/README.md` for details.
|
||||
|
||||
## Load testing script
|
||||
The `loadtest.js` script is provided to populate the Redis streams with event data.
|
||||
|
||||
### Running
|
||||
```
|
||||
node loadtest.js
|
||||
```
|
||||
|
||||
The script will populate data continually until the script is exited using CTRL-C.
|
||||
|
||||
**Note:** `loadtest.js` and test client application are configured to use the server's local `config.json` values, so care should be taken to not overwrite any sensitive data.
|
||||
|
|
@ -0,0 +1 @@
|
|||
v14.15.5
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
# Test client application
|
||||
This Express web application is provided for testing the WebSocket server. It is not required for running the server application, and is provided here for testing and development purposes only.
|
||||
|
||||
## Running
|
||||
First, start the WebSocket server:
|
||||
```
|
||||
cd ..
|
||||
npm run dev-server
|
||||
```
|
||||
|
||||
Then run the client application:
|
||||
```
|
||||
cd client-ws-app
|
||||
npm install
|
||||
npm start
|
||||
```
|
||||
|
||||
Open http://127.0.0.1:3000 in your web browser.
|
||||
|
||||
You can customize the number of WebSocket connections by passing the count in the `sockets` query param, e.g. `http://127.0.0.1:3000?sockets=180`, though beware that browsers limit the number of open WebSocket connections to around 200.
|
||||
|
||||
Run in conjunction with the `loadtest.js` script to populate the Redis streams with event data.
|
||||
|
||||
**Note:** this test application is configured to use the server's local `config.json` values, so care should be taken to not overwrite any sensitive data.
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
var createError = require('http-errors');
|
||||
var express = require('express');
|
||||
var path = require('path');
|
||||
var cookieParser = require('cookie-parser');
|
||||
var logger = require('morgan');
|
||||
|
||||
var indexRouter = require('./routes/index');
|
||||
|
||||
var app = express();
|
||||
|
||||
// view engine setup
|
||||
app.set('views', path.join(__dirname, 'views'));
|
||||
app.set('view engine', 'jade');
|
||||
|
||||
app.use(logger('dev'));
|
||||
app.use(express.json());
|
||||
app.use(express.urlencoded({ extended: false }));
|
||||
app.use(cookieParser());
|
||||
app.use(express.static(path.join(__dirname, 'public')));
|
||||
|
||||
app.use('/', indexRouter);
|
||||
|
||||
// catch 404 and forward to error handler
|
||||
app.use(function (req, res, next) {
|
||||
next(createError(404));
|
||||
});
|
||||
|
||||
// error handler
|
||||
app.use(function (err, req, res) {
|
||||
// set locals, only providing error in development
|
||||
res.locals.message = err.message;
|
||||
res.locals.error = req.app.get('env') === 'development' ? err : {};
|
||||
|
||||
// render the error page
|
||||
res.status(err.status || 500);
|
||||
res.render('error');
|
||||
});
|
||||
|
||||
module.exports = app;
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Module dependencies.
|
||||
*/
|
||||
|
||||
var app = require('../app');
|
||||
var debug = require('debug')('client-ws-app:server');
|
||||
var http = require('http');
|
||||
|
||||
/**
|
||||
* Get port from environment and store in Express.
|
||||
*/
|
||||
|
||||
var port = normalizePort(process.env.PORT || '3000');
|
||||
app.set('port', port);
|
||||
|
||||
/**
|
||||
* Create HTTP server.
|
||||
*/
|
||||
|
||||
var server = http.createServer(app);
|
||||
|
||||
/**
|
||||
* Listen on provided port, on all network interfaces.
|
||||
*/
|
||||
|
||||
server.listen(port);
|
||||
server.on('error', onError);
|
||||
server.on('listening', onListening);
|
||||
|
||||
/**
|
||||
* Normalize a port into a number, string, or false.
|
||||
*/
|
||||
|
||||
function normalizePort(val) {
|
||||
var port = parseInt(val, 10);
|
||||
|
||||
if (isNaN(port)) {
|
||||
// named pipe
|
||||
return val;
|
||||
}
|
||||
|
||||
if (port >= 0) {
|
||||
// port number
|
||||
return port;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Event listener for HTTP server "error" event.
|
||||
*/
|
||||
|
||||
function onError(error) {
|
||||
if (error.syscall !== 'listen') {
|
||||
throw error;
|
||||
}
|
||||
|
||||
var bind = typeof port === 'string'
|
||||
? 'Pipe ' + port
|
||||
: 'Port ' + port;
|
||||
|
||||
// handle specific listen errors with friendly messages
|
||||
switch (error.code) {
|
||||
case 'EACCES':
|
||||
console.error(bind + ' requires elevated privileges');
|
||||
process.exit(1);
|
||||
break;
|
||||
case 'EADDRINUSE':
|
||||
console.error(bind + ' is already in use');
|
||||
process.exit(1);
|
||||
break;
|
||||
default:
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Event listener for HTTP server "listening" event.
|
||||
*/
|
||||
|
||||
function onListening() {
|
||||
var addr = server.address();
|
||||
var bind = typeof addr === 'string'
|
||||
? 'pipe ' + addr
|
||||
: 'port ' + addr.port;
|
||||
debug('Listening on ' + bind);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"name": "client-ws-app",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "node ./bin/www"
|
||||
},
|
||||
"dependencies": {
|
||||
"cookie-parser": "~1.4.4",
|
||||
"debug": "~2.6.9",
|
||||
"express": "~4.16.1",
|
||||
"http-errors": "~1.6.3",
|
||||
"jade": "~1.11.0",
|
||||
"jsonwebtoken": "^8.5.1",
|
||||
"morgan": "~1.9.1"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
let socketCount = 0;
|
||||
let messageCount = 0;
|
||||
let lastMessage;
|
||||
|
||||
function ts() {
|
||||
return new Date().getTime();
|
||||
}
|
||||
|
||||
const cookieName = document.getElementById('cookie').innerHTML;
|
||||
const tokenData = document.getElementById('tokens').innerHTML;
|
||||
const tokens = JSON.parse(tokenData);
|
||||
|
||||
function connect() {
|
||||
if (socketCount >= tokens.length) return;
|
||||
|
||||
// using https://github.com/js-cookie/js-cookie
|
||||
// eslint-disable-next-line no-undef
|
||||
Cookies.set(cookieName, tokens[socketCount], { path: '' });
|
||||
|
||||
// Create WebSocket connection.
|
||||
let url = `ws://127.0.0.1:8080?last_id=${ts()}`;
|
||||
const socket = new WebSocket(url);
|
||||
|
||||
// Connection opened
|
||||
socket.addEventListener('open', function () {
|
||||
socketCount++;
|
||||
document.getElementById('socket-count').innerHTML = socketCount;
|
||||
connect();
|
||||
|
||||
socket.send('Hello Server!');
|
||||
});
|
||||
|
||||
// Listen for messages
|
||||
socket.addEventListener('message', function (event) {
|
||||
messageCount++;
|
||||
lastMessage = event.data;
|
||||
});
|
||||
}
|
||||
|
||||
connect();
|
||||
|
||||
setInterval(() => {
|
||||
document.getElementById('message-count').innerHTML = messageCount;
|
||||
document.getElementById('message-debug').innerHTML = lastMessage;
|
||||
}, 250);
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
body {
|
||||
padding: 50px;
|
||||
font: 14px 'Lucida Grande', Helvetica, Arial, sans-serif;
|
||||
}
|
||||
|
||||
a {
|
||||
color: #00b7ff;
|
||||
}
|
||||
|
||||
span {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
code {
|
||||
display: inline-block;
|
||||
margin-top: 5px;
|
||||
border: 1px #999 solid;
|
||||
padding: 5px;
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const jwt = require('jsonwebtoken');
|
||||
const config = require('../../../config.json');
|
||||
|
||||
router.get('/', function (req, res) {
|
||||
let numTokens = req.query.sockets ? Number(req.query.sockets) : 100;
|
||||
let tokens = [];
|
||||
for (let i = 0; i < numTokens; i++) {
|
||||
const token = jwt.sign({ channel: String(i) }, config.jwtSecret);
|
||||
tokens.push(token);
|
||||
}
|
||||
|
||||
res.render('index', {
|
||||
tokens: JSON.stringify(tokens),
|
||||
c: config.jwtCookieName,
|
||||
});
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
//-
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
extends layout
|
||||
|
||||
block content
|
||||
h1= message
|
||||
h2= error.status
|
||||
pre #{error.stack}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
//-
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
extends layout
|
||||
|
||||
block content
|
||||
h1 Websocket test app
|
||||
#tokens(style="display:none")= tokens
|
||||
#cookie(style="display:none")= c
|
||||
|
||||
div Sockets connected:
|
||||
span#socket-count 0
|
||||
div Messages recevied:
|
||||
span#message-count 0
|
||||
div Last message received:
|
||||
code#message-debug
|
||||
|
||||
script(src="/javascripts/app.js")
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
//-
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
doctype html
|
||||
html
|
||||
head
|
||||
title= title
|
||||
link(rel='stylesheet', href='/stylesheets/style.css')
|
||||
script(src="https://cdn.jsdelivr.net/npm/js-cookie@rc/dist/js.cookie.min.js")
|
||||
|
||||
body
|
||||
block content
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
const { v4: uuidv4 } = require('uuid');
|
||||
const Redis = require('ioredis');
|
||||
const config = require('../config.json');
|
||||
const redis = new Redis(config.redis);
|
||||
|
||||
const numClients = 256;
|
||||
const globalEventStreamName = `${config.redisStreamPrefix}full`;
|
||||
|
||||
function pushData() {
|
||||
for (let i = 0; i < numClients; i++) {
|
||||
const channelId = String(i);
|
||||
const streamId = `${config.redisStreamPrefix}${channelId}`;
|
||||
const data = {
|
||||
channel_id: channelId,
|
||||
job_id: uuidv4(),
|
||||
status: 'pending',
|
||||
};
|
||||
|
||||
// push to channel stream
|
||||
redis
|
||||
.xadd(streamId, 'MAXLEN', 1000, '*', 'data', JSON.stringify(data))
|
||||
.then(resp => {
|
||||
console.log('stream response', resp);
|
||||
});
|
||||
|
||||
// push to firehose (all events) stream
|
||||
redis
|
||||
.xadd(
|
||||
globalEventStreamName,
|
||||
'MAXLEN',
|
||||
100000,
|
||||
'*',
|
||||
'data',
|
||||
JSON.stringify(data),
|
||||
)
|
||||
.then(resp => {
|
||||
console.log('stream response', resp);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pushData();
|
||||
setInterval(pushData, 1000);
|
||||
|
|
@ -1134,6 +1134,7 @@ GLOBAL_ASYNC_QUERIES_REDIS_CONFIG = {
|
|||
"host": "127.0.0.1",
|
||||
"password": "",
|
||||
"db": 0,
|
||||
"ssl": False,
|
||||
}
|
||||
GLOBAL_ASYNC_QUERIES_REDIS_STREAM_PREFIX = "async-events-"
|
||||
GLOBAL_ASYNC_QUERIES_REDIS_STREAM_LIMIT = 1000
|
||||
|
|
@ -1143,6 +1144,7 @@ GLOBAL_ASYNC_QUERIES_JWT_COOKIE_SECURE = False
|
|||
GLOBAL_ASYNC_QUERIES_JWT_SECRET = "test-secret-change-me"
|
||||
GLOBAL_ASYNC_QUERIES_TRANSPORT = "polling"
|
||||
GLOBAL_ASYNC_QUERIES_POLLING_DELAY = 500
|
||||
GLOBAL_ASYNC_QUERIES_WEBSOCKET_URL = "ws://127.0.0.1:8080/"
|
||||
|
||||
# It's possible to add a dataset health check logic which is specific to your system.
|
||||
# It will get executed each time when user open a chart's explore view.
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from typing import Any, Dict, List, Optional, Tuple
|
|||
|
||||
import jwt
|
||||
import redis
|
||||
from flask import Flask, Request, Response, session
|
||||
from flask import Flask, request, Request, Response, session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -111,13 +111,14 @@ class AsyncQueryManager:
|
|||
def validate_session( # pylint: disable=unused-variable
|
||||
response: Response,
|
||||
) -> Response:
|
||||
reset_token = False
|
||||
user_id = session["user_id"] if "user_id" in session else None
|
||||
|
||||
if "async_channel_id" not in session or "async_user_id" not in session:
|
||||
reset_token = True
|
||||
elif user_id != session["async_user_id"]:
|
||||
reset_token = True
|
||||
reset_token = (
|
||||
not request.cookies.get(self._jwt_cookie_name)
|
||||
or "async_channel_id" not in session
|
||||
or "async_user_id" not in session
|
||||
or user_id != session["async_user_id"]
|
||||
)
|
||||
|
||||
if reset_token:
|
||||
async_channel_id = str(uuid.uuid4())
|
||||
|
|
@ -132,10 +133,6 @@ class AsyncQueryManager:
|
|||
value=token,
|
||||
httponly=True,
|
||||
secure=self._jwt_cookie_secure,
|
||||
# max_age=max_age or config.cookie_max_age,
|
||||
# domain=config.cookie_domain,
|
||||
# path=config.access_cookie_path,
|
||||
# samesite=config.cookie_samesite
|
||||
)
|
||||
|
||||
return response
|
||||
|
|
@ -148,8 +145,8 @@ class AsyncQueryManager:
|
|||
data = jwt.decode(token, self._jwt_secret, algorithms=["HS256"])
|
||||
return data
|
||||
|
||||
def parse_jwt_from_request(self, request: Request) -> Dict[str, Any]:
|
||||
token = request.cookies.get(self._jwt_cookie_name)
|
||||
def parse_jwt_from_request(self, req: Request) -> Dict[str, Any]:
|
||||
token = req.cookies.get(self._jwt_cookie_name)
|
||||
if not token:
|
||||
raise AsyncQueryTokenException("Token not preset")
|
||||
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ FRONTEND_CONF_KEYS = (
|
|||
"GLOBAL_ASYNC_QUERIES_POLLING_DELAY",
|
||||
"SQLALCHEMY_DOCS_URL",
|
||||
"SQLALCHEMY_DISPLAY_TEXT",
|
||||
"GLOBAL_ASYNC_QUERIES_WEBSOCKET_URL",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue