Webhook improvements

.github/scripts/ci.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+node --version
+npm install --unsafe-perm --no-audit --package-lock=false
+
+node <<'NODE'
+const fs = require('fs');
+const config = require('./srv_config.template.json');
+config.DB_HOST = '127.0.0.1';
+config.DB_PW = 'ci-password';
+fs.writeFileSync('srv_config.json', JSON.stringify(config));
+NODE
+
+node index.js > backend.log 2>&1 &
+backend_pid=$!
+cleanup() {
+    result=$?
+    if [ "$result" -ne 0 ]; then cat backend.log; fi
+    kill "$backend_pid" 2>/dev/null || true
+    exit "$result"
+}
+trap cleanup EXIT
+
+node <<'NODE'
+const http = require('http');
+const deadline = Date.now() + 60000;
+const check = () => {
+    const request = http.get('http://127.0.0.1:8000/settings', response => {
+        response.resume();
+        if (response.statusCode === 400) process.exit(0);
+        retry();
+    });
+    request.setTimeout(1000, () => request.destroy(new Error('Readiness request timed out')));
+    request.on('error', retry);
+};
+const retry = () => {
+    if (Date.now() >= deadline) {
+        console.error('Backend did not become ready within 60 seconds');
+        process.exit(1);
+    }
+    setTimeout(check, 500);
+};
+check();
+NODE
+
+# The MySQL service was initialized before this container started.
+node node_modules/mocha/bin/_mocha tests/*

.github/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Node 8 / MySQL 5.7.34
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    services:
+      mysql:
+        image: mysql:5.7.34
+        env:
+          MYSQL_ROOT_PASSWORD: ci-password
+          MYSQL_DATABASE: evnotify_test
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -pci-password"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=30
+    steps:
+      - uses: actions/checkout@v4
+      - name: Initialize test database
+        env:
+          MYSQL_CONTAINER: ${{ job.services.mysql.id }}
+        run: |
+          docker exec "$MYSQL_CONTAINER" mysql -uroot -pci-password --execute='SELECT VERSION()'
+          docker exec -i "$MYSQL_CONTAINER" mysql -uroot -pci-password evnotify_test < modules/db/db_template.sql
+      - name: Run backend and tests on Node 8.17.0
+        run: |
+          docker run --rm --network host \
+            --volume "$GITHUB_WORKSPACE:/app" --workdir /app \
+            node:8.17.0 bash .github/scripts/ci.sh

.hound.yml

@@ -0,0 +1,3 @@
+jshint:
+  enabled: true
+  config_file: .jshintrc

.jshintrc

@@ -2,9 +2,12 @@
     "undef": true,
     "unused": true,
     "esversion": 6,
+    "node": true,
+    "mocha": true,
+    "multistr": true,
     "globals": {
         "require": false,
         "module": false,
         "console": false
     }
-  }
+  }

CONTRIBUTING.md

@@ -28,7 +28,7 @@ The complete source code is available open source here, but out of the box your
 - You have to take care of keeping everything up-to-date - this means, your OS, packages and - the source code from this repository by itself. At the end I'll explain, how you can do this easily.
 - You may need to have your own domain - and a HTTPS certificate running on it - to encrypt the traffic that receives and sends out of the server. It's possible to run the server in HTTP mode, but that's not recommended.
 - Out of the box, you'll not be able to send notifications - to enable that feature, you'll need to provide some options in a config file that will be explained later. Keep in mind, that based on your needs, you need to register for some services, which may require some installation or other things (e.g. mail server, Telegram Bot Token, ..)
-- Out of the box, your server will not automatically track uncaught exceptions or analyze the usage. To be able to do so, you'll need to specify them in the config file, which will be explained later, as mentioned previously.
+- To track uncaught exceptions with Rollbar, configure `ROLLBAR_TOKEN` as described below.
 
 #### First-time-setup
 So, now we will setup the Backend for EVNotify. This step is required only once.
@@ -67,8 +67,6 @@ Here are optional keys, which can be specified within the _srv_config.json_ file
 - `FIREBASE_URL` - url of firebase project
 ###### Rollbar (automated error tracking)
 - `ROLLBAR_TOKEN` - the token of your Rollbar project
-###### Moesif (automated API traffic and usage analyzation)
-- `MOESIF_TOKEN` - the token of your Moesif project
 ###### GoingElectric (for stations finder)
 - `GE_API_URL` - the URL to the GoingElectric API (currently _https://api.goingelectric.de_)
 - `GE_API_KEY` - the key for the GoingElectric API (Format: `?key=XXXX`)

README.md

@@ -1,20 +1,74 @@
 # EVNotifyBackend
 
 [![Greenkeeper badge](https://badges.greenkeeper.io/EVNotify/EVNotifyBackend.svg)](https://greenkeeper.io/)
-[![Build Status](https://travis-ci.org/EVNotify/EVNotifyBackend.svg?branch=greenkeeper%2Finitial)](https://travis-ci.org/EVNotify/EVNotifyBackend)
+[![CI](https://github.com/EVNotify/EVNotifyBackend/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/EVNotify/EVNotifyBackend/actions/workflows/ci.yml)
 
-Backend for EVNotify (https://github.com/EVNotify/EVNotify)
+Backend for [EVNotify](https://github.com/EVNotify/EVNotify).
 
-### Contributing
-Feel free to contribute and report issues or suggestions.
-If you are a developer and want to contribute, have a look at the CONTRIBUTING.md file.
+## Features
 
-The App of EVNotify as well as the Backend has an integrated mechanism to track errors with a tool called Rollbar. This allows faster tracking of uncaught errors.
+- Account registration and authentication
+- Vehicle sync endpoints for SOC, location, and extended telemetry
+- Notifications via mail, push, and Telegram
+- Optional outbound webhooks for sync and notification events
+- Error tracking through Rollbar
 
-<a href="https://rollbar.com"><img src="https://rollbar.com/assets/badges/rollbar-partner-badge-light.png" alt="Rollbar Error Tracking" /></a>
+## Webhooks
+
+Webhooks are configured by an administrator on request, directly in the user's `settings.webhook` database column. `GET /settings` returns the configured URL. `PUT /settings` ignores any submitted `webhook` value and omits it from its response; users cannot change or disable delivery through the API. To disable delivery, an administrator sets the database value to `NULL` or an empty string.
+
+Stored webhook URLs must be absolute HTTPS URLs with a hostname and a valid port. HTTP and invalid URLs are logged and skipped before DNS lookup or delivery, without interrupting API requests. Administrators must replace existing HTTP URLs with HTTPS endpoints; stored values are not automatically changed.
+
+Destinations must resolve exclusively to public IPv4 or IPv6 addresses. Loopback, private, link-local (including cloud metadata), reserved, and other non-public addresses are blocked, including alternate IP spellings and IPv4-mapped IPv6 addresses containing blocked IPv4 addresses. DNS lookup failures or a lookup taking longer than 10 seconds skip delivery. Connections use a checked address without resolving again; proxies and redirects are disabled for all webhook requests. Blocked destinations are logged without changing the stored URL or interrupting API requests.
+
+The backend sends a JSON `POST` directly to the configured URL, including its path, after successful sync or notification requests:
+
+- `soc`
+- `location`
+- `extended`
+- `notification`
+
+The JSON payload contains:
+
+```json
+{
+  "akey": "ABC123",
+  "type": "soc",
+  "response": {}
+}
+```
+
+| Event type | `response` contents |
+| --- | --- |
+| `soc` | Submitted `display` and `bms` readings |
+| `location` | Submitted `location` object |
+| `extended` | Submitted telemetry body, with `token` removed |
+| `notification` | `soc_display`, `soc_bms`, and the submitted `abort` flag |
+
+The account's API `token` is included only as an outer payload field when the URL uses HTTPS and its hostname is exactly `evnotify.de`, `evnotify.com`, or a subdomain of either. Matching is case insensitive and requires a dot boundary: `api.evnotify.de` qualifies, but `fakeevnotify.de` and `evnotify.de.attacker.com` do not. Other destinations receive events without a token. Requests carrying tokens never follow redirects.
 
-For API Analytics and error tracking a tool called Moesif is used.
+Delivery happens in the background with a 10-second request timeout and no retries. Transport errors and non-2xx HTTP responses (including redirects) are logged. HTTP failure logs include the status code without URLs, credentials, payloads, or response bodies. A successful API response does not confirm webhook delivery. Rejected API requests and rate-limited notifications do not emit events.
 
-[Powered By Moesif API Analytics](https://www.moesif.com) 
+## Development
 
-[![Moesif API Analytics](https://cdn.moesif.com/images/moesif_badge.png)](https://www.moesif.com)
+Install dependencies with `npm install`. GitHub Actions runs the full test suite in Docker using Node 8.17.0 and MySQL 5.7.34, with a disposable test database.
+
+Run the isolated settings, webhook, sync, and notification tests without a backend or database:
+
+```bash
+./node_modules/.bin/mocha tests/settings/index.js tests/webhook/index.js tests/sync/index.js tests/notification/index.js
+```
+
+The full suite also includes account integration tests, which require a running backend at `TEST_URL` (or the configured local port) and a local test database configured in `srv_config.json`. Its setup creates the database and imports the schema:
+
+```bash
+npm test
+```
+
+## Contributing
+
+Feel free to contribute and report issues or suggestions. If you are a developer and want to contribute, have a look at [CONTRIBUTING.md](CONTRIBUTING.md).
+
+The app and backend use Rollbar to track uncaught errors and speed up debugging.
+
+<a href="https://rollbar.com"><img src="https://rollbar.com/assets/badges/rollbar-partner-badge-light.png" alt="Rollbar Error Tracking" /></a>

index.js

@@ -26,17 +26,6 @@ const express = require('express'),
         environment: ((srv_config.DEBUG) ? 'development' : 'production'),
         captureUnhandledRejections: true
     }) : false),
-    moesifExpress = require('moesif-express'),
-    moesif = ((srv_config.MOESIF_TOKEN) ? moesifExpress({
-        applicationId: srv_config.MOESIF_TOKEN,
-        identifyUser: req => req.body.akey,
-        getApiVersion: () => '2',
-        skip: req => {
-            const path = req.path.toLowerCase();
-
-            return path === '/location' || path === '/debug' || path === '/soc' || path === '/extended'
-        }
-    }) : false),
     db = require('./modules/db'),
     account = require('./modules/account'),
     sync = require('./modules/sync'),
@@ -97,9 +86,6 @@ app.use((req, res, next) => {
     next();
 });
 
-// moesif middleware
-if (moesif) app.use(moesif);
-
 if (srv_config.ROOKOUT_TOKEN) rookout.start({token: srv_config.ROOKOUT_TOKEN});
 
 // last activity track

modules/notification/index.js

@@ -49,7 +49,7 @@ const send = (req, res) => {
                         now, req.body.akey
                     ]);
                     // trigger webhook
-                    webhook.emit(req.body.akey, 'notification', {
+                    webhook.emit(req.body.akey, req.body.token, 'notification', {
                         soc_display: userObj.soc_display,
                         soc_bms: userObj.soc_bms,
                         abort: req.body.abort

modules/settings/index.js

@@ -15,7 +15,7 @@ const srv_config = require('./../../srv_config.json'),
  * @param {Function} callback callback function
  */
 const getSettings = (akey, callback) => {
-    db.query('SELECT email, telegram, abrp, push, soc, consumption, capacity, car, device, lng, summary FROM settings WHERE akey=?', [
+    db.query('SELECT email, telegram, abrp, push, soc, consumption, capacity, car, device, lng, summary, webhook FROM settings WHERE akey=?', [
         akey
     ], (err, dbRes) => {
         if (!err && dbRes && dbRes[0] && dbRes[0].email) dbRes[0].email = encryption.decrypt(dbRes[0].email);
@@ -30,7 +30,7 @@ const getSettings = (akey, callback) => {
  * @param {Function} callback callback function
  */
 const setSettings = (akey, settings, callback) => {
-    db.query('UPDATE settings SET email=?, telegram=?, push=?, soc=?, consumption=?, capacity=?, car=?, device=?, lng=?, summary=? WHERE akey=?', [
+    const values = [
         ((settings.email) ? encryption.encrypt(settings.email) : ''),
         settings.telegram,
         settings.push,
@@ -42,7 +42,10 @@ const setSettings = (akey, settings, callback) => {
         settings.lng,
         settings.summary,
         akey
-    ], (err, dbRes) => callback(err, ((!err && dbRes && dbRes[0]) ? dbRes[0] : null)));
+    ];
+
+    db.query('UPDATE settings SET email=?, telegram=?, push=?, soc=?, consumption=?, capacity=?, car=?, device=?, lng=?, summary=? WHERE akey=?', values,
+        (err, dbRes) => callback(err, ((!err && dbRes && dbRes[0]) ? dbRes[0] : null)));
 };
 
 module.exports = {
@@ -106,10 +109,12 @@ module.exports = {
             if (!err) {
                 if (valid) {
                     // set settings
-                    setSettings(req.body.akey, req.body.settings, (err, settings) => {
+                    setSettings(req.body.akey, req.body.settings, err => {
                         if (!err) {
+                            const updatedSettings = Object.assign({}, req.body.settings);
+                            delete updatedSettings.webhook;
                             res.json({
-                                settings: req.body.settings
+                                settings: updatedSettings
                             });
                         } else {
                             res.status(422).json({

modules/sync/index.js

@@ -7,6 +7,7 @@ const srv_config = require('./../../srv_config.json'),
     srv_errors = require('./../../srv_errors.json'),
     db = require('./../db'),
     abrp = require('./../integrations/abrp'),
+    webhook = require('./../webhook'),
     token = require('./../token');
 
 const updatedRecently = (now, dbTimestamp) => now - 5 < dbTimestamp;
@@ -171,6 +172,10 @@ module.exports = {
                         bms: req.body.bms
                     }, (err, synced) => {
                         if (!err && synced) {
+                            webhook.emit(req.body.akey, req.body.token, 'soc', {
+                                display: req.body.display,
+                                bms: req.body.bms
+                            });
                             res.json({
                                 synced: true
                             });
@@ -248,6 +253,7 @@ module.exports = {
                 if (valid) {
                     postExtended(req.body.akey, req.body, (err, synced) => {
                         if (!err && synced) {
+                            webhook.emit(req.body.akey, req.body.token, 'extended', req.body);
                             res.json({
                                 synced
                             });
@@ -314,7 +320,7 @@ module.exports = {
      */
     postLocation: (req, res) => {
         // check required params
-        if (!req.body.akey || !req.body.token || !req.body.location == null || typeof req.body.location !== 'object') {
+        if (!req.body.akey || !req.body.token || req.body.location == null || typeof req.body.location !== 'object') {
             return res.status(400).json({
                 error: srv_errors.INVALID_PARAMETERS
             });
@@ -325,6 +331,7 @@ module.exports = {
                 if (valid) {
                     postLocation(req.body.akey, req.body.location, (err, synced) => {
                         if (!err && synced) {
+                            webhook.emit(req.body.akey, req.body.token, 'location', req.body.location);
                             res.json({
                                 synced: true
                             });